2 * Copyright (C) ST-Ericsson SA 2007-2010
3 * Author: Per Friden <per.friden@stericsson.com> for ST-Ericsson
4 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/dmaengine.h>
11 #include <linux/platform_device.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/err.h>
16 #include <plat/ste_dma40.h>
18 #include "ste_dma40_ll.h"
20 #define D40_NAME "dma40"
22 #define D40_PHY_CHAN -1
24 /* For masking out/in 2 bit channel positions */
25 #define D40_CHAN_POS(chan) (2 * (chan / 2))
26 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
28 /* Maximum iterations taken before giving up suspending a channel */
29 #define D40_SUSPEND_MAX_IT 500
31 /* Hardware requirement on LCLA alignment */
32 #define LCLA_ALIGNMENT 0x40000
34 /* Max number of links per event group */
35 #define D40_LCLA_LINK_PER_EVENT_GRP 128
36 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
38 /* Attempts before giving up to trying to get pages that are aligned */
39 #define MAX_LCLA_ALLOC_ATTEMPTS 256
41 /* Bit markings for allocation map */
42 #define D40_ALLOC_FREE (1 << 31)
43 #define D40_ALLOC_PHY (1 << 30)
44 #define D40_ALLOC_LOG_FREE 0
46 /* Hardware designer of the block */
47 #define D40_HW_DESIGNER 0x8
50 * enum 40_command - The different commands and/or statuses.
52 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
53 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
54 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
55 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
60 D40_DMA_SUSPEND_REQ
= 2,
65 * struct d40_lli_pool - Structure for keeping LLIs in memory
67 * @base: Pointer to memory area when the pre_alloc_lli's are not large
68 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
69 * pre_alloc_lli is used.
70 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
71 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
72 * one buffer to one buffer.
77 /* Space for dst and src, plus an extra for padding */
78 u8 pre_alloc_lli
[3 * sizeof(struct d40_phy_lli
)];
82 * struct d40_desc - A descriptor is one DMA job.
84 * @lli_phy: LLI settings for physical channel. Both src and dst=
85 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
87 * @lli_log: Same as above but for logical channels.
88 * @lli_pool: The pool with two entries pre-allocated.
89 * @lli_len: Number of llis of current descriptor.
90 * @lli_current: Number of transfered llis.
91 * @lcla_alloc: Number of LCLA entries allocated.
92 * @txd: DMA engine struct. Used for among other things for communication
95 * @is_in_client_list: true if the client owns this descriptor.
96 * @is_hw_linked: true if this job will automatically be continued for
99 * This descriptor is used for both logical and physical transfers.
103 struct d40_phy_lli_bidir lli_phy
;
105 struct d40_log_lli_bidir lli_log
;
107 struct d40_lli_pool lli_pool
;
112 struct dma_async_tx_descriptor txd
;
113 struct list_head node
;
115 bool is_in_client_list
;
120 * struct d40_lcla_pool - LCLA pool settings and data.
122 * @base: The virtual address of LCLA. 18 bit aligned.
123 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
124 * This pointer is only there for clean-up on error.
125 * @pages: The number of pages needed for all physical channels.
126 * Only used later for clean-up on error
127 * @lock: Lock to protect the content in this struct.
128 * @alloc_map: big map over which LCLA entry is own by which job.
130 struct d40_lcla_pool
{
132 void *base_unaligned
;
135 struct d40_desc
**alloc_map
;
139 * struct d40_phy_res - struct for handling eventlines mapped to physical
142 * @lock: A lock protection this entity.
143 * @num: The physical channel number of this entity.
144 * @allocated_src: Bit mapped to show which src event line's are mapped to
145 * this physical channel. Can also be free or physically allocated.
146 * @allocated_dst: Same as for src but is dst.
147 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
160 * struct d40_chan - Struct that describes a channel.
162 * @lock: A spinlock to protect this struct.
163 * @log_num: The logical number, if any of this channel.
164 * @completed: Starts with 1, after first interrupt it is set to dma engine's
166 * @pending_tx: The number of pending transfers. Used between interrupt handler
168 * @busy: Set to true when transfer is ongoing on this channel.
169 * @phy_chan: Pointer to physical channel which this instance runs on. If this
170 * point is NULL, then the channel is not allocated.
171 * @chan: DMA engine handle.
172 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
173 * transfer and call client callback.
174 * @client: Cliented owned descriptor list.
175 * @active: Active descriptor.
176 * @queue: Queued jobs.
177 * @dma_cfg: The client configuration of this dma channel.
178 * @base: Pointer to the device instance struct.
179 * @src_def_cfg: Default cfg register setting for src.
180 * @dst_def_cfg: Default cfg register setting for dst.
181 * @log_def: Default logical channel settings.
182 * @lcla: Space for one dst src pair for logical channel transfers.
183 * @lcpa: Pointer to dst and src lcpa settings.
185 * This struct can either "be" a logical or a physical channel.
190 /* ID of the most recent completed transfer */
194 struct d40_phy_res
*phy_chan
;
195 struct dma_chan chan
;
196 struct tasklet_struct tasklet
;
197 struct list_head client
;
198 struct list_head active
;
199 struct list_head queue
;
200 struct stedma40_chan_cfg dma_cfg
;
201 struct d40_base
*base
;
202 /* Default register configurations */
205 struct d40_def_lcsp log_def
;
206 struct d40_log_lli_full
*lcpa
;
207 /* Runtime reconfiguration */
208 dma_addr_t runtime_addr
;
209 enum dma_data_direction runtime_direction
;
213 * struct d40_base - The big global struct, one for each probe'd instance.
215 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
216 * @execmd_lock: Lock for execute command usage since several channels share
217 * the same physical register.
218 * @dev: The device structure.
219 * @virtbase: The virtual base address of the DMA's register.
220 * @rev: silicon revision detected.
221 * @clk: Pointer to the DMA clock structure.
222 * @phy_start: Physical memory start of the DMA registers.
223 * @phy_size: Size of the DMA register map.
224 * @irq: The IRQ number.
225 * @num_phy_chans: The number of physical channels. Read from HW. This
226 * is the number of available channels for this driver, not counting "Secure
227 * mode" allocated physical channels.
228 * @num_log_chans: The number of logical channels. Calculated from
230 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
231 * @dma_slave: dma_device channels that can do only do slave transfers.
232 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
233 * @log_chans: Room for all possible logical channels in system.
234 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
235 * to log_chans entries.
236 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
237 * to phy_chans entries.
238 * @plat_data: Pointer to provided platform_data which is the driver
240 * @phy_res: Vector containing all physical channels.
241 * @lcla_pool: lcla pool settings and data.
242 * @lcpa_base: The virtual mapped address of LCPA.
243 * @phy_lcpa: The physical address of the LCPA.
244 * @lcpa_size: The size of the LCPA area.
245 * @desc_slab: cache for descriptors.
248 spinlock_t interrupt_lock
;
249 spinlock_t execmd_lock
;
251 void __iomem
*virtbase
;
254 phys_addr_t phy_start
;
255 resource_size_t phy_size
;
259 struct dma_device dma_both
;
260 struct dma_device dma_slave
;
261 struct dma_device dma_memcpy
;
262 struct d40_chan
*phy_chans
;
263 struct d40_chan
*log_chans
;
264 struct d40_chan
**lookup_log_chans
;
265 struct d40_chan
**lookup_phy_chans
;
266 struct stedma40_platform_data
*plat_data
;
267 /* Physical half channels */
268 struct d40_phy_res
*phy_res
;
269 struct d40_lcla_pool lcla_pool
;
272 resource_size_t lcpa_size
;
273 struct kmem_cache
*desc_slab
;
277 * struct d40_interrupt_lookup - lookup table for interrupt handler
279 * @src: Interrupt mask register.
280 * @clr: Interrupt clear register.
281 * @is_error: true if this is an error interrupt.
282 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
283 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
285 struct d40_interrupt_lookup
{
293 * struct d40_reg_val - simple lookup struct
295 * @reg: The register.
296 * @val: The value that belongs to the register in reg.
303 static int d40_pool_lli_alloc(struct d40_desc
*d40d
,
304 int lli_len
, bool is_log
)
310 align
= sizeof(struct d40_log_lli
);
312 align
= sizeof(struct d40_phy_lli
);
315 base
= d40d
->lli_pool
.pre_alloc_lli
;
316 d40d
->lli_pool
.size
= sizeof(d40d
->lli_pool
.pre_alloc_lli
);
317 d40d
->lli_pool
.base
= NULL
;
319 d40d
->lli_pool
.size
= ALIGN(lli_len
* 2 * align
, align
);
321 base
= kmalloc(d40d
->lli_pool
.size
+ align
, GFP_NOWAIT
);
322 d40d
->lli_pool
.base
= base
;
324 if (d40d
->lli_pool
.base
== NULL
)
329 d40d
->lli_log
.src
= PTR_ALIGN((struct d40_log_lli
*) base
,
331 d40d
->lli_log
.dst
= PTR_ALIGN(d40d
->lli_log
.src
+ lli_len
,
334 d40d
->lli_phy
.src
= PTR_ALIGN((struct d40_phy_lli
*)base
,
336 d40d
->lli_phy
.dst
= PTR_ALIGN(d40d
->lli_phy
.src
+ lli_len
,
343 static void d40_pool_lli_free(struct d40_desc
*d40d
)
345 kfree(d40d
->lli_pool
.base
);
346 d40d
->lli_pool
.base
= NULL
;
347 d40d
->lli_pool
.size
= 0;
348 d40d
->lli_log
.src
= NULL
;
349 d40d
->lli_log
.dst
= NULL
;
350 d40d
->lli_phy
.src
= NULL
;
351 d40d
->lli_phy
.dst
= NULL
;
354 static int d40_lcla_alloc_one(struct d40_chan
*d40c
,
355 struct d40_desc
*d40d
)
362 spin_lock_irqsave(&d40c
->base
->lcla_pool
.lock
, flags
);
364 p
= d40c
->phy_chan
->num
* D40_LCLA_LINK_PER_EVENT_GRP
;
367 * Allocate both src and dst at the same time, therefore the half
368 * start on 1 since 0 can't be used since zero is used as end marker.
370 for (i
= 1 ; i
< D40_LCLA_LINK_PER_EVENT_GRP
/ 2; i
++) {
371 if (!d40c
->base
->lcla_pool
.alloc_map
[p
+ i
]) {
372 d40c
->base
->lcla_pool
.alloc_map
[p
+ i
] = d40d
;
379 spin_unlock_irqrestore(&d40c
->base
->lcla_pool
.lock
, flags
);
384 static int d40_lcla_free_all(struct d40_chan
*d40c
,
385 struct d40_desc
*d40d
)
391 if (d40c
->log_num
== D40_PHY_CHAN
)
394 spin_lock_irqsave(&d40c
->base
->lcla_pool
.lock
, flags
);
396 for (i
= 1 ; i
< D40_LCLA_LINK_PER_EVENT_GRP
/ 2; i
++) {
397 if (d40c
->base
->lcla_pool
.alloc_map
[d40c
->phy_chan
->num
*
398 D40_LCLA_LINK_PER_EVENT_GRP
+ i
] == d40d
) {
399 d40c
->base
->lcla_pool
.alloc_map
[d40c
->phy_chan
->num
*
400 D40_LCLA_LINK_PER_EVENT_GRP
+ i
] = NULL
;
402 if (d40d
->lcla_alloc
== 0) {
409 spin_unlock_irqrestore(&d40c
->base
->lcla_pool
.lock
, flags
);
415 static void d40_desc_remove(struct d40_desc
*d40d
)
417 list_del(&d40d
->node
);
420 static struct d40_desc
*d40_desc_get(struct d40_chan
*d40c
)
425 if (!list_empty(&d40c
->client
)) {
426 list_for_each_entry_safe(d
, _d
, &d40c
->client
, node
)
427 if (async_tx_test_ack(&d
->txd
)) {
428 d40_pool_lli_free(d
);
433 d
= kmem_cache_alloc(d40c
->base
->desc_slab
, GFP_NOWAIT
);
435 memset(d
, 0, sizeof(struct d40_desc
));
436 INIT_LIST_HEAD(&d
->node
);
442 static void d40_desc_free(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
445 d40_lcla_free_all(d40c
, d40d
);
446 kmem_cache_free(d40c
->base
->desc_slab
, d40d
);
449 static void d40_desc_submit(struct d40_chan
*d40c
, struct d40_desc
*desc
)
451 list_add_tail(&desc
->node
, &d40c
->active
);
454 static void d40_desc_load(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
456 int curr_lcla
= -EINVAL
, next_lcla
;
458 if (d40c
->log_num
== D40_PHY_CHAN
) {
459 d40_phy_lli_write(d40c
->base
->virtbase
,
463 d40d
->lli_current
= d40d
->lli_len
;
466 if ((d40d
->lli_len
- d40d
->lli_current
) > 1)
467 curr_lcla
= d40_lcla_alloc_one(d40c
, d40d
);
469 d40_log_lli_lcpa_write(d40c
->lcpa
,
470 &d40d
->lli_log
.dst
[d40d
->lli_current
],
471 &d40d
->lli_log
.src
[d40d
->lli_current
],
475 for (; d40d
->lli_current
< d40d
->lli_len
; d40d
->lli_current
++) {
476 struct d40_log_lli
*lcla
;
478 if (d40d
->lli_current
+ 1 < d40d
->lli_len
)
479 next_lcla
= d40_lcla_alloc_one(d40c
, d40d
);
483 lcla
= d40c
->base
->lcla_pool
.base
+
484 d40c
->phy_chan
->num
* 1024 +
487 d40_log_lli_lcla_write(lcla
,
488 &d40d
->lli_log
.dst
[d40d
->lli_current
],
489 &d40d
->lli_log
.src
[d40d
->lli_current
],
492 (void) dma_map_single(d40c
->base
->dev
, lcla
,
493 2 * sizeof(struct d40_log_lli
),
496 curr_lcla
= next_lcla
;
498 if (curr_lcla
== -EINVAL
) {
507 static struct d40_desc
*d40_first_active_get(struct d40_chan
*d40c
)
511 if (list_empty(&d40c
->active
))
514 d
= list_first_entry(&d40c
->active
,
520 static void d40_desc_queue(struct d40_chan
*d40c
, struct d40_desc
*desc
)
522 list_add_tail(&desc
->node
, &d40c
->queue
);
525 static struct d40_desc
*d40_first_queued(struct d40_chan
*d40c
)
529 if (list_empty(&d40c
->queue
))
532 d
= list_first_entry(&d40c
->queue
,
538 static struct d40_desc
*d40_last_queued(struct d40_chan
*d40c
)
542 if (list_empty(&d40c
->queue
))
544 list_for_each_entry(d
, &d40c
->queue
, node
)
545 if (list_is_last(&d
->node
, &d40c
->queue
))
550 /* Support functions for logical channels */
553 static int d40_channel_execute_command(struct d40_chan
*d40c
,
554 enum d40_command command
)
558 void __iomem
*active_reg
;
563 spin_lock_irqsave(&d40c
->base
->execmd_lock
, flags
);
565 if (d40c
->phy_chan
->num
% 2 == 0)
566 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
568 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
570 if (command
== D40_DMA_SUSPEND_REQ
) {
571 status
= (readl(active_reg
) &
572 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
573 D40_CHAN_POS(d40c
->phy_chan
->num
);
575 if (status
== D40_DMA_SUSPENDED
|| status
== D40_DMA_STOP
)
579 wmask
= 0xffffffff & ~(D40_CHAN_POS_MASK(d40c
->phy_chan
->num
));
580 writel(wmask
| (command
<< D40_CHAN_POS(d40c
->phy_chan
->num
)),
583 if (command
== D40_DMA_SUSPEND_REQ
) {
585 for (i
= 0 ; i
< D40_SUSPEND_MAX_IT
; i
++) {
586 status
= (readl(active_reg
) &
587 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
588 D40_CHAN_POS(d40c
->phy_chan
->num
);
592 * Reduce the number of bus accesses while
593 * waiting for the DMA to suspend.
597 if (status
== D40_DMA_STOP
||
598 status
== D40_DMA_SUSPENDED
)
602 if (i
== D40_SUSPEND_MAX_IT
) {
603 dev_err(&d40c
->chan
.dev
->device
,
604 "[%s]: unable to suspend the chl %d (log: %d) status %x\n",
605 __func__
, d40c
->phy_chan
->num
, d40c
->log_num
,
613 spin_unlock_irqrestore(&d40c
->base
->execmd_lock
, flags
);
617 static void d40_term_all(struct d40_chan
*d40c
)
619 struct d40_desc
*d40d
;
621 /* Release active descriptors */
622 while ((d40d
= d40_first_active_get(d40c
))) {
623 d40_desc_remove(d40d
);
624 d40_desc_free(d40c
, d40d
);
627 /* Release queued descriptors waiting for transfer */
628 while ((d40d
= d40_first_queued(d40c
))) {
629 d40_desc_remove(d40d
);
630 d40_desc_free(d40c
, d40d
);
634 d40c
->pending_tx
= 0;
638 static void d40_config_set_event(struct d40_chan
*d40c
, bool do_enable
)
643 /* Notice, that disable requires the physical channel to be stopped */
645 val
= D40_ACTIVATE_EVENTLINE
;
647 val
= D40_DEACTIVATE_EVENTLINE
;
649 spin_lock_irqsave(&d40c
->phy_chan
->lock
, flags
);
651 /* Enable event line connected to device (or memcpy) */
652 if ((d40c
->dma_cfg
.dir
== STEDMA40_PERIPH_TO_MEM
) ||
653 (d40c
->dma_cfg
.dir
== STEDMA40_PERIPH_TO_PERIPH
)) {
654 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.src_dev_type
);
656 writel((val
<< D40_EVENTLINE_POS(event
)) |
657 ~D40_EVENTLINE_MASK(event
),
658 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
659 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
662 if (d40c
->dma_cfg
.dir
!= STEDMA40_PERIPH_TO_MEM
) {
663 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dst_dev_type
);
665 writel((val
<< D40_EVENTLINE_POS(event
)) |
666 ~D40_EVENTLINE_MASK(event
),
667 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
668 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
672 spin_unlock_irqrestore(&d40c
->phy_chan
->lock
, flags
);
675 static u32
d40_chan_has_events(struct d40_chan
*d40c
)
679 val
= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
680 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
683 val
|= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
684 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
689 static void d40_config_write(struct d40_chan
*d40c
)
694 /* Odd addresses are even addresses + 4 */
695 addr_base
= (d40c
->phy_chan
->num
% 2) * 4;
696 /* Setup channel mode to logical or physical */
697 var
= ((u32
)(d40c
->log_num
!= D40_PHY_CHAN
) + 1) <<
698 D40_CHAN_POS(d40c
->phy_chan
->num
);
699 writel(var
, d40c
->base
->virtbase
+ D40_DREG_PRMSE
+ addr_base
);
701 /* Setup operational mode option register */
702 var
= ((d40c
->dma_cfg
.channel_type
>> STEDMA40_INFO_CH_MODE_OPT_POS
) &
703 0x3) << D40_CHAN_POS(d40c
->phy_chan
->num
);
705 writel(var
, d40c
->base
->virtbase
+ D40_DREG_PRMOE
+ addr_base
);
707 if (d40c
->log_num
!= D40_PHY_CHAN
) {
708 /* Set default config for CFG reg */
709 writel(d40c
->src_def_cfg
,
710 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
711 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
713 writel(d40c
->dst_def_cfg
,
714 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
715 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
718 /* Set LIDX for lcla */
719 writel((d40c
->phy_chan
->num
<< D40_SREG_ELEM_LOG_LIDX_POS
) &
720 D40_SREG_ELEM_LOG_LIDX_MASK
,
721 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
722 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
725 writel((d40c
->phy_chan
->num
<< D40_SREG_ELEM_LOG_LIDX_POS
) &
726 D40_SREG_ELEM_LOG_LIDX_MASK
,
727 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
728 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
734 static u32
d40_residue(struct d40_chan
*d40c
)
738 if (d40c
->log_num
!= D40_PHY_CHAN
)
739 num_elt
= (readl(&d40c
->lcpa
->lcsp2
) & D40_MEM_LCSP2_ECNT_MASK
)
740 >> D40_MEM_LCSP2_ECNT_POS
;
742 num_elt
= (readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
743 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
744 D40_CHAN_REG_SDELT
) &
745 D40_SREG_ELEM_PHY_ECNT_MASK
) >>
746 D40_SREG_ELEM_PHY_ECNT_POS
;
747 return num_elt
* (1 << d40c
->dma_cfg
.dst_info
.data_width
);
750 static bool d40_tx_is_linked(struct d40_chan
*d40c
)
754 if (d40c
->log_num
!= D40_PHY_CHAN
)
755 is_link
= readl(&d40c
->lcpa
->lcsp3
) & D40_MEM_LCSP3_DLOS_MASK
;
757 is_link
= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
758 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
759 D40_CHAN_REG_SDLNK
) &
760 D40_SREG_LNK_PHYS_LNK_MASK
;
764 static int d40_pause(struct dma_chan
*chan
)
766 struct d40_chan
*d40c
=
767 container_of(chan
, struct d40_chan
, chan
);
774 spin_lock_irqsave(&d40c
->lock
, flags
);
776 res
= d40_channel_execute_command(d40c
, D40_DMA_SUSPEND_REQ
);
778 if (d40c
->log_num
!= D40_PHY_CHAN
) {
779 d40_config_set_event(d40c
, false);
780 /* Resume the other logical channels if any */
781 if (d40_chan_has_events(d40c
))
782 res
= d40_channel_execute_command(d40c
,
787 spin_unlock_irqrestore(&d40c
->lock
, flags
);
791 static int d40_resume(struct dma_chan
*chan
)
793 struct d40_chan
*d40c
=
794 container_of(chan
, struct d40_chan
, chan
);
801 spin_lock_irqsave(&d40c
->lock
, flags
);
803 if (d40c
->base
->rev
== 0)
804 if (d40c
->log_num
!= D40_PHY_CHAN
) {
805 res
= d40_channel_execute_command(d40c
,
806 D40_DMA_SUSPEND_REQ
);
810 /* If bytes left to transfer or linked tx resume job */
811 if (d40_residue(d40c
) || d40_tx_is_linked(d40c
)) {
813 if (d40c
->log_num
!= D40_PHY_CHAN
)
814 d40_config_set_event(d40c
, true);
816 res
= d40_channel_execute_command(d40c
, D40_DMA_RUN
);
820 spin_unlock_irqrestore(&d40c
->lock
, flags
);
824 static void d40_tx_submit_log(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
829 static void d40_tx_submit_phy(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
831 struct d40_desc
*d40d_prev
= NULL
;
835 if (!list_empty(&d40c
->queue
))
836 d40d_prev
= d40_last_queued(d40c
);
837 else if (!list_empty(&d40c
->active
))
838 d40d_prev
= d40_first_active_get(d40c
);
843 /* Here we try to join this job with previous jobs */
844 val
= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
845 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
848 /* Figure out which link we're currently transmitting */
849 for (i
= 0; i
< d40d_prev
->lli_len
; i
++)
850 if (val
== d40d_prev
->lli_phy
.src
[i
].reg_lnk
)
853 val
= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
854 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
855 D40_CHAN_REG_SSELT
) >> D40_SREG_ELEM_LOG_ECNT_POS
;
857 if (i
== (d40d_prev
->lli_len
- 1) && val
> 0) {
858 /* Change the current one */
859 writel(virt_to_phys(d40d
->lli_phy
.src
),
860 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
861 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
863 writel(virt_to_phys(d40d
->lli_phy
.dst
),
864 d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
865 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
868 d40d
->is_hw_linked
= true;
870 } else if (i
< d40d_prev
->lli_len
) {
871 (void) dma_unmap_single(d40c
->base
->dev
,
872 virt_to_phys(d40d_prev
->lli_phy
.src
),
873 d40d_prev
->lli_pool
.size
,
876 /* Keep the settings */
877 val
= d40d_prev
->lli_phy
.src
[d40d_prev
->lli_len
- 1].reg_lnk
&
878 ~D40_SREG_LNK_PHYS_LNK_MASK
;
879 d40d_prev
->lli_phy
.src
[d40d_prev
->lli_len
- 1].reg_lnk
=
880 val
| virt_to_phys(d40d
->lli_phy
.src
);
882 val
= d40d_prev
->lli_phy
.dst
[d40d_prev
->lli_len
- 1].reg_lnk
&
883 ~D40_SREG_LNK_PHYS_LNK_MASK
;
884 d40d_prev
->lli_phy
.dst
[d40d_prev
->lli_len
- 1].reg_lnk
=
885 val
| virt_to_phys(d40d
->lli_phy
.dst
);
887 (void) dma_map_single(d40c
->base
->dev
,
888 d40d_prev
->lli_phy
.src
,
889 d40d_prev
->lli_pool
.size
,
891 d40d
->is_hw_linked
= true;
895 static dma_cookie_t
d40_tx_submit(struct dma_async_tx_descriptor
*tx
)
897 struct d40_chan
*d40c
= container_of(tx
->chan
,
900 struct d40_desc
*d40d
= container_of(tx
, struct d40_desc
, txd
);
903 (void) d40_pause(&d40c
->chan
);
905 spin_lock_irqsave(&d40c
->lock
, flags
);
909 if (d40c
->chan
.cookie
< 0)
910 d40c
->chan
.cookie
= 1;
912 d40d
->txd
.cookie
= d40c
->chan
.cookie
;
914 if (d40c
->log_num
== D40_PHY_CHAN
)
915 d40_tx_submit_phy(d40c
, d40d
);
917 d40_tx_submit_log(d40c
, d40d
);
919 d40_desc_queue(d40c
, d40d
);
921 spin_unlock_irqrestore(&d40c
->lock
, flags
);
923 (void) d40_resume(&d40c
->chan
);
928 static int d40_start(struct d40_chan
*d40c
)
930 if (d40c
->base
->rev
== 0) {
933 if (d40c
->log_num
!= D40_PHY_CHAN
) {
934 err
= d40_channel_execute_command(d40c
,
935 D40_DMA_SUSPEND_REQ
);
941 if (d40c
->log_num
!= D40_PHY_CHAN
)
942 d40_config_set_event(d40c
, true);
944 return d40_channel_execute_command(d40c
, D40_DMA_RUN
);
947 static struct d40_desc
*d40_queue_start(struct d40_chan
*d40c
)
949 struct d40_desc
*d40d
;
952 /* Start queued jobs, if any */
953 d40d
= d40_first_queued(d40c
);
958 /* Remove from queue */
959 d40_desc_remove(d40d
);
961 /* Add to active queue */
962 d40_desc_submit(d40c
, d40d
);
965 * If this job is already linked in hw,
969 if (!d40d
->is_hw_linked
) {
970 /* Initiate DMA job */
971 d40_desc_load(d40c
, d40d
);
974 err
= d40_start(d40c
);
984 /* called from interrupt context */
985 static void dma_tc_handle(struct d40_chan
*d40c
)
987 struct d40_desc
*d40d
;
989 /* Get first active entry from list */
990 d40d
= d40_first_active_get(d40c
);
995 d40_lcla_free_all(d40c
, d40d
);
997 if (d40d
->lli_current
< d40d
->lli_len
) {
998 d40_desc_load(d40c
, d40d
);
1000 (void) d40_start(d40c
);
1004 if (d40_queue_start(d40c
) == NULL
)
1008 tasklet_schedule(&d40c
->tasklet
);
1012 static void dma_tasklet(unsigned long data
)
1014 struct d40_chan
*d40c
= (struct d40_chan
*) data
;
1015 struct d40_desc
*d40d
;
1016 unsigned long flags
;
1017 dma_async_tx_callback callback
;
1018 void *callback_param
;
1020 spin_lock_irqsave(&d40c
->lock
, flags
);
1022 /* Get first active entry from list */
1023 d40d
= d40_first_active_get(d40c
);
1028 d40c
->completed
= d40d
->txd
.cookie
;
1031 * If terminating a channel pending_tx is set to zero.
1032 * This prevents any finished active jobs to return to the client.
1034 if (d40c
->pending_tx
== 0) {
1035 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1039 /* Callback to client */
1040 callback
= d40d
->txd
.callback
;
1041 callback_param
= d40d
->txd
.callback_param
;
1043 if (async_tx_test_ack(&d40d
->txd
)) {
1044 d40_pool_lli_free(d40d
);
1045 d40_desc_remove(d40d
);
1046 d40_desc_free(d40c
, d40d
);
1048 if (!d40d
->is_in_client_list
) {
1049 d40_desc_remove(d40d
);
1050 d40_lcla_free_all(d40c
, d40d
);
1051 list_add_tail(&d40d
->node
, &d40c
->client
);
1052 d40d
->is_in_client_list
= true;
1058 if (d40c
->pending_tx
)
1059 tasklet_schedule(&d40c
->tasklet
);
1061 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1063 if (callback
&& (d40d
->txd
.flags
& DMA_PREP_INTERRUPT
))
1064 callback(callback_param
);
1069 /* Rescue manouver if receiving double interrupts */
1070 if (d40c
->pending_tx
> 0)
1072 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1075 static irqreturn_t
d40_handle_interrupt(int irq
, void *data
)
1077 static const struct d40_interrupt_lookup il
[] = {
1078 {D40_DREG_LCTIS0
, D40_DREG_LCICR0
, false, 0},
1079 {D40_DREG_LCTIS1
, D40_DREG_LCICR1
, false, 32},
1080 {D40_DREG_LCTIS2
, D40_DREG_LCICR2
, false, 64},
1081 {D40_DREG_LCTIS3
, D40_DREG_LCICR3
, false, 96},
1082 {D40_DREG_LCEIS0
, D40_DREG_LCICR0
, true, 0},
1083 {D40_DREG_LCEIS1
, D40_DREG_LCICR1
, true, 32},
1084 {D40_DREG_LCEIS2
, D40_DREG_LCICR2
, true, 64},
1085 {D40_DREG_LCEIS3
, D40_DREG_LCICR3
, true, 96},
1086 {D40_DREG_PCTIS
, D40_DREG_PCICR
, false, D40_PHY_CHAN
},
1087 {D40_DREG_PCEIS
, D40_DREG_PCICR
, true, D40_PHY_CHAN
},
1091 u32 regs
[ARRAY_SIZE(il
)];
1095 struct d40_chan
*d40c
;
1096 unsigned long flags
;
1097 struct d40_base
*base
= data
;
1099 spin_lock_irqsave(&base
->interrupt_lock
, flags
);
1101 /* Read interrupt status of both logical and physical channels */
1102 for (i
= 0; i
< ARRAY_SIZE(il
); i
++)
1103 regs
[i
] = readl(base
->virtbase
+ il
[i
].src
);
1107 chan
= find_next_bit((unsigned long *)regs
,
1108 BITS_PER_LONG
* ARRAY_SIZE(il
), chan
+ 1);
1110 /* No more set bits found? */
1111 if (chan
== BITS_PER_LONG
* ARRAY_SIZE(il
))
1114 row
= chan
/ BITS_PER_LONG
;
1115 idx
= chan
& (BITS_PER_LONG
- 1);
1118 writel(1 << idx
, base
->virtbase
+ il
[row
].clr
);
1120 if (il
[row
].offset
== D40_PHY_CHAN
)
1121 d40c
= base
->lookup_phy_chans
[idx
];
1123 d40c
= base
->lookup_log_chans
[il
[row
].offset
+ idx
];
1124 spin_lock(&d40c
->lock
);
1126 if (!il
[row
].is_error
)
1127 dma_tc_handle(d40c
);
1130 "[%s] IRQ chan: %ld offset %d idx %d\n",
1131 __func__
, chan
, il
[row
].offset
, idx
);
1133 spin_unlock(&d40c
->lock
);
1136 spin_unlock_irqrestore(&base
->interrupt_lock
, flags
);
1141 static int d40_validate_conf(struct d40_chan
*d40c
,
1142 struct stedma40_chan_cfg
*conf
)
1145 u32 dst_event_group
= D40_TYPE_TO_GROUP(conf
->dst_dev_type
);
1146 u32 src_event_group
= D40_TYPE_TO_GROUP(conf
->src_dev_type
);
1147 bool is_log
= (conf
->channel_type
& STEDMA40_CHANNEL_IN_OPER_MODE
)
1148 == STEDMA40_CHANNEL_IN_LOG_MODE
;
1151 dev_err(&d40c
->chan
.dev
->device
, "[%s] Invalid direction.\n",
1156 if (conf
->dst_dev_type
!= STEDMA40_DEV_DST_MEMORY
&&
1157 d40c
->base
->plat_data
->dev_tx
[conf
->dst_dev_type
] == 0 &&
1158 d40c
->runtime_addr
== 0) {
1160 dev_err(&d40c
->chan
.dev
->device
,
1161 "[%s] Invalid TX channel address (%d)\n",
1162 __func__
, conf
->dst_dev_type
);
1166 if (conf
->src_dev_type
!= STEDMA40_DEV_SRC_MEMORY
&&
1167 d40c
->base
->plat_data
->dev_rx
[conf
->src_dev_type
] == 0 &&
1168 d40c
->runtime_addr
== 0) {
1169 dev_err(&d40c
->chan
.dev
->device
,
1170 "[%s] Invalid RX channel address (%d)\n",
1171 __func__
, conf
->src_dev_type
);
1175 if (conf
->dir
== STEDMA40_MEM_TO_PERIPH
&&
1176 dst_event_group
== STEDMA40_DEV_DST_MEMORY
) {
1177 dev_err(&d40c
->chan
.dev
->device
, "[%s] Invalid dst\n",
1182 if (conf
->dir
== STEDMA40_PERIPH_TO_MEM
&&
1183 src_event_group
== STEDMA40_DEV_SRC_MEMORY
) {
1184 dev_err(&d40c
->chan
.dev
->device
, "[%s] Invalid src\n",
1189 if (src_event_group
== STEDMA40_DEV_SRC_MEMORY
&&
1190 dst_event_group
== STEDMA40_DEV_DST_MEMORY
&& is_log
) {
1191 dev_err(&d40c
->chan
.dev
->device
,
1192 "[%s] No event line\n", __func__
);
1196 if (conf
->dir
== STEDMA40_PERIPH_TO_PERIPH
&&
1197 (src_event_group
!= dst_event_group
)) {
1198 dev_err(&d40c
->chan
.dev
->device
,
1199 "[%s] Invalid event group\n", __func__
);
1203 if (conf
->dir
== STEDMA40_PERIPH_TO_PERIPH
) {
1205 * DMAC HW supports it. Will be added to this driver,
1206 * in case any dma client requires it.
1208 dev_err(&d40c
->chan
.dev
->device
,
1209 "[%s] periph to periph not supported\n",
1217 static bool d40_alloc_mask_set(struct d40_phy_res
*phy
, bool is_src
,
1218 int log_event_line
, bool is_log
)
1220 unsigned long flags
;
1221 spin_lock_irqsave(&phy
->lock
, flags
);
1223 /* Physical interrupts are masked per physical full channel */
1224 if (phy
->allocated_src
== D40_ALLOC_FREE
&&
1225 phy
->allocated_dst
== D40_ALLOC_FREE
) {
1226 phy
->allocated_dst
= D40_ALLOC_PHY
;
1227 phy
->allocated_src
= D40_ALLOC_PHY
;
1233 /* Logical channel */
1235 if (phy
->allocated_src
== D40_ALLOC_PHY
)
1238 if (phy
->allocated_src
== D40_ALLOC_FREE
)
1239 phy
->allocated_src
= D40_ALLOC_LOG_FREE
;
1241 if (!(phy
->allocated_src
& (1 << log_event_line
))) {
1242 phy
->allocated_src
|= 1 << log_event_line
;
1247 if (phy
->allocated_dst
== D40_ALLOC_PHY
)
1250 if (phy
->allocated_dst
== D40_ALLOC_FREE
)
1251 phy
->allocated_dst
= D40_ALLOC_LOG_FREE
;
1253 if (!(phy
->allocated_dst
& (1 << log_event_line
))) {
1254 phy
->allocated_dst
|= 1 << log_event_line
;
1261 spin_unlock_irqrestore(&phy
->lock
, flags
);
1264 spin_unlock_irqrestore(&phy
->lock
, flags
);
1268 static bool d40_alloc_mask_free(struct d40_phy_res
*phy
, bool is_src
,
1271 unsigned long flags
;
1272 bool is_free
= false;
1274 spin_lock_irqsave(&phy
->lock
, flags
);
1275 if (!log_event_line
) {
1276 phy
->allocated_dst
= D40_ALLOC_FREE
;
1277 phy
->allocated_src
= D40_ALLOC_FREE
;
1282 /* Logical channel */
1284 phy
->allocated_src
&= ~(1 << log_event_line
);
1285 if (phy
->allocated_src
== D40_ALLOC_LOG_FREE
)
1286 phy
->allocated_src
= D40_ALLOC_FREE
;
1288 phy
->allocated_dst
&= ~(1 << log_event_line
);
1289 if (phy
->allocated_dst
== D40_ALLOC_LOG_FREE
)
1290 phy
->allocated_dst
= D40_ALLOC_FREE
;
1293 is_free
= ((phy
->allocated_src
| phy
->allocated_dst
) ==
1297 spin_unlock_irqrestore(&phy
->lock
, flags
);
1302 static int d40_allocate_channel(struct d40_chan
*d40c
)
1307 struct d40_phy_res
*phys
;
1312 bool is_log
= (d40c
->dma_cfg
.channel_type
&
1313 STEDMA40_CHANNEL_IN_OPER_MODE
)
1314 == STEDMA40_CHANNEL_IN_LOG_MODE
;
1317 phys
= d40c
->base
->phy_res
;
1319 if (d40c
->dma_cfg
.dir
== STEDMA40_PERIPH_TO_MEM
) {
1320 dev_type
= d40c
->dma_cfg
.src_dev_type
;
1321 log_num
= 2 * dev_type
;
1323 } else if (d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_PERIPH
||
1324 d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_MEM
) {
1325 /* dst event lines are used for logical memcpy */
1326 dev_type
= d40c
->dma_cfg
.dst_dev_type
;
1327 log_num
= 2 * dev_type
+ 1;
1332 event_group
= D40_TYPE_TO_GROUP(dev_type
);
1333 event_line
= D40_TYPE_TO_EVENT(dev_type
);
1336 if (d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_MEM
) {
1337 /* Find physical half channel */
1338 for (i
= 0; i
< d40c
->base
->num_phy_chans
; i
++) {
1340 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1345 for (j
= 0; j
< d40c
->base
->num_phy_chans
; j
+= 8) {
1346 int phy_num
= j
+ event_group
* 2;
1347 for (i
= phy_num
; i
< phy_num
+ 2; i
++) {
1348 if (d40_alloc_mask_set(&phys
[i
],
1357 d40c
->phy_chan
= &phys
[i
];
1358 d40c
->log_num
= D40_PHY_CHAN
;
1364 /* Find logical channel */
1365 for (j
= 0; j
< d40c
->base
->num_phy_chans
; j
+= 8) {
1366 int phy_num
= j
+ event_group
* 2;
1368 * Spread logical channels across all available physical rather
1369 * than pack every logical channel at the first available phy
1373 for (i
= phy_num
; i
< phy_num
+ 2; i
++) {
1374 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1375 event_line
, is_log
))
1379 for (i
= phy_num
+ 1; i
>= phy_num
; i
--) {
1380 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1381 event_line
, is_log
))
1389 d40c
->phy_chan
= &phys
[i
];
1390 d40c
->log_num
= log_num
;
1394 d40c
->base
->lookup_log_chans
[d40c
->log_num
] = d40c
;
1396 d40c
->base
->lookup_phy_chans
[d40c
->phy_chan
->num
] = d40c
;
1402 static int d40_config_memcpy(struct d40_chan
*d40c
)
1404 dma_cap_mask_t cap
= d40c
->chan
.device
->cap_mask
;
1406 if (dma_has_cap(DMA_MEMCPY
, cap
) && !dma_has_cap(DMA_SLAVE
, cap
)) {
1407 d40c
->dma_cfg
= *d40c
->base
->plat_data
->memcpy_conf_log
;
1408 d40c
->dma_cfg
.src_dev_type
= STEDMA40_DEV_SRC_MEMORY
;
1409 d40c
->dma_cfg
.dst_dev_type
= d40c
->base
->plat_data
->
1410 memcpy
[d40c
->chan
.chan_id
];
1412 } else if (dma_has_cap(DMA_MEMCPY
, cap
) &&
1413 dma_has_cap(DMA_SLAVE
, cap
)) {
1414 d40c
->dma_cfg
= *d40c
->base
->plat_data
->memcpy_conf_phy
;
1416 dev_err(&d40c
->chan
.dev
->device
, "[%s] No memcpy\n",
1425 static int d40_free_dma(struct d40_chan
*d40c
)
1430 struct d40_phy_res
*phy
= d40c
->phy_chan
;
1433 struct d40_desc
*_d
;
1436 /* Terminate all queued and active transfers */
1439 /* Release client owned descriptors */
1440 if (!list_empty(&d40c
->client
))
1441 list_for_each_entry_safe(d
, _d
, &d40c
->client
, node
) {
1442 d40_pool_lli_free(d
);
1444 d40_desc_free(d40c
, d
);
1448 dev_err(&d40c
->chan
.dev
->device
, "[%s] phy == null\n",
1453 if (phy
->allocated_src
== D40_ALLOC_FREE
&&
1454 phy
->allocated_dst
== D40_ALLOC_FREE
) {
1455 dev_err(&d40c
->chan
.dev
->device
, "[%s] channel already free\n",
1460 if (d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_PERIPH
||
1461 d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_MEM
) {
1462 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dst_dev_type
);
1464 } else if (d40c
->dma_cfg
.dir
== STEDMA40_PERIPH_TO_MEM
) {
1465 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.src_dev_type
);
1468 dev_err(&d40c
->chan
.dev
->device
,
1469 "[%s] Unknown direction\n", __func__
);
1473 res
= d40_channel_execute_command(d40c
, D40_DMA_SUSPEND_REQ
);
1475 dev_err(&d40c
->chan
.dev
->device
, "[%s] suspend failed\n",
1480 if (d40c
->log_num
!= D40_PHY_CHAN
) {
1481 /* Release logical channel, deactivate the event line */
1483 d40_config_set_event(d40c
, false);
1484 d40c
->base
->lookup_log_chans
[d40c
->log_num
] = NULL
;
1487 * Check if there are more logical allocation
1488 * on this phy channel.
1490 if (!d40_alloc_mask_free(phy
, is_src
, event
)) {
1491 /* Resume the other logical channels if any */
1492 if (d40_chan_has_events(d40c
)) {
1493 res
= d40_channel_execute_command(d40c
,
1496 dev_err(&d40c
->chan
.dev
->device
,
1497 "[%s] Executing RUN command\n",
1505 (void) d40_alloc_mask_free(phy
, is_src
, 0);
1508 /* Release physical channel */
1509 res
= d40_channel_execute_command(d40c
, D40_DMA_STOP
);
1511 dev_err(&d40c
->chan
.dev
->device
,
1512 "[%s] Failed to stop channel\n", __func__
);
1515 d40c
->phy_chan
= NULL
;
1516 /* Invalidate channel type */
1517 d40c
->dma_cfg
.channel_type
= 0;
1518 d40c
->base
->lookup_phy_chans
[phy
->num
] = NULL
;
1523 static bool d40_is_paused(struct d40_chan
*d40c
)
1525 bool is_paused
= false;
1526 unsigned long flags
;
1527 void __iomem
*active_reg
;
1531 spin_lock_irqsave(&d40c
->lock
, flags
);
1533 if (d40c
->log_num
== D40_PHY_CHAN
) {
1534 if (d40c
->phy_chan
->num
% 2 == 0)
1535 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
1537 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
1539 status
= (readl(active_reg
) &
1540 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1541 D40_CHAN_POS(d40c
->phy_chan
->num
);
1542 if (status
== D40_DMA_SUSPENDED
|| status
== D40_DMA_STOP
)
1548 if (d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_PERIPH
||
1549 d40c
->dma_cfg
.dir
== STEDMA40_MEM_TO_MEM
) {
1550 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dst_dev_type
);
1551 status
= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
1552 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
1553 D40_CHAN_REG_SDLNK
);
1554 } else if (d40c
->dma_cfg
.dir
== STEDMA40_PERIPH_TO_MEM
) {
1555 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.src_dev_type
);
1556 status
= readl(d40c
->base
->virtbase
+ D40_DREG_PCBASE
+
1557 d40c
->phy_chan
->num
* D40_DREG_PCDELTA
+
1558 D40_CHAN_REG_SSLNK
);
1560 dev_err(&d40c
->chan
.dev
->device
,
1561 "[%s] Unknown direction\n", __func__
);
1565 status
= (status
& D40_EVENTLINE_MASK(event
)) >>
1566 D40_EVENTLINE_POS(event
);
1568 if (status
!= D40_DMA_RUN
)
1571 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1577 static u32
stedma40_residue(struct dma_chan
*chan
)
1579 struct d40_chan
*d40c
=
1580 container_of(chan
, struct d40_chan
, chan
);
1582 unsigned long flags
;
1584 spin_lock_irqsave(&d40c
->lock
, flags
);
1585 bytes_left
= d40_residue(d40c
);
1586 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1591 /* Public DMA functions in addition to the DMA engine framework */
1593 int stedma40_set_psize(struct dma_chan
*chan
,
1597 struct d40_chan
*d40c
=
1598 container_of(chan
, struct d40_chan
, chan
);
1599 unsigned long flags
;
1601 spin_lock_irqsave(&d40c
->lock
, flags
);
1603 if (d40c
->log_num
!= D40_PHY_CHAN
) {
1604 d40c
->log_def
.lcsp1
&= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK
;
1605 d40c
->log_def
.lcsp3
&= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK
;
1606 d40c
->log_def
.lcsp1
|= src_psize
<<
1607 D40_MEM_LCSP1_SCFG_PSIZE_POS
;
1608 d40c
->log_def
.lcsp3
|= dst_psize
<<
1609 D40_MEM_LCSP1_SCFG_PSIZE_POS
;
1613 if (src_psize
== STEDMA40_PSIZE_PHY_1
)
1614 d40c
->src_def_cfg
&= ~(1 << D40_SREG_CFG_PHY_PEN_POS
);
1616 d40c
->src_def_cfg
|= 1 << D40_SREG_CFG_PHY_PEN_POS
;
1617 d40c
->src_def_cfg
&= ~(STEDMA40_PSIZE_PHY_16
<<
1618 D40_SREG_CFG_PSIZE_POS
);
1619 d40c
->src_def_cfg
|= src_psize
<< D40_SREG_CFG_PSIZE_POS
;
1622 if (dst_psize
== STEDMA40_PSIZE_PHY_1
)
1623 d40c
->dst_def_cfg
&= ~(1 << D40_SREG_CFG_PHY_PEN_POS
);
1625 d40c
->dst_def_cfg
|= 1 << D40_SREG_CFG_PHY_PEN_POS
;
1626 d40c
->dst_def_cfg
&= ~(STEDMA40_PSIZE_PHY_16
<<
1627 D40_SREG_CFG_PSIZE_POS
);
1628 d40c
->dst_def_cfg
|= dst_psize
<< D40_SREG_CFG_PSIZE_POS
;
1631 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1634 EXPORT_SYMBOL(stedma40_set_psize
);
1636 struct dma_async_tx_descriptor
*stedma40_memcpy_sg(struct dma_chan
*chan
,
1637 struct scatterlist
*sgl_dst
,
1638 struct scatterlist
*sgl_src
,
1639 unsigned int sgl_len
,
1640 unsigned long dma_flags
)
1643 struct d40_desc
*d40d
;
1644 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
,
1646 unsigned long flags
;
1648 if (d40c
->phy_chan
== NULL
) {
1649 dev_err(&d40c
->chan
.dev
->device
,
1650 "[%s] Unallocated channel.\n", __func__
);
1651 return ERR_PTR(-EINVAL
);
1654 spin_lock_irqsave(&d40c
->lock
, flags
);
1655 d40d
= d40_desc_get(d40c
);
1660 d40d
->lli_len
= sgl_len
;
1661 d40d
->lli_current
= 0;
1662 d40d
->txd
.flags
= dma_flags
;
1664 if (d40c
->log_num
!= D40_PHY_CHAN
) {
1666 if (d40_pool_lli_alloc(d40d
, sgl_len
, true) < 0) {
1667 dev_err(&d40c
->chan
.dev
->device
,
1668 "[%s] Out of memory\n", __func__
);
1672 (void) d40_log_sg_to_lli(sgl_src
,
1675 d40c
->log_def
.lcsp1
,
1676 d40c
->dma_cfg
.src_info
.data_width
);
1678 (void) d40_log_sg_to_lli(sgl_dst
,
1681 d40c
->log_def
.lcsp3
,
1682 d40c
->dma_cfg
.dst_info
.data_width
);
1684 if (d40_pool_lli_alloc(d40d
, sgl_len
, false) < 0) {
1685 dev_err(&d40c
->chan
.dev
->device
,
1686 "[%s] Out of memory\n", __func__
);
1690 res
= d40_phy_sg_to_lli(sgl_src
,
1694 virt_to_phys(d40d
->lli_phy
.src
),
1696 d40c
->dma_cfg
.src_info
.data_width
,
1697 d40c
->dma_cfg
.src_info
.psize
);
1702 res
= d40_phy_sg_to_lli(sgl_dst
,
1706 virt_to_phys(d40d
->lli_phy
.dst
),
1708 d40c
->dma_cfg
.dst_info
.data_width
,
1709 d40c
->dma_cfg
.dst_info
.psize
);
1714 (void) dma_map_single(d40c
->base
->dev
, d40d
->lli_phy
.src
,
1715 d40d
->lli_pool
.size
, DMA_TO_DEVICE
);
1718 dma_async_tx_descriptor_init(&d40d
->txd
, chan
);
1720 d40d
->txd
.tx_submit
= d40_tx_submit
;
1722 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1726 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1729 EXPORT_SYMBOL(stedma40_memcpy_sg
);
1731 bool stedma40_filter(struct dma_chan
*chan
, void *data
)
1733 struct stedma40_chan_cfg
*info
= data
;
1734 struct d40_chan
*d40c
=
1735 container_of(chan
, struct d40_chan
, chan
);
1739 err
= d40_validate_conf(d40c
, info
);
1741 d40c
->dma_cfg
= *info
;
1743 err
= d40_config_memcpy(d40c
);
1747 EXPORT_SYMBOL(stedma40_filter
);
1749 /* DMA ENGINE functions */
1750 static int d40_alloc_chan_resources(struct dma_chan
*chan
)
1753 unsigned long flags
;
1754 struct d40_chan
*d40c
=
1755 container_of(chan
, struct d40_chan
, chan
);
1757 spin_lock_irqsave(&d40c
->lock
, flags
);
1759 d40c
->completed
= chan
->cookie
= 1;
1762 * If no dma configuration is set (channel_type == 0)
1763 * use default configuration (memcpy)
1765 if (d40c
->dma_cfg
.channel_type
== 0) {
1767 err
= d40_config_memcpy(d40c
);
1769 dev_err(&d40c
->chan
.dev
->device
,
1770 "[%s] Failed to configure memcpy channel\n",
1775 is_free_phy
= (d40c
->phy_chan
== NULL
);
1777 err
= d40_allocate_channel(d40c
);
1779 dev_err(&d40c
->chan
.dev
->device
,
1780 "[%s] Failed to allocate channel\n", __func__
);
1784 /* Fill in basic CFG register values */
1785 d40_phy_cfg(&d40c
->dma_cfg
, &d40c
->src_def_cfg
,
1786 &d40c
->dst_def_cfg
, d40c
->log_num
!= D40_PHY_CHAN
);
1788 if (d40c
->log_num
!= D40_PHY_CHAN
) {
1789 d40_log_cfg(&d40c
->dma_cfg
,
1790 &d40c
->log_def
.lcsp1
, &d40c
->log_def
.lcsp3
);
1792 if (d40c
->dma_cfg
.dir
== STEDMA40_PERIPH_TO_MEM
)
1793 d40c
->lcpa
= d40c
->base
->lcpa_base
+
1794 d40c
->dma_cfg
.src_dev_type
* D40_LCPA_CHAN_SIZE
;
1796 d40c
->lcpa
= d40c
->base
->lcpa_base
+
1797 d40c
->dma_cfg
.dst_dev_type
*
1798 D40_LCPA_CHAN_SIZE
+ D40_LCPA_CHAN_DST_DELTA
;
1802 * Only write channel configuration to the DMA if the physical
1803 * resource is free. In case of multiple logical channels
1804 * on the same physical resource, only the first write is necessary.
1807 d40_config_write(d40c
);
1809 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1813 static void d40_free_chan_resources(struct dma_chan
*chan
)
1815 struct d40_chan
*d40c
=
1816 container_of(chan
, struct d40_chan
, chan
);
1818 unsigned long flags
;
1820 if (d40c
->phy_chan
== NULL
) {
1821 dev_err(&d40c
->chan
.dev
->device
,
1822 "[%s] Cannot free unallocated channel\n", __func__
);
1827 spin_lock_irqsave(&d40c
->lock
, flags
);
1829 err
= d40_free_dma(d40c
);
1832 dev_err(&d40c
->chan
.dev
->device
,
1833 "[%s] Failed to free channel\n", __func__
);
1834 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1837 static struct dma_async_tx_descriptor
*d40_prep_memcpy(struct dma_chan
*chan
,
1841 unsigned long dma_flags
)
1843 struct d40_desc
*d40d
;
1844 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
,
1846 unsigned long flags
;
1849 if (d40c
->phy_chan
== NULL
) {
1850 dev_err(&d40c
->chan
.dev
->device
,
1851 "[%s] Channel is not allocated.\n", __func__
);
1852 return ERR_PTR(-EINVAL
);
1855 spin_lock_irqsave(&d40c
->lock
, flags
);
1856 d40d
= d40_desc_get(d40c
);
1859 dev_err(&d40c
->chan
.dev
->device
,
1860 "[%s] Descriptor is NULL\n", __func__
);
1864 d40d
->txd
.flags
= dma_flags
;
1866 dma_async_tx_descriptor_init(&d40d
->txd
, chan
);
1868 d40d
->txd
.tx_submit
= d40_tx_submit
;
1870 if (d40c
->log_num
!= D40_PHY_CHAN
) {
1872 if (d40_pool_lli_alloc(d40d
, 1, true) < 0) {
1873 dev_err(&d40c
->chan
.dev
->device
,
1874 "[%s] Out of memory\n", __func__
);
1878 d40d
->lli_current
= 0;
1880 d40_log_fill_lli(d40d
->lli_log
.src
,
1883 d40c
->log_def
.lcsp1
,
1884 d40c
->dma_cfg
.src_info
.data_width
,
1887 d40_log_fill_lli(d40d
->lli_log
.dst
,
1890 d40c
->log_def
.lcsp3
,
1891 d40c
->dma_cfg
.dst_info
.data_width
,
1896 if (d40_pool_lli_alloc(d40d
, 1, false) < 0) {
1897 dev_err(&d40c
->chan
.dev
->device
,
1898 "[%s] Out of memory\n", __func__
);
1902 err
= d40_phy_fill_lli(d40d
->lli_phy
.src
,
1905 d40c
->dma_cfg
.src_info
.psize
,
1909 d40c
->dma_cfg
.src_info
.data_width
,
1914 err
= d40_phy_fill_lli(d40d
->lli_phy
.dst
,
1917 d40c
->dma_cfg
.dst_info
.psize
,
1921 d40c
->dma_cfg
.dst_info
.data_width
,
1927 (void) dma_map_single(d40c
->base
->dev
, d40d
->lli_phy
.src
,
1928 d40d
->lli_pool
.size
, DMA_TO_DEVICE
);
1931 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1935 dev_err(&d40c
->chan
.dev
->device
,
1936 "[%s] Failed filling in PHY LLI\n", __func__
);
1937 d40_pool_lli_free(d40d
);
1939 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1943 static int d40_prep_slave_sg_log(struct d40_desc
*d40d
,
1944 struct d40_chan
*d40c
,
1945 struct scatterlist
*sgl
,
1946 unsigned int sg_len
,
1947 enum dma_data_direction direction
,
1948 unsigned long dma_flags
)
1950 dma_addr_t dev_addr
= 0;
1953 if (d40_pool_lli_alloc(d40d
, sg_len
, true) < 0) {
1954 dev_err(&d40c
->chan
.dev
->device
,
1955 "[%s] Out of memory\n", __func__
);
1959 d40d
->lli_len
= sg_len
;
1960 d40d
->lli_current
= 0;
1962 if (direction
== DMA_FROM_DEVICE
)
1963 if (d40c
->runtime_addr
)
1964 dev_addr
= d40c
->runtime_addr
;
1966 dev_addr
= d40c
->base
->plat_data
->dev_rx
[d40c
->dma_cfg
.src_dev_type
];
1967 else if (direction
== DMA_TO_DEVICE
)
1968 if (d40c
->runtime_addr
)
1969 dev_addr
= d40c
->runtime_addr
;
1971 dev_addr
= d40c
->base
->plat_data
->dev_tx
[d40c
->dma_cfg
.dst_dev_type
];
1976 total_size
= d40_log_sg_to_dev(sgl
, sg_len
,
1979 d40c
->dma_cfg
.src_info
.data_width
,
1980 d40c
->dma_cfg
.dst_info
.data_width
,
1990 static int d40_prep_slave_sg_phy(struct d40_desc
*d40d
,
1991 struct d40_chan
*d40c
,
1992 struct scatterlist
*sgl
,
1993 unsigned int sgl_len
,
1994 enum dma_data_direction direction
,
1995 unsigned long dma_flags
)
1997 dma_addr_t src_dev_addr
;
1998 dma_addr_t dst_dev_addr
;
2001 if (d40_pool_lli_alloc(d40d
, sgl_len
, false) < 0) {
2002 dev_err(&d40c
->chan
.dev
->device
,
2003 "[%s] Out of memory\n", __func__
);
2007 d40d
->lli_len
= sgl_len
;
2008 d40d
->lli_current
= 0;
2010 if (direction
== DMA_FROM_DEVICE
) {
2012 if (d40c
->runtime_addr
)
2013 src_dev_addr
= d40c
->runtime_addr
;
2015 src_dev_addr
= d40c
->base
->plat_data
->dev_rx
[d40c
->dma_cfg
.src_dev_type
];
2016 } else if (direction
== DMA_TO_DEVICE
) {
2017 if (d40c
->runtime_addr
)
2018 dst_dev_addr
= d40c
->runtime_addr
;
2020 dst_dev_addr
= d40c
->base
->plat_data
->dev_tx
[d40c
->dma_cfg
.dst_dev_type
];
2025 res
= d40_phy_sg_to_lli(sgl
,
2029 virt_to_phys(d40d
->lli_phy
.src
),
2031 d40c
->dma_cfg
.src_info
.data_width
,
2032 d40c
->dma_cfg
.src_info
.psize
);
2036 res
= d40_phy_sg_to_lli(sgl
,
2040 virt_to_phys(d40d
->lli_phy
.dst
),
2042 d40c
->dma_cfg
.dst_info
.data_width
,
2043 d40c
->dma_cfg
.dst_info
.psize
);
2047 (void) dma_map_single(d40c
->base
->dev
, d40d
->lli_phy
.src
,
2048 d40d
->lli_pool
.size
, DMA_TO_DEVICE
);
2052 static struct dma_async_tx_descriptor
*d40_prep_slave_sg(struct dma_chan
*chan
,
2053 struct scatterlist
*sgl
,
2054 unsigned int sg_len
,
2055 enum dma_data_direction direction
,
2056 unsigned long dma_flags
)
2058 struct d40_desc
*d40d
;
2059 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
,
2061 unsigned long flags
;
2064 if (d40c
->phy_chan
== NULL
) {
2065 dev_err(&d40c
->chan
.dev
->device
,
2066 "[%s] Cannot prepare unallocated channel\n", __func__
);
2067 return ERR_PTR(-EINVAL
);
2070 if (d40c
->dma_cfg
.pre_transfer
)
2071 d40c
->dma_cfg
.pre_transfer(chan
,
2072 d40c
->dma_cfg
.pre_transfer_data
,
2075 spin_lock_irqsave(&d40c
->lock
, flags
);
2076 d40d
= d40_desc_get(d40c
);
2077 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2082 if (d40c
->log_num
!= D40_PHY_CHAN
)
2083 err
= d40_prep_slave_sg_log(d40d
, d40c
, sgl
, sg_len
,
2084 direction
, dma_flags
);
2086 err
= d40_prep_slave_sg_phy(d40d
, d40c
, sgl
, sg_len
,
2087 direction
, dma_flags
);
2089 dev_err(&d40c
->chan
.dev
->device
,
2090 "[%s] Failed to prepare %s slave sg job: %d\n",
2092 d40c
->log_num
!= D40_PHY_CHAN
? "log" : "phy", err
);
2096 d40d
->txd
.flags
= dma_flags
;
2098 dma_async_tx_descriptor_init(&d40d
->txd
, chan
);
2100 d40d
->txd
.tx_submit
= d40_tx_submit
;
2105 static enum dma_status
d40_tx_status(struct dma_chan
*chan
,
2106 dma_cookie_t cookie
,
2107 struct dma_tx_state
*txstate
)
2109 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2110 dma_cookie_t last_used
;
2111 dma_cookie_t last_complete
;
2114 if (d40c
->phy_chan
== NULL
) {
2115 dev_err(&d40c
->chan
.dev
->device
,
2116 "[%s] Cannot read status of unallocated channel\n",
2121 last_complete
= d40c
->completed
;
2122 last_used
= chan
->cookie
;
2124 if (d40_is_paused(d40c
))
2127 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
2129 dma_set_tx_state(txstate
, last_complete
, last_used
,
2130 stedma40_residue(chan
));
2135 static void d40_issue_pending(struct dma_chan
*chan
)
2137 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2138 unsigned long flags
;
2140 if (d40c
->phy_chan
== NULL
) {
2141 dev_err(&d40c
->chan
.dev
->device
,
2142 "[%s] Channel is not allocated!\n", __func__
);
2146 spin_lock_irqsave(&d40c
->lock
, flags
);
2148 /* Busy means that pending jobs are already being processed */
2150 (void) d40_queue_start(d40c
);
2152 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2155 /* Runtime reconfiguration extension */
2156 static void d40_set_runtime_config(struct dma_chan
*chan
,
2157 struct dma_slave_config
*config
)
2159 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2160 struct stedma40_chan_cfg
*cfg
= &d40c
->dma_cfg
;
2161 enum dma_slave_buswidth config_addr_width
;
2162 dma_addr_t config_addr
;
2163 u32 config_maxburst
;
2164 enum stedma40_periph_data_width addr_width
;
2167 if (config
->direction
== DMA_FROM_DEVICE
) {
2168 dma_addr_t dev_addr_rx
=
2169 d40c
->base
->plat_data
->dev_rx
[cfg
->src_dev_type
];
2171 config_addr
= config
->src_addr
;
2173 dev_dbg(d40c
->base
->dev
,
2174 "channel has a pre-wired RX address %08x "
2175 "overriding with %08x\n",
2176 dev_addr_rx
, config_addr
);
2177 if (cfg
->dir
!= STEDMA40_PERIPH_TO_MEM
)
2178 dev_dbg(d40c
->base
->dev
,
2179 "channel was not configured for peripheral "
2180 "to memory transfer (%d) overriding\n",
2182 cfg
->dir
= STEDMA40_PERIPH_TO_MEM
;
2184 config_addr_width
= config
->src_addr_width
;
2185 config_maxburst
= config
->src_maxburst
;
2187 } else if (config
->direction
== DMA_TO_DEVICE
) {
2188 dma_addr_t dev_addr_tx
=
2189 d40c
->base
->plat_data
->dev_tx
[cfg
->dst_dev_type
];
2191 config_addr
= config
->dst_addr
;
2193 dev_dbg(d40c
->base
->dev
,
2194 "channel has a pre-wired TX address %08x "
2195 "overriding with %08x\n",
2196 dev_addr_tx
, config_addr
);
2197 if (cfg
->dir
!= STEDMA40_MEM_TO_PERIPH
)
2198 dev_dbg(d40c
->base
->dev
,
2199 "channel was not configured for memory "
2200 "to peripheral transfer (%d) overriding\n",
2202 cfg
->dir
= STEDMA40_MEM_TO_PERIPH
;
2204 config_addr_width
= config
->dst_addr_width
;
2205 config_maxburst
= config
->dst_maxburst
;
2208 dev_err(d40c
->base
->dev
,
2209 "unrecognized channel direction %d\n",
2214 switch (config_addr_width
) {
2215 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
2216 addr_width
= STEDMA40_BYTE_WIDTH
;
2218 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
2219 addr_width
= STEDMA40_HALFWORD_WIDTH
;
2221 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
2222 addr_width
= STEDMA40_WORD_WIDTH
;
2224 case DMA_SLAVE_BUSWIDTH_8_BYTES
:
2225 addr_width
= STEDMA40_DOUBLEWORD_WIDTH
;
2228 dev_err(d40c
->base
->dev
,
2229 "illegal peripheral address width "
2231 config
->src_addr_width
);
2235 if (config_maxburst
>= 16)
2236 psize
= STEDMA40_PSIZE_LOG_16
;
2237 else if (config_maxburst
>= 8)
2238 psize
= STEDMA40_PSIZE_LOG_8
;
2239 else if (config_maxburst
>= 4)
2240 psize
= STEDMA40_PSIZE_LOG_4
;
2242 psize
= STEDMA40_PSIZE_LOG_1
;
2244 /* Set up all the endpoint configs */
2245 cfg
->src_info
.data_width
= addr_width
;
2246 cfg
->src_info
.psize
= psize
;
2247 cfg
->src_info
.endianess
= STEDMA40_LITTLE_ENDIAN
;
2248 cfg
->src_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
;
2249 cfg
->dst_info
.data_width
= addr_width
;
2250 cfg
->dst_info
.psize
= psize
;
2251 cfg
->dst_info
.endianess
= STEDMA40_LITTLE_ENDIAN
;
2252 cfg
->dst_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
;
2254 /* These settings will take precedence later */
2255 d40c
->runtime_addr
= config_addr
;
2256 d40c
->runtime_direction
= config
->direction
;
2257 dev_dbg(d40c
->base
->dev
,
2258 "configured channel %s for %s, data width %d, "
2259 "maxburst %d bytes, LE, no flow control\n",
2260 dma_chan_name(chan
),
2261 (config
->direction
== DMA_FROM_DEVICE
) ? "RX" : "TX",
2266 static int d40_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
2269 unsigned long flags
;
2270 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2272 if (d40c
->phy_chan
== NULL
) {
2273 dev_err(&d40c
->chan
.dev
->device
,
2274 "[%s] Channel is not allocated!\n", __func__
);
2279 case DMA_TERMINATE_ALL
:
2280 spin_lock_irqsave(&d40c
->lock
, flags
);
2282 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2285 return d40_pause(chan
);
2287 return d40_resume(chan
);
2288 case DMA_SLAVE_CONFIG
:
2289 d40_set_runtime_config(chan
,
2290 (struct dma_slave_config
*) arg
);
2296 /* Other commands are unimplemented */
2300 /* Initialization functions */
2302 static void __init
d40_chan_init(struct d40_base
*base
, struct dma_device
*dma
,
2303 struct d40_chan
*chans
, int offset
,
2307 struct d40_chan
*d40c
;
2309 INIT_LIST_HEAD(&dma
->channels
);
2311 for (i
= offset
; i
< offset
+ num_chans
; i
++) {
2314 d40c
->chan
.device
= dma
;
2316 spin_lock_init(&d40c
->lock
);
2318 d40c
->log_num
= D40_PHY_CHAN
;
2320 INIT_LIST_HEAD(&d40c
->active
);
2321 INIT_LIST_HEAD(&d40c
->queue
);
2322 INIT_LIST_HEAD(&d40c
->client
);
2324 tasklet_init(&d40c
->tasklet
, dma_tasklet
,
2325 (unsigned long) d40c
);
2327 list_add_tail(&d40c
->chan
.device_node
,
2332 static int __init
d40_dmaengine_init(struct d40_base
*base
,
2333 int num_reserved_chans
)
2337 d40_chan_init(base
, &base
->dma_slave
, base
->log_chans
,
2338 0, base
->num_log_chans
);
2340 dma_cap_zero(base
->dma_slave
.cap_mask
);
2341 dma_cap_set(DMA_SLAVE
, base
->dma_slave
.cap_mask
);
2343 base
->dma_slave
.device_alloc_chan_resources
= d40_alloc_chan_resources
;
2344 base
->dma_slave
.device_free_chan_resources
= d40_free_chan_resources
;
2345 base
->dma_slave
.device_prep_dma_memcpy
= d40_prep_memcpy
;
2346 base
->dma_slave
.device_prep_slave_sg
= d40_prep_slave_sg
;
2347 base
->dma_slave
.device_tx_status
= d40_tx_status
;
2348 base
->dma_slave
.device_issue_pending
= d40_issue_pending
;
2349 base
->dma_slave
.device_control
= d40_control
;
2350 base
->dma_slave
.dev
= base
->dev
;
2352 err
= dma_async_device_register(&base
->dma_slave
);
2356 "[%s] Failed to register slave channels\n",
2361 d40_chan_init(base
, &base
->dma_memcpy
, base
->log_chans
,
2362 base
->num_log_chans
, base
->plat_data
->memcpy_len
);
2364 dma_cap_zero(base
->dma_memcpy
.cap_mask
);
2365 dma_cap_set(DMA_MEMCPY
, base
->dma_memcpy
.cap_mask
);
2367 base
->dma_memcpy
.device_alloc_chan_resources
= d40_alloc_chan_resources
;
2368 base
->dma_memcpy
.device_free_chan_resources
= d40_free_chan_resources
;
2369 base
->dma_memcpy
.device_prep_dma_memcpy
= d40_prep_memcpy
;
2370 base
->dma_memcpy
.device_prep_slave_sg
= d40_prep_slave_sg
;
2371 base
->dma_memcpy
.device_tx_status
= d40_tx_status
;
2372 base
->dma_memcpy
.device_issue_pending
= d40_issue_pending
;
2373 base
->dma_memcpy
.device_control
= d40_control
;
2374 base
->dma_memcpy
.dev
= base
->dev
;
2376 * This controller can only access address at even
2377 * 32bit boundaries, i.e. 2^2
2379 base
->dma_memcpy
.copy_align
= 2;
2381 err
= dma_async_device_register(&base
->dma_memcpy
);
2385 "[%s] Failed to regsiter memcpy only channels\n",
2390 d40_chan_init(base
, &base
->dma_both
, base
->phy_chans
,
2391 0, num_reserved_chans
);
2393 dma_cap_zero(base
->dma_both
.cap_mask
);
2394 dma_cap_set(DMA_SLAVE
, base
->dma_both
.cap_mask
);
2395 dma_cap_set(DMA_MEMCPY
, base
->dma_both
.cap_mask
);
2397 base
->dma_both
.device_alloc_chan_resources
= d40_alloc_chan_resources
;
2398 base
->dma_both
.device_free_chan_resources
= d40_free_chan_resources
;
2399 base
->dma_both
.device_prep_dma_memcpy
= d40_prep_memcpy
;
2400 base
->dma_both
.device_prep_slave_sg
= d40_prep_slave_sg
;
2401 base
->dma_both
.device_tx_status
= d40_tx_status
;
2402 base
->dma_both
.device_issue_pending
= d40_issue_pending
;
2403 base
->dma_both
.device_control
= d40_control
;
2404 base
->dma_both
.dev
= base
->dev
;
2405 base
->dma_both
.copy_align
= 2;
2406 err
= dma_async_device_register(&base
->dma_both
);
2410 "[%s] Failed to register logical and physical capable channels\n",
2416 dma_async_device_unregister(&base
->dma_memcpy
);
2418 dma_async_device_unregister(&base
->dma_slave
);
2423 /* Initialization functions. */
2425 static int __init
d40_phy_res_init(struct d40_base
*base
)
2428 int num_phy_chans_avail
= 0;
2430 int odd_even_bit
= -2;
2432 val
[0] = readl(base
->virtbase
+ D40_DREG_PRSME
);
2433 val
[1] = readl(base
->virtbase
+ D40_DREG_PRSMO
);
2435 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
2436 base
->phy_res
[i
].num
= i
;
2437 odd_even_bit
+= 2 * ((i
% 2) == 0);
2438 if (((val
[i
% 2] >> odd_even_bit
) & 3) == 1) {
2439 /* Mark security only channels as occupied */
2440 base
->phy_res
[i
].allocated_src
= D40_ALLOC_PHY
;
2441 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_PHY
;
2443 base
->phy_res
[i
].allocated_src
= D40_ALLOC_FREE
;
2444 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_FREE
;
2445 num_phy_chans_avail
++;
2447 spin_lock_init(&base
->phy_res
[i
].lock
);
2450 /* Mark disabled channels as occupied */
2451 for (i
= 0; base
->plat_data
->disabled_channels
[i
] != -1; i
++) {
2452 base
->phy_res
[i
].allocated_src
= D40_ALLOC_PHY
;
2453 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_PHY
;
2454 num_phy_chans_avail
--;
2457 dev_info(base
->dev
, "%d of %d physical DMA channels available\n",
2458 num_phy_chans_avail
, base
->num_phy_chans
);
2460 /* Verify settings extended vs standard */
2461 val
[0] = readl(base
->virtbase
+ D40_DREG_PRTYP
);
2463 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
2465 if (base
->phy_res
[i
].allocated_src
== D40_ALLOC_FREE
&&
2466 (val
[0] & 0x3) != 1)
2468 "[%s] INFO: channel %d is misconfigured (%d)\n",
2469 __func__
, i
, val
[0] & 0x3);
2471 val
[0] = val
[0] >> 2;
2474 return num_phy_chans_avail
;
2477 static struct d40_base
* __init
d40_hw_detect_init(struct platform_device
*pdev
)
2479 static const struct d40_reg_val dma_id_regs
[] = {
2481 { .reg
= D40_DREG_PERIPHID0
, .val
= 0x0040},
2482 { .reg
= D40_DREG_PERIPHID1
, .val
= 0x0000},
2484 * D40_DREG_PERIPHID2 Depends on HW revision:
2485 * MOP500/HREF ED has 0x0008,
2487 * HREF V1 has 0x0028
2489 { .reg
= D40_DREG_PERIPHID3
, .val
= 0x0000},
2492 { .reg
= D40_DREG_CELLID0
, .val
= 0x000d},
2493 { .reg
= D40_DREG_CELLID1
, .val
= 0x00f0},
2494 { .reg
= D40_DREG_CELLID2
, .val
= 0x0005},
2495 { .reg
= D40_DREG_CELLID3
, .val
= 0x00b1}
2497 struct stedma40_platform_data
*plat_data
;
2498 struct clk
*clk
= NULL
;
2499 void __iomem
*virtbase
= NULL
;
2500 struct resource
*res
= NULL
;
2501 struct d40_base
*base
= NULL
;
2502 int num_log_chans
= 0;
2508 clk
= clk_get(&pdev
->dev
, NULL
);
2511 dev_err(&pdev
->dev
, "[%s] No matching clock found\n",
2518 /* Get IO for DMAC base address */
2519 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "base");
2523 if (request_mem_region(res
->start
, resource_size(res
),
2524 D40_NAME
" I/O base") == NULL
)
2527 virtbase
= ioremap(res
->start
, resource_size(res
));
2531 /* HW version check */
2532 for (i
= 0; i
< ARRAY_SIZE(dma_id_regs
); i
++) {
2533 if (dma_id_regs
[i
].val
!=
2534 readl(virtbase
+ dma_id_regs
[i
].reg
)) {
2536 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2540 readl(virtbase
+ dma_id_regs
[i
].reg
));
2545 /* Get silicon revision and designer */
2546 val
= readl(virtbase
+ D40_DREG_PERIPHID2
);
2548 if ((val
& D40_DREG_PERIPHID2_DESIGNER_MASK
) !=
2551 "[%s] Unknown designer! Got %x wanted %x\n",
2552 __func__
, val
& D40_DREG_PERIPHID2_DESIGNER_MASK
,
2557 rev
= (val
& D40_DREG_PERIPHID2_REV_MASK
) >>
2558 D40_DREG_PERIPHID2_REV_POS
;
2560 /* The number of physical channels on this HW */
2561 num_phy_chans
= 4 * (readl(virtbase
+ D40_DREG_ICFG
) & 0x7) + 4;
2563 dev_info(&pdev
->dev
, "hardware revision: %d @ 0x%x\n",
2566 plat_data
= pdev
->dev
.platform_data
;
2568 /* Count the number of logical channels in use */
2569 for (i
= 0; i
< plat_data
->dev_len
; i
++)
2570 if (plat_data
->dev_rx
[i
] != 0)
2573 for (i
= 0; i
< plat_data
->dev_len
; i
++)
2574 if (plat_data
->dev_tx
[i
] != 0)
2577 base
= kzalloc(ALIGN(sizeof(struct d40_base
), 4) +
2578 (num_phy_chans
+ num_log_chans
+ plat_data
->memcpy_len
) *
2579 sizeof(struct d40_chan
), GFP_KERNEL
);
2582 dev_err(&pdev
->dev
, "[%s] Out of memory\n", __func__
);
2588 base
->num_phy_chans
= num_phy_chans
;
2589 base
->num_log_chans
= num_log_chans
;
2590 base
->phy_start
= res
->start
;
2591 base
->phy_size
= resource_size(res
);
2592 base
->virtbase
= virtbase
;
2593 base
->plat_data
= plat_data
;
2594 base
->dev
= &pdev
->dev
;
2595 base
->phy_chans
= ((void *)base
) + ALIGN(sizeof(struct d40_base
), 4);
2596 base
->log_chans
= &base
->phy_chans
[num_phy_chans
];
2598 base
->phy_res
= kzalloc(num_phy_chans
* sizeof(struct d40_phy_res
),
2603 base
->lookup_phy_chans
= kzalloc(num_phy_chans
*
2604 sizeof(struct d40_chan
*),
2606 if (!base
->lookup_phy_chans
)
2609 if (num_log_chans
+ plat_data
->memcpy_len
) {
2611 * The max number of logical channels are event lines for all
2612 * src devices and dst devices
2614 base
->lookup_log_chans
= kzalloc(plat_data
->dev_len
* 2 *
2615 sizeof(struct d40_chan
*),
2617 if (!base
->lookup_log_chans
)
2621 base
->lcla_pool
.alloc_map
= kzalloc(num_phy_chans
*
2622 sizeof(struct d40_desc
*) *
2623 D40_LCLA_LINK_PER_EVENT_GRP
,
2625 if (!base
->lcla_pool
.alloc_map
)
2628 base
->desc_slab
= kmem_cache_create(D40_NAME
, sizeof(struct d40_desc
),
2629 0, SLAB_HWCACHE_ALIGN
,
2631 if (base
->desc_slab
== NULL
)
2644 release_mem_region(res
->start
,
2645 resource_size(res
));
2650 kfree(base
->lcla_pool
.alloc_map
);
2651 kfree(base
->lookup_log_chans
);
2652 kfree(base
->lookup_phy_chans
);
2653 kfree(base
->phy_res
);
2660 static void __init
d40_hw_init(struct d40_base
*base
)
2663 static const struct d40_reg_val dma_init_reg
[] = {
2664 /* Clock every part of the DMA block from start */
2665 { .reg
= D40_DREG_GCC
, .val
= 0x0000ff01},
2667 /* Interrupts on all logical channels */
2668 { .reg
= D40_DREG_LCMIS0
, .val
= 0xFFFFFFFF},
2669 { .reg
= D40_DREG_LCMIS1
, .val
= 0xFFFFFFFF},
2670 { .reg
= D40_DREG_LCMIS2
, .val
= 0xFFFFFFFF},
2671 { .reg
= D40_DREG_LCMIS3
, .val
= 0xFFFFFFFF},
2672 { .reg
= D40_DREG_LCICR0
, .val
= 0xFFFFFFFF},
2673 { .reg
= D40_DREG_LCICR1
, .val
= 0xFFFFFFFF},
2674 { .reg
= D40_DREG_LCICR2
, .val
= 0xFFFFFFFF},
2675 { .reg
= D40_DREG_LCICR3
, .val
= 0xFFFFFFFF},
2676 { .reg
= D40_DREG_LCTIS0
, .val
= 0xFFFFFFFF},
2677 { .reg
= D40_DREG_LCTIS1
, .val
= 0xFFFFFFFF},
2678 { .reg
= D40_DREG_LCTIS2
, .val
= 0xFFFFFFFF},
2679 { .reg
= D40_DREG_LCTIS3
, .val
= 0xFFFFFFFF}
2682 u32 prmseo
[2] = {0, 0};
2683 u32 activeo
[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2687 for (i
= 0; i
< ARRAY_SIZE(dma_init_reg
); i
++)
2688 writel(dma_init_reg
[i
].val
,
2689 base
->virtbase
+ dma_init_reg
[i
].reg
);
2691 /* Configure all our dma channels to default settings */
2692 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
2694 activeo
[i
% 2] = activeo
[i
% 2] << 2;
2696 if (base
->phy_res
[base
->num_phy_chans
- i
- 1].allocated_src
2698 activeo
[i
% 2] |= 3;
2702 /* Enable interrupt # */
2703 pcmis
= (pcmis
<< 1) | 1;
2705 /* Clear interrupt # */
2706 pcicr
= (pcicr
<< 1) | 1;
2708 /* Set channel to physical mode */
2709 prmseo
[i
% 2] = prmseo
[i
% 2] << 2;
2714 writel(prmseo
[1], base
->virtbase
+ D40_DREG_PRMSE
);
2715 writel(prmseo
[0], base
->virtbase
+ D40_DREG_PRMSO
);
2716 writel(activeo
[1], base
->virtbase
+ D40_DREG_ACTIVE
);
2717 writel(activeo
[0], base
->virtbase
+ D40_DREG_ACTIVO
);
2719 /* Write which interrupt to enable */
2720 writel(pcmis
, base
->virtbase
+ D40_DREG_PCMIS
);
2722 /* Write which interrupt to clear */
2723 writel(pcicr
, base
->virtbase
+ D40_DREG_PCICR
);
2727 static int __init
d40_lcla_allocate(struct d40_base
*base
)
2729 unsigned long *page_list
;
2734 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2735 * To full fill this hardware requirement without wasting 256 kb
2736 * we allocate pages until we get an aligned one.
2738 page_list
= kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS
,
2746 /* Calculating how many pages that are required */
2747 base
->lcla_pool
.pages
= SZ_1K
* base
->num_phy_chans
/ PAGE_SIZE
;
2749 for (i
= 0; i
< MAX_LCLA_ALLOC_ATTEMPTS
; i
++) {
2750 page_list
[i
] = __get_free_pages(GFP_KERNEL
,
2751 base
->lcla_pool
.pages
);
2752 if (!page_list
[i
]) {
2755 "[%s] Failed to allocate %d pages.\n",
2756 __func__
, base
->lcla_pool
.pages
);
2758 for (j
= 0; j
< i
; j
++)
2759 free_pages(page_list
[j
], base
->lcla_pool
.pages
);
2763 if ((virt_to_phys((void *)page_list
[i
]) &
2764 (LCLA_ALIGNMENT
- 1)) == 0)
2768 for (j
= 0; j
< i
; j
++)
2769 free_pages(page_list
[j
], base
->lcla_pool
.pages
);
2771 if (i
< MAX_LCLA_ALLOC_ATTEMPTS
) {
2772 base
->lcla_pool
.base
= (void *)page_list
[i
];
2775 * After many attempts and no succees with finding the correct
2776 * alignment, try with allocating a big buffer.
2779 "[%s] Failed to get %d pages @ 18 bit align.\n",
2780 __func__
, base
->lcla_pool
.pages
);
2781 base
->lcla_pool
.base_unaligned
= kmalloc(SZ_1K
*
2782 base
->num_phy_chans
+
2785 if (!base
->lcla_pool
.base_unaligned
) {
2790 base
->lcla_pool
.base
= PTR_ALIGN(base
->lcla_pool
.base_unaligned
,
2794 writel(virt_to_phys(base
->lcla_pool
.base
),
2795 base
->virtbase
+ D40_DREG_LCLA
);
2801 static int __init
d40_probe(struct platform_device
*pdev
)
2805 struct d40_base
*base
;
2806 struct resource
*res
= NULL
;
2807 int num_reserved_chans
;
2810 base
= d40_hw_detect_init(pdev
);
2815 num_reserved_chans
= d40_phy_res_init(base
);
2817 platform_set_drvdata(pdev
, base
);
2819 spin_lock_init(&base
->interrupt_lock
);
2820 spin_lock_init(&base
->execmd_lock
);
2822 /* Get IO for logical channel parameter address */
2823 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "lcpa");
2827 "[%s] No \"lcpa\" memory resource\n",
2831 base
->lcpa_size
= resource_size(res
);
2832 base
->phy_lcpa
= res
->start
;
2834 if (request_mem_region(res
->start
, resource_size(res
),
2835 D40_NAME
" I/O lcpa") == NULL
) {
2838 "[%s] Failed to request LCPA region 0x%x-0x%x\n",
2839 __func__
, res
->start
, res
->end
);
2843 /* We make use of ESRAM memory for this. */
2844 val
= readl(base
->virtbase
+ D40_DREG_LCPA
);
2845 if (res
->start
!= val
&& val
!= 0) {
2846 dev_warn(&pdev
->dev
,
2847 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2848 __func__
, val
, res
->start
);
2850 writel(res
->start
, base
->virtbase
+ D40_DREG_LCPA
);
2852 base
->lcpa_base
= ioremap(res
->start
, resource_size(res
));
2853 if (!base
->lcpa_base
) {
2856 "[%s] Failed to ioremap LCPA region\n",
2861 ret
= d40_lcla_allocate(base
);
2863 dev_err(&pdev
->dev
, "[%s] Failed to allocate LCLA area\n",
2868 spin_lock_init(&base
->lcla_pool
.lock
);
2870 base
->irq
= platform_get_irq(pdev
, 0);
2872 ret
= request_irq(base
->irq
, d40_handle_interrupt
, 0, D40_NAME
, base
);
2875 dev_err(&pdev
->dev
, "[%s] No IRQ defined\n", __func__
);
2879 err
= d40_dmaengine_init(base
, num_reserved_chans
);
2885 dev_info(base
->dev
, "initialized\n");
2890 if (base
->desc_slab
)
2891 kmem_cache_destroy(base
->desc_slab
);
2893 iounmap(base
->virtbase
);
2894 if (!base
->lcla_pool
.base_unaligned
&& base
->lcla_pool
.base
)
2895 free_pages((unsigned long)base
->lcla_pool
.base
,
2896 base
->lcla_pool
.pages
);
2898 kfree(base
->lcla_pool
.base_unaligned
);
2901 release_mem_region(base
->phy_lcpa
,
2903 if (base
->phy_start
)
2904 release_mem_region(base
->phy_start
,
2907 clk_disable(base
->clk
);
2911 kfree(base
->lcla_pool
.alloc_map
);
2912 kfree(base
->lookup_log_chans
);
2913 kfree(base
->lookup_phy_chans
);
2914 kfree(base
->phy_res
);
2918 dev_err(&pdev
->dev
, "[%s] probe failed\n", __func__
);
2922 static struct platform_driver d40_driver
= {
2924 .owner
= THIS_MODULE
,
2929 int __init
stedma40_init(void)
2931 return platform_driver_probe(&d40_driver
, d40_probe
);
2933 arch_initcall(stedma40_init
);