projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
dmaengine: ste_dma40: Move rev error-check up to revision acquisition
[deliverable/linux.git] / drivers / dma / ste_dma40.c
1 /*
2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
6 * License terms: GNU General Public License (GPL) version 2
7 */
8
9 #include <linux/dma-mapping.h>
10 #include <linux/kernel.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/dmaengine.h>
14 #include <linux/platform_device.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/pm.h>
18 #include <linux/pm_runtime.h>
19 #include <linux/err.h>
20 #include <linux/amba/bus.h>
21 #include <linux/regulator/consumer.h>
22 #include <linux/platform_data/dma-ste-dma40.h>
23
24 #include "dmaengine.h"
25 #include "ste_dma40_ll.h"
26
27 #define D40_NAME "dma40"
28
29 #define D40_PHY_CHAN -1
30
31 /* For masking out/in 2 bit channel positions */
32 #define D40_CHAN_POS(chan) (2 * (chan / 2))
33 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
34
35 /* Maximum iterations taken before giving up suspending a channel */
36 #define D40_SUSPEND_MAX_IT 500
37
38 /* Milliseconds */
39 #define DMA40_AUTOSUSPEND_DELAY 100
40
41 /* Hardware requirement on LCLA alignment */
42 #define LCLA_ALIGNMENT 0x40000
43
44 /* Max number of links per event group */
45 #define D40_LCLA_LINK_PER_EVENT_GRP 128
46 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
47
48 /* Max number of logical channels per physical channel */
49 #define D40_MAX_LOG_CHAN_PER_PHY 32
50
51 /* Attempts before giving up to trying to get pages that are aligned */
52 #define MAX_LCLA_ALLOC_ATTEMPTS 256
53
54 /* Bit markings for allocation map */
55 #define D40_ALLOC_FREE (1 << 31)
56 #define D40_ALLOC_PHY (1 << 30)
57 #define D40_ALLOC_LOG_FREE 0
58
59 /* Reserved event lines for memcpy only. */
60 #define DB8500_DMA_MEMCPY_EV_0 51
61 #define DB8500_DMA_MEMCPY_EV_1 56
62 #define DB8500_DMA_MEMCPY_EV_2 57
63 #define DB8500_DMA_MEMCPY_EV_3 58
64 #define DB8500_DMA_MEMCPY_EV_4 59
65 #define DB8500_DMA_MEMCPY_EV_5 60
66
67 static int dma40_memcpy_channels[] = {
68 DB8500_DMA_MEMCPY_EV_0,
69 DB8500_DMA_MEMCPY_EV_1,
70 DB8500_DMA_MEMCPY_EV_2,
71 DB8500_DMA_MEMCPY_EV_3,
72 DB8500_DMA_MEMCPY_EV_4,
73 DB8500_DMA_MEMCPY_EV_5,
74 };
75
76 /* Default configuration for physcial memcpy */
77 struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
78 .mode = STEDMA40_MODE_PHYSICAL,
79 .dir = STEDMA40_MEM_TO_MEM,
80
81 .src_info.data_width = STEDMA40_BYTE_WIDTH,
82 .src_info.psize = STEDMA40_PSIZE_PHY_1,
83 .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
84
85 .dst_info.data_width = STEDMA40_BYTE_WIDTH,
86 .dst_info.psize = STEDMA40_PSIZE_PHY_1,
87 .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
88 };
89
90 /* Default configuration for logical memcpy */
91 struct stedma40_chan_cfg dma40_memcpy_conf_log = {
92 .mode = STEDMA40_MODE_LOGICAL,
93 .dir = STEDMA40_MEM_TO_MEM,
94
95 .src_info.data_width = STEDMA40_BYTE_WIDTH,
96 .src_info.psize = STEDMA40_PSIZE_LOG_1,
97 .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
98
99 .dst_info.data_width = STEDMA40_BYTE_WIDTH,
100 .dst_info.psize = STEDMA40_PSIZE_LOG_1,
101 .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
102 };
103
104 /**
105 * enum 40_command - The different commands and/or statuses.
106 *
107 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
108 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
109 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
110 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
111 */
112 enum d40_command {
113 D40_DMA_STOP = 0,
114 D40_DMA_RUN = 1,
115 D40_DMA_SUSPEND_REQ = 2,
116 D40_DMA_SUSPENDED = 3
117 };
118
119 /*
120 * enum d40_events - The different Event Enables for the event lines.
121 *
122 * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
123 * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
124 * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
125 * @D40_ROUND_EVENTLINE: Status check for event line.
126 */
127
128 enum d40_events {
129 D40_DEACTIVATE_EVENTLINE = 0,
130 D40_ACTIVATE_EVENTLINE = 1,
131 D40_SUSPEND_REQ_EVENTLINE = 2,
132 D40_ROUND_EVENTLINE = 3
133 };
134
135 /*
136 * These are the registers that has to be saved and later restored
137 * when the DMA hw is powered off.
138 * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
139 */
140 static u32 d40_backup_regs[] = {
141 D40_DREG_LCPA,
142 D40_DREG_LCLA,
143 D40_DREG_PRMSE,
144 D40_DREG_PRMSO,
145 D40_DREG_PRMOE,
146 D40_DREG_PRMOO,
147 };
148
149 #define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
150
151 /*
152 * since 9540 and 8540 has the same HW revision
153 * use v4a for 9540 or ealier
154 * use v4b for 8540 or later
155 * HW revision:
156 * DB8500ed has revision 0
157 * DB8500v1 has revision 2
158 * DB8500v2 has revision 3
159 * AP9540v1 has revision 4
160 * DB8540v1 has revision 4
161 * TODO: Check if all these registers have to be saved/restored on dma40 v4a
162 */
163 static u32 d40_backup_regs_v4a[] = {
164 D40_DREG_PSEG1,
165 D40_DREG_PSEG2,
166 D40_DREG_PSEG3,
167 D40_DREG_PSEG4,
168 D40_DREG_PCEG1,
169 D40_DREG_PCEG2,
170 D40_DREG_PCEG3,
171 D40_DREG_PCEG4,
172 D40_DREG_RSEG1,
173 D40_DREG_RSEG2,
174 D40_DREG_RSEG3,
175 D40_DREG_RSEG4,
176 D40_DREG_RCEG1,
177 D40_DREG_RCEG2,
178 D40_DREG_RCEG3,
179 D40_DREG_RCEG4,
180 };
181
182 #define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
183
184 static u32 d40_backup_regs_v4b[] = {
185 D40_DREG_CPSEG1,
186 D40_DREG_CPSEG2,
187 D40_DREG_CPSEG3,
188 D40_DREG_CPSEG4,
189 D40_DREG_CPSEG5,
190 D40_DREG_CPCEG1,
191 D40_DREG_CPCEG2,
192 D40_DREG_CPCEG3,
193 D40_DREG_CPCEG4,
194 D40_DREG_CPCEG5,
195 D40_DREG_CRSEG1,
196 D40_DREG_CRSEG2,
197 D40_DREG_CRSEG3,
198 D40_DREG_CRSEG4,
199 D40_DREG_CRSEG5,
200 D40_DREG_CRCEG1,
201 D40_DREG_CRCEG2,
202 D40_DREG_CRCEG3,
203 D40_DREG_CRCEG4,
204 D40_DREG_CRCEG5,
205 };
206
207 #define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
208
209 static u32 d40_backup_regs_chan[] = {
210 D40_CHAN_REG_SSCFG,
211 D40_CHAN_REG_SSELT,
212 D40_CHAN_REG_SSPTR,
213 D40_CHAN_REG_SSLNK,
214 D40_CHAN_REG_SDCFG,
215 D40_CHAN_REG_SDELT,
216 D40_CHAN_REG_SDPTR,
217 D40_CHAN_REG_SDLNK,
218 };
219
220 #define BACKUP_REGS_SZ_MAX ((BACKUP_REGS_SZ_V4A > BACKUP_REGS_SZ_V4B) ? \
221 BACKUP_REGS_SZ_V4A : BACKUP_REGS_SZ_V4B)
222
223 /**
224 * struct d40_interrupt_lookup - lookup table for interrupt handler
225 *
226 * @src: Interrupt mask register.
227 * @clr: Interrupt clear register.
228 * @is_error: true if this is an error interrupt.
229 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
230 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
231 */
232 struct d40_interrupt_lookup {
233 u32 src;
234 u32 clr;
235 bool is_error;
236 int offset;
237 };
238
239
240 static struct d40_interrupt_lookup il_v4a[] = {
241 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
242 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
243 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
244 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
245 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
246 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
247 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
248 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
249 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
250 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
251 };
252
253 static struct d40_interrupt_lookup il_v4b[] = {
254 {D40_DREG_CLCTIS1, D40_DREG_CLCICR1, false, 0},
255 {D40_DREG_CLCTIS2, D40_DREG_CLCICR2, false, 32},
256 {D40_DREG_CLCTIS3, D40_DREG_CLCICR3, false, 64},
257 {D40_DREG_CLCTIS4, D40_DREG_CLCICR4, false, 96},
258 {D40_DREG_CLCTIS5, D40_DREG_CLCICR5, false, 128},
259 {D40_DREG_CLCEIS1, D40_DREG_CLCICR1, true, 0},
260 {D40_DREG_CLCEIS2, D40_DREG_CLCICR2, true, 32},
261 {D40_DREG_CLCEIS3, D40_DREG_CLCICR3, true, 64},
262 {D40_DREG_CLCEIS4, D40_DREG_CLCICR4, true, 96},
263 {D40_DREG_CLCEIS5, D40_DREG_CLCICR5, true, 128},
264 {D40_DREG_CPCTIS, D40_DREG_CPCICR, false, D40_PHY_CHAN},
265 {D40_DREG_CPCEIS, D40_DREG_CPCICR, true, D40_PHY_CHAN},
266 };
267
268 /**
269 * struct d40_reg_val - simple lookup struct
270 *
271 * @reg: The register.
272 * @val: The value that belongs to the register in reg.
273 */
274 struct d40_reg_val {
275 unsigned int reg;
276 unsigned int val;
277 };
278
279 static __initdata struct d40_reg_val dma_init_reg_v4a[] = {
280 /* Clock every part of the DMA block from start */
281 { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
282
283 /* Interrupts on all logical channels */
284 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
285 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
286 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
287 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
288 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
289 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
290 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
291 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
292 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
293 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
294 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
295 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
296 };
297 static __initdata struct d40_reg_val dma_init_reg_v4b[] = {
298 /* Clock every part of the DMA block from start */
299 { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
300
301 /* Interrupts on all logical channels */
302 { .reg = D40_DREG_CLCMIS1, .val = 0xFFFFFFFF},
303 { .reg = D40_DREG_CLCMIS2, .val = 0xFFFFFFFF},
304 { .reg = D40_DREG_CLCMIS3, .val = 0xFFFFFFFF},
305 { .reg = D40_DREG_CLCMIS4, .val = 0xFFFFFFFF},
306 { .reg = D40_DREG_CLCMIS5, .val = 0xFFFFFFFF},
307 { .reg = D40_DREG_CLCICR1, .val = 0xFFFFFFFF},
308 { .reg = D40_DREG_CLCICR2, .val = 0xFFFFFFFF},
309 { .reg = D40_DREG_CLCICR3, .val = 0xFFFFFFFF},
310 { .reg = D40_DREG_CLCICR4, .val = 0xFFFFFFFF},
311 { .reg = D40_DREG_CLCICR5, .val = 0xFFFFFFFF},
312 { .reg = D40_DREG_CLCTIS1, .val = 0xFFFFFFFF},
313 { .reg = D40_DREG_CLCTIS2, .val = 0xFFFFFFFF},
314 { .reg = D40_DREG_CLCTIS3, .val = 0xFFFFFFFF},
315 { .reg = D40_DREG_CLCTIS4, .val = 0xFFFFFFFF},
316 { .reg = D40_DREG_CLCTIS5, .val = 0xFFFFFFFF}
317 };
318
319 /**
320 * struct d40_lli_pool - Structure for keeping LLIs in memory
321 *
322 * @base: Pointer to memory area when the pre_alloc_lli's are not large
323 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
324 * pre_alloc_lli is used.
325 * @dma_addr: DMA address, if mapped
326 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
327 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
328 * one buffer to one buffer.
329 */
330 struct d40_lli_pool {
331 void *base;
332 int size;
333 dma_addr_t dma_addr;
334 /* Space for dst and src, plus an extra for padding */
335 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
336 };
337
338 /**
339 * struct d40_desc - A descriptor is one DMA job.
340 *
341 * @lli_phy: LLI settings for physical channel. Both src and dst=
342 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
343 * lli_len equals one.
344 * @lli_log: Same as above but for logical channels.
345 * @lli_pool: The pool with two entries pre-allocated.
346 * @lli_len: Number of llis of current descriptor.
347 * @lli_current: Number of transferred llis.
348 * @lcla_alloc: Number of LCLA entries allocated.
349 * @txd: DMA engine struct. Used for among other things for communication
350 * during a transfer.
351 * @node: List entry.
352 * @is_in_client_list: true if the client owns this descriptor.
353 * @cyclic: true if this is a cyclic job
354 *
355 * This descriptor is used for both logical and physical transfers.
356 */
357 struct d40_desc {
358 /* LLI physical */
359 struct d40_phy_lli_bidir lli_phy;
360 /* LLI logical */
361 struct d40_log_lli_bidir lli_log;
362
363 struct d40_lli_pool lli_pool;
364 int lli_len;
365 int lli_current;
366 int lcla_alloc;
367
368 struct dma_async_tx_descriptor txd;
369 struct list_head node;
370
371 bool is_in_client_list;
372 bool cyclic;
373 };
374
375 /**
376 * struct d40_lcla_pool - LCLA pool settings and data.
377 *
378 * @base: The virtual address of LCLA. 18 bit aligned.
379 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
380 * This pointer is only there for clean-up on error.
381 * @pages: The number of pages needed for all physical channels.
382 * Only used later for clean-up on error
383 * @lock: Lock to protect the content in this struct.
384 * @alloc_map: big map over which LCLA entry is own by which job.
385 */
386 struct d40_lcla_pool {
387 void *base;
388 dma_addr_t dma_addr;
389 void *base_unaligned;
390 int pages;
391 spinlock_t lock;
392 struct d40_desc **alloc_map;
393 };
394
395 /**
396 * struct d40_phy_res - struct for handling eventlines mapped to physical
397 * channels.
398 *
399 * @lock: A lock protection this entity.
400 * @reserved: True if used by secure world or otherwise.
401 * @num: The physical channel number of this entity.
402 * @allocated_src: Bit mapped to show which src event line's are mapped to
403 * this physical channel. Can also be free or physically allocated.
404 * @allocated_dst: Same as for src but is dst.
405 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
406 * event line number.
407 * @use_soft_lli: To mark if the linked lists of channel are managed by SW.
408 */
409 struct d40_phy_res {
410 spinlock_t lock;
411 bool reserved;
412 int num;
413 u32 allocated_src;
414 u32 allocated_dst;
415 bool use_soft_lli;
416 };
417
418 struct d40_base;
419
420 /**
421 * struct d40_chan - Struct that describes a channel.
422 *
423 * @lock: A spinlock to protect this struct.
424 * @log_num: The logical number, if any of this channel.
425 * @pending_tx: The number of pending transfers. Used between interrupt handler
426 * and tasklet.
427 * @busy: Set to true when transfer is ongoing on this channel.
428 * @phy_chan: Pointer to physical channel which this instance runs on. If this
429 * point is NULL, then the channel is not allocated.
430 * @chan: DMA engine handle.
431 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
432 * transfer and call client callback.
433 * @client: Cliented owned descriptor list.
434 * @pending_queue: Submitted jobs, to be issued by issue_pending()
435 * @active: Active descriptor.
436 * @done: Completed jobs
437 * @queue: Queued jobs.
438 * @prepare_queue: Prepared jobs.
439 * @dma_cfg: The client configuration of this dma channel.
440 * @configured: whether the dma_cfg configuration is valid
441 * @base: Pointer to the device instance struct.
442 * @src_def_cfg: Default cfg register setting for src.
443 * @dst_def_cfg: Default cfg register setting for dst.
444 * @log_def: Default logical channel settings.
445 * @lcpa: Pointer to dst and src lcpa settings.
446 * @runtime_addr: runtime configured address.
447 * @runtime_direction: runtime configured direction.
448 *
449 * This struct can either "be" a logical or a physical channel.
450 */
451 struct d40_chan {
452 spinlock_t lock;
453 int log_num;
454 int pending_tx;
455 bool busy;
456 struct d40_phy_res *phy_chan;
457 struct dma_chan chan;
458 struct tasklet_struct tasklet;
459 struct list_head client;
460 struct list_head pending_queue;
461 struct list_head active;
462 struct list_head done;
463 struct list_head queue;
464 struct list_head prepare_queue;
465 struct stedma40_chan_cfg dma_cfg;
466 bool configured;
467 struct d40_base *base;
468 /* Default register configurations */
469 u32 src_def_cfg;
470 u32 dst_def_cfg;
471 struct d40_def_lcsp log_def;
472 struct d40_log_lli_full *lcpa;
473 /* Runtime reconfiguration */
474 dma_addr_t runtime_addr;
475 enum dma_transfer_direction runtime_direction;
476 };
477
478 /**
479 * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
480 * controller
481 *
482 * @backup: the pointer to the registers address array for backup
483 * @backup_size: the size of the registers address array for backup
484 * @realtime_en: the realtime enable register
485 * @realtime_clear: the realtime clear register
486 * @high_prio_en: the high priority enable register
487 * @high_prio_clear: the high priority clear register
488 * @interrupt_en: the interrupt enable register
489 * @interrupt_clear: the interrupt clear register
490 * @il: the pointer to struct d40_interrupt_lookup
491 * @il_size: the size of d40_interrupt_lookup array
492 * @init_reg: the pointer to the struct d40_reg_val
493 * @init_reg_size: the size of d40_reg_val array
494 */
495 struct d40_gen_dmac {
496 u32 *backup;
497 u32 backup_size;
498 u32 realtime_en;
499 u32 realtime_clear;
500 u32 high_prio_en;
501 u32 high_prio_clear;
502 u32 interrupt_en;
503 u32 interrupt_clear;
504 struct d40_interrupt_lookup *il;
505 u32 il_size;
506 struct d40_reg_val *init_reg;
507 u32 init_reg_size;
508 };
509
510 /**
511 * struct d40_base - The big global struct, one for each probe'd instance.
512 *
513 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
514 * @execmd_lock: Lock for execute command usage since several channels share
515 * the same physical register.
516 * @dev: The device structure.
517 * @virtbase: The virtual base address of the DMA's register.
518 * @rev: silicon revision detected.
519 * @clk: Pointer to the DMA clock structure.
520 * @phy_start: Physical memory start of the DMA registers.
521 * @phy_size: Size of the DMA register map.
522 * @irq: The IRQ number.
523 * @num_phy_chans: The number of physical channels. Read from HW. This
524 * is the number of available channels for this driver, not counting "Secure
525 * mode" allocated physical channels.
526 * @num_log_chans: The number of logical channels. Calculated from
527 * num_phy_chans.
528 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
529 * @dma_slave: dma_device channels that can do only do slave transfers.
530 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
531 * @phy_chans: Room for all possible physical channels in system.
532 * @log_chans: Room for all possible logical channels in system.
533 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
534 * to log_chans entries.
535 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
536 * to phy_chans entries.
537 * @plat_data: Pointer to provided platform_data which is the driver
538 * configuration.
539 * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
540 * @phy_res: Vector containing all physical channels.
541 * @lcla_pool: lcla pool settings and data.
542 * @lcpa_base: The virtual mapped address of LCPA.
543 * @phy_lcpa: The physical address of the LCPA.
544 * @lcpa_size: The size of the LCPA area.
545 * @desc_slab: cache for descriptors.
546 * @reg_val_backup: Here the values of some hardware registers are stored
547 * before the DMA is powered off. They are restored when the power is back on.
548 * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
549 * later
550 * @reg_val_backup_chan: Backup data for standard channel parameter registers.
551 * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
552 * @initialized: true if the dma has been initialized
553 * @gen_dmac: the struct for generic registers values to represent u8500/8540
554 * DMA controller
555 */
556 struct d40_base {
557 spinlock_t interrupt_lock;
558 spinlock_t execmd_lock;
559 struct device *dev;
560 void __iomem *virtbase;
561 u8 rev:4;
562 struct clk *clk;
563 phys_addr_t phy_start;
564 resource_size_t phy_size;
565 int irq;
566 int num_phy_chans;
567 int num_log_chans;
568 struct device_dma_parameters dma_parms;
569 struct dma_device dma_both;
570 struct dma_device dma_slave;
571 struct dma_device dma_memcpy;
572 struct d40_chan *phy_chans;
573 struct d40_chan *log_chans;
574 struct d40_chan **lookup_log_chans;
575 struct d40_chan **lookup_phy_chans;
576 struct stedma40_platform_data *plat_data;
577 struct regulator *lcpa_regulator;
578 /* Physical half channels */
579 struct d40_phy_res *phy_res;
580 struct d40_lcla_pool lcla_pool;
581 void *lcpa_base;
582 dma_addr_t phy_lcpa;
583 resource_size_t lcpa_size;
584 struct kmem_cache *desc_slab;
585 u32 reg_val_backup[BACKUP_REGS_SZ];
586 u32 reg_val_backup_v4[BACKUP_REGS_SZ_MAX];
587 u32 *reg_val_backup_chan;
588 u16 gcc_pwr_off_mask;
589 bool initialized;
590 struct d40_gen_dmac gen_dmac;
591 };
592
593 static struct device *chan2dev(struct d40_chan *d40c)
594 {
595 return &d40c->chan.dev->device;
596 }
597
598 static bool chan_is_physical(struct d40_chan *chan)
599 {
600 return chan->log_num == D40_PHY_CHAN;
601 }
602
603 static bool chan_is_logical(struct d40_chan *chan)
604 {
605 return !chan_is_physical(chan);
606 }
607
608 static void __iomem *chan_base(struct d40_chan *chan)
609 {
610 return chan->base->virtbase + D40_DREG_PCBASE +
611 chan->phy_chan->num * D40_DREG_PCDELTA;
612 }
613
614 #define d40_err(dev, format, arg...) \
615 dev_err(dev, "[%s] " format, __func__, ## arg)
616
617 #define chan_err(d40c, format, arg...) \
618 d40_err(chan2dev(d40c), format, ## arg)
619
620 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
621 int lli_len)
622 {
623 bool is_log = chan_is_logical(d40c);
624 u32 align;
625 void *base;
626
627 if (is_log)
628 align = sizeof(struct d40_log_lli);
629 else
630 align = sizeof(struct d40_phy_lli);
631
632 if (lli_len == 1) {
633 base = d40d->lli_pool.pre_alloc_lli;
634 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
635 d40d->lli_pool.base = NULL;
636 } else {
637 d40d->lli_pool.size = lli_len * 2 * align;
638
639 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
640 d40d->lli_pool.base = base;
641
642 if (d40d->lli_pool.base == NULL)
643 return -ENOMEM;
644 }
645
646 if (is_log) {
647 d40d->lli_log.src = PTR_ALIGN(base, align);
648 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
649
650 d40d->lli_pool.dma_addr = 0;
651 } else {
652 d40d->lli_phy.src = PTR_ALIGN(base, align);
653 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
654
655 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
656 d40d->lli_phy.src,
657 d40d->lli_pool.size,
658 DMA_TO_DEVICE);
659
660 if (dma_mapping_error(d40c->base->dev,
661 d40d->lli_pool.dma_addr)) {
662 kfree(d40d->lli_pool.base);
663 d40d->lli_pool.base = NULL;
664 d40d->lli_pool.dma_addr = 0;
665 return -ENOMEM;
666 }
667 }
668
669 return 0;
670 }
671
672 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
673 {
674 if (d40d->lli_pool.dma_addr)
675 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
676 d40d->lli_pool.size, DMA_TO_DEVICE);
677
678 kfree(d40d->lli_pool.base);
679 d40d->lli_pool.base = NULL;
680 d40d->lli_pool.size = 0;
681 d40d->lli_log.src = NULL;
682 d40d->lli_log.dst = NULL;
683 d40d->lli_phy.src = NULL;
684 d40d->lli_phy.dst = NULL;
685 }
686
687 static int d40_lcla_alloc_one(struct d40_chan *d40c,
688 struct d40_desc *d40d)
689 {
690 unsigned long flags;
691 int i;
692 int ret = -EINVAL;
693
694 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
695
696 /*
697 * Allocate both src and dst at the same time, therefore the half
698 * start on 1 since 0 can't be used since zero is used as end marker.
699 */
700 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
701 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
702
703 if (!d40c->base->lcla_pool.alloc_map[idx]) {
704 d40c->base->lcla_pool.alloc_map[idx] = d40d;
705 d40d->lcla_alloc++;
706 ret = i;
707 break;
708 }
709 }
710
711 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
712
713 return ret;
714 }
715
716 static int d40_lcla_free_all(struct d40_chan *d40c,
717 struct d40_desc *d40d)
718 {
719 unsigned long flags;
720 int i;
721 int ret = -EINVAL;
722
723 if (chan_is_physical(d40c))
724 return 0;
725
726 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
727
728 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
729 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
730
731 if (d40c->base->lcla_pool.alloc_map[idx] == d40d) {
732 d40c->base->lcla_pool.alloc_map[idx] = NULL;
733 d40d->lcla_alloc--;
734 if (d40d->lcla_alloc == 0) {
735 ret = 0;
736 break;
737 }
738 }
739 }
740
741 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
742
743 return ret;
744
745 }
746
747 static void d40_desc_remove(struct d40_desc *d40d)
748 {
749 list_del(&d40d->node);
750 }
751
752 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
753 {
754 struct d40_desc *desc = NULL;
755
756 if (!list_empty(&d40c->client)) {
757 struct d40_desc *d;
758 struct d40_desc *_d;
759
760 list_for_each_entry_safe(d, _d, &d40c->client, node) {
761 if (async_tx_test_ack(&d->txd)) {
762 d40_desc_remove(d);
763 desc = d;
764 memset(desc, 0, sizeof(*desc));
765 break;
766 }
767 }
768 }
769
770 if (!desc)
771 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
772
773 if (desc)
774 INIT_LIST_HEAD(&desc->node);
775
776 return desc;
777 }
778
779 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
780 {
781
782 d40_pool_lli_free(d40c, d40d);
783 d40_lcla_free_all(d40c, d40d);
784 kmem_cache_free(d40c->base->desc_slab, d40d);
785 }
786
787 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
788 {
789 list_add_tail(&desc->node, &d40c->active);
790 }
791
792 static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
793 {
794 struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
795 struct d40_phy_lli *lli_src = desc->lli_phy.src;
796 void __iomem *base = chan_base(chan);
797
798 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
799 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
800 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
801 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
802
803 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
804 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
805 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
806 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
807 }
808
809 static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc)
810 {
811 list_add_tail(&desc->node, &d40c->done);
812 }
813
814 static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
815 {
816 struct d40_lcla_pool *pool = &chan->base->lcla_pool;
817 struct d40_log_lli_bidir *lli = &desc->lli_log;
818 int lli_current = desc->lli_current;
819 int lli_len = desc->lli_len;
820 bool cyclic = desc->cyclic;
821 int curr_lcla = -EINVAL;
822 int first_lcla = 0;
823 bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
824 bool linkback;
825
826 /*
827 * We may have partially running cyclic transfers, in case we did't get
828 * enough LCLA entries.
829 */
830 linkback = cyclic && lli_current == 0;
831
832 /*
833 * For linkback, we need one LCLA even with only one link, because we
834 * can't link back to the one in LCPA space
835 */
836 if (linkback || (lli_len - lli_current > 1)) {
837 /*
838 * If the channel is expected to use only soft_lli don't
839 * allocate a lcla. This is to avoid a HW issue that exists
840 * in some controller during a peripheral to memory transfer
841 * that uses linked lists.
842 */
843 if (!(chan->phy_chan->use_soft_lli &&
844 chan->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM))
845 curr_lcla = d40_lcla_alloc_one(chan, desc);
846
847 first_lcla = curr_lcla;
848 }
849
850 /*
851 * For linkback, we normally load the LCPA in the loop since we need to
852 * link it to the second LCLA and not the first. However, if we
853 * couldn't even get a first LCLA, then we have to run in LCPA and
854 * reload manually.
855 */
856 if (!linkback || curr_lcla == -EINVAL) {
857 unsigned int flags = 0;
858
859 if (curr_lcla == -EINVAL)
860 flags |= LLI_TERM_INT;
861
862 d40_log_lli_lcpa_write(chan->lcpa,
863 &lli->dst[lli_current],
864 &lli->src[lli_current],
865 curr_lcla,
866 flags);
867 lli_current++;
868 }
869
870 if (curr_lcla < 0)
871 goto out;
872
873 for (; lli_current < lli_len; lli_current++) {
874 unsigned int lcla_offset = chan->phy_chan->num * 1024 +
875 8 * curr_lcla * 2;
876 struct d40_log_lli *lcla = pool->base + lcla_offset;
877 unsigned int flags = 0;
878 int next_lcla;
879
880 if (lli_current + 1 < lli_len)
881 next_lcla = d40_lcla_alloc_one(chan, desc);
882 else
883 next_lcla = linkback ? first_lcla : -EINVAL;
884
885 if (cyclic || next_lcla == -EINVAL)
886 flags |= LLI_TERM_INT;
887
888 if (linkback && curr_lcla == first_lcla) {
889 /* First link goes in both LCPA and LCLA */
890 d40_log_lli_lcpa_write(chan->lcpa,
891 &lli->dst[lli_current],
892 &lli->src[lli_current],
893 next_lcla, flags);
894 }
895
896 /*
897 * One unused LCLA in the cyclic case if the very first
898 * next_lcla fails...
899 */
900 d40_log_lli_lcla_write(lcla,
901 &lli->dst[lli_current],
902 &lli->src[lli_current],
903 next_lcla, flags);
904
905 /*
906 * Cache maintenance is not needed if lcla is
907 * mapped in esram
908 */
909 if (!use_esram_lcla) {
910 dma_sync_single_range_for_device(chan->base->dev,
911 pool->dma_addr, lcla_offset,
912 2 * sizeof(struct d40_log_lli),
913 DMA_TO_DEVICE);
914 }
915 curr_lcla = next_lcla;
916
917 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
918 lli_current++;
919 break;
920 }
921 }
922
923 out:
924 desc->lli_current = lli_current;
925 }
926
927 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
928 {
929 if (chan_is_physical(d40c)) {
930 d40_phy_lli_load(d40c, d40d);
931 d40d->lli_current = d40d->lli_len;
932 } else
933 d40_log_lli_to_lcxa(d40c, d40d);
934 }
935
936 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
937 {
938 struct d40_desc *d;
939
940 if (list_empty(&d40c->active))
941 return NULL;
942
943 d = list_first_entry(&d40c->active,
944 struct d40_desc,
945 node);
946 return d;
947 }
948
949 /* remove desc from current queue and add it to the pending_queue */
950 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
951 {
952 d40_desc_remove(desc);
953 desc->is_in_client_list = false;
954 list_add_tail(&desc->node, &d40c->pending_queue);
955 }
956
957 static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
958 {
959 struct d40_desc *d;
960
961 if (list_empty(&d40c->pending_queue))
962 return NULL;
963
964 d = list_first_entry(&d40c->pending_queue,
965 struct d40_desc,
966 node);
967 return d;
968 }
969
970 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
971 {
972 struct d40_desc *d;
973
974 if (list_empty(&d40c->queue))
975 return NULL;
976
977 d = list_first_entry(&d40c->queue,
978 struct d40_desc,
979 node);
980 return d;
981 }
982
983 static struct d40_desc *d40_first_done(struct d40_chan *d40c)
984 {
985 if (list_empty(&d40c->done))
986 return NULL;
987
988 return list_first_entry(&d40c->done, struct d40_desc, node);
989 }
990
991 static int d40_psize_2_burst_size(bool is_log, int psize)
992 {
993 if (is_log) {
994 if (psize == STEDMA40_PSIZE_LOG_1)
995 return 1;
996 } else {
997 if (psize == STEDMA40_PSIZE_PHY_1)
998 return 1;
999 }
1000
1001 return 2 << psize;
1002 }
1003
1004 /*
1005 * The dma only supports transmitting packages up to
1006 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
1007 * dma elements required to send the entire sg list
1008 */
1009 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
1010 {
1011 int dmalen;
1012 u32 max_w = max(data_width1, data_width2);
1013 u32 min_w = min(data_width1, data_width2);
1014 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
1015
1016 if (seg_max > STEDMA40_MAX_SEG_SIZE)
1017 seg_max -= (1 << max_w);
1018
1019 if (!IS_ALIGNED(size, 1 << max_w))
1020 return -EINVAL;
1021
1022 if (size <= seg_max)
1023 dmalen = 1;
1024 else {
1025 dmalen = size / seg_max;
1026 if (dmalen * seg_max < size)
1027 dmalen++;
1028 }
1029 return dmalen;
1030 }
1031
1032 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
1033 u32 data_width1, u32 data_width2)
1034 {
1035 struct scatterlist *sg;
1036 int i;
1037 int len = 0;
1038 int ret;
1039
1040 for_each_sg(sgl, sg, sg_len, i) {
1041 ret = d40_size_2_dmalen(sg_dma_len(sg),
1042 data_width1, data_width2);
1043 if (ret < 0)
1044 return ret;
1045 len += ret;
1046 }
1047 return len;
1048 }
1049
1050
1051 #ifdef CONFIG_PM
1052 static void dma40_backup(void __iomem *baseaddr, u32 *backup,
1053 u32 *regaddr, int num, bool save)
1054 {
1055 int i;
1056
1057 for (i = 0; i < num; i++) {
1058 void __iomem *addr = baseaddr + regaddr[i];
1059
1060 if (save)
1061 backup[i] = readl_relaxed(addr);
1062 else
1063 writel_relaxed(backup[i], addr);
1064 }
1065 }
1066
1067 static void d40_save_restore_registers(struct d40_base *base, bool save)
1068 {
1069 int i;
1070
1071 /* Save/Restore channel specific registers */
1072 for (i = 0; i < base->num_phy_chans; i++) {
1073 void __iomem *addr;
1074 int idx;
1075
1076 if (base->phy_res[i].reserved)
1077 continue;
1078
1079 addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
1080 idx = i * ARRAY_SIZE(d40_backup_regs_chan);
1081
1082 dma40_backup(addr, &base->reg_val_backup_chan[idx],
1083 d40_backup_regs_chan,
1084 ARRAY_SIZE(d40_backup_regs_chan),
1085 save);
1086 }
1087
1088 /* Save/Restore global registers */
1089 dma40_backup(base->virtbase, base->reg_val_backup,
1090 d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
1091 save);
1092
1093 /* Save/Restore registers only existing on dma40 v3 and later */
1094 if (base->gen_dmac.backup)
1095 dma40_backup(base->virtbase, base->reg_val_backup_v4,
1096 base->gen_dmac.backup,
1097 base->gen_dmac.backup_size,
1098 save);
1099 }
1100 #else
1101 static void d40_save_restore_registers(struct d40_base *base, bool save)
1102 {
1103 }
1104 #endif
1105
1106 static int __d40_execute_command_phy(struct d40_chan *d40c,
1107 enum d40_command command)
1108 {
1109 u32 status;
1110 int i;
1111 void __iomem *active_reg;
1112 int ret = 0;
1113 unsigned long flags;
1114 u32 wmask;
1115
1116 if (command == D40_DMA_STOP) {
1117 ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ);
1118 if (ret)
1119 return ret;
1120 }
1121
1122 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
1123
1124 if (d40c->phy_chan->num % 2 == 0)
1125 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1126 else
1127 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1128
1129 if (command == D40_DMA_SUSPEND_REQ) {
1130 status = (readl(active_reg) &
1131 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1132 D40_CHAN_POS(d40c->phy_chan->num);
1133
1134 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1135 goto done;
1136 }
1137
1138 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
1139 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
1140 active_reg);
1141
1142 if (command == D40_DMA_SUSPEND_REQ) {
1143
1144 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
1145 status = (readl(active_reg) &
1146 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1147 D40_CHAN_POS(d40c->phy_chan->num);
1148
1149 cpu_relax();
1150 /*
1151 * Reduce the number of bus accesses while
1152 * waiting for the DMA to suspend.
1153 */
1154 udelay(3);
1155
1156 if (status == D40_DMA_STOP ||
1157 status == D40_DMA_SUSPENDED)
1158 break;
1159 }
1160
1161 if (i == D40_SUSPEND_MAX_IT) {
1162 chan_err(d40c,
1163 "unable to suspend the chl %d (log: %d) status %x\n",
1164 d40c->phy_chan->num, d40c->log_num,
1165 status);
1166 dump_stack();
1167 ret = -EBUSY;
1168 }
1169
1170 }
1171 done:
1172 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
1173 return ret;
1174 }
1175
1176 static void d40_term_all(struct d40_chan *d40c)
1177 {
1178 struct d40_desc *d40d;
1179 struct d40_desc *_d;
1180
1181 /* Release completed descriptors */
1182 while ((d40d = d40_first_done(d40c))) {
1183 d40_desc_remove(d40d);
1184 d40_desc_free(d40c, d40d);
1185 }
1186
1187 /* Release active descriptors */
1188 while ((d40d = d40_first_active_get(d40c))) {
1189 d40_desc_remove(d40d);
1190 d40_desc_free(d40c, d40d);
1191 }
1192
1193 /* Release queued descriptors waiting for transfer */
1194 while ((d40d = d40_first_queued(d40c))) {
1195 d40_desc_remove(d40d);
1196 d40_desc_free(d40c, d40d);
1197 }
1198
1199 /* Release pending descriptors */
1200 while ((d40d = d40_first_pending(d40c))) {
1201 d40_desc_remove(d40d);
1202 d40_desc_free(d40c, d40d);
1203 }
1204
1205 /* Release client owned descriptors */
1206 if (!list_empty(&d40c->client))
1207 list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
1208 d40_desc_remove(d40d);
1209 d40_desc_free(d40c, d40d);
1210 }
1211
1212 /* Release descriptors in prepare queue */
1213 if (!list_empty(&d40c->prepare_queue))
1214 list_for_each_entry_safe(d40d, _d,
1215 &d40c->prepare_queue, node) {
1216 d40_desc_remove(d40d);
1217 d40_desc_free(d40c, d40d);
1218 }
1219
1220 d40c->pending_tx = 0;
1221 }
1222
1223 static void __d40_config_set_event(struct d40_chan *d40c,
1224 enum d40_events event_type, u32 event,
1225 int reg)
1226 {
1227 void __iomem *addr = chan_base(d40c) + reg;
1228 int tries;
1229 u32 status;
1230
1231 switch (event_type) {
1232
1233 case D40_DEACTIVATE_EVENTLINE:
1234
1235 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
1236 | ~D40_EVENTLINE_MASK(event), addr);
1237 break;
1238
1239 case D40_SUSPEND_REQ_EVENTLINE:
1240 status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1241 D40_EVENTLINE_POS(event);
1242
1243 if (status == D40_DEACTIVATE_EVENTLINE ||
1244 status == D40_SUSPEND_REQ_EVENTLINE)
1245 break;
1246
1247 writel((D40_SUSPEND_REQ_EVENTLINE << D40_EVENTLINE_POS(event))
1248 | ~D40_EVENTLINE_MASK(event), addr);
1249
1250 for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {
1251
1252 status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1253 D40_EVENTLINE_POS(event);
1254
1255 cpu_relax();
1256 /*
1257 * Reduce the number of bus accesses while
1258 * waiting for the DMA to suspend.
1259 */
1260 udelay(3);
1261
1262 if (status == D40_DEACTIVATE_EVENTLINE)
1263 break;
1264 }
1265
1266 if (tries == D40_SUSPEND_MAX_IT) {
1267 chan_err(d40c,
1268 "unable to stop the event_line chl %d (log: %d)"
1269 "status %x\n", d40c->phy_chan->num,
1270 d40c->log_num, status);
1271 }
1272 break;
1273
1274 case D40_ACTIVATE_EVENTLINE:
1275 /*
1276 * The hardware sometimes doesn't register the enable when src and dst
1277 * event lines are active on the same logical channel. Retry to ensure
1278 * it does. Usually only one retry is sufficient.
1279 */
1280 tries = 100;
1281 while (--tries) {
1282 writel((D40_ACTIVATE_EVENTLINE <<
1283 D40_EVENTLINE_POS(event)) |
1284 ~D40_EVENTLINE_MASK(event), addr);
1285
1286 if (readl(addr) & D40_EVENTLINE_MASK(event))
1287 break;
1288 }
1289
1290 if (tries != 99)
1291 dev_dbg(chan2dev(d40c),
1292 "[%s] workaround enable S%cLNK (%d tries)\n",
1293 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
1294 100 - tries);
1295
1296 WARN_ON(!tries);
1297 break;
1298
1299 case D40_ROUND_EVENTLINE:
1300 BUG();
1301 break;
1302
1303 }
1304 }
1305
1306 static void d40_config_set_event(struct d40_chan *d40c,
1307 enum d40_events event_type)
1308 {
1309 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
1310
1311 /* Enable event line connected to device (or memcpy) */
1312 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
1313 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1314 __d40_config_set_event(d40c, event_type, event,
1315 D40_CHAN_REG_SSLNK);
1316
1317 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM)
1318 __d40_config_set_event(d40c, event_type, event,
1319 D40_CHAN_REG_SDLNK);
1320 }
1321
1322 static u32 d40_chan_has_events(struct d40_chan *d40c)
1323 {
1324 void __iomem *chanbase = chan_base(d40c);
1325 u32 val;
1326
1327 val = readl(chanbase + D40_CHAN_REG_SSLNK);
1328 val |= readl(chanbase + D40_CHAN_REG_SDLNK);
1329
1330 return val;
1331 }
1332
1333 static int
1334 __d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)
1335 {
1336 unsigned long flags;
1337 int ret = 0;
1338 u32 active_status;
1339 void __iomem *active_reg;
1340
1341 if (d40c->phy_chan->num % 2 == 0)
1342 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1343 else
1344 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1345
1346
1347 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
1348
1349 switch (command) {
1350 case D40_DMA_STOP:
1351 case D40_DMA_SUSPEND_REQ:
1352
1353 active_status = (readl(active_reg) &
1354 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1355 D40_CHAN_POS(d40c->phy_chan->num);
1356
1357 if (active_status == D40_DMA_RUN)
1358 d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE);
1359 else
1360 d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE);
1361
1362 if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))
1363 ret = __d40_execute_command_phy(d40c, command);
1364
1365 break;
1366
1367 case D40_DMA_RUN:
1368
1369 d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE);
1370 ret = __d40_execute_command_phy(d40c, command);
1371 break;
1372
1373 case D40_DMA_SUSPENDED:
1374 BUG();
1375 break;
1376 }
1377
1378 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
1379 return ret;
1380 }
1381
1382 static int d40_channel_execute_command(struct d40_chan *d40c,
1383 enum d40_command command)
1384 {
1385 if (chan_is_logical(d40c))
1386 return __d40_execute_command_log(d40c, command);
1387 else
1388 return __d40_execute_command_phy(d40c, command);
1389 }
1390
1391 static u32 d40_get_prmo(struct d40_chan *d40c)
1392 {
1393 static const unsigned int phy_map[] = {
1394 [STEDMA40_PCHAN_BASIC_MODE]
1395 = D40_DREG_PRMO_PCHAN_BASIC,
1396 [STEDMA40_PCHAN_MODULO_MODE]
1397 = D40_DREG_PRMO_PCHAN_MODULO,
1398 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
1399 = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
1400 };
1401 static const unsigned int log_map[] = {
1402 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
1403 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
1404 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
1405 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
1406 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
1407 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
1408 };
1409
1410 if (chan_is_physical(d40c))
1411 return phy_map[d40c->dma_cfg.mode_opt];
1412 else
1413 return log_map[d40c->dma_cfg.mode_opt];
1414 }
1415
1416 static void d40_config_write(struct d40_chan *d40c)
1417 {
1418 u32 addr_base;
1419 u32 var;
1420
1421 /* Odd addresses are even addresses + 4 */
1422 addr_base = (d40c->phy_chan->num % 2) * 4;
1423 /* Setup channel mode to logical or physical */
1424 var = ((u32)(chan_is_logical(d40c)) + 1) <<
1425 D40_CHAN_POS(d40c->phy_chan->num);
1426 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
1427
1428 /* Setup operational mode option register */
1429 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
1430
1431 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
1432
1433 if (chan_is_logical(d40c)) {
1434 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
1435 & D40_SREG_ELEM_LOG_LIDX_MASK;
1436 void __iomem *chanbase = chan_base(d40c);
1437
1438 /* Set default config for CFG reg */
1439 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
1440 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
1441
1442 /* Set LIDX for lcla */
1443 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
1444 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
1445
1446 /* Clear LNK which will be used by d40_chan_has_events() */
1447 writel(0, chanbase + D40_CHAN_REG_SSLNK);
1448 writel(0, chanbase + D40_CHAN_REG_SDLNK);
1449 }
1450 }
1451
1452 static u32 d40_residue(struct d40_chan *d40c)
1453 {
1454 u32 num_elt;
1455
1456 if (chan_is_logical(d40c))
1457 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
1458 >> D40_MEM_LCSP2_ECNT_POS;
1459 else {
1460 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
1461 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
1462 >> D40_SREG_ELEM_PHY_ECNT_POS;
1463 }
1464
1465 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1466 }
1467
1468 static bool d40_tx_is_linked(struct d40_chan *d40c)
1469 {
1470 bool is_link;
1471
1472 if (chan_is_logical(d40c))
1473 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
1474 else
1475 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
1476 & D40_SREG_LNK_PHYS_LNK_MASK;
1477
1478 return is_link;
1479 }
1480
1481 static int d40_pause(struct d40_chan *d40c)
1482 {
1483 int res = 0;
1484 unsigned long flags;
1485
1486 if (!d40c->busy)
1487 return 0;
1488
1489 pm_runtime_get_sync(d40c->base->dev);
1490 spin_lock_irqsave(&d40c->lock, flags);
1491
1492 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1493
1494 pm_runtime_mark_last_busy(d40c->base->dev);
1495 pm_runtime_put_autosuspend(d40c->base->dev);
1496 spin_unlock_irqrestore(&d40c->lock, flags);
1497 return res;
1498 }
1499
1500 static int d40_resume(struct d40_chan *d40c)
1501 {
1502 int res = 0;
1503 unsigned long flags;
1504
1505 if (!d40c->busy)
1506 return 0;
1507
1508 spin_lock_irqsave(&d40c->lock, flags);
1509 pm_runtime_get_sync(d40c->base->dev);
1510
1511 /* If bytes left to transfer or linked tx resume job */
1512 if (d40_residue(d40c) || d40_tx_is_linked(d40c))
1513 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1514
1515 pm_runtime_mark_last_busy(d40c->base->dev);
1516 pm_runtime_put_autosuspend(d40c->base->dev);
1517 spin_unlock_irqrestore(&d40c->lock, flags);
1518 return res;
1519 }
1520
1521 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1522 {
1523 struct d40_chan *d40c = container_of(tx->chan,
1524 struct d40_chan,
1525 chan);
1526 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1527 unsigned long flags;
1528 dma_cookie_t cookie;
1529
1530 spin_lock_irqsave(&d40c->lock, flags);
1531 cookie = dma_cookie_assign(tx);
1532 d40_desc_queue(d40c, d40d);
1533 spin_unlock_irqrestore(&d40c->lock, flags);
1534
1535 return cookie;
1536 }
1537
1538 static int d40_start(struct d40_chan *d40c)
1539 {
1540 return d40_channel_execute_command(d40c, D40_DMA_RUN);
1541 }
1542
1543 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1544 {
1545 struct d40_desc *d40d;
1546 int err;
1547
1548 /* Start queued jobs, if any */
1549 d40d = d40_first_queued(d40c);
1550
1551 if (d40d != NULL) {
1552 if (!d40c->busy) {
1553 d40c->busy = true;
1554 pm_runtime_get_sync(d40c->base->dev);
1555 }
1556
1557 /* Remove from queue */
1558 d40_desc_remove(d40d);
1559
1560 /* Add to active queue */
1561 d40_desc_submit(d40c, d40d);
1562
1563 /* Initiate DMA job */
1564 d40_desc_load(d40c, d40d);
1565
1566 /* Start dma job */
1567 err = d40_start(d40c);
1568
1569 if (err)
1570 return NULL;
1571 }
1572
1573 return d40d;
1574 }
1575
1576 /* called from interrupt context */
1577 static void dma_tc_handle(struct d40_chan *d40c)
1578 {
1579 struct d40_desc *d40d;
1580
1581 /* Get first active entry from list */
1582 d40d = d40_first_active_get(d40c);
1583
1584 if (d40d == NULL)
1585 return;
1586
1587 if (d40d->cyclic) {
1588 /*
1589 * If this was a paritially loaded list, we need to reloaded
1590 * it, and only when the list is completed. We need to check
1591 * for done because the interrupt will hit for every link, and
1592 * not just the last one.
1593 */
1594 if (d40d->lli_current < d40d->lli_len
1595 && !d40_tx_is_linked(d40c)
1596 && !d40_residue(d40c)) {
1597 d40_lcla_free_all(d40c, d40d);
1598 d40_desc_load(d40c, d40d);
1599 (void) d40_start(d40c);
1600
1601 if (d40d->lli_current == d40d->lli_len)
1602 d40d->lli_current = 0;
1603 }
1604 } else {
1605 d40_lcla_free_all(d40c, d40d);
1606
1607 if (d40d->lli_current < d40d->lli_len) {
1608 d40_desc_load(d40c, d40d);
1609 /* Start dma job */
1610 (void) d40_start(d40c);
1611 return;
1612 }
1613
1614 if (d40_queue_start(d40c) == NULL)
1615 d40c->busy = false;
1616 pm_runtime_mark_last_busy(d40c->base->dev);
1617 pm_runtime_put_autosuspend(d40c->base->dev);
1618
1619 d40_desc_remove(d40d);
1620 d40_desc_done(d40c, d40d);
1621 }
1622
1623 d40c->pending_tx++;
1624 tasklet_schedule(&d40c->tasklet);
1625
1626 }
1627
1628 static void dma_tasklet(unsigned long data)
1629 {
1630 struct d40_chan *d40c = (struct d40_chan *) data;
1631 struct d40_desc *d40d;
1632 unsigned long flags;
1633 dma_async_tx_callback callback;
1634 void *callback_param;
1635
1636 spin_lock_irqsave(&d40c->lock, flags);
1637
1638 /* Get first entry from the done list */
1639 d40d = d40_first_done(d40c);
1640 if (d40d == NULL) {
1641 /* Check if we have reached here for cyclic job */
1642 d40d = d40_first_active_get(d40c);
1643 if (d40d == NULL || !d40d->cyclic)
1644 goto err;
1645 }
1646
1647 if (!d40d->cyclic)
1648 dma_cookie_complete(&d40d->txd);
1649
1650 /*
1651 * If terminating a channel pending_tx is set to zero.
1652 * This prevents any finished active jobs to return to the client.
1653 */
1654 if (d40c->pending_tx == 0) {
1655 spin_unlock_irqrestore(&d40c->lock, flags);
1656 return;
1657 }
1658
1659 /* Callback to client */
1660 callback = d40d->txd.callback;
1661 callback_param = d40d->txd.callback_param;
1662
1663 if (!d40d->cyclic) {
1664 if (async_tx_test_ack(&d40d->txd)) {
1665 d40_desc_remove(d40d);
1666 d40_desc_free(d40c, d40d);
1667 } else if (!d40d->is_in_client_list) {
1668 d40_desc_remove(d40d);
1669 d40_lcla_free_all(d40c, d40d);
1670 list_add_tail(&d40d->node, &d40c->client);
1671 d40d->is_in_client_list = true;
1672 }
1673 }
1674
1675 d40c->pending_tx--;
1676
1677 if (d40c->pending_tx)
1678 tasklet_schedule(&d40c->tasklet);
1679
1680 spin_unlock_irqrestore(&d40c->lock, flags);
1681
1682 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1683 callback(callback_param);
1684
1685 return;
1686
1687 err:
1688 /* Rescue manouver if receiving double interrupts */
1689 if (d40c->pending_tx > 0)
1690 d40c->pending_tx--;
1691 spin_unlock_irqrestore(&d40c->lock, flags);
1692 }
1693
1694 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1695 {
1696 int i;
1697 u32 idx;
1698 u32 row;
1699 long chan = -1;
1700 struct d40_chan *d40c;
1701 unsigned long flags;
1702 struct d40_base *base = data;
1703 u32 regs[base->gen_dmac.il_size];
1704 struct d40_interrupt_lookup *il = base->gen_dmac.il;
1705 u32 il_size = base->gen_dmac.il_size;
1706
1707 spin_lock_irqsave(&base->interrupt_lock, flags);
1708
1709 /* Read interrupt status of both logical and physical channels */
1710 for (i = 0; i < il_size; i++)
1711 regs[i] = readl(base->virtbase + il[i].src);
1712
1713 for (;;) {
1714
1715 chan = find_next_bit((unsigned long *)regs,
1716 BITS_PER_LONG * il_size, chan + 1);
1717
1718 /* No more set bits found? */
1719 if (chan == BITS_PER_LONG * il_size)
1720 break;
1721
1722 row = chan / BITS_PER_LONG;
1723 idx = chan & (BITS_PER_LONG - 1);
1724
1725 if (il[row].offset == D40_PHY_CHAN)
1726 d40c = base->lookup_phy_chans[idx];
1727 else
1728 d40c = base->lookup_log_chans[il[row].offset + idx];
1729
1730 if (!d40c) {
1731 /*
1732 * No error because this can happen if something else
1733 * in the system is using the channel.
1734 */
1735 continue;
1736 }
1737
1738 /* ACK interrupt */
1739 writel(1 << idx, base->virtbase + il[row].clr);
1740
1741 spin_lock(&d40c->lock);
1742
1743 if (!il[row].is_error)
1744 dma_tc_handle(d40c);
1745 else
1746 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1747 chan, il[row].offset, idx);
1748
1749 spin_unlock(&d40c->lock);
1750 }
1751
1752 spin_unlock_irqrestore(&base->interrupt_lock, flags);
1753
1754 return IRQ_HANDLED;
1755 }
1756
1757 static int d40_validate_conf(struct d40_chan *d40c,
1758 struct stedma40_chan_cfg *conf)
1759 {
1760 int res = 0;
1761 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1762
1763 if (!conf->dir) {
1764 chan_err(d40c, "Invalid direction.\n");
1765 res = -EINVAL;
1766 }
1767
1768 if ((is_log && conf->dev_type > d40c->base->num_log_chans) ||
1769 (!is_log && conf->dev_type > d40c->base->num_phy_chans) ||
1770 (conf->dev_type < 0)) {
1771 chan_err(d40c, "Invalid device type (%d)\n", conf->dev_type);
1772 res = -EINVAL;
1773 }
1774
1775 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1776 d40c->base->plat_data->dev_tx[conf->dev_type] == 0 &&
1777 d40c->runtime_addr == 0) {
1778 chan_err(d40c, "Invalid TX channel address (%d)\n",
1779 conf->dev_type);
1780 res = -EINVAL;
1781 }
1782
1783 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1784 d40c->base->plat_data->dev_rx[conf->dev_type] == 0 &&
1785 d40c->runtime_addr == 0) {
1786 chan_err(d40c, "Invalid RX channel address (%d)\n",
1787 conf->dev_type);
1788 res = -EINVAL;
1789 }
1790
1791 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1792 /*
1793 * DMAC HW supports it. Will be added to this driver,
1794 * in case any dma client requires it.
1795 */
1796 chan_err(d40c, "periph to periph not supported\n");
1797 res = -EINVAL;
1798 }
1799
1800 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1801 (1 << conf->src_info.data_width) !=
1802 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1803 (1 << conf->dst_info.data_width)) {
1804 /*
1805 * The DMAC hardware only supports
1806 * src (burst x width) == dst (burst x width)
1807 */
1808
1809 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1810 res = -EINVAL;
1811 }
1812
1813 return res;
1814 }
1815
1816 static bool d40_alloc_mask_set(struct d40_phy_res *phy,
1817 bool is_src, int log_event_line, bool is_log,
1818 bool *first_user)
1819 {
1820 unsigned long flags;
1821 spin_lock_irqsave(&phy->lock, flags);
1822
1823 *first_user = ((phy->allocated_src | phy->allocated_dst)
1824 == D40_ALLOC_FREE);
1825
1826 if (!is_log) {
1827 /* Physical interrupts are masked per physical full channel */
1828 if (phy->allocated_src == D40_ALLOC_FREE &&
1829 phy->allocated_dst == D40_ALLOC_FREE) {
1830 phy->allocated_dst = D40_ALLOC_PHY;
1831 phy->allocated_src = D40_ALLOC_PHY;
1832 goto found;
1833 } else
1834 goto not_found;
1835 }
1836
1837 /* Logical channel */
1838 if (is_src) {
1839 if (phy->allocated_src == D40_ALLOC_PHY)
1840 goto not_found;
1841
1842 if (phy->allocated_src == D40_ALLOC_FREE)
1843 phy->allocated_src = D40_ALLOC_LOG_FREE;
1844
1845 if (!(phy->allocated_src & (1 << log_event_line))) {
1846 phy->allocated_src |= 1 << log_event_line;
1847 goto found;
1848 } else
1849 goto not_found;
1850 } else {
1851 if (phy->allocated_dst == D40_ALLOC_PHY)
1852 goto not_found;
1853
1854 if (phy->allocated_dst == D40_ALLOC_FREE)
1855 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1856
1857 if (!(phy->allocated_dst & (1 << log_event_line))) {
1858 phy->allocated_dst |= 1 << log_event_line;
1859 goto found;
1860 } else
1861 goto not_found;
1862 }
1863
1864 not_found:
1865 spin_unlock_irqrestore(&phy->lock, flags);
1866 return false;
1867 found:
1868 spin_unlock_irqrestore(&phy->lock, flags);
1869 return true;
1870 }
1871
1872 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1873 int log_event_line)
1874 {
1875 unsigned long flags;
1876 bool is_free = false;
1877
1878 spin_lock_irqsave(&phy->lock, flags);
1879 if (!log_event_line) {
1880 phy->allocated_dst = D40_ALLOC_FREE;
1881 phy->allocated_src = D40_ALLOC_FREE;
1882 is_free = true;
1883 goto out;
1884 }
1885
1886 /* Logical channel */
1887 if (is_src) {
1888 phy->allocated_src &= ~(1 << log_event_line);
1889 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1890 phy->allocated_src = D40_ALLOC_FREE;
1891 } else {
1892 phy->allocated_dst &= ~(1 << log_event_line);
1893 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1894 phy->allocated_dst = D40_ALLOC_FREE;
1895 }
1896
1897 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1898 D40_ALLOC_FREE);
1899
1900 out:
1901 spin_unlock_irqrestore(&phy->lock, flags);
1902
1903 return is_free;
1904 }
1905
1906 static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
1907 {
1908 int dev_type = d40c->dma_cfg.dev_type;
1909 int event_group;
1910 int event_line;
1911 struct d40_phy_res *phys;
1912 int i;
1913 int j;
1914 int log_num;
1915 int num_phy_chans;
1916 bool is_src;
1917 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1918
1919 phys = d40c->base->phy_res;
1920 num_phy_chans = d40c->base->num_phy_chans;
1921
1922 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1923 log_num = 2 * dev_type;
1924 is_src = true;
1925 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1926 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1927 /* dst event lines are used for logical memcpy */
1928 log_num = 2 * dev_type + 1;
1929 is_src = false;
1930 } else
1931 return -EINVAL;
1932
1933 event_group = D40_TYPE_TO_GROUP(dev_type);
1934 event_line = D40_TYPE_TO_EVENT(dev_type);
1935
1936 if (!is_log) {
1937 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1938 /* Find physical half channel */
1939 if (d40c->dma_cfg.use_fixed_channel) {
1940 i = d40c->dma_cfg.phy_channel;
1941 if (d40_alloc_mask_set(&phys[i], is_src,
1942 0, is_log,
1943 first_phy_user))
1944 goto found_phy;
1945 } else {
1946 for (i = 0; i < num_phy_chans; i++) {
1947 if (d40_alloc_mask_set(&phys[i], is_src,
1948 0, is_log,
1949 first_phy_user))
1950 goto found_phy;
1951 }
1952 }
1953 } else
1954 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1955 int phy_num = j + event_group * 2;
1956 for (i = phy_num; i < phy_num + 2; i++) {
1957 if (d40_alloc_mask_set(&phys[i],
1958 is_src,
1959 0,
1960 is_log,
1961 first_phy_user))
1962 goto found_phy;
1963 }
1964 }
1965 return -EINVAL;
1966 found_phy:
1967 d40c->phy_chan = &phys[i];
1968 d40c->log_num = D40_PHY_CHAN;
1969 goto out;
1970 }
1971 if (dev_type == -1)
1972 return -EINVAL;
1973
1974 /* Find logical channel */
1975 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1976 int phy_num = j + event_group * 2;
1977
1978 if (d40c->dma_cfg.use_fixed_channel) {
1979 i = d40c->dma_cfg.phy_channel;
1980
1981 if ((i != phy_num) && (i != phy_num + 1)) {
1982 dev_err(chan2dev(d40c),
1983 "invalid fixed phy channel %d\n", i);
1984 return -EINVAL;
1985 }
1986
1987 if (d40_alloc_mask_set(&phys[i], is_src, event_line,
1988 is_log, first_phy_user))
1989 goto found_log;
1990
1991 dev_err(chan2dev(d40c),
1992 "could not allocate fixed phy channel %d\n", i);
1993 return -EINVAL;
1994 }
1995
1996 /*
1997 * Spread logical channels across all available physical rather
1998 * than pack every logical channel at the first available phy
1999 * channels.
2000 */
2001 if (is_src) {
2002 for (i = phy_num; i < phy_num + 2; i++) {
2003 if (d40_alloc_mask_set(&phys[i], is_src,
2004 event_line, is_log,
2005 first_phy_user))
2006 goto found_log;
2007 }
2008 } else {
2009 for (i = phy_num + 1; i >= phy_num; i--) {
2010 if (d40_alloc_mask_set(&phys[i], is_src,
2011 event_line, is_log,
2012 first_phy_user))
2013 goto found_log;
2014 }
2015 }
2016 }
2017 return -EINVAL;
2018
2019 found_log:
2020 d40c->phy_chan = &phys[i];
2021 d40c->log_num = log_num;
2022 out:
2023
2024 if (is_log)
2025 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
2026 else
2027 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
2028
2029 return 0;
2030
2031 }
2032
2033 static int d40_config_memcpy(struct d40_chan *d40c)
2034 {
2035 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
2036
2037 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
2038 d40c->dma_cfg = dma40_memcpy_conf_log;
2039 d40c->dma_cfg.dev_type = dma40_memcpy_channels[d40c->chan.chan_id];
2040
2041 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
2042 dma_has_cap(DMA_SLAVE, cap)) {
2043 d40c->dma_cfg = dma40_memcpy_conf_phy;
2044 } else {
2045 chan_err(d40c, "No memcpy\n");
2046 return -EINVAL;
2047 }
2048
2049 return 0;
2050 }
2051
2052 static int d40_free_dma(struct d40_chan *d40c)
2053 {
2054
2055 int res = 0;
2056 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
2057 struct d40_phy_res *phy = d40c->phy_chan;
2058 bool is_src;
2059
2060 /* Terminate all queued and active transfers */
2061 d40_term_all(d40c);
2062
2063 if (phy == NULL) {
2064 chan_err(d40c, "phy == null\n");
2065 return -EINVAL;
2066 }
2067
2068 if (phy->allocated_src == D40_ALLOC_FREE &&
2069 phy->allocated_dst == D40_ALLOC_FREE) {
2070 chan_err(d40c, "channel already free\n");
2071 return -EINVAL;
2072 }
2073
2074 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
2075 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM)
2076 is_src = false;
2077 else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
2078 is_src = true;
2079 else {
2080 chan_err(d40c, "Unknown direction\n");
2081 return -EINVAL;
2082 }
2083
2084 pm_runtime_get_sync(d40c->base->dev);
2085 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
2086 if (res) {
2087 chan_err(d40c, "stop failed\n");
2088 goto out;
2089 }
2090
2091 d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);
2092
2093 if (chan_is_logical(d40c))
2094 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
2095 else
2096 d40c->base->lookup_phy_chans[phy->num] = NULL;
2097
2098 if (d40c->busy) {
2099 pm_runtime_mark_last_busy(d40c->base->dev);
2100 pm_runtime_put_autosuspend(d40c->base->dev);
2101 }
2102
2103 d40c->busy = false;
2104 d40c->phy_chan = NULL;
2105 d40c->configured = false;
2106 out:
2107
2108 pm_runtime_mark_last_busy(d40c->base->dev);
2109 pm_runtime_put_autosuspend(d40c->base->dev);
2110 return res;
2111 }
2112
2113 static bool d40_is_paused(struct d40_chan *d40c)
2114 {
2115 void __iomem *chanbase = chan_base(d40c);
2116 bool is_paused = false;
2117 unsigned long flags;
2118 void __iomem *active_reg;
2119 u32 status;
2120 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
2121
2122 spin_lock_irqsave(&d40c->lock, flags);
2123
2124 if (chan_is_physical(d40c)) {
2125 if (d40c->phy_chan->num % 2 == 0)
2126 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
2127 else
2128 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
2129
2130 status = (readl(active_reg) &
2131 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
2132 D40_CHAN_POS(d40c->phy_chan->num);
2133 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
2134 is_paused = true;
2135
2136 goto _exit;
2137 }
2138
2139 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
2140 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
2141 status = readl(chanbase + D40_CHAN_REG_SDLNK);
2142 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
2143 status = readl(chanbase + D40_CHAN_REG_SSLNK);
2144 } else {
2145 chan_err(d40c, "Unknown direction\n");
2146 goto _exit;
2147 }
2148
2149 status = (status & D40_EVENTLINE_MASK(event)) >>
2150 D40_EVENTLINE_POS(event);
2151
2152 if (status != D40_DMA_RUN)
2153 is_paused = true;
2154 _exit:
2155 spin_unlock_irqrestore(&d40c->lock, flags);
2156 return is_paused;
2157
2158 }
2159
2160 static u32 stedma40_residue(struct dma_chan *chan)
2161 {
2162 struct d40_chan *d40c =
2163 container_of(chan, struct d40_chan, chan);
2164 u32 bytes_left;
2165 unsigned long flags;
2166
2167 spin_lock_irqsave(&d40c->lock, flags);
2168 bytes_left = d40_residue(d40c);
2169 spin_unlock_irqrestore(&d40c->lock, flags);
2170
2171 return bytes_left;
2172 }
2173
2174 static int
2175 d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
2176 struct scatterlist *sg_src, struct scatterlist *sg_dst,
2177 unsigned int sg_len, dma_addr_t src_dev_addr,
2178 dma_addr_t dst_dev_addr)
2179 {
2180 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2181 struct stedma40_half_channel_info *src_info = &cfg->src_info;
2182 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
2183 int ret;
2184
2185 ret = d40_log_sg_to_lli(sg_src, sg_len,
2186 src_dev_addr,
2187 desc->lli_log.src,
2188 chan->log_def.lcsp1,
2189 src_info->data_width,
2190 dst_info->data_width);
2191
2192 ret = d40_log_sg_to_lli(sg_dst, sg_len,
2193 dst_dev_addr,
2194 desc->lli_log.dst,
2195 chan->log_def.lcsp3,
2196 dst_info->data_width,
2197 src_info->data_width);
2198
2199 return ret < 0 ? ret : 0;
2200 }
2201
2202 static int
2203 d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
2204 struct scatterlist *sg_src, struct scatterlist *sg_dst,
2205 unsigned int sg_len, dma_addr_t src_dev_addr,
2206 dma_addr_t dst_dev_addr)
2207 {
2208 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2209 struct stedma40_half_channel_info *src_info = &cfg->src_info;
2210 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
2211 unsigned long flags = 0;
2212 int ret;
2213
2214 if (desc->cyclic)
2215 flags |= LLI_CYCLIC | LLI_TERM_INT;
2216
2217 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
2218 desc->lli_phy.src,
2219 virt_to_phys(desc->lli_phy.src),
2220 chan->src_def_cfg,
2221 src_info, dst_info, flags);
2222
2223 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
2224 desc->lli_phy.dst,
2225 virt_to_phys(desc->lli_phy.dst),
2226 chan->dst_def_cfg,
2227 dst_info, src_info, flags);
2228
2229 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
2230 desc->lli_pool.size, DMA_TO_DEVICE);
2231
2232 return ret < 0 ? ret : 0;
2233 }
2234
2235 static struct d40_desc *
2236 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
2237 unsigned int sg_len, unsigned long dma_flags)
2238 {
2239 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2240 struct d40_desc *desc;
2241 int ret;
2242
2243 desc = d40_desc_get(chan);
2244 if (!desc)
2245 return NULL;
2246
2247 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
2248 cfg->dst_info.data_width);
2249 if (desc->lli_len < 0) {
2250 chan_err(chan, "Unaligned size\n");
2251 goto err;
2252 }
2253
2254 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
2255 if (ret < 0) {
2256 chan_err(chan, "Could not allocate lli\n");
2257 goto err;
2258 }
2259
2260 desc->lli_current = 0;
2261 desc->txd.flags = dma_flags;
2262 desc->txd.tx_submit = d40_tx_submit;
2263
2264 dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
2265
2266 return desc;
2267
2268 err:
2269 d40_desc_free(chan, desc);
2270 return NULL;
2271 }
2272
2273 static dma_addr_t
2274 d40_get_dev_addr(struct d40_chan *chan, enum dma_transfer_direction direction)
2275 {
2276 struct stedma40_platform_data *plat = chan->base->plat_data;
2277 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2278 dma_addr_t addr = 0;
2279
2280 if (chan->runtime_addr)
2281 return chan->runtime_addr;
2282
2283 if (direction == DMA_DEV_TO_MEM)
2284 addr = plat->dev_rx[cfg->dev_type];
2285 else if (direction == DMA_MEM_TO_DEV)
2286 addr = plat->dev_tx[cfg->dev_type];
2287
2288 return addr;
2289 }
2290
2291 static struct dma_async_tx_descriptor *
2292 d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
2293 struct scatterlist *sg_dst, unsigned int sg_len,
2294 enum dma_transfer_direction direction, unsigned long dma_flags)
2295 {
2296 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
2297 dma_addr_t src_dev_addr = 0;
2298 dma_addr_t dst_dev_addr = 0;
2299 struct d40_desc *desc;
2300 unsigned long flags;
2301 int ret;
2302
2303 if (!chan->phy_chan) {
2304 chan_err(chan, "Cannot prepare unallocated channel\n");
2305 return NULL;
2306 }
2307
2308 spin_lock_irqsave(&chan->lock, flags);
2309
2310 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
2311 if (desc == NULL)
2312 goto err;
2313
2314 if (sg_next(&sg_src[sg_len - 1]) == sg_src)
2315 desc->cyclic = true;
2316
2317 if (direction != DMA_TRANS_NONE) {
2318 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
2319
2320 if (direction == DMA_DEV_TO_MEM)
2321 src_dev_addr = dev_addr;
2322 else if (direction == DMA_MEM_TO_DEV)
2323 dst_dev_addr = dev_addr;
2324 }
2325
2326 if (chan_is_logical(chan))
2327 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
2328 sg_len, src_dev_addr, dst_dev_addr);
2329 else
2330 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
2331 sg_len, src_dev_addr, dst_dev_addr);
2332
2333 if (ret) {
2334 chan_err(chan, "Failed to prepare %s sg job: %d\n",
2335 chan_is_logical(chan) ? "log" : "phy", ret);
2336 goto err;
2337 }
2338
2339 /*
2340 * add descriptor to the prepare queue in order to be able
2341 * to free them later in terminate_all
2342 */
2343 list_add_tail(&desc->node, &chan->prepare_queue);
2344
2345 spin_unlock_irqrestore(&chan->lock, flags);
2346
2347 return &desc->txd;
2348
2349 err:
2350 if (desc)
2351 d40_desc_free(chan, desc);
2352 spin_unlock_irqrestore(&chan->lock, flags);
2353 return NULL;
2354 }
2355
2356 bool stedma40_filter(struct dma_chan *chan, void *data)
2357 {
2358 struct stedma40_chan_cfg *info = data;
2359 struct d40_chan *d40c =
2360 container_of(chan, struct d40_chan, chan);
2361 int err;
2362
2363 if (data) {
2364 err = d40_validate_conf(d40c, info);
2365 if (!err)
2366 d40c->dma_cfg = *info;
2367 } else
2368 err = d40_config_memcpy(d40c);
2369
2370 if (!err)
2371 d40c->configured = true;
2372
2373 return err == 0;
2374 }
2375 EXPORT_SYMBOL(stedma40_filter);
2376
2377 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
2378 {
2379 bool realtime = d40c->dma_cfg.realtime;
2380 bool highprio = d40c->dma_cfg.high_priority;
2381 u32 rtreg;
2382 u32 event = D40_TYPE_TO_EVENT(dev_type);
2383 u32 group = D40_TYPE_TO_GROUP(dev_type);
2384 u32 bit = 1 << event;
2385 u32 prioreg;
2386 struct d40_gen_dmac *dmac = &d40c->base->gen_dmac;
2387
2388 rtreg = realtime ? dmac->realtime_en : dmac->realtime_clear;
2389 /*
2390 * Due to a hardware bug, in some cases a logical channel triggered by
2391 * a high priority destination event line can generate extra packet
2392 * transactions.
2393 *
2394 * The workaround is to not set the high priority level for the
2395 * destination event lines that trigger logical channels.
2396 */
2397 if (!src && chan_is_logical(d40c))
2398 highprio = false;
2399
2400 prioreg = highprio ? dmac->high_prio_en : dmac->high_prio_clear;
2401
2402 /* Destination event lines are stored in the upper halfword */
2403 if (!src)
2404 bit <<= 16;
2405
2406 writel(bit, d40c->base->virtbase + prioreg + group * 4);
2407 writel(bit, d40c->base->virtbase + rtreg + group * 4);
2408 }
2409
2410 static void d40_set_prio_realtime(struct d40_chan *d40c)
2411 {
2412 if (d40c->base->rev < 3)
2413 return;
2414
2415 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
2416 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
2417 __d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, true);
2418
2419 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
2420 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
2421 __d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, false);
2422 }
2423
2424 /* DMA ENGINE functions */
2425 static int d40_alloc_chan_resources(struct dma_chan *chan)
2426 {
2427 int err;
2428 unsigned long flags;
2429 struct d40_chan *d40c =
2430 container_of(chan, struct d40_chan, chan);
2431 bool is_free_phy;
2432 spin_lock_irqsave(&d40c->lock, flags);
2433
2434 dma_cookie_init(chan);
2435
2436 /* If no dma configuration is set use default configuration (memcpy) */
2437 if (!d40c->configured) {
2438 err = d40_config_memcpy(d40c);
2439 if (err) {
2440 chan_err(d40c, "Failed to configure memcpy channel\n");
2441 goto fail;
2442 }
2443 }
2444
2445 err = d40_allocate_channel(d40c, &is_free_phy);
2446 if (err) {
2447 chan_err(d40c, "Failed to allocate channel\n");
2448 d40c->configured = false;
2449 goto fail;
2450 }
2451
2452 pm_runtime_get_sync(d40c->base->dev);
2453 /* Fill in basic CFG register values */
2454 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
2455 &d40c->dst_def_cfg, chan_is_logical(d40c));
2456
2457 d40_set_prio_realtime(d40c);
2458
2459 if (chan_is_logical(d40c)) {
2460 d40_log_cfg(&d40c->dma_cfg,
2461 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2462
2463 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
2464 d40c->lcpa = d40c->base->lcpa_base +
2465 d40c->dma_cfg.dev_type * D40_LCPA_CHAN_SIZE;
2466 else
2467 d40c->lcpa = d40c->base->lcpa_base +
2468 d40c->dma_cfg.dev_type *
2469 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
2470 }
2471
2472 dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
2473 chan_is_logical(d40c) ? "logical" : "physical",
2474 d40c->phy_chan->num,
2475 d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
2476
2477
2478 /*
2479 * Only write channel configuration to the DMA if the physical
2480 * resource is free. In case of multiple logical channels
2481 * on the same physical resource, only the first write is necessary.
2482 */
2483 if (is_free_phy)
2484 d40_config_write(d40c);
2485 fail:
2486 pm_runtime_mark_last_busy(d40c->base->dev);
2487 pm_runtime_put_autosuspend(d40c->base->dev);
2488 spin_unlock_irqrestore(&d40c->lock, flags);
2489 return err;
2490 }
2491
2492 static void d40_free_chan_resources(struct dma_chan *chan)
2493 {
2494 struct d40_chan *d40c =
2495 container_of(chan, struct d40_chan, chan);
2496 int err;
2497 unsigned long flags;
2498
2499 if (d40c->phy_chan == NULL) {
2500 chan_err(d40c, "Cannot free unallocated channel\n");
2501 return;
2502 }
2503
2504 spin_lock_irqsave(&d40c->lock, flags);
2505
2506 err = d40_free_dma(d40c);
2507
2508 if (err)
2509 chan_err(d40c, "Failed to free channel\n");
2510 spin_unlock_irqrestore(&d40c->lock, flags);
2511 }
2512
2513 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2514 dma_addr_t dst,
2515 dma_addr_t src,
2516 size_t size,
2517 unsigned long dma_flags)
2518 {
2519 struct scatterlist dst_sg;
2520 struct scatterlist src_sg;
2521
2522 sg_init_table(&dst_sg, 1);
2523 sg_init_table(&src_sg, 1);
2524
2525 sg_dma_address(&dst_sg) = dst;
2526 sg_dma_address(&src_sg) = src;
2527
2528 sg_dma_len(&dst_sg) = size;
2529 sg_dma_len(&src_sg) = size;
2530
2531 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
2532 }
2533
2534 static struct dma_async_tx_descriptor *
2535 d40_prep_memcpy_sg(struct dma_chan *chan,
2536 struct scatterlist *dst_sg, unsigned int dst_nents,
2537 struct scatterlist *src_sg, unsigned int src_nents,
2538 unsigned long dma_flags)
2539 {
2540 if (dst_nents != src_nents)
2541 return NULL;
2542
2543 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
2544 }
2545
2546 static struct dma_async_tx_descriptor *
2547 d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2548 unsigned int sg_len, enum dma_transfer_direction direction,
2549 unsigned long dma_flags, void *context)
2550 {
2551 if (!is_slave_direction(direction))
2552 return NULL;
2553
2554 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
2555 }
2556
2557 static struct dma_async_tx_descriptor *
2558 dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2559 size_t buf_len, size_t period_len,
2560 enum dma_transfer_direction direction, unsigned long flags,
2561 void *context)
2562 {
2563 unsigned int periods = buf_len / period_len;
2564 struct dma_async_tx_descriptor *txd;
2565 struct scatterlist *sg;
2566 int i;
2567
2568 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);
2569 for (i = 0; i < periods; i++) {
2570 sg_dma_address(&sg[i]) = dma_addr;
2571 sg_dma_len(&sg[i]) = period_len;
2572 dma_addr += period_len;
2573 }
2574
2575 sg[periods].offset = 0;
2576 sg_dma_len(&sg[periods]) = 0;
2577 sg[periods].page_link =
2578 ((unsigned long)sg | 0x01) & ~0x02;
2579
2580 txd = d40_prep_sg(chan, sg, sg, periods, direction,
2581 DMA_PREP_INTERRUPT);
2582
2583 kfree(sg);
2584
2585 return txd;
2586 }
2587
2588 static enum dma_status d40_tx_status(struct dma_chan *chan,
2589 dma_cookie_t cookie,
2590 struct dma_tx_state *txstate)
2591 {
2592 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2593 enum dma_status ret;
2594
2595 if (d40c->phy_chan == NULL) {
2596 chan_err(d40c, "Cannot read status of unallocated channel\n");
2597 return -EINVAL;
2598 }
2599
2600 ret = dma_cookie_status(chan, cookie, txstate);
2601 if (ret != DMA_SUCCESS)
2602 dma_set_residue(txstate, stedma40_residue(chan));
2603
2604 if (d40_is_paused(d40c))
2605 ret = DMA_PAUSED;
2606
2607 return ret;
2608 }
2609
2610 static void d40_issue_pending(struct dma_chan *chan)
2611 {
2612 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2613 unsigned long flags;
2614
2615 if (d40c->phy_chan == NULL) {
2616 chan_err(d40c, "Channel is not allocated!\n");
2617 return;
2618 }
2619
2620 spin_lock_irqsave(&d40c->lock, flags);
2621
2622 list_splice_tail_init(&d40c->pending_queue, &d40c->queue);
2623
2624 /* Busy means that queued jobs are already being processed */
2625 if (!d40c->busy)
2626 (void) d40_queue_start(d40c);
2627
2628 spin_unlock_irqrestore(&d40c->lock, flags);
2629 }
2630
2631 static void d40_terminate_all(struct dma_chan *chan)
2632 {
2633 unsigned long flags;
2634 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2635 int ret;
2636
2637 spin_lock_irqsave(&d40c->lock, flags);
2638
2639 pm_runtime_get_sync(d40c->base->dev);
2640 ret = d40_channel_execute_command(d40c, D40_DMA_STOP);
2641 if (ret)
2642 chan_err(d40c, "Failed to stop channel\n");
2643
2644 d40_term_all(d40c);
2645 pm_runtime_mark_last_busy(d40c->base->dev);
2646 pm_runtime_put_autosuspend(d40c->base->dev);
2647 if (d40c->busy) {
2648 pm_runtime_mark_last_busy(d40c->base->dev);
2649 pm_runtime_put_autosuspend(d40c->base->dev);
2650 }
2651 d40c->busy = false;
2652
2653 spin_unlock_irqrestore(&d40c->lock, flags);
2654 }
2655
2656 static int
2657 dma40_config_to_halfchannel(struct d40_chan *d40c,
2658 struct stedma40_half_channel_info *info,
2659 enum dma_slave_buswidth width,
2660 u32 maxburst)
2661 {
2662 enum stedma40_periph_data_width addr_width;
2663 int psize;
2664
2665 switch (width) {
2666 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2667 addr_width = STEDMA40_BYTE_WIDTH;
2668 break;
2669 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2670 addr_width = STEDMA40_HALFWORD_WIDTH;
2671 break;
2672 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2673 addr_width = STEDMA40_WORD_WIDTH;
2674 break;
2675 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2676 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2677 break;
2678 default:
2679 dev_err(d40c->base->dev,
2680 "illegal peripheral address width "
2681 "requested (%d)\n",
2682 width);
2683 return -EINVAL;
2684 }
2685
2686 if (chan_is_logical(d40c)) {
2687 if (maxburst >= 16)
2688 psize = STEDMA40_PSIZE_LOG_16;
2689 else if (maxburst >= 8)
2690 psize = STEDMA40_PSIZE_LOG_8;
2691 else if (maxburst >= 4)
2692 psize = STEDMA40_PSIZE_LOG_4;
2693 else
2694 psize = STEDMA40_PSIZE_LOG_1;
2695 } else {
2696 if (maxburst >= 16)
2697 psize = STEDMA40_PSIZE_PHY_16;
2698 else if (maxburst >= 8)
2699 psize = STEDMA40_PSIZE_PHY_8;
2700 else if (maxburst >= 4)
2701 psize = STEDMA40_PSIZE_PHY_4;
2702 else
2703 psize = STEDMA40_PSIZE_PHY_1;
2704 }
2705
2706 info->data_width = addr_width;
2707 info->psize = psize;
2708 info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2709
2710 return 0;
2711 }
2712
2713 /* Runtime reconfiguration extension */
2714 static int d40_set_runtime_config(struct dma_chan *chan,
2715 struct dma_slave_config *config)
2716 {
2717 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2718 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2719 enum dma_slave_buswidth src_addr_width, dst_addr_width;
2720 dma_addr_t config_addr;
2721 u32 src_maxburst, dst_maxburst;
2722 int ret;
2723
2724 src_addr_width = config->src_addr_width;
2725 src_maxburst = config->src_maxburst;
2726 dst_addr_width = config->dst_addr_width;
2727 dst_maxburst = config->dst_maxburst;
2728
2729 if (config->direction == DMA_DEV_TO_MEM) {
2730 dma_addr_t dev_addr_rx =
2731 d40c->base->plat_data->dev_rx[cfg->dev_type];
2732
2733 config_addr = config->src_addr;
2734 if (dev_addr_rx)
2735 dev_dbg(d40c->base->dev,
2736 "channel has a pre-wired RX address %08x "
2737 "overriding with %08x\n",
2738 dev_addr_rx, config_addr);
2739 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2740 dev_dbg(d40c->base->dev,
2741 "channel was not configured for peripheral "
2742 "to memory transfer (%d) overriding\n",
2743 cfg->dir);
2744 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2745
2746 /* Configure the memory side */
2747 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2748 dst_addr_width = src_addr_width;
2749 if (dst_maxburst == 0)
2750 dst_maxburst = src_maxburst;
2751
2752 } else if (config->direction == DMA_MEM_TO_DEV) {
2753 dma_addr_t dev_addr_tx =
2754 d40c->base->plat_data->dev_tx[cfg->dev_type];
2755
2756 config_addr = config->dst_addr;
2757 if (dev_addr_tx)
2758 dev_dbg(d40c->base->dev,
2759 "channel has a pre-wired TX address %08x "
2760 "overriding with %08x\n",
2761 dev_addr_tx, config_addr);
2762 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2763 dev_dbg(d40c->base->dev,
2764 "channel was not configured for memory "
2765 "to peripheral transfer (%d) overriding\n",
2766 cfg->dir);
2767 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2768
2769 /* Configure the memory side */
2770 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2771 src_addr_width = dst_addr_width;
2772 if (src_maxburst == 0)
2773 src_maxburst = dst_maxburst;
2774 } else {
2775 dev_err(d40c->base->dev,
2776 "unrecognized channel direction %d\n",
2777 config->direction);
2778 return -EINVAL;
2779 }
2780
2781 if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
2782 dev_err(d40c->base->dev,
2783 "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2784 src_maxburst,
2785 src_addr_width,
2786 dst_maxburst,
2787 dst_addr_width);
2788 return -EINVAL;
2789 }
2790
2791 if (src_maxburst > 16) {
2792 src_maxburst = 16;
2793 dst_maxburst = src_maxburst * src_addr_width / dst_addr_width;
2794 } else if (dst_maxburst > 16) {
2795 dst_maxburst = 16;
2796 src_maxburst = dst_maxburst * dst_addr_width / src_addr_width;
2797 }
2798
2799 ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,
2800 src_addr_width,
2801 src_maxburst);
2802 if (ret)
2803 return ret;
2804
2805 ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,
2806 dst_addr_width,
2807 dst_maxburst);
2808 if (ret)
2809 return ret;
2810
2811 /* Fill in register values */
2812 if (chan_is_logical(d40c))
2813 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2814 else
2815 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2816 &d40c->dst_def_cfg, false);
2817
2818 /* These settings will take precedence later */
2819 d40c->runtime_addr = config_addr;
2820 d40c->runtime_direction = config->direction;
2821 dev_dbg(d40c->base->dev,
2822 "configured channel %s for %s, data width %d/%d, "
2823 "maxburst %d/%d elements, LE, no flow control\n",
2824 dma_chan_name(chan),
2825 (config->direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
2826 src_addr_width, dst_addr_width,
2827 src_maxburst, dst_maxburst);
2828
2829 return 0;
2830 }
2831
2832 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2833 unsigned long arg)
2834 {
2835 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2836
2837 if (d40c->phy_chan == NULL) {
2838 chan_err(d40c, "Channel is not allocated!\n");
2839 return -EINVAL;
2840 }
2841
2842 switch (cmd) {
2843 case DMA_TERMINATE_ALL:
2844 d40_terminate_all(chan);
2845 return 0;
2846 case DMA_PAUSE:
2847 return d40_pause(d40c);
2848 case DMA_RESUME:
2849 return d40_resume(d40c);
2850 case DMA_SLAVE_CONFIG:
2851 return d40_set_runtime_config(chan,
2852 (struct dma_slave_config *) arg);
2853 default:
2854 break;
2855 }
2856
2857 /* Other commands are unimplemented */
2858 return -ENXIO;
2859 }
2860
2861 /* Initialization functions */
2862
2863 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2864 struct d40_chan *chans, int offset,
2865 int num_chans)
2866 {
2867 int i = 0;
2868 struct d40_chan *d40c;
2869
2870 INIT_LIST_HEAD(&dma->channels);
2871
2872 for (i = offset; i < offset + num_chans; i++) {
2873 d40c = &chans[i];
2874 d40c->base = base;
2875 d40c->chan.device = dma;
2876
2877 spin_lock_init(&d40c->lock);
2878
2879 d40c->log_num = D40_PHY_CHAN;
2880
2881 INIT_LIST_HEAD(&d40c->done);
2882 INIT_LIST_HEAD(&d40c->active);
2883 INIT_LIST_HEAD(&d40c->queue);
2884 INIT_LIST_HEAD(&d40c->pending_queue);
2885 INIT_LIST_HEAD(&d40c->client);
2886 INIT_LIST_HEAD(&d40c->prepare_queue);
2887
2888 tasklet_init(&d40c->tasklet, dma_tasklet,
2889 (unsigned long) d40c);
2890
2891 list_add_tail(&d40c->chan.device_node,
2892 &dma->channels);
2893 }
2894 }
2895
2896 static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2897 {
2898 if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2899 dev->device_prep_slave_sg = d40_prep_slave_sg;
2900
2901 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2902 dev->device_prep_dma_memcpy = d40_prep_memcpy;
2903
2904 /*
2905 * This controller can only access address at even
2906 * 32bit boundaries, i.e. 2^2
2907 */
2908 dev->copy_align = 2;
2909 }
2910
2911 if (dma_has_cap(DMA_SG, dev->cap_mask))
2912 dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2913
2914 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2915 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2916
2917 dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2918 dev->device_free_chan_resources = d40_free_chan_resources;
2919 dev->device_issue_pending = d40_issue_pending;
2920 dev->device_tx_status = d40_tx_status;
2921 dev->device_control = d40_control;
2922 dev->dev = base->dev;
2923 }
2924
2925 static int __init d40_dmaengine_init(struct d40_base *base,
2926 int num_reserved_chans)
2927 {
2928 int err ;
2929
2930 d40_chan_init(base, &base->dma_slave, base->log_chans,
2931 0, base->num_log_chans);
2932
2933 dma_cap_zero(base->dma_slave.cap_mask);
2934 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2935 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2936
2937 d40_ops_init(base, &base->dma_slave);
2938
2939 err = dma_async_device_register(&base->dma_slave);
2940
2941 if (err) {
2942 d40_err(base->dev, "Failed to register slave channels\n");
2943 goto failure1;
2944 }
2945
2946 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2947 base->num_log_chans, ARRAY_SIZE(dma40_memcpy_channels));
2948
2949 dma_cap_zero(base->dma_memcpy.cap_mask);
2950 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2951 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
2952
2953 d40_ops_init(base, &base->dma_memcpy);
2954
2955 err = dma_async_device_register(&base->dma_memcpy);
2956
2957 if (err) {
2958 d40_err(base->dev,
2959 "Failed to regsiter memcpy only channels\n");
2960 goto failure2;
2961 }
2962
2963 d40_chan_init(base, &base->dma_both, base->phy_chans,
2964 0, num_reserved_chans);
2965
2966 dma_cap_zero(base->dma_both.cap_mask);
2967 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2968 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2969 dma_cap_set(DMA_SG, base->dma_both.cap_mask);
2970 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2971
2972 d40_ops_init(base, &base->dma_both);
2973 err = dma_async_device_register(&base->dma_both);
2974
2975 if (err) {
2976 d40_err(base->dev,
2977 "Failed to register logical and physical capable channels\n");
2978 goto failure3;
2979 }
2980 return 0;
2981 failure3:
2982 dma_async_device_unregister(&base->dma_memcpy);
2983 failure2:
2984 dma_async_device_unregister(&base->dma_slave);
2985 failure1:
2986 return err;
2987 }
2988
2989 /* Suspend resume functionality */
2990 #ifdef CONFIG_PM
2991 static int dma40_pm_suspend(struct device *dev)
2992 {
2993 struct platform_device *pdev = to_platform_device(dev);
2994 struct d40_base *base = platform_get_drvdata(pdev);
2995 int ret = 0;
2996
2997 if (base->lcpa_regulator)
2998 ret = regulator_disable(base->lcpa_regulator);
2999 return ret;
3000 }
3001
3002 static int dma40_runtime_suspend(struct device *dev)
3003 {
3004 struct platform_device *pdev = to_platform_device(dev);
3005 struct d40_base *base = platform_get_drvdata(pdev);
3006
3007 d40_save_restore_registers(base, true);
3008
3009 /* Don't disable/enable clocks for v1 due to HW bugs */
3010 if (base->rev != 1)
3011 writel_relaxed(base->gcc_pwr_off_mask,
3012 base->virtbase + D40_DREG_GCC);
3013
3014 return 0;
3015 }
3016
3017 static int dma40_runtime_resume(struct device *dev)
3018 {
3019 struct platform_device *pdev = to_platform_device(dev);
3020 struct d40_base *base = platform_get_drvdata(pdev);
3021
3022 if (base->initialized)
3023 d40_save_restore_registers(base, false);
3024
3025 writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
3026 base->virtbase + D40_DREG_GCC);
3027 return 0;
3028 }
3029
3030 static int dma40_resume(struct device *dev)
3031 {
3032 struct platform_device *pdev = to_platform_device(dev);
3033 struct d40_base *base = platform_get_drvdata(pdev);
3034 int ret = 0;
3035
3036 if (base->lcpa_regulator)
3037 ret = regulator_enable(base->lcpa_regulator);
3038
3039 return ret;
3040 }
3041
3042 static const struct dev_pm_ops dma40_pm_ops = {
3043 .suspend = dma40_pm_suspend,
3044 .runtime_suspend = dma40_runtime_suspend,
3045 .runtime_resume = dma40_runtime_resume,
3046 .resume = dma40_resume,
3047 };
3048 #define DMA40_PM_OPS (&dma40_pm_ops)
3049 #else
3050 #define DMA40_PM_OPS NULL
3051 #endif
3052
3053 /* Initialization functions. */
3054
3055 static int __init d40_phy_res_init(struct d40_base *base)
3056 {
3057 int i;
3058 int num_phy_chans_avail = 0;
3059 u32 val[2];
3060 int odd_even_bit = -2;
3061 int gcc = D40_DREG_GCC_ENA;
3062
3063 val[0] = readl(base->virtbase + D40_DREG_PRSME);
3064 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
3065
3066 for (i = 0; i < base->num_phy_chans; i++) {
3067 base->phy_res[i].num = i;
3068 odd_even_bit += 2 * ((i % 2) == 0);
3069 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
3070 /* Mark security only channels as occupied */
3071 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
3072 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
3073 base->phy_res[i].reserved = true;
3074 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3075 D40_DREG_GCC_SRC);
3076 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3077 D40_DREG_GCC_DST);
3078
3079
3080 } else {
3081 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
3082 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
3083 base->phy_res[i].reserved = false;
3084 num_phy_chans_avail++;
3085 }
3086 spin_lock_init(&base->phy_res[i].lock);
3087 }
3088
3089 /* Mark disabled channels as occupied */
3090 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
3091 int chan = base->plat_data->disabled_channels[i];
3092
3093 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
3094 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
3095 base->phy_res[chan].reserved = true;
3096 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3097 D40_DREG_GCC_SRC);
3098 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3099 D40_DREG_GCC_DST);
3100 num_phy_chans_avail--;
3101 }
3102
3103 /* Mark soft_lli channels */
3104 for (i = 0; i < base->plat_data->num_of_soft_lli_chans; i++) {
3105 int chan = base->plat_data->soft_lli_chans[i];
3106
3107 base->phy_res[chan].use_soft_lli = true;
3108 }
3109
3110 dev_info(base->dev, "%d of %d physical DMA channels available\n",
3111 num_phy_chans_avail, base->num_phy_chans);
3112
3113 /* Verify settings extended vs standard */
3114 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
3115
3116 for (i = 0; i < base->num_phy_chans; i++) {
3117
3118 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
3119 (val[0] & 0x3) != 1)
3120 dev_info(base->dev,
3121 "[%s] INFO: channel %d is misconfigured (%d)\n",
3122 __func__, i, val[0] & 0x3);
3123
3124 val[0] = val[0] >> 2;
3125 }
3126
3127 /*
3128 * To keep things simple, Enable all clocks initially.
3129 * The clocks will get managed later post channel allocation.
3130 * The clocks for the event lines on which reserved channels exists
3131 * are not managed here.
3132 */
3133 writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
3134 base->gcc_pwr_off_mask = gcc;
3135
3136 return num_phy_chans_avail;
3137 }
3138
3139 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
3140 {
3141 struct stedma40_platform_data *plat_data;
3142 struct clk *clk = NULL;
3143 void __iomem *virtbase = NULL;
3144 struct resource *res = NULL;
3145 struct d40_base *base = NULL;
3146 int num_log_chans = 0;
3147 int num_phy_chans;
3148 int clk_ret = -EINVAL;
3149 int i;
3150 u32 pid;
3151 u32 cid;
3152 u8 rev;
3153
3154 clk = clk_get(&pdev->dev, NULL);
3155 if (IS_ERR(clk)) {
3156 d40_err(&pdev->dev, "No matching clock found\n");
3157 goto failure;
3158 }
3159
3160 clk_ret = clk_prepare_enable(clk);
3161 if (clk_ret) {
3162 d40_err(&pdev->dev, "Failed to prepare/enable clock\n");
3163 goto failure;
3164 }
3165
3166 /* Get IO for DMAC base address */
3167 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3168 if (!res)
3169 goto failure;
3170
3171 if (request_mem_region(res->start, resource_size(res),
3172 D40_NAME " I/O base") == NULL)
3173 goto failure;
3174
3175 virtbase = ioremap(res->start, resource_size(res));
3176 if (!virtbase)
3177 goto failure;
3178
3179 /* This is just a regular AMBA PrimeCell ID actually */
3180 for (pid = 0, i = 0; i < 4; i++)
3181 pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i)
3182 & 255) << (i * 8);
3183 for (cid = 0, i = 0; i < 4; i++)
3184 cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i)
3185 & 255) << (i * 8);
3186
3187 if (cid != AMBA_CID) {
3188 d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
3189 goto failure;
3190 }
3191 if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
3192 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
3193 AMBA_MANF_BITS(pid),
3194 AMBA_VENDOR_ST);
3195 goto failure;
3196 }
3197 /*
3198 * HW revision:
3199 * DB8500ed has revision 0
3200 * ? has revision 1
3201 * DB8500v1 has revision 2
3202 * DB8500v2 has revision 3
3203 * AP9540v1 has revision 4
3204 * DB8540v1 has revision 4
3205 */
3206 rev = AMBA_REV_BITS(pid);
3207 if (rev < 2) {
3208 d40_err(&pdev->dev, "hardware revision: %d is not supported", rev);
3209 goto failure;
3210 }
3211
3212 plat_data = pdev->dev.platform_data;
3213
3214 /* The number of physical channels on this HW */
3215 if (plat_data->num_of_phy_chans)
3216 num_phy_chans = plat_data->num_of_phy_chans;
3217 else
3218 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
3219
3220 num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY;
3221
3222 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x with %d physical channels\n",
3223 rev, res->start, num_phy_chans);
3224
3225 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
3226 (num_phy_chans + num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) *
3227 sizeof(struct d40_chan), GFP_KERNEL);
3228
3229 if (base == NULL) {
3230 d40_err(&pdev->dev, "Out of memory\n");
3231 goto failure;
3232 }
3233
3234 base->rev = rev;
3235 base->clk = clk;
3236 base->num_phy_chans = num_phy_chans;
3237 base->num_log_chans = num_log_chans;
3238 base->phy_start = res->start;
3239 base->phy_size = resource_size(res);
3240 base->virtbase = virtbase;
3241 base->plat_data = plat_data;
3242 base->dev = &pdev->dev;
3243 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
3244 base->log_chans = &base->phy_chans[num_phy_chans];
3245
3246 if (base->plat_data->num_of_phy_chans == 14) {
3247 base->gen_dmac.backup = d40_backup_regs_v4b;
3248 base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B;
3249 base->gen_dmac.interrupt_en = D40_DREG_CPCMIS;
3250 base->gen_dmac.interrupt_clear = D40_DREG_CPCICR;
3251 base->gen_dmac.realtime_en = D40_DREG_CRSEG1;
3252 base->gen_dmac.realtime_clear = D40_DREG_CRCEG1;
3253 base->gen_dmac.high_prio_en = D40_DREG_CPSEG1;
3254 base->gen_dmac.high_prio_clear = D40_DREG_CPCEG1;
3255 base->gen_dmac.il = il_v4b;
3256 base->gen_dmac.il_size = ARRAY_SIZE(il_v4b);
3257 base->gen_dmac.init_reg = dma_init_reg_v4b;
3258 base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b);
3259 } else {
3260 if (base->rev >= 3) {
3261 base->gen_dmac.backup = d40_backup_regs_v4a;
3262 base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A;
3263 }
3264 base->gen_dmac.interrupt_en = D40_DREG_PCMIS;
3265 base->gen_dmac.interrupt_clear = D40_DREG_PCICR;
3266 base->gen_dmac.realtime_en = D40_DREG_RSEG1;
3267 base->gen_dmac.realtime_clear = D40_DREG_RCEG1;
3268 base->gen_dmac.high_prio_en = D40_DREG_PSEG1;
3269 base->gen_dmac.high_prio_clear = D40_DREG_PCEG1;
3270 base->gen_dmac.il = il_v4a;
3271 base->gen_dmac.il_size = ARRAY_SIZE(il_v4a);
3272 base->gen_dmac.init_reg = dma_init_reg_v4a;
3273 base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);
3274 }
3275
3276 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
3277 GFP_KERNEL);
3278 if (!base->phy_res)
3279 goto failure;
3280
3281 base->lookup_phy_chans = kzalloc(num_phy_chans *
3282 sizeof(struct d40_chan *),
3283 GFP_KERNEL);
3284 if (!base->lookup_phy_chans)
3285 goto failure;
3286
3287 base->lookup_log_chans = kzalloc(num_log_chans *
3288 sizeof(struct d40_chan *),
3289 GFP_KERNEL);
3290 if (!base->lookup_log_chans)
3291 goto failure;
3292
3293 base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
3294 sizeof(d40_backup_regs_chan),
3295 GFP_KERNEL);
3296 if (!base->reg_val_backup_chan)
3297 goto failure;
3298
3299 base->lcla_pool.alloc_map =
3300 kzalloc(num_phy_chans * sizeof(struct d40_desc *)
3301 * D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
3302 if (!base->lcla_pool.alloc_map)
3303 goto failure;
3304
3305 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
3306 0, SLAB_HWCACHE_ALIGN,
3307 NULL);
3308 if (base->desc_slab == NULL)
3309 goto failure;
3310
3311 return base;
3312
3313 failure:
3314 if (!clk_ret)
3315 clk_disable_unprepare(clk);
3316 if (!IS_ERR(clk))
3317 clk_put(clk);
3318 if (virtbase)
3319 iounmap(virtbase);
3320 if (res)
3321 release_mem_region(res->start,
3322 resource_size(res));
3323 if (virtbase)
3324 iounmap(virtbase);
3325
3326 if (base) {
3327 kfree(base->lcla_pool.alloc_map);
3328 kfree(base->reg_val_backup_chan);
3329 kfree(base->lookup_log_chans);
3330 kfree(base->lookup_phy_chans);
3331 kfree(base->phy_res);
3332 kfree(base);
3333 }
3334
3335 return NULL;
3336 }
3337
3338 static void __init d40_hw_init(struct d40_base *base)
3339 {
3340
3341 int i;
3342 u32 prmseo[2] = {0, 0};
3343 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3344 u32 pcmis = 0;
3345 u32 pcicr = 0;
3346 struct d40_reg_val *dma_init_reg = base->gen_dmac.init_reg;
3347 u32 reg_size = base->gen_dmac.init_reg_size;
3348
3349 for (i = 0; i < reg_size; i++)
3350 writel(dma_init_reg[i].val,
3351 base->virtbase + dma_init_reg[i].reg);
3352
3353 /* Configure all our dma channels to default settings */
3354 for (i = 0; i < base->num_phy_chans; i++) {
3355
3356 activeo[i % 2] = activeo[i % 2] << 2;
3357
3358 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
3359 == D40_ALLOC_PHY) {
3360 activeo[i % 2] |= 3;
3361 continue;
3362 }
3363
3364 /* Enable interrupt # */
3365 pcmis = (pcmis << 1) | 1;
3366
3367 /* Clear interrupt # */
3368 pcicr = (pcicr << 1) | 1;
3369
3370 /* Set channel to physical mode */
3371 prmseo[i % 2] = prmseo[i % 2] << 2;
3372 prmseo[i % 2] |= 1;
3373
3374 }
3375
3376 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
3377 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
3378 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
3379 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
3380
3381 /* Write which interrupt to enable */
3382 writel(pcmis, base->virtbase + base->gen_dmac.interrupt_en);
3383
3384 /* Write which interrupt to clear */
3385 writel(pcicr, base->virtbase + base->gen_dmac.interrupt_clear);
3386
3387 /* These are __initdata and cannot be accessed after init */
3388 base->gen_dmac.init_reg = NULL;
3389 base->gen_dmac.init_reg_size = 0;
3390 }
3391
3392 static int __init d40_lcla_allocate(struct d40_base *base)
3393 {
3394 struct d40_lcla_pool *pool = &base->lcla_pool;
3395 unsigned long *page_list;
3396 int i, j;
3397 int ret = 0;
3398
3399 /*
3400 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3401 * To full fill this hardware requirement without wasting 256 kb
3402 * we allocate pages until we get an aligned one.
3403 */
3404 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
3405 GFP_KERNEL);
3406
3407 if (!page_list) {
3408 ret = -ENOMEM;
3409 goto failure;
3410 }
3411
3412 /* Calculating how many pages that are required */
3413 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
3414
3415 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
3416 page_list[i] = __get_free_pages(GFP_KERNEL,
3417 base->lcla_pool.pages);
3418 if (!page_list[i]) {
3419
3420 d40_err(base->dev, "Failed to allocate %d pages.\n",
3421 base->lcla_pool.pages);
3422
3423 for (j = 0; j < i; j++)
3424 free_pages(page_list[j], base->lcla_pool.pages);
3425 goto failure;
3426 }
3427
3428 if ((virt_to_phys((void *)page_list[i]) &
3429 (LCLA_ALIGNMENT - 1)) == 0)
3430 break;
3431 }
3432
3433 for (j = 0; j < i; j++)
3434 free_pages(page_list[j], base->lcla_pool.pages);
3435
3436 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
3437 base->lcla_pool.base = (void *)page_list[i];
3438 } else {
3439 /*
3440 * After many attempts and no succees with finding the correct
3441 * alignment, try with allocating a big buffer.
3442 */
3443 dev_warn(base->dev,
3444 "[%s] Failed to get %d pages @ 18 bit align.\n",
3445 __func__, base->lcla_pool.pages);
3446 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
3447 base->num_phy_chans +
3448 LCLA_ALIGNMENT,
3449 GFP_KERNEL);
3450 if (!base->lcla_pool.base_unaligned) {
3451 ret = -ENOMEM;
3452 goto failure;
3453 }
3454
3455 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
3456 LCLA_ALIGNMENT);
3457 }
3458
3459 pool->dma_addr = dma_map_single(base->dev, pool->base,
3460 SZ_1K * base->num_phy_chans,
3461 DMA_TO_DEVICE);
3462 if (dma_mapping_error(base->dev, pool->dma_addr)) {
3463 pool->dma_addr = 0;
3464 ret = -ENOMEM;
3465 goto failure;
3466 }
3467
3468 writel(virt_to_phys(base->lcla_pool.base),
3469 base->virtbase + D40_DREG_LCLA);
3470 failure:
3471 kfree(page_list);
3472 return ret;
3473 }
3474
3475 static int __init d40_probe(struct platform_device *pdev)
3476 {
3477 int err;
3478 int ret = -ENOENT;
3479 struct d40_base *base;
3480 struct resource *res = NULL;
3481 int num_reserved_chans;
3482 u32 val;
3483
3484 base = d40_hw_detect_init(pdev);
3485
3486 if (!base)
3487 goto failure;
3488
3489 num_reserved_chans = d40_phy_res_init(base);
3490
3491 platform_set_drvdata(pdev, base);
3492
3493 spin_lock_init(&base->interrupt_lock);
3494 spin_lock_init(&base->execmd_lock);
3495
3496 /* Get IO for logical channel parameter address */
3497 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
3498 if (!res) {
3499 ret = -ENOENT;
3500 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
3501 goto failure;
3502 }
3503 base->lcpa_size = resource_size(res);
3504 base->phy_lcpa = res->start;
3505
3506 if (request_mem_region(res->start, resource_size(res),
3507 D40_NAME " I/O lcpa") == NULL) {
3508 ret = -EBUSY;
3509 d40_err(&pdev->dev,
3510 "Failed to request LCPA region 0x%x-0x%x\n",
3511 res->start, res->end);
3512 goto failure;
3513 }
3514
3515 /* We make use of ESRAM memory for this. */
3516 val = readl(base->virtbase + D40_DREG_LCPA);
3517 if (res->start != val && val != 0) {
3518 dev_warn(&pdev->dev,
3519 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
3520 __func__, val, res->start);
3521 } else
3522 writel(res->start, base->virtbase + D40_DREG_LCPA);
3523
3524 base->lcpa_base = ioremap(res->start, resource_size(res));
3525 if (!base->lcpa_base) {
3526 ret = -ENOMEM;
3527 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
3528 goto failure;
3529 }
3530 /* If lcla has to be located in ESRAM we don't need to allocate */
3531 if (base->plat_data->use_esram_lcla) {
3532 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
3533 "lcla_esram");
3534 if (!res) {
3535 ret = -ENOENT;
3536 d40_err(&pdev->dev,
3537 "No \"lcla_esram\" memory resource\n");
3538 goto failure;
3539 }
3540 base->lcla_pool.base = ioremap(res->start,
3541 resource_size(res));
3542 if (!base->lcla_pool.base) {
3543 ret = -ENOMEM;
3544 d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
3545 goto failure;
3546 }
3547 writel(res->start, base->virtbase + D40_DREG_LCLA);
3548
3549 } else {
3550 ret = d40_lcla_allocate(base);
3551 if (ret) {
3552 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
3553 goto failure;
3554 }
3555 }
3556
3557 spin_lock_init(&base->lcla_pool.lock);
3558
3559 base->irq = platform_get_irq(pdev, 0);
3560
3561 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
3562 if (ret) {
3563 d40_err(&pdev->dev, "No IRQ defined\n");
3564 goto failure;
3565 }
3566
3567 pm_runtime_irq_safe(base->dev);
3568 pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);
3569 pm_runtime_use_autosuspend(base->dev);
3570 pm_runtime_enable(base->dev);
3571 pm_runtime_resume(base->dev);
3572
3573 if (base->plat_data->use_esram_lcla) {
3574
3575 base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");
3576 if (IS_ERR(base->lcpa_regulator)) {
3577 d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
3578 base->lcpa_regulator = NULL;
3579 goto failure;
3580 }
3581
3582 ret = regulator_enable(base->lcpa_regulator);
3583 if (ret) {
3584 d40_err(&pdev->dev,
3585 "Failed to enable lcpa_regulator\n");
3586 regulator_put(base->lcpa_regulator);
3587 base->lcpa_regulator = NULL;
3588 goto failure;
3589 }
3590 }
3591
3592 base->initialized = true;
3593 err = d40_dmaengine_init(base, num_reserved_chans);
3594 if (err)
3595 goto failure;
3596
3597 base->dev->dma_parms = &base->dma_parms;
3598 err = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE);
3599 if (err) {
3600 d40_err(&pdev->dev, "Failed to set dma max seg size\n");
3601 goto failure;
3602 }
3603
3604 d40_hw_init(base);
3605
3606 dev_info(base->dev, "initialized\n");
3607 return 0;
3608
3609 failure:
3610 if (base) {
3611 if (base->desc_slab)
3612 kmem_cache_destroy(base->desc_slab);
3613 if (base->virtbase)
3614 iounmap(base->virtbase);
3615
3616 if (base->lcla_pool.base && base->plat_data->use_esram_lcla) {
3617 iounmap(base->lcla_pool.base);
3618 base->lcla_pool.base = NULL;
3619 }
3620
3621 if (base->lcla_pool.dma_addr)
3622 dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
3623 SZ_1K * base->num_phy_chans,
3624 DMA_TO_DEVICE);
3625
3626 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3627 free_pages((unsigned long)base->lcla_pool.base,
3628 base->lcla_pool.pages);
3629
3630 kfree(base->lcla_pool.base_unaligned);
3631
3632 if (base->phy_lcpa)
3633 release_mem_region(base->phy_lcpa,
3634 base->lcpa_size);
3635 if (base->phy_start)
3636 release_mem_region(base->phy_start,
3637 base->phy_size);
3638 if (base->clk) {
3639 clk_disable_unprepare(base->clk);
3640 clk_put(base->clk);
3641 }
3642
3643 if (base->lcpa_regulator) {
3644 regulator_disable(base->lcpa_regulator);
3645 regulator_put(base->lcpa_regulator);
3646 }
3647
3648 kfree(base->lcla_pool.alloc_map);
3649 kfree(base->lookup_log_chans);
3650 kfree(base->lookup_phy_chans);
3651 kfree(base->phy_res);
3652 kfree(base);
3653 }
3654
3655 d40_err(&pdev->dev, "probe failed\n");
3656 return ret;
3657 }
3658
3659 static struct platform_driver d40_driver = {
3660 .driver = {
3661 .owner = THIS_MODULE,
3662 .name = D40_NAME,
3663 .pm = DMA40_PM_OPS,
3664 },
3665 };
3666
3667 static int __init stedma40_init(void)
3668 {
3669 return platform_driver_probe(&d40_driver, d40_probe);
3670 }
3671 subsys_initcall(stedma40_init);
Linux kernel - Deliverables
This page took 0.165796 seconds and 5 git commands to generate.