projects / deliverable / linux.git / blob
? search:


a2c5a60bc2e204d0047363cbbc491ab198b2e559
[deliverable/linux.git] / drivers / dma / dw_dmac.c
1 /*
2 * Core driver for the Synopsys DesignWare DMA Controller
3 *
4 * Copyright (C) 2007-2008 Atmel Corporation
5 * Copyright (C) 2010-2011 ST Microelectronics
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12 #include <linux/bitops.h>
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/dmaengine.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/init.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/of.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/slab.h>
25
26 #include "dw_dmac_regs.h"
27 #include "dmaengine.h"
28
29 /*
30 * This supports the Synopsys "DesignWare AHB Central DMA Controller",
31 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
32 * of which use ARM any more). See the "Databook" from Synopsys for
33 * information beyond what licensees probably provide.
34 *
35 * The driver has currently been tested only with the Atmel AT32AP7000,
36 * which does not support descriptor writeback.
37 */
38
39 static inline unsigned int dwc_get_dms(struct dw_dma_slave *slave)
40 {
41 return slave ? slave->dst_master : 0;
42 }
43
44 static inline unsigned int dwc_get_sms(struct dw_dma_slave *slave)
45 {
46 return slave ? slave->src_master : 1;
47 }
48
49 #define DWC_DEFAULT_CTLLO(_chan) ({ \
50 struct dw_dma_slave *__slave = (_chan->private); \
51 struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan); \
52 struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
53 int _dms = dwc_get_dms(__slave); \
54 int _sms = dwc_get_sms(__slave); \
55 u8 _smsize = __slave ? _sconfig->src_maxburst : \
56 DW_DMA_MSIZE_16; \
57 u8 _dmsize = __slave ? _sconfig->dst_maxburst : \
58 DW_DMA_MSIZE_16; \
59 \
60 (DWC_CTLL_DST_MSIZE(_dmsize) \
61 | DWC_CTLL_SRC_MSIZE(_smsize) \
62 | DWC_CTLL_LLP_D_EN \
63 | DWC_CTLL_LLP_S_EN \
64 | DWC_CTLL_DMS(_dms) \
65 | DWC_CTLL_SMS(_sms)); \
66 })
67
68 /*
69 * Number of descriptors to allocate for each channel. This should be
70 * made configurable somehow; preferably, the clients (at least the
71 * ones using slave transfers) should be able to give us a hint.
72 */
73 #define NR_DESCS_PER_CHANNEL 64
74
75 /*----------------------------------------------------------------------*/
76
77 /*
78 * Because we're not relying on writeback from the controller (it may not
79 * even be configured into the core!) we don't need to use dma_pool. These
80 * descriptors -- and associated data -- are cacheable. We do need to make
81 * sure their dcache entries are written back before handing them off to
82 * the controller, though.
83 */
84
85 static struct device *chan2dev(struct dma_chan *chan)
86 {
87 return &chan->dev->device;
88 }
89 static struct device *chan2parent(struct dma_chan *chan)
90 {
91 return chan->dev->device.parent;
92 }
93
94 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
95 {
96 return to_dw_desc(dwc->active_list.next);
97 }
98
99 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
100 {
101 struct dw_desc *desc, *_desc;
102 struct dw_desc *ret = NULL;
103 unsigned int i = 0;
104 unsigned long flags;
105
106 spin_lock_irqsave(&dwc->lock, flags);
107 list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
108 i++;
109 if (async_tx_test_ack(&desc->txd)) {
110 list_del(&desc->desc_node);
111 ret = desc;
112 break;
113 }
114 dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
115 }
116 spin_unlock_irqrestore(&dwc->lock, flags);
117
118 dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
119
120 return ret;
121 }
122
123 static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
124 {
125 struct dw_desc *child;
126
127 list_for_each_entry(child, &desc->tx_list, desc_node)
128 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
129 child->txd.phys, sizeof(child->lli),
130 DMA_TO_DEVICE);
131 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
132 desc->txd.phys, sizeof(desc->lli),
133 DMA_TO_DEVICE);
134 }
135
136 /*
137 * Move a descriptor, including any children, to the free list.
138 * `desc' must not be on any lists.
139 */
140 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
141 {
142 unsigned long flags;
143
144 if (desc) {
145 struct dw_desc *child;
146
147 dwc_sync_desc_for_cpu(dwc, desc);
148
149 spin_lock_irqsave(&dwc->lock, flags);
150 list_for_each_entry(child, &desc->tx_list, desc_node)
151 dev_vdbg(chan2dev(&dwc->chan),
152 "moving child desc %p to freelist\n",
153 child);
154 list_splice_init(&desc->tx_list, &dwc->free_list);
155 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
156 list_add(&desc->desc_node, &dwc->free_list);
157 spin_unlock_irqrestore(&dwc->lock, flags);
158 }
159 }
160
161 static void dwc_initialize(struct dw_dma_chan *dwc)
162 {
163 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
164 struct dw_dma_slave *dws = dwc->chan.private;
165 u32 cfghi = DWC_CFGH_FIFO_MODE;
166 u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
167
168 if (dwc->initialized == true)
169 return;
170
171 if (dws) {
172 /*
173 * We need controller-specific data to set up slave
174 * transfers.
175 */
176 BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
177
178 cfghi = dws->cfg_hi;
179 cfglo |= dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
180 } else {
181 if (dwc->direction == DMA_MEM_TO_DEV)
182 cfghi = DWC_CFGH_DST_PER(dwc->dma_sconfig.slave_id);
183 else if (dwc->direction == DMA_DEV_TO_MEM)
184 cfghi = DWC_CFGH_SRC_PER(dwc->dma_sconfig.slave_id);
185 }
186
187 channel_writel(dwc, CFG_LO, cfglo);
188 channel_writel(dwc, CFG_HI, cfghi);
189
190 /* Enable interrupts */
191 channel_set_bit(dw, MASK.XFER, dwc->mask);
192 channel_set_bit(dw, MASK.ERROR, dwc->mask);
193
194 dwc->initialized = true;
195 }
196
197 /*----------------------------------------------------------------------*/
198
199 static inline unsigned int dwc_fast_fls(unsigned long long v)
200 {
201 /*
202 * We can be a lot more clever here, but this should take care
203 * of the most common optimization.
204 */
205 if (!(v & 7))
206 return 3;
207 else if (!(v & 3))
208 return 2;
209 else if (!(v & 1))
210 return 1;
211 return 0;
212 }
213
214 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
215 {
216 dev_err(chan2dev(&dwc->chan),
217 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
218 channel_readl(dwc, SAR),
219 channel_readl(dwc, DAR),
220 channel_readl(dwc, LLP),
221 channel_readl(dwc, CTL_HI),
222 channel_readl(dwc, CTL_LO));
223 }
224
225 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
226 {
227 channel_clear_bit(dw, CH_EN, dwc->mask);
228 while (dma_readl(dw, CH_EN) & dwc->mask)
229 cpu_relax();
230 }
231
232 /*----------------------------------------------------------------------*/
233
234 /* Perform single block transfer */
235 static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
236 struct dw_desc *desc)
237 {
238 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
239 u32 ctllo;
240
241 /* Software emulation of LLP mode relies on interrupts to continue
242 * multi block transfer. */
243 ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN;
244
245 channel_writel(dwc, SAR, desc->lli.sar);
246 channel_writel(dwc, DAR, desc->lli.dar);
247 channel_writel(dwc, CTL_LO, ctllo);
248 channel_writel(dwc, CTL_HI, desc->lli.ctlhi);
249 channel_set_bit(dw, CH_EN, dwc->mask);
250
251 /* Move pointer to next descriptor */
252 dwc->tx_node_active = dwc->tx_node_active->next;
253 }
254
255 /* Called with dwc->lock held and bh disabled */
256 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
257 {
258 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
259 unsigned long was_soft_llp;
260
261 /* ASSERT: channel is idle */
262 if (dma_readl(dw, CH_EN) & dwc->mask) {
263 dev_err(chan2dev(&dwc->chan),
264 "BUG: Attempted to start non-idle channel\n");
265 dwc_dump_chan_regs(dwc);
266
267 /* The tasklet will hopefully advance the queue... */
268 return;
269 }
270
271 if (dwc->nollp) {
272 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
273 &dwc->flags);
274 if (was_soft_llp) {
275 dev_err(chan2dev(&dwc->chan),
276 "BUG: Attempted to start new LLP transfer "
277 "inside ongoing one\n");
278 return;
279 }
280
281 dwc_initialize(dwc);
282
283 dwc->tx_list = &first->tx_list;
284 dwc->tx_node_active = &first->tx_list;
285
286 dwc_do_single_block(dwc, first);
287
288 return;
289 }
290
291 dwc_initialize(dwc);
292
293 channel_writel(dwc, LLP, first->txd.phys);
294 channel_writel(dwc, CTL_LO,
295 DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
296 channel_writel(dwc, CTL_HI, 0);
297 channel_set_bit(dw, CH_EN, dwc->mask);
298 }
299
300 /*----------------------------------------------------------------------*/
301
302 static void
303 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
304 bool callback_required)
305 {
306 dma_async_tx_callback callback = NULL;
307 void *param = NULL;
308 struct dma_async_tx_descriptor *txd = &desc->txd;
309 struct dw_desc *child;
310 unsigned long flags;
311
312 dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
313
314 spin_lock_irqsave(&dwc->lock, flags);
315 dma_cookie_complete(txd);
316 if (callback_required) {
317 callback = txd->callback;
318 param = txd->callback_param;
319 }
320
321 dwc_sync_desc_for_cpu(dwc, desc);
322
323 /* async_tx_ack */
324 list_for_each_entry(child, &desc->tx_list, desc_node)
325 async_tx_ack(&child->txd);
326 async_tx_ack(&desc->txd);
327
328 list_splice_init(&desc->tx_list, &dwc->free_list);
329 list_move(&desc->desc_node, &dwc->free_list);
330
331 if (!dwc->chan.private) {
332 struct device *parent = chan2parent(&dwc->chan);
333 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
334 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
335 dma_unmap_single(parent, desc->lli.dar,
336 desc->len, DMA_FROM_DEVICE);
337 else
338 dma_unmap_page(parent, desc->lli.dar,
339 desc->len, DMA_FROM_DEVICE);
340 }
341 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
342 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
343 dma_unmap_single(parent, desc->lli.sar,
344 desc->len, DMA_TO_DEVICE);
345 else
346 dma_unmap_page(parent, desc->lli.sar,
347 desc->len, DMA_TO_DEVICE);
348 }
349 }
350
351 spin_unlock_irqrestore(&dwc->lock, flags);
352
353 if (callback)
354 callback(param);
355 }
356
357 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
358 {
359 struct dw_desc *desc, *_desc;
360 LIST_HEAD(list);
361 unsigned long flags;
362
363 spin_lock_irqsave(&dwc->lock, flags);
364 if (dma_readl(dw, CH_EN) & dwc->mask) {
365 dev_err(chan2dev(&dwc->chan),
366 "BUG: XFER bit set, but channel not idle!\n");
367
368 /* Try to continue after resetting the channel... */
369 dwc_chan_disable(dw, dwc);
370 }
371
372 /*
373 * Submit queued descriptors ASAP, i.e. before we go through
374 * the completed ones.
375 */
376 list_splice_init(&dwc->active_list, &list);
377 if (!list_empty(&dwc->queue)) {
378 list_move(dwc->queue.next, &dwc->active_list);
379 dwc_dostart(dwc, dwc_first_active(dwc));
380 }
381
382 spin_unlock_irqrestore(&dwc->lock, flags);
383
384 list_for_each_entry_safe(desc, _desc, &list, desc_node)
385 dwc_descriptor_complete(dwc, desc, true);
386 }
387
388 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
389 {
390 dma_addr_t llp;
391 struct dw_desc *desc, *_desc;
392 struct dw_desc *child;
393 u32 status_xfer;
394 unsigned long flags;
395
396 spin_lock_irqsave(&dwc->lock, flags);
397 llp = channel_readl(dwc, LLP);
398 status_xfer = dma_readl(dw, RAW.XFER);
399
400 if (status_xfer & dwc->mask) {
401 /* Everything we've submitted is done */
402 dma_writel(dw, CLEAR.XFER, dwc->mask);
403 spin_unlock_irqrestore(&dwc->lock, flags);
404
405 dwc_complete_all(dw, dwc);
406 return;
407 }
408
409 if (list_empty(&dwc->active_list)) {
410 spin_unlock_irqrestore(&dwc->lock, flags);
411 return;
412 }
413
414 dev_vdbg(chan2dev(&dwc->chan), "%s: llp=0x%llx\n", __func__,
415 (unsigned long long)llp);
416
417 list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
418 /* check first descriptors addr */
419 if (desc->txd.phys == llp) {
420 spin_unlock_irqrestore(&dwc->lock, flags);
421 return;
422 }
423
424 /* check first descriptors llp */
425 if (desc->lli.llp == llp) {
426 /* This one is currently in progress */
427 spin_unlock_irqrestore(&dwc->lock, flags);
428 return;
429 }
430
431 list_for_each_entry(child, &desc->tx_list, desc_node)
432 if (child->lli.llp == llp) {
433 /* Currently in progress */
434 spin_unlock_irqrestore(&dwc->lock, flags);
435 return;
436 }
437
438 /*
439 * No descriptors so far seem to be in progress, i.e.
440 * this one must be done.
441 */
442 spin_unlock_irqrestore(&dwc->lock, flags);
443 dwc_descriptor_complete(dwc, desc, true);
444 spin_lock_irqsave(&dwc->lock, flags);
445 }
446
447 dev_err(chan2dev(&dwc->chan),
448 "BUG: All descriptors done, but channel not idle!\n");
449
450 /* Try to continue after resetting the channel... */
451 dwc_chan_disable(dw, dwc);
452
453 if (!list_empty(&dwc->queue)) {
454 list_move(dwc->queue.next, &dwc->active_list);
455 dwc_dostart(dwc, dwc_first_active(dwc));
456 }
457 spin_unlock_irqrestore(&dwc->lock, flags);
458 }
459
460 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
461 {
462 dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
463 lli->sar, lli->dar, lli->llp, lli->ctlhi, lli->ctllo);
464 }
465
466 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
467 {
468 struct dw_desc *bad_desc;
469 struct dw_desc *child;
470 unsigned long flags;
471
472 dwc_scan_descriptors(dw, dwc);
473
474 spin_lock_irqsave(&dwc->lock, flags);
475
476 /*
477 * The descriptor currently at the head of the active list is
478 * borked. Since we don't have any way to report errors, we'll
479 * just have to scream loudly and try to carry on.
480 */
481 bad_desc = dwc_first_active(dwc);
482 list_del_init(&bad_desc->desc_node);
483 list_move(dwc->queue.next, dwc->active_list.prev);
484
485 /* Clear the error flag and try to restart the controller */
486 dma_writel(dw, CLEAR.ERROR, dwc->mask);
487 if (!list_empty(&dwc->active_list))
488 dwc_dostart(dwc, dwc_first_active(dwc));
489
490 /*
491 * WARN may seem harsh, but since this only happens
492 * when someone submits a bad physical address in a
493 * descriptor, we should consider ourselves lucky that the
494 * controller flagged an error instead of scribbling over
495 * random memory locations.
496 */
497 dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
498 " cookie: %d\n", bad_desc->txd.cookie);
499 dwc_dump_lli(dwc, &bad_desc->lli);
500 list_for_each_entry(child, &bad_desc->tx_list, desc_node)
501 dwc_dump_lli(dwc, &child->lli);
502
503 spin_unlock_irqrestore(&dwc->lock, flags);
504
505 /* Pretend the descriptor completed successfully */
506 dwc_descriptor_complete(dwc, bad_desc, true);
507 }
508
509 /* --------------------- Cyclic DMA API extensions -------------------- */
510
511 inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
512 {
513 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
514 return channel_readl(dwc, SAR);
515 }
516 EXPORT_SYMBOL(dw_dma_get_src_addr);
517
518 inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
519 {
520 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
521 return channel_readl(dwc, DAR);
522 }
523 EXPORT_SYMBOL(dw_dma_get_dst_addr);
524
525 /* called with dwc->lock held and all DMAC interrupts disabled */
526 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
527 u32 status_err, u32 status_xfer)
528 {
529 unsigned long flags;
530
531 if (dwc->mask) {
532 void (*callback)(void *param);
533 void *callback_param;
534
535 dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
536 channel_readl(dwc, LLP));
537
538 callback = dwc->cdesc->period_callback;
539 callback_param = dwc->cdesc->period_callback_param;
540
541 if (callback)
542 callback(callback_param);
543 }
544
545 /*
546 * Error and transfer complete are highly unlikely, and will most
547 * likely be due to a configuration error by the user.
548 */
549 if (unlikely(status_err & dwc->mask) ||
550 unlikely(status_xfer & dwc->mask)) {
551 int i;
552
553 dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
554 "interrupt, stopping DMA transfer\n",
555 status_xfer ? "xfer" : "error");
556
557 spin_lock_irqsave(&dwc->lock, flags);
558
559 dwc_dump_chan_regs(dwc);
560
561 dwc_chan_disable(dw, dwc);
562
563 /* make sure DMA does not restart by loading a new list */
564 channel_writel(dwc, LLP, 0);
565 channel_writel(dwc, CTL_LO, 0);
566 channel_writel(dwc, CTL_HI, 0);
567
568 dma_writel(dw, CLEAR.ERROR, dwc->mask);
569 dma_writel(dw, CLEAR.XFER, dwc->mask);
570
571 for (i = 0; i < dwc->cdesc->periods; i++)
572 dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
573
574 spin_unlock_irqrestore(&dwc->lock, flags);
575 }
576 }
577
578 /* ------------------------------------------------------------------------- */
579
580 static void dw_dma_tasklet(unsigned long data)
581 {
582 struct dw_dma *dw = (struct dw_dma *)data;
583 struct dw_dma_chan *dwc;
584 u32 status_xfer;
585 u32 status_err;
586 int i;
587
588 status_xfer = dma_readl(dw, RAW.XFER);
589 status_err = dma_readl(dw, RAW.ERROR);
590
591 dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
592
593 for (i = 0; i < dw->dma.chancnt; i++) {
594 dwc = &dw->chan[i];
595 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
596 dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
597 else if (status_err & (1 << i))
598 dwc_handle_error(dw, dwc);
599 else if (status_xfer & (1 << i)) {
600 unsigned long flags;
601
602 spin_lock_irqsave(&dwc->lock, flags);
603 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
604 if (dwc->tx_node_active != dwc->tx_list) {
605 struct dw_desc *desc =
606 to_dw_desc(dwc->tx_node_active);
607
608 dma_writel(dw, CLEAR.XFER, dwc->mask);
609
610 dwc_do_single_block(dwc, desc);
611
612 spin_unlock_irqrestore(&dwc->lock, flags);
613 continue;
614 }
615 /* we are done here */
616 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
617 }
618 spin_unlock_irqrestore(&dwc->lock, flags);
619
620 dwc_scan_descriptors(dw, dwc);
621 }
622 }
623
624 /*
625 * Re-enable interrupts.
626 */
627 channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
628 channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
629 }
630
631 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
632 {
633 struct dw_dma *dw = dev_id;
634 u32 status;
635
636 dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__,
637 dma_readl(dw, STATUS_INT));
638
639 /*
640 * Just disable the interrupts. We'll turn them back on in the
641 * softirq handler.
642 */
643 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
644 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
645
646 status = dma_readl(dw, STATUS_INT);
647 if (status) {
648 dev_err(dw->dma.dev,
649 "BUG: Unexpected interrupts pending: 0x%x\n",
650 status);
651
652 /* Try to recover */
653 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
654 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
655 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
656 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
657 }
658
659 tasklet_schedule(&dw->tasklet);
660
661 return IRQ_HANDLED;
662 }
663
664 /*----------------------------------------------------------------------*/
665
666 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
667 {
668 struct dw_desc *desc = txd_to_dw_desc(tx);
669 struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
670 dma_cookie_t cookie;
671 unsigned long flags;
672
673 spin_lock_irqsave(&dwc->lock, flags);
674 cookie = dma_cookie_assign(tx);
675
676 /*
677 * REVISIT: We should attempt to chain as many descriptors as
678 * possible, perhaps even appending to those already submitted
679 * for DMA. But this is hard to do in a race-free manner.
680 */
681 if (list_empty(&dwc->active_list)) {
682 dev_vdbg(chan2dev(tx->chan), "%s: started %u\n", __func__,
683 desc->txd.cookie);
684 list_add_tail(&desc->desc_node, &dwc->active_list);
685 dwc_dostart(dwc, dwc_first_active(dwc));
686 } else {
687 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__,
688 desc->txd.cookie);
689
690 list_add_tail(&desc->desc_node, &dwc->queue);
691 }
692
693 spin_unlock_irqrestore(&dwc->lock, flags);
694
695 return cookie;
696 }
697
698 static struct dma_async_tx_descriptor *
699 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
700 size_t len, unsigned long flags)
701 {
702 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
703 struct dw_dma_slave *dws = chan->private;
704 struct dw_desc *desc;
705 struct dw_desc *first;
706 struct dw_desc *prev;
707 size_t xfer_count;
708 size_t offset;
709 unsigned int src_width;
710 unsigned int dst_width;
711 unsigned int data_width;
712 u32 ctllo;
713
714 dev_vdbg(chan2dev(chan),
715 "%s: d0x%llx s0x%llx l0x%zx f0x%lx\n", __func__,
716 (unsigned long long)dest, (unsigned long long)src,
717 len, flags);
718
719 if (unlikely(!len)) {
720 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
721 return NULL;
722 }
723
724 dwc->direction = DMA_MEM_TO_MEM;
725
726 data_width = min_t(unsigned int, dwc->dw->data_width[dwc_get_sms(dws)],
727 dwc->dw->data_width[dwc_get_dms(dws)]);
728
729 src_width = dst_width = min_t(unsigned int, data_width,
730 dwc_fast_fls(src | dest | len));
731
732 ctllo = DWC_DEFAULT_CTLLO(chan)
733 | DWC_CTLL_DST_WIDTH(dst_width)
734 | DWC_CTLL_SRC_WIDTH(src_width)
735 | DWC_CTLL_DST_INC
736 | DWC_CTLL_SRC_INC
737 | DWC_CTLL_FC_M2M;
738 prev = first = NULL;
739
740 for (offset = 0; offset < len; offset += xfer_count << src_width) {
741 xfer_count = min_t(size_t, (len - offset) >> src_width,
742 dwc->block_size);
743
744 desc = dwc_desc_get(dwc);
745 if (!desc)
746 goto err_desc_get;
747
748 desc->lli.sar = src + offset;
749 desc->lli.dar = dest + offset;
750 desc->lli.ctllo = ctllo;
751 desc->lli.ctlhi = xfer_count;
752
753 if (!first) {
754 first = desc;
755 } else {
756 prev->lli.llp = desc->txd.phys;
757 dma_sync_single_for_device(chan2parent(chan),
758 prev->txd.phys, sizeof(prev->lli),
759 DMA_TO_DEVICE);
760 list_add_tail(&desc->desc_node,
761 &first->tx_list);
762 }
763 prev = desc;
764 }
765
766
767 if (flags & DMA_PREP_INTERRUPT)
768 /* Trigger interrupt after last block */
769 prev->lli.ctllo |= DWC_CTLL_INT_EN;
770
771 prev->lli.llp = 0;
772 dma_sync_single_for_device(chan2parent(chan),
773 prev->txd.phys, sizeof(prev->lli),
774 DMA_TO_DEVICE);
775
776 first->txd.flags = flags;
777 first->len = len;
778
779 return &first->txd;
780
781 err_desc_get:
782 dwc_desc_put(dwc, first);
783 return NULL;
784 }
785
786 static struct dma_async_tx_descriptor *
787 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
788 unsigned int sg_len, enum dma_transfer_direction direction,
789 unsigned long flags, void *context)
790 {
791 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
792 struct dw_dma_slave *dws = chan->private;
793 struct dma_slave_config *sconfig = &dwc->dma_sconfig;
794 struct dw_desc *prev;
795 struct dw_desc *first;
796 u32 ctllo;
797 dma_addr_t reg;
798 unsigned int reg_width;
799 unsigned int mem_width;
800 unsigned int data_width;
801 unsigned int i;
802 struct scatterlist *sg;
803 size_t total_len = 0;
804
805 dev_vdbg(chan2dev(chan), "%s\n", __func__);
806
807 if (unlikely(!dws || !sg_len))
808 return NULL;
809
810 dwc->direction = direction;
811
812 prev = first = NULL;
813
814 switch (direction) {
815 case DMA_MEM_TO_DEV:
816 reg_width = __fls(sconfig->dst_addr_width);
817 reg = sconfig->dst_addr;
818 ctllo = (DWC_DEFAULT_CTLLO(chan)
819 | DWC_CTLL_DST_WIDTH(reg_width)
820 | DWC_CTLL_DST_FIX
821 | DWC_CTLL_SRC_INC);
822
823 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
824 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
825
826 data_width = dwc->dw->data_width[dwc_get_sms(dws)];
827
828 for_each_sg(sgl, sg, sg_len, i) {
829 struct dw_desc *desc;
830 u32 len, dlen, mem;
831
832 mem = sg_dma_address(sg);
833 len = sg_dma_len(sg);
834
835 mem_width = min_t(unsigned int,
836 data_width, dwc_fast_fls(mem | len));
837
838 slave_sg_todev_fill_desc:
839 desc = dwc_desc_get(dwc);
840 if (!desc) {
841 dev_err(chan2dev(chan),
842 "not enough descriptors available\n");
843 goto err_desc_get;
844 }
845
846 desc->lli.sar = mem;
847 desc->lli.dar = reg;
848 desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
849 if ((len >> mem_width) > dwc->block_size) {
850 dlen = dwc->block_size << mem_width;
851 mem += dlen;
852 len -= dlen;
853 } else {
854 dlen = len;
855 len = 0;
856 }
857
858 desc->lli.ctlhi = dlen >> mem_width;
859
860 if (!first) {
861 first = desc;
862 } else {
863 prev->lli.llp = desc->txd.phys;
864 dma_sync_single_for_device(chan2parent(chan),
865 prev->txd.phys,
866 sizeof(prev->lli),
867 DMA_TO_DEVICE);
868 list_add_tail(&desc->desc_node,
869 &first->tx_list);
870 }
871 prev = desc;
872 total_len += dlen;
873
874 if (len)
875 goto slave_sg_todev_fill_desc;
876 }
877 break;
878 case DMA_DEV_TO_MEM:
879 reg_width = __fls(sconfig->src_addr_width);
880 reg = sconfig->src_addr;
881 ctllo = (DWC_DEFAULT_CTLLO(chan)
882 | DWC_CTLL_SRC_WIDTH(reg_width)
883 | DWC_CTLL_DST_INC
884 | DWC_CTLL_SRC_FIX);
885
886 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
887 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
888
889 data_width = dwc->dw->data_width[dwc_get_dms(dws)];
890
891 for_each_sg(sgl, sg, sg_len, i) {
892 struct dw_desc *desc;
893 u32 len, dlen, mem;
894
895 mem = sg_dma_address(sg);
896 len = sg_dma_len(sg);
897
898 mem_width = min_t(unsigned int,
899 data_width, dwc_fast_fls(mem | len));
900
901 slave_sg_fromdev_fill_desc:
902 desc = dwc_desc_get(dwc);
903 if (!desc) {
904 dev_err(chan2dev(chan),
905 "not enough descriptors available\n");
906 goto err_desc_get;
907 }
908
909 desc->lli.sar = reg;
910 desc->lli.dar = mem;
911 desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
912 if ((len >> reg_width) > dwc->block_size) {
913 dlen = dwc->block_size << reg_width;
914 mem += dlen;
915 len -= dlen;
916 } else {
917 dlen = len;
918 len = 0;
919 }
920 desc->lli.ctlhi = dlen >> reg_width;
921
922 if (!first) {
923 first = desc;
924 } else {
925 prev->lli.llp = desc->txd.phys;
926 dma_sync_single_for_device(chan2parent(chan),
927 prev->txd.phys,
928 sizeof(prev->lli),
929 DMA_TO_DEVICE);
930 list_add_tail(&desc->desc_node,
931 &first->tx_list);
932 }
933 prev = desc;
934 total_len += dlen;
935
936 if (len)
937 goto slave_sg_fromdev_fill_desc;
938 }
939 break;
940 default:
941 return NULL;
942 }
943
944 if (flags & DMA_PREP_INTERRUPT)
945 /* Trigger interrupt after last block */
946 prev->lli.ctllo |= DWC_CTLL_INT_EN;
947
948 prev->lli.llp = 0;
949 dma_sync_single_for_device(chan2parent(chan),
950 prev->txd.phys, sizeof(prev->lli),
951 DMA_TO_DEVICE);
952
953 first->len = total_len;
954
955 return &first->txd;
956
957 err_desc_get:
958 dwc_desc_put(dwc, first);
959 return NULL;
960 }
961
962 /*
963 * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
964 * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
965 *
966 * NOTE: burst size 2 is not supported by controller.
967 *
968 * This can be done by finding least significant bit set: n & (n - 1)
969 */
970 static inline void convert_burst(u32 *maxburst)
971 {
972 if (*maxburst > 1)
973 *maxburst = fls(*maxburst) - 2;
974 else
975 *maxburst = 0;
976 }
977
978 static int
979 set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
980 {
981 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
982
983 /* Check if it is chan is configured for slave transfers */
984 if (!chan->private)
985 return -EINVAL;
986
987 memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
988 dwc->direction = sconfig->direction;
989
990 convert_burst(&dwc->dma_sconfig.src_maxburst);
991 convert_burst(&dwc->dma_sconfig.dst_maxburst);
992
993 return 0;
994 }
995
996 static inline void dwc_chan_pause(struct dw_dma_chan *dwc)
997 {
998 u32 cfglo = channel_readl(dwc, CFG_LO);
999
1000 channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
1001 while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY))
1002 cpu_relax();
1003
1004 dwc->paused = true;
1005 }
1006
1007 static inline void dwc_chan_resume(struct dw_dma_chan *dwc)
1008 {
1009 u32 cfglo = channel_readl(dwc, CFG_LO);
1010
1011 channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
1012
1013 dwc->paused = false;
1014 }
1015
1016 static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1017 unsigned long arg)
1018 {
1019 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1020 struct dw_dma *dw = to_dw_dma(chan->device);
1021 struct dw_desc *desc, *_desc;
1022 unsigned long flags;
1023 LIST_HEAD(list);
1024
1025 if (cmd == DMA_PAUSE) {
1026 spin_lock_irqsave(&dwc->lock, flags);
1027
1028 dwc_chan_pause(dwc);
1029
1030 spin_unlock_irqrestore(&dwc->lock, flags);
1031 } else if (cmd == DMA_RESUME) {
1032 if (!dwc->paused)
1033 return 0;
1034
1035 spin_lock_irqsave(&dwc->lock, flags);
1036
1037 dwc_chan_resume(dwc);
1038
1039 spin_unlock_irqrestore(&dwc->lock, flags);
1040 } else if (cmd == DMA_TERMINATE_ALL) {
1041 spin_lock_irqsave(&dwc->lock, flags);
1042
1043 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
1044
1045 dwc_chan_disable(dw, dwc);
1046
1047 dwc->paused = false;
1048
1049 /* active_list entries will end up before queued entries */
1050 list_splice_init(&dwc->queue, &list);
1051 list_splice_init(&dwc->active_list, &list);
1052
1053 spin_unlock_irqrestore(&dwc->lock, flags);
1054
1055 /* Flush all pending and queued descriptors */
1056 list_for_each_entry_safe(desc, _desc, &list, desc_node)
1057 dwc_descriptor_complete(dwc, desc, false);
1058 } else if (cmd == DMA_SLAVE_CONFIG) {
1059 return set_runtime_config(chan, (struct dma_slave_config *)arg);
1060 } else {
1061 return -ENXIO;
1062 }
1063
1064 return 0;
1065 }
1066
1067 static enum dma_status
1068 dwc_tx_status(struct dma_chan *chan,
1069 dma_cookie_t cookie,
1070 struct dma_tx_state *txstate)
1071 {
1072 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1073 enum dma_status ret;
1074
1075 ret = dma_cookie_status(chan, cookie, txstate);
1076 if (ret != DMA_SUCCESS) {
1077 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1078
1079 ret = dma_cookie_status(chan, cookie, txstate);
1080 }
1081
1082 if (ret != DMA_SUCCESS)
1083 dma_set_residue(txstate, dwc_first_active(dwc)->len);
1084
1085 if (dwc->paused)
1086 return DMA_PAUSED;
1087
1088 return ret;
1089 }
1090
1091 static void dwc_issue_pending(struct dma_chan *chan)
1092 {
1093 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1094
1095 if (!list_empty(&dwc->queue))
1096 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1097 }
1098
1099 static int dwc_alloc_chan_resources(struct dma_chan *chan)
1100 {
1101 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1102 struct dw_dma *dw = to_dw_dma(chan->device);
1103 struct dw_desc *desc;
1104 int i;
1105 unsigned long flags;
1106 int ret;
1107
1108 dev_vdbg(chan2dev(chan), "%s\n", __func__);
1109
1110 /* ASSERT: channel is idle */
1111 if (dma_readl(dw, CH_EN) & dwc->mask) {
1112 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
1113 return -EIO;
1114 }
1115
1116 dma_cookie_init(chan);
1117
1118 /*
1119 * NOTE: some controllers may have additional features that we
1120 * need to initialize here, like "scatter-gather" (which
1121 * doesn't mean what you think it means), and status writeback.
1122 */
1123
1124 spin_lock_irqsave(&dwc->lock, flags);
1125 i = dwc->descs_allocated;
1126 while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
1127 spin_unlock_irqrestore(&dwc->lock, flags);
1128
1129 desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
1130 if (!desc)
1131 goto err_desc_alloc;
1132
1133 INIT_LIST_HEAD(&desc->tx_list);
1134 dma_async_tx_descriptor_init(&desc->txd, chan);
1135 desc->txd.tx_submit = dwc_tx_submit;
1136 desc->txd.flags = DMA_CTRL_ACK;
1137 desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
1138 sizeof(desc->lli), DMA_TO_DEVICE);
1139 ret = dma_mapping_error(chan2parent(chan), desc->txd.phys);
1140 if (ret)
1141 goto err_desc_alloc;
1142
1143 dwc_desc_put(dwc, desc);
1144
1145 spin_lock_irqsave(&dwc->lock, flags);
1146 i = ++dwc->descs_allocated;
1147 }
1148
1149 spin_unlock_irqrestore(&dwc->lock, flags);
1150
1151 dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
1152
1153 return i;
1154
1155 err_desc_alloc:
1156 kfree(desc);
1157
1158 dev_info(chan2dev(chan), "only allocated %d descriptors\n", i);
1159
1160 return i;
1161 }
1162
1163 static void dwc_free_chan_resources(struct dma_chan *chan)
1164 {
1165 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1166 struct dw_dma *dw = to_dw_dma(chan->device);
1167 struct dw_desc *desc, *_desc;
1168 unsigned long flags;
1169 LIST_HEAD(list);
1170
1171 dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1172 dwc->descs_allocated);
1173
1174 /* ASSERT: channel is idle */
1175 BUG_ON(!list_empty(&dwc->active_list));
1176 BUG_ON(!list_empty(&dwc->queue));
1177 BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1178
1179 spin_lock_irqsave(&dwc->lock, flags);
1180 list_splice_init(&dwc->free_list, &list);
1181 dwc->descs_allocated = 0;
1182 dwc->initialized = false;
1183
1184 /* Disable interrupts */
1185 channel_clear_bit(dw, MASK.XFER, dwc->mask);
1186 channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1187
1188 spin_unlock_irqrestore(&dwc->lock, flags);
1189
1190 list_for_each_entry_safe(desc, _desc, &list, desc_node) {
1191 dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
1192 dma_unmap_single(chan2parent(chan), desc->txd.phys,
1193 sizeof(desc->lli), DMA_TO_DEVICE);
1194 kfree(desc);
1195 }
1196
1197 dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1198 }
1199
1200 bool dw_dma_generic_filter(struct dma_chan *chan, void *param)
1201 {
1202 struct dw_dma *dw = to_dw_dma(chan->device);
1203 static struct dw_dma *last_dw;
1204 static char *last_bus_id;
1205 int i = -1;
1206
1207 /*
1208 * dmaengine framework calls this routine for all channels of all dma
1209 * controller, until true is returned. If 'param' bus_id is not
1210 * registered with a dma controller (dw), then there is no need of
1211 * running below function for all channels of dw.
1212 *
1213 * This block of code does this by saving the parameters of last
1214 * failure. If dw and param are same, i.e. trying on same dw with
1215 * different channel, return false.
1216 */
1217 if ((last_dw == dw) && (last_bus_id == param))
1218 return false;
1219 /*
1220 * Return true:
1221 * - If dw_dma's platform data is not filled with slave info, then all
1222 * dma controllers are fine for transfer.
1223 * - Or if param is NULL
1224 */
1225 if (!dw->sd || !param)
1226 return true;
1227
1228 while (++i < dw->sd_count) {
1229 if (!strcmp(dw->sd[i].bus_id, param)) {
1230 chan->private = &dw->sd[i];
1231 last_dw = NULL;
1232 last_bus_id = NULL;
1233
1234 return true;
1235 }
1236 }
1237
1238 last_dw = dw;
1239 last_bus_id = param;
1240 return false;
1241 }
1242 EXPORT_SYMBOL(dw_dma_generic_filter);
1243
1244 /* --------------------- Cyclic DMA API extensions -------------------- */
1245
1246 /**
1247 * dw_dma_cyclic_start - start the cyclic DMA transfer
1248 * @chan: the DMA channel to start
1249 *
1250 * Must be called with soft interrupts disabled. Returns zero on success or
1251 * -errno on failure.
1252 */
1253 int dw_dma_cyclic_start(struct dma_chan *chan)
1254 {
1255 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1256 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1257 unsigned long flags;
1258
1259 if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
1260 dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
1261 return -ENODEV;
1262 }
1263
1264 spin_lock_irqsave(&dwc->lock, flags);
1265
1266 /* assert channel is idle */
1267 if (dma_readl(dw, CH_EN) & dwc->mask) {
1268 dev_err(chan2dev(&dwc->chan),
1269 "BUG: Attempted to start non-idle channel\n");
1270 dwc_dump_chan_regs(dwc);
1271 spin_unlock_irqrestore(&dwc->lock, flags);
1272 return -EBUSY;
1273 }
1274
1275 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1276 dma_writel(dw, CLEAR.XFER, dwc->mask);
1277
1278 /* setup DMAC channel registers */
1279 channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
1280 channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
1281 channel_writel(dwc, CTL_HI, 0);
1282
1283 channel_set_bit(dw, CH_EN, dwc->mask);
1284
1285 spin_unlock_irqrestore(&dwc->lock, flags);
1286
1287 return 0;
1288 }
1289 EXPORT_SYMBOL(dw_dma_cyclic_start);
1290
1291 /**
1292 * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1293 * @chan: the DMA channel to stop
1294 *
1295 * Must be called with soft interrupts disabled.
1296 */
1297 void dw_dma_cyclic_stop(struct dma_chan *chan)
1298 {
1299 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1300 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1301 unsigned long flags;
1302
1303 spin_lock_irqsave(&dwc->lock, flags);
1304
1305 dwc_chan_disable(dw, dwc);
1306
1307 spin_unlock_irqrestore(&dwc->lock, flags);
1308 }
1309 EXPORT_SYMBOL(dw_dma_cyclic_stop);
1310
1311 /**
1312 * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1313 * @chan: the DMA channel to prepare
1314 * @buf_addr: physical DMA address where the buffer starts
1315 * @buf_len: total number of bytes for the entire buffer
1316 * @period_len: number of bytes for each period
1317 * @direction: transfer direction, to or from device
1318 *
1319 * Must be called before trying to start the transfer. Returns a valid struct
1320 * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1321 */
1322 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
1323 dma_addr_t buf_addr, size_t buf_len, size_t period_len,
1324 enum dma_transfer_direction direction)
1325 {
1326 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1327 struct dma_slave_config *sconfig = &dwc->dma_sconfig;
1328 struct dw_cyclic_desc *cdesc;
1329 struct dw_cyclic_desc *retval = NULL;
1330 struct dw_desc *desc;
1331 struct dw_desc *last = NULL;
1332 unsigned long was_cyclic;
1333 unsigned int reg_width;
1334 unsigned int periods;
1335 unsigned int i;
1336 unsigned long flags;
1337
1338 spin_lock_irqsave(&dwc->lock, flags);
1339 if (dwc->nollp) {
1340 spin_unlock_irqrestore(&dwc->lock, flags);
1341 dev_dbg(chan2dev(&dwc->chan),
1342 "channel doesn't support LLP transfers\n");
1343 return ERR_PTR(-EINVAL);
1344 }
1345
1346 if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
1347 spin_unlock_irqrestore(&dwc->lock, flags);
1348 dev_dbg(chan2dev(&dwc->chan),
1349 "queue and/or active list are not empty\n");
1350 return ERR_PTR(-EBUSY);
1351 }
1352
1353 was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1354 spin_unlock_irqrestore(&dwc->lock, flags);
1355 if (was_cyclic) {
1356 dev_dbg(chan2dev(&dwc->chan),
1357 "channel already prepared for cyclic DMA\n");
1358 return ERR_PTR(-EBUSY);
1359 }
1360
1361 retval = ERR_PTR(-EINVAL);
1362
1363 if (unlikely(!is_slave_direction(direction)))
1364 goto out_err;
1365
1366 dwc->direction = direction;
1367
1368 if (direction == DMA_MEM_TO_DEV)
1369 reg_width = __ffs(sconfig->dst_addr_width);
1370 else
1371 reg_width = __ffs(sconfig->src_addr_width);
1372
1373 periods = buf_len / period_len;
1374
1375 /* Check for too big/unaligned periods and unaligned DMA buffer. */
1376 if (period_len > (dwc->block_size << reg_width))
1377 goto out_err;
1378 if (unlikely(period_len & ((1 << reg_width) - 1)))
1379 goto out_err;
1380 if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1381 goto out_err;
1382
1383 retval = ERR_PTR(-ENOMEM);
1384
1385 if (periods > NR_DESCS_PER_CHANNEL)
1386 goto out_err;
1387
1388 cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
1389 if (!cdesc)
1390 goto out_err;
1391
1392 cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
1393 if (!cdesc->desc)
1394 goto out_err_alloc;
1395
1396 for (i = 0; i < periods; i++) {
1397 desc = dwc_desc_get(dwc);
1398 if (!desc)
1399 goto out_err_desc_get;
1400
1401 switch (direction) {
1402 case DMA_MEM_TO_DEV:
1403 desc->lli.dar = sconfig->dst_addr;
1404 desc->lli.sar = buf_addr + (period_len * i);
1405 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1406 | DWC_CTLL_DST_WIDTH(reg_width)
1407 | DWC_CTLL_SRC_WIDTH(reg_width)
1408 | DWC_CTLL_DST_FIX
1409 | DWC_CTLL_SRC_INC
1410 | DWC_CTLL_INT_EN);
1411
1412 desc->lli.ctllo |= sconfig->device_fc ?
1413 DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
1414 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
1415
1416 break;
1417 case DMA_DEV_TO_MEM:
1418 desc->lli.dar = buf_addr + (period_len * i);
1419 desc->lli.sar = sconfig->src_addr;
1420 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1421 | DWC_CTLL_SRC_WIDTH(reg_width)
1422 | DWC_CTLL_DST_WIDTH(reg_width)
1423 | DWC_CTLL_DST_INC
1424 | DWC_CTLL_SRC_FIX
1425 | DWC_CTLL_INT_EN);
1426
1427 desc->lli.ctllo |= sconfig->device_fc ?
1428 DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
1429 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
1430
1431 break;
1432 default:
1433 break;
1434 }
1435
1436 desc->lli.ctlhi = (period_len >> reg_width);
1437 cdesc->desc[i] = desc;
1438
1439 if (last) {
1440 last->lli.llp = desc->txd.phys;
1441 dma_sync_single_for_device(chan2parent(chan),
1442 last->txd.phys, sizeof(last->lli),
1443 DMA_TO_DEVICE);
1444 }
1445
1446 last = desc;
1447 }
1448
1449 /* lets make a cyclic list */
1450 last->lli.llp = cdesc->desc[0]->txd.phys;
1451 dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
1452 sizeof(last->lli), DMA_TO_DEVICE);
1453
1454 dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu "
1455 "period %zu periods %d\n", (unsigned long long)buf_addr,
1456 buf_len, period_len, periods);
1457
1458 cdesc->periods = periods;
1459 dwc->cdesc = cdesc;
1460
1461 return cdesc;
1462
1463 out_err_desc_get:
1464 while (i--)
1465 dwc_desc_put(dwc, cdesc->desc[i]);
1466 out_err_alloc:
1467 kfree(cdesc);
1468 out_err:
1469 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1470 return (struct dw_cyclic_desc *)retval;
1471 }
1472 EXPORT_SYMBOL(dw_dma_cyclic_prep);
1473
1474 /**
1475 * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1476 * @chan: the DMA channel to free
1477 */
1478 void dw_dma_cyclic_free(struct dma_chan *chan)
1479 {
1480 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1481 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1482 struct dw_cyclic_desc *cdesc = dwc->cdesc;
1483 int i;
1484 unsigned long flags;
1485
1486 dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);
1487
1488 if (!cdesc)
1489 return;
1490
1491 spin_lock_irqsave(&dwc->lock, flags);
1492
1493 dwc_chan_disable(dw, dwc);
1494
1495 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1496 dma_writel(dw, CLEAR.XFER, dwc->mask);
1497
1498 spin_unlock_irqrestore(&dwc->lock, flags);
1499
1500 for (i = 0; i < cdesc->periods; i++)
1501 dwc_desc_put(dwc, cdesc->desc[i]);
1502
1503 kfree(cdesc->desc);
1504 kfree(cdesc);
1505
1506 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1507 }
1508 EXPORT_SYMBOL(dw_dma_cyclic_free);
1509
1510 /*----------------------------------------------------------------------*/
1511
1512 static void dw_dma_off(struct dw_dma *dw)
1513 {
1514 int i;
1515
1516 dma_writel(dw, CFG, 0);
1517
1518 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1519 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1520 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1521 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1522
1523 while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1524 cpu_relax();
1525
1526 for (i = 0; i < dw->dma.chancnt; i++)
1527 dw->chan[i].initialized = false;
1528 }
1529
1530 #ifdef CONFIG_OF
1531 static struct dw_dma_platform_data *
1532 dw_dma_parse_dt(struct platform_device *pdev)
1533 {
1534 struct device_node *sn, *cn, *np = pdev->dev.of_node;
1535 struct dw_dma_platform_data *pdata;
1536 struct dw_dma_slave *sd;
1537 u32 tmp, arr[4];
1538
1539 if (!np) {
1540 dev_err(&pdev->dev, "Missing DT data\n");
1541 return NULL;
1542 }
1543
1544 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1545 if (!pdata)
1546 return NULL;
1547
1548 if (of_property_read_u32(np, "nr_channels", &pdata->nr_channels))
1549 return NULL;
1550
1551 if (of_property_read_bool(np, "is_private"))
1552 pdata->is_private = true;
1553
1554 if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
1555 pdata->chan_allocation_order = (unsigned char)tmp;
1556
1557 if (!of_property_read_u32(np, "chan_priority", &tmp))
1558 pdata->chan_priority = tmp;
1559
1560 if (!of_property_read_u32(np, "block_size", &tmp))
1561 pdata->block_size = tmp;
1562
1563 if (!of_property_read_u32(np, "nr_masters", &tmp)) {
1564 if (tmp > 4)
1565 return NULL;
1566
1567 pdata->nr_masters = tmp;
1568 }
1569
1570 if (!of_property_read_u32_array(np, "data_width", arr,
1571 pdata->nr_masters))
1572 for (tmp = 0; tmp < pdata->nr_masters; tmp++)
1573 pdata->data_width[tmp] = arr[tmp];
1574
1575 /* parse slave data */
1576 sn = of_find_node_by_name(np, "slave_info");
1577 if (!sn)
1578 return pdata;
1579
1580 /* calculate number of slaves */
1581 tmp = of_get_child_count(sn);
1582 if (!tmp)
1583 return NULL;
1584
1585 sd = devm_kzalloc(&pdev->dev, sizeof(*sd) * tmp, GFP_KERNEL);
1586 if (!sd)
1587 return NULL;
1588
1589 pdata->sd = sd;
1590 pdata->sd_count = tmp;
1591
1592 for_each_child_of_node(sn, cn) {
1593 sd->dma_dev = &pdev->dev;
1594 of_property_read_string(cn, "bus_id", &sd->bus_id);
1595 of_property_read_u32(cn, "cfg_hi", &sd->cfg_hi);
1596 of_property_read_u32(cn, "cfg_lo", &sd->cfg_lo);
1597 if (!of_property_read_u32(cn, "src_master", &tmp))
1598 sd->src_master = tmp;
1599
1600 if (!of_property_read_u32(cn, "dst_master", &tmp))
1601 sd->dst_master = tmp;
1602 sd++;
1603 }
1604
1605 return pdata;
1606 }
1607 #else
1608 static inline struct dw_dma_platform_data *
1609 dw_dma_parse_dt(struct platform_device *pdev)
1610 {
1611 return NULL;
1612 }
1613 #endif
1614
1615 static int dw_probe(struct platform_device *pdev)
1616 {
1617 struct dw_dma_platform_data *pdata;
1618 struct resource *io;
1619 struct dw_dma *dw;
1620 size_t size;
1621 void __iomem *regs;
1622 bool autocfg;
1623 unsigned int dw_params;
1624 unsigned int nr_channels;
1625 unsigned int max_blk_size = 0;
1626 int irq;
1627 int err;
1628 int i;
1629
1630 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1631 if (!io)
1632 return -EINVAL;
1633
1634 irq = platform_get_irq(pdev, 0);
1635 if (irq < 0)
1636 return irq;
1637
1638 regs = devm_request_and_ioremap(&pdev->dev, io);
1639 if (!regs)
1640 return -EBUSY;
1641
1642 dw_params = dma_read_byaddr(regs, DW_PARAMS);
1643 autocfg = dw_params >> DW_PARAMS_EN & 0x1;
1644
1645 pdata = dev_get_platdata(&pdev->dev);
1646 if (!pdata)
1647 pdata = dw_dma_parse_dt(pdev);
1648
1649 if (!pdata && autocfg) {
1650 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1651 if (!pdata)
1652 return -ENOMEM;
1653
1654 /* Fill platform data with the default values */
1655 pdata->is_private = true;
1656 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
1657 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
1658 } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
1659 return -EINVAL;
1660
1661 if (autocfg)
1662 nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 0x7) + 1;
1663 else
1664 nr_channels = pdata->nr_channels;
1665
1666 size = sizeof(struct dw_dma) + nr_channels * sizeof(struct dw_dma_chan);
1667 dw = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
1668 if (!dw)
1669 return -ENOMEM;
1670
1671 dw->clk = devm_clk_get(&pdev->dev, "hclk");
1672 if (IS_ERR(dw->clk))
1673 return PTR_ERR(dw->clk);
1674 clk_prepare_enable(dw->clk);
1675
1676 dw->regs = regs;
1677 dw->sd = pdata->sd;
1678 dw->sd_count = pdata->sd_count;
1679
1680 /* get hardware configuration parameters */
1681 if (autocfg) {
1682 max_blk_size = dma_readl(dw, MAX_BLK_SIZE);
1683
1684 dw->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
1685 for (i = 0; i < dw->nr_masters; i++) {
1686 dw->data_width[i] =
1687 (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3) + 2;
1688 }
1689 } else {
1690 dw->nr_masters = pdata->nr_masters;
1691 memcpy(dw->data_width, pdata->data_width, 4);
1692 }
1693
1694 /* Calculate all channel mask before DMA setup */
1695 dw->all_chan_mask = (1 << nr_channels) - 1;
1696
1697 /* force dma off, just in case */
1698 dw_dma_off(dw);
1699
1700 /* disable BLOCK interrupts as well */
1701 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1702
1703 err = devm_request_irq(&pdev->dev, irq, dw_dma_interrupt, 0,
1704 "dw_dmac", dw);
1705 if (err)
1706 return err;
1707
1708 platform_set_drvdata(pdev, dw);
1709
1710 tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1711
1712 INIT_LIST_HEAD(&dw->dma.channels);
1713 for (i = 0; i < nr_channels; i++) {
1714 struct dw_dma_chan *dwc = &dw->chan[i];
1715 int r = nr_channels - i - 1;
1716
1717 dwc->chan.device = &dw->dma;
1718 dma_cookie_init(&dwc->chan);
1719 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1720 list_add_tail(&dwc->chan.device_node,
1721 &dw->dma.channels);
1722 else
1723 list_add(&dwc->chan.device_node, &dw->dma.channels);
1724
1725 /* 7 is highest priority & 0 is lowest. */
1726 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1727 dwc->priority = r;
1728 else
1729 dwc->priority = i;
1730
1731 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1732 spin_lock_init(&dwc->lock);
1733 dwc->mask = 1 << i;
1734
1735 INIT_LIST_HEAD(&dwc->active_list);
1736 INIT_LIST_HEAD(&dwc->queue);
1737 INIT_LIST_HEAD(&dwc->free_list);
1738
1739 channel_clear_bit(dw, CH_EN, dwc->mask);
1740
1741 dwc->dw = dw;
1742 dwc->direction = DMA_TRANS_NONE;
1743
1744 /* hardware configuration */
1745 if (autocfg) {
1746 unsigned int dwc_params;
1747
1748 dwc_params = dma_read_byaddr(regs + r * sizeof(u32),
1749 DWC_PARAMS);
1750
1751 /* Decode maximum block size for given channel. The
1752 * stored 4 bit value represents blocks from 0x00 for 3
1753 * up to 0x0a for 4095. */
1754 dwc->block_size =
1755 (4 << ((max_blk_size >> 4 * i) & 0xf)) - 1;
1756 dwc->nollp =
1757 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
1758 } else {
1759 dwc->block_size = pdata->block_size;
1760
1761 /* Check if channel supports multi block transfer */
1762 channel_writel(dwc, LLP, 0xfffffffc);
1763 dwc->nollp =
1764 (channel_readl(dwc, LLP) & 0xfffffffc) == 0;
1765 channel_writel(dwc, LLP, 0);
1766 }
1767 }
1768
1769 /* Clear all interrupts on all channels. */
1770 dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1771 dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1772 dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1773 dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1774 dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1775
1776 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1777 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1778 if (pdata->is_private)
1779 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1780 dw->dma.dev = &pdev->dev;
1781 dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1782 dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1783
1784 dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1785
1786 dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1787 dw->dma.device_control = dwc_control;
1788
1789 dw->dma.device_tx_status = dwc_tx_status;
1790 dw->dma.device_issue_pending = dwc_issue_pending;
1791
1792 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1793
1794 dev_info(&pdev->dev, "DesignWare DMA Controller, %d channels\n",
1795 nr_channels);
1796
1797 dma_async_device_register(&dw->dma);
1798
1799 return 0;
1800 }
1801
1802 static int __devexit dw_remove(struct platform_device *pdev)
1803 {
1804 struct dw_dma *dw = platform_get_drvdata(pdev);
1805 struct dw_dma_chan *dwc, *_dwc;
1806
1807 dw_dma_off(dw);
1808 dma_async_device_unregister(&dw->dma);
1809
1810 tasklet_kill(&dw->tasklet);
1811
1812 list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1813 chan.device_node) {
1814 list_del(&dwc->chan.device_node);
1815 channel_clear_bit(dw, CH_EN, dwc->mask);
1816 }
1817
1818 return 0;
1819 }
1820
1821 static void dw_shutdown(struct platform_device *pdev)
1822 {
1823 struct dw_dma *dw = platform_get_drvdata(pdev);
1824
1825 dw_dma_off(dw);
1826 clk_disable_unprepare(dw->clk);
1827 }
1828
1829 static int dw_suspend_noirq(struct device *dev)
1830 {
1831 struct platform_device *pdev = to_platform_device(dev);
1832 struct dw_dma *dw = platform_get_drvdata(pdev);
1833
1834 dw_dma_off(dw);
1835 clk_disable_unprepare(dw->clk);
1836
1837 return 0;
1838 }
1839
1840 static int dw_resume_noirq(struct device *dev)
1841 {
1842 struct platform_device *pdev = to_platform_device(dev);
1843 struct dw_dma *dw = platform_get_drvdata(pdev);
1844
1845 clk_prepare_enable(dw->clk);
1846 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1847
1848 return 0;
1849 }
1850
1851 static const struct dev_pm_ops dw_dev_pm_ops = {
1852 .suspend_noirq = dw_suspend_noirq,
1853 .resume_noirq = dw_resume_noirq,
1854 .freeze_noirq = dw_suspend_noirq,
1855 .thaw_noirq = dw_resume_noirq,
1856 .restore_noirq = dw_resume_noirq,
1857 .poweroff_noirq = dw_suspend_noirq,
1858 };
1859
1860 #ifdef CONFIG_OF
1861 static const struct of_device_id dw_dma_id_table[] = {
1862 { .compatible = "snps,dma-spear1340" },
1863 {}
1864 };
1865 MODULE_DEVICE_TABLE(of, dw_dma_id_table);
1866 #endif
1867
1868 static struct platform_driver dw_driver = {
1869 .probe = dw_probe,
1870 .remove = dw_remove,
1871 .shutdown = dw_shutdown,
1872 .driver = {
1873 .name = "dw_dmac",
1874 .pm = &dw_dev_pm_ops,
1875 .of_match_table = of_match_ptr(dw_dma_id_table),
1876 },
1877 };
1878
1879 static int __init dw_init(void)
1880 {
1881 return platform_driver_register(&dw_driver);
1882 }
1883 subsys_initcall(dw_init);
1884
1885 static void __exit dw_exit(void)
1886 {
1887 platform_driver_unregister(&dw_driver);
1888 }
1889 module_exit(dw_exit);
1890
1891 MODULE_LICENSE("GPL v2");
1892 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1893 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1894 MODULE_AUTHOR("Viresh Kumar <viresh.linux@gmail.com>");
Linux kernel - Deliverables
This page took 0.084122 seconds and 4 git commands to generate.