projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
rapidio/tsi721_dma: fix pending transaction queue handling
[deliverable/linux.git] / drivers / rapidio / devices / tsi721_dma.c
CommitLineData
9eaa3d9b
AB		 1/*
2 * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge
3 *
50835e97 4 * Copyright (c) 2011-2014 Integrated Device Technology, Inc.
9eaa3d9b
AB		 5 * Alexandre Bounine <alexandre.bounine@idt.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the Free
9 * Software Foundation; either version 2 of the License, or (at your option)
10 * any later version.
11 *
12 * This program is distributed in the hope that it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
50835e97
AB		 17 * The full GNU General Public License is included in this distribution in the
18 * file called COPYING.
9eaa3d9b
AB		 19 */
20
21#include <linux/io.h>
22#include <linux/errno.h>
23#include <linux/init.h>
24#include <linux/ioport.h>
25#include <linux/kernel.h>
26#include <linux/module.h>
27#include <linux/pci.h>
28#include <linux/rio.h>
29#include <linux/rio_drv.h>
30#include <linux/dma-mapping.h>
31#include <linux/interrupt.h>
32#include <linux/kfifo.h>
33#include <linux/delay.h>
50835e97 34#include "../../dma/dmaengine.h"
9eaa3d9b
AB		 35
36#include "tsi721.h"
37
50835e97
AB		 38#define TSI721_DMA_TX_QUEUE_SZ 16 /* number of transaction descriptors */
39
40#ifdef CONFIG_PCI_MSI
41static irqreturn_t tsi721_bdma_msix(int irq, void *ptr);
42#endif
43static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
44
45static unsigned int dma_desc_per_channel = 128;
46module_param(dma_desc_per_channel, uint, S_IWUSR | S_IRUGO);
47MODULE_PARM_DESC(dma_desc_per_channel,
48 "Number of DMA descriptors per channel (default: 128)");
49
9eaa3d9b
AB		 50static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan)
51{
52 return container_of(chan, struct tsi721_bdma_chan, dchan);
53}
54
55static inline struct tsi721_device *to_tsi721(struct dma_device *ddev)
56{
57 return container_of(ddev, struct rio_mport, dma)->priv;
58}
59
60static inline
61struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd)
62{
63 return container_of(txd, struct tsi721_tx_desc, txd);
64}
65
50835e97 66static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num)
9eaa3d9b
AB		 67{
68 struct tsi721_dma_desc *bd_ptr;
69 struct device *dev = bdma_chan->dchan.device->dev;
70 u64 *sts_ptr;
71 dma_addr_t bd_phys;
72 dma_addr_t sts_phys;
73 int sts_size;
50835e97
AB		 74#ifdef CONFIG_PCI_MSI
75 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
76#endif
9eaa3d9b
AB		 77
78 dev_dbg(dev, "Init Block DMA Engine, CH%d\n", bdma_chan->id);
79
50835e97
AB		 80 /*
81 * Allocate space for DMA descriptors
82 * (add an extra element for link descriptor)
83 */
9eaa3d9b 84 bd_ptr = dma_zalloc_coherent(dev,
50835e97 85 (bd_num + 1) * sizeof(struct tsi721_dma_desc),
9eaa3d9b
AB		 86 &bd_phys, GFP_KERNEL);
87 if (!bd_ptr)
88 return -ENOMEM;
89
50835e97 90 bdma_chan->bd_num = bd_num;
9eaa3d9b
AB		 91 bdma_chan->bd_phys = bd_phys;
92 bdma_chan->bd_base = bd_ptr;
93
94 dev_dbg(dev, "DMA descriptors @ %p (phys = %llx)\n",
95 bd_ptr, (unsigned long long)bd_phys);
96
97 /* Allocate space for descriptor status FIFO */
50835e97
AB		 98 sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ?
99 (bd_num + 1) : TSI721_DMA_MINSTSSZ;
9eaa3d9b
AB		 100 sts_size = roundup_pow_of_two(sts_size);
101 sts_ptr = dma_zalloc_coherent(dev,
102 sts_size * sizeof(struct tsi721_dma_sts),
103 &sts_phys, GFP_KERNEL);
104 if (!sts_ptr) {
105 /* Free space allocated for DMA descriptors */
106 dma_free_coherent(dev,
50835e97 107 (bd_num + 1) * sizeof(struct tsi721_dma_desc),
9eaa3d9b
AB		 108 bd_ptr, bd_phys);
109 bdma_chan->bd_base = NULL;
110 return -ENOMEM;
111 }
112
113 bdma_chan->sts_phys = sts_phys;
114 bdma_chan->sts_base = sts_ptr;
115 bdma_chan->sts_size = sts_size;
116
117 dev_dbg(dev,
118 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
119 sts_ptr, (unsigned long long)sts_phys, sts_size);
120
50835e97
AB		 121 /* Initialize DMA descriptors ring using added link descriptor */
122 bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29);
123 bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys &
9eaa3d9b 124 TSI721_DMAC_DPTRL_MASK);
50835e97 125 bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32);
9eaa3d9b
AB		 126
127 /* Setup DMA descriptor pointers */
128 iowrite32(((u64)bd_phys >> 32),
129 bdma_chan->regs + TSI721_DMAC_DPTRH);
130 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
131 bdma_chan->regs + TSI721_DMAC_DPTRL);
132
133 /* Setup descriptor status FIFO */
134 iowrite32(((u64)sts_phys >> 32),
135 bdma_chan->regs + TSI721_DMAC_DSBH);
136 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
137 bdma_chan->regs + TSI721_DMAC_DSBL);
138 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
139 bdma_chan->regs + TSI721_DMAC_DSSZ);
140
141 /* Clear interrupt bits */
142 iowrite32(TSI721_DMAC_INT_ALL,
143 bdma_chan->regs + TSI721_DMAC_INT);
144
145 ioread32(bdma_chan->regs + TSI721_DMAC_INT);
146
50835e97
AB		 147#ifdef CONFIG_PCI_MSI
148 /* Request interrupt service if we are in MSI-X mode */
149 if (priv->flags & TSI721_USING_MSIX) {
150 int rc, idx;
151
152 idx = TSI721_VECT_DMA0_DONE + bdma_chan->id;
153
154 rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
155 priv->msix[idx].irq_name, (void *)bdma_chan);
156
157 if (rc) {
158 dev_dbg(dev, "Unable to get MSI-X for BDMA%d-DONE\n",
159 bdma_chan->id);
160 goto err_out;
161 }
162
163 idx = TSI721_VECT_DMA0_INT + bdma_chan->id;
164
165 rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
166 priv->msix[idx].irq_name, (void *)bdma_chan);
167
168 if (rc) {
169 dev_dbg(dev, "Unable to get MSI-X for BDMA%d-INT\n",
170 bdma_chan->id);
171 free_irq(
172 priv->msix[TSI721_VECT_DMA0_DONE +
173 bdma_chan->id].vector,
174 (void *)bdma_chan);
175 }
176
177err_out:
178 if (rc) {
179 /* Free space allocated for DMA descriptors */
180 dma_free_coherent(dev,
181 (bd_num + 1) * sizeof(struct tsi721_dma_desc),
182 bd_ptr, bd_phys);
183 bdma_chan->bd_base = NULL;
184
185 /* Free space allocated for status descriptors */
186 dma_free_coherent(dev,
187 sts_size * sizeof(struct tsi721_dma_sts),
188 sts_ptr, sts_phys);
189 bdma_chan->sts_base = NULL;
190
191 return -EIO;
192 }
193 }
194#endif /* CONFIG_PCI_MSI */
195
9eaa3d9b
AB		 196 /* Toggle DMA channel initialization */
197 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
198 ioread32(bdma_chan->regs + TSI721_DMAC_CTL);
199 bdma_chan->wr_count = bdma_chan->wr_count_next = 0;
200 bdma_chan->sts_rdptr = 0;
201 udelay(10);
202
203 return 0;
204}
205
206static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan)
207{
208 u32 ch_stat;
50835e97
AB		 209#ifdef CONFIG_PCI_MSI
210 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
211#endif
9eaa3d9b
AB		 212
213 if (bdma_chan->bd_base == NULL)
214 return 0;
215
216 /* Check if DMA channel still running */
217 ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
218 if (ch_stat & TSI721_DMAC_STS_RUN)
219 return -EFAULT;
220
221 /* Put DMA channel into init state */
222 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
223
50835e97
AB		 224#ifdef CONFIG_PCI_MSI
225 if (priv->flags & TSI721_USING_MSIX) {
226 free_irq(priv->msix[TSI721_VECT_DMA0_DONE +
227 bdma_chan->id].vector, (void *)bdma_chan);
228 free_irq(priv->msix[TSI721_VECT_DMA0_INT +
229 bdma_chan->id].vector, (void *)bdma_chan);
230 }
231#endif /* CONFIG_PCI_MSI */
232
9eaa3d9b
AB		 233 /* Free space allocated for DMA descriptors */
234 dma_free_coherent(bdma_chan->dchan.device->dev,
50835e97 235 (bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc),
9eaa3d9b
AB		 236 bdma_chan->bd_base, bdma_chan->bd_phys);
237 bdma_chan->bd_base = NULL;
238
239 /* Free space allocated for status FIFO */
240 dma_free_coherent(bdma_chan->dchan.device->dev,
241 bdma_chan->sts_size * sizeof(struct tsi721_dma_sts),
242 bdma_chan->sts_base, bdma_chan->sts_phys);
243 bdma_chan->sts_base = NULL;
244 return 0;
245}
246
247static void
248tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable)
249{
250 if (enable) {
251 /* Clear pending BDMA channel interrupts */
252 iowrite32(TSI721_DMAC_INT_ALL,
253 bdma_chan->regs + TSI721_DMAC_INT);
254 ioread32(bdma_chan->regs + TSI721_DMAC_INT);
255 /* Enable BDMA channel interrupts */
256 iowrite32(TSI721_DMAC_INT_ALL,
257 bdma_chan->regs + TSI721_DMAC_INTE);
258 } else {
259 /* Disable BDMA channel interrupts */
260 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
261 /* Clear pending BDMA channel interrupts */
262 iowrite32(TSI721_DMAC_INT_ALL,
263 bdma_chan->regs + TSI721_DMAC_INT);
264 }
265
266}
267
268static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan)
269{
270 u32 sts;
271
272 sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
273 return ((sts & TSI721_DMAC_STS_RUN) == 0);
274}
275
276void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan)
277{
278 /* Disable BDMA channel interrupts */
279 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
04379dff
AB		 280 if (bdma_chan->active)
281 tasklet_schedule(&bdma_chan->tasklet);
9eaa3d9b
AB		 282}
283
284#ifdef CONFIG_PCI_MSI
285/**
286 * tsi721_omsg_msix - MSI-X interrupt handler for BDMA channels
287 * @irq: Linux interrupt number
288 * @ptr: Pointer to interrupt-specific data (BDMA channel structure)
289 *
290 * Handles BDMA channel interrupts signaled using MSI-X.
291 */
292static irqreturn_t tsi721_bdma_msix(int irq, void *ptr)
293{
294 struct tsi721_bdma_chan *bdma_chan = ptr;
295
296 tsi721_bdma_handler(bdma_chan);
297 return IRQ_HANDLED;
298}
299#endif /* CONFIG_PCI_MSI */
300
301/* Must be called with the spinlock held */
302static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan)
303{
304 if (!tsi721_dma_is_idle(bdma_chan)) {
305 dev_err(bdma_chan->dchan.device->dev,
306 "BUG: Attempt to start non-idle channel\n");
307 return;
308 }
309
310 if (bdma_chan->wr_count == bdma_chan->wr_count_next) {
311 dev_err(bdma_chan->dchan.device->dev,
312 "BUG: Attempt to start DMA with no BDs ready\n");
313 return;
314 }
315
316 dev_dbg(bdma_chan->dchan.device->dev,
50835e97
AB		 317 "%s: chan_%d (wrc=%d)\n", __func__, bdma_chan->id,
318 bdma_chan->wr_count_next);
9eaa3d9b
AB		 319
320 iowrite32(bdma_chan->wr_count_next,
321 bdma_chan->regs + TSI721_DMAC_DWRCNT);
322 ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT);
323
324 bdma_chan->wr_count = bdma_chan->wr_count_next;
325}
326
9eaa3d9b 327static int
50835e97
AB		 328tsi721_desc_fill_init(struct tsi721_tx_desc *desc,
329 struct tsi721_dma_desc *bd_ptr,
330 struct scatterlist *sg, u32 sys_size)
9eaa3d9b 331{
9eaa3d9b
AB		 332 u64 rio_addr;
333
50835e97
AB		 334 if (bd_ptr == NULL)
335 return -EINVAL;
336
9eaa3d9b
AB		 337 /* Initialize DMA descriptor */
338 bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) |
50835e97 339 (desc->rtype << 19) | desc->destid);
9eaa3d9b 340 bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |
40f847ba 341 (sys_size << 26));
9eaa3d9b
AB		 342 rio_addr = (desc->rio_addr >> 2) |
343 ((u64)(desc->rio_addr_u & 0x3) << 62);
344 bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff);
345 bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32);
346 bd_ptr->t1.bufptr_lo = cpu_to_le32(
347 (u64)sg_dma_address(sg) & 0xffffffff);
348 bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32);
349 bd_ptr->t1.s_dist = 0;
350 bd_ptr->t1.s_size = 0;
351
352 return 0;
353}
354
40f847ba 355static int
50835e97 356tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt)
40f847ba 357{
50835e97
AB		 358 if (bd_ptr == NULL)
359 return -EINVAL;
40f847ba
AB		 360
361 /* Update DMA descriptor */
50835e97 362 if (interrupt)
40f847ba 363 bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF);
50835e97 364 bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1);
40f847ba
AB		 365
366 return 0;
367}
368
50835e97
AB		 369static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan,
370 struct tsi721_tx_desc *desc)
9eaa3d9b
AB		 371{
372 struct dma_async_tx_descriptor *txd = &desc->txd;
373 dma_async_tx_callback callback = txd->callback;
374 void *param = txd->callback_param;
375
9eaa3d9b 376 list_move(&desc->desc_node, &bdma_chan->free_list);
9eaa3d9b
AB		 377
378 if (callback)
379 callback(param);
380}
381
9eaa3d9b
AB		 382static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan)
383{
384 u32 srd_ptr;
385 u64 *sts_ptr;
386 int i, j;
387
388 /* Check and clear descriptor status FIFO entries */
389 srd_ptr = bdma_chan->sts_rdptr;
390 sts_ptr = bdma_chan->sts_base;
391 j = srd_ptr * 8;
392 while (sts_ptr[j]) {
393 for (i = 0; i < 8 && sts_ptr[j]; i++, j++)
394 sts_ptr[j] = 0;
395
396 ++srd_ptr;
397 srd_ptr %= bdma_chan->sts_size;
398 j = srd_ptr * 8;
399 }
400
401 iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP);
402 bdma_chan->sts_rdptr = srd_ptr;
403}
404
50835e97
AB		 405/* Must be called with the channel spinlock held */
406static int tsi721_submit_sg(struct tsi721_tx_desc *desc)
407{
408 struct dma_chan *dchan = desc->txd.chan;
409 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
410 u32 sys_size;
411 u64 rio_addr;
412 dma_addr_t next_addr;
413 u32 bcount;
414 struct scatterlist *sg;
415 unsigned int i;
416 int err = 0;
417 struct tsi721_dma_desc *bd_ptr = NULL;
418 u32 idx, rd_idx;
419 u32 add_count = 0;
420
421 if (!tsi721_dma_is_idle(bdma_chan)) {
422 dev_err(bdma_chan->dchan.device->dev,
423 "BUG: Attempt to use non-idle channel\n");
424 return -EIO;
425 }
426
427 /*
428 * Fill DMA channel's hardware buffer descriptors.
429 * (NOTE: RapidIO destination address is limited to 64 bits for now)
430 */
431 rio_addr = desc->rio_addr;
432 next_addr = -1;
433 bcount = 0;
434 sys_size = dma_to_mport(bdma_chan->dchan.device)->sys_size;
435
436 rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT);
437 rd_idx %= (bdma_chan->bd_num + 1);
438
439 idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1);
440 if (idx == bdma_chan->bd_num) {
441 /* wrap around link descriptor */
442 idx = 0;
443 add_count++;
444 }
445
446 dev_dbg(dchan->device->dev, "%s: BD ring status: rdi=%d wri=%d\n",
447 __func__, rd_idx, idx);
448
449 for_each_sg(desc->sg, sg, desc->sg_len, i) {
450
451 dev_dbg(dchan->device->dev, "sg%d/%d addr: 0x%llx len: %d\n",
452 i, desc->sg_len,
453 (unsigned long long)sg_dma_address(sg), sg_dma_len(sg));
454
455 if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) {
456 dev_err(dchan->device->dev,
457 "%s: SG entry %d is too large\n", __func__, i);
458 err = -EINVAL;
459 break;
460 }
461
462 /*
463 * If this sg entry forms contiguous block with previous one,
464 * try to merge it into existing DMA descriptor
465 */
466 if (next_addr == sg_dma_address(sg) &&
467 bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) {
468 /* Adjust byte count of the descriptor */
469 bcount += sg_dma_len(sg);
470 goto entry_done;
471 } else if (next_addr != -1) {
472 /* Finalize descriptor using total byte count value */
473 tsi721_desc_fill_end(bd_ptr, bcount, 0);
474 dev_dbg(dchan->device->dev,
475 "%s: prev desc final len: %d\n",
476 __func__, bcount);
477 }
478
479 desc->rio_addr = rio_addr;
480
481 if (i && idx == rd_idx) {
482 dev_dbg(dchan->device->dev,
483 "%s: HW descriptor ring is full @ %d\n",
484 __func__, i);
485 desc->sg = sg;
486 desc->sg_len -= i;
487 break;
488 }
489
490 bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx];
491 err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);
492 if (err) {
493 dev_err(dchan->device->dev,
494 "Failed to build desc: err=%d\n", err);
495 break;
496 }
497
498 dev_dbg(dchan->device->dev, "bd_ptr = %p did=%d raddr=0x%llx\n",
499 bd_ptr, desc->destid, desc->rio_addr);
500
501 next_addr = sg_dma_address(sg);
502 bcount = sg_dma_len(sg);
503
504 add_count++;
505 if (++idx == bdma_chan->bd_num) {
506 /* wrap around link descriptor */
507 idx = 0;
508 add_count++;
509 }
510
511entry_done:
512 if (sg_is_last(sg)) {
513 tsi721_desc_fill_end(bd_ptr, bcount, 0);
514 dev_dbg(dchan->device->dev, "%s: last desc final len: %d\n",
515 __func__, bcount);
516 desc->sg_len = 0;
517 } else {
518 rio_addr += sg_dma_len(sg);
519 next_addr += sg_dma_len(sg);
520 }
521 }
522
523 if (!err)
524 bdma_chan->wr_count_next += add_count;
525
526 return err;
527}
528
d2a321f3
AB		 529static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan,
530 struct tsi721_tx_desc *desc)
9eaa3d9b 531{
50835e97
AB		 532 int err;
533
534 dev_dbg(bdma_chan->dchan.device->dev, "%s: Enter\n", __func__);
535
d2a321f3
AB		 536 if (!tsi721_dma_is_idle(bdma_chan))
537 return;
538
50835e97 539 /*
d2a321f3
AB		 540 * If there is no data transfer in progress, fetch new descriptor from
541 * the pending queue.
542 */
543
544 if (desc == NULL && bdma_chan->active_tx == NULL &&
545 !list_empty(&bdma_chan->queue)) {
546 desc = list_first_entry(&bdma_chan->queue,
547 struct tsi721_tx_desc, desc_node);
548 list_del_init((&desc->desc_node));
549 bdma_chan->active_tx = desc;
550 }
50835e97 551
d2a321f3 552 if (desc) {
50835e97
AB		 553 err = tsi721_submit_sg(desc);
554 if (!err)
555 tsi721_start_dma(bdma_chan);
556 else {
557 tsi721_dma_tx_err(bdma_chan, desc);
558 dev_dbg(bdma_chan->dchan.device->dev,
559 "ERR: tsi721_submit_sg failed with err=%d\n",
560 err);
561 }
9eaa3d9b 562 }
50835e97
AB		 563
564 dev_dbg(bdma_chan->dchan.device->dev, "%s: Exit\n", __func__);
9eaa3d9b
AB		 565}
566
567static void tsi721_dma_tasklet(unsigned long data)
568{
569 struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data;
570 u32 dmac_int, dmac_sts;
571
572 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
573 dev_dbg(bdma_chan->dchan.device->dev, "%s: DMAC%d_INT = 0x%x\n",
574 __func__, bdma_chan->id, dmac_int);
575 /* Clear channel interrupts */
576 iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT);
577
578 if (dmac_int & TSI721_DMAC_INT_ERR) {
579 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
580 dev_err(bdma_chan->dchan.device->dev,
581 "%s: DMA ERROR - DMAC%d_STS = 0x%x\n",
582 __func__, bdma_chan->id, dmac_sts);
d2a321f3
AB		 583
584 spin_lock(&bdma_chan->lock);
585 bdma_chan->active_tx = NULL;
586 spin_unlock(&bdma_chan->lock);
9eaa3d9b
AB		 587 }
588
589 if (dmac_int & TSI721_DMAC_INT_STFULL) {
590 dev_err(bdma_chan->dchan.device->dev,
591 "%s: DMAC%d descriptor status FIFO is full\n",
592 __func__, bdma_chan->id);
593 }
594
595 if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) {
50835e97
AB		 596 struct tsi721_tx_desc *desc;
597
9eaa3d9b
AB		 598 tsi721_clr_stat(bdma_chan);
599 spin_lock(&bdma_chan->lock);
d2a321f3 600 desc = bdma_chan->active_tx;
50835e97
AB		 601
602 if (desc->sg_len == 0) {
603 dma_async_tx_callback callback = NULL;
604 void *param = NULL;
605
606 desc->status = DMA_COMPLETE;
607 dma_cookie_complete(&desc->txd);
608 if (desc->txd.flags & DMA_PREP_INTERRUPT) {
609 callback = desc->txd.callback;
610 param = desc->txd.callback_param;
611 }
d2a321f3
AB		 612 list_add(&desc->desc_node, &bdma_chan->free_list);
613 bdma_chan->active_tx = NULL;
50835e97
AB		 614 spin_unlock(&bdma_chan->lock);
615 if (callback)
616 callback(param);
617 spin_lock(&bdma_chan->lock);
618 }
619
d2a321f3 620 tsi721_advance_work(bdma_chan, bdma_chan->active_tx);
9eaa3d9b
AB		 621 spin_unlock(&bdma_chan->lock);
622 }
623
624 /* Re-Enable BDMA channel interrupts */
625 iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE);
626}
627
628static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd)
629{
630 struct tsi721_tx_desc *desc = to_tsi721_desc(txd);
631 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan);
632 dma_cookie_t cookie;
633
50835e97
AB		 634 /* Check if the descriptor is detached from any lists */
635 if (!list_empty(&desc->desc_node)) {
636 dev_err(bdma_chan->dchan.device->dev,
637 "%s: wrong state of descriptor %p\n", __func__, txd);
638 return -EIO;
639 }
9eaa3d9b 640
50835e97 641 spin_lock_bh(&bdma_chan->lock);
9eaa3d9b 642
50835e97
AB		 643 if (!bdma_chan->active) {
644 spin_unlock_bh(&bdma_chan->lock);
645 return -ENODEV;
9eaa3d9b
AB		 646 }
647
50835e97
AB		 648 cookie = dma_cookie_assign(txd);
649 desc->status = DMA_IN_PROGRESS;
650 list_add_tail(&desc->desc_node, &bdma_chan->queue);
651
9eaa3d9b
AB		 652 spin_unlock_bh(&bdma_chan->lock);
653 return cookie;
654}
655
656static int tsi721_alloc_chan_resources(struct dma_chan *dchan)
657{
658 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
9eaa3d9b 659 struct tsi721_tx_desc *desc = NULL;
9eaa3d9b 660 int i;
50835e97
AB		 661
662 dev_dbg(dchan->device->dev, "%s: for channel %d\n",
663 __func__, bdma_chan->id);
9eaa3d9b
AB		 664
665 if (bdma_chan->bd_base)
50835e97 666 return TSI721_DMA_TX_QUEUE_SZ;
9eaa3d9b
AB		 667
668 /* Initialize BDMA channel */
50835e97 669 if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) {
9eaa3d9b
AB		 670 dev_err(dchan->device->dev, "Unable to initialize data DMA"
671 " channel %d, aborting\n", bdma_chan->id);
50835e97 672 return -ENODEV;
9eaa3d9b
AB		 673 }
674
50835e97
AB		 675 /* Allocate queue of transaction descriptors */
676 desc = kcalloc(TSI721_DMA_TX_QUEUE_SZ, sizeof(struct tsi721_tx_desc),
9eaa3d9b
AB		 677 GFP_KERNEL);
678 if (!desc) {
679 dev_err(dchan->device->dev,
680 "Failed to allocate logical descriptors\n");
50835e97
AB		 681 tsi721_bdma_ch_free(bdma_chan);
682 return -ENOMEM;
9eaa3d9b
AB		 683 }
684
685 bdma_chan->tx_desc = desc;
686
50835e97 687 for (i = 0; i < TSI721_DMA_TX_QUEUE_SZ; i++) {
9eaa3d9b
AB		 688 dma_async_tx_descriptor_init(&desc[i].txd, dchan);
689 desc[i].txd.tx_submit = tsi721_tx_submit;
690 desc[i].txd.flags = DMA_CTRL_ACK;
50835e97 691 list_add(&desc[i].desc_node, &bdma_chan->free_list);
9eaa3d9b
AB		 692 }
693
50835e97 694 dma_cookie_init(dchan);
9eaa3d9b 695
04379dff 696 bdma_chan->active = true;
9eaa3d9b
AB		 697 tsi721_bdma_interrupt_enable(bdma_chan, 1);
698
50835e97 699 return TSI721_DMA_TX_QUEUE_SZ;
9eaa3d9b
AB		 700}
701
50835e97 702static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan)
9eaa3d9b 703{
50835e97 704 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
04379dff
AB		 705
706#ifdef CONFIG_PCI_MSI
707 if (priv->flags & TSI721_USING_MSIX) {
708 synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE +
709 bdma_chan->id].vector);
710 synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT +
711 bdma_chan->id].vector);
712 } else
713#endif
714 synchronize_irq(priv->pdev->irq);
50835e97 715}
04379dff 716
50835e97
AB		 717static void tsi721_free_chan_resources(struct dma_chan *dchan)
718{
719 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
9eaa3d9b 720
50835e97
AB		 721 dev_dbg(dchan->device->dev, "%s: for channel %d\n",
722 __func__, bdma_chan->id);
9eaa3d9b 723
50835e97
AB		 724 if (bdma_chan->bd_base == NULL)
725 return;
9eaa3d9b 726
50835e97
AB		 727 tsi721_bdma_interrupt_enable(bdma_chan, 0);
728 bdma_chan->active = false;
729 tsi721_sync_dma_irq(bdma_chan);
730 tasklet_kill(&bdma_chan->tasklet);
731 INIT_LIST_HEAD(&bdma_chan->free_list);
9eaa3d9b 732 kfree(bdma_chan->tx_desc);
50835e97 733 tsi721_bdma_ch_free(bdma_chan);
9eaa3d9b
AB		 734}
735
736static
737enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
738 struct dma_tx_state *txstate)
739{
50835e97 740 return dma_cookie_status(dchan, cookie, txstate);
9eaa3d9b
AB		 741}
742
743static void tsi721_issue_pending(struct dma_chan *dchan)
744{
745 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
746
50835e97 747 dev_dbg(dchan->device->dev, "%s: Enter\n", __func__);
9eaa3d9b 748
d2a321f3 749 spin_lock_bh(&bdma_chan->lock);
50835e97 750 if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) {
d2a321f3 751 tsi721_advance_work(bdma_chan, NULL);
50835e97 752 }
d2a321f3 753 spin_unlock_bh(&bdma_chan->lock);
9eaa3d9b
AB		 754}
755
756static
757struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan,
758 struct scatterlist *sgl, unsigned int sg_len,
759 enum dma_transfer_direction dir, unsigned long flags,
760 void *tinfo)
761{
762 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
50835e97 763 struct tsi721_tx_desc *desc, *_d;
9eaa3d9b 764 struct rio_dma_ext *rext = tinfo;
9eaa3d9b 765 enum dma_rtype rtype;
50835e97 766 struct dma_async_tx_descriptor *txd = NULL;
9eaa3d9b
AB		 767
768 if (!sgl || !sg_len) {
769 dev_err(dchan->device->dev, "%s: No SG list\n", __func__);
770 return NULL;
771 }
772
50835e97
AB		 773 dev_dbg(dchan->device->dev, "%s: %s\n", __func__,
774 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE");
775
9eaa3d9b
AB		 776 if (dir == DMA_DEV_TO_MEM)
777 rtype = NREAD;
778 else if (dir == DMA_MEM_TO_DEV) {
779 switch (rext->wr_type) {
780 case RDW_ALL_NWRITE:
781 rtype = ALL_NWRITE;
782 break;
783 case RDW_ALL_NWRITE_R:
784 rtype = ALL_NWRITE_R;
785 break;
786 case RDW_LAST_NWRITE_R:
787 default:
788 rtype = LAST_NWRITE_R;
789 break;
790 }
791 } else {
792 dev_err(dchan->device->dev,
793 "%s: Unsupported DMA direction option\n", __func__);
794 return NULL;
795 }
796
50835e97 797 spin_lock_bh(&bdma_chan->lock);
40f847ba 798
50835e97
AB		 799 list_for_each_entry_safe(desc, _d, &bdma_chan->free_list, desc_node) {
800 if (async_tx_test_ack(&desc->txd)) {
801 list_del_init(&desc->desc_node);
802 desc->destid = rext->destid;
803 desc->rio_addr = rext->rio_addr;
804 desc->rio_addr_u = 0;
805 desc->rtype = rtype;
806 desc->sg_len = sg_len;
807 desc->sg = sgl;
808 txd = &desc->txd;
809 txd->flags = flags;
810 break;
40f847ba 811 }
9eaa3d9b
AB		 812 }
813
50835e97 814 spin_unlock_bh(&bdma_chan->lock);
9eaa3d9b 815
50835e97 816 return txd;
9eaa3d9b
AB		 817}
818
7664cfe0 819static int tsi721_terminate_all(struct dma_chan *dchan)
9eaa3d9b
AB		 820{
821 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
822 struct tsi721_tx_desc *desc, *_d;
50835e97 823 u32 dmac_int;
9eaa3d9b
AB		 824 LIST_HEAD(list);
825
826 dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
827
9eaa3d9b
AB		 828 spin_lock_bh(&bdma_chan->lock);
829
50835e97
AB		 830 bdma_chan->active = false;
831
832 if (!tsi721_dma_is_idle(bdma_chan)) {
833 /* make sure to stop the transfer */
834 iowrite32(TSI721_DMAC_CTL_SUSP,
835 bdma_chan->regs + TSI721_DMAC_CTL);
836
837 /* Wait until DMA channel stops */
838 do {
839 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
840 } while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0);
841 }
9eaa3d9b 842
d2a321f3
AB		 843 if (bdma_chan->active_tx)
844 list_add(&bdma_chan->active_tx->desc_node, &list);
9eaa3d9b
AB		 845 list_splice_init(&bdma_chan->queue, &list);
846
847 list_for_each_entry_safe(desc, _d, &list, desc_node)
50835e97 848 tsi721_dma_tx_err(bdma_chan, desc);
9eaa3d9b
AB		 849
850 spin_unlock_bh(&bdma_chan->lock);
851
852 return 0;
853}
854
305c891e 855int tsi721_register_dma(struct tsi721_device *priv)
9eaa3d9b
AB		 856{
857 int i;
50835e97 858 int nr_channels = 0;
9eaa3d9b
AB		 859 int err;
860 struct rio_mport *mport = priv->mport;
861
9eaa3d9b
AB		 862 INIT_LIST_HEAD(&mport->dma.channels);
863
50835e97 864 for (i = 0; i < TSI721_DMA_MAXCH; i++) {
9eaa3d9b
AB		 865 struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i];
866
867 if (i == TSI721_DMACH_MAINT)
868 continue;
869
9eaa3d9b
AB		 870 bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i);
871
872 bdma_chan->dchan.device = &mport->dma;
873 bdma_chan->dchan.cookie = 1;
874 bdma_chan->dchan.chan_id = i;
875 bdma_chan->id = i;
04379dff 876 bdma_chan->active = false;
9eaa3d9b
AB		 877
878 spin_lock_init(&bdma_chan->lock);
879
d2a321f3 880 bdma_chan->active_tx = NULL;
9eaa3d9b
AB		 881 INIT_LIST_HEAD(&bdma_chan->queue);
882 INIT_LIST_HEAD(&bdma_chan->free_list);
883
884 tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet,
885 (unsigned long)bdma_chan);
9eaa3d9b
AB		 886 list_add_tail(&bdma_chan->dchan.device_node,
887 &mport->dma.channels);
50835e97 888 nr_channels++;
9eaa3d9b
AB		 889 }
890
50835e97 891 mport->dma.chancnt = nr_channels;
9eaa3d9b
AB		 892 dma_cap_zero(mport->dma.cap_mask);
893 dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask);
894 dma_cap_set(DMA_SLAVE, mport->dma.cap_mask);
895
50835e97 896 mport->dma.dev = &priv->pdev->dev;
9eaa3d9b
AB		 897 mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources;
898 mport->dma.device_free_chan_resources = tsi721_free_chan_resources;
899 mport->dma.device_tx_status = tsi721_tx_status;
900 mport->dma.device_issue_pending = tsi721_issue_pending;
901 mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg;
7664cfe0 902 mport->dma.device_terminate_all = tsi721_terminate_all;
9eaa3d9b
AB		 903
904 err = dma_async_device_register(&mport->dma);
905 if (err)
906 dev_err(&priv->pdev->dev, "Failed to register DMA device\n");
907
908 return err;
909}
Linux kernel - Deliverables
This page took 0.26004 seconds and 5 git commands to generate.