Commit | Line | Data |
---|---|---|
4de3a8fa CC |
1 | /* |
2 | * arch/arm/mach-tegra/dma.c | |
3 | * | |
4 | * System DMA driver for NVIDIA Tegra SoCs | |
5 | * | |
6 | * Copyright (c) 2008-2009, NVIDIA Corporation. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License as published by | |
10 | * the Free Software Foundation; either version 2 of the License, or | |
11 | * (at your option) any later version. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
14 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
15 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
16 | * more details. | |
17 | * | |
18 | * You should have received a copy of the GNU General Public License along | |
19 | * with this program; if not, write to the Free Software Foundation, Inc., | |
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |
21 | */ | |
22 | ||
23 | #include <linux/io.h> | |
24 | #include <linux/interrupt.h> | |
25 | #include <linux/module.h> | |
26 | #include <linux/spinlock.h> | |
27 | #include <linux/err.h> | |
28 | #include <linux/irq.h> | |
29 | #include <linux/delay.h> | |
1ca00347 | 30 | #include <linux/clk.h> |
4de3a8fa CC |
31 | #include <mach/dma.h> |
32 | #include <mach/irqs.h> | |
33 | #include <mach/iomap.h> | |
2ea67fd1 | 34 | #include <mach/suspend.h> |
4de3a8fa | 35 | |
e2f91578 OJ |
36 | #include "apbio.h" |
37 | ||
4de3a8fa CC |
38 | #define APB_DMA_GEN 0x000 |
39 | #define GEN_ENABLE (1<<31) | |
40 | ||
41 | #define APB_DMA_CNTRL 0x010 | |
42 | ||
43 | #define APB_DMA_IRQ_MASK 0x01c | |
44 | ||
45 | #define APB_DMA_IRQ_MASK_SET 0x020 | |
46 | ||
47 | #define APB_DMA_CHAN_CSR 0x000 | |
48 | #define CSR_ENB (1<<31) | |
49 | #define CSR_IE_EOC (1<<30) | |
50 | #define CSR_HOLD (1<<29) | |
51 | #define CSR_DIR (1<<28) | |
52 | #define CSR_ONCE (1<<27) | |
53 | #define CSR_FLOW (1<<21) | |
54 | #define CSR_REQ_SEL_SHIFT 16 | |
4de3a8fa CC |
55 | #define CSR_WCOUNT_SHIFT 2 |
56 | #define CSR_WCOUNT_MASK 0xFFFC | |
57 | ||
58 | #define APB_DMA_CHAN_STA 0x004 | |
59 | #define STA_BUSY (1<<31) | |
60 | #define STA_ISE_EOC (1<<30) | |
61 | #define STA_HALT (1<<29) | |
62 | #define STA_PING_PONG (1<<28) | |
63 | #define STA_COUNT_SHIFT 2 | |
64 | #define STA_COUNT_MASK 0xFFFC | |
65 | ||
66 | #define APB_DMA_CHAN_AHB_PTR 0x010 | |
67 | ||
68 | #define APB_DMA_CHAN_AHB_SEQ 0x014 | |
69 | #define AHB_SEQ_INTR_ENB (1<<31) | |
70 | #define AHB_SEQ_BUS_WIDTH_SHIFT 28 | |
71 | #define AHB_SEQ_BUS_WIDTH_MASK (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT) | |
72 | #define AHB_SEQ_BUS_WIDTH_8 (0<<AHB_SEQ_BUS_WIDTH_SHIFT) | |
73 | #define AHB_SEQ_BUS_WIDTH_16 (1<<AHB_SEQ_BUS_WIDTH_SHIFT) | |
74 | #define AHB_SEQ_BUS_WIDTH_32 (2<<AHB_SEQ_BUS_WIDTH_SHIFT) | |
75 | #define AHB_SEQ_BUS_WIDTH_64 (3<<AHB_SEQ_BUS_WIDTH_SHIFT) | |
76 | #define AHB_SEQ_BUS_WIDTH_128 (4<<AHB_SEQ_BUS_WIDTH_SHIFT) | |
77 | #define AHB_SEQ_DATA_SWAP (1<<27) | |
78 | #define AHB_SEQ_BURST_MASK (0x7<<24) | |
79 | #define AHB_SEQ_BURST_1 (4<<24) | |
80 | #define AHB_SEQ_BURST_4 (5<<24) | |
81 | #define AHB_SEQ_BURST_8 (6<<24) | |
82 | #define AHB_SEQ_DBL_BUF (1<<19) | |
83 | #define AHB_SEQ_WRAP_SHIFT 16 | |
84 | #define AHB_SEQ_WRAP_MASK (0x7<<AHB_SEQ_WRAP_SHIFT) | |
85 | ||
86 | #define APB_DMA_CHAN_APB_PTR 0x018 | |
87 | ||
88 | #define APB_DMA_CHAN_APB_SEQ 0x01c | |
89 | #define APB_SEQ_BUS_WIDTH_SHIFT 28 | |
90 | #define APB_SEQ_BUS_WIDTH_MASK (0x7<<APB_SEQ_BUS_WIDTH_SHIFT) | |
91 | #define APB_SEQ_BUS_WIDTH_8 (0<<APB_SEQ_BUS_WIDTH_SHIFT) | |
92 | #define APB_SEQ_BUS_WIDTH_16 (1<<APB_SEQ_BUS_WIDTH_SHIFT) | |
93 | #define APB_SEQ_BUS_WIDTH_32 (2<<APB_SEQ_BUS_WIDTH_SHIFT) | |
94 | #define APB_SEQ_BUS_WIDTH_64 (3<<APB_SEQ_BUS_WIDTH_SHIFT) | |
95 | #define APB_SEQ_BUS_WIDTH_128 (4<<APB_SEQ_BUS_WIDTH_SHIFT) | |
96 | #define APB_SEQ_DATA_SWAP (1<<27) | |
97 | #define APB_SEQ_WRAP_SHIFT 16 | |
98 | #define APB_SEQ_WRAP_MASK (0x7<<APB_SEQ_WRAP_SHIFT) | |
99 | ||
100 | #define TEGRA_SYSTEM_DMA_CH_NR 16 | |
101 | #define TEGRA_SYSTEM_DMA_AVP_CH_NUM 4 | |
102 | #define TEGRA_SYSTEM_DMA_CH_MIN 0 | |
103 | #define TEGRA_SYSTEM_DMA_CH_MAX \ | |
104 | (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1) | |
105 | ||
106 | #define NV_DMA_MAX_TRASFER_SIZE 0x10000 | |
107 | ||
cf28cba0 | 108 | static const unsigned int ahb_addr_wrap_table[8] = { |
4de3a8fa CC |
109 | 0, 32, 64, 128, 256, 512, 1024, 2048 |
110 | }; | |
111 | ||
cf28cba0 OJ |
112 | static const unsigned int apb_addr_wrap_table[8] = { |
113 | 0, 1, 2, 4, 8, 16, 32, 64 | |
114 | }; | |
4de3a8fa | 115 | |
cf28cba0 OJ |
116 | static const unsigned int bus_width_table[5] = { |
117 | 8, 16, 32, 64, 128 | |
118 | }; | |
4de3a8fa CC |
119 | |
120 | #define TEGRA_DMA_NAME_SIZE 16 | |
121 | struct tegra_dma_channel { | |
122 | struct list_head list; | |
123 | int id; | |
124 | spinlock_t lock; | |
125 | char name[TEGRA_DMA_NAME_SIZE]; | |
126 | void __iomem *addr; | |
127 | int mode; | |
128 | int irq; | |
5789fee9 | 129 | int req_transfer_count; |
4de3a8fa CC |
130 | }; |
131 | ||
132 | #define NV_DMA_MAX_CHANNELS 32 | |
133 | ||
ccac0515 | 134 | static bool tegra_dma_initialized; |
5789fee9 | 135 | static DEFINE_MUTEX(tegra_dma_lock); |
cb3732d0 | 136 | static DEFINE_SPINLOCK(enable_lock); |
5789fee9 | 137 | |
4de3a8fa CC |
138 | static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS); |
139 | static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS]; | |
140 | ||
141 | static void tegra_dma_update_hw(struct tegra_dma_channel *ch, | |
142 | struct tegra_dma_req *req); | |
143 | static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch, | |
144 | struct tegra_dma_req *req); | |
4de3a8fa CC |
145 | static void tegra_dma_stop(struct tegra_dma_channel *ch); |
146 | ||
147 | void tegra_dma_flush(struct tegra_dma_channel *ch) | |
148 | { | |
149 | } | |
150 | EXPORT_SYMBOL(tegra_dma_flush); | |
151 | ||
152 | void tegra_dma_dequeue(struct tegra_dma_channel *ch) | |
153 | { | |
154 | struct tegra_dma_req *req; | |
155 | ||
5789fee9 CC |
156 | if (tegra_dma_is_empty(ch)) |
157 | return; | |
158 | ||
4de3a8fa CC |
159 | req = list_entry(ch->list.next, typeof(*req), node); |
160 | ||
161 | tegra_dma_dequeue_req(ch, req); | |
162 | return; | |
163 | } | |
164 | ||
cf28cba0 | 165 | static void tegra_dma_stop(struct tegra_dma_channel *ch) |
4de3a8fa | 166 | { |
5789fee9 CC |
167 | u32 csr; |
168 | u32 status; | |
4de3a8fa | 169 | |
5789fee9 | 170 | csr = readl(ch->addr + APB_DMA_CHAN_CSR); |
4de3a8fa CC |
171 | csr &= ~CSR_IE_EOC; |
172 | writel(csr, ch->addr + APB_DMA_CHAN_CSR); | |
173 | ||
174 | csr &= ~CSR_ENB; | |
175 | writel(csr, ch->addr + APB_DMA_CHAN_CSR); | |
176 | ||
177 | status = readl(ch->addr + APB_DMA_CHAN_STA); | |
178 | if (status & STA_ISE_EOC) | |
179 | writel(status, ch->addr + APB_DMA_CHAN_STA); | |
180 | } | |
181 | ||
cf28cba0 | 182 | static int tegra_dma_cancel(struct tegra_dma_channel *ch) |
4de3a8fa | 183 | { |
4de3a8fa CC |
184 | unsigned long irq_flags; |
185 | ||
186 | spin_lock_irqsave(&ch->lock, irq_flags); | |
187 | while (!list_empty(&ch->list)) | |
188 | list_del(ch->list.next); | |
189 | ||
4de3a8fa CC |
190 | tegra_dma_stop(ch); |
191 | ||
192 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
193 | return 0; | |
194 | } | |
195 | ||
cb3732d0 LD |
196 | static unsigned int get_channel_status(struct tegra_dma_channel *ch, |
197 | struct tegra_dma_req *req, bool is_stop_dma) | |
198 | { | |
199 | void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); | |
200 | unsigned int status; | |
201 | ||
202 | if (is_stop_dma) { | |
203 | /* | |
204 | * STOP the DMA and get the transfer count. | |
205 | * Getting the transfer count is tricky. | |
206 | * - Globally disable DMA on all channels | |
207 | * - Read the channel's status register to know the number | |
208 | * of pending bytes to be transfered. | |
209 | * - Stop the dma channel | |
210 | * - Globally re-enable DMA to resume other transfers | |
211 | */ | |
212 | spin_lock(&enable_lock); | |
213 | writel(0, addr + APB_DMA_GEN); | |
214 | udelay(20); | |
215 | status = readl(ch->addr + APB_DMA_CHAN_STA); | |
216 | tegra_dma_stop(ch); | |
217 | writel(GEN_ENABLE, addr + APB_DMA_GEN); | |
218 | spin_unlock(&enable_lock); | |
219 | if (status & STA_ISE_EOC) { | |
220 | pr_err("Got Dma Int here clearing"); | |
221 | writel(status, ch->addr + APB_DMA_CHAN_STA); | |
222 | } | |
223 | req->status = TEGRA_DMA_REQ_ERROR_ABORTED; | |
224 | } else { | |
225 | status = readl(ch->addr + APB_DMA_CHAN_STA); | |
226 | } | |
227 | return status; | |
228 | } | |
229 | ||
230 | /* should be called with the channel lock held */ | |
231 | static unsigned int dma_active_count(struct tegra_dma_channel *ch, | |
232 | struct tegra_dma_req *req, unsigned int status) | |
233 | { | |
234 | unsigned int to_transfer; | |
235 | unsigned int req_transfer_count; | |
236 | unsigned int bytes_transferred; | |
237 | ||
238 | to_transfer = ((status & STA_COUNT_MASK) >> STA_COUNT_SHIFT) + 1; | |
239 | req_transfer_count = ch->req_transfer_count + 1; | |
240 | bytes_transferred = req_transfer_count; | |
241 | if (status & STA_BUSY) | |
242 | bytes_transferred -= to_transfer; | |
243 | /* | |
244 | * In continuous transfer mode, DMA only tracks the count of the | |
245 | * half DMA buffer. So, if the DMA already finished half the DMA | |
246 | * then add the half buffer to the completed count. | |
247 | */ | |
248 | if (ch->mode & TEGRA_DMA_MODE_CONTINOUS) { | |
249 | if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) | |
250 | bytes_transferred += req_transfer_count; | |
251 | if (status & STA_ISE_EOC) | |
252 | bytes_transferred += req_transfer_count; | |
253 | } | |
254 | bytes_transferred *= 4; | |
255 | return bytes_transferred; | |
256 | } | |
257 | ||
4de3a8fa CC |
258 | int tegra_dma_dequeue_req(struct tegra_dma_channel *ch, |
259 | struct tegra_dma_req *_req) | |
260 | { | |
4de3a8fa CC |
261 | unsigned int status; |
262 | struct tegra_dma_req *req = NULL; | |
263 | int found = 0; | |
264 | unsigned long irq_flags; | |
cb3732d0 | 265 | int stop = 0; |
4de3a8fa CC |
266 | |
267 | spin_lock_irqsave(&ch->lock, irq_flags); | |
cb3732d0 LD |
268 | |
269 | if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req) | |
270 | stop = 1; | |
271 | ||
4de3a8fa CC |
272 | list_for_each_entry(req, &ch->list, node) { |
273 | if (req == _req) { | |
274 | list_del(&req->node); | |
275 | found = 1; | |
276 | break; | |
277 | } | |
278 | } | |
279 | if (!found) { | |
280 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
281 | return 0; | |
282 | } | |
283 | ||
cb3732d0 LD |
284 | if (!stop) |
285 | goto skip_stop_dma; | |
4de3a8fa | 286 | |
cb3732d0 LD |
287 | status = get_channel_status(ch, req, true); |
288 | req->bytes_transferred = dma_active_count(ch, req, status); | |
4de3a8fa | 289 | |
4de3a8fa CC |
290 | if (!list_empty(&ch->list)) { |
291 | /* if the list is not empty, queue the next request */ | |
292 | struct tegra_dma_req *next_req; | |
293 | next_req = list_entry(ch->list.next, | |
294 | typeof(*next_req), node); | |
295 | tegra_dma_update_hw(ch, next_req); | |
296 | } | |
cb3732d0 LD |
297 | |
298 | skip_stop_dma: | |
4de3a8fa CC |
299 | req->status = -TEGRA_DMA_REQ_ERROR_ABORTED; |
300 | ||
301 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
302 | ||
303 | /* Callback should be called without any lock */ | |
304 | req->complete(req); | |
305 | return 0; | |
306 | } | |
307 | EXPORT_SYMBOL(tegra_dma_dequeue_req); | |
308 | ||
309 | bool tegra_dma_is_empty(struct tegra_dma_channel *ch) | |
310 | { | |
311 | unsigned long irq_flags; | |
312 | bool is_empty; | |
313 | ||
314 | spin_lock_irqsave(&ch->lock, irq_flags); | |
315 | if (list_empty(&ch->list)) | |
316 | is_empty = true; | |
317 | else | |
318 | is_empty = false; | |
319 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
320 | return is_empty; | |
321 | } | |
322 | EXPORT_SYMBOL(tegra_dma_is_empty); | |
323 | ||
324 | bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch, | |
325 | struct tegra_dma_req *_req) | |
326 | { | |
327 | unsigned long irq_flags; | |
328 | struct tegra_dma_req *req; | |
329 | ||
330 | spin_lock_irqsave(&ch->lock, irq_flags); | |
331 | list_for_each_entry(req, &ch->list, node) { | |
332 | if (req == _req) { | |
333 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
334 | return true; | |
335 | } | |
336 | } | |
337 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
338 | return false; | |
339 | } | |
340 | EXPORT_SYMBOL(tegra_dma_is_req_inflight); | |
341 | ||
342 | int tegra_dma_enqueue_req(struct tegra_dma_channel *ch, | |
343 | struct tegra_dma_req *req) | |
344 | { | |
345 | unsigned long irq_flags; | |
499ef7a5 | 346 | struct tegra_dma_req *_req; |
4de3a8fa CC |
347 | int start_dma = 0; |
348 | ||
349 | if (req->size > NV_DMA_MAX_TRASFER_SIZE || | |
350 | req->source_addr & 0x3 || req->dest_addr & 0x3) { | |
351 | pr_err("Invalid DMA request for channel %d\n", ch->id); | |
352 | return -EINVAL; | |
353 | } | |
354 | ||
355 | spin_lock_irqsave(&ch->lock, irq_flags); | |
356 | ||
499ef7a5 SW |
357 | list_for_each_entry(_req, &ch->list, node) { |
358 | if (req == _req) { | |
359 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
360 | return -EEXIST; | |
361 | } | |
362 | } | |
363 | ||
4de3a8fa CC |
364 | req->bytes_transferred = 0; |
365 | req->status = 0; | |
366 | req->buffer_status = 0; | |
367 | if (list_empty(&ch->list)) | |
368 | start_dma = 1; | |
369 | ||
370 | list_add_tail(&req->node, &ch->list); | |
371 | ||
372 | if (start_dma) | |
373 | tegra_dma_update_hw(ch, req); | |
374 | ||
375 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
376 | ||
377 | return 0; | |
378 | } | |
379 | EXPORT_SYMBOL(tegra_dma_enqueue_req); | |
380 | ||
381 | struct tegra_dma_channel *tegra_dma_allocate_channel(int mode) | |
382 | { | |
383 | int channel; | |
5789fee9 CC |
384 | struct tegra_dma_channel *ch = NULL; |
385 | ||
13ae3d5b | 386 | if (!tegra_dma_initialized) |
ccac0515 SW |
387 | return NULL; |
388 | ||
5789fee9 | 389 | mutex_lock(&tegra_dma_lock); |
4de3a8fa CC |
390 | |
391 | /* first channel is the shared channel */ | |
392 | if (mode & TEGRA_DMA_SHARED) { | |
393 | channel = TEGRA_SYSTEM_DMA_CH_MIN; | |
394 | } else { | |
395 | channel = find_first_zero_bit(channel_usage, | |
396 | ARRAY_SIZE(dma_channels)); | |
397 | if (channel >= ARRAY_SIZE(dma_channels)) | |
5789fee9 | 398 | goto out; |
4de3a8fa CC |
399 | } |
400 | __set_bit(channel, channel_usage); | |
401 | ch = &dma_channels[channel]; | |
402 | ch->mode = mode; | |
5789fee9 CC |
403 | |
404 | out: | |
405 | mutex_unlock(&tegra_dma_lock); | |
4de3a8fa CC |
406 | return ch; |
407 | } | |
408 | EXPORT_SYMBOL(tegra_dma_allocate_channel); | |
409 | ||
410 | void tegra_dma_free_channel(struct tegra_dma_channel *ch) | |
411 | { | |
412 | if (ch->mode & TEGRA_DMA_SHARED) | |
413 | return; | |
414 | tegra_dma_cancel(ch); | |
5789fee9 | 415 | mutex_lock(&tegra_dma_lock); |
4de3a8fa | 416 | __clear_bit(ch->id, channel_usage); |
5789fee9 | 417 | mutex_unlock(&tegra_dma_lock); |
4de3a8fa CC |
418 | } |
419 | EXPORT_SYMBOL(tegra_dma_free_channel); | |
420 | ||
421 | static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch, | |
422 | struct tegra_dma_req *req) | |
423 | { | |
5789fee9 CC |
424 | u32 apb_ptr; |
425 | u32 ahb_ptr; | |
426 | ||
4de3a8fa | 427 | if (req->to_memory) { |
5789fee9 CC |
428 | apb_ptr = req->source_addr; |
429 | ahb_ptr = req->dest_addr; | |
4de3a8fa | 430 | } else { |
5789fee9 CC |
431 | apb_ptr = req->dest_addr; |
432 | ahb_ptr = req->source_addr; | |
4de3a8fa | 433 | } |
5789fee9 CC |
434 | writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR); |
435 | writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR); | |
4de3a8fa CC |
436 | |
437 | req->status = TEGRA_DMA_REQ_INFLIGHT; | |
438 | return; | |
439 | } | |
440 | ||
441 | static void tegra_dma_update_hw(struct tegra_dma_channel *ch, | |
442 | struct tegra_dma_req *req) | |
443 | { | |
444 | int ahb_addr_wrap; | |
445 | int apb_addr_wrap; | |
446 | int ahb_bus_width; | |
447 | int apb_bus_width; | |
448 | int index; | |
4de3a8fa | 449 | |
5789fee9 CC |
450 | u32 ahb_seq; |
451 | u32 apb_seq; | |
452 | u32 ahb_ptr; | |
453 | u32 apb_ptr; | |
454 | u32 csr; | |
455 | ||
456 | csr = CSR_IE_EOC | CSR_FLOW; | |
457 | ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1; | |
458 | apb_seq = 0; | |
4de3a8fa | 459 | |
5789fee9 | 460 | csr |= req->req_sel << CSR_REQ_SEL_SHIFT; |
4de3a8fa CC |
461 | |
462 | /* One shot mode is always single buffered, | |
463 | * continuous mode is always double buffered | |
464 | * */ | |
465 | if (ch->mode & TEGRA_DMA_MODE_ONESHOT) { | |
5789fee9 CC |
466 | csr |= CSR_ONCE; |
467 | ch->req_transfer_count = (req->size >> 2) - 1; | |
4de3a8fa | 468 | } else { |
5789fee9 | 469 | ahb_seq |= AHB_SEQ_DBL_BUF; |
4de3a8fa CC |
470 | |
471 | /* In double buffered mode, we set the size to half the | |
472 | * requested size and interrupt when half the buffer | |
473 | * is full */ | |
5789fee9 | 474 | ch->req_transfer_count = (req->size >> 3) - 1; |
4de3a8fa CC |
475 | } |
476 | ||
5789fee9 CC |
477 | csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT; |
478 | ||
4de3a8fa | 479 | if (req->to_memory) { |
5789fee9 CC |
480 | apb_ptr = req->source_addr; |
481 | ahb_ptr = req->dest_addr; | |
4de3a8fa CC |
482 | |
483 | apb_addr_wrap = req->source_wrap; | |
484 | ahb_addr_wrap = req->dest_wrap; | |
485 | apb_bus_width = req->source_bus_width; | |
486 | ahb_bus_width = req->dest_bus_width; | |
487 | ||
488 | } else { | |
5789fee9 CC |
489 | csr |= CSR_DIR; |
490 | apb_ptr = req->dest_addr; | |
491 | ahb_ptr = req->source_addr; | |
4de3a8fa CC |
492 | |
493 | apb_addr_wrap = req->dest_wrap; | |
494 | ahb_addr_wrap = req->source_wrap; | |
495 | apb_bus_width = req->dest_bus_width; | |
496 | ahb_bus_width = req->source_bus_width; | |
497 | } | |
498 | ||
499 | apb_addr_wrap >>= 2; | |
500 | ahb_addr_wrap >>= 2; | |
501 | ||
502 | /* set address wrap for APB size */ | |
503 | index = 0; | |
504 | do { | |
505 | if (apb_addr_wrap_table[index] == apb_addr_wrap) | |
506 | break; | |
507 | index++; | |
508 | } while (index < ARRAY_SIZE(apb_addr_wrap_table)); | |
509 | BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table)); | |
5789fee9 | 510 | apb_seq |= index << APB_SEQ_WRAP_SHIFT; |
4de3a8fa CC |
511 | |
512 | /* set address wrap for AHB size */ | |
513 | index = 0; | |
514 | do { | |
515 | if (ahb_addr_wrap_table[index] == ahb_addr_wrap) | |
516 | break; | |
517 | index++; | |
518 | } while (index < ARRAY_SIZE(ahb_addr_wrap_table)); | |
519 | BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table)); | |
5789fee9 | 520 | ahb_seq |= index << AHB_SEQ_WRAP_SHIFT; |
4de3a8fa CC |
521 | |
522 | for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) { | |
523 | if (bus_width_table[index] == ahb_bus_width) | |
524 | break; | |
525 | } | |
526 | BUG_ON(index == ARRAY_SIZE(bus_width_table)); | |
5789fee9 | 527 | ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT; |
4de3a8fa CC |
528 | |
529 | for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) { | |
530 | if (bus_width_table[index] == apb_bus_width) | |
531 | break; | |
532 | } | |
533 | BUG_ON(index == ARRAY_SIZE(bus_width_table)); | |
5789fee9 | 534 | apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT; |
4de3a8fa | 535 | |
5789fee9 CC |
536 | writel(csr, ch->addr + APB_DMA_CHAN_CSR); |
537 | writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ); | |
538 | writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR); | |
539 | writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ); | |
540 | writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR); | |
4de3a8fa | 541 | |
5789fee9 | 542 | csr |= CSR_ENB; |
4de3a8fa CC |
543 | writel(csr, ch->addr + APB_DMA_CHAN_CSR); |
544 | ||
545 | req->status = TEGRA_DMA_REQ_INFLIGHT; | |
546 | } | |
547 | ||
4de3a8fa CC |
548 | static void handle_oneshot_dma(struct tegra_dma_channel *ch) |
549 | { | |
550 | struct tegra_dma_req *req; | |
5789fee9 | 551 | unsigned long irq_flags; |
4de3a8fa | 552 | |
5789fee9 | 553 | spin_lock_irqsave(&ch->lock, irq_flags); |
4de3a8fa | 554 | if (list_empty(&ch->list)) { |
5789fee9 | 555 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
556 | return; |
557 | } | |
558 | ||
559 | req = list_entry(ch->list.next, typeof(*req), node); | |
560 | if (req) { | |
561 | int bytes_transferred; | |
562 | ||
5789fee9 | 563 | bytes_transferred = ch->req_transfer_count; |
4de3a8fa CC |
564 | bytes_transferred += 1; |
565 | bytes_transferred <<= 2; | |
566 | ||
567 | list_del(&req->node); | |
568 | req->bytes_transferred = bytes_transferred; | |
569 | req->status = TEGRA_DMA_REQ_SUCCESS; | |
570 | ||
5789fee9 | 571 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
572 | /* Callback should be called without any lock */ |
573 | pr_debug("%s: transferred %d bytes\n", __func__, | |
574 | req->bytes_transferred); | |
575 | req->complete(req); | |
5789fee9 | 576 | spin_lock_irqsave(&ch->lock, irq_flags); |
4de3a8fa CC |
577 | } |
578 | ||
579 | if (!list_empty(&ch->list)) { | |
580 | req = list_entry(ch->list.next, typeof(*req), node); | |
581 | /* the complete function we just called may have enqueued | |
582 | another req, in which case dma has already started */ | |
583 | if (req->status != TEGRA_DMA_REQ_INFLIGHT) | |
584 | tegra_dma_update_hw(ch, req); | |
585 | } | |
5789fee9 | 586 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
587 | } |
588 | ||
589 | static void handle_continuous_dma(struct tegra_dma_channel *ch) | |
590 | { | |
591 | struct tegra_dma_req *req; | |
5789fee9 | 592 | unsigned long irq_flags; |
4de3a8fa | 593 | |
5789fee9 | 594 | spin_lock_irqsave(&ch->lock, irq_flags); |
4de3a8fa | 595 | if (list_empty(&ch->list)) { |
5789fee9 | 596 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
597 | return; |
598 | } | |
599 | ||
600 | req = list_entry(ch->list.next, typeof(*req), node); | |
601 | if (req) { | |
602 | if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) { | |
5789fee9 CC |
603 | bool is_dma_ping_complete; |
604 | is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA) | |
605 | & STA_PING_PONG) ? true : false; | |
606 | if (req->to_memory) | |
607 | is_dma_ping_complete = !is_dma_ping_complete; | |
608 | /* Out of sync - Release current buffer */ | |
609 | if (!is_dma_ping_complete) { | |
610 | int bytes_transferred; | |
611 | ||
612 | bytes_transferred = ch->req_transfer_count; | |
613 | bytes_transferred += 1; | |
614 | bytes_transferred <<= 3; | |
615 | req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL; | |
616 | req->bytes_transferred = bytes_transferred; | |
617 | req->status = TEGRA_DMA_REQ_SUCCESS; | |
618 | tegra_dma_stop(ch); | |
619 | ||
620 | if (!list_is_last(&req->node, &ch->list)) { | |
621 | struct tegra_dma_req *next_req; | |
622 | ||
623 | next_req = list_entry(req->node.next, | |
624 | typeof(*next_req), node); | |
625 | tegra_dma_update_hw(ch, next_req); | |
626 | } | |
627 | ||
628 | list_del(&req->node); | |
629 | ||
630 | /* DMA lock is NOT held when callbak is called */ | |
631 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
632 | req->complete(req); | |
633 | return; | |
634 | } | |
4de3a8fa CC |
635 | /* Load the next request into the hardware, if available |
636 | * */ | |
637 | if (!list_is_last(&req->node, &ch->list)) { | |
638 | struct tegra_dma_req *next_req; | |
639 | ||
640 | next_req = list_entry(req->node.next, | |
641 | typeof(*next_req), node); | |
642 | tegra_dma_update_hw_partial(ch, next_req); | |
643 | } | |
644 | req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL; | |
645 | req->status = TEGRA_DMA_REQ_SUCCESS; | |
646 | /* DMA lock is NOT held when callback is called */ | |
5789fee9 | 647 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
648 | if (likely(req->threshold)) |
649 | req->threshold(req); | |
650 | return; | |
651 | ||
652 | } else if (req->buffer_status == | |
653 | TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) { | |
654 | /* Callback when the buffer is completely full (i.e on | |
655 | * the second interrupt */ | |
656 | int bytes_transferred; | |
657 | ||
5789fee9 | 658 | bytes_transferred = ch->req_transfer_count; |
4de3a8fa CC |
659 | bytes_transferred += 1; |
660 | bytes_transferred <<= 3; | |
661 | ||
662 | req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL; | |
663 | req->bytes_transferred = bytes_transferred; | |
664 | req->status = TEGRA_DMA_REQ_SUCCESS; | |
665 | list_del(&req->node); | |
666 | ||
667 | /* DMA lock is NOT held when callbak is called */ | |
5789fee9 | 668 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
669 | req->complete(req); |
670 | return; | |
671 | ||
672 | } else { | |
673 | BUG(); | |
674 | } | |
675 | } | |
5789fee9 | 676 | spin_unlock_irqrestore(&ch->lock, irq_flags); |
4de3a8fa CC |
677 | } |
678 | ||
679 | static irqreturn_t dma_isr(int irq, void *data) | |
680 | { | |
681 | struct tegra_dma_channel *ch = data; | |
682 | unsigned long status; | |
683 | ||
684 | status = readl(ch->addr + APB_DMA_CHAN_STA); | |
685 | if (status & STA_ISE_EOC) | |
686 | writel(status, ch->addr + APB_DMA_CHAN_STA); | |
687 | else { | |
688 | pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id); | |
689 | return IRQ_HANDLED; | |
690 | } | |
691 | return IRQ_WAKE_THREAD; | |
692 | } | |
693 | ||
694 | static irqreturn_t dma_thread_fn(int irq, void *data) | |
695 | { | |
696 | struct tegra_dma_channel *ch = data; | |
697 | ||
698 | if (ch->mode & TEGRA_DMA_MODE_ONESHOT) | |
699 | handle_oneshot_dma(ch); | |
700 | else | |
701 | handle_continuous_dma(ch); | |
702 | ||
703 | ||
704 | return IRQ_HANDLED; | |
705 | } | |
706 | ||
707 | int __init tegra_dma_init(void) | |
708 | { | |
709 | int ret = 0; | |
710 | int i; | |
711 | unsigned int irq; | |
712 | void __iomem *addr; | |
1ca00347 SW |
713 | struct clk *c; |
714 | ||
ccac0515 SW |
715 | bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS); |
716 | ||
72d967ed | 717 | c = clk_get_sys("tegra-apbdma", NULL); |
1ca00347 SW |
718 | if (IS_ERR(c)) { |
719 | pr_err("Unable to get clock for APB DMA\n"); | |
720 | ret = PTR_ERR(c); | |
721 | goto fail; | |
722 | } | |
6a5278d0 | 723 | ret = clk_prepare_enable(c); |
1ca00347 SW |
724 | if (ret != 0) { |
725 | pr_err("Unable to enable clock for APB DMA\n"); | |
726 | goto fail; | |
727 | } | |
4de3a8fa CC |
728 | |
729 | addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); | |
730 | writel(GEN_ENABLE, addr + APB_DMA_GEN); | |
731 | writel(0, addr + APB_DMA_CNTRL); | |
732 | writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX), | |
733 | addr + APB_DMA_IRQ_MASK_SET); | |
734 | ||
4de3a8fa CC |
735 | for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) { |
736 | struct tegra_dma_channel *ch = &dma_channels[i]; | |
737 | ||
4de3a8fa CC |
738 | ch->id = i; |
739 | snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i); | |
740 | ||
741 | ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE + | |
742 | TEGRA_APB_DMA_CH0_SIZE * i); | |
743 | ||
744 | spin_lock_init(&ch->lock); | |
745 | INIT_LIST_HEAD(&ch->list); | |
4de3a8fa CC |
746 | |
747 | irq = INT_APB_DMA_CH0 + i; | |
748 | ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0, | |
749 | dma_channels[i].name, ch); | |
750 | if (ret) { | |
751 | pr_err("Failed to register IRQ %d for DMA %d\n", | |
752 | irq, i); | |
753 | goto fail; | |
754 | } | |
755 | ch->irq = irq; | |
ccac0515 SW |
756 | |
757 | __clear_bit(i, channel_usage); | |
4de3a8fa CC |
758 | } |
759 | /* mark the shared channel allocated */ | |
760 | __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage); | |
761 | ||
ccac0515 | 762 | tegra_dma_initialized = true; |
4de3a8fa | 763 | |
ccac0515 | 764 | return 0; |
4de3a8fa CC |
765 | fail: |
766 | writel(0, addr + APB_DMA_GEN); | |
767 | for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) { | |
768 | struct tegra_dma_channel *ch = &dma_channels[i]; | |
769 | if (ch->irq) | |
770 | free_irq(ch->irq, ch); | |
771 | } | |
772 | return ret; | |
773 | } | |
dc54c23b | 774 | postcore_initcall(tegra_dma_init); |
4de3a8fa CC |
775 | |
776 | #ifdef CONFIG_PM | |
777 | static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3]; | |
778 | ||
779 | void tegra_dma_suspend(void) | |
780 | { | |
781 | void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); | |
782 | u32 *ctx = apb_dma; | |
783 | int i; | |
784 | ||
785 | *ctx++ = readl(addr + APB_DMA_GEN); | |
786 | *ctx++ = readl(addr + APB_DMA_CNTRL); | |
787 | *ctx++ = readl(addr + APB_DMA_IRQ_MASK); | |
788 | ||
789 | for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) { | |
790 | addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE + | |
791 | TEGRA_APB_DMA_CH0_SIZE * i); | |
792 | ||
793 | *ctx++ = readl(addr + APB_DMA_CHAN_CSR); | |
794 | *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR); | |
795 | *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ); | |
796 | *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR); | |
797 | *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ); | |
798 | } | |
799 | } | |
800 | ||
801 | void tegra_dma_resume(void) | |
802 | { | |
803 | void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); | |
804 | u32 *ctx = apb_dma; | |
805 | int i; | |
806 | ||
807 | writel(*ctx++, addr + APB_DMA_GEN); | |
808 | writel(*ctx++, addr + APB_DMA_CNTRL); | |
809 | writel(*ctx++, addr + APB_DMA_IRQ_MASK); | |
810 | ||
811 | for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) { | |
812 | addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE + | |
813 | TEGRA_APB_DMA_CH0_SIZE * i); | |
814 | ||
815 | writel(*ctx++, addr + APB_DMA_CHAN_CSR); | |
816 | writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR); | |
817 | writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ); | |
818 | writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR); | |
819 | writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ); | |
820 | } | |
821 | } | |
822 | ||
823 | #endif |