Commit | Line | Data |
---|---|---|
1394f032 | 1 | /* |
dd3dd384 | 2 | * bfin_dma_5xx.c - Blackfin DMA implementation |
1394f032 | 3 | * |
9c417a43 | 4 | * Copyright 2004-2008 Analog Devices Inc. |
dd3dd384 | 5 | * Licensed under the GPL-2 or later. |
1394f032 BW |
6 | */ |
7 | ||
8 | #include <linux/errno.h> | |
dd3dd384 MF |
9 | #include <linux/interrupt.h> |
10 | #include <linux/kernel.h> | |
1394f032 | 11 | #include <linux/module.h> |
dd3dd384 | 12 | #include <linux/param.h> |
d642a8ad | 13 | #include <linux/proc_fs.h> |
1394f032 | 14 | #include <linux/sched.h> |
d642a8ad | 15 | #include <linux/seq_file.h> |
dd3dd384 | 16 | #include <linux/spinlock.h> |
1394f032 | 17 | |
24a07a12 | 18 | #include <asm/blackfin.h> |
1394f032 | 19 | #include <asm/cacheflush.h> |
dd3dd384 MF |
20 | #include <asm/dma.h> |
21 | #include <asm/uaccess.h> | |
837ec2d5 | 22 | #include <asm/early_printk.h> |
1394f032 | 23 | |
76068c3c RG |
24 | /* |
25 | * To make sure we work around 05000119 - we always check DMA_DONE bit, | |
26 | * never the DMA_RUN bit | |
27 | */ | |
28 | ||
9c417a43 MF |
29 | struct dma_channel dma_ch[MAX_DMA_CHANNELS]; |
30 | EXPORT_SYMBOL(dma_ch); | |
1394f032 | 31 | |
a161bb05 | 32 | static int __init blackfin_dma_init(void) |
1394f032 BW |
33 | { |
34 | int i; | |
35 | ||
36 | printk(KERN_INFO "Blackfin DMA Controller\n"); | |
37 | ||
211daf9d | 38 | for (i = 0; i < MAX_DMA_CHANNELS; i++) { |
1394f032 | 39 | dma_ch[i].chan_status = DMA_CHANNEL_FREE; |
77955664 | 40 | dma_ch[i].regs = dma_io_base_addr[i]; |
1394f032 BW |
41 | mutex_init(&(dma_ch[i].dmalock)); |
42 | } | |
23ee968d | 43 | /* Mark MEMDMA Channel 0 as requested since we're using it internally */ |
d642a8ad GY |
44 | request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy"); |
45 | request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy"); | |
a924db7c MH |
46 | |
47 | #if defined(CONFIG_DEB_DMA_URGENT) | |
48 | bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE() | |
49 | | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT); | |
50 | #endif | |
d642a8ad | 51 | |
1394f032 BW |
52 | return 0; |
53 | } | |
1394f032 BW |
54 | arch_initcall(blackfin_dma_init); |
55 | ||
d642a8ad | 56 | #ifdef CONFIG_PROC_FS |
d642a8ad GY |
57 | static int proc_dma_show(struct seq_file *m, void *v) |
58 | { | |
59 | int i; | |
60 | ||
dd3dd384 | 61 | for (i = 0; i < MAX_DMA_CHANNELS; ++i) |
d642a8ad GY |
62 | if (dma_ch[i].chan_status != DMA_CHANNEL_FREE) |
63 | seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id); | |
64 | ||
65 | return 0; | |
66 | } | |
67 | ||
68 | static int proc_dma_open(struct inode *inode, struct file *file) | |
69 | { | |
70 | return single_open(file, proc_dma_show, NULL); | |
71 | } | |
72 | ||
73 | static const struct file_operations proc_dma_operations = { | |
74 | .open = proc_dma_open, | |
75 | .read = seq_read, | |
76 | .llseek = seq_lseek, | |
77 | .release = single_release, | |
78 | }; | |
79 | ||
80 | static int __init proc_dma_init(void) | |
81 | { | |
82 | return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL; | |
83 | } | |
84 | late_initcall(proc_dma_init); | |
85 | #endif | |
86 | ||
9c417a43 MF |
87 | /** |
88 | * request_dma - request a DMA channel | |
89 | * | |
90 | * Request the specific DMA channel from the system if it's available. | |
91 | */ | |
99532fd2 | 92 | int request_dma(unsigned int channel, const char *device_id) |
1394f032 | 93 | { |
1394f032 | 94 | pr_debug("request_dma() : BEGIN \n"); |
d642a8ad GY |
95 | |
96 | if (device_id == NULL) | |
97 | printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel); | |
5ce998cf MH |
98 | |
99 | #if defined(CONFIG_BF561) && ANOMALY_05000182 | |
100 | if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) { | |
101 | if (get_cclk() > 500000000) { | |
102 | printk(KERN_WARNING | |
103 | "Request IMDMA failed due to ANOMALY 05000182\n"); | |
104 | return -EFAULT; | |
105 | } | |
106 | } | |
107 | #endif | |
108 | ||
1394f032 BW |
109 | mutex_lock(&(dma_ch[channel].dmalock)); |
110 | ||
111 | if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED) | |
112 | || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) { | |
113 | mutex_unlock(&(dma_ch[channel].dmalock)); | |
114 | pr_debug("DMA CHANNEL IN USE \n"); | |
115 | return -EBUSY; | |
116 | } else { | |
117 | dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED; | |
118 | pr_debug("DMA CHANNEL IS ALLOCATED \n"); | |
119 | } | |
120 | ||
121 | mutex_unlock(&(dma_ch[channel].dmalock)); | |
122 | ||
8b01eaff | 123 | #ifdef CONFIG_BF54x |
549aaa84 | 124 | if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) { |
ab2375f2 SZ |
125 | unsigned int per_map; |
126 | per_map = dma_ch[channel].regs->peripheral_map & 0xFFF; | |
127 | if (strncmp(device_id, "BFIN_UART", 9) == 0) | |
128 | dma_ch[channel].regs->peripheral_map = per_map | | |
5be36d22 | 129 | ((channel - CH_UART2_RX + 0xC)<<12); |
ab2375f2 SZ |
130 | else |
131 | dma_ch[channel].regs->peripheral_map = per_map | | |
5be36d22 | 132 | ((channel - CH_UART2_RX + 0x6)<<12); |
549aaa84 | 133 | } |
8b01eaff SZ |
134 | #endif |
135 | ||
1394f032 | 136 | dma_ch[channel].device_id = device_id; |
9b011407 | 137 | dma_ch[channel].irq = 0; |
1394f032 BW |
138 | |
139 | /* This is to be enabled by putting a restriction - | |
140 | * you have to request DMA, before doing any operations on | |
141 | * descriptor/channel | |
142 | */ | |
143 | pr_debug("request_dma() : END \n"); | |
596b565b | 144 | return 0; |
1394f032 BW |
145 | } |
146 | EXPORT_SYMBOL(request_dma); | |
147 | ||
68532bda | 148 | int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data) |
1394f032 | 149 | { |
ac860751 RK |
150 | BUG_ON(channel >= MAX_DMA_CHANNELS || |
151 | dma_ch[channel].chan_status == DMA_CHANNEL_FREE); | |
1394f032 BW |
152 | |
153 | if (callback != NULL) { | |
8f1cc233 MF |
154 | int ret; |
155 | unsigned int irq = channel2irq(channel); | |
1394f032 | 156 | |
8f1cc233 MF |
157 | ret = request_irq(irq, callback, IRQF_DISABLED, |
158 | dma_ch[channel].device_id, data); | |
159 | if (ret) | |
160 | return ret; | |
161 | ||
162 | dma_ch[channel].irq = irq; | |
163 | dma_ch[channel].data = data; | |
1394f032 BW |
164 | } |
165 | return 0; | |
166 | } | |
167 | EXPORT_SYMBOL(set_dma_callback); | |
168 | ||
9c417a43 MF |
169 | /** |
170 | * clear_dma_buffer - clear DMA fifos for specified channel | |
171 | * | |
172 | * Set the Buffer Clear bit in the Configuration register of specific DMA | |
173 | * channel. This will stop the descriptor based DMA operation. | |
174 | */ | |
175 | static void clear_dma_buffer(unsigned int channel) | |
176 | { | |
177 | dma_ch[channel].regs->cfg |= RESTART; | |
178 | SSYNC(); | |
179 | dma_ch[channel].regs->cfg &= ~RESTART; | |
180 | } | |
181 | ||
1394f032 BW |
182 | void free_dma(unsigned int channel) |
183 | { | |
1394f032 | 184 | pr_debug("freedma() : BEGIN \n"); |
ac860751 RK |
185 | BUG_ON(channel >= MAX_DMA_CHANNELS || |
186 | dma_ch[channel].chan_status == DMA_CHANNEL_FREE); | |
1394f032 BW |
187 | |
188 | /* Halt the DMA */ | |
189 | disable_dma(channel); | |
190 | clear_dma_buffer(channel); | |
191 | ||
9b011407 | 192 | if (dma_ch[channel].irq) |
a2ba8b19 | 193 | free_irq(dma_ch[channel].irq, dma_ch[channel].data); |
1394f032 BW |
194 | |
195 | /* Clear the DMA Variable in the Channel */ | |
196 | mutex_lock(&(dma_ch[channel].dmalock)); | |
197 | dma_ch[channel].chan_status = DMA_CHANNEL_FREE; | |
198 | mutex_unlock(&(dma_ch[channel].dmalock)); | |
199 | ||
200 | pr_debug("freedma() : END \n"); | |
201 | } | |
202 | EXPORT_SYMBOL(free_dma); | |
203 | ||
1efc80b5 | 204 | #ifdef CONFIG_PM |
c9e0020d MF |
205 | # ifndef MAX_DMA_SUSPEND_CHANNELS |
206 | # define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS | |
207 | # endif | |
1efc80b5 MH |
208 | int blackfin_dma_suspend(void) |
209 | { | |
210 | int i; | |
211 | ||
c9e0020d | 212 | for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) { |
1efc80b5 MH |
213 | if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) { |
214 | printk(KERN_ERR "DMA Channel %d failed to suspend\n", i); | |
215 | return -EBUSY; | |
216 | } | |
217 | ||
218 | dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map; | |
219 | } | |
220 | ||
221 | return 0; | |
222 | } | |
223 | ||
224 | void blackfin_dma_resume(void) | |
225 | { | |
226 | int i; | |
c9e0020d | 227 | for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) |
1efc80b5 MH |
228 | dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map; |
229 | } | |
230 | #endif | |
231 | ||
dd3dd384 MF |
232 | /** |
233 | * blackfin_dma_early_init - minimal DMA init | |
234 | * | |
235 | * Setup a few DMA registers so we can safely do DMA transfers early on in | |
236 | * the kernel booting process. Really this just means using dma_memcpy(). | |
237 | */ | |
238 | void __init blackfin_dma_early_init(void) | |
1394f032 | 239 | { |
837ec2d5 | 240 | early_shadow_stamp(); |
1394f032 | 241 | bfin_write_MDMA_S0_CONFIG(0); |
fecbd736 RG |
242 | bfin_write_MDMA_S1_CONFIG(0); |
243 | } | |
244 | ||
245 | void __init early_dma_memcpy(void *pdst, const void *psrc, size_t size) | |
246 | { | |
247 | unsigned long dst = (unsigned long)pdst; | |
248 | unsigned long src = (unsigned long)psrc; | |
249 | struct dma_register *dst_ch, *src_ch; | |
250 | ||
837ec2d5 RG |
251 | early_shadow_stamp(); |
252 | ||
fecbd736 RG |
253 | /* We assume that everything is 4 byte aligned, so include |
254 | * a basic sanity check | |
255 | */ | |
256 | BUG_ON(dst % 4); | |
257 | BUG_ON(src % 4); | |
258 | BUG_ON(size % 4); | |
259 | ||
fecbd736 RG |
260 | src_ch = 0; |
261 | /* Find an avalible memDMA channel */ | |
262 | while (1) { | |
532f07ca | 263 | if (src_ch == (struct dma_register *)MDMA_S0_NEXT_DESC_PTR) { |
fecbd736 RG |
264 | dst_ch = (struct dma_register *)MDMA_D1_NEXT_DESC_PTR; |
265 | src_ch = (struct dma_register *)MDMA_S1_NEXT_DESC_PTR; | |
532f07ca MF |
266 | } else { |
267 | dst_ch = (struct dma_register *)MDMA_D0_NEXT_DESC_PTR; | |
268 | src_ch = (struct dma_register *)MDMA_S0_NEXT_DESC_PTR; | |
fecbd736 RG |
269 | } |
270 | ||
532f07ca MF |
271 | if (!bfin_read16(&src_ch->cfg)) |
272 | break; | |
273 | else if (bfin_read16(&dst_ch->irq_status) & DMA_DONE) { | |
274 | bfin_write16(&src_ch->cfg, 0); | |
fecbd736 | 275 | break; |
fecbd736 | 276 | } |
fecbd736 RG |
277 | } |
278 | ||
532f07ca MF |
279 | /* Force a sync in case a previous config reset on this channel |
280 | * occurred. This is needed so subsequent writes to DMA registers | |
281 | * are not spuriously lost/corrupted. | |
282 | */ | |
283 | __builtin_bfin_ssync(); | |
284 | ||
fecbd736 RG |
285 | /* Destination */ |
286 | bfin_write32(&dst_ch->start_addr, dst); | |
287 | bfin_write16(&dst_ch->x_count, size >> 2); | |
288 | bfin_write16(&dst_ch->x_modify, 1 << 2); | |
289 | bfin_write16(&dst_ch->irq_status, DMA_DONE | DMA_ERR); | |
290 | ||
291 | /* Source */ | |
292 | bfin_write32(&src_ch->start_addr, src); | |
293 | bfin_write16(&src_ch->x_count, size >> 2); | |
294 | bfin_write16(&src_ch->x_modify, 1 << 2); | |
295 | bfin_write16(&src_ch->irq_status, DMA_DONE | DMA_ERR); | |
296 | ||
297 | /* Enable */ | |
298 | bfin_write16(&src_ch->cfg, DMAEN | WDSIZE_32); | |
299 | bfin_write16(&dst_ch->cfg, WNR | DI_EN | DMAEN | WDSIZE_32); | |
300 | ||
301 | /* Since we are atomic now, don't use the workaround ssync */ | |
302 | __builtin_bfin_ssync(); | |
303 | } | |
304 | ||
305 | void __init early_dma_memcpy_done(void) | |
306 | { | |
837ec2d5 RG |
307 | early_shadow_stamp(); |
308 | ||
fecbd736 RG |
309 | while ((bfin_read_MDMA_S0_CONFIG() && !(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) || |
310 | (bfin_read_MDMA_S1_CONFIG() && !(bfin_read_MDMA_D1_IRQ_STATUS() & DMA_DONE))) | |
311 | continue; | |
312 | ||
313 | bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR); | |
314 | bfin_write_MDMA_D1_IRQ_STATUS(DMA_DONE | DMA_ERR); | |
315 | /* | |
316 | * Now that DMA is done, we would normally flush cache, but | |
317 | * i/d cache isn't running this early, so we don't bother, | |
318 | * and just clear out the DMA channel for next time | |
319 | */ | |
320 | bfin_write_MDMA_S0_CONFIG(0); | |
321 | bfin_write_MDMA_S1_CONFIG(0); | |
322 | bfin_write_MDMA_D0_CONFIG(0); | |
323 | bfin_write_MDMA_D1_CONFIG(0); | |
324 | ||
325 | __builtin_bfin_ssync(); | |
1394f032 | 326 | } |
5f9a3e89 | 327 | |
49946e73 | 328 | /** |
dd3dd384 | 329 | * __dma_memcpy - program the MDMA registers |
49946e73 | 330 | * |
dd3dd384 MF |
331 | * Actually program MDMA0 and wait for the transfer to finish. Disable IRQs |
332 | * while programming registers so that everything is fully configured. Wait | |
333 | * for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE | |
334 | * check will make sure we don't clobber any existing transfer. | |
49946e73 | 335 | */ |
dd3dd384 | 336 | static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf) |
23ee968d | 337 | { |
dd3dd384 | 338 | static DEFINE_SPINLOCK(mdma_lock); |
23ee968d | 339 | unsigned long flags; |
1f83b8f1 | 340 | |
dd3dd384 MF |
341 | spin_lock_irqsave(&mdma_lock, flags); |
342 | ||
41245ac5 MF |
343 | /* Force a sync in case a previous config reset on this channel |
344 | * occurred. This is needed so subsequent writes to DMA registers | |
345 | * are not spuriously lost/corrupted. Do it under irq lock and | |
346 | * without the anomaly version (because we are atomic already). | |
347 | */ | |
348 | __builtin_bfin_ssync(); | |
349 | ||
dd3dd384 MF |
350 | if (bfin_read_MDMA_S0_CONFIG()) |
351 | while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) | |
352 | continue; | |
353 | ||
354 | if (conf & DMA2D) { | |
355 | /* For larger bit sizes, we've already divided down cnt so it | |
356 | * is no longer a multiple of 64k. So we have to break down | |
357 | * the limit here so it is a multiple of the incoming size. | |
358 | * There is no limitation here in terms of total size other | |
359 | * than the hardware though as the bits lost in the shift are | |
360 | * made up by MODIFY (== we can hit the whole address space). | |
361 | * X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4 | |
362 | */ | |
363 | u32 shift = abs(dmod) >> 1; | |
364 | size_t ycnt = cnt >> (16 - shift); | |
365 | cnt = 1 << (16 - shift); | |
366 | bfin_write_MDMA_D0_Y_COUNT(ycnt); | |
367 | bfin_write_MDMA_S0_Y_COUNT(ycnt); | |
368 | bfin_write_MDMA_D0_Y_MODIFY(dmod); | |
369 | bfin_write_MDMA_S0_Y_MODIFY(smod); | |
370 | } | |
1f83b8f1 | 371 | |
dd3dd384 MF |
372 | bfin_write_MDMA_D0_START_ADDR(daddr); |
373 | bfin_write_MDMA_D0_X_COUNT(cnt); | |
374 | bfin_write_MDMA_D0_X_MODIFY(dmod); | |
23ee968d MH |
375 | bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR); |
376 | ||
dd3dd384 MF |
377 | bfin_write_MDMA_S0_START_ADDR(saddr); |
378 | bfin_write_MDMA_S0_X_COUNT(cnt); | |
379 | bfin_write_MDMA_S0_X_MODIFY(smod); | |
23ee968d MH |
380 | bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR); |
381 | ||
dd3dd384 MF |
382 | bfin_write_MDMA_S0_CONFIG(DMAEN | conf); |
383 | bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | conf); | |
384 | ||
385 | spin_unlock_irqrestore(&mdma_lock, flags); | |
23ee968d | 386 | |
1a7d91d6 MH |
387 | SSYNC(); |
388 | ||
dd3dd384 MF |
389 | while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) |
390 | if (bfin_read_MDMA_S0_CONFIG()) | |
391 | continue; | |
392 | else | |
393 | return; | |
23ee968d MH |
394 | |
395 | bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR); | |
396 | ||
397 | bfin_write_MDMA_S0_CONFIG(0); | |
398 | bfin_write_MDMA_D0_CONFIG(0); | |
23ee968d | 399 | } |
23ee968d | 400 | |
dd3dd384 MF |
401 | /** |
402 | * _dma_memcpy - translate C memcpy settings into MDMA settings | |
403 | * | |
404 | * Handle all the high level steps before we touch the MDMA registers. So | |
7ad883a9 | 405 | * handle direction, tweaking of sizes, and formatting of addresses. |
dd3dd384 MF |
406 | */ |
407 | static void *_dma_memcpy(void *pdst, const void *psrc, size_t size) | |
23ee968d | 408 | { |
dd3dd384 MF |
409 | u32 conf, shift; |
410 | s16 mod; | |
411 | unsigned long dst = (unsigned long)pdst; | |
412 | unsigned long src = (unsigned long)psrc; | |
1f83b8f1 | 413 | |
dd3dd384 MF |
414 | if (size == 0) |
415 | return NULL; | |
1a7d91d6 | 416 | |
dd3dd384 MF |
417 | if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) { |
418 | conf = WDSIZE_32; | |
419 | shift = 2; | |
420 | } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) { | |
421 | conf = WDSIZE_16; | |
422 | shift = 1; | |
423 | } else { | |
424 | conf = WDSIZE_8; | |
425 | shift = 0; | |
426 | } | |
23ee968d | 427 | |
dd3dd384 MF |
428 | /* If the two memory regions have a chance of overlapping, make |
429 | * sure the memcpy still works as expected. Do this by having the | |
430 | * copy run backwards instead. | |
431 | */ | |
432 | mod = 1 << shift; | |
433 | if (src < dst) { | |
434 | mod *= -1; | |
435 | dst += size + mod; | |
436 | src += size + mod; | |
437 | } | |
438 | size >>= shift; | |
23ee968d | 439 | |
dd3dd384 MF |
440 | if (size > 0x10000) |
441 | conf |= DMA2D; | |
23ee968d | 442 | |
dd3dd384 | 443 | __dma_memcpy(dst, mod, src, mod, size, conf); |
23ee968d | 444 | |
dd3dd384 | 445 | return pdst; |
23ee968d | 446 | } |
23ee968d | 447 | |
dd3dd384 MF |
448 | /** |
449 | * dma_memcpy - DMA memcpy under mutex lock | |
450 | * | |
451 | * Do not check arguments before starting the DMA memcpy. Break the transfer | |
452 | * up into two pieces. The first transfer is in multiples of 64k and the | |
453 | * second transfer is the piece smaller than 64k. | |
454 | */ | |
7ad883a9 | 455 | void *dma_memcpy(void *pdst, const void *psrc, size_t size) |
23ee968d | 456 | { |
7ad883a9 MF |
457 | unsigned long dst = (unsigned long)pdst; |
458 | unsigned long src = (unsigned long)psrc; | |
dd3dd384 | 459 | size_t bulk, rest; |
7ad883a9 | 460 | |
67834fa9 | 461 | if (bfin_addr_dcacheable(src)) |
7ad883a9 MF |
462 | blackfin_dcache_flush_range(src, src + size); |
463 | ||
67834fa9 | 464 | if (bfin_addr_dcacheable(dst)) |
7ad883a9 MF |
465 | blackfin_dcache_invalidate_range(dst, dst + size); |
466 | ||
dd3dd384 MF |
467 | bulk = size & ~0xffff; |
468 | rest = size - bulk; | |
469 | if (bulk) | |
7ad883a9 MF |
470 | _dma_memcpy(pdst, psrc, bulk); |
471 | _dma_memcpy(pdst + bulk, psrc + bulk, rest); | |
472 | return pdst; | |
23ee968d | 473 | } |
dd3dd384 | 474 | EXPORT_SYMBOL(dma_memcpy); |
23ee968d | 475 | |
dd3dd384 MF |
476 | /** |
477 | * safe_dma_memcpy - DMA memcpy w/argument checking | |
478 | * | |
479 | * Verify arguments are safe before heading to dma_memcpy(). | |
480 | */ | |
481 | void *safe_dma_memcpy(void *dst, const void *src, size_t size) | |
23ee968d | 482 | { |
dd3dd384 MF |
483 | if (!access_ok(VERIFY_WRITE, dst, size)) |
484 | return NULL; | |
485 | if (!access_ok(VERIFY_READ, src, size)) | |
486 | return NULL; | |
487 | return dma_memcpy(dst, src, size); | |
23ee968d | 488 | } |
dd3dd384 | 489 | EXPORT_SYMBOL(safe_dma_memcpy); |
23ee968d | 490 | |
dd3dd384 MF |
491 | static void _dma_out(unsigned long addr, unsigned long buf, unsigned short len, |
492 | u16 size, u16 dma_size) | |
23ee968d | 493 | { |
dd3dd384 MF |
494 | blackfin_dcache_flush_range(buf, buf + len * size); |
495 | __dma_memcpy(addr, 0, buf, size, len, dma_size); | |
23ee968d | 496 | } |
23ee968d | 497 | |
dd3dd384 MF |
498 | static void _dma_in(unsigned long addr, unsigned long buf, unsigned short len, |
499 | u16 size, u16 dma_size) | |
23ee968d | 500 | { |
dd3dd384 MF |
501 | blackfin_dcache_invalidate_range(buf, buf + len * size); |
502 | __dma_memcpy(buf, size, addr, 0, len, dma_size); | |
23ee968d | 503 | } |
dd3dd384 MF |
504 | |
505 | #define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \ | |
506 | void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned short len) \ | |
507 | { \ | |
508 | _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \ | |
509 | } \ | |
510 | EXPORT_SYMBOL(dma_##io##s##bwl) | |
511 | MAKE_DMA_IO(out, b, 1, 8, const); | |
512 | MAKE_DMA_IO(in, b, 1, 8, ); | |
513 | MAKE_DMA_IO(out, w, 2, 16, const); | |
514 | MAKE_DMA_IO(in, w, 2, 16, ); | |
515 | MAKE_DMA_IO(out, l, 4, 32, const); | |
516 | MAKE_DMA_IO(in, l, 4, 32, ); |