Commit | Line | Data |
---|---|---|
4a776f0a HS |
1 | /* |
2 | * DMA Engine test module | |
3 | * | |
4 | * Copyright (C) 2007 Atmel Corporation | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | */ | |
10 | #include <linux/delay.h> | |
11 | #include <linux/dmaengine.h> | |
12 | #include <linux/init.h> | |
13 | #include <linux/kthread.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/moduleparam.h> | |
16 | #include <linux/random.h> | |
5a0e3ad6 | 17 | #include <linux/slab.h> |
4a776f0a HS |
18 | #include <linux/wait.h> |
19 | ||
20 | static unsigned int test_buf_size = 16384; | |
21 | module_param(test_buf_size, uint, S_IRUGO); | |
22 | MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); | |
23 | ||
06190d84 | 24 | static char test_channel[20]; |
4a776f0a HS |
25 | module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); |
26 | MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); | |
27 | ||
06190d84 | 28 | static char test_device[20]; |
4a776f0a HS |
29 | module_param_string(device, test_device, sizeof(test_device), S_IRUGO); |
30 | MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); | |
31 | ||
32 | static unsigned int threads_per_chan = 1; | |
33 | module_param(threads_per_chan, uint, S_IRUGO); | |
34 | MODULE_PARM_DESC(threads_per_chan, | |
35 | "Number of threads to start per channel (default: 1)"); | |
36 | ||
37 | static unsigned int max_channels; | |
38 | module_param(max_channels, uint, S_IRUGO); | |
33df8ca0 | 39 | MODULE_PARM_DESC(max_channels, |
4a776f0a HS |
40 | "Maximum number of channels to use (default: all)"); |
41 | ||
0a2ff57d NF |
42 | static unsigned int iterations; |
43 | module_param(iterations, uint, S_IRUGO); | |
44 | MODULE_PARM_DESC(iterations, | |
45 | "Iterations before stopping test (default: infinite)"); | |
46 | ||
b54d5cb9 DW |
47 | static unsigned int xor_sources = 3; |
48 | module_param(xor_sources, uint, S_IRUGO); | |
49 | MODULE_PARM_DESC(xor_sources, | |
50 | "Number of xor source buffers (default: 3)"); | |
51 | ||
58691d64 DW |
52 | static unsigned int pq_sources = 3; |
53 | module_param(pq_sources, uint, S_IRUGO); | |
54 | MODULE_PARM_DESC(pq_sources, | |
55 | "Number of p+q source buffers (default: 3)"); | |
56 | ||
4a776f0a HS |
57 | /* |
58 | * Initialization patterns. All bytes in the source buffer has bit 7 | |
59 | * set, all bytes in the destination buffer has bit 7 cleared. | |
60 | * | |
61 | * Bit 6 is set for all bytes which are to be copied by the DMA | |
62 | * engine. Bit 5 is set for all bytes which are to be overwritten by | |
63 | * the DMA engine. | |
64 | * | |
65 | * The remaining bits are the inverse of a counter which increments by | |
66 | * one for each byte address. | |
67 | */ | |
68 | #define PATTERN_SRC 0x80 | |
69 | #define PATTERN_DST 0x00 | |
70 | #define PATTERN_COPY 0x40 | |
71 | #define PATTERN_OVERWRITE 0x20 | |
72 | #define PATTERN_COUNT_MASK 0x1f | |
73 | ||
74 | struct dmatest_thread { | |
75 | struct list_head node; | |
76 | struct task_struct *task; | |
77 | struct dma_chan *chan; | |
b54d5cb9 DW |
78 | u8 **srcs; |
79 | u8 **dsts; | |
80 | enum dma_transaction_type type; | |
4a776f0a HS |
81 | }; |
82 | ||
83 | struct dmatest_chan { | |
84 | struct list_head node; | |
85 | struct dma_chan *chan; | |
86 | struct list_head threads; | |
87 | }; | |
88 | ||
89 | /* | |
90 | * These are protected by dma_list_mutex since they're only used by | |
33df8ca0 | 91 | * the DMA filter function callback |
4a776f0a HS |
92 | */ |
93 | static LIST_HEAD(dmatest_channels); | |
94 | static unsigned int nr_channels; | |
95 | ||
96 | static bool dmatest_match_channel(struct dma_chan *chan) | |
97 | { | |
98 | if (test_channel[0] == '\0') | |
99 | return true; | |
41d5e59c | 100 | return strcmp(dma_chan_name(chan), test_channel) == 0; |
4a776f0a HS |
101 | } |
102 | ||
103 | static bool dmatest_match_device(struct dma_device *device) | |
104 | { | |
105 | if (test_device[0] == '\0') | |
106 | return true; | |
06190d84 | 107 | return strcmp(dev_name(device->dev), test_device) == 0; |
4a776f0a HS |
108 | } |
109 | ||
110 | static unsigned long dmatest_random(void) | |
111 | { | |
112 | unsigned long buf; | |
113 | ||
114 | get_random_bytes(&buf, sizeof(buf)); | |
115 | return buf; | |
116 | } | |
117 | ||
b54d5cb9 | 118 | static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) |
4a776f0a HS |
119 | { |
120 | unsigned int i; | |
b54d5cb9 DW |
121 | u8 *buf; |
122 | ||
123 | for (; (buf = *bufs); bufs++) { | |
124 | for (i = 0; i < start; i++) | |
125 | buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); | |
126 | for ( ; i < start + len; i++) | |
127 | buf[i] = PATTERN_SRC | PATTERN_COPY | |
c019894e | 128 | | (~i & PATTERN_COUNT_MASK); |
b54d5cb9 DW |
129 | for ( ; i < test_buf_size; i++) |
130 | buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); | |
131 | buf++; | |
132 | } | |
4a776f0a HS |
133 | } |
134 | ||
b54d5cb9 | 135 | static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) |
4a776f0a HS |
136 | { |
137 | unsigned int i; | |
b54d5cb9 DW |
138 | u8 *buf; |
139 | ||
140 | for (; (buf = *bufs); bufs++) { | |
141 | for (i = 0; i < start; i++) | |
142 | buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); | |
143 | for ( ; i < start + len; i++) | |
144 | buf[i] = PATTERN_DST | PATTERN_OVERWRITE | |
145 | | (~i & PATTERN_COUNT_MASK); | |
146 | for ( ; i < test_buf_size; i++) | |
147 | buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); | |
148 | } | |
4a776f0a HS |
149 | } |
150 | ||
151 | static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, | |
152 | unsigned int counter, bool is_srcbuf) | |
153 | { | |
154 | u8 diff = actual ^ pattern; | |
155 | u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); | |
156 | const char *thread_name = current->comm; | |
157 | ||
158 | if (is_srcbuf) | |
159 | pr_warning("%s: srcbuf[0x%x] overwritten!" | |
160 | " Expected %02x, got %02x\n", | |
161 | thread_name, index, expected, actual); | |
162 | else if ((pattern & PATTERN_COPY) | |
163 | && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) | |
164 | pr_warning("%s: dstbuf[0x%x] not copied!" | |
165 | " Expected %02x, got %02x\n", | |
166 | thread_name, index, expected, actual); | |
167 | else if (diff & PATTERN_SRC) | |
168 | pr_warning("%s: dstbuf[0x%x] was copied!" | |
169 | " Expected %02x, got %02x\n", | |
170 | thread_name, index, expected, actual); | |
171 | else | |
172 | pr_warning("%s: dstbuf[0x%x] mismatch!" | |
173 | " Expected %02x, got %02x\n", | |
174 | thread_name, index, expected, actual); | |
175 | } | |
176 | ||
b54d5cb9 | 177 | static unsigned int dmatest_verify(u8 **bufs, unsigned int start, |
4a776f0a HS |
178 | unsigned int end, unsigned int counter, u8 pattern, |
179 | bool is_srcbuf) | |
180 | { | |
181 | unsigned int i; | |
182 | unsigned int error_count = 0; | |
183 | u8 actual; | |
b54d5cb9 DW |
184 | u8 expected; |
185 | u8 *buf; | |
186 | unsigned int counter_orig = counter; | |
187 | ||
188 | for (; (buf = *bufs); bufs++) { | |
189 | counter = counter_orig; | |
190 | for (i = start; i < end; i++) { | |
191 | actual = buf[i]; | |
192 | expected = pattern | (~counter & PATTERN_COUNT_MASK); | |
193 | if (actual != expected) { | |
194 | if (error_count < 32) | |
195 | dmatest_mismatch(actual, pattern, i, | |
196 | counter, is_srcbuf); | |
197 | error_count++; | |
198 | } | |
199 | counter++; | |
4a776f0a | 200 | } |
4a776f0a HS |
201 | } |
202 | ||
203 | if (error_count > 32) | |
204 | pr_warning("%s: %u errors suppressed\n", | |
205 | current->comm, error_count - 32); | |
206 | ||
207 | return error_count; | |
208 | } | |
209 | ||
e44e0aa3 DW |
210 | static void dmatest_callback(void *completion) |
211 | { | |
212 | complete(completion); | |
213 | } | |
214 | ||
4a776f0a HS |
215 | /* |
216 | * This function repeatedly tests DMA transfers of various lengths and | |
b54d5cb9 DW |
217 | * offsets for a given operation type until it is told to exit by |
218 | * kthread_stop(). There may be multiple threads running this function | |
219 | * in parallel for a single channel, and there may be multiple channels | |
220 | * being tested in parallel. | |
4a776f0a HS |
221 | * |
222 | * Before each test, the source and destination buffer is initialized | |
223 | * with a known pattern. This pattern is different depending on | |
224 | * whether it's in an area which is supposed to be copied or | |
225 | * overwritten, and different in the source and destination buffers. | |
226 | * So if the DMA engine doesn't copy exactly what we tell it to copy, | |
227 | * we'll notice. | |
228 | */ | |
229 | static int dmatest_func(void *data) | |
230 | { | |
231 | struct dmatest_thread *thread = data; | |
232 | struct dma_chan *chan; | |
233 | const char *thread_name; | |
234 | unsigned int src_off, dst_off, len; | |
235 | unsigned int error_count; | |
236 | unsigned int failed_tests = 0; | |
237 | unsigned int total_tests = 0; | |
238 | dma_cookie_t cookie; | |
239 | enum dma_status status; | |
b54d5cb9 | 240 | enum dma_ctrl_flags flags; |
94de648d | 241 | u8 pq_coefs[pq_sources + 1]; |
4a776f0a | 242 | int ret; |
b54d5cb9 DW |
243 | int src_cnt; |
244 | int dst_cnt; | |
245 | int i; | |
4a776f0a HS |
246 | |
247 | thread_name = current->comm; | |
248 | ||
249 | ret = -ENOMEM; | |
4a776f0a HS |
250 | |
251 | smp_rmb(); | |
252 | chan = thread->chan; | |
b54d5cb9 DW |
253 | if (thread->type == DMA_MEMCPY) |
254 | src_cnt = dst_cnt = 1; | |
255 | else if (thread->type == DMA_XOR) { | |
256 | src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ | |
257 | dst_cnt = 1; | |
58691d64 DW |
258 | } else if (thread->type == DMA_PQ) { |
259 | src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ | |
260 | dst_cnt = 2; | |
94de648d | 261 | for (i = 0; i < src_cnt; i++) |
58691d64 | 262 | pq_coefs[i] = 1; |
b54d5cb9 DW |
263 | } else |
264 | goto err_srcs; | |
265 | ||
266 | thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); | |
267 | if (!thread->srcs) | |
268 | goto err_srcs; | |
269 | for (i = 0; i < src_cnt; i++) { | |
270 | thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); | |
271 | if (!thread->srcs[i]) | |
272 | goto err_srcbuf; | |
273 | } | |
274 | thread->srcs[i] = NULL; | |
275 | ||
276 | thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); | |
277 | if (!thread->dsts) | |
278 | goto err_dsts; | |
279 | for (i = 0; i < dst_cnt; i++) { | |
280 | thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); | |
281 | if (!thread->dsts[i]) | |
282 | goto err_dstbuf; | |
283 | } | |
284 | thread->dsts[i] = NULL; | |
285 | ||
e44e0aa3 DW |
286 | set_user_nice(current, 10); |
287 | ||
288 | flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT; | |
4a776f0a | 289 | |
0a2ff57d NF |
290 | while (!kthread_should_stop() |
291 | && !(iterations && total_tests >= iterations)) { | |
d86be86e | 292 | struct dma_device *dev = chan->device; |
b54d5cb9 DW |
293 | struct dma_async_tx_descriptor *tx = NULL; |
294 | dma_addr_t dma_srcs[src_cnt]; | |
295 | dma_addr_t dma_dsts[dst_cnt]; | |
e44e0aa3 DW |
296 | struct completion cmp; |
297 | unsigned long tmo = msecs_to_jiffies(3000); | |
83544ae9 | 298 | u8 align = 0; |
d86be86e | 299 | |
4a776f0a HS |
300 | total_tests++; |
301 | ||
83544ae9 DW |
302 | /* honor alignment restrictions */ |
303 | if (thread->type == DMA_MEMCPY) | |
304 | align = dev->copy_align; | |
305 | else if (thread->type == DMA_XOR) | |
306 | align = dev->xor_align; | |
307 | else if (thread->type == DMA_PQ) | |
308 | align = dev->pq_align; | |
309 | ||
cfe4f275 GL |
310 | if (1 << align > test_buf_size) { |
311 | pr_err("%u-byte buffer too small for %d-byte alignment\n", | |
312 | test_buf_size, 1 << align); | |
313 | break; | |
314 | } | |
315 | ||
316 | len = dmatest_random() % test_buf_size + 1; | |
83544ae9 | 317 | len = (len >> align) << align; |
cfe4f275 GL |
318 | if (!len) |
319 | len = 1 << align; | |
320 | src_off = dmatest_random() % (test_buf_size - len + 1); | |
321 | dst_off = dmatest_random() % (test_buf_size - len + 1); | |
322 | ||
83544ae9 DW |
323 | src_off = (src_off >> align) << align; |
324 | dst_off = (dst_off >> align) << align; | |
325 | ||
b54d5cb9 DW |
326 | dmatest_init_srcs(thread->srcs, src_off, len); |
327 | dmatest_init_dsts(thread->dsts, dst_off, len); | |
4a776f0a | 328 | |
b54d5cb9 DW |
329 | for (i = 0; i < src_cnt; i++) { |
330 | u8 *buf = thread->srcs[i] + src_off; | |
331 | ||
332 | dma_srcs[i] = dma_map_single(dev->dev, buf, len, | |
333 | DMA_TO_DEVICE); | |
334 | } | |
d86be86e | 335 | /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ |
b54d5cb9 DW |
336 | for (i = 0; i < dst_cnt; i++) { |
337 | dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], | |
338 | test_buf_size, | |
339 | DMA_BIDIRECTIONAL); | |
340 | } | |
341 | ||
83544ae9 | 342 | |
b54d5cb9 DW |
343 | if (thread->type == DMA_MEMCPY) |
344 | tx = dev->device_prep_dma_memcpy(chan, | |
345 | dma_dsts[0] + dst_off, | |
346 | dma_srcs[0], len, | |
347 | flags); | |
348 | else if (thread->type == DMA_XOR) | |
349 | tx = dev->device_prep_dma_xor(chan, | |
350 | dma_dsts[0] + dst_off, | |
67b9124f | 351 | dma_srcs, src_cnt, |
b54d5cb9 | 352 | len, flags); |
58691d64 DW |
353 | else if (thread->type == DMA_PQ) { |
354 | dma_addr_t dma_pq[dst_cnt]; | |
355 | ||
356 | for (i = 0; i < dst_cnt; i++) | |
357 | dma_pq[i] = dma_dsts[i] + dst_off; | |
358 | tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, | |
94de648d | 359 | src_cnt, pq_coefs, |
58691d64 DW |
360 | len, flags); |
361 | } | |
d86be86e | 362 | |
d86be86e | 363 | if (!tx) { |
b54d5cb9 DW |
364 | for (i = 0; i < src_cnt; i++) |
365 | dma_unmap_single(dev->dev, dma_srcs[i], len, | |
366 | DMA_TO_DEVICE); | |
367 | for (i = 0; i < dst_cnt; i++) | |
368 | dma_unmap_single(dev->dev, dma_dsts[i], | |
369 | test_buf_size, | |
370 | DMA_BIDIRECTIONAL); | |
d86be86e AN |
371 | pr_warning("%s: #%u: prep error with src_off=0x%x " |
372 | "dst_off=0x%x len=0x%x\n", | |
373 | thread_name, total_tests - 1, | |
374 | src_off, dst_off, len); | |
375 | msleep(100); | |
376 | failed_tests++; | |
377 | continue; | |
378 | } | |
e44e0aa3 DW |
379 | |
380 | init_completion(&cmp); | |
381 | tx->callback = dmatest_callback; | |
382 | tx->callback_param = &cmp; | |
d86be86e AN |
383 | cookie = tx->tx_submit(tx); |
384 | ||
4a776f0a HS |
385 | if (dma_submit_error(cookie)) { |
386 | pr_warning("%s: #%u: submit error %d with src_off=0x%x " | |
387 | "dst_off=0x%x len=0x%x\n", | |
388 | thread_name, total_tests - 1, cookie, | |
389 | src_off, dst_off, len); | |
390 | msleep(100); | |
391 | failed_tests++; | |
392 | continue; | |
393 | } | |
b54d5cb9 | 394 | dma_async_issue_pending(chan); |
4a776f0a | 395 | |
e44e0aa3 DW |
396 | tmo = wait_for_completion_timeout(&cmp, tmo); |
397 | status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); | |
4a776f0a | 398 | |
e44e0aa3 DW |
399 | if (tmo == 0) { |
400 | pr_warning("%s: #%u: test timed out\n", | |
401 | thread_name, total_tests - 1); | |
402 | failed_tests++; | |
403 | continue; | |
404 | } else if (status != DMA_SUCCESS) { | |
405 | pr_warning("%s: #%u: got completion callback," | |
406 | " but status is \'%s\'\n", | |
407 | thread_name, total_tests - 1, | |
408 | status == DMA_ERROR ? "error" : "in progress"); | |
4a776f0a HS |
409 | failed_tests++; |
410 | continue; | |
411 | } | |
e44e0aa3 | 412 | |
d86be86e | 413 | /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ |
b54d5cb9 DW |
414 | for (i = 0; i < dst_cnt; i++) |
415 | dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, | |
416 | DMA_BIDIRECTIONAL); | |
4a776f0a HS |
417 | |
418 | error_count = 0; | |
419 | ||
420 | pr_debug("%s: verifying source buffer...\n", thread_name); | |
b54d5cb9 | 421 | error_count += dmatest_verify(thread->srcs, 0, src_off, |
4a776f0a | 422 | 0, PATTERN_SRC, true); |
b54d5cb9 | 423 | error_count += dmatest_verify(thread->srcs, src_off, |
4a776f0a HS |
424 | src_off + len, src_off, |
425 | PATTERN_SRC | PATTERN_COPY, true); | |
b54d5cb9 | 426 | error_count += dmatest_verify(thread->srcs, src_off + len, |
4a776f0a HS |
427 | test_buf_size, src_off + len, |
428 | PATTERN_SRC, true); | |
429 | ||
430 | pr_debug("%s: verifying dest buffer...\n", | |
431 | thread->task->comm); | |
b54d5cb9 | 432 | error_count += dmatest_verify(thread->dsts, 0, dst_off, |
4a776f0a | 433 | 0, PATTERN_DST, false); |
b54d5cb9 | 434 | error_count += dmatest_verify(thread->dsts, dst_off, |
4a776f0a HS |
435 | dst_off + len, src_off, |
436 | PATTERN_SRC | PATTERN_COPY, false); | |
b54d5cb9 | 437 | error_count += dmatest_verify(thread->dsts, dst_off + len, |
4a776f0a HS |
438 | test_buf_size, dst_off + len, |
439 | PATTERN_DST, false); | |
440 | ||
441 | if (error_count) { | |
442 | pr_warning("%s: #%u: %u errors with " | |
443 | "src_off=0x%x dst_off=0x%x len=0x%x\n", | |
444 | thread_name, total_tests - 1, error_count, | |
445 | src_off, dst_off, len); | |
446 | failed_tests++; | |
447 | } else { | |
448 | pr_debug("%s: #%u: No errors with " | |
449 | "src_off=0x%x dst_off=0x%x len=0x%x\n", | |
450 | thread_name, total_tests - 1, | |
451 | src_off, dst_off, len); | |
452 | } | |
453 | } | |
454 | ||
455 | ret = 0; | |
b54d5cb9 DW |
456 | for (i = 0; thread->dsts[i]; i++) |
457 | kfree(thread->dsts[i]); | |
4a776f0a | 458 | err_dstbuf: |
b54d5cb9 DW |
459 | kfree(thread->dsts); |
460 | err_dsts: | |
461 | for (i = 0; thread->srcs[i]; i++) | |
462 | kfree(thread->srcs[i]); | |
4a776f0a | 463 | err_srcbuf: |
b54d5cb9 DW |
464 | kfree(thread->srcs); |
465 | err_srcs: | |
4a776f0a HS |
466 | pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", |
467 | thread_name, total_tests, failed_tests, ret); | |
0a2ff57d NF |
468 | |
469 | if (iterations > 0) | |
470 | while (!kthread_should_stop()) { | |
b953df7c | 471 | DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); |
0a2ff57d NF |
472 | interruptible_sleep_on(&wait_dmatest_exit); |
473 | } | |
474 | ||
4a776f0a HS |
475 | return ret; |
476 | } | |
477 | ||
478 | static void dmatest_cleanup_channel(struct dmatest_chan *dtc) | |
479 | { | |
480 | struct dmatest_thread *thread; | |
481 | struct dmatest_thread *_thread; | |
482 | int ret; | |
483 | ||
484 | list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { | |
485 | ret = kthread_stop(thread->task); | |
486 | pr_debug("dmatest: thread %s exited with status %d\n", | |
487 | thread->task->comm, ret); | |
488 | list_del(&thread->node); | |
489 | kfree(thread); | |
490 | } | |
491 | kfree(dtc); | |
492 | } | |
493 | ||
b54d5cb9 | 494 | static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) |
4a776f0a | 495 | { |
b54d5cb9 DW |
496 | struct dmatest_thread *thread; |
497 | struct dma_chan *chan = dtc->chan; | |
498 | char *op; | |
499 | unsigned int i; | |
4a776f0a | 500 | |
b54d5cb9 DW |
501 | if (type == DMA_MEMCPY) |
502 | op = "copy"; | |
503 | else if (type == DMA_XOR) | |
504 | op = "xor"; | |
58691d64 DW |
505 | else if (type == DMA_PQ) |
506 | op = "pq"; | |
b54d5cb9 DW |
507 | else |
508 | return -EINVAL; | |
4a776f0a HS |
509 | |
510 | for (i = 0; i < threads_per_chan; i++) { | |
511 | thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); | |
512 | if (!thread) { | |
b54d5cb9 DW |
513 | pr_warning("dmatest: No memory for %s-%s%u\n", |
514 | dma_chan_name(chan), op, i); | |
515 | ||
4a776f0a HS |
516 | break; |
517 | } | |
518 | thread->chan = dtc->chan; | |
b54d5cb9 | 519 | thread->type = type; |
4a776f0a | 520 | smp_wmb(); |
b54d5cb9 DW |
521 | thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", |
522 | dma_chan_name(chan), op, i); | |
4a776f0a | 523 | if (IS_ERR(thread->task)) { |
b54d5cb9 DW |
524 | pr_warning("dmatest: Failed to run thread %s-%s%u\n", |
525 | dma_chan_name(chan), op, i); | |
4a776f0a HS |
526 | kfree(thread); |
527 | break; | |
528 | } | |
529 | ||
530 | /* srcbuf and dstbuf are allocated by the thread itself */ | |
531 | ||
532 | list_add_tail(&thread->node, &dtc->threads); | |
533 | } | |
534 | ||
b54d5cb9 DW |
535 | return i; |
536 | } | |
537 | ||
538 | static int dmatest_add_channel(struct dma_chan *chan) | |
539 | { | |
540 | struct dmatest_chan *dtc; | |
541 | struct dma_device *dma_dev = chan->device; | |
542 | unsigned int thread_count = 0; | |
b9033e68 | 543 | int cnt; |
b54d5cb9 DW |
544 | |
545 | dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); | |
546 | if (!dtc) { | |
547 | pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); | |
548 | return -ENOMEM; | |
549 | } | |
550 | ||
551 | dtc->chan = chan; | |
552 | INIT_LIST_HEAD(&dtc->threads); | |
553 | ||
554 | if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { | |
555 | cnt = dmatest_add_threads(dtc, DMA_MEMCPY); | |
f1aef8b6 | 556 | thread_count += cnt > 0 ? cnt : 0; |
b54d5cb9 DW |
557 | } |
558 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { | |
559 | cnt = dmatest_add_threads(dtc, DMA_XOR); | |
f1aef8b6 | 560 | thread_count += cnt > 0 ? cnt : 0; |
b54d5cb9 | 561 | } |
58691d64 DW |
562 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { |
563 | cnt = dmatest_add_threads(dtc, DMA_PQ); | |
564 | thread_count += cnt > 0 ?: 0; | |
565 | } | |
b54d5cb9 DW |
566 | |
567 | pr_info("dmatest: Started %u threads using %s\n", | |
568 | thread_count, dma_chan_name(chan)); | |
4a776f0a HS |
569 | |
570 | list_add_tail(&dtc->node, &dmatest_channels); | |
571 | nr_channels++; | |
572 | ||
33df8ca0 | 573 | return 0; |
4a776f0a HS |
574 | } |
575 | ||
7dd60251 | 576 | static bool filter(struct dma_chan *chan, void *param) |
4a776f0a | 577 | { |
33df8ca0 | 578 | if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) |
7dd60251 | 579 | return false; |
33df8ca0 | 580 | else |
7dd60251 | 581 | return true; |
4a776f0a HS |
582 | } |
583 | ||
4a776f0a HS |
584 | static int __init dmatest_init(void) |
585 | { | |
33df8ca0 DW |
586 | dma_cap_mask_t mask; |
587 | struct dma_chan *chan; | |
588 | int err = 0; | |
589 | ||
590 | dma_cap_zero(mask); | |
591 | dma_cap_set(DMA_MEMCPY, mask); | |
592 | for (;;) { | |
593 | chan = dma_request_channel(mask, filter, NULL); | |
594 | if (chan) { | |
595 | err = dmatest_add_channel(chan); | |
c56c81ab | 596 | if (err) { |
33df8ca0 DW |
597 | dma_release_channel(chan); |
598 | break; /* add_channel failed, punt */ | |
599 | } | |
600 | } else | |
601 | break; /* no more channels available */ | |
602 | if (max_channels && nr_channels >= max_channels) | |
603 | break; /* we have all we need */ | |
604 | } | |
4a776f0a | 605 | |
33df8ca0 | 606 | return err; |
4a776f0a | 607 | } |
33df8ca0 DW |
608 | /* when compiled-in wait for drivers to load first */ |
609 | late_initcall(dmatest_init); | |
4a776f0a HS |
610 | |
611 | static void __exit dmatest_exit(void) | |
612 | { | |
33df8ca0 | 613 | struct dmatest_chan *dtc, *_dtc; |
7cbd4877 | 614 | struct dma_chan *chan; |
33df8ca0 DW |
615 | |
616 | list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { | |
617 | list_del(&dtc->node); | |
7cbd4877 | 618 | chan = dtc->chan; |
33df8ca0 DW |
619 | dmatest_cleanup_channel(dtc); |
620 | pr_debug("dmatest: dropped channel %s\n", | |
7cbd4877 DW |
621 | dma_chan_name(chan)); |
622 | dma_release_channel(chan); | |
33df8ca0 | 623 | } |
4a776f0a HS |
624 | } |
625 | module_exit(dmatest_exit); | |
626 | ||
627 | MODULE_AUTHOR("Haavard Skinnemoen <hskinnemoen@atmel.com>"); | |
628 | MODULE_LICENSE("GPL v2"); |