Commit | Line | Data |
---|---|---|
8a7b6a77 DJ |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright(c) 2015 Intel Corporation. All rights reserved. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * BSD LICENSE | |
14 | * | |
15 | * Copyright(c) 2015 Intel Corporation. All rights reserved. | |
16 | * | |
17 | * Redistribution and use in source and binary forms, with or without | |
18 | * modification, are permitted provided that the following conditions | |
19 | * are met: | |
20 | * | |
21 | * * Redistributions of source code must retain the above copyright | |
22 | * notice, this list of conditions and the following disclaimer. | |
23 | * * Redistributions in binary form must reproduce the above copy | |
24 | * notice, this list of conditions and the following disclaimer in | |
25 | * the documentation and/or other materials provided with the | |
26 | * distribution. | |
27 | * * Neither the name of Intel Corporation nor the names of its | |
28 | * contributors may be used to endorse or promote products derived | |
29 | * from this software without specific prior written permission. | |
30 | * | |
31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
32 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
33 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
34 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
35 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
36 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
37 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
38 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
39 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
40 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
41 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
42 | * | |
43 | * PCIe NTB Perf Linux driver | |
44 | */ | |
45 | ||
46 | #include <linux/init.h> | |
47 | #include <linux/kernel.h> | |
48 | #include <linux/module.h> | |
49 | #include <linux/kthread.h> | |
50 | #include <linux/time.h> | |
51 | #include <linux/timer.h> | |
52 | #include <linux/dma-mapping.h> | |
53 | #include <linux/pci.h> | |
54 | #include <linux/slab.h> | |
55 | #include <linux/spinlock.h> | |
56 | #include <linux/debugfs.h> | |
57 | #include <linux/dmaengine.h> | |
58 | #include <linux/delay.h> | |
59 | #include <linux/sizes.h> | |
60 | #include <linux/ntb.h> | |
61 | ||
62 | #define DRIVER_NAME "ntb_perf" | |
63 | #define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool" | |
64 | ||
65 | #define DRIVER_LICENSE "Dual BSD/GPL" | |
66 | #define DRIVER_VERSION "1.0" | |
67 | #define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>" | |
68 | ||
69 | #define PERF_LINK_DOWN_TIMEOUT 10 | |
70 | #define PERF_VERSION 0xffff0001 | |
71 | #define MAX_THREADS 32 | |
72 | #define MAX_TEST_SIZE SZ_1M | |
73 | #define MAX_SRCS 32 | |
74 | #define DMA_OUT_RESOURCE_TO 50 | |
75 | #define DMA_RETRIES 20 | |
76 | #define SZ_4G (1ULL << 32) | |
77 | #define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */ | |
78 | ||
79 | MODULE_LICENSE(DRIVER_LICENSE); | |
80 | MODULE_VERSION(DRIVER_VERSION); | |
81 | MODULE_AUTHOR(DRIVER_AUTHOR); | |
82 | MODULE_DESCRIPTION(DRIVER_DESCRIPTION); | |
83 | ||
84 | static struct dentry *perf_debugfs_dir; | |
85 | ||
86 | static unsigned int seg_order = 19; /* 512K */ | |
87 | module_param(seg_order, uint, 0644); | |
88 | MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing"); | |
89 | ||
90 | static unsigned int run_order = 32; /* 4G */ | |
91 | module_param(run_order, uint, 0644); | |
92 | MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer"); | |
93 | ||
94 | static bool use_dma; /* default to 0 */ | |
95 | module_param(use_dma, bool, 0644); | |
96 | MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance"); | |
97 | ||
98 | struct perf_mw { | |
99 | phys_addr_t phys_addr; | |
100 | resource_size_t phys_size; | |
101 | resource_size_t xlat_align; | |
102 | resource_size_t xlat_align_size; | |
103 | void __iomem *vbase; | |
104 | size_t xlat_size; | |
105 | size_t buf_size; | |
106 | void *virt_addr; | |
107 | dma_addr_t dma_addr; | |
108 | }; | |
109 | ||
110 | struct perf_ctx; | |
111 | ||
112 | struct pthr_ctx { | |
113 | struct task_struct *thread; | |
114 | struct perf_ctx *perf; | |
115 | atomic_t dma_sync; | |
116 | struct dma_chan *dma_chan; | |
117 | int dma_prep_err; | |
118 | int src_idx; | |
119 | void *srcs[MAX_SRCS]; | |
120 | }; | |
121 | ||
122 | struct perf_ctx { | |
123 | struct ntb_dev *ntb; | |
124 | spinlock_t db_lock; | |
125 | struct perf_mw mw; | |
126 | bool link_is_up; | |
127 | struct work_struct link_cleanup; | |
128 | struct delayed_work link_work; | |
129 | struct dentry *debugfs_node_dir; | |
130 | struct dentry *debugfs_run; | |
131 | struct dentry *debugfs_threads; | |
132 | u8 perf_threads; | |
133 | bool run; | |
134 | struct pthr_ctx pthr_ctx[MAX_THREADS]; | |
135 | atomic_t tsync; | |
136 | }; | |
137 | ||
138 | enum { | |
139 | VERSION = 0, | |
140 | MW_SZ_HIGH, | |
141 | MW_SZ_LOW, | |
142 | SPAD_MSG, | |
143 | SPAD_ACK, | |
144 | MAX_SPAD | |
145 | }; | |
146 | ||
147 | static void perf_link_event(void *ctx) | |
148 | { | |
149 | struct perf_ctx *perf = ctx; | |
150 | ||
151 | if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) | |
152 | schedule_delayed_work(&perf->link_work, 2*HZ); | |
153 | else | |
154 | schedule_work(&perf->link_cleanup); | |
155 | } | |
156 | ||
157 | static void perf_db_event(void *ctx, int vec) | |
158 | { | |
159 | struct perf_ctx *perf = ctx; | |
160 | u64 db_bits, db_mask; | |
161 | ||
162 | db_mask = ntb_db_vector_mask(perf->ntb, vec); | |
163 | db_bits = ntb_db_read(perf->ntb); | |
164 | ||
165 | dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", | |
166 | vec, db_mask, db_bits); | |
167 | } | |
168 | ||
169 | static const struct ntb_ctx_ops perf_ops = { | |
170 | .link_event = perf_link_event, | |
171 | .db_event = perf_db_event, | |
172 | }; | |
173 | ||
174 | static void perf_copy_callback(void *data) | |
175 | { | |
176 | struct pthr_ctx *pctx = data; | |
177 | ||
178 | atomic_dec(&pctx->dma_sync); | |
179 | } | |
180 | ||
1985a881 | 181 | static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst, |
8a7b6a77 DJ |
182 | char *src, size_t size) |
183 | { | |
184 | struct perf_ctx *perf = pctx->perf; | |
185 | struct dma_async_tx_descriptor *txd; | |
186 | struct dma_chan *chan = pctx->dma_chan; | |
187 | struct dma_device *device; | |
188 | struct dmaengine_unmap_data *unmap; | |
189 | dma_cookie_t cookie; | |
190 | size_t src_off, dst_off; | |
191 | struct perf_mw *mw = &perf->mw; | |
1985a881 AB |
192 | void __iomem *vbase; |
193 | void __iomem *dst_vaddr; | |
8a7b6a77 DJ |
194 | dma_addr_t dst_phys; |
195 | int retries = 0; | |
196 | ||
197 | if (!use_dma) { | |
198 | memcpy_toio(dst, src, size); | |
199 | return size; | |
200 | } | |
201 | ||
202 | if (!chan) { | |
203 | dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); | |
204 | return -EINVAL; | |
205 | } | |
206 | ||
207 | device = chan->device; | |
1985a881 AB |
208 | src_off = (uintptr_t)src & ~PAGE_MASK; |
209 | dst_off = (uintptr_t __force)dst & ~PAGE_MASK; | |
8a7b6a77 DJ |
210 | |
211 | if (!is_dma_copy_aligned(device, src_off, dst_off, size)) | |
212 | return -ENODEV; | |
213 | ||
1985a881 AB |
214 | vbase = mw->vbase; |
215 | dst_vaddr = dst; | |
8a7b6a77 DJ |
216 | dst_phys = mw->phys_addr + (dst_vaddr - vbase); |
217 | ||
218 | unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); | |
219 | if (!unmap) | |
220 | return -ENOMEM; | |
221 | ||
222 | unmap->len = size; | |
223 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), | |
224 | src_off, size, DMA_TO_DEVICE); | |
225 | if (dma_mapping_error(device->dev, unmap->addr[0])) | |
226 | goto err_get_unmap; | |
227 | ||
228 | unmap->to_cnt = 1; | |
229 | ||
230 | do { | |
231 | txd = device->device_prep_dma_memcpy(chan, dst_phys, | |
232 | unmap->addr[0], | |
233 | size, DMA_PREP_INTERRUPT); | |
234 | if (!txd) { | |
235 | set_current_state(TASK_INTERRUPTIBLE); | |
236 | schedule_timeout(DMA_OUT_RESOURCE_TO); | |
237 | } | |
238 | } while (!txd && (++retries < DMA_RETRIES)); | |
239 | ||
240 | if (!txd) { | |
241 | pctx->dma_prep_err++; | |
242 | goto err_get_unmap; | |
243 | } | |
244 | ||
245 | txd->callback = perf_copy_callback; | |
246 | txd->callback_param = pctx; | |
247 | dma_set_unmap(txd, unmap); | |
248 | ||
249 | cookie = dmaengine_submit(txd); | |
250 | if (dma_submit_error(cookie)) | |
251 | goto err_set_unmap; | |
252 | ||
253 | atomic_inc(&pctx->dma_sync); | |
254 | dma_async_issue_pending(chan); | |
255 | ||
256 | return size; | |
257 | ||
258 | err_set_unmap: | |
259 | dmaengine_unmap_put(unmap); | |
260 | err_get_unmap: | |
261 | dmaengine_unmap_put(unmap); | |
262 | return 0; | |
263 | } | |
264 | ||
1985a881 | 265 | static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src, |
8a7b6a77 DJ |
266 | u64 buf_size, u64 win_size, u64 total) |
267 | { | |
268 | int chunks, total_chunks, i; | |
269 | int copied_chunks = 0; | |
270 | u64 copied = 0, result; | |
1985a881 | 271 | char __iomem *tmp = dst; |
8a7b6a77 DJ |
272 | u64 perf, diff_us; |
273 | ktime_t kstart, kstop, kdiff; | |
274 | ||
275 | chunks = div64_u64(win_size, buf_size); | |
276 | total_chunks = div64_u64(total, buf_size); | |
277 | kstart = ktime_get(); | |
278 | ||
279 | for (i = 0; i < total_chunks; i++) { | |
280 | result = perf_copy(pctx, tmp, src, buf_size); | |
281 | copied += result; | |
282 | copied_chunks++; | |
283 | if (copied_chunks == chunks) { | |
284 | tmp = dst; | |
285 | copied_chunks = 0; | |
286 | } else | |
287 | tmp += buf_size; | |
288 | ||
289 | /* Probably should schedule every 4GB to prevent soft hang. */ | |
290 | if (((copied % SZ_4G) == 0) && !use_dma) { | |
291 | set_current_state(TASK_INTERRUPTIBLE); | |
292 | schedule_timeout(1); | |
293 | } | |
294 | } | |
295 | ||
296 | if (use_dma) { | |
297 | pr_info("%s: All DMA descriptors submitted\n", current->comm); | |
298 | while (atomic_read(&pctx->dma_sync) != 0) | |
299 | msleep(20); | |
300 | } | |
301 | ||
302 | kstop = ktime_get(); | |
303 | kdiff = ktime_sub(kstop, kstart); | |
304 | diff_us = ktime_to_us(kdiff); | |
305 | ||
306 | pr_info("%s: copied %llu bytes\n", current->comm, copied); | |
307 | ||
308 | pr_info("%s: lasted %llu usecs\n", current->comm, diff_us); | |
309 | ||
310 | perf = div64_u64(copied, diff_us); | |
311 | ||
312 | pr_info("%s: MBytes/s: %llu\n", current->comm, perf); | |
313 | ||
314 | return 0; | |
315 | } | |
316 | ||
317 | static bool perf_dma_filter_fn(struct dma_chan *chan, void *node) | |
318 | { | |
319 | return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; | |
320 | } | |
321 | ||
322 | static int ntb_perf_thread(void *data) | |
323 | { | |
324 | struct pthr_ctx *pctx = data; | |
325 | struct perf_ctx *perf = pctx->perf; | |
326 | struct pci_dev *pdev = perf->ntb->pdev; | |
327 | struct perf_mw *mw = &perf->mw; | |
1985a881 | 328 | char __iomem *dst; |
8a7b6a77 DJ |
329 | u64 win_size, buf_size, total; |
330 | void *src; | |
331 | int rc, node, i; | |
332 | struct dma_chan *dma_chan = NULL; | |
333 | ||
334 | pr_info("kthread %s starting...\n", current->comm); | |
335 | ||
336 | node = dev_to_node(&pdev->dev); | |
337 | ||
338 | if (use_dma && !pctx->dma_chan) { | |
339 | dma_cap_mask_t dma_mask; | |
340 | ||
341 | dma_cap_zero(dma_mask); | |
342 | dma_cap_set(DMA_MEMCPY, dma_mask); | |
343 | dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn, | |
344 | (void *)(unsigned long)node); | |
345 | if (!dma_chan) { | |
346 | pr_warn("%s: cannot acquire DMA channel, quitting\n", | |
347 | current->comm); | |
348 | return -ENODEV; | |
349 | } | |
350 | pctx->dma_chan = dma_chan; | |
351 | } | |
352 | ||
353 | for (i = 0; i < MAX_SRCS; i++) { | |
354 | pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node); | |
355 | if (!pctx->srcs[i]) { | |
356 | rc = -ENOMEM; | |
357 | goto err; | |
358 | } | |
359 | } | |
360 | ||
361 | win_size = mw->phys_size; | |
362 | buf_size = 1ULL << seg_order; | |
363 | total = 1ULL << run_order; | |
364 | ||
365 | if (buf_size > MAX_TEST_SIZE) | |
366 | buf_size = MAX_TEST_SIZE; | |
367 | ||
1985a881 | 368 | dst = (char __iomem *)mw->vbase; |
8a7b6a77 DJ |
369 | |
370 | atomic_inc(&perf->tsync); | |
371 | while (atomic_read(&perf->tsync) != perf->perf_threads) | |
372 | schedule(); | |
373 | ||
374 | src = pctx->srcs[pctx->src_idx]; | |
375 | pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1); | |
376 | ||
377 | rc = perf_move_data(pctx, dst, src, buf_size, win_size, total); | |
378 | ||
379 | atomic_dec(&perf->tsync); | |
380 | ||
381 | if (rc < 0) { | |
382 | pr_err("%s: failed\n", current->comm); | |
383 | rc = -ENXIO; | |
384 | goto err; | |
385 | } | |
386 | ||
387 | for (i = 0; i < MAX_SRCS; i++) { | |
388 | kfree(pctx->srcs[i]); | |
389 | pctx->srcs[i] = NULL; | |
390 | } | |
391 | ||
392 | return 0; | |
393 | ||
394 | err: | |
395 | for (i = 0; i < MAX_SRCS; i++) { | |
396 | kfree(pctx->srcs[i]); | |
397 | pctx->srcs[i] = NULL; | |
398 | } | |
399 | ||
400 | if (dma_chan) { | |
401 | dma_release_channel(dma_chan); | |
402 | pctx->dma_chan = NULL; | |
403 | } | |
404 | ||
405 | return rc; | |
406 | } | |
407 | ||
408 | static void perf_free_mw(struct perf_ctx *perf) | |
409 | { | |
410 | struct perf_mw *mw = &perf->mw; | |
411 | struct pci_dev *pdev = perf->ntb->pdev; | |
412 | ||
413 | if (!mw->virt_addr) | |
414 | return; | |
415 | ||
416 | ntb_mw_clear_trans(perf->ntb, 0); | |
417 | dma_free_coherent(&pdev->dev, mw->buf_size, | |
418 | mw->virt_addr, mw->dma_addr); | |
419 | mw->xlat_size = 0; | |
420 | mw->buf_size = 0; | |
421 | mw->virt_addr = NULL; | |
422 | } | |
423 | ||
424 | static int perf_set_mw(struct perf_ctx *perf, resource_size_t size) | |
425 | { | |
426 | struct perf_mw *mw = &perf->mw; | |
427 | size_t xlat_size, buf_size; | |
ee5f750f | 428 | int rc; |
8a7b6a77 DJ |
429 | |
430 | if (!size) | |
431 | return -EINVAL; | |
432 | ||
433 | xlat_size = round_up(size, mw->xlat_align_size); | |
434 | buf_size = round_up(size, mw->xlat_align); | |
435 | ||
436 | if (mw->xlat_size == xlat_size) | |
437 | return 0; | |
438 | ||
439 | if (mw->buf_size) | |
440 | perf_free_mw(perf); | |
441 | ||
442 | mw->xlat_size = xlat_size; | |
443 | mw->buf_size = buf_size; | |
444 | ||
445 | mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size, | |
446 | &mw->dma_addr, GFP_KERNEL); | |
447 | if (!mw->virt_addr) { | |
448 | mw->xlat_size = 0; | |
449 | mw->buf_size = 0; | |
450 | } | |
451 | ||
ee5f750f DJ |
452 | rc = ntb_mw_set_trans(perf->ntb, 0, mw->dma_addr, mw->xlat_size); |
453 | if (rc) { | |
454 | dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n"); | |
455 | perf_free_mw(perf); | |
456 | return -EIO; | |
457 | } | |
458 | ||
8a7b6a77 DJ |
459 | return 0; |
460 | } | |
461 | ||
462 | static void perf_link_work(struct work_struct *work) | |
463 | { | |
464 | struct perf_ctx *perf = | |
465 | container_of(work, struct perf_ctx, link_work.work); | |
466 | struct ntb_dev *ndev = perf->ntb; | |
467 | struct pci_dev *pdev = ndev->pdev; | |
468 | u32 val; | |
469 | u64 size; | |
470 | int rc; | |
471 | ||
472 | dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); | |
473 | ||
474 | size = perf->mw.phys_size; | |
475 | ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size)); | |
476 | ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size)); | |
477 | ntb_peer_spad_write(ndev, VERSION, PERF_VERSION); | |
478 | ||
479 | /* now read what peer wrote */ | |
480 | val = ntb_spad_read(ndev, VERSION); | |
481 | if (val != PERF_VERSION) { | |
482 | dev_dbg(&pdev->dev, "Remote version = %#x\n", val); | |
483 | goto out; | |
484 | } | |
485 | ||
486 | val = ntb_spad_read(ndev, MW_SZ_HIGH); | |
487 | size = (u64)val << 32; | |
488 | ||
489 | val = ntb_spad_read(ndev, MW_SZ_LOW); | |
490 | size |= val; | |
491 | ||
492 | dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size); | |
493 | ||
494 | rc = perf_set_mw(perf, size); | |
495 | if (rc) | |
496 | goto out1; | |
497 | ||
498 | perf->link_is_up = true; | |
499 | ||
500 | return; | |
501 | ||
502 | out1: | |
503 | perf_free_mw(perf); | |
504 | ||
505 | out: | |
506 | if (ntb_link_is_up(ndev, NULL, NULL) == 1) | |
507 | schedule_delayed_work(&perf->link_work, | |
508 | msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT)); | |
509 | } | |
510 | ||
511 | static void perf_link_cleanup(struct work_struct *work) | |
512 | { | |
513 | struct perf_ctx *perf = container_of(work, | |
514 | struct perf_ctx, | |
515 | link_cleanup); | |
516 | ||
517 | dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); | |
518 | ||
519 | if (!perf->link_is_up) | |
520 | cancel_delayed_work_sync(&perf->link_work); | |
521 | } | |
522 | ||
523 | static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf) | |
524 | { | |
525 | struct perf_mw *mw; | |
526 | int rc; | |
527 | ||
528 | mw = &perf->mw; | |
529 | ||
530 | rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size, | |
531 | &mw->xlat_align, &mw->xlat_align_size); | |
532 | if (rc) | |
533 | return rc; | |
534 | ||
535 | perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size); | |
536 | if (!mw->vbase) | |
537 | return -ENOMEM; | |
538 | ||
539 | return 0; | |
540 | } | |
541 | ||
542 | static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf, | |
543 | size_t count, loff_t *offp) | |
544 | { | |
545 | struct perf_ctx *perf = filp->private_data; | |
546 | char *buf; | |
547 | ssize_t ret, out_offset; | |
548 | ||
549 | if (!perf) | |
550 | return 0; | |
551 | ||
552 | buf = kmalloc(64, GFP_KERNEL); | |
2572c7fb SM |
553 | if (!buf) |
554 | return -ENOMEM; | |
8a7b6a77 DJ |
555 | out_offset = snprintf(buf, 64, "%d\n", perf->run); |
556 | ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); | |
557 | kfree(buf); | |
558 | ||
559 | return ret; | |
560 | } | |
561 | ||
838850ee DJ |
562 | static void threads_cleanup(struct perf_ctx *perf) |
563 | { | |
564 | struct pthr_ctx *pctx; | |
565 | int i; | |
566 | ||
567 | perf->run = false; | |
568 | for (i = 0; i < MAX_THREADS; i++) { | |
569 | pctx = &perf->pthr_ctx[i]; | |
570 | if (pctx->thread) { | |
571 | kthread_stop(pctx->thread); | |
572 | pctx->thread = NULL; | |
573 | } | |
574 | } | |
575 | } | |
576 | ||
8a7b6a77 DJ |
577 | static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf, |
578 | size_t count, loff_t *offp) | |
579 | { | |
580 | struct perf_ctx *perf = filp->private_data; | |
581 | int node, i; | |
582 | ||
583 | if (!perf->link_is_up) | |
584 | return 0; | |
585 | ||
586 | if (perf->perf_threads == 0) | |
587 | return 0; | |
588 | ||
589 | if (atomic_read(&perf->tsync) == 0) | |
590 | perf->run = false; | |
591 | ||
838850ee DJ |
592 | if (perf->run) |
593 | threads_cleanup(perf); | |
594 | else { | |
8a7b6a77 DJ |
595 | perf->run = true; |
596 | ||
597 | if (perf->perf_threads > MAX_THREADS) { | |
598 | perf->perf_threads = MAX_THREADS; | |
599 | pr_info("Reset total threads to: %u\n", MAX_THREADS); | |
600 | } | |
601 | ||
602 | /* no greater than 1M */ | |
603 | if (seg_order > MAX_SEG_ORDER) { | |
604 | seg_order = MAX_SEG_ORDER; | |
605 | pr_info("Fix seg_order to %u\n", seg_order); | |
606 | } | |
607 | ||
608 | if (run_order < seg_order) { | |
609 | run_order = seg_order; | |
610 | pr_info("Fix run_order to %u\n", run_order); | |
611 | } | |
612 | ||
613 | node = dev_to_node(&perf->ntb->pdev->dev); | |
614 | /* launch kernel thread */ | |
615 | for (i = 0; i < perf->perf_threads; i++) { | |
616 | struct pthr_ctx *pctx; | |
617 | ||
618 | pctx = &perf->pthr_ctx[i]; | |
619 | atomic_set(&pctx->dma_sync, 0); | |
620 | pctx->perf = perf; | |
621 | pctx->thread = | |
622 | kthread_create_on_node(ntb_perf_thread, | |
623 | (void *)pctx, | |
624 | node, "ntb_perf %d", i); | |
ddc8f6fe | 625 | if (IS_ERR(pctx->thread)) { |
838850ee DJ |
626 | pctx->thread = NULL; |
627 | goto err; | |
ddc8f6fe DJ |
628 | } else |
629 | wake_up_process(pctx->thread); | |
8a7b6a77 DJ |
630 | |
631 | if (perf->run == false) | |
632 | return -ENXIO; | |
633 | } | |
634 | ||
635 | } | |
636 | ||
637 | return count; | |
838850ee DJ |
638 | |
639 | err: | |
640 | threads_cleanup(perf); | |
641 | return -ENXIO; | |
8a7b6a77 DJ |
642 | } |
643 | ||
644 | static const struct file_operations ntb_perf_debugfs_run = { | |
645 | .owner = THIS_MODULE, | |
646 | .open = simple_open, | |
647 | .read = debugfs_run_read, | |
648 | .write = debugfs_run_write, | |
649 | }; | |
650 | ||
651 | static int perf_debugfs_setup(struct perf_ctx *perf) | |
652 | { | |
653 | struct pci_dev *pdev = perf->ntb->pdev; | |
654 | ||
655 | if (!debugfs_initialized()) | |
656 | return -ENODEV; | |
657 | ||
658 | if (!perf_debugfs_dir) { | |
659 | perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); | |
660 | if (!perf_debugfs_dir) | |
661 | return -ENODEV; | |
662 | } | |
663 | ||
664 | perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev), | |
665 | perf_debugfs_dir); | |
666 | if (!perf->debugfs_node_dir) | |
667 | return -ENODEV; | |
668 | ||
669 | perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR, | |
670 | perf->debugfs_node_dir, perf, | |
671 | &ntb_perf_debugfs_run); | |
672 | if (!perf->debugfs_run) | |
673 | return -ENODEV; | |
674 | ||
675 | perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR, | |
676 | perf->debugfs_node_dir, | |
677 | &perf->perf_threads); | |
678 | if (!perf->debugfs_threads) | |
679 | return -ENODEV; | |
680 | ||
681 | return 0; | |
682 | } | |
683 | ||
684 | static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb) | |
685 | { | |
686 | struct pci_dev *pdev = ntb->pdev; | |
687 | struct perf_ctx *perf; | |
688 | int node; | |
689 | int rc = 0; | |
690 | ||
691 | node = dev_to_node(&pdev->dev); | |
692 | ||
693 | perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node); | |
694 | if (!perf) { | |
695 | rc = -ENOMEM; | |
696 | goto err_perf; | |
697 | } | |
698 | ||
699 | perf->ntb = ntb; | |
700 | perf->perf_threads = 1; | |
701 | atomic_set(&perf->tsync, 0); | |
702 | perf->run = false; | |
703 | spin_lock_init(&perf->db_lock); | |
704 | perf_setup_mw(ntb, perf); | |
705 | INIT_DELAYED_WORK(&perf->link_work, perf_link_work); | |
706 | INIT_WORK(&perf->link_cleanup, perf_link_cleanup); | |
707 | ||
708 | rc = ntb_set_ctx(ntb, perf, &perf_ops); | |
709 | if (rc) | |
710 | goto err_ctx; | |
711 | ||
712 | perf->link_is_up = false; | |
713 | ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); | |
714 | ntb_link_event(ntb); | |
715 | ||
716 | rc = perf_debugfs_setup(perf); | |
717 | if (rc) | |
718 | goto err_ctx; | |
719 | ||
720 | return 0; | |
721 | ||
722 | err_ctx: | |
723 | cancel_delayed_work_sync(&perf->link_work); | |
724 | cancel_work_sync(&perf->link_cleanup); | |
725 | kfree(perf); | |
726 | err_perf: | |
727 | return rc; | |
728 | } | |
729 | ||
730 | static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb) | |
731 | { | |
732 | struct perf_ctx *perf = ntb->ctx; | |
733 | int i; | |
734 | ||
735 | dev_dbg(&perf->ntb->dev, "%s called\n", __func__); | |
736 | ||
737 | cancel_delayed_work_sync(&perf->link_work); | |
738 | cancel_work_sync(&perf->link_cleanup); | |
739 | ||
740 | ntb_clear_ctx(ntb); | |
741 | ntb_link_disable(ntb); | |
742 | ||
743 | debugfs_remove_recursive(perf_debugfs_dir); | |
744 | perf_debugfs_dir = NULL; | |
745 | ||
746 | if (use_dma) { | |
747 | for (i = 0; i < MAX_THREADS; i++) { | |
748 | struct pthr_ctx *pctx = &perf->pthr_ctx[i]; | |
749 | ||
750 | if (pctx->dma_chan) | |
751 | dma_release_channel(pctx->dma_chan); | |
752 | } | |
753 | } | |
754 | ||
755 | kfree(perf); | |
756 | } | |
757 | ||
758 | static struct ntb_client perf_client = { | |
759 | .ops = { | |
760 | .probe = perf_probe, | |
761 | .remove = perf_remove, | |
762 | }, | |
763 | }; | |
764 | module_ntb_client(perf_client); |