Commit | Line | Data |
---|---|---|
f931551b RC |
1 | /* |
2 | * Copyright (c) 2007, 2008, 2009 QLogic Corporation. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | #include <linux/mm.h> | |
33 | #include <linux/types.h> | |
34 | #include <linux/device.h> | |
35 | #include <linux/dmapool.h> | |
36 | #include <linux/slab.h> | |
37 | #include <linux/list.h> | |
38 | #include <linux/highmem.h> | |
39 | #include <linux/io.h> | |
40 | #include <linux/uio.h> | |
41 | #include <linux/rbtree.h> | |
42 | #include <linux/spinlock.h> | |
43 | #include <linux/delay.h> | |
44 | ||
45 | #include "qib.h" | |
46 | #include "qib_user_sdma.h" | |
47 | ||
48 | /* minimum size of header */ | |
49 | #define QIB_USER_SDMA_MIN_HEADER_LENGTH 64 | |
50 | /* expected size of headers (for dma_pool) */ | |
51 | #define QIB_USER_SDMA_EXP_HEADER_LENGTH 64 | |
52 | /* attempt to drain the queue for 5secs */ | |
53 | #define QIB_USER_SDMA_DRAIN_TIMEOUT 500 | |
54 | ||
55 | struct qib_user_sdma_pkt { | |
56 | u8 naddr; /* dimension of addr (1..3) ... */ | |
57 | u32 counter; /* sdma pkts queued counter for this entry */ | |
58 | u64 added; /* global descq number of entries */ | |
59 | ||
60 | struct { | |
61 | u32 offset; /* offset for kvaddr, addr */ | |
62 | u32 length; /* length in page */ | |
63 | u8 put_page; /* should we put_page? */ | |
64 | u8 dma_mapped; /* is page dma_mapped? */ | |
65 | struct page *page; /* may be NULL (coherent mem) */ | |
66 | void *kvaddr; /* FIXME: only for pio hack */ | |
67 | dma_addr_t addr; | |
68 | } addr[4]; /* max pages, any more and we coalesce */ | |
69 | struct list_head list; /* list element */ | |
70 | }; | |
71 | ||
72 | struct qib_user_sdma_queue { | |
73 | /* | |
74 | * pkts sent to dma engine are queued on this | |
75 | * list head. the type of the elements of this | |
76 | * list are struct qib_user_sdma_pkt... | |
77 | */ | |
78 | struct list_head sent; | |
79 | ||
80 | /* headers with expected length are allocated from here... */ | |
81 | char header_cache_name[64]; | |
82 | struct dma_pool *header_cache; | |
83 | ||
84 | /* packets are allocated from the slab cache... */ | |
85 | char pkt_slab_name[64]; | |
86 | struct kmem_cache *pkt_slab; | |
87 | ||
88 | /* as packets go on the queued queue, they are counted... */ | |
89 | u32 counter; | |
90 | u32 sent_counter; | |
91 | ||
92 | /* dma page table */ | |
93 | struct rb_root dma_pages_root; | |
94 | ||
95 | /* protect everything above... */ | |
96 | struct mutex lock; | |
97 | }; | |
98 | ||
99 | struct qib_user_sdma_queue * | |
100 | qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt) | |
101 | { | |
102 | struct qib_user_sdma_queue *pq = | |
103 | kmalloc(sizeof(struct qib_user_sdma_queue), GFP_KERNEL); | |
104 | ||
105 | if (!pq) | |
106 | goto done; | |
107 | ||
108 | pq->counter = 0; | |
109 | pq->sent_counter = 0; | |
110 | INIT_LIST_HEAD(&pq->sent); | |
111 | ||
112 | mutex_init(&pq->lock); | |
113 | ||
114 | snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name), | |
115 | "qib-user-sdma-pkts-%u-%02u.%02u", unit, ctxt, sctxt); | |
116 | pq->pkt_slab = kmem_cache_create(pq->pkt_slab_name, | |
117 | sizeof(struct qib_user_sdma_pkt), | |
118 | 0, 0, NULL); | |
119 | ||
120 | if (!pq->pkt_slab) | |
121 | goto err_kfree; | |
122 | ||
123 | snprintf(pq->header_cache_name, sizeof(pq->header_cache_name), | |
124 | "qib-user-sdma-headers-%u-%02u.%02u", unit, ctxt, sctxt); | |
125 | pq->header_cache = dma_pool_create(pq->header_cache_name, | |
126 | dev, | |
127 | QIB_USER_SDMA_EXP_HEADER_LENGTH, | |
128 | 4, 0); | |
129 | if (!pq->header_cache) | |
130 | goto err_slab; | |
131 | ||
132 | pq->dma_pages_root = RB_ROOT; | |
133 | ||
134 | goto done; | |
135 | ||
136 | err_slab: | |
137 | kmem_cache_destroy(pq->pkt_slab); | |
138 | err_kfree: | |
139 | kfree(pq); | |
140 | pq = NULL; | |
141 | ||
142 | done: | |
143 | return pq; | |
144 | } | |
145 | ||
146 | static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt, | |
147 | int i, size_t offset, size_t len, | |
148 | int put_page, int dma_mapped, | |
149 | struct page *page, | |
150 | void *kvaddr, dma_addr_t dma_addr) | |
151 | { | |
152 | pkt->addr[i].offset = offset; | |
153 | pkt->addr[i].length = len; | |
154 | pkt->addr[i].put_page = put_page; | |
155 | pkt->addr[i].dma_mapped = dma_mapped; | |
156 | pkt->addr[i].page = page; | |
157 | pkt->addr[i].kvaddr = kvaddr; | |
158 | pkt->addr[i].addr = dma_addr; | |
159 | } | |
160 | ||
161 | static void qib_user_sdma_init_header(struct qib_user_sdma_pkt *pkt, | |
162 | u32 counter, size_t offset, | |
163 | size_t len, int dma_mapped, | |
164 | struct page *page, | |
165 | void *kvaddr, dma_addr_t dma_addr) | |
166 | { | |
167 | pkt->naddr = 1; | |
168 | pkt->counter = counter; | |
169 | qib_user_sdma_init_frag(pkt, 0, offset, len, 0, dma_mapped, page, | |
170 | kvaddr, dma_addr); | |
171 | } | |
172 | ||
173 | /* we've too many pages in the iovec, coalesce to a single page */ | |
174 | static int qib_user_sdma_coalesce(const struct qib_devdata *dd, | |
175 | struct qib_user_sdma_pkt *pkt, | |
176 | const struct iovec *iov, | |
177 | unsigned long niov) | |
178 | { | |
179 | int ret = 0; | |
180 | struct page *page = alloc_page(GFP_KERNEL); | |
181 | void *mpage_save; | |
182 | char *mpage; | |
183 | int i; | |
184 | int len = 0; | |
185 | dma_addr_t dma_addr; | |
186 | ||
187 | if (!page) { | |
188 | ret = -ENOMEM; | |
189 | goto done; | |
190 | } | |
191 | ||
192 | mpage = kmap(page); | |
193 | mpage_save = mpage; | |
194 | for (i = 0; i < niov; i++) { | |
195 | int cfur; | |
196 | ||
197 | cfur = copy_from_user(mpage, | |
198 | iov[i].iov_base, iov[i].iov_len); | |
199 | if (cfur) { | |
200 | ret = -EFAULT; | |
201 | goto free_unmap; | |
202 | } | |
203 | ||
204 | mpage += iov[i].iov_len; | |
205 | len += iov[i].iov_len; | |
206 | } | |
207 | ||
208 | dma_addr = dma_map_page(&dd->pcidev->dev, page, 0, len, | |
209 | DMA_TO_DEVICE); | |
210 | if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) { | |
211 | ret = -ENOMEM; | |
212 | goto free_unmap; | |
213 | } | |
214 | ||
215 | qib_user_sdma_init_frag(pkt, 1, 0, len, 0, 1, page, mpage_save, | |
216 | dma_addr); | |
217 | pkt->naddr = 2; | |
218 | ||
219 | goto done; | |
220 | ||
221 | free_unmap: | |
222 | kunmap(page); | |
223 | __free_page(page); | |
224 | done: | |
225 | return ret; | |
226 | } | |
227 | ||
228 | /* | |
229 | * How many pages in this iovec element? | |
230 | */ | |
231 | static int qib_user_sdma_num_pages(const struct iovec *iov) | |
232 | { | |
233 | const unsigned long addr = (unsigned long) iov->iov_base; | |
234 | const unsigned long len = iov->iov_len; | |
235 | const unsigned long spage = addr & PAGE_MASK; | |
236 | const unsigned long epage = (addr + len - 1) & PAGE_MASK; | |
237 | ||
238 | return 1 + ((epage - spage) >> PAGE_SHIFT); | |
239 | } | |
240 | ||
241 | /* | |
25985edc | 242 | * Truncate length to page boundary. |
f931551b RC |
243 | */ |
244 | static int qib_user_sdma_page_length(unsigned long addr, unsigned long len) | |
245 | { | |
246 | const unsigned long offset = addr & ~PAGE_MASK; | |
247 | ||
248 | return ((offset + len) > PAGE_SIZE) ? (PAGE_SIZE - offset) : len; | |
249 | } | |
250 | ||
251 | static void qib_user_sdma_free_pkt_frag(struct device *dev, | |
252 | struct qib_user_sdma_queue *pq, | |
253 | struct qib_user_sdma_pkt *pkt, | |
254 | int frag) | |
255 | { | |
256 | const int i = frag; | |
257 | ||
258 | if (pkt->addr[i].page) { | |
259 | if (pkt->addr[i].dma_mapped) | |
260 | dma_unmap_page(dev, | |
261 | pkt->addr[i].addr, | |
262 | pkt->addr[i].length, | |
263 | DMA_TO_DEVICE); | |
264 | ||
265 | if (pkt->addr[i].kvaddr) | |
266 | kunmap(pkt->addr[i].page); | |
267 | ||
268 | if (pkt->addr[i].put_page) | |
269 | put_page(pkt->addr[i].page); | |
270 | else | |
271 | __free_page(pkt->addr[i].page); | |
272 | } else if (pkt->addr[i].kvaddr) | |
273 | /* free coherent mem from cache... */ | |
274 | dma_pool_free(pq->header_cache, | |
275 | pkt->addr[i].kvaddr, pkt->addr[i].addr); | |
276 | } | |
277 | ||
278 | /* return number of pages pinned... */ | |
279 | static int qib_user_sdma_pin_pages(const struct qib_devdata *dd, | |
280 | struct qib_user_sdma_pkt *pkt, | |
281 | unsigned long addr, int tlen, int npages) | |
282 | { | |
283 | struct page *pages[2]; | |
284 | int j; | |
285 | int ret; | |
286 | ||
287 | ret = get_user_pages(current, current->mm, addr, | |
288 | npages, 0, 1, pages, NULL); | |
289 | ||
290 | if (ret != npages) { | |
291 | int i; | |
292 | ||
293 | for (i = 0; i < ret; i++) | |
294 | put_page(pages[i]); | |
295 | ||
296 | ret = -ENOMEM; | |
297 | goto done; | |
298 | } | |
299 | ||
300 | for (j = 0; j < npages; j++) { | |
301 | /* map the pages... */ | |
302 | const int flen = qib_user_sdma_page_length(addr, tlen); | |
303 | dma_addr_t dma_addr = | |
304 | dma_map_page(&dd->pcidev->dev, | |
305 | pages[j], 0, flen, DMA_TO_DEVICE); | |
306 | unsigned long fofs = addr & ~PAGE_MASK; | |
307 | ||
308 | if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) { | |
309 | ret = -ENOMEM; | |
310 | goto done; | |
311 | } | |
312 | ||
313 | qib_user_sdma_init_frag(pkt, pkt->naddr, fofs, flen, 1, 1, | |
314 | pages[j], kmap(pages[j]), dma_addr); | |
315 | ||
316 | pkt->naddr++; | |
317 | addr += flen; | |
318 | tlen -= flen; | |
319 | } | |
320 | ||
321 | done: | |
322 | return ret; | |
323 | } | |
324 | ||
325 | static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd, | |
326 | struct qib_user_sdma_queue *pq, | |
327 | struct qib_user_sdma_pkt *pkt, | |
328 | const struct iovec *iov, | |
329 | unsigned long niov) | |
330 | { | |
331 | int ret = 0; | |
332 | unsigned long idx; | |
333 | ||
334 | for (idx = 0; idx < niov; idx++) { | |
335 | const int npages = qib_user_sdma_num_pages(iov + idx); | |
336 | const unsigned long addr = (unsigned long) iov[idx].iov_base; | |
337 | ||
338 | ret = qib_user_sdma_pin_pages(dd, pkt, addr, | |
339 | iov[idx].iov_len, npages); | |
340 | if (ret < 0) | |
341 | goto free_pkt; | |
342 | } | |
343 | ||
344 | goto done; | |
345 | ||
346 | free_pkt: | |
347 | for (idx = 0; idx < pkt->naddr; idx++) | |
348 | qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx); | |
349 | ||
350 | done: | |
351 | return ret; | |
352 | } | |
353 | ||
354 | static int qib_user_sdma_init_payload(const struct qib_devdata *dd, | |
355 | struct qib_user_sdma_queue *pq, | |
356 | struct qib_user_sdma_pkt *pkt, | |
357 | const struct iovec *iov, | |
358 | unsigned long niov, int npages) | |
359 | { | |
360 | int ret = 0; | |
361 | ||
362 | if (npages >= ARRAY_SIZE(pkt->addr)) | |
363 | ret = qib_user_sdma_coalesce(dd, pkt, iov, niov); | |
364 | else | |
365 | ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov); | |
366 | ||
367 | return ret; | |
368 | } | |
369 | ||
370 | /* free a packet list -- return counter value of last packet */ | |
371 | static void qib_user_sdma_free_pkt_list(struct device *dev, | |
372 | struct qib_user_sdma_queue *pq, | |
373 | struct list_head *list) | |
374 | { | |
375 | struct qib_user_sdma_pkt *pkt, *pkt_next; | |
376 | ||
377 | list_for_each_entry_safe(pkt, pkt_next, list, list) { | |
378 | int i; | |
379 | ||
380 | for (i = 0; i < pkt->naddr; i++) | |
381 | qib_user_sdma_free_pkt_frag(dev, pq, pkt, i); | |
382 | ||
383 | kmem_cache_free(pq->pkt_slab, pkt); | |
384 | } | |
f73df408 | 385 | INIT_LIST_HEAD(list); |
f931551b RC |
386 | } |
387 | ||
388 | /* | |
389 | * copy headers, coalesce etc -- pq->lock must be held | |
390 | * | |
391 | * we queue all the packets to list, returning the | |
392 | * number of bytes total. list must be empty initially, | |
393 | * as, if there is an error we clean it... | |
394 | */ | |
395 | static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd, | |
396 | struct qib_user_sdma_queue *pq, | |
397 | struct list_head *list, | |
398 | const struct iovec *iov, | |
399 | unsigned long niov, | |
400 | int maxpkts) | |
401 | { | |
402 | unsigned long idx = 0; | |
403 | int ret = 0; | |
404 | int npkts = 0; | |
405 | struct page *page = NULL; | |
406 | __le32 *pbc; | |
407 | dma_addr_t dma_addr; | |
408 | struct qib_user_sdma_pkt *pkt = NULL; | |
409 | size_t len; | |
410 | size_t nw; | |
411 | u32 counter = pq->counter; | |
412 | int dma_mapped = 0; | |
413 | ||
414 | while (idx < niov && npkts < maxpkts) { | |
415 | const unsigned long addr = (unsigned long) iov[idx].iov_base; | |
416 | const unsigned long idx_save = idx; | |
417 | unsigned pktnw; | |
418 | unsigned pktnwc; | |
419 | int nfrags = 0; | |
420 | int npages = 0; | |
421 | int cfur; | |
422 | ||
423 | dma_mapped = 0; | |
424 | len = iov[idx].iov_len; | |
425 | nw = len >> 2; | |
426 | page = NULL; | |
427 | ||
428 | pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL); | |
429 | if (!pkt) { | |
430 | ret = -ENOMEM; | |
431 | goto free_list; | |
432 | } | |
433 | ||
434 | if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH || | |
435 | len > PAGE_SIZE || len & 3 || addr & 3) { | |
436 | ret = -EINVAL; | |
437 | goto free_pkt; | |
438 | } | |
439 | ||
440 | if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH) | |
441 | pbc = dma_pool_alloc(pq->header_cache, GFP_KERNEL, | |
442 | &dma_addr); | |
443 | else | |
444 | pbc = NULL; | |
445 | ||
446 | if (!pbc) { | |
447 | page = alloc_page(GFP_KERNEL); | |
448 | if (!page) { | |
449 | ret = -ENOMEM; | |
450 | goto free_pkt; | |
451 | } | |
452 | pbc = kmap(page); | |
453 | } | |
454 | ||
455 | cfur = copy_from_user(pbc, iov[idx].iov_base, len); | |
456 | if (cfur) { | |
457 | ret = -EFAULT; | |
458 | goto free_pbc; | |
459 | } | |
460 | ||
461 | /* | |
462 | * This assignment is a bit strange. it's because the | |
463 | * the pbc counts the number of 32 bit words in the full | |
464 | * packet _except_ the first word of the pbc itself... | |
465 | */ | |
466 | pktnwc = nw - 1; | |
467 | ||
468 | /* | |
469 | * pktnw computation yields the number of 32 bit words | |
470 | * that the caller has indicated in the PBC. note that | |
471 | * this is one less than the total number of words that | |
472 | * goes to the send DMA engine as the first 32 bit word | |
473 | * of the PBC itself is not counted. Armed with this count, | |
474 | * we can verify that the packet is consistent with the | |
475 | * iovec lengths. | |
476 | */ | |
477 | pktnw = le32_to_cpu(*pbc) & QIB_PBC_LENGTH_MASK; | |
478 | if (pktnw < pktnwc || pktnw > pktnwc + (PAGE_SIZE >> 2)) { | |
479 | ret = -EINVAL; | |
480 | goto free_pbc; | |
481 | } | |
482 | ||
483 | idx++; | |
484 | while (pktnwc < pktnw && idx < niov) { | |
485 | const size_t slen = iov[idx].iov_len; | |
486 | const unsigned long faddr = | |
487 | (unsigned long) iov[idx].iov_base; | |
488 | ||
489 | if (slen & 3 || faddr & 3 || !slen || | |
490 | slen > PAGE_SIZE) { | |
491 | ret = -EINVAL; | |
492 | goto free_pbc; | |
493 | } | |
494 | ||
495 | npages++; | |
496 | if ((faddr & PAGE_MASK) != | |
497 | ((faddr + slen - 1) & PAGE_MASK)) | |
498 | npages++; | |
499 | ||
500 | pktnwc += slen >> 2; | |
501 | idx++; | |
502 | nfrags++; | |
503 | } | |
504 | ||
505 | if (pktnwc != pktnw) { | |
506 | ret = -EINVAL; | |
507 | goto free_pbc; | |
508 | } | |
509 | ||
510 | if (page) { | |
511 | dma_addr = dma_map_page(&dd->pcidev->dev, | |
512 | page, 0, len, DMA_TO_DEVICE); | |
513 | if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) { | |
514 | ret = -ENOMEM; | |
515 | goto free_pbc; | |
516 | } | |
517 | ||
518 | dma_mapped = 1; | |
519 | } | |
520 | ||
521 | qib_user_sdma_init_header(pkt, counter, 0, len, dma_mapped, | |
522 | page, pbc, dma_addr); | |
523 | ||
524 | if (nfrags) { | |
525 | ret = qib_user_sdma_init_payload(dd, pq, pkt, | |
526 | iov + idx_save + 1, | |
527 | nfrags, npages); | |
528 | if (ret < 0) | |
529 | goto free_pbc_dma; | |
530 | } | |
531 | ||
532 | counter++; | |
533 | npkts++; | |
534 | ||
535 | list_add_tail(&pkt->list, list); | |
536 | } | |
537 | ||
538 | ret = idx; | |
539 | goto done; | |
540 | ||
541 | free_pbc_dma: | |
542 | if (dma_mapped) | |
543 | dma_unmap_page(&dd->pcidev->dev, dma_addr, len, DMA_TO_DEVICE); | |
544 | free_pbc: | |
545 | if (page) { | |
546 | kunmap(page); | |
547 | __free_page(page); | |
548 | } else | |
549 | dma_pool_free(pq->header_cache, pbc, dma_addr); | |
550 | free_pkt: | |
551 | kmem_cache_free(pq->pkt_slab, pkt); | |
552 | free_list: | |
553 | qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list); | |
554 | done: | |
555 | return ret; | |
556 | } | |
557 | ||
558 | static void qib_user_sdma_set_complete_counter(struct qib_user_sdma_queue *pq, | |
559 | u32 c) | |
560 | { | |
561 | pq->sent_counter = c; | |
562 | } | |
563 | ||
564 | /* try to clean out queue -- needs pq->lock */ | |
565 | static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd, | |
566 | struct qib_user_sdma_queue *pq) | |
567 | { | |
568 | struct qib_devdata *dd = ppd->dd; | |
569 | struct list_head free_list; | |
570 | struct qib_user_sdma_pkt *pkt; | |
571 | struct qib_user_sdma_pkt *pkt_prev; | |
572 | int ret = 0; | |
573 | ||
574 | INIT_LIST_HEAD(&free_list); | |
575 | ||
576 | list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) { | |
577 | s64 descd = ppd->sdma_descq_removed - pkt->added; | |
578 | ||
579 | if (descd < 0) | |
580 | break; | |
581 | ||
582 | list_move_tail(&pkt->list, &free_list); | |
583 | ||
584 | /* one more packet cleaned */ | |
585 | ret++; | |
586 | } | |
587 | ||
588 | if (!list_empty(&free_list)) { | |
589 | u32 counter; | |
590 | ||
591 | pkt = list_entry(free_list.prev, | |
592 | struct qib_user_sdma_pkt, list); | |
593 | counter = pkt->counter; | |
594 | ||
595 | qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list); | |
596 | qib_user_sdma_set_complete_counter(pq, counter); | |
597 | } | |
598 | ||
599 | return ret; | |
600 | } | |
601 | ||
602 | void qib_user_sdma_queue_destroy(struct qib_user_sdma_queue *pq) | |
603 | { | |
604 | if (!pq) | |
605 | return; | |
606 | ||
607 | kmem_cache_destroy(pq->pkt_slab); | |
608 | dma_pool_destroy(pq->header_cache); | |
609 | kfree(pq); | |
610 | } | |
611 | ||
612 | /* clean descriptor queue, returns > 0 if some elements cleaned */ | |
613 | static int qib_user_sdma_hwqueue_clean(struct qib_pportdata *ppd) | |
614 | { | |
615 | int ret; | |
616 | unsigned long flags; | |
617 | ||
618 | spin_lock_irqsave(&ppd->sdma_lock, flags); | |
619 | ret = qib_sdma_make_progress(ppd); | |
620 | spin_unlock_irqrestore(&ppd->sdma_lock, flags); | |
621 | ||
622 | return ret; | |
623 | } | |
624 | ||
625 | /* we're in close, drain packets so that we can cleanup successfully... */ | |
626 | void qib_user_sdma_queue_drain(struct qib_pportdata *ppd, | |
627 | struct qib_user_sdma_queue *pq) | |
628 | { | |
629 | struct qib_devdata *dd = ppd->dd; | |
630 | int i; | |
631 | ||
632 | if (!pq) | |
633 | return; | |
634 | ||
635 | for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) { | |
636 | mutex_lock(&pq->lock); | |
637 | if (list_empty(&pq->sent)) { | |
638 | mutex_unlock(&pq->lock); | |
639 | break; | |
640 | } | |
641 | qib_user_sdma_hwqueue_clean(ppd); | |
642 | qib_user_sdma_queue_clean(ppd, pq); | |
643 | mutex_unlock(&pq->lock); | |
644 | msleep(10); | |
645 | } | |
646 | ||
647 | if (!list_empty(&pq->sent)) { | |
648 | struct list_head free_list; | |
649 | ||
650 | qib_dev_err(dd, "user sdma lists not empty: forcing!\n"); | |
651 | INIT_LIST_HEAD(&free_list); | |
652 | mutex_lock(&pq->lock); | |
653 | list_splice_init(&pq->sent, &free_list); | |
654 | qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list); | |
655 | mutex_unlock(&pq->lock); | |
656 | } | |
657 | } | |
658 | ||
659 | static inline __le64 qib_sdma_make_desc0(struct qib_pportdata *ppd, | |
660 | u64 addr, u64 dwlen, u64 dwoffset) | |
661 | { | |
662 | u8 tmpgen; | |
663 | ||
664 | tmpgen = ppd->sdma_generation; | |
665 | ||
666 | return cpu_to_le64(/* SDmaPhyAddr[31:0] */ | |
667 | ((addr & 0xfffffffcULL) << 32) | | |
668 | /* SDmaGeneration[1:0] */ | |
669 | ((tmpgen & 3ULL) << 30) | | |
670 | /* SDmaDwordCount[10:0] */ | |
671 | ((dwlen & 0x7ffULL) << 16) | | |
672 | /* SDmaBufOffset[12:2] */ | |
673 | (dwoffset & 0x7ffULL)); | |
674 | } | |
675 | ||
676 | static inline __le64 qib_sdma_make_first_desc0(__le64 descq) | |
677 | { | |
678 | return descq | cpu_to_le64(1ULL << 12); | |
679 | } | |
680 | ||
681 | static inline __le64 qib_sdma_make_last_desc0(__le64 descq) | |
682 | { | |
683 | /* last */ /* dma head */ | |
684 | return descq | cpu_to_le64(1ULL << 11 | 1ULL << 13); | |
685 | } | |
686 | ||
687 | static inline __le64 qib_sdma_make_desc1(u64 addr) | |
688 | { | |
689 | /* SDmaPhyAddr[47:32] */ | |
690 | return cpu_to_le64(addr >> 32); | |
691 | } | |
692 | ||
693 | static void qib_user_sdma_send_frag(struct qib_pportdata *ppd, | |
694 | struct qib_user_sdma_pkt *pkt, int idx, | |
695 | unsigned ofs, u16 tail) | |
696 | { | |
697 | const u64 addr = (u64) pkt->addr[idx].addr + | |
698 | (u64) pkt->addr[idx].offset; | |
699 | const u64 dwlen = (u64) pkt->addr[idx].length / 4; | |
700 | __le64 *descqp; | |
701 | __le64 descq0; | |
702 | ||
703 | descqp = &ppd->sdma_descq[tail].qw[0]; | |
704 | ||
705 | descq0 = qib_sdma_make_desc0(ppd, addr, dwlen, ofs); | |
706 | if (idx == 0) | |
707 | descq0 = qib_sdma_make_first_desc0(descq0); | |
708 | if (idx == pkt->naddr - 1) | |
709 | descq0 = qib_sdma_make_last_desc0(descq0); | |
710 | ||
711 | descqp[0] = descq0; | |
712 | descqp[1] = qib_sdma_make_desc1(addr); | |
713 | } | |
714 | ||
715 | /* pq->lock must be held, get packets on the wire... */ | |
716 | static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd, | |
717 | struct qib_user_sdma_queue *pq, | |
718 | struct list_head *pktlist) | |
719 | { | |
720 | struct qib_devdata *dd = ppd->dd; | |
721 | int ret = 0; | |
722 | unsigned long flags; | |
723 | u16 tail; | |
724 | u8 generation; | |
725 | u64 descq_added; | |
726 | ||
727 | if (list_empty(pktlist)) | |
728 | return 0; | |
729 | ||
730 | if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE))) | |
731 | return -ECOMM; | |
732 | ||
733 | spin_lock_irqsave(&ppd->sdma_lock, flags); | |
734 | ||
735 | /* keep a copy for restoring purposes in case of problems */ | |
736 | generation = ppd->sdma_generation; | |
737 | descq_added = ppd->sdma_descq_added; | |
738 | ||
739 | if (unlikely(!__qib_sdma_running(ppd))) { | |
740 | ret = -ECOMM; | |
741 | goto unlock; | |
742 | } | |
743 | ||
744 | tail = ppd->sdma_descq_tail; | |
745 | while (!list_empty(pktlist)) { | |
746 | struct qib_user_sdma_pkt *pkt = | |
747 | list_entry(pktlist->next, struct qib_user_sdma_pkt, | |
748 | list); | |
749 | int i; | |
750 | unsigned ofs = 0; | |
751 | u16 dtail = tail; | |
752 | ||
753 | if (pkt->naddr > qib_sdma_descq_freecnt(ppd)) | |
754 | goto unlock_check_tail; | |
755 | ||
756 | for (i = 0; i < pkt->naddr; i++) { | |
757 | qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail); | |
758 | ofs += pkt->addr[i].length >> 2; | |
759 | ||
760 | if (++tail == ppd->sdma_descq_cnt) { | |
761 | tail = 0; | |
762 | ++ppd->sdma_generation; | |
763 | } | |
764 | } | |
765 | ||
766 | if ((ofs << 2) > ppd->ibmaxlen) { | |
767 | ret = -EMSGSIZE; | |
768 | goto unlock; | |
769 | } | |
770 | ||
771 | /* | |
772 | * If the packet is >= 2KB mtu equivalent, we have to use | |
773 | * the large buffers, and have to mark each descriptor as | |
774 | * part of a large buffer packet. | |
775 | */ | |
776 | if (ofs > dd->piosize2kmax_dwords) { | |
777 | for (i = 0; i < pkt->naddr; i++) { | |
778 | ppd->sdma_descq[dtail].qw[0] |= | |
779 | cpu_to_le64(1ULL << 14); | |
780 | if (++dtail == ppd->sdma_descq_cnt) | |
781 | dtail = 0; | |
782 | } | |
783 | } | |
784 | ||
785 | ppd->sdma_descq_added += pkt->naddr; | |
786 | pkt->added = ppd->sdma_descq_added; | |
787 | list_move_tail(&pkt->list, &pq->sent); | |
788 | ret++; | |
789 | } | |
790 | ||
791 | unlock_check_tail: | |
792 | /* advance the tail on the chip if necessary */ | |
793 | if (ppd->sdma_descq_tail != tail) | |
794 | dd->f_sdma_update_tail(ppd, tail); | |
795 | ||
796 | unlock: | |
797 | if (unlikely(ret < 0)) { | |
798 | ppd->sdma_generation = generation; | |
799 | ppd->sdma_descq_added = descq_added; | |
800 | } | |
801 | spin_unlock_irqrestore(&ppd->sdma_lock, flags); | |
802 | ||
803 | return ret; | |
804 | } | |
805 | ||
806 | int qib_user_sdma_writev(struct qib_ctxtdata *rcd, | |
807 | struct qib_user_sdma_queue *pq, | |
808 | const struct iovec *iov, | |
809 | unsigned long dim) | |
810 | { | |
811 | struct qib_devdata *dd = rcd->dd; | |
812 | struct qib_pportdata *ppd = rcd->ppd; | |
813 | int ret = 0; | |
814 | struct list_head list; | |
815 | int npkts = 0; | |
816 | ||
817 | INIT_LIST_HEAD(&list); | |
818 | ||
819 | mutex_lock(&pq->lock); | |
820 | ||
821 | /* why not -ECOMM like qib_user_sdma_push_pkts() below? */ | |
822 | if (!qib_sdma_running(ppd)) | |
823 | goto done_unlock; | |
824 | ||
825 | if (ppd->sdma_descq_added != ppd->sdma_descq_removed) { | |
826 | qib_user_sdma_hwqueue_clean(ppd); | |
827 | qib_user_sdma_queue_clean(ppd, pq); | |
828 | } | |
829 | ||
830 | while (dim) { | |
831 | const int mxp = 8; | |
832 | ||
833 | down_write(¤t->mm->mmap_sem); | |
834 | ret = qib_user_sdma_queue_pkts(dd, pq, &list, iov, dim, mxp); | |
835 | up_write(¤t->mm->mmap_sem); | |
836 | ||
837 | if (ret <= 0) | |
838 | goto done_unlock; | |
839 | else { | |
840 | dim -= ret; | |
841 | iov += ret; | |
842 | } | |
843 | ||
844 | /* force packets onto the sdma hw queue... */ | |
845 | if (!list_empty(&list)) { | |
846 | /* | |
847 | * Lazily clean hw queue. the 4 is a guess of about | |
848 | * how many sdma descriptors a packet will take (it | |
849 | * doesn't have to be perfect). | |
850 | */ | |
851 | if (qib_sdma_descq_freecnt(ppd) < ret * 4) { | |
852 | qib_user_sdma_hwqueue_clean(ppd); | |
853 | qib_user_sdma_queue_clean(ppd, pq); | |
854 | } | |
855 | ||
856 | ret = qib_user_sdma_push_pkts(ppd, pq, &list); | |
857 | if (ret < 0) | |
858 | goto done_unlock; | |
859 | else { | |
860 | npkts += ret; | |
861 | pq->counter += ret; | |
862 | ||
863 | if (!list_empty(&list)) | |
864 | goto done_unlock; | |
865 | } | |
866 | } | |
867 | } | |
868 | ||
869 | done_unlock: | |
870 | if (!list_empty(&list)) | |
871 | qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &list); | |
872 | mutex_unlock(&pq->lock); | |
873 | ||
874 | return (ret < 0) ? ret : npkts; | |
875 | } | |
876 | ||
877 | int qib_user_sdma_make_progress(struct qib_pportdata *ppd, | |
878 | struct qib_user_sdma_queue *pq) | |
879 | { | |
880 | int ret = 0; | |
881 | ||
882 | mutex_lock(&pq->lock); | |
883 | qib_user_sdma_hwqueue_clean(ppd); | |
884 | ret = qib_user_sdma_queue_clean(ppd, pq); | |
885 | mutex_unlock(&pq->lock); | |
886 | ||
887 | return ret; | |
888 | } | |
889 | ||
890 | u32 qib_user_sdma_complete_counter(const struct qib_user_sdma_queue *pq) | |
891 | { | |
892 | return pq ? pq->sent_counter : 0; | |
893 | } | |
894 | ||
895 | u32 qib_user_sdma_inflight_counter(struct qib_user_sdma_queue *pq) | |
896 | { | |
897 | return pq ? pq->counter : 0; | |
898 | } |