Commit | Line | Data |
---|---|---|
f931551b RC |
1 | /* |
2 | * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. | |
3 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. | |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include <rdma/ib_umem.h> | |
35 | #include <rdma/ib_smi.h> | |
36 | ||
37 | #include "qib.h" | |
38 | ||
39 | /* Fast memory region */ | |
40 | struct qib_fmr { | |
41 | struct ib_fmr ibfmr; | |
f931551b RC |
42 | struct qib_mregion mr; /* must be last */ |
43 | }; | |
44 | ||
45 | static inline struct qib_fmr *to_ifmr(struct ib_fmr *ibfmr) | |
46 | { | |
47 | return container_of(ibfmr, struct qib_fmr, ibfmr); | |
48 | } | |
49 | ||
6a82649f MM |
50 | static int init_qib_mregion(struct qib_mregion *mr, struct ib_pd *pd, |
51 | int count) | |
52 | { | |
53 | int m, i = 0; | |
54 | int rval = 0; | |
55 | ||
56 | m = (count + QIB_SEGSZ - 1) / QIB_SEGSZ; | |
57 | for (; i < m; i++) { | |
041af0bb | 58 | mr->map[i] = kzalloc(sizeof(*mr->map[0]), GFP_KERNEL); |
6a82649f MM |
59 | if (!mr->map[i]) |
60 | goto bail; | |
61 | } | |
62 | mr->mapsz = m; | |
63 | init_completion(&mr->comp); | |
64 | /* count returning the ptr to user */ | |
65 | atomic_set(&mr->refcount, 1); | |
66 | mr->pd = pd; | |
67 | mr->max_segs = count; | |
68 | out: | |
69 | return rval; | |
70 | bail: | |
71 | while (i) | |
72 | kfree(mr->map[--i]); | |
73 | rval = -ENOMEM; | |
74 | goto out; | |
75 | } | |
76 | ||
77 | static void deinit_qib_mregion(struct qib_mregion *mr) | |
78 | { | |
79 | int i = mr->mapsz; | |
80 | ||
81 | mr->mapsz = 0; | |
82 | while (i) | |
83 | kfree(mr->map[--i]); | |
84 | } | |
85 | ||
86 | ||
f931551b RC |
87 | /** |
88 | * qib_get_dma_mr - get a DMA memory region | |
89 | * @pd: protection domain for this memory region | |
90 | * @acc: access flags | |
91 | * | |
92 | * Returns the memory region on success, otherwise returns an errno. | |
93 | * Note that all DMA addresses should be created via the | |
94 | * struct ib_dma_mapping_ops functions (see qib_dma.c). | |
95 | */ | |
96 | struct ib_mr *qib_get_dma_mr(struct ib_pd *pd, int acc) | |
97 | { | |
6a82649f | 98 | struct qib_mr *mr = NULL; |
f931551b | 99 | struct ib_mr *ret; |
6a82649f | 100 | int rval; |
f931551b RC |
101 | |
102 | if (to_ipd(pd)->user) { | |
103 | ret = ERR_PTR(-EPERM); | |
104 | goto bail; | |
105 | } | |
106 | ||
041af0bb | 107 | mr = kzalloc(sizeof(*mr), GFP_KERNEL); |
f931551b RC |
108 | if (!mr) { |
109 | ret = ERR_PTR(-ENOMEM); | |
110 | goto bail; | |
111 | } | |
112 | ||
6a82649f MM |
113 | rval = init_qib_mregion(&mr->mr, pd, 0); |
114 | if (rval) { | |
115 | ret = ERR_PTR(rval); | |
116 | goto bail; | |
117 | } | |
f931551b | 118 | |
f931551b | 119 | |
6a82649f MM |
120 | rval = qib_alloc_lkey(&mr->mr, 1); |
121 | if (rval) { | |
122 | ret = ERR_PTR(rval); | |
123 | goto bail_mregion; | |
124 | } | |
125 | ||
126 | mr->mr.access_flags = acc; | |
f931551b | 127 | ret = &mr->ibmr; |
6a82649f MM |
128 | done: |
129 | return ret; | |
f931551b | 130 | |
6a82649f MM |
131 | bail_mregion: |
132 | deinit_qib_mregion(&mr->mr); | |
f931551b | 133 | bail: |
6a82649f MM |
134 | kfree(mr); |
135 | goto done; | |
f931551b RC |
136 | } |
137 | ||
6a82649f | 138 | static struct qib_mr *alloc_mr(int count, struct ib_pd *pd) |
f931551b RC |
139 | { |
140 | struct qib_mr *mr; | |
6a82649f MM |
141 | int rval = -ENOMEM; |
142 | int m; | |
f931551b RC |
143 | |
144 | /* Allocate struct plus pointers to first level page tables. */ | |
145 | m = (count + QIB_SEGSZ - 1) / QIB_SEGSZ; | |
041af0bb | 146 | mr = kzalloc(sizeof(*mr) + m * sizeof(mr->mr.map[0]), GFP_KERNEL); |
f931551b | 147 | if (!mr) |
6a82649f | 148 | goto bail; |
f931551b | 149 | |
6a82649f MM |
150 | rval = init_qib_mregion(&mr->mr, pd, count); |
151 | if (rval) | |
152 | goto bail; | |
f931551b RC |
153 | /* |
154 | * ib_reg_phys_mr() will initialize mr->ibmr except for | |
155 | * lkey and rkey. | |
156 | */ | |
6a82649f MM |
157 | rval = qib_alloc_lkey(&mr->mr, 0); |
158 | if (rval) | |
159 | goto bail_mregion; | |
f931551b RC |
160 | mr->ibmr.lkey = mr->mr.lkey; |
161 | mr->ibmr.rkey = mr->mr.lkey; | |
6a82649f MM |
162 | done: |
163 | return mr; | |
f931551b | 164 | |
6a82649f MM |
165 | bail_mregion: |
166 | deinit_qib_mregion(&mr->mr); | |
f931551b | 167 | bail: |
f931551b | 168 | kfree(mr); |
6a82649f MM |
169 | mr = ERR_PTR(rval); |
170 | goto done; | |
f931551b RC |
171 | } |
172 | ||
173 | /** | |
174 | * qib_reg_phys_mr - register a physical memory region | |
175 | * @pd: protection domain for this memory region | |
176 | * @buffer_list: pointer to the list of physical buffers to register | |
177 | * @num_phys_buf: the number of physical buffers to register | |
178 | * @iova_start: the starting address passed over IB which maps to this MR | |
179 | * | |
180 | * Returns the memory region on success, otherwise returns an errno. | |
181 | */ | |
182 | struct ib_mr *qib_reg_phys_mr(struct ib_pd *pd, | |
183 | struct ib_phys_buf *buffer_list, | |
184 | int num_phys_buf, int acc, u64 *iova_start) | |
185 | { | |
186 | struct qib_mr *mr; | |
187 | int n, m, i; | |
188 | struct ib_mr *ret; | |
189 | ||
6a82649f MM |
190 | mr = alloc_mr(num_phys_buf, pd); |
191 | if (IS_ERR(mr)) { | |
192 | ret = (struct ib_mr *)mr; | |
f931551b RC |
193 | goto bail; |
194 | } | |
195 | ||
f931551b RC |
196 | mr->mr.user_base = *iova_start; |
197 | mr->mr.iova = *iova_start; | |
f931551b | 198 | mr->mr.access_flags = acc; |
f931551b RC |
199 | |
200 | m = 0; | |
201 | n = 0; | |
202 | for (i = 0; i < num_phys_buf; i++) { | |
203 | mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr; | |
204 | mr->mr.map[m]->segs[n].length = buffer_list[i].size; | |
205 | mr->mr.length += buffer_list[i].size; | |
206 | n++; | |
207 | if (n == QIB_SEGSZ) { | |
208 | m++; | |
209 | n = 0; | |
210 | } | |
211 | } | |
212 | ||
213 | ret = &mr->ibmr; | |
214 | ||
215 | bail: | |
216 | return ret; | |
217 | } | |
218 | ||
219 | /** | |
220 | * qib_reg_user_mr - register a userspace memory region | |
221 | * @pd: protection domain for this memory region | |
222 | * @start: starting userspace address | |
223 | * @length: length of region to register | |
f931551b RC |
224 | * @mr_access_flags: access flags for this memory region |
225 | * @udata: unused by the QLogic_IB driver | |
226 | * | |
227 | * Returns the memory region on success, otherwise returns an errno. | |
228 | */ | |
229 | struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, | |
230 | u64 virt_addr, int mr_access_flags, | |
231 | struct ib_udata *udata) | |
232 | { | |
233 | struct qib_mr *mr; | |
234 | struct ib_umem *umem; | |
eeb8461e YH |
235 | struct scatterlist *sg; |
236 | int n, m, entry; | |
f931551b RC |
237 | struct ib_mr *ret; |
238 | ||
239 | if (length == 0) { | |
240 | ret = ERR_PTR(-EINVAL); | |
241 | goto bail; | |
242 | } | |
243 | ||
244 | umem = ib_umem_get(pd->uobject->context, start, length, | |
245 | mr_access_flags, 0); | |
246 | if (IS_ERR(umem)) | |
247 | return (void *) umem; | |
248 | ||
eeb8461e | 249 | n = umem->nmap; |
f931551b | 250 | |
6a82649f MM |
251 | mr = alloc_mr(n, pd); |
252 | if (IS_ERR(mr)) { | |
253 | ret = (struct ib_mr *)mr; | |
f931551b RC |
254 | ib_umem_release(umem); |
255 | goto bail; | |
256 | } | |
257 | ||
f931551b RC |
258 | mr->mr.user_base = start; |
259 | mr->mr.iova = virt_addr; | |
260 | mr->mr.length = length; | |
406f9e5f | 261 | mr->mr.offset = ib_umem_offset(umem); |
f931551b RC |
262 | mr->mr.access_flags = mr_access_flags; |
263 | mr->umem = umem; | |
264 | ||
2a600f14 MM |
265 | if (is_power_of_2(umem->page_size)) |
266 | mr->mr.page_shift = ilog2(umem->page_size); | |
f931551b RC |
267 | m = 0; |
268 | n = 0; | |
eeb8461e | 269 | for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { |
f931551b RC |
270 | void *vaddr; |
271 | ||
eeb8461e | 272 | vaddr = page_address(sg_page(sg)); |
f931551b RC |
273 | if (!vaddr) { |
274 | ret = ERR_PTR(-EINVAL); | |
275 | goto bail; | |
276 | } | |
277 | mr->mr.map[m]->segs[n].vaddr = vaddr; | |
278 | mr->mr.map[m]->segs[n].length = umem->page_size; | |
279 | n++; | |
280 | if (n == QIB_SEGSZ) { | |
281 | m++; | |
282 | n = 0; | |
283 | } | |
f931551b RC |
284 | } |
285 | ret = &mr->ibmr; | |
286 | ||
287 | bail: | |
288 | return ret; | |
289 | } | |
290 | ||
291 | /** | |
292 | * qib_dereg_mr - unregister and free a memory region | |
293 | * @ibmr: the memory region to free | |
294 | * | |
295 | * Returns 0 on success. | |
296 | * | |
297 | * Note that this is called to free MRs created by qib_get_dma_mr() | |
298 | * or qib_reg_user_mr(). | |
299 | */ | |
300 | int qib_dereg_mr(struct ib_mr *ibmr) | |
301 | { | |
302 | struct qib_mr *mr = to_imr(ibmr); | |
6a82649f MM |
303 | int ret = 0; |
304 | unsigned long timeout; | |
305 | ||
38071a46 | 306 | kfree(mr->pages); |
6a82649f MM |
307 | qib_free_lkey(&mr->mr); |
308 | ||
309 | qib_put_mr(&mr->mr); /* will set completion if last */ | |
310 | timeout = wait_for_completion_timeout(&mr->mr.comp, | |
311 | 5 * HZ); | |
312 | if (!timeout) { | |
313 | qib_get_mr(&mr->mr); | |
314 | ret = -EBUSY; | |
315 | goto out; | |
316 | } | |
317 | deinit_qib_mregion(&mr->mr); | |
f931551b RC |
318 | if (mr->umem) |
319 | ib_umem_release(mr->umem); | |
320 | kfree(mr); | |
6a82649f MM |
321 | out: |
322 | return ret; | |
f931551b RC |
323 | } |
324 | ||
325 | /* | |
326 | * Allocate a memory region usable with the | |
b8533ecc | 327 | * IB_WR_REG_MR send work request. |
f931551b RC |
328 | * |
329 | * Return the memory region on success, otherwise return an errno. | |
330 | */ | |
1302f845 SG |
331 | struct ib_mr *qib_alloc_mr(struct ib_pd *pd, |
332 | enum ib_mr_type mr_type, | |
333 | u32 max_num_sg) | |
f931551b RC |
334 | { |
335 | struct qib_mr *mr; | |
336 | ||
1302f845 SG |
337 | if (mr_type != IB_MR_TYPE_MEM_REG) |
338 | return ERR_PTR(-EINVAL); | |
339 | ||
340 | mr = alloc_mr(max_num_sg, pd); | |
6a82649f MM |
341 | if (IS_ERR(mr)) |
342 | return (struct ib_mr *)mr; | |
f931551b | 343 | |
38071a46 SG |
344 | mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); |
345 | if (!mr->pages) | |
346 | goto err; | |
347 | ||
f931551b | 348 | return &mr->ibmr; |
38071a46 SG |
349 | |
350 | err: | |
351 | qib_dereg_mr(&mr->ibmr); | |
352 | return ERR_PTR(-ENOMEM); | |
353 | } | |
354 | ||
355 | static int qib_set_page(struct ib_mr *ibmr, u64 addr) | |
356 | { | |
357 | struct qib_mr *mr = to_imr(ibmr); | |
358 | ||
359 | if (unlikely(mr->npages == mr->mr.max_segs)) | |
360 | return -ENOMEM; | |
361 | ||
362 | mr->pages[mr->npages++] = addr; | |
363 | ||
364 | return 0; | |
365 | } | |
366 | ||
367 | int qib_map_mr_sg(struct ib_mr *ibmr, | |
368 | struct scatterlist *sg, | |
369 | int sg_nents) | |
370 | { | |
371 | struct qib_mr *mr = to_imr(ibmr); | |
372 | ||
373 | mr->npages = 0; | |
374 | ||
375 | return ib_sg_to_pages(ibmr, sg, sg_nents, qib_set_page); | |
f931551b RC |
376 | } |
377 | ||
f931551b RC |
378 | /** |
379 | * qib_alloc_fmr - allocate a fast memory region | |
380 | * @pd: the protection domain for this memory region | |
381 | * @mr_access_flags: access flags for this memory region | |
382 | * @fmr_attr: fast memory region attributes | |
383 | * | |
384 | * Returns the memory region on success, otherwise returns an errno. | |
385 | */ | |
386 | struct ib_fmr *qib_alloc_fmr(struct ib_pd *pd, int mr_access_flags, | |
387 | struct ib_fmr_attr *fmr_attr) | |
388 | { | |
389 | struct qib_fmr *fmr; | |
6a82649f | 390 | int m; |
f931551b | 391 | struct ib_fmr *ret; |
6a82649f | 392 | int rval = -ENOMEM; |
f931551b RC |
393 | |
394 | /* Allocate struct plus pointers to first level page tables. */ | |
395 | m = (fmr_attr->max_pages + QIB_SEGSZ - 1) / QIB_SEGSZ; | |
041af0bb | 396 | fmr = kzalloc(sizeof(*fmr) + m * sizeof(fmr->mr.map[0]), GFP_KERNEL); |
f931551b RC |
397 | if (!fmr) |
398 | goto bail; | |
399 | ||
6a82649f MM |
400 | rval = init_qib_mregion(&fmr->mr, pd, fmr_attr->max_pages); |
401 | if (rval) | |
402 | goto bail; | |
f931551b RC |
403 | |
404 | /* | |
405 | * ib_alloc_fmr() will initialize fmr->ibfmr except for lkey & | |
406 | * rkey. | |
407 | */ | |
6a82649f MM |
408 | rval = qib_alloc_lkey(&fmr->mr, 0); |
409 | if (rval) | |
410 | goto bail_mregion; | |
f931551b RC |
411 | fmr->ibfmr.rkey = fmr->mr.lkey; |
412 | fmr->ibfmr.lkey = fmr->mr.lkey; | |
413 | /* | |
414 | * Resources are allocated but no valid mapping (RKEY can't be | |
415 | * used). | |
416 | */ | |
f931551b RC |
417 | fmr->mr.access_flags = mr_access_flags; |
418 | fmr->mr.max_segs = fmr_attr->max_pages; | |
2a600f14 | 419 | fmr->mr.page_shift = fmr_attr->page_shift; |
f931551b | 420 | |
f931551b | 421 | ret = &fmr->ibfmr; |
6a82649f MM |
422 | done: |
423 | return ret; | |
f931551b | 424 | |
6a82649f MM |
425 | bail_mregion: |
426 | deinit_qib_mregion(&fmr->mr); | |
f931551b | 427 | bail: |
f931551b | 428 | kfree(fmr); |
6a82649f MM |
429 | ret = ERR_PTR(rval); |
430 | goto done; | |
f931551b RC |
431 | } |
432 | ||
433 | /** | |
434 | * qib_map_phys_fmr - set up a fast memory region | |
435 | * @ibmfr: the fast memory region to set up | |
436 | * @page_list: the list of pages to associate with the fast memory region | |
437 | * @list_len: the number of pages to associate with the fast memory region | |
438 | * @iova: the virtual address of the start of the fast memory region | |
439 | * | |
440 | * This may be called from interrupt context. | |
441 | */ | |
442 | ||
443 | int qib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, | |
444 | int list_len, u64 iova) | |
445 | { | |
446 | struct qib_fmr *fmr = to_ifmr(ibfmr); | |
447 | struct qib_lkey_table *rkt; | |
448 | unsigned long flags; | |
449 | int m, n, i; | |
450 | u32 ps; | |
451 | int ret; | |
452 | ||
6a82649f MM |
453 | i = atomic_read(&fmr->mr.refcount); |
454 | if (i > 2) | |
f931551b RC |
455 | return -EBUSY; |
456 | ||
457 | if (list_len > fmr->mr.max_segs) { | |
458 | ret = -EINVAL; | |
459 | goto bail; | |
460 | } | |
461 | rkt = &to_idev(ibfmr->device)->lk_table; | |
462 | spin_lock_irqsave(&rkt->lock, flags); | |
463 | fmr->mr.user_base = iova; | |
464 | fmr->mr.iova = iova; | |
2a600f14 | 465 | ps = 1 << fmr->mr.page_shift; |
f931551b RC |
466 | fmr->mr.length = list_len * ps; |
467 | m = 0; | |
468 | n = 0; | |
469 | for (i = 0; i < list_len; i++) { | |
470 | fmr->mr.map[m]->segs[n].vaddr = (void *) page_list[i]; | |
471 | fmr->mr.map[m]->segs[n].length = ps; | |
472 | if (++n == QIB_SEGSZ) { | |
473 | m++; | |
474 | n = 0; | |
475 | } | |
476 | } | |
477 | spin_unlock_irqrestore(&rkt->lock, flags); | |
478 | ret = 0; | |
479 | ||
480 | bail: | |
481 | return ret; | |
482 | } | |
483 | ||
484 | /** | |
485 | * qib_unmap_fmr - unmap fast memory regions | |
486 | * @fmr_list: the list of fast memory regions to unmap | |
487 | * | |
488 | * Returns 0 on success. | |
489 | */ | |
490 | int qib_unmap_fmr(struct list_head *fmr_list) | |
491 | { | |
492 | struct qib_fmr *fmr; | |
493 | struct qib_lkey_table *rkt; | |
494 | unsigned long flags; | |
495 | ||
496 | list_for_each_entry(fmr, fmr_list, ibfmr.list) { | |
497 | rkt = &to_idev(fmr->ibfmr.device)->lk_table; | |
498 | spin_lock_irqsave(&rkt->lock, flags); | |
499 | fmr->mr.user_base = 0; | |
500 | fmr->mr.iova = 0; | |
501 | fmr->mr.length = 0; | |
502 | spin_unlock_irqrestore(&rkt->lock, flags); | |
503 | } | |
504 | return 0; | |
505 | } | |
506 | ||
507 | /** | |
508 | * qib_dealloc_fmr - deallocate a fast memory region | |
509 | * @ibfmr: the fast memory region to deallocate | |
510 | * | |
511 | * Returns 0 on success. | |
512 | */ | |
513 | int qib_dealloc_fmr(struct ib_fmr *ibfmr) | |
514 | { | |
515 | struct qib_fmr *fmr = to_ifmr(ibfmr); | |
6a82649f MM |
516 | int ret = 0; |
517 | unsigned long timeout; | |
518 | ||
519 | qib_free_lkey(&fmr->mr); | |
520 | qib_put_mr(&fmr->mr); /* will set completion if last */ | |
521 | timeout = wait_for_completion_timeout(&fmr->mr.comp, | |
522 | 5 * HZ); | |
523 | if (!timeout) { | |
524 | qib_get_mr(&fmr->mr); | |
525 | ret = -EBUSY; | |
526 | goto out; | |
527 | } | |
528 | deinit_qib_mregion(&fmr->mr); | |
f931551b | 529 | kfree(fmr); |
6a82649f MM |
530 | out: |
531 | return ret; | |
f931551b | 532 | } |
8aac4cc3 MM |
533 | |
534 | void mr_rcu_callback(struct rcu_head *list) | |
535 | { | |
536 | struct qib_mregion *mr = container_of(list, struct qib_mregion, list); | |
537 | ||
538 | complete(&mr->comp); | |
539 | } |