2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/module.h>
34 #include <linux/moduleparam.h>
35 #include <rdma/ib_umem.h>
36 #include <linux/atomic.h>
37 #include <rdma/ib_user_verbs.h>
42 module_param(use_dsgl
, int, 0644);
43 MODULE_PARM_DESC(use_dsgl
, "Use DSGL for PBL/FastReg (default=0)");
45 #define T4_ULPTX_MIN_IO 32
46 #define C4IW_MAX_INLINE_SIZE 96
47 #define T4_ULPTX_MAX_DMA 1024
48 #define C4IW_INLINE_THRESHOLD 128
50 static int inline_threshold
= C4IW_INLINE_THRESHOLD
;
51 module_param(inline_threshold
, int, 0644);
52 MODULE_PARM_DESC(inline_threshold
, "inline vs dsgl threshold (default=128)");
54 static int mr_exceeds_hw_limits(struct c4iw_dev
*dev
, u64 length
)
56 return (is_t4(dev
->rdev
.lldi
.adapter_type
) ||
57 is_t5(dev
->rdev
.lldi
.adapter_type
)) &&
58 length
>= 8*1024*1024*1024ULL;
61 static int _c4iw_write_mem_dma_aligned(struct c4iw_rdev
*rdev
, u32 addr
,
62 u32 len
, dma_addr_t data
, int wait
)
65 struct ulp_mem_io
*req
;
66 struct ulptx_sgl
*sgl
;
69 struct c4iw_wr_wait wr_wait
;
74 c4iw_init_wr_wait(&wr_wait
);
75 wr_len
= roundup(sizeof(*req
) + sizeof(*sgl
), 16);
77 skb
= alloc_skb(wr_len
, GFP_KERNEL
);
80 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
82 req
= (struct ulp_mem_io
*)__skb_put(skb
, wr_len
);
83 memset(req
, 0, wr_len
);
84 INIT_ULPTX_WR(req
, wr_len
, 0, 0);
85 req
->wr
.wr_hi
= cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR
) |
86 (wait
? FW_WR_COMPL_F
: 0));
87 req
->wr
.wr_lo
= wait
? (__force __be64
)(unsigned long) &wr_wait
: 0L;
88 req
->wr
.wr_mid
= cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(wr_len
, 16)));
89 req
->cmd
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE
) |
90 T5_ULP_MEMIO_ORDER_V(1) |
91 T5_ULP_MEMIO_FID_V(rdev
->lldi
.rxq_ids
[0]));
92 req
->dlen
= cpu_to_be32(ULP_MEMIO_DATA_LEN_V(len
>>5));
93 req
->len16
= cpu_to_be32(DIV_ROUND_UP(wr_len
-sizeof(req
->wr
), 16));
94 req
->lock_addr
= cpu_to_be32(ULP_MEMIO_ADDR_V(addr
));
96 sgl
= (struct ulptx_sgl
*)(req
+ 1);
97 sgl
->cmd_nsge
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_DSGL
) |
99 sgl
->len0
= cpu_to_be32(len
);
100 sgl
->addr0
= cpu_to_be64(data
);
102 ret
= c4iw_ofld_send(rdev
, skb
);
106 ret
= c4iw_wait_for_reply(rdev
, &wr_wait
, 0, 0, __func__
);
110 static int _c4iw_write_mem_inline(struct c4iw_rdev
*rdev
, u32 addr
, u32 len
,
114 struct ulp_mem_io
*req
;
115 struct ulptx_idata
*sc
;
116 u8 wr_len
, *to_dp
, *from_dp
;
117 int copy_len
, num_wqe
, i
, ret
= 0;
118 struct c4iw_wr_wait wr_wait
;
119 __be32 cmd
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE
));
121 if (is_t4(rdev
->lldi
.adapter_type
))
122 cmd
|= cpu_to_be32(ULP_MEMIO_ORDER_F
);
124 cmd
|= cpu_to_be32(T5_ULP_MEMIO_IMM_F
);
127 PDBG("%s addr 0x%x len %u\n", __func__
, addr
, len
);
128 num_wqe
= DIV_ROUND_UP(len
, C4IW_MAX_INLINE_SIZE
);
129 c4iw_init_wr_wait(&wr_wait
);
130 for (i
= 0; i
< num_wqe
; i
++) {
132 copy_len
= len
> C4IW_MAX_INLINE_SIZE
? C4IW_MAX_INLINE_SIZE
:
134 wr_len
= roundup(sizeof *req
+ sizeof *sc
+
135 roundup(copy_len
, T4_ULPTX_MIN_IO
), 16);
137 skb
= alloc_skb(wr_len
, GFP_KERNEL
);
140 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
142 req
= (struct ulp_mem_io
*)__skb_put(skb
, wr_len
);
143 memset(req
, 0, wr_len
);
144 INIT_ULPTX_WR(req
, wr_len
, 0, 0);
146 if (i
== (num_wqe
-1)) {
147 req
->wr
.wr_hi
= cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR
) |
149 req
->wr
.wr_lo
= (__force __be64
)(unsigned long)&wr_wait
;
151 req
->wr
.wr_hi
= cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR
));
152 req
->wr
.wr_mid
= cpu_to_be32(
153 FW_WR_LEN16_V(DIV_ROUND_UP(wr_len
, 16)));
156 req
->dlen
= cpu_to_be32(ULP_MEMIO_DATA_LEN_V(
157 DIV_ROUND_UP(copy_len
, T4_ULPTX_MIN_IO
)));
158 req
->len16
= cpu_to_be32(DIV_ROUND_UP(wr_len
-sizeof(req
->wr
),
160 req
->lock_addr
= cpu_to_be32(ULP_MEMIO_ADDR_V(addr
+ i
* 3));
162 sc
= (struct ulptx_idata
*)(req
+ 1);
163 sc
->cmd_more
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM
));
164 sc
->len
= cpu_to_be32(roundup(copy_len
, T4_ULPTX_MIN_IO
));
166 to_dp
= (u8
*)(sc
+ 1);
167 from_dp
= (u8
*)data
+ i
* C4IW_MAX_INLINE_SIZE
;
169 memcpy(to_dp
, from_dp
, copy_len
);
171 memset(to_dp
, 0, copy_len
);
172 if (copy_len
% T4_ULPTX_MIN_IO
)
173 memset(to_dp
+ copy_len
, 0, T4_ULPTX_MIN_IO
-
174 (copy_len
% T4_ULPTX_MIN_IO
));
175 ret
= c4iw_ofld_send(rdev
, skb
);
178 len
-= C4IW_MAX_INLINE_SIZE
;
181 ret
= c4iw_wait_for_reply(rdev
, &wr_wait
, 0, 0, __func__
);
185 static int _c4iw_write_mem_dma(struct c4iw_rdev
*rdev
, u32 addr
, u32 len
, void *data
)
193 daddr
= dma_map_single(&rdev
->lldi
.pdev
->dev
, data
, len
, DMA_TO_DEVICE
);
194 if (dma_mapping_error(&rdev
->lldi
.pdev
->dev
, daddr
))
198 while (remain
> inline_threshold
) {
199 if (remain
< T4_ULPTX_MAX_DMA
) {
200 if (remain
& ~T4_ULPTX_MIN_IO
)
201 dmalen
= remain
& ~(T4_ULPTX_MIN_IO
-1);
205 dmalen
= T4_ULPTX_MAX_DMA
;
207 ret
= _c4iw_write_mem_dma_aligned(rdev
, addr
, dmalen
, daddr
,
216 ret
= _c4iw_write_mem_inline(rdev
, addr
, remain
, data
);
218 dma_unmap_single(&rdev
->lldi
.pdev
->dev
, save
, len
, DMA_TO_DEVICE
);
223 * write len bytes of data into addr (32B aligned address)
224 * If data is NULL, clear len byte of memory to zero.
226 static int write_adapter_mem(struct c4iw_rdev
*rdev
, u32 addr
, u32 len
,
229 if (is_t5(rdev
->lldi
.adapter_type
) && use_dsgl
) {
230 if (len
> inline_threshold
) {
231 if (_c4iw_write_mem_dma(rdev
, addr
, len
, data
)) {
232 printk_ratelimited(KERN_WARNING
234 " failure (non fatal)\n",
235 pci_name(rdev
->lldi
.pdev
));
236 return _c4iw_write_mem_inline(rdev
, addr
, len
,
241 return _c4iw_write_mem_inline(rdev
, addr
, len
, data
);
243 return _c4iw_write_mem_inline(rdev
, addr
, len
, data
);
247 * Build and write a TPT entry.
248 * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size,
249 * pbl_size and pbl_addr
252 static int write_tpt_entry(struct c4iw_rdev
*rdev
, u32 reset_tpt_entry
,
253 u32
*stag
, u8 stag_state
, u32 pdid
,
254 enum fw_ri_stag_type type
, enum fw_ri_mem_perms perm
,
255 int bind_enabled
, u32 zbva
, u64 to
,
256 u64 len
, u8 page_size
, u32 pbl_size
, u32 pbl_addr
)
259 struct fw_ri_tpte tpt
;
263 if (c4iw_fatal_error(rdev
))
266 stag_state
= stag_state
> 0;
267 stag_idx
= (*stag
) >> 8;
269 if ((!reset_tpt_entry
) && (*stag
== T4_STAG_UNSET
)) {
270 stag_idx
= c4iw_get_resource(&rdev
->resource
.tpt_table
);
272 mutex_lock(&rdev
->stats
.lock
);
273 rdev
->stats
.stag
.fail
++;
274 mutex_unlock(&rdev
->stats
.lock
);
277 mutex_lock(&rdev
->stats
.lock
);
278 rdev
->stats
.stag
.cur
+= 32;
279 if (rdev
->stats
.stag
.cur
> rdev
->stats
.stag
.max
)
280 rdev
->stats
.stag
.max
= rdev
->stats
.stag
.cur
;
281 mutex_unlock(&rdev
->stats
.lock
);
282 *stag
= (stag_idx
<< 8) | (atomic_inc_return(&key
) & 0xff);
284 PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
285 __func__
, stag_state
, type
, pdid
, stag_idx
);
287 /* write TPT entry */
289 memset(&tpt
, 0, sizeof(tpt
));
291 tpt
.valid_to_pdid
= cpu_to_be32(FW_RI_TPTE_VALID_F
|
292 FW_RI_TPTE_STAGKEY_V((*stag
& FW_RI_TPTE_STAGKEY_M
)) |
293 FW_RI_TPTE_STAGSTATE_V(stag_state
) |
294 FW_RI_TPTE_STAGTYPE_V(type
) | FW_RI_TPTE_PDID_V(pdid
));
295 tpt
.locread_to_qpid
= cpu_to_be32(FW_RI_TPTE_PERM_V(perm
) |
296 (bind_enabled
? FW_RI_TPTE_MWBINDEN_F
: 0) |
297 FW_RI_TPTE_ADDRTYPE_V((zbva
? FW_RI_ZERO_BASED_TO
:
299 FW_RI_TPTE_PS_V(page_size
));
300 tpt
.nosnoop_pbladdr
= !pbl_size
? 0 : cpu_to_be32(
301 FW_RI_TPTE_PBLADDR_V(PBL_OFF(rdev
, pbl_addr
)>>3));
302 tpt
.len_lo
= cpu_to_be32((u32
)(len
& 0xffffffffUL
));
303 tpt
.va_hi
= cpu_to_be32((u32
)(to
>> 32));
304 tpt
.va_lo_fbo
= cpu_to_be32((u32
)(to
& 0xffffffffUL
));
305 tpt
.dca_mwbcnt_pstag
= cpu_to_be32(0);
306 tpt
.len_hi
= cpu_to_be32((u32
)(len
>> 32));
308 err
= write_adapter_mem(rdev
, stag_idx
+
309 (rdev
->lldi
.vr
->stag
.start
>> 5),
312 if (reset_tpt_entry
) {
313 c4iw_put_resource(&rdev
->resource
.tpt_table
, stag_idx
);
314 mutex_lock(&rdev
->stats
.lock
);
315 rdev
->stats
.stag
.cur
-= 32;
316 mutex_unlock(&rdev
->stats
.lock
);
321 static int write_pbl(struct c4iw_rdev
*rdev
, __be64
*pbl
,
322 u32 pbl_addr
, u32 pbl_size
)
326 PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
327 __func__
, pbl_addr
, rdev
->lldi
.vr
->pbl
.start
,
330 err
= write_adapter_mem(rdev
, pbl_addr
>> 5, pbl_size
<< 3, pbl
);
334 static int dereg_mem(struct c4iw_rdev
*rdev
, u32 stag
, u32 pbl_size
,
337 return write_tpt_entry(rdev
, 1, &stag
, 0, 0, 0, 0, 0, 0, 0UL, 0, 0,
341 static int allocate_window(struct c4iw_rdev
*rdev
, u32
* stag
, u32 pdid
)
343 *stag
= T4_STAG_UNSET
;
344 return write_tpt_entry(rdev
, 0, stag
, 0, pdid
, FW_RI_STAG_MW
, 0, 0, 0,
348 static int deallocate_window(struct c4iw_rdev
*rdev
, u32 stag
)
350 return write_tpt_entry(rdev
, 1, &stag
, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0,
354 static int allocate_stag(struct c4iw_rdev
*rdev
, u32
*stag
, u32 pdid
,
355 u32 pbl_size
, u32 pbl_addr
)
357 *stag
= T4_STAG_UNSET
;
358 return write_tpt_entry(rdev
, 0, stag
, 0, pdid
, FW_RI_STAG_NSMR
, 0, 0, 0,
359 0UL, 0, 0, pbl_size
, pbl_addr
);
362 static int finish_mem_reg(struct c4iw_mr
*mhp
, u32 stag
)
367 mhp
->attr
.stag
= stag
;
369 mhp
->ibmr
.rkey
= mhp
->ibmr
.lkey
= stag
;
370 PDBG("%s mmid 0x%x mhp %p\n", __func__
, mmid
, mhp
);
371 return insert_handle(mhp
->rhp
, &mhp
->rhp
->mmidr
, mhp
, mmid
);
374 static int register_mem(struct c4iw_dev
*rhp
, struct c4iw_pd
*php
,
375 struct c4iw_mr
*mhp
, int shift
)
377 u32 stag
= T4_STAG_UNSET
;
380 ret
= write_tpt_entry(&rhp
->rdev
, 0, &stag
, 1, mhp
->attr
.pdid
,
381 FW_RI_STAG_NSMR
, mhp
->attr
.len
?
383 mhp
->attr
.mw_bind_enable
, mhp
->attr
.zbva
,
384 mhp
->attr
.va_fbo
, mhp
->attr
.len
?
385 mhp
->attr
.len
: -1, shift
- 12,
386 mhp
->attr
.pbl_size
, mhp
->attr
.pbl_addr
);
390 ret
= finish_mem_reg(mhp
, stag
);
392 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
397 static int alloc_pbl(struct c4iw_mr
*mhp
, int npages
)
399 mhp
->attr
.pbl_addr
= c4iw_pblpool_alloc(&mhp
->rhp
->rdev
,
402 if (!mhp
->attr
.pbl_addr
)
405 mhp
->attr
.pbl_size
= npages
;
410 struct ib_mr
*c4iw_get_dma_mr(struct ib_pd
*pd
, int acc
)
412 struct c4iw_dev
*rhp
;
416 u32 stag
= T4_STAG_UNSET
;
418 PDBG("%s ib_pd %p\n", __func__
, pd
);
419 php
= to_c4iw_pd(pd
);
422 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
424 return ERR_PTR(-ENOMEM
);
427 mhp
->attr
.pdid
= php
->pdid
;
428 mhp
->attr
.perms
= c4iw_ib_to_tpt_access(acc
);
429 mhp
->attr
.mw_bind_enable
= (acc
&IB_ACCESS_MW_BIND
) == IB_ACCESS_MW_BIND
;
431 mhp
->attr
.va_fbo
= 0;
432 mhp
->attr
.page_size
= 0;
433 mhp
->attr
.len
= ~0ULL;
434 mhp
->attr
.pbl_size
= 0;
436 ret
= write_tpt_entry(&rhp
->rdev
, 0, &stag
, 1, php
->pdid
,
437 FW_RI_STAG_NSMR
, mhp
->attr
.perms
,
438 mhp
->attr
.mw_bind_enable
, 0, 0, ~0ULL, 0, 0, 0);
442 ret
= finish_mem_reg(mhp
, stag
);
447 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
454 struct ib_mr
*c4iw_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
455 u64 virt
, int acc
, struct ib_udata
*udata
)
461 struct scatterlist
*sg
;
462 struct c4iw_dev
*rhp
;
466 PDBG("%s ib_pd %p\n", __func__
, pd
);
469 return ERR_PTR(-EINVAL
);
471 if ((length
+ start
) < start
)
472 return ERR_PTR(-EINVAL
);
474 php
= to_c4iw_pd(pd
);
477 if (mr_exceeds_hw_limits(rhp
, length
))
478 return ERR_PTR(-EINVAL
);
480 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
482 return ERR_PTR(-ENOMEM
);
486 mhp
->umem
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
487 if (IS_ERR(mhp
->umem
)) {
488 err
= PTR_ERR(mhp
->umem
);
493 shift
= ffs(mhp
->umem
->page_size
) - 1;
496 err
= alloc_pbl(mhp
, n
);
500 pages
= (__be64
*) __get_free_page(GFP_KERNEL
);
508 for_each_sg(mhp
->umem
->sg_head
.sgl
, sg
, mhp
->umem
->nmap
, entry
) {
509 len
= sg_dma_len(sg
) >> shift
;
510 for (k
= 0; k
< len
; ++k
) {
511 pages
[i
++] = cpu_to_be64(sg_dma_address(sg
) +
512 mhp
->umem
->page_size
* k
);
513 if (i
== PAGE_SIZE
/ sizeof *pages
) {
514 err
= write_pbl(&mhp
->rhp
->rdev
,
516 mhp
->attr
.pbl_addr
+ (n
<< 3), i
);
526 err
= write_pbl(&mhp
->rhp
->rdev
, pages
,
527 mhp
->attr
.pbl_addr
+ (n
<< 3), i
);
530 free_page((unsigned long) pages
);
534 mhp
->attr
.pdid
= php
->pdid
;
536 mhp
->attr
.perms
= c4iw_ib_to_tpt_access(acc
);
537 mhp
->attr
.va_fbo
= virt
;
538 mhp
->attr
.page_size
= shift
- 12;
539 mhp
->attr
.len
= length
;
541 err
= register_mem(rhp
, php
, mhp
, shift
);
548 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
549 mhp
->attr
.pbl_size
<< 3);
552 ib_umem_release(mhp
->umem
);
557 struct ib_mw
*c4iw_alloc_mw(struct ib_pd
*pd
, enum ib_mw_type type
,
558 struct ib_udata
*udata
)
560 struct c4iw_dev
*rhp
;
567 if (type
!= IB_MW_TYPE_1
)
568 return ERR_PTR(-EINVAL
);
570 php
= to_c4iw_pd(pd
);
572 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
574 return ERR_PTR(-ENOMEM
);
575 ret
= allocate_window(&rhp
->rdev
, &stag
, php
->pdid
);
581 mhp
->attr
.pdid
= php
->pdid
;
582 mhp
->attr
.type
= FW_RI_STAG_MW
;
583 mhp
->attr
.stag
= stag
;
585 mhp
->ibmw
.rkey
= stag
;
586 if (insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
)) {
587 deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
589 return ERR_PTR(-ENOMEM
);
591 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__
, mmid
, mhp
, stag
);
595 int c4iw_dealloc_mw(struct ib_mw
*mw
)
597 struct c4iw_dev
*rhp
;
601 mhp
= to_c4iw_mw(mw
);
603 mmid
= (mw
->rkey
) >> 8;
604 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
605 deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
607 PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__
, mw
, mmid
, mhp
);
611 struct ib_mr
*c4iw_alloc_mr(struct ib_pd
*pd
,
612 enum ib_mr_type mr_type
,
615 struct c4iw_dev
*rhp
;
621 int length
= roundup(max_num_sg
* sizeof(u64
), 32);
623 php
= to_c4iw_pd(pd
);
626 if (mr_type
!= IB_MR_TYPE_MEM_REG
||
627 max_num_sg
> t4_max_fr_depth(&rhp
->rdev
.lldi
.ulptx_memwrite_dsgl
&&
629 return ERR_PTR(-EINVAL
);
631 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
637 mhp
->mpl
= dma_alloc_coherent(&rhp
->rdev
.lldi
.pdev
->dev
,
638 length
, &mhp
->mpl_addr
, GFP_KERNEL
);
643 mhp
->max_mpl_len
= length
;
646 ret
= alloc_pbl(mhp
, max_num_sg
);
649 mhp
->attr
.pbl_size
= max_num_sg
;
650 ret
= allocate_stag(&rhp
->rdev
, &stag
, php
->pdid
,
651 mhp
->attr
.pbl_size
, mhp
->attr
.pbl_addr
);
654 mhp
->attr
.pdid
= php
->pdid
;
655 mhp
->attr
.type
= FW_RI_STAG_NSMR
;
656 mhp
->attr
.stag
= stag
;
659 mhp
->ibmr
.rkey
= mhp
->ibmr
.lkey
= stag
;
660 if (insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
)) {
665 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__
, mmid
, mhp
, stag
);
668 dereg_mem(&rhp
->rdev
, stag
, mhp
->attr
.pbl_size
,
671 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
672 mhp
->attr
.pbl_size
<< 3);
674 dma_free_coherent(&mhp
->rhp
->rdev
.lldi
.pdev
->dev
,
675 mhp
->max_mpl_len
, mhp
->mpl
, mhp
->mpl_addr
);
682 static int c4iw_set_page(struct ib_mr
*ibmr
, u64 addr
)
684 struct c4iw_mr
*mhp
= to_c4iw_mr(ibmr
);
686 if (unlikely(mhp
->mpl_len
== mhp
->max_mpl_len
))
689 mhp
->mpl
[mhp
->mpl_len
++] = addr
;
694 int c4iw_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
, int sg_nents
,
695 unsigned int *sg_offset
)
697 struct c4iw_mr
*mhp
= to_c4iw_mr(ibmr
);
701 return ib_sg_to_pages(ibmr
, sg
, sg_nents
, sg_offset
, c4iw_set_page
);
704 int c4iw_dereg_mr(struct ib_mr
*ib_mr
)
706 struct c4iw_dev
*rhp
;
710 PDBG("%s ib_mr %p\n", __func__
, ib_mr
);
712 mhp
= to_c4iw_mr(ib_mr
);
714 mmid
= mhp
->attr
.stag
>> 8;
715 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
717 dma_free_coherent(&mhp
->rhp
->rdev
.lldi
.pdev
->dev
,
718 mhp
->max_mpl_len
, mhp
->mpl
, mhp
->mpl_addr
);
719 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
721 if (mhp
->attr
.pbl_size
)
722 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
723 mhp
->attr
.pbl_size
<< 3);
725 kfree((void *) (unsigned long) mhp
->kva
);
727 ib_umem_release(mhp
->umem
);
728 PDBG("%s mmid 0x%x ptr %p\n", __func__
, mmid
, mhp
);
This page took 0.057676 seconds and 5 git commands to generate.