1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/random.h>
38 #include <linux/highmem.h>
39 #include <linux/time.h>
40 #include <asm/byteorder.h>
42 #include <rdma/ib_verbs.h>
43 #include <rdma/iw_cm.h>
44 #include <rdma/ib_user_verbs.h>
45 #include <rdma/ib_umem.h>
49 * i40iw_query_device - get device attributes
50 * @ibdev: device pointer from stack
51 * @props: returning device attributes
54 static int i40iw_query_device(struct ib_device
*ibdev
,
55 struct ib_device_attr
*props
,
56 struct ib_udata
*udata
)
58 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
60 if (udata
->inlen
|| udata
->outlen
)
62 memset(props
, 0, sizeof(*props
));
63 ether_addr_copy((u8
*)&props
->sys_image_guid
, iwdev
->netdev
->dev_addr
);
64 props
->fw_ver
= I40IW_FW_VERSION
;
65 props
->device_cap_flags
= iwdev
->device_cap_flags
;
66 props
->vendor_id
= iwdev
->ldev
->pcidev
->vendor
;
67 props
->vendor_part_id
= iwdev
->ldev
->pcidev
->device
;
68 props
->hw_ver
= (u32
)iwdev
->sc_dev
.hw_rev
;
69 props
->max_mr_size
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
70 props
->max_qp
= iwdev
->max_qp
;
71 props
->max_qp_wr
= (I40IW_MAX_WQ_ENTRIES
>> 2) - 1;
72 props
->max_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
73 props
->max_cq
= iwdev
->max_cq
;
74 props
->max_cqe
= iwdev
->max_cqe
;
75 props
->max_mr
= iwdev
->max_mr
;
76 props
->max_pd
= iwdev
->max_pd
;
77 props
->max_sge_rd
= 1;
78 props
->max_qp_rd_atom
= I40IW_MAX_IRD_SIZE
;
79 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
80 props
->atomic_cap
= IB_ATOMIC_NONE
;
81 props
->max_map_per_fmr
= 1;
86 * i40iw_query_port - get port attrubutes
87 * @ibdev: device pointer from stack
88 * @port: port number for query
89 * @props: returning device attributes
91 static int i40iw_query_port(struct ib_device
*ibdev
,
93 struct ib_port_attr
*props
)
95 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
96 struct net_device
*netdev
= iwdev
->netdev
;
98 memset(props
, 0, sizeof(*props
));
100 props
->max_mtu
= IB_MTU_4096
;
101 if (netdev
->mtu
>= 4096)
102 props
->active_mtu
= IB_MTU_4096
;
103 else if (netdev
->mtu
>= 2048)
104 props
->active_mtu
= IB_MTU_2048
;
105 else if (netdev
->mtu
>= 1024)
106 props
->active_mtu
= IB_MTU_1024
;
107 else if (netdev
->mtu
>= 512)
108 props
->active_mtu
= IB_MTU_512
;
110 props
->active_mtu
= IB_MTU_256
;
113 if (netif_carrier_ok(iwdev
->netdev
))
114 props
->state
= IB_PORT_ACTIVE
;
116 props
->state
= IB_PORT_DOWN
;
117 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
118 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
119 props
->gid_tbl_len
= 1;
120 props
->pkey_tbl_len
= 1;
121 props
->active_width
= IB_WIDTH_4X
;
122 props
->active_speed
= 1;
123 props
->max_msg_sz
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
128 * i40iw_alloc_ucontext - Allocate the user context data structure
129 * @ibdev: device pointer from stack
132 * This keeps track of all objects associated with a particular
135 static struct ib_ucontext
*i40iw_alloc_ucontext(struct ib_device
*ibdev
,
136 struct ib_udata
*udata
)
138 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
139 struct i40iw_alloc_ucontext_req req
;
140 struct i40iw_alloc_ucontext_resp uresp
;
141 struct i40iw_ucontext
*ucontext
;
143 if (ib_copy_from_udata(&req
, udata
, sizeof(req
)))
144 return ERR_PTR(-EINVAL
);
146 if (req
.userspace_ver
!= I40IW_ABI_USERSPACE_VER
) {
147 i40iw_pr_err("Invalid userspace driver version detected. Detected version %d, should be %d\n",
148 req
.userspace_ver
, I40IW_ABI_USERSPACE_VER
);
149 return ERR_PTR(-EINVAL
);
152 memset(&uresp
, 0, sizeof(uresp
));
153 uresp
.max_qps
= iwdev
->max_qp
;
154 uresp
.max_pds
= iwdev
->max_pd
;
155 uresp
.wq_size
= iwdev
->max_qp_wr
* 2;
156 uresp
.kernel_ver
= I40IW_ABI_KERNEL_VER
;
158 ucontext
= kzalloc(sizeof(*ucontext
), GFP_KERNEL
);
160 return ERR_PTR(-ENOMEM
);
162 ucontext
->iwdev
= iwdev
;
164 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
166 return ERR_PTR(-EFAULT
);
169 INIT_LIST_HEAD(&ucontext
->cq_reg_mem_list
);
170 spin_lock_init(&ucontext
->cq_reg_mem_list_lock
);
171 INIT_LIST_HEAD(&ucontext
->qp_reg_mem_list
);
172 spin_lock_init(&ucontext
->qp_reg_mem_list_lock
);
174 return &ucontext
->ibucontext
;
178 * i40iw_dealloc_ucontext - deallocate the user context data structure
179 * @context: user context created during alloc
181 static int i40iw_dealloc_ucontext(struct ib_ucontext
*context
)
183 struct i40iw_ucontext
*ucontext
= to_ucontext(context
);
186 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
187 if (!list_empty(&ucontext
->cq_reg_mem_list
)) {
188 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
191 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
192 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
193 if (!list_empty(&ucontext
->qp_reg_mem_list
)) {
194 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
197 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
204 * i40iw_mmap - user memory map
205 * @context: context created during alloc
206 * @vma: kernel info for user memory map
208 static int i40iw_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
210 struct i40iw_ucontext
*ucontext
;
214 ucontext
= to_ucontext(context
);
215 if (ucontext
->iwdev
->sc_dev
.is_pf
) {
216 db_addr_offset
= I40IW_DB_ADDR_OFFSET
;
217 push_offset
= I40IW_PUSH_OFFSET
;
219 vma
->vm_pgoff
+= I40IW_PF_FIRST_PUSH_PAGE_INDEX
- 1;
221 db_addr_offset
= I40IW_VF_DB_ADDR_OFFSET
;
222 push_offset
= I40IW_VF_PUSH_OFFSET
;
224 vma
->vm_pgoff
+= I40IW_VF_FIRST_PUSH_PAGE_INDEX
- 1;
227 vma
->vm_pgoff
+= db_addr_offset
>> PAGE_SHIFT
;
229 if (vma
->vm_pgoff
== (db_addr_offset
>> PAGE_SHIFT
)) {
230 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
231 vma
->vm_private_data
= ucontext
;
233 if ((vma
->vm_pgoff
- (push_offset
>> PAGE_SHIFT
)) % 2)
234 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
236 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
239 if (io_remap_pfn_range(vma
, vma
->vm_start
,
240 vma
->vm_pgoff
+ (pci_resource_start(ucontext
->iwdev
->ldev
->pcidev
, 0) >> PAGE_SHIFT
),
241 PAGE_SIZE
, vma
->vm_page_prot
))
248 * i40iw_alloc_push_page - allocate a push page for qp
249 * @iwdev: iwarp device
250 * @qp: hardware control qp
252 static void i40iw_alloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
254 struct i40iw_cqp_request
*cqp_request
;
255 struct cqp_commands_info
*cqp_info
;
256 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
257 enum i40iw_status_code status
;
259 if (qp
->push_idx
!= I40IW_INVALID_PUSH_PAGE_INDEX
)
262 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
266 atomic_inc(&cqp_request
->refcount
);
268 cqp_info
= &cqp_request
->info
;
269 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
270 cqp_info
->post_sq
= 1;
272 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= dev
->qs_handle
;
273 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 0;
274 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
275 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
277 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
279 qp
->push_idx
= cqp_request
->compl_info
.op_ret_val
;
281 i40iw_pr_err("CQP-OP Push page fail");
282 i40iw_put_cqp_request(&iwdev
->cqp
, cqp_request
);
286 * i40iw_dealloc_push_page - free a push page for qp
287 * @iwdev: iwarp device
288 * @qp: hardware control qp
290 static void i40iw_dealloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
292 struct i40iw_cqp_request
*cqp_request
;
293 struct cqp_commands_info
*cqp_info
;
294 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
295 enum i40iw_status_code status
;
297 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
)
300 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, false);
304 cqp_info
= &cqp_request
->info
;
305 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
306 cqp_info
->post_sq
= 1;
308 cqp_info
->in
.u
.manage_push_page
.info
.push_idx
= qp
->push_idx
;
309 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= dev
->qs_handle
;
310 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 1;
311 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
312 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
314 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
316 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
318 i40iw_pr_err("CQP-OP Push page fail");
322 * i40iw_alloc_pd - allocate protection domain
323 * @ibdev: device pointer from stack
324 * @context: user context created during alloc
327 static struct ib_pd
*i40iw_alloc_pd(struct ib_device
*ibdev
,
328 struct ib_ucontext
*context
,
329 struct ib_udata
*udata
)
331 struct i40iw_pd
*iwpd
;
332 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
333 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
334 struct i40iw_alloc_pd_resp uresp
;
335 struct i40iw_sc_pd
*sc_pd
;
339 err
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_pds
,
340 iwdev
->max_pd
, &pd_id
, &iwdev
->next_pd
);
342 i40iw_pr_err("alloc resource failed\n");
346 iwpd
= kzalloc(sizeof(*iwpd
), GFP_KERNEL
);
352 sc_pd
= &iwpd
->sc_pd
;
353 dev
->iw_pd_ops
->pd_init(dev
, sc_pd
, pd_id
);
356 memset(&uresp
, 0, sizeof(uresp
));
358 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
364 i40iw_add_pdusecount(iwpd
);
369 i40iw_free_resource(iwdev
, iwdev
->allocated_pds
, pd_id
);
374 * i40iw_dealloc_pd - deallocate pd
375 * @ibpd: ptr of pd to be deallocated
377 static int i40iw_dealloc_pd(struct ib_pd
*ibpd
)
379 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
380 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
382 i40iw_rem_pdusecount(iwpd
, iwdev
);
387 * i40iw_qp_roundup - return round up qp ring size
388 * @wr_ring_size: ring size to round up
390 static int i40iw_qp_roundup(u32 wr_ring_size
)
394 if (wr_ring_size
< I40IWQP_SW_MIN_WQSIZE
)
395 wr_ring_size
= I40IWQP_SW_MIN_WQSIZE
;
397 for (wr_ring_size
--; scount
<= 16; scount
*= 2)
398 wr_ring_size
|= wr_ring_size
>> scount
;
399 return ++wr_ring_size
;
403 * i40iw_get_pbl - Retrieve pbl from a list given a virtual
405 * @va: user virtual address
406 * @pbl_list: pbl list to search in (QP's or CQ's)
408 static struct i40iw_pbl
*i40iw_get_pbl(unsigned long va
,
409 struct list_head
*pbl_list
)
411 struct i40iw_pbl
*iwpbl
;
413 list_for_each_entry(iwpbl
, pbl_list
, list
) {
414 if (iwpbl
->user_base
== va
) {
415 list_del(&iwpbl
->list
);
423 * i40iw_free_qp_resources - free up memory resources for qp
424 * @iwdev: iwarp device
425 * @iwqp: qp ptr (user or kernel)
426 * @qp_num: qp number assigned
428 void i40iw_free_qp_resources(struct i40iw_device
*iwdev
,
429 struct i40iw_qp
*iwqp
,
432 i40iw_dealloc_push_page(iwdev
, &iwqp
->sc_qp
);
434 i40iw_free_resource(iwdev
, iwdev
->allocated_qps
, qp_num
);
435 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->q2_ctx_mem
);
436 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->kqp
.dma_mem
);
437 kfree(iwqp
->kqp
.wrid_mem
);
438 iwqp
->kqp
.wrid_mem
= NULL
;
439 kfree(iwqp
->allocated_buffer
);
443 * i40iw_clean_cqes - clean cq entries for qp
444 * @iwqp: qp ptr (user or kernel)
447 static void i40iw_clean_cqes(struct i40iw_qp
*iwqp
, struct i40iw_cq
*iwcq
)
449 struct i40iw_cq_uk
*ukcq
= &iwcq
->sc_cq
.cq_uk
;
451 ukcq
->ops
.iw_cq_clean(&iwqp
->sc_qp
.qp_uk
, ukcq
);
455 * i40iw_destroy_qp - destroy qp
456 * @ibqp: qp's ib pointer also to get to device's qp address
458 static int i40iw_destroy_qp(struct ib_qp
*ibqp
)
460 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
464 if (iwqp
->ibqp_state
>= IB_QPS_INIT
&& iwqp
->ibqp_state
< IB_QPS_RTS
)
465 i40iw_next_iw_state(iwqp
, I40IW_QP_STATE_ERROR
, 0, 0, 0);
467 if (!iwqp
->user_mode
) {
469 i40iw_clean_cqes(iwqp
, iwqp
->iwscq
);
470 if (iwqp
->iwrcq
!= iwqp
->iwscq
)
471 i40iw_clean_cqes(iwqp
, iwqp
->iwrcq
);
475 i40iw_rem_ref(&iwqp
->ibqp
);
480 * i40iw_setup_virt_qp - setup for allocation of virtual qp
483 * @init_info: initialize info to return
485 static int i40iw_setup_virt_qp(struct i40iw_device
*iwdev
,
486 struct i40iw_qp
*iwqp
,
487 struct i40iw_qp_init_info
*init_info
)
489 struct i40iw_pbl
*iwpbl
= iwqp
->iwpbl
;
490 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
492 iwqp
->page
= qpmr
->sq_page
;
493 init_info
->shadow_area_pa
= cpu_to_le64(qpmr
->shadow
);
494 if (iwpbl
->pbl_allocated
) {
495 init_info
->virtual_map
= true;
496 init_info
->sq_pa
= qpmr
->sq_pbl
.idx
;
497 init_info
->rq_pa
= qpmr
->rq_pbl
.idx
;
499 init_info
->sq_pa
= qpmr
->sq_pbl
.addr
;
500 init_info
->rq_pa
= qpmr
->rq_pbl
.addr
;
506 * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
507 * @iwdev: iwarp device
508 * @iwqp: qp ptr (user or kernel)
509 * @info: initialize info to return
511 static int i40iw_setup_kmode_qp(struct i40iw_device
*iwdev
,
512 struct i40iw_qp
*iwqp
,
513 struct i40iw_qp_init_info
*info
)
515 struct i40iw_dma_mem
*mem
= &iwqp
->kqp
.dma_mem
;
516 u32 sqdepth
, rqdepth
;
517 u32 sq_size
, rq_size
;
520 enum i40iw_status_code status
;
521 struct i40iw_qp_uk_init_info
*ukinfo
= &info
->qp_uk_init_info
;
523 ukinfo
->max_sq_frag_cnt
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
525 sq_size
= i40iw_qp_roundup(ukinfo
->sq_size
+ 1);
526 rq_size
= i40iw_qp_roundup(ukinfo
->rq_size
+ 1);
528 status
= i40iw_get_wqe_shift(sq_size
, ukinfo
->max_sq_frag_cnt
, ukinfo
->max_inline_data
, &sqshift
);
530 status
= i40iw_get_wqe_shift(rq_size
, ukinfo
->max_rq_frag_cnt
, 0, &rqshift
);
535 sqdepth
= sq_size
<< sqshift
;
536 rqdepth
= rq_size
<< rqshift
;
538 size
= sqdepth
* sizeof(struct i40iw_sq_uk_wr_trk_info
) + (rqdepth
<< 3);
539 iwqp
->kqp
.wrid_mem
= kzalloc(size
, GFP_KERNEL
);
541 ukinfo
->sq_wrtrk_array
= (struct i40iw_sq_uk_wr_trk_info
*)iwqp
->kqp
.wrid_mem
;
542 if (!ukinfo
->sq_wrtrk_array
)
545 ukinfo
->rq_wrid_array
= (u64
*)&ukinfo
->sq_wrtrk_array
[sqdepth
];
547 size
= (sqdepth
+ rqdepth
) * I40IW_QP_WQE_MIN_SIZE
;
548 size
+= (I40IW_SHADOW_AREA_SIZE
<< 3);
550 status
= i40iw_allocate_dma_mem(iwdev
->sc_dev
.hw
, mem
, size
, 256);
552 kfree(ukinfo
->sq_wrtrk_array
);
553 ukinfo
->sq_wrtrk_array
= NULL
;
557 ukinfo
->sq
= mem
->va
;
558 info
->sq_pa
= mem
->pa
;
560 ukinfo
->rq
= &ukinfo
->sq
[sqdepth
];
561 info
->rq_pa
= info
->sq_pa
+ (sqdepth
* I40IW_QP_WQE_MIN_SIZE
);
563 ukinfo
->shadow_area
= ukinfo
->rq
[rqdepth
].elem
;
564 info
->shadow_area_pa
= info
->rq_pa
+ (rqdepth
* I40IW_QP_WQE_MIN_SIZE
);
566 ukinfo
->sq_size
= sq_size
;
567 ukinfo
->rq_size
= rq_size
;
568 ukinfo
->qp_id
= iwqp
->ibqp
.qp_num
;
573 * i40iw_create_qp - create qp
575 * @init_attr: attributes for qp
576 * @udata: user data for create qp
578 static struct ib_qp
*i40iw_create_qp(struct ib_pd
*ibpd
,
579 struct ib_qp_init_attr
*init_attr
,
580 struct ib_udata
*udata
)
582 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
583 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
584 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
585 struct i40iw_qp
*iwqp
;
586 struct i40iw_ucontext
*ucontext
;
587 struct i40iw_create_qp_req req
;
588 struct i40iw_create_qp_resp uresp
;
591 enum i40iw_status_code ret
;
595 struct i40iw_sc_qp
*qp
;
596 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
597 struct i40iw_qp_init_info init_info
;
598 struct i40iw_create_qp_info
*qp_info
;
599 struct i40iw_cqp_request
*cqp_request
;
600 struct cqp_commands_info
*cqp_info
;
602 struct i40iw_qp_host_ctx_info
*ctx_info
;
603 struct i40iwarp_offload_info
*iwarp_info
;
606 if (init_attr
->create_flags
)
607 return ERR_PTR(-EINVAL
);
608 if (init_attr
->cap
.max_inline_data
> I40IW_MAX_INLINE_DATA_SIZE
)
609 init_attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
611 if (init_attr
->cap
.max_send_sge
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
612 init_attr
->cap
.max_send_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
614 memset(&init_info
, 0, sizeof(init_info
));
616 sq_size
= init_attr
->cap
.max_send_wr
;
617 rq_size
= init_attr
->cap
.max_recv_wr
;
619 init_info
.qp_uk_init_info
.sq_size
= sq_size
;
620 init_info
.qp_uk_init_info
.rq_size
= rq_size
;
621 init_info
.qp_uk_init_info
.max_sq_frag_cnt
= init_attr
->cap
.max_send_sge
;
622 init_info
.qp_uk_init_info
.max_rq_frag_cnt
= init_attr
->cap
.max_recv_sge
;
623 init_info
.qp_uk_init_info
.max_inline_data
= init_attr
->cap
.max_inline_data
;
625 mem
= kzalloc(sizeof(*iwqp
), GFP_KERNEL
);
627 return ERR_PTR(-ENOMEM
);
629 iwqp
= (struct i40iw_qp
*)mem
;
631 qp
->back_qp
= (void *)iwqp
;
632 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
634 iwqp
->ctx_info
.iwarp_info
= &iwqp
->iwarp_info
;
636 if (i40iw_allocate_dma_mem(dev
->hw
,
638 I40IW_Q2_BUFFER_SIZE
+ I40IW_QP_CTX_SIZE
,
640 i40iw_pr_err("dma_mem failed\n");
645 init_info
.q2
= iwqp
->q2_ctx_mem
.va
;
646 init_info
.q2_pa
= iwqp
->q2_ctx_mem
.pa
;
648 init_info
.host_ctx
= (void *)init_info
.q2
+ I40IW_Q2_BUFFER_SIZE
;
649 init_info
.host_ctx_pa
= init_info
.q2_pa
+ I40IW_Q2_BUFFER_SIZE
;
651 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_qps
, iwdev
->max_qp
,
652 &qp_num
, &iwdev
->next_qp
);
654 i40iw_pr_err("qp resource\n");
658 iwqp
->allocated_buffer
= mem
;
661 iwqp
->ibqp
.qp_num
= qp_num
;
663 iwqp
->iwscq
= to_iwcq(init_attr
->send_cq
);
664 iwqp
->iwrcq
= to_iwcq(init_attr
->recv_cq
);
666 iwqp
->host_ctx
.va
= init_info
.host_ctx
;
667 iwqp
->host_ctx
.pa
= init_info
.host_ctx_pa
;
668 iwqp
->host_ctx
.size
= I40IW_QP_CTX_SIZE
;
670 init_info
.pd
= &iwpd
->sc_pd
;
671 init_info
.qp_uk_init_info
.qp_id
= iwqp
->ibqp
.qp_num
;
672 iwqp
->ctx_info
.qp_compl_ctx
= (uintptr_t)qp
;
674 if (init_attr
->qp_type
!= IB_QPT_RC
) {
678 if (iwdev
->push_mode
)
679 i40iw_alloc_push_page(iwdev
, qp
);
681 err_code
= ib_copy_from_udata(&req
, udata
, sizeof(req
));
683 i40iw_pr_err("ib_copy_from_data\n");
686 iwqp
->ctx_info
.qp_compl_ctx
= req
.user_compl_ctx
;
687 if (ibpd
->uobject
&& ibpd
->uobject
->context
) {
689 ucontext
= to_ucontext(ibpd
->uobject
->context
);
691 if (req
.user_wqe_buffers
) {
693 &ucontext
->qp_reg_mem_list_lock
, flags
);
694 iwqp
->iwpbl
= i40iw_get_pbl(
695 (unsigned long)req
.user_wqe_buffers
,
696 &ucontext
->qp_reg_mem_list
);
697 spin_unlock_irqrestore(
698 &ucontext
->qp_reg_mem_list_lock
, flags
);
702 i40iw_pr_err("no pbl info\n");
707 err_code
= i40iw_setup_virt_qp(iwdev
, iwqp
, &init_info
);
709 err_code
= i40iw_setup_kmode_qp(iwdev
, iwqp
, &init_info
);
713 i40iw_pr_err("setup qp failed\n");
717 init_info
.type
= I40IW_QP_TYPE_IWARP
;
718 ret
= dev
->iw_priv_qp_ops
->qp_init(qp
, &init_info
);
721 i40iw_pr_err("qp_init fail\n");
724 ctx_info
= &iwqp
->ctx_info
;
725 iwarp_info
= &iwqp
->iwarp_info
;
726 iwarp_info
->rd_enable
= true;
727 iwarp_info
->wr_rdresp_en
= true;
728 if (!iwqp
->user_mode
) {
729 iwarp_info
->fast_reg_en
= true;
730 iwarp_info
->priv_mode_en
= true;
732 iwarp_info
->ddp_ver
= 1;
733 iwarp_info
->rdmap_ver
= 1;
735 ctx_info
->iwarp_info_valid
= true;
736 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
737 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
738 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
) {
739 ctx_info
->push_mode_en
= false;
741 ctx_info
->push_mode_en
= true;
742 ctx_info
->push_idx
= qp
->push_idx
;
745 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
746 (u64
*)iwqp
->host_ctx
.va
,
748 ctx_info
->iwarp_info_valid
= false;
749 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
754 cqp_info
= &cqp_request
->info
;
755 qp_info
= &cqp_request
->info
.in
.u
.qp_create
.info
;
757 memset(qp_info
, 0, sizeof(*qp_info
));
759 qp_info
->cq_num_valid
= true;
760 qp_info
->next_iwarp_state
= I40IW_QP_STATE_IDLE
;
762 cqp_info
->cqp_cmd
= OP_QP_CREATE
;
763 cqp_info
->post_sq
= 1;
764 cqp_info
->in
.u
.qp_create
.qp
= qp
;
765 cqp_info
->in
.u
.qp_create
.scratch
= (uintptr_t)cqp_request
;
766 ret
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
768 i40iw_pr_err("CQP-OP QP create fail");
773 i40iw_add_ref(&iwqp
->ibqp
);
774 spin_lock_init(&iwqp
->lock
);
775 iwqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
776 iwdev
->qp_table
[qp_num
] = iwqp
;
777 i40iw_add_pdusecount(iwqp
->iwpd
);
778 if (ibpd
->uobject
&& udata
) {
779 memset(&uresp
, 0, sizeof(uresp
));
780 uresp
.actual_sq_size
= sq_size
;
781 uresp
.actual_rq_size
= rq_size
;
782 uresp
.qp_id
= qp_num
;
783 uresp
.push_idx
= qp
->push_idx
;
784 err_code
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
786 i40iw_pr_err("copy_to_udata failed\n");
787 i40iw_destroy_qp(&iwqp
->ibqp
);
788 /* let the completion of the qp destroy free the qp */
789 return ERR_PTR(err_code
);
792 init_completion(&iwqp
->sq_drained
);
793 init_completion(&iwqp
->rq_drained
);
797 i40iw_free_qp_resources(iwdev
, iwqp
, qp_num
);
799 return ERR_PTR(err_code
);
803 * i40iw_query - query qp attributes
805 * @attr: attributes pointer
806 * @attr_mask: Not used
807 * @init_attr: qp attributes to return
809 static int i40iw_query_qp(struct ib_qp
*ibqp
,
810 struct ib_qp_attr
*attr
,
812 struct ib_qp_init_attr
*init_attr
)
814 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
815 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
817 attr
->qp_access_flags
= 0;
818 attr
->cap
.max_send_wr
= qp
->qp_uk
.sq_size
;
819 attr
->cap
.max_recv_wr
= qp
->qp_uk
.rq_size
;
820 attr
->cap
.max_recv_sge
= 1;
821 attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
822 init_attr
->event_handler
= iwqp
->ibqp
.event_handler
;
823 init_attr
->qp_context
= iwqp
->ibqp
.qp_context
;
824 init_attr
->send_cq
= iwqp
->ibqp
.send_cq
;
825 init_attr
->recv_cq
= iwqp
->ibqp
.recv_cq
;
826 init_attr
->srq
= iwqp
->ibqp
.srq
;
827 init_attr
->cap
= attr
->cap
;
832 * i40iw_hw_modify_qp - setup cqp for modify qp
833 * @iwdev: iwarp device
834 * @iwqp: qp ptr (user or kernel)
835 * @info: info for modify qp
836 * @wait: flag to wait or not for modify qp completion
838 void i40iw_hw_modify_qp(struct i40iw_device
*iwdev
, struct i40iw_qp
*iwqp
,
839 struct i40iw_modify_qp_info
*info
, bool wait
)
841 enum i40iw_status_code status
;
842 struct i40iw_cqp_request
*cqp_request
;
843 struct cqp_commands_info
*cqp_info
;
844 struct i40iw_modify_qp_info
*m_info
;
846 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, wait
);
850 cqp_info
= &cqp_request
->info
;
851 m_info
= &cqp_info
->in
.u
.qp_modify
.info
;
852 memcpy(m_info
, info
, sizeof(*m_info
));
853 cqp_info
->cqp_cmd
= OP_QP_MODIFY
;
854 cqp_info
->post_sq
= 1;
855 cqp_info
->in
.u
.qp_modify
.qp
= &iwqp
->sc_qp
;
856 cqp_info
->in
.u
.qp_modify
.scratch
= (uintptr_t)cqp_request
;
857 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
859 i40iw_pr_err("CQP-OP Modify QP fail");
863 * i40iw_modify_qp - modify qp request
864 * @ibqp: qp's pointer for modify
865 * @attr: access attributes
866 * @attr_mask: state mask
869 int i40iw_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
870 int attr_mask
, struct ib_udata
*udata
)
872 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
873 struct i40iw_device
*iwdev
= iwqp
->iwdev
;
874 struct i40iw_qp_host_ctx_info
*ctx_info
;
875 struct i40iwarp_offload_info
*iwarp_info
;
876 struct i40iw_modify_qp_info info
;
877 u8 issue_modify_qp
= 0;
882 memset(&info
, 0, sizeof(info
));
883 ctx_info
= &iwqp
->ctx_info
;
884 iwarp_info
= &iwqp
->iwarp_info
;
886 spin_lock_irqsave(&iwqp
->lock
, flags
);
888 if (attr_mask
& IB_QP_STATE
) {
889 switch (attr
->qp_state
) {
892 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_IDLE
) {
896 if (iwqp
->iwarp_state
== I40IW_QP_STATE_INVALID
) {
897 info
.next_iwarp_state
= I40IW_QP_STATE_IDLE
;
902 if ((iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) ||
909 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_ESTABLISHED
;
911 info
.next_iwarp_state
= I40IW_QP_STATE_RTS
;
912 info
.tcp_ctx_valid
= true;
913 info
.ord_valid
= true;
914 info
.arp_cache_idx_valid
= true;
915 info
.cq_num_valid
= true;
918 if (iwqp
->hw_iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) {
922 if ((iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_CLOSING
) ||
923 (iwqp
->iwarp_state
< (u32
)I40IW_QP_STATE_RTS
)) {
927 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_CLOSING
) {
931 info
.next_iwarp_state
= I40IW_QP_STATE_CLOSING
;
935 if (iwqp
->iwarp_state
>= (u32
)I40IW_QP_STATE_TERMINATE
) {
939 info
.next_iwarp_state
= I40IW_QP_STATE_TERMINATE
;
944 if (iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_ERROR
) {
948 if (iwqp
->sc_qp
.term_flags
)
949 del_timer(&iwqp
->terminate_timer
);
950 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
951 if ((iwqp
->hw_tcp_state
> I40IW_TCP_STATE_CLOSED
) &&
953 (iwqp
->hw_tcp_state
!= I40IW_TCP_STATE_TIME_WAIT
))
954 info
.reset_tcp_conn
= true;
958 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
965 iwqp
->ibqp_state
= attr
->qp_state
;
968 iwqp
->iwarp_state
= info
.next_iwarp_state
;
970 info
.next_iwarp_state
= iwqp
->iwarp_state
;
972 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
973 ctx_info
->iwarp_info_valid
= true;
974 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
)
975 iwarp_info
->wr_rdresp_en
= true;
976 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
977 iwarp_info
->wr_rdresp_en
= true;
978 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
)
979 iwarp_info
->rd_enable
= true;
980 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
)
981 iwarp_info
->bind_en
= true;
983 if (iwqp
->user_mode
) {
984 iwarp_info
->rd_enable
= true;
985 iwarp_info
->wr_rdresp_en
= true;
986 iwarp_info
->priv_mode_en
= false;
990 if (ctx_info
->iwarp_info_valid
) {
991 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
994 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
995 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
996 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
997 (u64
*)iwqp
->host_ctx
.va
,
1000 i40iw_pr_err("setting QP context\n");
1006 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1008 if (issue_modify_qp
)
1009 i40iw_hw_modify_qp(iwdev
, iwqp
, &info
, true);
1011 if (issue_modify_qp
&& (iwqp
->ibqp_state
> IB_QPS_RTS
)) {
1013 if (iwqp
->cm_id
&& iwqp
->hw_tcp_state
) {
1014 spin_lock_irqsave(&iwqp
->lock
, flags
);
1015 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_CLOSED
;
1016 iwqp
->last_aeq
= I40IW_AE_RESET_SENT
;
1017 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1023 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1028 * cq_free_resources - free up recources for cq
1029 * @iwdev: iwarp device
1032 static void cq_free_resources(struct i40iw_device
*iwdev
, struct i40iw_cq
*iwcq
)
1034 struct i40iw_sc_cq
*cq
= &iwcq
->sc_cq
;
1036 if (!iwcq
->user_mode
)
1037 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwcq
->kmem
);
1038 i40iw_free_resource(iwdev
, iwdev
->allocated_cqs
, cq
->cq_uk
.cq_id
);
1042 * cq_wq_destroy - send cq destroy cqp
1043 * @iwdev: iwarp device
1044 * @cq: hardware control cq
1046 static void cq_wq_destroy(struct i40iw_device
*iwdev
, struct i40iw_sc_cq
*cq
)
1048 enum i40iw_status_code status
;
1049 struct i40iw_cqp_request
*cqp_request
;
1050 struct cqp_commands_info
*cqp_info
;
1052 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1056 cqp_info
= &cqp_request
->info
;
1058 cqp_info
->cqp_cmd
= OP_CQ_DESTROY
;
1059 cqp_info
->post_sq
= 1;
1060 cqp_info
->in
.u
.cq_destroy
.cq
= cq
;
1061 cqp_info
->in
.u
.cq_destroy
.scratch
= (uintptr_t)cqp_request
;
1062 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1064 i40iw_pr_err("CQP-OP Destroy QP fail");
1068 * i40iw_destroy_cq - destroy cq
1069 * @ib_cq: cq pointer
1071 static int i40iw_destroy_cq(struct ib_cq
*ib_cq
)
1073 struct i40iw_cq
*iwcq
;
1074 struct i40iw_device
*iwdev
;
1075 struct i40iw_sc_cq
*cq
;
1078 i40iw_pr_err("ib_cq == NULL\n");
1082 iwcq
= to_iwcq(ib_cq
);
1083 iwdev
= to_iwdev(ib_cq
->device
);
1085 cq_wq_destroy(iwdev
, cq
);
1086 cq_free_resources(iwdev
, iwcq
);
1092 * i40iw_create_cq - create cq
1093 * @ibdev: device pointer from stack
1094 * @attr: attributes for cq
1095 * @context: user context created during alloc
1098 static struct ib_cq
*i40iw_create_cq(struct ib_device
*ibdev
,
1099 const struct ib_cq_init_attr
*attr
,
1100 struct ib_ucontext
*context
,
1101 struct ib_udata
*udata
)
1103 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
1104 struct i40iw_cq
*iwcq
;
1105 struct i40iw_pbl
*iwpbl
;
1107 struct i40iw_sc_cq
*cq
;
1108 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1109 struct i40iw_cq_init_info info
;
1110 enum i40iw_status_code status
;
1111 struct i40iw_cqp_request
*cqp_request
;
1112 struct cqp_commands_info
*cqp_info
;
1113 struct i40iw_cq_uk_init_info
*ukinfo
= &info
.cq_uk_init_info
;
1114 unsigned long flags
;
1116 int entries
= attr
->cqe
;
1118 if (entries
> iwdev
->max_cqe
)
1119 return ERR_PTR(-EINVAL
);
1121 iwcq
= kzalloc(sizeof(*iwcq
), GFP_KERNEL
);
1123 return ERR_PTR(-ENOMEM
);
1125 memset(&info
, 0, sizeof(info
));
1127 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_cqs
,
1128 iwdev
->max_cq
, &cq_num
,
1134 cq
->back_cq
= (void *)iwcq
;
1135 spin_lock_init(&iwcq
->lock
);
1138 ukinfo
->cq_size
= max(entries
, 4);
1139 ukinfo
->cq_id
= cq_num
;
1140 iwcq
->ibcq
.cqe
= info
.cq_uk_init_info
.cq_size
;
1143 info
.ceq_id_valid
= true;
1145 info
.type
= I40IW_CQ_TYPE_IWARP
;
1147 struct i40iw_ucontext
*ucontext
;
1148 struct i40iw_create_cq_req req
;
1149 struct i40iw_cq_mr
*cqmr
;
1151 memset(&req
, 0, sizeof(req
));
1152 iwcq
->user_mode
= true;
1153 ucontext
= to_ucontext(context
);
1154 if (ib_copy_from_udata(&req
, udata
, sizeof(struct i40iw_create_cq_req
)))
1155 goto cq_free_resources
;
1157 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1158 iwpbl
= i40iw_get_pbl((unsigned long)req
.user_cq_buffer
,
1159 &ucontext
->cq_reg_mem_list
);
1160 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1163 goto cq_free_resources
;
1166 iwcq
->iwpbl
= iwpbl
;
1167 iwcq
->cq_mem_size
= 0;
1168 cqmr
= &iwpbl
->cq_mr
;
1169 info
.shadow_area_pa
= cpu_to_le64(cqmr
->shadow
);
1170 if (iwpbl
->pbl_allocated
) {
1171 info
.virtual_map
= true;
1172 info
.pbl_chunk_size
= 1;
1173 info
.first_pm_pbl_idx
= cqmr
->cq_pbl
.idx
;
1175 info
.cq_base_pa
= cqmr
->cq_pbl
.addr
;
1178 /* Kmode allocations */
1182 rsize
= info
.cq_uk_init_info
.cq_size
* sizeof(struct i40iw_cqe
);
1183 rsize
= round_up(rsize
, 256);
1184 shadow
= I40IW_SHADOW_AREA_SIZE
<< 3;
1185 status
= i40iw_allocate_dma_mem(dev
->hw
, &iwcq
->kmem
,
1186 rsize
+ shadow
, 256);
1189 goto cq_free_resources
;
1191 ukinfo
->cq_base
= iwcq
->kmem
.va
;
1192 info
.cq_base_pa
= iwcq
->kmem
.pa
;
1193 info
.shadow_area_pa
= info
.cq_base_pa
+ rsize
;
1194 ukinfo
->shadow_area
= iwcq
->kmem
.va
+ rsize
;
1197 if (dev
->iw_priv_cq_ops
->cq_init(cq
, &info
)) {
1198 i40iw_pr_err("init cq fail\n");
1200 goto cq_free_resources
;
1203 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1206 goto cq_free_resources
;
1209 cqp_info
= &cqp_request
->info
;
1210 cqp_info
->cqp_cmd
= OP_CQ_CREATE
;
1211 cqp_info
->post_sq
= 1;
1212 cqp_info
->in
.u
.cq_create
.cq
= cq
;
1213 cqp_info
->in
.u
.cq_create
.scratch
= (uintptr_t)cqp_request
;
1214 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1216 i40iw_pr_err("CQP-OP Create QP fail");
1218 goto cq_free_resources
;
1222 struct i40iw_create_cq_resp resp
;
1224 memset(&resp
, 0, sizeof(resp
));
1225 resp
.cq_id
= info
.cq_uk_init_info
.cq_id
;
1226 resp
.cq_size
= info
.cq_uk_init_info
.cq_size
;
1227 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
1228 i40iw_pr_err("copy to user data\n");
1234 return (struct ib_cq
*)iwcq
;
1237 cq_wq_destroy(iwdev
, cq
);
1239 cq_free_resources(iwdev
, iwcq
);
1242 return ERR_PTR(err_code
);
1246 * i40iw_get_user_access - get hw access from IB access
1247 * @acc: IB access to return hw access
1249 static inline u16
i40iw_get_user_access(int acc
)
1253 access
|= (acc
& IB_ACCESS_LOCAL_WRITE
) ? I40IW_ACCESS_FLAGS_LOCALWRITE
: 0;
1254 access
|= (acc
& IB_ACCESS_REMOTE_WRITE
) ? I40IW_ACCESS_FLAGS_REMOTEWRITE
: 0;
1255 access
|= (acc
& IB_ACCESS_REMOTE_READ
) ? I40IW_ACCESS_FLAGS_REMOTEREAD
: 0;
1256 access
|= (acc
& IB_ACCESS_MW_BIND
) ? I40IW_ACCESS_FLAGS_BIND_WINDOW
: 0;
1261 * i40iw_free_stag - free stag resource
1262 * @iwdev: iwarp device
1263 * @stag: stag to free
1265 static void i40iw_free_stag(struct i40iw_device
*iwdev
, u32 stag
)
1269 stag_idx
= (stag
& iwdev
->mr_stagmask
) >> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1270 i40iw_free_resource(iwdev
, iwdev
->allocated_mrs
, stag_idx
);
1274 * i40iw_create_stag - create random stag
1275 * @iwdev: iwarp device
1277 static u32
i40iw_create_stag(struct i40iw_device
*iwdev
)
1281 u32 next_stag_index
;
1287 get_random_bytes(&random
, sizeof(random
));
1288 consumer_key
= (u8
)random
;
1290 driver_key
= random
& ~iwdev
->mr_stagmask
;
1291 next_stag_index
= (random
& iwdev
->mr_stagmask
) >> 8;
1292 next_stag_index
%= iwdev
->max_mr
;
1294 ret
= i40iw_alloc_resource(iwdev
,
1295 iwdev
->allocated_mrs
, iwdev
->max_mr
,
1296 &stag_index
, &next_stag_index
);
1298 stag
= stag_index
<< I40IW_CQPSQ_STAG_IDX_SHIFT
;
1300 stag
+= (u32
)consumer_key
;
1306 * i40iw_next_pbl_addr - Get next pbl address
1307 * @palloc: Poiner to allocated pbles
1308 * @pbl: pointer to a pble
1309 * @pinfo: info pointer
1312 static inline u64
*i40iw_next_pbl_addr(struct i40iw_pble_alloc
*palloc
,
1314 struct i40iw_pble_info
**pinfo
,
1318 if ((!(*pinfo
)) || (*idx
!= (*pinfo
)->cnt
))
1322 return (u64
*)(*pinfo
)->addr
;
1326 * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally
1327 * @iwmr: iwmr for IB's user page addresses
1328 * @pbl: ple pointer to save 1 level or 0 level pble
1329 * @level: indicated level 0, 1 or 2
1331 static void i40iw_copy_user_pgaddrs(struct i40iw_mr
*iwmr
,
1333 enum i40iw_pble_level level
)
1335 struct ib_umem
*region
= iwmr
->region
;
1336 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1337 int chunk_pages
, entry
, pg_shift
, i
;
1338 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1339 struct i40iw_pble_info
*pinfo
;
1340 struct scatterlist
*sg
;
1343 pinfo
= (level
== I40IW_LEVEL_1
) ? NULL
: palloc
->level2
.leaf
;
1344 pg_shift
= ffs(region
->page_size
) - 1;
1345 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
1346 chunk_pages
= sg_dma_len(sg
) >> pg_shift
;
1347 if ((iwmr
->type
== IW_MEMREG_TYPE_QP
) &&
1348 !iwpbl
->qp_mr
.sq_page
)
1349 iwpbl
->qp_mr
.sq_page
= sg_page(sg
);
1350 for (i
= 0; i
< chunk_pages
; i
++) {
1351 *pbl
= cpu_to_le64(sg_dma_address(sg
) + region
->page_size
* i
);
1352 pbl
= i40iw_next_pbl_addr(palloc
, pbl
, &pinfo
, &idx
);
1358 * i40iw_setup_pbles - copy user pg address to pble's
1359 * @iwdev: iwarp device
1360 * @iwmr: mr pointer for this memory registration
1361 * @use_pbles: flag if to use pble's or memory (level 0)
1363 static int i40iw_setup_pbles(struct i40iw_device
*iwdev
,
1364 struct i40iw_mr
*iwmr
,
1367 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1368 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1369 struct i40iw_pble_info
*pinfo
;
1371 enum i40iw_status_code status
;
1372 enum i40iw_pble_level level
= I40IW_LEVEL_1
;
1374 if (!use_pbles
&& (iwmr
->page_cnt
> MAX_SAVE_PAGE_ADDRS
))
1378 mutex_lock(&iwdev
->pbl_mutex
);
1379 status
= i40iw_get_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
, palloc
, iwmr
->page_cnt
);
1380 mutex_unlock(&iwdev
->pbl_mutex
);
1384 iwpbl
->pbl_allocated
= true;
1385 level
= palloc
->level
;
1386 pinfo
= (level
== I40IW_LEVEL_1
) ? &palloc
->level1
: palloc
->level2
.leaf
;
1387 pbl
= (u64
*)pinfo
->addr
;
1389 pbl
= iwmr
->pgaddrmem
;
1392 i40iw_copy_user_pgaddrs(iwmr
, pbl
, level
);
1397 * i40iw_handle_q_mem - handle memory for qp and cq
1398 * @iwdev: iwarp device
1399 * @req: information for q memory management
1400 * @iwpbl: pble struct
1401 * @use_pbles: flag to use pble
1403 static int i40iw_handle_q_mem(struct i40iw_device
*iwdev
,
1404 struct i40iw_mem_reg_req
*req
,
1405 struct i40iw_pbl
*iwpbl
,
1408 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1409 struct i40iw_mr
*iwmr
= iwpbl
->iwmr
;
1410 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
1411 struct i40iw_cq_mr
*cqmr
= &iwpbl
->cq_mr
;
1412 struct i40iw_hmc_pble
*hmc_p
;
1413 u64
*arr
= iwmr
->pgaddrmem
;
1417 total
= req
->sq_pages
+ req
->rq_pages
+ req
->cq_pages
;
1419 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1422 if (use_pbles
&& (palloc
->level
!= I40IW_LEVEL_1
)) {
1423 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1424 iwpbl
->pbl_allocated
= false;
1429 arr
= (u64
*)palloc
->level1
.addr
;
1430 if (req
->reg_type
== IW_MEMREG_TYPE_QP
) {
1431 hmc_p
= &qpmr
->sq_pbl
;
1432 qpmr
->shadow
= (dma_addr_t
)arr
[total
];
1434 hmc_p
->idx
= palloc
->level1
.idx
;
1435 hmc_p
= &qpmr
->rq_pbl
;
1436 hmc_p
->idx
= palloc
->level1
.idx
+ req
->sq_pages
;
1438 hmc_p
->addr
= arr
[0];
1439 hmc_p
= &qpmr
->rq_pbl
;
1440 hmc_p
->addr
= arr
[1];
1443 hmc_p
= &cqmr
->cq_pbl
;
1444 cqmr
->shadow
= (dma_addr_t
)arr
[total
];
1446 hmc_p
->idx
= palloc
->level1
.idx
;
1448 hmc_p
->addr
= arr
[0];
1454 * i40iw_hw_alloc_stag - cqp command to allocate stag
1455 * @iwdev: iwarp device
1456 * @iwmr: iwarp mr pointer
1458 static int i40iw_hw_alloc_stag(struct i40iw_device
*iwdev
, struct i40iw_mr
*iwmr
)
1460 struct i40iw_allocate_stag_info
*info
;
1461 struct i40iw_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
1462 enum i40iw_status_code status
;
1464 struct i40iw_cqp_request
*cqp_request
;
1465 struct cqp_commands_info
*cqp_info
;
1467 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1471 cqp_info
= &cqp_request
->info
;
1472 info
= &cqp_info
->in
.u
.alloc_stag
.info
;
1473 memset(info
, 0, sizeof(*info
));
1474 info
->page_size
= PAGE_SIZE
;
1475 info
->stag_idx
= iwmr
->stag
>> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1476 info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1477 info
->total_len
= iwmr
->length
;
1478 cqp_info
->cqp_cmd
= OP_ALLOC_STAG
;
1479 cqp_info
->post_sq
= 1;
1480 cqp_info
->in
.u
.alloc_stag
.dev
= &iwdev
->sc_dev
;
1481 cqp_info
->in
.u
.alloc_stag
.scratch
= (uintptr_t)cqp_request
;
1483 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1486 i40iw_pr_err("CQP-OP MR Reg fail");
1492 * i40iw_alloc_mr - register stag for fast memory registration
1494 * @mr_type: memory for stag registrion
1495 * @max_num_sg: man number of pages
1497 static struct ib_mr
*i40iw_alloc_mr(struct ib_pd
*pd
,
1498 enum ib_mr_type mr_type
,
1501 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1502 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1503 struct i40iw_pble_alloc
*palloc
;
1504 struct i40iw_pbl
*iwpbl
;
1505 struct i40iw_mr
*iwmr
;
1506 enum i40iw_status_code status
;
1508 int err_code
= -ENOMEM
;
1510 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1512 return ERR_PTR(-ENOMEM
);
1514 stag
= i40iw_create_stag(iwdev
);
1516 err_code
= -EOVERFLOW
;
1520 iwmr
->ibmr
.rkey
= stag
;
1521 iwmr
->ibmr
.lkey
= stag
;
1523 iwmr
->ibmr
.device
= pd
->device
;
1524 iwpbl
= &iwmr
->iwpbl
;
1526 iwmr
->type
= IW_MEMREG_TYPE_MEM
;
1527 palloc
= &iwpbl
->pble_alloc
;
1528 iwmr
->page_cnt
= max_num_sg
;
1529 mutex_lock(&iwdev
->pbl_mutex
);
1530 status
= i40iw_get_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
, palloc
, iwmr
->page_cnt
);
1531 mutex_unlock(&iwdev
->pbl_mutex
);
1535 if (palloc
->level
!= I40IW_LEVEL_1
)
1537 err_code
= i40iw_hw_alloc_stag(iwdev
, iwmr
);
1540 iwpbl
->pbl_allocated
= true;
1541 i40iw_add_pdusecount(iwpd
);
1544 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1546 i40iw_free_stag(iwdev
, stag
);
1549 return ERR_PTR(err_code
);
1553 * i40iw_set_page - populate pbl list for fmr
1554 * @ibmr: ib mem to access iwarp mr pointer
1555 * @addr: page dma address fro pbl list
1557 static int i40iw_set_page(struct ib_mr
*ibmr
, u64 addr
)
1559 struct i40iw_mr
*iwmr
= to_iwmr(ibmr
);
1560 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1561 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1564 if (unlikely(iwmr
->npages
== iwmr
->page_cnt
))
1567 pbl
= (u64
*)palloc
->level1
.addr
;
1568 pbl
[iwmr
->npages
++] = cpu_to_le64(addr
);
1573 * i40iw_map_mr_sg - map of sg list for fmr
1574 * @ibmr: ib mem to access iwarp mr pointer
1575 * @sg: scatter gather list for fmr
1576 * @sg_nents: number of sg pages
1578 static int i40iw_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
, int sg_nents
)
1580 struct i40iw_mr
*iwmr
= to_iwmr(ibmr
);
1583 return ib_sg_to_pages(ibmr
, sg
, sg_nents
, i40iw_set_page
);
1587 * i40iw_drain_sq - drain the send queue
1588 * @ibqp: ib qp pointer
1590 static void i40iw_drain_sq(struct ib_qp
*ibqp
)
1592 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
1593 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
1595 if (I40IW_RING_MORE_WORK(qp
->qp_uk
.sq_ring
))
1596 wait_for_completion(&iwqp
->sq_drained
);
1600 * i40iw_drain_rq - drain the receive queue
1601 * @ibqp: ib qp pointer
1603 static void i40iw_drain_rq(struct ib_qp
*ibqp
)
1605 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
1606 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
1608 if (I40IW_RING_MORE_WORK(qp
->qp_uk
.rq_ring
))
1609 wait_for_completion(&iwqp
->rq_drained
);
1613 * i40iw_hwreg_mr - send cqp command for memory registration
1614 * @iwdev: iwarp device
1615 * @iwmr: iwarp mr pointer
1616 * @access: access for MR
1618 static int i40iw_hwreg_mr(struct i40iw_device
*iwdev
,
1619 struct i40iw_mr
*iwmr
,
1622 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1623 struct i40iw_reg_ns_stag_info
*stag_info
;
1624 struct i40iw_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
1625 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1626 enum i40iw_status_code status
;
1628 struct i40iw_cqp_request
*cqp_request
;
1629 struct cqp_commands_info
*cqp_info
;
1631 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1635 cqp_info
= &cqp_request
->info
;
1636 stag_info
= &cqp_info
->in
.u
.mr_reg_non_shared
.info
;
1637 memset(stag_info
, 0, sizeof(*stag_info
));
1638 stag_info
->va
= (void *)(unsigned long)iwpbl
->user_base
;
1639 stag_info
->stag_idx
= iwmr
->stag
>> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1640 stag_info
->stag_key
= (u8
)iwmr
->stag
;
1641 stag_info
->total_len
= iwmr
->length
;
1642 stag_info
->access_rights
= access
;
1643 stag_info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1644 stag_info
->addr_type
= I40IW_ADDR_TYPE_VA_BASED
;
1646 if (iwmr
->page_cnt
> 1) {
1647 if (palloc
->level
== I40IW_LEVEL_1
) {
1648 stag_info
->first_pm_pbl_index
= palloc
->level1
.idx
;
1649 stag_info
->chunk_size
= 1;
1651 stag_info
->first_pm_pbl_index
= palloc
->level2
.root
.idx
;
1652 stag_info
->chunk_size
= 3;
1655 stag_info
->reg_addr_pa
= iwmr
->pgaddrmem
[0];
1658 cqp_info
->cqp_cmd
= OP_MR_REG_NON_SHARED
;
1659 cqp_info
->post_sq
= 1;
1660 cqp_info
->in
.u
.mr_reg_non_shared
.dev
= &iwdev
->sc_dev
;
1661 cqp_info
->in
.u
.mr_reg_non_shared
.scratch
= (uintptr_t)cqp_request
;
1663 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1666 i40iw_pr_err("CQP-OP MR Reg fail");
1672 * i40iw_reg_user_mr - Register a user memory region
1674 * @start: virtual start address
1675 * @length: length of mr
1676 * @virt: virtual address
1677 * @acc: access of mr
1680 static struct ib_mr
*i40iw_reg_user_mr(struct ib_pd
*pd
,
1685 struct ib_udata
*udata
)
1687 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1688 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1689 struct i40iw_ucontext
*ucontext
;
1690 struct i40iw_pble_alloc
*palloc
;
1691 struct i40iw_pbl
*iwpbl
;
1692 struct i40iw_mr
*iwmr
;
1693 struct ib_umem
*region
;
1694 struct i40iw_mem_reg_req req
;
1699 bool use_pbles
= false;
1700 unsigned long flags
;
1703 if (length
> I40IW_MAX_MR_SIZE
)
1704 return ERR_PTR(-EINVAL
);
1705 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
1707 return (struct ib_mr
*)region
;
1709 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
1710 ib_umem_release(region
);
1711 return ERR_PTR(-EFAULT
);
1714 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1716 ib_umem_release(region
);
1717 return ERR_PTR(-ENOMEM
);
1720 iwpbl
= &iwmr
->iwpbl
;
1722 iwmr
->region
= region
;
1724 iwmr
->ibmr
.device
= pd
->device
;
1725 ucontext
= to_ucontext(pd
->uobject
->context
);
1726 region_length
= region
->length
+ (start
& 0xfff);
1727 pbl_depth
= region_length
>> 12;
1728 pbl_depth
+= (region_length
& (4096 - 1)) ? 1 : 0;
1729 iwmr
->length
= region
->length
;
1731 iwpbl
->user_base
= virt
;
1732 palloc
= &iwpbl
->pble_alloc
;
1734 iwmr
->type
= req
.reg_type
;
1735 iwmr
->page_cnt
= (u32
)pbl_depth
;
1737 switch (req
.reg_type
) {
1738 case IW_MEMREG_TYPE_QP
:
1739 use_pbles
= ((req
.sq_pages
+ req
.rq_pages
) > 2);
1740 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1743 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1744 list_add_tail(&iwpbl
->list
, &ucontext
->qp_reg_mem_list
);
1745 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1747 case IW_MEMREG_TYPE_CQ
:
1748 use_pbles
= (req
.cq_pages
> 1);
1749 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1753 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1754 list_add_tail(&iwpbl
->list
, &ucontext
->cq_reg_mem_list
);
1755 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1757 case IW_MEMREG_TYPE_MEM
:
1758 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1760 use_pbles
= (iwmr
->page_cnt
!= 1);
1761 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1765 access
|= i40iw_get_user_access(acc
);
1766 stag
= i40iw_create_stag(iwdev
);
1773 iwmr
->ibmr
.rkey
= stag
;
1774 iwmr
->ibmr
.lkey
= stag
;
1776 err
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1778 i40iw_free_stag(iwdev
, stag
);
1786 iwmr
->type
= req
.reg_type
;
1787 if (req
.reg_type
== IW_MEMREG_TYPE_MEM
)
1788 i40iw_add_pdusecount(iwpd
);
1792 if (palloc
->level
!= I40IW_LEVEL_0
)
1793 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1794 ib_umem_release(region
);
1796 return ERR_PTR(err
);
1800 * i40iw_reg_phys_mr - register kernel physical memory
1802 * @addr: physical address of memory to register
1803 * @size: size of memory to register
1804 * @acc: Access rights
1805 * @iova_start: start of virtual address for physical buffers
1807 struct ib_mr
*i40iw_reg_phys_mr(struct ib_pd
*pd
,
1813 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1814 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1815 struct i40iw_pbl
*iwpbl
;
1816 struct i40iw_mr
*iwmr
;
1817 enum i40iw_status_code status
;
1819 u16 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1822 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1824 return ERR_PTR(-ENOMEM
);
1826 iwmr
->ibmr
.device
= pd
->device
;
1827 iwpbl
= &iwmr
->iwpbl
;
1829 iwmr
->type
= IW_MEMREG_TYPE_MEM
;
1830 iwpbl
->user_base
= *iova_start
;
1831 stag
= i40iw_create_stag(iwdev
);
1836 access
|= i40iw_get_user_access(acc
);
1838 iwmr
->ibmr
.rkey
= stag
;
1839 iwmr
->ibmr
.lkey
= stag
;
1841 iwmr
->pgaddrmem
[0] = addr
;
1842 status
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1844 i40iw_free_stag(iwdev
, stag
);
1849 i40iw_add_pdusecount(iwpd
);
1853 return ERR_PTR(ret
);
1857 * i40iw_get_dma_mr - register physical mem
1859 * @acc: access for memory
1861 static struct ib_mr
*i40iw_get_dma_mr(struct ib_pd
*pd
, int acc
)
1865 return i40iw_reg_phys_mr(pd
, 0, 0xffffffffffULL
, acc
, &kva
);
1869 * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
1870 * @iwmr: iwmr for IB's user page addresses
1871 * @ucontext: ptr to user context
1873 static void i40iw_del_memlist(struct i40iw_mr
*iwmr
,
1874 struct i40iw_ucontext
*ucontext
)
1876 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1877 unsigned long flags
;
1879 switch (iwmr
->type
) {
1880 case IW_MEMREG_TYPE_CQ
:
1881 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1882 if (!list_empty(&ucontext
->cq_reg_mem_list
))
1883 list_del(&iwpbl
->list
);
1884 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1886 case IW_MEMREG_TYPE_QP
:
1887 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1888 if (!list_empty(&ucontext
->qp_reg_mem_list
))
1889 list_del(&iwpbl
->list
);
1890 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1898 * i40iw_dereg_mr - deregister mr
1899 * @ib_mr: mr ptr for dereg
1901 static int i40iw_dereg_mr(struct ib_mr
*ib_mr
)
1903 struct ib_pd
*ibpd
= ib_mr
->pd
;
1904 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
1905 struct i40iw_mr
*iwmr
= to_iwmr(ib_mr
);
1906 struct i40iw_device
*iwdev
= to_iwdev(ib_mr
->device
);
1907 enum i40iw_status_code status
;
1908 struct i40iw_dealloc_stag_info
*info
;
1909 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1910 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1911 struct i40iw_cqp_request
*cqp_request
;
1912 struct cqp_commands_info
*cqp_info
;
1916 ib_umem_release(iwmr
->region
);
1918 if (iwmr
->type
!= IW_MEMREG_TYPE_MEM
) {
1919 if (ibpd
->uobject
) {
1920 struct i40iw_ucontext
*ucontext
;
1922 ucontext
= to_ucontext(ibpd
->uobject
->context
);
1923 i40iw_del_memlist(iwmr
, ucontext
);
1925 if (iwpbl
->pbl_allocated
)
1926 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1932 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1936 cqp_info
= &cqp_request
->info
;
1937 info
= &cqp_info
->in
.u
.dealloc_stag
.info
;
1938 memset(info
, 0, sizeof(*info
));
1940 info
->pd_id
= cpu_to_le32(iwpd
->sc_pd
.pd_id
& 0x00007fff);
1941 info
->stag_idx
= RS_64_1(ib_mr
->rkey
, I40IW_CQPSQ_STAG_IDX_SHIFT
);
1942 stag_idx
= info
->stag_idx
;
1944 if (iwpbl
->pbl_allocated
)
1945 info
->dealloc_pbl
= true;
1947 cqp_info
->cqp_cmd
= OP_DEALLOC_STAG
;
1948 cqp_info
->post_sq
= 1;
1949 cqp_info
->in
.u
.dealloc_stag
.dev
= &iwdev
->sc_dev
;
1950 cqp_info
->in
.u
.dealloc_stag
.scratch
= (uintptr_t)cqp_request
;
1951 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1953 i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx
);
1954 i40iw_rem_pdusecount(iwpd
, iwdev
);
1955 i40iw_free_stag(iwdev
, iwmr
->stag
);
1956 if (iwpbl
->pbl_allocated
)
1957 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1965 static ssize_t
i40iw_show_rev(struct device
*dev
,
1966 struct device_attribute
*attr
, char *buf
)
1968 struct i40iw_ib_device
*iwibdev
= container_of(dev
,
1969 struct i40iw_ib_device
,
1971 u32 hw_rev
= iwibdev
->iwdev
->sc_dev
.hw_rev
;
1973 return sprintf(buf
, "%x\n", hw_rev
);
1979 static ssize_t
i40iw_show_fw_ver(struct device
*dev
,
1980 struct device_attribute
*attr
, char *buf
)
1982 u32 firmware_version
= I40IW_FW_VERSION
;
1984 return sprintf(buf
, "%u.%u\n", firmware_version
,
1985 (firmware_version
& 0x000000ff));
1991 static ssize_t
i40iw_show_hca(struct device
*dev
,
1992 struct device_attribute
*attr
, char *buf
)
1994 return sprintf(buf
, "I40IW\n");
2000 static ssize_t
i40iw_show_board(struct device
*dev
,
2001 struct device_attribute
*attr
,
2004 return sprintf(buf
, "%.*s\n", 32, "I40IW Board ID");
2007 static DEVICE_ATTR(hw_rev
, S_IRUGO
, i40iw_show_rev
, NULL
);
2008 static DEVICE_ATTR(fw_ver
, S_IRUGO
, i40iw_show_fw_ver
, NULL
);
2009 static DEVICE_ATTR(hca_type
, S_IRUGO
, i40iw_show_hca
, NULL
);
2010 static DEVICE_ATTR(board_id
, S_IRUGO
, i40iw_show_board
, NULL
);
2012 static struct device_attribute
*i40iw_dev_attributes
[] = {
2020 * i40iw_copy_sg_list - copy sg list for qp
2021 * @sg_list: copied into sg_list
2022 * @sgl: copy from sgl
2023 * @num_sges: count of sg entries
2025 static void i40iw_copy_sg_list(struct i40iw_sge
*sg_list
, struct ib_sge
*sgl
, int num_sges
)
2029 for (i
= 0; (i
< num_sges
) && (i
< I40IW_MAX_WQ_FRAGMENT_COUNT
); i
++) {
2030 sg_list
[i
].tag_off
= sgl
[i
].addr
;
2031 sg_list
[i
].len
= sgl
[i
].length
;
2032 sg_list
[i
].stag
= sgl
[i
].lkey
;
2037 * i40iw_post_send - kernel application wr
2038 * @ibqp: qp ptr for wr
2039 * @ib_wr: work request ptr
2040 * @bad_wr: return of bad wr if err
2042 static int i40iw_post_send(struct ib_qp
*ibqp
,
2043 struct ib_send_wr
*ib_wr
,
2044 struct ib_send_wr
**bad_wr
)
2046 struct i40iw_qp
*iwqp
;
2047 struct i40iw_qp_uk
*ukqp
;
2048 struct i40iw_post_sq_info info
;
2049 enum i40iw_status_code ret
;
2051 unsigned long flags
;
2054 iwqp
= (struct i40iw_qp
*)ibqp
;
2055 ukqp
= &iwqp
->sc_qp
.qp_uk
;
2057 spin_lock_irqsave(&iwqp
->lock
, flags
);
2060 memset(&info
, 0, sizeof(info
));
2061 info
.wr_id
= (u64
)(ib_wr
->wr_id
);
2062 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || iwqp
->sig_all
)
2063 info
.signaled
= true;
2064 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
2065 info
.read_fence
= true;
2067 switch (ib_wr
->opcode
) {
2070 case IB_WR_SEND_WITH_INV
:
2071 if (ib_wr
->opcode
== IB_WR_SEND
) {
2072 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
2073 info
.op_type
= I40IW_OP_TYPE_SEND_SOL
;
2075 info
.op_type
= I40IW_OP_TYPE_SEND
;
2077 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
2078 info
.op_type
= I40IW_OP_TYPE_SEND_SOL_INV
;
2080 info
.op_type
= I40IW_OP_TYPE_SEND_INV
;
2083 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
2084 info
.op
.inline_send
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
2085 info
.op
.inline_send
.len
= ib_wr
->sg_list
[0].length
;
2086 ret
= ukqp
->ops
.iw_inline_send(ukqp
, &info
, ib_wr
->ex
.invalidate_rkey
, false);
2088 info
.op
.send
.num_sges
= ib_wr
->num_sge
;
2089 info
.op
.send
.sg_list
= (struct i40iw_sge
*)ib_wr
->sg_list
;
2090 ret
= ukqp
->ops
.iw_send(ukqp
, &info
, ib_wr
->ex
.invalidate_rkey
, false);
2096 case IB_WR_RDMA_WRITE
:
2097 info
.op_type
= I40IW_OP_TYPE_RDMA_WRITE
;
2099 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
2100 info
.op
.inline_rdma_write
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
2101 info
.op
.inline_rdma_write
.len
= ib_wr
->sg_list
[0].length
;
2102 info
.op
.inline_rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
2103 info
.op
.inline_rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
2104 info
.op
.inline_rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
2105 ret
= ukqp
->ops
.iw_inline_rdma_write(ukqp
, &info
, false);
2107 info
.op
.rdma_write
.lo_sg_list
= (void *)ib_wr
->sg_list
;
2108 info
.op
.rdma_write
.num_lo_sges
= ib_wr
->num_sge
;
2109 info
.op
.rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
2110 info
.op
.rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
2111 info
.op
.rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
2112 ret
= ukqp
->ops
.iw_rdma_write(ukqp
, &info
, false);
2118 case IB_WR_RDMA_READ_WITH_INV
:
2121 case IB_WR_RDMA_READ
:
2122 info
.op_type
= I40IW_OP_TYPE_RDMA_READ
;
2123 info
.op
.rdma_read
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
2124 info
.op
.rdma_read
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
2125 info
.op
.rdma_read
.rem_addr
.len
= ib_wr
->sg_list
->length
;
2126 info
.op
.rdma_read
.lo_addr
.tag_off
= ib_wr
->sg_list
->addr
;
2127 info
.op
.rdma_read
.lo_addr
.stag
= ib_wr
->sg_list
->lkey
;
2128 info
.op
.rdma_read
.lo_addr
.len
= ib_wr
->sg_list
->length
;
2129 ret
= ukqp
->ops
.iw_rdma_read(ukqp
, &info
, inv_stag
, false);
2133 case IB_WR_LOCAL_INV
:
2134 info
.op_type
= I40IW_OP_TYPE_INV_STAG
;
2135 info
.op
.inv_local_stag
.target_stag
= ib_wr
->ex
.invalidate_rkey
;
2136 ret
= ukqp
->ops
.iw_stag_local_invalidate(ukqp
, &info
, true);
2142 struct i40iw_mr
*iwmr
= to_iwmr(reg_wr(ib_wr
)->mr
);
2143 int page_shift
= ilog2(reg_wr(ib_wr
)->mr
->page_size
);
2144 int flags
= reg_wr(ib_wr
)->access
;
2145 struct i40iw_pble_alloc
*palloc
= &iwmr
->iwpbl
.pble_alloc
;
2146 struct i40iw_sc_dev
*dev
= &iwqp
->iwdev
->sc_dev
;
2147 struct i40iw_fast_reg_stag_info info
;
2149 info
.access_rights
= I40IW_ACCESS_FLAGS_LOCALREAD
;
2150 info
.access_rights
|= i40iw_get_user_access(flags
);
2151 info
.stag_key
= reg_wr(ib_wr
)->key
& 0xff;
2152 info
.stag_idx
= reg_wr(ib_wr
)->key
>> 8;
2153 info
.wr_id
= ib_wr
->wr_id
;
2155 info
.addr_type
= I40IW_ADDR_TYPE_VA_BASED
;
2156 info
.va
= (void *)(uintptr_t)iwmr
->ibmr
.iova
;
2157 info
.total_len
= iwmr
->ibmr
.length
;
2158 info
.first_pm_pbl_index
= palloc
->level1
.idx
;
2159 info
.local_fence
= ib_wr
->send_flags
& IB_SEND_FENCE
;
2160 info
.signaled
= ib_wr
->send_flags
& IB_SEND_SIGNALED
;
2162 if (page_shift
== 21)
2163 info
.page_size
= 1; /* 2M page */
2165 ret
= dev
->iw_priv_qp_ops
->iw_mr_fast_register(&iwqp
->sc_qp
, &info
, true);
2172 i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
2179 ib_wr
= ib_wr
->next
;
2185 ukqp
->ops
.iw_qp_post_wr(ukqp
);
2186 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
2192 * i40iw_post_recv - post receive wr for kernel application
2193 * @ibqp: ib qp pointer
2194 * @ib_wr: work request for receive
2195 * @bad_wr: bad wr caused an error
2197 static int i40iw_post_recv(struct ib_qp
*ibqp
,
2198 struct ib_recv_wr
*ib_wr
,
2199 struct ib_recv_wr
**bad_wr
)
2201 struct i40iw_qp
*iwqp
;
2202 struct i40iw_qp_uk
*ukqp
;
2203 struct i40iw_post_rq_info post_recv
;
2204 struct i40iw_sge sg_list
[I40IW_MAX_WQ_FRAGMENT_COUNT
];
2205 enum i40iw_status_code ret
= 0;
2206 unsigned long flags
;
2208 iwqp
= (struct i40iw_qp
*)ibqp
;
2209 ukqp
= &iwqp
->sc_qp
.qp_uk
;
2211 memset(&post_recv
, 0, sizeof(post_recv
));
2212 spin_lock_irqsave(&iwqp
->lock
, flags
);
2214 post_recv
.num_sges
= ib_wr
->num_sge
;
2215 post_recv
.wr_id
= ib_wr
->wr_id
;
2216 i40iw_copy_sg_list(sg_list
, ib_wr
->sg_list
, ib_wr
->num_sge
);
2217 post_recv
.sg_list
= sg_list
;
2218 ret
= ukqp
->ops
.iw_post_receive(ukqp
, &post_recv
);
2220 i40iw_pr_err(" post_recv err %d\n", ret
);
2224 ib_wr
= ib_wr
->next
;
2227 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
2234 * i40iw_poll_cq - poll cq for completion (kernel apps)
2236 * @num_entries: number of entries to poll
2237 * @entry: wr of entry completed
2239 static int i40iw_poll_cq(struct ib_cq
*ibcq
,
2241 struct ib_wc
*entry
)
2243 struct i40iw_cq
*iwcq
;
2245 struct i40iw_cq_poll_info cq_poll_info
;
2246 enum i40iw_status_code ret
;
2247 struct i40iw_cq_uk
*ukcq
;
2248 struct i40iw_sc_qp
*qp
;
2249 struct i40iw_qp
*iwqp
;
2250 unsigned long flags
;
2252 iwcq
= (struct i40iw_cq
*)ibcq
;
2253 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2255 spin_lock_irqsave(&iwcq
->lock
, flags
);
2256 while (cqe_count
< num_entries
) {
2257 ret
= ukcq
->ops
.iw_cq_poll_completion(ukcq
, &cq_poll_info
, true);
2258 if (ret
== I40IW_ERR_QUEUE_EMPTY
) {
2265 entry
->wc_flags
= 0;
2266 entry
->wr_id
= cq_poll_info
.wr_id
;
2267 if (cq_poll_info
.error
) {
2268 entry
->status
= IB_WC_WR_FLUSH_ERR
;
2269 entry
->vendor_err
= cq_poll_info
.major_err
<< 16 | cq_poll_info
.minor_err
;
2271 entry
->status
= IB_WC_SUCCESS
;
2274 switch (cq_poll_info
.op_type
) {
2275 case I40IW_OP_TYPE_RDMA_WRITE
:
2276 entry
->opcode
= IB_WC_RDMA_WRITE
;
2278 case I40IW_OP_TYPE_RDMA_READ_INV_STAG
:
2279 case I40IW_OP_TYPE_RDMA_READ
:
2280 entry
->opcode
= IB_WC_RDMA_READ
;
2282 case I40IW_OP_TYPE_SEND_SOL
:
2283 case I40IW_OP_TYPE_SEND_SOL_INV
:
2284 case I40IW_OP_TYPE_SEND_INV
:
2285 case I40IW_OP_TYPE_SEND
:
2286 entry
->opcode
= IB_WC_SEND
;
2288 case I40IW_OP_TYPE_REC
:
2289 entry
->opcode
= IB_WC_RECV
;
2292 entry
->opcode
= IB_WC_RECV
;
2296 entry
->ex
.imm_data
= 0;
2297 qp
= (struct i40iw_sc_qp
*)cq_poll_info
.qp_handle
;
2298 entry
->qp
= (struct ib_qp
*)qp
->back_qp
;
2299 entry
->src_qp
= cq_poll_info
.qp_id
;
2300 iwqp
= (struct i40iw_qp
*)qp
->back_qp
;
2301 if (iwqp
->iwarp_state
> I40IW_QP_STATE_RTS
) {
2302 if (!I40IW_RING_MORE_WORK(qp
->qp_uk
.sq_ring
))
2303 complete(&iwqp
->sq_drained
);
2304 if (!I40IW_RING_MORE_WORK(qp
->qp_uk
.rq_ring
))
2305 complete(&iwqp
->rq_drained
);
2307 entry
->byte_len
= cq_poll_info
.bytes_xfered
;
2311 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
2316 * i40iw_req_notify_cq - arm cq kernel application
2318 * @notify_flags: notofication flags
2320 static int i40iw_req_notify_cq(struct ib_cq
*ibcq
,
2321 enum ib_cq_notify_flags notify_flags
)
2323 struct i40iw_cq
*iwcq
;
2324 struct i40iw_cq_uk
*ukcq
;
2325 enum i40iw_completion_notify cq_notify
= IW_CQ_COMPL_SOLICITED
;
2327 iwcq
= (struct i40iw_cq
*)ibcq
;
2328 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2329 if (notify_flags
== IB_CQ_NEXT_COMP
)
2330 cq_notify
= IW_CQ_COMPL_EVENT
;
2331 ukcq
->ops
.iw_cq_request_notification(ukcq
, cq_notify
);
2336 * i40iw_port_immutable - return port's immutable data
2337 * @ibdev: ib dev struct
2338 * @port_num: port number
2339 * @immutable: immutable data for the port return
2341 static int i40iw_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
2342 struct ib_port_immutable
*immutable
)
2344 struct ib_port_attr attr
;
2347 err
= i40iw_query_port(ibdev
, port_num
, &attr
);
2352 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
2353 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
2354 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
2360 * i40iw_get_protocol_stats - Populates the rdma_stats structure
2361 * @ibdev: ib dev struct
2362 * @stats: iw protocol stats struct
2364 static int i40iw_get_protocol_stats(struct ib_device
*ibdev
,
2365 union rdma_protocol_stats
*stats
)
2367 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2368 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
2369 struct i40iw_dev_pestat
*devstat
= &dev
->dev_pestat
;
2370 struct i40iw_dev_hw_stats
*hw_stats
= &devstat
->hw_stats
;
2371 struct timespec curr_time
;
2372 static struct timespec last_rd_time
= {0, 0};
2373 unsigned long flags
;
2375 curr_time
= current_kernel_time();
2376 memset(stats
, 0, sizeof(*stats
));
2379 spin_lock_irqsave(&devstat
->stats_lock
, flags
);
2380 devstat
->ops
.iw_hw_stat_read_all(devstat
,
2381 &devstat
->hw_stats
);
2382 spin_unlock_irqrestore(&devstat
->stats_lock
, flags
);
2384 if (((u64
)curr_time
.tv_sec
- (u64
)last_rd_time
.tv_sec
) > 1)
2385 if (i40iw_vchnl_vf_get_pe_stats(dev
, &devstat
->hw_stats
))
2389 stats
->iw
.ipInReceives
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] +
2390 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
];
2391 stats
->iw
.ipInTruncatedPkts
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] +
2392 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
];
2393 stats
->iw
.ipInDiscards
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] +
2394 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
];
2395 stats
->iw
.ipOutNoRoutes
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] +
2396 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
];
2397 stats
->iw
.ipReasmReqds
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] +
2398 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
];
2399 stats
->iw
.ipFragCreates
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] +
2400 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
];
2401 stats
->iw
.ipInMcastPkts
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] +
2402 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
];
2403 stats
->iw
.ipOutMcastPkts
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] +
2404 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6TXMCPKTS
];
2405 stats
->iw
.tcpOutSegs
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
];
2406 stats
->iw
.tcpInSegs
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
];
2407 stats
->iw
.tcpRetransSegs
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
];
2409 last_rd_time
= curr_time
;
2414 * i40iw_query_gid - Query port GID
2415 * @ibdev: device pointer from stack
2416 * @port: port number
2417 * @index: Entry index
2420 static int i40iw_query_gid(struct ib_device
*ibdev
,
2425 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2427 memset(gid
->raw
, 0, sizeof(gid
->raw
));
2428 ether_addr_copy(gid
->raw
, iwdev
->netdev
->dev_addr
);
2433 * i40iw_modify_port Modify port properties
2434 * @ibdev: device pointer from stack
2435 * @port: port number
2436 * @port_modify_mask: mask for port modifications
2437 * @props: port properties
2439 static int i40iw_modify_port(struct ib_device
*ibdev
,
2441 int port_modify_mask
,
2442 struct ib_port_modify
*props
)
2448 * i40iw_query_pkey - Query partition key
2449 * @ibdev: device pointer from stack
2450 * @port: port number
2451 * @index: index of pkey
2452 * @pkey: pointer to store the pkey
2454 static int i40iw_query_pkey(struct ib_device
*ibdev
,
2464 * i40iw_create_ah - create address handle
2466 * @ah_attr: address handle attributes
2468 static struct ib_ah
*i40iw_create_ah(struct ib_pd
*ibpd
,
2469 struct ib_ah_attr
*attr
)
2471 return ERR_PTR(-ENOSYS
);
2475 * i40iw_destroy_ah - Destroy address handle
2476 * @ah: pointer to address handle
2478 static int i40iw_destroy_ah(struct ib_ah
*ah
)
2484 * i40iw_init_rdma_device - initialization of iwarp device
2485 * @iwdev: iwarp device
2487 static struct i40iw_ib_device
*i40iw_init_rdma_device(struct i40iw_device
*iwdev
)
2489 struct i40iw_ib_device
*iwibdev
;
2490 struct net_device
*netdev
= iwdev
->netdev
;
2491 struct pci_dev
*pcidev
= (struct pci_dev
*)iwdev
->hw
.dev_context
;
2493 iwibdev
= (struct i40iw_ib_device
*)ib_alloc_device(sizeof(*iwibdev
));
2495 i40iw_pr_err("iwdev == NULL\n");
2498 strlcpy(iwibdev
->ibdev
.name
, "i40iw%d", IB_DEVICE_NAME_MAX
);
2499 iwibdev
->ibdev
.owner
= THIS_MODULE
;
2500 iwdev
->iwibdev
= iwibdev
;
2501 iwibdev
->iwdev
= iwdev
;
2503 iwibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
2504 ether_addr_copy((u8
*)&iwibdev
->ibdev
.node_guid
, netdev
->dev_addr
);
2506 iwibdev
->ibdev
.uverbs_cmd_mask
=
2507 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
2508 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
2509 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
2510 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
2511 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
2512 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
2513 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
2514 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
2515 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
2516 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
2517 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
2518 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
2519 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
2520 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
2521 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
2522 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
2523 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
2524 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
2525 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
2526 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
2527 iwibdev
->ibdev
.phys_port_cnt
= 1;
2528 iwibdev
->ibdev
.num_comp_vectors
= 1;
2529 iwibdev
->ibdev
.dma_device
= &pcidev
->dev
;
2530 iwibdev
->ibdev
.dev
.parent
= &pcidev
->dev
;
2531 iwibdev
->ibdev
.query_port
= i40iw_query_port
;
2532 iwibdev
->ibdev
.modify_port
= i40iw_modify_port
;
2533 iwibdev
->ibdev
.query_pkey
= i40iw_query_pkey
;
2534 iwibdev
->ibdev
.query_gid
= i40iw_query_gid
;
2535 iwibdev
->ibdev
.alloc_ucontext
= i40iw_alloc_ucontext
;
2536 iwibdev
->ibdev
.dealloc_ucontext
= i40iw_dealloc_ucontext
;
2537 iwibdev
->ibdev
.mmap
= i40iw_mmap
;
2538 iwibdev
->ibdev
.alloc_pd
= i40iw_alloc_pd
;
2539 iwibdev
->ibdev
.dealloc_pd
= i40iw_dealloc_pd
;
2540 iwibdev
->ibdev
.create_qp
= i40iw_create_qp
;
2541 iwibdev
->ibdev
.modify_qp
= i40iw_modify_qp
;
2542 iwibdev
->ibdev
.query_qp
= i40iw_query_qp
;
2543 iwibdev
->ibdev
.destroy_qp
= i40iw_destroy_qp
;
2544 iwibdev
->ibdev
.create_cq
= i40iw_create_cq
;
2545 iwibdev
->ibdev
.destroy_cq
= i40iw_destroy_cq
;
2546 iwibdev
->ibdev
.get_dma_mr
= i40iw_get_dma_mr
;
2547 iwibdev
->ibdev
.reg_user_mr
= i40iw_reg_user_mr
;
2548 iwibdev
->ibdev
.dereg_mr
= i40iw_dereg_mr
;
2549 iwibdev
->ibdev
.get_protocol_stats
= i40iw_get_protocol_stats
;
2550 iwibdev
->ibdev
.query_device
= i40iw_query_device
;
2551 iwibdev
->ibdev
.create_ah
= i40iw_create_ah
;
2552 iwibdev
->ibdev
.destroy_ah
= i40iw_destroy_ah
;
2553 iwibdev
->ibdev
.drain_sq
= i40iw_drain_sq
;
2554 iwibdev
->ibdev
.drain_rq
= i40iw_drain_rq
;
2555 iwibdev
->ibdev
.alloc_mr
= i40iw_alloc_mr
;
2556 iwibdev
->ibdev
.map_mr_sg
= i40iw_map_mr_sg
;
2557 iwibdev
->ibdev
.iwcm
= kzalloc(sizeof(*iwibdev
->ibdev
.iwcm
), GFP_KERNEL
);
2558 if (!iwibdev
->ibdev
.iwcm
) {
2559 ib_dealloc_device(&iwibdev
->ibdev
);
2560 i40iw_pr_err("iwcm == NULL\n");
2564 iwibdev
->ibdev
.iwcm
->add_ref
= i40iw_add_ref
;
2565 iwibdev
->ibdev
.iwcm
->rem_ref
= i40iw_rem_ref
;
2566 iwibdev
->ibdev
.iwcm
->get_qp
= i40iw_get_qp
;
2567 iwibdev
->ibdev
.iwcm
->connect
= i40iw_connect
;
2568 iwibdev
->ibdev
.iwcm
->accept
= i40iw_accept
;
2569 iwibdev
->ibdev
.iwcm
->reject
= i40iw_reject
;
2570 iwibdev
->ibdev
.iwcm
->create_listen
= i40iw_create_listen
;
2571 iwibdev
->ibdev
.iwcm
->destroy_listen
= i40iw_destroy_listen
;
2572 memcpy(iwibdev
->ibdev
.iwcm
->ifname
, netdev
->name
,
2573 sizeof(iwibdev
->ibdev
.iwcm
->ifname
));
2574 iwibdev
->ibdev
.get_port_immutable
= i40iw_port_immutable
;
2575 iwibdev
->ibdev
.poll_cq
= i40iw_poll_cq
;
2576 iwibdev
->ibdev
.req_notify_cq
= i40iw_req_notify_cq
;
2577 iwibdev
->ibdev
.post_send
= i40iw_post_send
;
2578 iwibdev
->ibdev
.post_recv
= i40iw_post_recv
;
2584 * i40iw_port_ibevent - indicate port event
2585 * @iwdev: iwarp device
2587 void i40iw_port_ibevent(struct i40iw_device
*iwdev
)
2589 struct i40iw_ib_device
*iwibdev
= iwdev
->iwibdev
;
2590 struct ib_event event
;
2592 event
.device
= &iwibdev
->ibdev
;
2593 event
.element
.port_num
= 1;
2594 event
.event
= iwdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
2595 ib_dispatch_event(&event
);
2599 * i40iw_unregister_rdma_device - unregister of iwarp from IB
2600 * @iwibdev: rdma device ptr
2602 static void i40iw_unregister_rdma_device(struct i40iw_ib_device
*iwibdev
)
2606 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
)
2607 device_remove_file(&iwibdev
->ibdev
.dev
,
2608 i40iw_dev_attributes
[i
]);
2609 ib_unregister_device(&iwibdev
->ibdev
);
2613 * i40iw_destroy_rdma_device - destroy rdma device and free resources
2614 * @iwibdev: IB device ptr
2616 void i40iw_destroy_rdma_device(struct i40iw_ib_device
*iwibdev
)
2621 i40iw_unregister_rdma_device(iwibdev
);
2622 kfree(iwibdev
->ibdev
.iwcm
);
2623 iwibdev
->ibdev
.iwcm
= NULL
;
2624 ib_dealloc_device(&iwibdev
->ibdev
);
2628 * i40iw_register_rdma_device - register iwarp device to IB
2629 * @iwdev: iwarp device
2631 int i40iw_register_rdma_device(struct i40iw_device
*iwdev
)
2634 struct i40iw_ib_device
*iwibdev
;
2636 iwdev
->iwibdev
= i40iw_init_rdma_device(iwdev
);
2637 if (!iwdev
->iwibdev
)
2639 iwibdev
= iwdev
->iwibdev
;
2641 ret
= ib_register_device(&iwibdev
->ibdev
, NULL
);
2645 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
) {
2647 device_create_file(&iwibdev
->ibdev
.dev
,
2648 i40iw_dev_attributes
[i
]);
2652 device_remove_file(&iwibdev
->ibdev
.dev
, i40iw_dev_attributes
[i
]);
2654 ib_unregister_device(&iwibdev
->ibdev
);
2660 kfree(iwdev
->iwibdev
->ibdev
.iwcm
);
2661 iwdev
->iwibdev
->ibdev
.iwcm
= NULL
;
2662 ib_dealloc_device(&iwdev
->iwibdev
->ibdev
);