1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/random.h>
38 #include <linux/highmem.h>
39 #include <linux/time.h>
40 #include <asm/byteorder.h>
42 #include <rdma/ib_verbs.h>
43 #include <rdma/iw_cm.h>
44 #include <rdma/ib_user_verbs.h>
45 #include <rdma/ib_umem.h>
49 * i40iw_query_device - get device attributes
50 * @ibdev: device pointer from stack
51 * @props: returning device attributes
54 static int i40iw_query_device(struct ib_device
*ibdev
,
55 struct ib_device_attr
*props
,
56 struct ib_udata
*udata
)
58 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
60 if (udata
->inlen
|| udata
->outlen
)
62 memset(props
, 0, sizeof(*props
));
63 ether_addr_copy((u8
*)&props
->sys_image_guid
, iwdev
->netdev
->dev_addr
);
64 props
->fw_ver
= I40IW_FW_VERSION
;
65 props
->device_cap_flags
= iwdev
->device_cap_flags
;
66 props
->vendor_id
= iwdev
->vendor_id
;
67 props
->vendor_part_id
= iwdev
->vendor_part_id
;
68 props
->hw_ver
= (u32
)iwdev
->sc_dev
.hw_rev
;
69 props
->max_mr_size
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
70 props
->max_qp
= iwdev
->max_qp
;
71 props
->max_qp_wr
= (I40IW_MAX_WQ_ENTRIES
>> 2) - 1;
72 props
->max_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
73 props
->max_cq
= iwdev
->max_cq
;
74 props
->max_cqe
= iwdev
->max_cqe
;
75 props
->max_mr
= iwdev
->max_mr
;
76 props
->max_pd
= iwdev
->max_pd
;
77 props
->max_sge_rd
= 1;
78 props
->max_qp_rd_atom
= I40IW_MAX_IRD_SIZE
;
79 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
80 props
->atomic_cap
= IB_ATOMIC_NONE
;
81 props
->max_map_per_fmr
= 1;
86 * i40iw_query_port - get port attrubutes
87 * @ibdev: device pointer from stack
88 * @port: port number for query
89 * @props: returning device attributes
91 static int i40iw_query_port(struct ib_device
*ibdev
,
93 struct ib_port_attr
*props
)
95 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
96 struct net_device
*netdev
= iwdev
->netdev
;
98 memset(props
, 0, sizeof(*props
));
100 props
->max_mtu
= IB_MTU_4096
;
101 if (netdev
->mtu
>= 4096)
102 props
->active_mtu
= IB_MTU_4096
;
103 else if (netdev
->mtu
>= 2048)
104 props
->active_mtu
= IB_MTU_2048
;
105 else if (netdev
->mtu
>= 1024)
106 props
->active_mtu
= IB_MTU_1024
;
107 else if (netdev
->mtu
>= 512)
108 props
->active_mtu
= IB_MTU_512
;
110 props
->active_mtu
= IB_MTU_256
;
113 if (netif_carrier_ok(iwdev
->netdev
))
114 props
->state
= IB_PORT_ACTIVE
;
116 props
->state
= IB_PORT_DOWN
;
117 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
118 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
119 props
->gid_tbl_len
= 1;
120 props
->pkey_tbl_len
= 1;
121 props
->active_width
= IB_WIDTH_4X
;
122 props
->active_speed
= 1;
123 props
->max_msg_sz
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
128 * i40iw_alloc_ucontext - Allocate the user context data structure
129 * @ibdev: device pointer from stack
132 * This keeps track of all objects associated with a particular
135 static struct ib_ucontext
*i40iw_alloc_ucontext(struct ib_device
*ibdev
,
136 struct ib_udata
*udata
)
138 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
139 struct i40iw_alloc_ucontext_req req
;
140 struct i40iw_alloc_ucontext_resp uresp
;
141 struct i40iw_ucontext
*ucontext
;
143 if (ib_copy_from_udata(&req
, udata
, sizeof(req
)))
144 return ERR_PTR(-EINVAL
);
146 if (req
.userspace_ver
!= I40IW_ABI_USERSPACE_VER
) {
147 i40iw_pr_err("Invalid userspace driver version detected. Detected version %d, should be %d\n",
148 req
.userspace_ver
, I40IW_ABI_USERSPACE_VER
);
149 return ERR_PTR(-EINVAL
);
152 memset(&uresp
, 0, sizeof(uresp
));
153 uresp
.max_qps
= iwdev
->max_qp
;
154 uresp
.max_pds
= iwdev
->max_pd
;
155 uresp
.wq_size
= iwdev
->max_qp_wr
* 2;
156 uresp
.kernel_ver
= I40IW_ABI_KERNEL_VER
;
158 ucontext
= kzalloc(sizeof(*ucontext
), GFP_KERNEL
);
160 return ERR_PTR(-ENOMEM
);
162 ucontext
->iwdev
= iwdev
;
164 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
166 return ERR_PTR(-EFAULT
);
169 INIT_LIST_HEAD(&ucontext
->cq_reg_mem_list
);
170 spin_lock_init(&ucontext
->cq_reg_mem_list_lock
);
171 INIT_LIST_HEAD(&ucontext
->qp_reg_mem_list
);
172 spin_lock_init(&ucontext
->qp_reg_mem_list_lock
);
174 return &ucontext
->ibucontext
;
178 * i40iw_dealloc_ucontext - deallocate the user context data structure
179 * @context: user context created during alloc
181 static int i40iw_dealloc_ucontext(struct ib_ucontext
*context
)
183 struct i40iw_ucontext
*ucontext
= to_ucontext(context
);
186 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
187 if (!list_empty(&ucontext
->cq_reg_mem_list
)) {
188 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
191 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
192 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
193 if (!list_empty(&ucontext
->qp_reg_mem_list
)) {
194 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
197 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
204 * i40iw_mmap - user memory map
205 * @context: context created during alloc
206 * @vma: kernel info for user memory map
208 static int i40iw_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
210 struct i40iw_ucontext
*ucontext
;
214 ucontext
= to_ucontext(context
);
215 if (ucontext
->iwdev
->sc_dev
.is_pf
) {
216 db_addr_offset
= I40IW_DB_ADDR_OFFSET
;
217 push_offset
= I40IW_PUSH_OFFSET
;
219 vma
->vm_pgoff
+= I40IW_PF_FIRST_PUSH_PAGE_INDEX
- 1;
221 db_addr_offset
= I40IW_VF_DB_ADDR_OFFSET
;
222 push_offset
= I40IW_VF_PUSH_OFFSET
;
224 vma
->vm_pgoff
+= I40IW_VF_FIRST_PUSH_PAGE_INDEX
- 1;
227 vma
->vm_pgoff
+= db_addr_offset
>> PAGE_SHIFT
;
229 if (vma
->vm_pgoff
== (db_addr_offset
>> PAGE_SHIFT
)) {
230 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
231 vma
->vm_private_data
= ucontext
;
233 if ((vma
->vm_pgoff
- (push_offset
>> PAGE_SHIFT
)) % 2)
234 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
236 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
239 if (io_remap_pfn_range(vma
, vma
->vm_start
,
240 vma
->vm_pgoff
+ (pci_resource_start(ucontext
->iwdev
->ldev
->pcidev
, 0) >> PAGE_SHIFT
),
241 PAGE_SIZE
, vma
->vm_page_prot
))
248 * i40iw_alloc_push_page - allocate a push page for qp
249 * @iwdev: iwarp device
250 * @qp: hardware control qp
252 static void i40iw_alloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
254 struct i40iw_cqp_request
*cqp_request
;
255 struct cqp_commands_info
*cqp_info
;
256 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
257 enum i40iw_status_code status
;
259 if (qp
->push_idx
!= I40IW_INVALID_PUSH_PAGE_INDEX
)
262 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
266 atomic_inc(&cqp_request
->refcount
);
268 cqp_info
= &cqp_request
->info
;
269 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
270 cqp_info
->post_sq
= 1;
272 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= dev
->qs_handle
;
273 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 0;
274 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
275 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
277 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
279 qp
->push_idx
= cqp_request
->compl_info
.op_ret_val
;
281 i40iw_pr_err("CQP-OP Push page fail");
282 i40iw_put_cqp_request(&iwdev
->cqp
, cqp_request
);
286 * i40iw_dealloc_push_page - free a push page for qp
287 * @iwdev: iwarp device
288 * @qp: hardware control qp
290 static void i40iw_dealloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
292 struct i40iw_cqp_request
*cqp_request
;
293 struct cqp_commands_info
*cqp_info
;
294 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
295 enum i40iw_status_code status
;
297 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
)
300 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, false);
304 cqp_info
= &cqp_request
->info
;
305 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
306 cqp_info
->post_sq
= 1;
308 cqp_info
->in
.u
.manage_push_page
.info
.push_idx
= qp
->push_idx
;
309 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= dev
->qs_handle
;
310 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 1;
311 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
312 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
314 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
316 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
318 i40iw_pr_err("CQP-OP Push page fail");
322 * i40iw_alloc_pd - allocate protection domain
323 * @ibdev: device pointer from stack
324 * @context: user context created during alloc
327 static struct ib_pd
*i40iw_alloc_pd(struct ib_device
*ibdev
,
328 struct ib_ucontext
*context
,
329 struct ib_udata
*udata
)
331 struct i40iw_pd
*iwpd
;
332 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
333 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
334 struct i40iw_alloc_pd_resp uresp
;
335 struct i40iw_sc_pd
*sc_pd
;
339 err
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_pds
,
340 iwdev
->max_pd
, &pd_id
, &iwdev
->next_pd
);
342 i40iw_pr_err("alloc resource failed\n");
346 iwpd
= kzalloc(sizeof(*iwpd
), GFP_KERNEL
);
352 sc_pd
= &iwpd
->sc_pd
;
353 dev
->iw_pd_ops
->pd_init(dev
, sc_pd
, pd_id
);
356 memset(&uresp
, 0, sizeof(uresp
));
358 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
364 i40iw_add_pdusecount(iwpd
);
369 i40iw_free_resource(iwdev
, iwdev
->allocated_pds
, pd_id
);
374 * i40iw_dealloc_pd - deallocate pd
375 * @ibpd: ptr of pd to be deallocated
377 static int i40iw_dealloc_pd(struct ib_pd
*ibpd
)
379 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
380 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
382 i40iw_rem_pdusecount(iwpd
, iwdev
);
387 * i40iw_qp_roundup - return round up qp ring size
388 * @wr_ring_size: ring size to round up
390 static int i40iw_qp_roundup(u32 wr_ring_size
)
394 if (wr_ring_size
< I40IWQP_SW_MIN_WQSIZE
)
395 wr_ring_size
= I40IWQP_SW_MIN_WQSIZE
;
397 for (wr_ring_size
--; scount
<= 16; scount
*= 2)
398 wr_ring_size
|= wr_ring_size
>> scount
;
399 return ++wr_ring_size
;
403 * i40iw_get_pbl - Retrieve pbl from a list given a virtual
405 * @va: user virtual address
406 * @pbl_list: pbl list to search in (QP's or CQ's)
408 static struct i40iw_pbl
*i40iw_get_pbl(unsigned long va
,
409 struct list_head
*pbl_list
)
411 struct i40iw_pbl
*iwpbl
;
413 list_for_each_entry(iwpbl
, pbl_list
, list
) {
414 if (iwpbl
->user_base
== va
) {
415 list_del(&iwpbl
->list
);
423 * i40iw_free_qp_resources - free up memory resources for qp
424 * @iwdev: iwarp device
425 * @iwqp: qp ptr (user or kernel)
426 * @qp_num: qp number assigned
428 void i40iw_free_qp_resources(struct i40iw_device
*iwdev
,
429 struct i40iw_qp
*iwqp
,
432 i40iw_dealloc_push_page(iwdev
, &iwqp
->sc_qp
);
434 i40iw_free_resource(iwdev
, iwdev
->allocated_qps
, qp_num
);
435 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->q2_ctx_mem
);
436 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->kqp
.dma_mem
);
437 kfree(iwqp
->kqp
.wrid_mem
);
438 iwqp
->kqp
.wrid_mem
= NULL
;
439 kfree(iwqp
->allocated_buffer
);
440 iwqp
->allocated_buffer
= NULL
;
444 * i40iw_clean_cqes - clean cq entries for qp
445 * @iwqp: qp ptr (user or kernel)
448 static void i40iw_clean_cqes(struct i40iw_qp
*iwqp
, struct i40iw_cq
*iwcq
)
450 struct i40iw_cq_uk
*ukcq
= &iwcq
->sc_cq
.cq_uk
;
452 ukcq
->ops
.iw_cq_clean(&iwqp
->sc_qp
.qp_uk
, ukcq
);
456 * i40iw_destroy_qp - destroy qp
457 * @ibqp: qp's ib pointer also to get to device's qp address
459 static int i40iw_destroy_qp(struct ib_qp
*ibqp
)
461 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
465 if (iwqp
->ibqp_state
>= IB_QPS_INIT
&& iwqp
->ibqp_state
< IB_QPS_RTS
)
466 i40iw_next_iw_state(iwqp
, I40IW_QP_STATE_ERROR
, 0, 0, 0);
468 if (!iwqp
->user_mode
) {
470 i40iw_clean_cqes(iwqp
, iwqp
->iwscq
);
471 if (iwqp
->iwrcq
!= iwqp
->iwscq
)
472 i40iw_clean_cqes(iwqp
, iwqp
->iwrcq
);
476 i40iw_rem_ref(&iwqp
->ibqp
);
481 * i40iw_setup_virt_qp - setup for allocation of virtual qp
484 * @init_info: initialize info to return
486 static int i40iw_setup_virt_qp(struct i40iw_device
*iwdev
,
487 struct i40iw_qp
*iwqp
,
488 struct i40iw_qp_init_info
*init_info
)
490 struct i40iw_pbl
*iwpbl
= iwqp
->iwpbl
;
491 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
493 iwqp
->page
= qpmr
->sq_page
;
494 init_info
->shadow_area_pa
= cpu_to_le64(qpmr
->shadow
);
495 if (iwpbl
->pbl_allocated
) {
496 init_info
->virtual_map
= true;
497 init_info
->sq_pa
= qpmr
->sq_pbl
.idx
;
498 init_info
->rq_pa
= qpmr
->rq_pbl
.idx
;
500 init_info
->sq_pa
= qpmr
->sq_pbl
.addr
;
501 init_info
->rq_pa
= qpmr
->rq_pbl
.addr
;
507 * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
508 * @iwdev: iwarp device
509 * @iwqp: qp ptr (user or kernel)
510 * @info: initialize info to return
512 static int i40iw_setup_kmode_qp(struct i40iw_device
*iwdev
,
513 struct i40iw_qp
*iwqp
,
514 struct i40iw_qp_init_info
*info
)
516 struct i40iw_dma_mem
*mem
= &iwqp
->kqp
.dma_mem
;
517 u32 sqdepth
, rqdepth
;
518 u32 sq_size
, rq_size
;
521 enum i40iw_status_code status
;
522 struct i40iw_qp_uk_init_info
*ukinfo
= &info
->qp_uk_init_info
;
524 ukinfo
->max_sq_frag_cnt
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
526 sq_size
= i40iw_qp_roundup(ukinfo
->sq_size
+ 1);
527 rq_size
= i40iw_qp_roundup(ukinfo
->rq_size
+ 1);
529 status
= i40iw_get_wqe_shift(sq_size
, ukinfo
->max_sq_frag_cnt
, ukinfo
->max_inline_data
, &sqshift
);
531 status
= i40iw_get_wqe_shift(rq_size
, ukinfo
->max_rq_frag_cnt
, 0, &rqshift
);
536 sqdepth
= sq_size
<< sqshift
;
537 rqdepth
= rq_size
<< rqshift
;
539 size
= sqdepth
* sizeof(struct i40iw_sq_uk_wr_trk_info
) + (rqdepth
<< 3);
540 iwqp
->kqp
.wrid_mem
= kzalloc(size
, GFP_KERNEL
);
542 ukinfo
->sq_wrtrk_array
= (struct i40iw_sq_uk_wr_trk_info
*)iwqp
->kqp
.wrid_mem
;
543 if (!ukinfo
->sq_wrtrk_array
)
546 ukinfo
->rq_wrid_array
= (u64
*)&ukinfo
->sq_wrtrk_array
[sqdepth
];
548 size
= (sqdepth
+ rqdepth
) * I40IW_QP_WQE_MIN_SIZE
;
549 size
+= (I40IW_SHADOW_AREA_SIZE
<< 3);
551 status
= i40iw_allocate_dma_mem(iwdev
->sc_dev
.hw
, mem
, size
, 256);
553 kfree(ukinfo
->sq_wrtrk_array
);
554 ukinfo
->sq_wrtrk_array
= NULL
;
558 ukinfo
->sq
= mem
->va
;
559 info
->sq_pa
= mem
->pa
;
561 ukinfo
->rq
= &ukinfo
->sq
[sqdepth
];
562 info
->rq_pa
= info
->sq_pa
+ (sqdepth
* I40IW_QP_WQE_MIN_SIZE
);
564 ukinfo
->shadow_area
= ukinfo
->rq
[rqdepth
].elem
;
565 info
->shadow_area_pa
= info
->rq_pa
+ (rqdepth
* I40IW_QP_WQE_MIN_SIZE
);
567 ukinfo
->sq_size
= sq_size
;
568 ukinfo
->rq_size
= rq_size
;
569 ukinfo
->qp_id
= iwqp
->ibqp
.qp_num
;
574 * i40iw_create_qp - create qp
576 * @init_attr: attributes for qp
577 * @udata: user data for create qp
579 static struct ib_qp
*i40iw_create_qp(struct ib_pd
*ibpd
,
580 struct ib_qp_init_attr
*init_attr
,
581 struct ib_udata
*udata
)
583 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
584 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
585 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
586 struct i40iw_qp
*iwqp
;
587 struct i40iw_ucontext
*ucontext
;
588 struct i40iw_create_qp_req req
;
589 struct i40iw_create_qp_resp uresp
;
592 enum i40iw_status_code ret
;
596 struct i40iw_sc_qp
*qp
;
597 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
598 struct i40iw_qp_init_info init_info
;
599 struct i40iw_create_qp_info
*qp_info
;
600 struct i40iw_cqp_request
*cqp_request
;
601 struct cqp_commands_info
*cqp_info
;
603 struct i40iw_qp_host_ctx_info
*ctx_info
;
604 struct i40iwarp_offload_info
*iwarp_info
;
607 if (init_attr
->create_flags
)
608 return ERR_PTR(-EINVAL
);
609 if (init_attr
->cap
.max_inline_data
> I40IW_MAX_INLINE_DATA_SIZE
)
610 init_attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
612 if (init_attr
->cap
.max_send_sge
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
613 init_attr
->cap
.max_send_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
615 memset(&init_info
, 0, sizeof(init_info
));
617 sq_size
= init_attr
->cap
.max_send_wr
;
618 rq_size
= init_attr
->cap
.max_recv_wr
;
620 init_info
.qp_uk_init_info
.sq_size
= sq_size
;
621 init_info
.qp_uk_init_info
.rq_size
= rq_size
;
622 init_info
.qp_uk_init_info
.max_sq_frag_cnt
= init_attr
->cap
.max_send_sge
;
623 init_info
.qp_uk_init_info
.max_rq_frag_cnt
= init_attr
->cap
.max_recv_sge
;
624 init_info
.qp_uk_init_info
.max_inline_data
= init_attr
->cap
.max_inline_data
;
626 mem
= kzalloc(sizeof(*iwqp
), GFP_KERNEL
);
628 return ERR_PTR(-ENOMEM
);
630 iwqp
= (struct i40iw_qp
*)mem
;
632 qp
->back_qp
= (void *)iwqp
;
633 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
635 iwqp
->ctx_info
.iwarp_info
= &iwqp
->iwarp_info
;
637 if (i40iw_allocate_dma_mem(dev
->hw
,
639 I40IW_Q2_BUFFER_SIZE
+ I40IW_QP_CTX_SIZE
,
641 i40iw_pr_err("dma_mem failed\n");
646 init_info
.q2
= iwqp
->q2_ctx_mem
.va
;
647 init_info
.q2_pa
= iwqp
->q2_ctx_mem
.pa
;
649 init_info
.host_ctx
= (void *)init_info
.q2
+ I40IW_Q2_BUFFER_SIZE
;
650 init_info
.host_ctx_pa
= init_info
.q2_pa
+ I40IW_Q2_BUFFER_SIZE
;
652 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_qps
, iwdev
->max_qp
,
653 &qp_num
, &iwdev
->next_qp
);
655 i40iw_pr_err("qp resource\n");
659 iwqp
->allocated_buffer
= mem
;
662 iwqp
->ibqp
.qp_num
= qp_num
;
664 iwqp
->iwscq
= to_iwcq(init_attr
->send_cq
);
665 iwqp
->iwrcq
= to_iwcq(init_attr
->recv_cq
);
667 iwqp
->host_ctx
.va
= init_info
.host_ctx
;
668 iwqp
->host_ctx
.pa
= init_info
.host_ctx_pa
;
669 iwqp
->host_ctx
.size
= I40IW_QP_CTX_SIZE
;
671 init_info
.pd
= &iwpd
->sc_pd
;
672 init_info
.qp_uk_init_info
.qp_id
= iwqp
->ibqp
.qp_num
;
673 iwqp
->ctx_info
.qp_compl_ctx
= (uintptr_t)qp
;
675 if (init_attr
->qp_type
!= IB_QPT_RC
) {
679 if (iwdev
->push_mode
)
680 i40iw_alloc_push_page(iwdev
, qp
);
682 err_code
= ib_copy_from_udata(&req
, udata
, sizeof(req
));
684 i40iw_pr_err("ib_copy_from_data\n");
687 iwqp
->ctx_info
.qp_compl_ctx
= req
.user_compl_ctx
;
688 if (ibpd
->uobject
&& ibpd
->uobject
->context
) {
690 ucontext
= to_ucontext(ibpd
->uobject
->context
);
692 if (req
.user_wqe_buffers
) {
694 &ucontext
->qp_reg_mem_list_lock
, flags
);
695 iwqp
->iwpbl
= i40iw_get_pbl(
696 (unsigned long)req
.user_wqe_buffers
,
697 &ucontext
->qp_reg_mem_list
);
698 spin_unlock_irqrestore(
699 &ucontext
->qp_reg_mem_list_lock
, flags
);
703 i40iw_pr_err("no pbl info\n");
708 err_code
= i40iw_setup_virt_qp(iwdev
, iwqp
, &init_info
);
710 err_code
= i40iw_setup_kmode_qp(iwdev
, iwqp
, &init_info
);
714 i40iw_pr_err("setup qp failed\n");
718 init_info
.type
= I40IW_QP_TYPE_IWARP
;
719 ret
= dev
->iw_priv_qp_ops
->qp_init(qp
, &init_info
);
722 i40iw_pr_err("qp_init fail\n");
725 ctx_info
= &iwqp
->ctx_info
;
726 iwarp_info
= &iwqp
->iwarp_info
;
727 iwarp_info
->rd_enable
= true;
728 iwarp_info
->wr_rdresp_en
= true;
729 if (!iwqp
->user_mode
)
730 iwarp_info
->priv_mode_en
= true;
731 iwarp_info
->ddp_ver
= 1;
732 iwarp_info
->rdmap_ver
= 1;
734 ctx_info
->iwarp_info_valid
= true;
735 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
736 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
737 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
) {
738 ctx_info
->push_mode_en
= false;
740 ctx_info
->push_mode_en
= true;
741 ctx_info
->push_idx
= qp
->push_idx
;
744 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
745 (u64
*)iwqp
->host_ctx
.va
,
747 ctx_info
->iwarp_info_valid
= false;
748 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
753 cqp_info
= &cqp_request
->info
;
754 qp_info
= &cqp_request
->info
.in
.u
.qp_create
.info
;
756 memset(qp_info
, 0, sizeof(*qp_info
));
758 qp_info
->cq_num_valid
= true;
759 qp_info
->next_iwarp_state
= I40IW_QP_STATE_IDLE
;
761 cqp_info
->cqp_cmd
= OP_QP_CREATE
;
762 cqp_info
->post_sq
= 1;
763 cqp_info
->in
.u
.qp_create
.qp
= qp
;
764 cqp_info
->in
.u
.qp_create
.scratch
= (uintptr_t)cqp_request
;
765 ret
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
767 i40iw_pr_err("CQP-OP QP create fail");
772 i40iw_add_ref(&iwqp
->ibqp
);
773 spin_lock_init(&iwqp
->lock
);
774 iwqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
775 iwdev
->qp_table
[qp_num
] = iwqp
;
776 i40iw_add_pdusecount(iwqp
->iwpd
);
777 if (ibpd
->uobject
&& udata
) {
778 memset(&uresp
, 0, sizeof(uresp
));
779 uresp
.actual_sq_size
= sq_size
;
780 uresp
.actual_rq_size
= rq_size
;
781 uresp
.qp_id
= qp_num
;
782 uresp
.push_idx
= qp
->push_idx
;
783 err_code
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
785 i40iw_pr_err("copy_to_udata failed\n");
786 i40iw_destroy_qp(&iwqp
->ibqp
);
787 /* let the completion of the qp destroy free the qp */
788 return ERR_PTR(err_code
);
794 i40iw_free_qp_resources(iwdev
, iwqp
, qp_num
);
796 return ERR_PTR(err_code
);
800 * i40iw_query - query qp attributes
802 * @attr: attributes pointer
803 * @attr_mask: Not used
804 * @init_attr: qp attributes to return
806 static int i40iw_query_qp(struct ib_qp
*ibqp
,
807 struct ib_qp_attr
*attr
,
809 struct ib_qp_init_attr
*init_attr
)
811 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
812 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
814 attr
->qp_access_flags
= 0;
815 attr
->cap
.max_send_wr
= qp
->qp_uk
.sq_size
;
816 attr
->cap
.max_recv_wr
= qp
->qp_uk
.rq_size
;
817 attr
->cap
.max_recv_sge
= 1;
818 attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
819 init_attr
->event_handler
= iwqp
->ibqp
.event_handler
;
820 init_attr
->qp_context
= iwqp
->ibqp
.qp_context
;
821 init_attr
->send_cq
= iwqp
->ibqp
.send_cq
;
822 init_attr
->recv_cq
= iwqp
->ibqp
.recv_cq
;
823 init_attr
->srq
= iwqp
->ibqp
.srq
;
824 init_attr
->cap
= attr
->cap
;
829 * i40iw_hw_modify_qp - setup cqp for modify qp
830 * @iwdev: iwarp device
831 * @iwqp: qp ptr (user or kernel)
832 * @info: info for modify qp
833 * @wait: flag to wait or not for modify qp completion
835 void i40iw_hw_modify_qp(struct i40iw_device
*iwdev
, struct i40iw_qp
*iwqp
,
836 struct i40iw_modify_qp_info
*info
, bool wait
)
838 enum i40iw_status_code status
;
839 struct i40iw_cqp_request
*cqp_request
;
840 struct cqp_commands_info
*cqp_info
;
841 struct i40iw_modify_qp_info
*m_info
;
843 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, wait
);
847 cqp_info
= &cqp_request
->info
;
848 m_info
= &cqp_info
->in
.u
.qp_modify
.info
;
849 memcpy(m_info
, info
, sizeof(*m_info
));
850 cqp_info
->cqp_cmd
= OP_QP_MODIFY
;
851 cqp_info
->post_sq
= 1;
852 cqp_info
->in
.u
.qp_modify
.qp
= &iwqp
->sc_qp
;
853 cqp_info
->in
.u
.qp_modify
.scratch
= (uintptr_t)cqp_request
;
854 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
856 i40iw_pr_err("CQP-OP Modify QP fail");
860 * i40iw_modify_qp - modify qp request
861 * @ibqp: qp's pointer for modify
862 * @attr: access attributes
863 * @attr_mask: state mask
866 int i40iw_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
867 int attr_mask
, struct ib_udata
*udata
)
869 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
870 struct i40iw_device
*iwdev
= iwqp
->iwdev
;
871 struct i40iw_qp_host_ctx_info
*ctx_info
;
872 struct i40iwarp_offload_info
*iwarp_info
;
873 struct i40iw_modify_qp_info info
;
874 u8 issue_modify_qp
= 0;
879 memset(&info
, 0, sizeof(info
));
880 ctx_info
= &iwqp
->ctx_info
;
881 iwarp_info
= &iwqp
->iwarp_info
;
883 spin_lock_irqsave(&iwqp
->lock
, flags
);
885 if (attr_mask
& IB_QP_STATE
) {
886 switch (attr
->qp_state
) {
889 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_IDLE
) {
893 if (iwqp
->iwarp_state
== I40IW_QP_STATE_INVALID
) {
894 info
.next_iwarp_state
= I40IW_QP_STATE_IDLE
;
899 if ((iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) ||
906 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_ESTABLISHED
;
908 info
.next_iwarp_state
= I40IW_QP_STATE_RTS
;
909 info
.tcp_ctx_valid
= true;
910 info
.ord_valid
= true;
911 info
.arp_cache_idx_valid
= true;
912 info
.cq_num_valid
= true;
915 if (iwqp
->hw_iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) {
919 if ((iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_CLOSING
) ||
920 (iwqp
->iwarp_state
< (u32
)I40IW_QP_STATE_RTS
)) {
924 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_CLOSING
) {
928 info
.next_iwarp_state
= I40IW_QP_STATE_CLOSING
;
932 if (iwqp
->iwarp_state
>= (u32
)I40IW_QP_STATE_TERMINATE
) {
936 info
.next_iwarp_state
= I40IW_QP_STATE_TERMINATE
;
941 if (iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_ERROR
) {
945 if (iwqp
->sc_qp
.term_flags
)
946 del_timer(&iwqp
->terminate_timer
);
947 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
948 if ((iwqp
->hw_tcp_state
> I40IW_TCP_STATE_CLOSED
) &&
950 (iwqp
->hw_tcp_state
!= I40IW_TCP_STATE_TIME_WAIT
))
951 info
.reset_tcp_conn
= true;
955 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
962 iwqp
->ibqp_state
= attr
->qp_state
;
965 iwqp
->iwarp_state
= info
.next_iwarp_state
;
967 info
.next_iwarp_state
= iwqp
->iwarp_state
;
969 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
970 ctx_info
->iwarp_info_valid
= true;
971 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
)
972 iwarp_info
->wr_rdresp_en
= true;
973 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
974 iwarp_info
->wr_rdresp_en
= true;
975 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
)
976 iwarp_info
->rd_enable
= true;
977 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
)
978 iwarp_info
->bind_en
= true;
980 if (iwqp
->user_mode
) {
981 iwarp_info
->rd_enable
= true;
982 iwarp_info
->wr_rdresp_en
= true;
983 iwarp_info
->priv_mode_en
= false;
987 if (ctx_info
->iwarp_info_valid
) {
988 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
991 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
992 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
993 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
994 (u64
*)iwqp
->host_ctx
.va
,
997 i40iw_pr_err("setting QP context\n");
1003 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1005 if (issue_modify_qp
)
1006 i40iw_hw_modify_qp(iwdev
, iwqp
, &info
, true);
1008 if (issue_modify_qp
&& (iwqp
->ibqp_state
> IB_QPS_RTS
)) {
1010 if (iwqp
->cm_id
&& iwqp
->hw_tcp_state
) {
1011 spin_lock_irqsave(&iwqp
->lock
, flags
);
1012 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_CLOSED
;
1013 iwqp
->last_aeq
= I40IW_AE_RESET_SENT
;
1014 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1020 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1025 * cq_free_resources - free up recources for cq
1026 * @iwdev: iwarp device
1029 static void cq_free_resources(struct i40iw_device
*iwdev
, struct i40iw_cq
*iwcq
)
1031 struct i40iw_sc_cq
*cq
= &iwcq
->sc_cq
;
1033 if (!iwcq
->user_mode
)
1034 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwcq
->kmem
);
1035 i40iw_free_resource(iwdev
, iwdev
->allocated_cqs
, cq
->cq_uk
.cq_id
);
1039 * cq_wq_destroy - send cq destroy cqp
1040 * @iwdev: iwarp device
1041 * @cq: hardware control cq
1043 static void cq_wq_destroy(struct i40iw_device
*iwdev
, struct i40iw_sc_cq
*cq
)
1045 enum i40iw_status_code status
;
1046 struct i40iw_cqp_request
*cqp_request
;
1047 struct cqp_commands_info
*cqp_info
;
1049 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1053 cqp_info
= &cqp_request
->info
;
1055 cqp_info
->cqp_cmd
= OP_CQ_DESTROY
;
1056 cqp_info
->post_sq
= 1;
1057 cqp_info
->in
.u
.cq_destroy
.cq
= cq
;
1058 cqp_info
->in
.u
.cq_destroy
.scratch
= (uintptr_t)cqp_request
;
1059 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1061 i40iw_pr_err("CQP-OP Destroy QP fail");
1065 * i40iw_destroy_cq - destroy cq
1066 * @ib_cq: cq pointer
1068 static int i40iw_destroy_cq(struct ib_cq
*ib_cq
)
1070 struct i40iw_cq
*iwcq
;
1071 struct i40iw_device
*iwdev
;
1072 struct i40iw_sc_cq
*cq
;
1075 i40iw_pr_err("ib_cq == NULL\n");
1079 iwcq
= to_iwcq(ib_cq
);
1080 iwdev
= to_iwdev(ib_cq
->device
);
1082 cq_wq_destroy(iwdev
, cq
);
1083 cq_free_resources(iwdev
, iwcq
);
1089 * i40iw_create_cq - create cq
1090 * @ibdev: device pointer from stack
1091 * @attr: attributes for cq
1092 * @context: user context created during alloc
1095 static struct ib_cq
*i40iw_create_cq(struct ib_device
*ibdev
,
1096 const struct ib_cq_init_attr
*attr
,
1097 struct ib_ucontext
*context
,
1098 struct ib_udata
*udata
)
1100 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
1101 struct i40iw_cq
*iwcq
;
1102 struct i40iw_pbl
*iwpbl
;
1104 struct i40iw_sc_cq
*cq
;
1105 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1106 struct i40iw_cq_init_info info
;
1107 enum i40iw_status_code status
;
1108 struct i40iw_cqp_request
*cqp_request
;
1109 struct cqp_commands_info
*cqp_info
;
1110 struct i40iw_cq_uk_init_info
*ukinfo
= &info
.cq_uk_init_info
;
1111 unsigned long flags
;
1113 int entries
= attr
->cqe
;
1115 if (entries
> iwdev
->max_cqe
)
1116 return ERR_PTR(-EINVAL
);
1118 iwcq
= kzalloc(sizeof(*iwcq
), GFP_KERNEL
);
1120 return ERR_PTR(-ENOMEM
);
1122 memset(&info
, 0, sizeof(info
));
1124 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_cqs
,
1125 iwdev
->max_cq
, &cq_num
,
1131 cq
->back_cq
= (void *)iwcq
;
1132 spin_lock_init(&iwcq
->lock
);
1135 ukinfo
->cq_size
= max(entries
, 4);
1136 ukinfo
->cq_id
= cq_num
;
1137 iwcq
->ibcq
.cqe
= info
.cq_uk_init_info
.cq_size
;
1140 info
.ceq_id_valid
= true;
1142 info
.type
= I40IW_CQ_TYPE_IWARP
;
1144 struct i40iw_ucontext
*ucontext
;
1145 struct i40iw_create_cq_req req
;
1146 struct i40iw_cq_mr
*cqmr
;
1148 memset(&req
, 0, sizeof(req
));
1149 iwcq
->user_mode
= true;
1150 ucontext
= to_ucontext(context
);
1151 if (ib_copy_from_udata(&req
, udata
, sizeof(struct i40iw_create_cq_req
)))
1152 goto cq_free_resources
;
1154 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1155 iwpbl
= i40iw_get_pbl((unsigned long)req
.user_cq_buffer
,
1156 &ucontext
->cq_reg_mem_list
);
1157 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1160 goto cq_free_resources
;
1163 iwcq
->iwpbl
= iwpbl
;
1164 iwcq
->cq_mem_size
= 0;
1165 cqmr
= &iwpbl
->cq_mr
;
1166 info
.shadow_area_pa
= cpu_to_le64(cqmr
->shadow
);
1167 if (iwpbl
->pbl_allocated
) {
1168 info
.virtual_map
= true;
1169 info
.pbl_chunk_size
= 1;
1170 info
.first_pm_pbl_idx
= cqmr
->cq_pbl
.idx
;
1172 info
.cq_base_pa
= cqmr
->cq_pbl
.addr
;
1175 /* Kmode allocations */
1179 rsize
= info
.cq_uk_init_info
.cq_size
* sizeof(struct i40iw_cqe
);
1180 rsize
= round_up(rsize
, 256);
1181 shadow
= I40IW_SHADOW_AREA_SIZE
<< 3;
1182 status
= i40iw_allocate_dma_mem(dev
->hw
, &iwcq
->kmem
,
1183 rsize
+ shadow
, 256);
1186 goto cq_free_resources
;
1188 ukinfo
->cq_base
= iwcq
->kmem
.va
;
1189 info
.cq_base_pa
= iwcq
->kmem
.pa
;
1190 info
.shadow_area_pa
= info
.cq_base_pa
+ rsize
;
1191 ukinfo
->shadow_area
= iwcq
->kmem
.va
+ rsize
;
1194 if (dev
->iw_priv_cq_ops
->cq_init(cq
, &info
)) {
1195 i40iw_pr_err("init cq fail\n");
1197 goto cq_free_resources
;
1200 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1203 goto cq_free_resources
;
1206 cqp_info
= &cqp_request
->info
;
1207 cqp_info
->cqp_cmd
= OP_CQ_CREATE
;
1208 cqp_info
->post_sq
= 1;
1209 cqp_info
->in
.u
.cq_create
.cq
= cq
;
1210 cqp_info
->in
.u
.cq_create
.scratch
= (uintptr_t)cqp_request
;
1211 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1213 i40iw_pr_err("CQP-OP Create QP fail");
1215 goto cq_free_resources
;
1219 struct i40iw_create_cq_resp resp
;
1221 memset(&resp
, 0, sizeof(resp
));
1222 resp
.cq_id
= info
.cq_uk_init_info
.cq_id
;
1223 resp
.cq_size
= info
.cq_uk_init_info
.cq_size
;
1224 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
1225 i40iw_pr_err("copy to user data\n");
1231 return (struct ib_cq
*)iwcq
;
1234 cq_wq_destroy(iwdev
, cq
);
1236 cq_free_resources(iwdev
, iwcq
);
1239 return ERR_PTR(err_code
);
1243 * i40iw_get_user_access - get hw access from IB access
1244 * @acc: IB access to return hw access
1246 static inline u16
i40iw_get_user_access(int acc
)
1250 access
|= (acc
& IB_ACCESS_LOCAL_WRITE
) ? I40IW_ACCESS_FLAGS_LOCALWRITE
: 0;
1251 access
|= (acc
& IB_ACCESS_REMOTE_WRITE
) ? I40IW_ACCESS_FLAGS_REMOTEWRITE
: 0;
1252 access
|= (acc
& IB_ACCESS_REMOTE_READ
) ? I40IW_ACCESS_FLAGS_REMOTEREAD
: 0;
1253 access
|= (acc
& IB_ACCESS_MW_BIND
) ? I40IW_ACCESS_FLAGS_BIND_WINDOW
: 0;
1258 * i40iw_free_stag - free stag resource
1259 * @iwdev: iwarp device
1260 * @stag: stag to free
1262 static void i40iw_free_stag(struct i40iw_device
*iwdev
, u32 stag
)
1266 stag_idx
= (stag
& iwdev
->mr_stagmask
) >> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1267 i40iw_free_resource(iwdev
, iwdev
->allocated_mrs
, stag_idx
);
1271 * i40iw_create_stag - create random stag
1272 * @iwdev: iwarp device
1274 static u32
i40iw_create_stag(struct i40iw_device
*iwdev
)
1278 u32 next_stag_index
;
1284 get_random_bytes(&random
, sizeof(random
));
1285 consumer_key
= (u8
)random
;
1287 driver_key
= random
& ~iwdev
->mr_stagmask
;
1288 next_stag_index
= (random
& iwdev
->mr_stagmask
) >> 8;
1289 next_stag_index
%= iwdev
->max_mr
;
1291 ret
= i40iw_alloc_resource(iwdev
,
1292 iwdev
->allocated_mrs
, iwdev
->max_mr
,
1293 &stag_index
, &next_stag_index
);
1295 stag
= stag_index
<< I40IW_CQPSQ_STAG_IDX_SHIFT
;
1297 stag
+= (u32
)consumer_key
;
1303 * i40iw_next_pbl_addr - Get next pbl address
1304 * @palloc: Poiner to allocated pbles
1305 * @pbl: pointer to a pble
1306 * @pinfo: info pointer
1309 static inline u64
*i40iw_next_pbl_addr(struct i40iw_pble_alloc
*palloc
,
1311 struct i40iw_pble_info
**pinfo
,
1315 if ((!(*pinfo
)) || (*idx
!= (*pinfo
)->cnt
))
1319 return (u64
*)(*pinfo
)->addr
;
1323 * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally
1324 * @iwmr: iwmr for IB's user page addresses
1325 * @pbl: ple pointer to save 1 level or 0 level pble
1326 * @level: indicated level 0, 1 or 2
1328 static void i40iw_copy_user_pgaddrs(struct i40iw_mr
*iwmr
,
1330 enum i40iw_pble_level level
)
1332 struct ib_umem
*region
= iwmr
->region
;
1333 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1334 int chunk_pages
, entry
, pg_shift
, i
;
1335 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1336 struct i40iw_pble_info
*pinfo
;
1337 struct scatterlist
*sg
;
1340 pinfo
= (level
== I40IW_LEVEL_1
) ? NULL
: palloc
->level2
.leaf
;
1341 pg_shift
= ffs(region
->page_size
) - 1;
1342 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
1343 chunk_pages
= sg_dma_len(sg
) >> pg_shift
;
1344 if ((iwmr
->type
== IW_MEMREG_TYPE_QP
) &&
1345 !iwpbl
->qp_mr
.sq_page
)
1346 iwpbl
->qp_mr
.sq_page
= sg_page(sg
);
1347 for (i
= 0; i
< chunk_pages
; i
++) {
1348 *pbl
= cpu_to_le64(sg_dma_address(sg
) + region
->page_size
* i
);
1349 pbl
= i40iw_next_pbl_addr(palloc
, pbl
, &pinfo
, &idx
);
1355 * i40iw_setup_pbles - copy user pg address to pble's
1356 * @iwdev: iwarp device
1357 * @iwmr: mr pointer for this memory registration
1358 * @use_pbles: flag if to use pble's or memory (level 0)
1360 static int i40iw_setup_pbles(struct i40iw_device
*iwdev
,
1361 struct i40iw_mr
*iwmr
,
1364 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1365 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1366 struct i40iw_pble_info
*pinfo
;
1368 enum i40iw_status_code status
;
1369 enum i40iw_pble_level level
= I40IW_LEVEL_1
;
1371 if (!use_pbles
&& (iwmr
->page_cnt
> MAX_SAVE_PAGE_ADDRS
))
1375 mutex_lock(&iwdev
->pbl_mutex
);
1376 status
= i40iw_get_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
, palloc
, iwmr
->page_cnt
);
1377 mutex_unlock(&iwdev
->pbl_mutex
);
1381 iwpbl
->pbl_allocated
= true;
1382 level
= palloc
->level
;
1383 pinfo
= (level
== I40IW_LEVEL_1
) ? &palloc
->level1
: palloc
->level2
.leaf
;
1384 pbl
= (u64
*)pinfo
->addr
;
1386 pbl
= iwmr
->pgaddrmem
;
1389 i40iw_copy_user_pgaddrs(iwmr
, pbl
, level
);
1394 * i40iw_handle_q_mem - handle memory for qp and cq
1395 * @iwdev: iwarp device
1396 * @req: information for q memory management
1397 * @iwpbl: pble struct
1398 * @use_pbles: flag to use pble
1400 static int i40iw_handle_q_mem(struct i40iw_device
*iwdev
,
1401 struct i40iw_mem_reg_req
*req
,
1402 struct i40iw_pbl
*iwpbl
,
1405 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1406 struct i40iw_mr
*iwmr
= iwpbl
->iwmr
;
1407 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
1408 struct i40iw_cq_mr
*cqmr
= &iwpbl
->cq_mr
;
1409 struct i40iw_hmc_pble
*hmc_p
;
1410 u64
*arr
= iwmr
->pgaddrmem
;
1414 total
= req
->sq_pages
+ req
->rq_pages
+ req
->cq_pages
;
1416 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1419 if (use_pbles
&& (palloc
->level
!= I40IW_LEVEL_1
)) {
1420 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1421 iwpbl
->pbl_allocated
= false;
1426 arr
= (u64
*)palloc
->level1
.addr
;
1427 if (req
->reg_type
== IW_MEMREG_TYPE_QP
) {
1428 hmc_p
= &qpmr
->sq_pbl
;
1429 qpmr
->shadow
= (dma_addr_t
)arr
[total
];
1431 hmc_p
->idx
= palloc
->level1
.idx
;
1432 hmc_p
= &qpmr
->rq_pbl
;
1433 hmc_p
->idx
= palloc
->level1
.idx
+ req
->sq_pages
;
1435 hmc_p
->addr
= arr
[0];
1436 hmc_p
= &qpmr
->rq_pbl
;
1437 hmc_p
->addr
= arr
[1];
1440 hmc_p
= &cqmr
->cq_pbl
;
1441 cqmr
->shadow
= (dma_addr_t
)arr
[total
];
1443 hmc_p
->idx
= palloc
->level1
.idx
;
1445 hmc_p
->addr
= arr
[0];
1451 * i40iw_hwreg_mr - send cqp command for memory registration
1452 * @iwdev: iwarp device
1453 * @iwmr: iwarp mr pointer
1454 * @access: access for MR
1456 static int i40iw_hwreg_mr(struct i40iw_device
*iwdev
,
1457 struct i40iw_mr
*iwmr
,
1460 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1461 struct i40iw_reg_ns_stag_info
*stag_info
;
1462 struct i40iw_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
1463 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1464 enum i40iw_status_code status
;
1466 struct i40iw_cqp_request
*cqp_request
;
1467 struct cqp_commands_info
*cqp_info
;
1469 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1473 cqp_info
= &cqp_request
->info
;
1474 stag_info
= &cqp_info
->in
.u
.mr_reg_non_shared
.info
;
1475 memset(stag_info
, 0, sizeof(*stag_info
));
1476 stag_info
->va
= (void *)(unsigned long)iwpbl
->user_base
;
1477 stag_info
->stag_idx
= iwmr
->stag
>> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1478 stag_info
->stag_key
= (u8
)iwmr
->stag
;
1479 stag_info
->total_len
= iwmr
->length
;
1480 stag_info
->access_rights
= access
;
1481 stag_info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1482 stag_info
->addr_type
= I40IW_ADDR_TYPE_VA_BASED
;
1484 if (iwmr
->page_cnt
> 1) {
1485 if (palloc
->level
== I40IW_LEVEL_1
) {
1486 stag_info
->first_pm_pbl_index
= palloc
->level1
.idx
;
1487 stag_info
->chunk_size
= 1;
1489 stag_info
->first_pm_pbl_index
= palloc
->level2
.root
.idx
;
1490 stag_info
->chunk_size
= 3;
1493 stag_info
->reg_addr_pa
= iwmr
->pgaddrmem
[0];
1496 cqp_info
->cqp_cmd
= OP_MR_REG_NON_SHARED
;
1497 cqp_info
->post_sq
= 1;
1498 cqp_info
->in
.u
.mr_reg_non_shared
.dev
= &iwdev
->sc_dev
;
1499 cqp_info
->in
.u
.mr_reg_non_shared
.scratch
= (uintptr_t)cqp_request
;
1501 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1504 i40iw_pr_err("CQP-OP MR Reg fail");
1510 * i40iw_reg_user_mr - Register a user memory region
1512 * @start: virtual start address
1513 * @length: length of mr
1514 * @virt: virtual address
1515 * @acc: access of mr
1518 static struct ib_mr
*i40iw_reg_user_mr(struct ib_pd
*pd
,
1523 struct ib_udata
*udata
)
1525 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1526 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1527 struct i40iw_ucontext
*ucontext
;
1528 struct i40iw_pble_alloc
*palloc
;
1529 struct i40iw_pbl
*iwpbl
;
1530 struct i40iw_mr
*iwmr
;
1531 struct ib_umem
*region
;
1532 struct i40iw_mem_reg_req req
;
1537 bool use_pbles
= false;
1538 unsigned long flags
;
1541 if (length
> I40IW_MAX_MR_SIZE
)
1542 return ERR_PTR(-EINVAL
);
1543 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
1545 return (struct ib_mr
*)region
;
1547 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
1548 ib_umem_release(region
);
1549 return ERR_PTR(-EFAULT
);
1552 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1554 ib_umem_release(region
);
1555 return ERR_PTR(-ENOMEM
);
1558 iwpbl
= &iwmr
->iwpbl
;
1560 iwmr
->region
= region
;
1562 iwmr
->ibmr
.device
= pd
->device
;
1563 ucontext
= to_ucontext(pd
->uobject
->context
);
1564 region_length
= region
->length
+ (start
& 0xfff);
1565 pbl_depth
= region_length
>> 12;
1566 pbl_depth
+= (region_length
& (4096 - 1)) ? 1 : 0;
1567 iwmr
->length
= region
->length
;
1569 iwpbl
->user_base
= virt
;
1570 palloc
= &iwpbl
->pble_alloc
;
1572 iwmr
->type
= req
.reg_type
;
1573 iwmr
->page_cnt
= (u32
)pbl_depth
;
1575 switch (req
.reg_type
) {
1576 case IW_MEMREG_TYPE_QP
:
1577 use_pbles
= ((req
.sq_pages
+ req
.rq_pages
) > 2);
1578 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1581 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1582 list_add_tail(&iwpbl
->list
, &ucontext
->qp_reg_mem_list
);
1583 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1585 case IW_MEMREG_TYPE_CQ
:
1586 use_pbles
= (req
.cq_pages
> 1);
1587 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1591 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1592 list_add_tail(&iwpbl
->list
, &ucontext
->cq_reg_mem_list
);
1593 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1595 case IW_MEMREG_TYPE_MEM
:
1596 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1598 use_pbles
= (iwmr
->page_cnt
!= 1);
1599 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1603 access
|= i40iw_get_user_access(acc
);
1604 stag
= i40iw_create_stag(iwdev
);
1611 iwmr
->ibmr
.rkey
= stag
;
1612 iwmr
->ibmr
.lkey
= stag
;
1614 err
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1616 i40iw_free_stag(iwdev
, stag
);
1624 iwmr
->type
= req
.reg_type
;
1625 if (req
.reg_type
== IW_MEMREG_TYPE_MEM
)
1626 i40iw_add_pdusecount(iwpd
);
1630 if (palloc
->level
!= I40IW_LEVEL_0
)
1631 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1632 ib_umem_release(region
);
1634 return ERR_PTR(err
);
1638 * i40iw_reg_phys_mr - register kernel physical memory
1640 * @addr: physical address of memory to register
1641 * @size: size of memory to register
1642 * @acc: Access rights
1643 * @iova_start: start of virtual address for physical buffers
1645 struct ib_mr
*i40iw_reg_phys_mr(struct ib_pd
*pd
,
1651 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1652 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1653 struct i40iw_pbl
*iwpbl
;
1654 struct i40iw_mr
*iwmr
;
1655 enum i40iw_status_code status
;
1657 u16 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1660 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1662 return ERR_PTR(-ENOMEM
);
1664 iwmr
->ibmr
.device
= pd
->device
;
1665 iwpbl
= &iwmr
->iwpbl
;
1667 iwmr
->type
= IW_MEMREG_TYPE_MEM
;
1668 iwpbl
->user_base
= *iova_start
;
1669 stag
= i40iw_create_stag(iwdev
);
1674 access
|= i40iw_get_user_access(acc
);
1676 iwmr
->ibmr
.rkey
= stag
;
1677 iwmr
->ibmr
.lkey
= stag
;
1679 iwmr
->pgaddrmem
[0] = addr
;
1680 status
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1682 i40iw_free_stag(iwdev
, stag
);
1687 i40iw_add_pdusecount(iwpd
);
1691 return ERR_PTR(ret
);
1695 * i40iw_get_dma_mr - register physical mem
1697 * @acc: access for memory
1699 static struct ib_mr
*i40iw_get_dma_mr(struct ib_pd
*pd
, int acc
)
1703 return i40iw_reg_phys_mr(pd
, 0, 0xffffffffffULL
, acc
, &kva
);
1707 * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
1708 * @iwmr: iwmr for IB's user page addresses
1709 * @ucontext: ptr to user context
1711 static void i40iw_del_memlist(struct i40iw_mr
*iwmr
,
1712 struct i40iw_ucontext
*ucontext
)
1714 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1715 unsigned long flags
;
1717 switch (iwmr
->type
) {
1718 case IW_MEMREG_TYPE_CQ
:
1719 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1720 if (!list_empty(&ucontext
->cq_reg_mem_list
))
1721 list_del(&iwpbl
->list
);
1722 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1724 case IW_MEMREG_TYPE_QP
:
1725 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1726 if (!list_empty(&ucontext
->qp_reg_mem_list
))
1727 list_del(&iwpbl
->list
);
1728 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1736 * i40iw_dereg_mr - deregister mr
1737 * @ib_mr: mr ptr for dereg
1739 static int i40iw_dereg_mr(struct ib_mr
*ib_mr
)
1741 struct ib_pd
*ibpd
= ib_mr
->pd
;
1742 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
1743 struct i40iw_mr
*iwmr
= to_iwmr(ib_mr
);
1744 struct i40iw_device
*iwdev
= to_iwdev(ib_mr
->device
);
1745 enum i40iw_status_code status
;
1746 struct i40iw_dealloc_stag_info
*info
;
1747 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1748 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1749 struct i40iw_cqp_request
*cqp_request
;
1750 struct cqp_commands_info
*cqp_info
;
1754 ib_umem_release(iwmr
->region
);
1756 if (iwmr
->type
!= IW_MEMREG_TYPE_MEM
) {
1757 if (ibpd
->uobject
) {
1758 struct i40iw_ucontext
*ucontext
;
1760 ucontext
= to_ucontext(ibpd
->uobject
->context
);
1761 i40iw_del_memlist(iwmr
, ucontext
);
1763 if (iwpbl
->pbl_allocated
)
1764 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1770 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1774 cqp_info
= &cqp_request
->info
;
1775 info
= &cqp_info
->in
.u
.dealloc_stag
.info
;
1776 memset(info
, 0, sizeof(*info
));
1778 info
->pd_id
= cpu_to_le32(iwpd
->sc_pd
.pd_id
& 0x00007fff);
1779 info
->stag_idx
= RS_64_1(ib_mr
->rkey
, I40IW_CQPSQ_STAG_IDX_SHIFT
);
1780 stag_idx
= info
->stag_idx
;
1782 if (iwpbl
->pbl_allocated
)
1783 info
->dealloc_pbl
= true;
1785 cqp_info
->cqp_cmd
= OP_DEALLOC_STAG
;
1786 cqp_info
->post_sq
= 1;
1787 cqp_info
->in
.u
.dealloc_stag
.dev
= &iwdev
->sc_dev
;
1788 cqp_info
->in
.u
.dealloc_stag
.scratch
= (uintptr_t)cqp_request
;
1789 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1791 i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx
);
1792 i40iw_rem_pdusecount(iwpd
, iwdev
);
1793 i40iw_free_stag(iwdev
, iwmr
->stag
);
1794 if (iwpbl
->pbl_allocated
)
1795 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1803 static ssize_t
i40iw_show_rev(struct device
*dev
,
1804 struct device_attribute
*attr
, char *buf
)
1806 struct i40iw_ib_device
*iwibdev
= container_of(dev
,
1807 struct i40iw_ib_device
,
1809 u32 hw_rev
= iwibdev
->iwdev
->sc_dev
.hw_rev
;
1811 return sprintf(buf
, "%x\n", hw_rev
);
1817 static ssize_t
i40iw_show_fw_ver(struct device
*dev
,
1818 struct device_attribute
*attr
, char *buf
)
1820 u32 firmware_version
= I40IW_FW_VERSION
;
1822 return sprintf(buf
, "%u.%u\n", firmware_version
,
1823 (firmware_version
& 0x000000ff));
1829 static ssize_t
i40iw_show_hca(struct device
*dev
,
1830 struct device_attribute
*attr
, char *buf
)
1832 return sprintf(buf
, "I40IW\n");
1838 static ssize_t
i40iw_show_board(struct device
*dev
,
1839 struct device_attribute
*attr
,
1842 return sprintf(buf
, "%.*s\n", 32, "I40IW Board ID");
1845 static DEVICE_ATTR(hw_rev
, S_IRUGO
, i40iw_show_rev
, NULL
);
1846 static DEVICE_ATTR(fw_ver
, S_IRUGO
, i40iw_show_fw_ver
, NULL
);
1847 static DEVICE_ATTR(hca_type
, S_IRUGO
, i40iw_show_hca
, NULL
);
1848 static DEVICE_ATTR(board_id
, S_IRUGO
, i40iw_show_board
, NULL
);
1850 static struct device_attribute
*i40iw_dev_attributes
[] = {
1858 * i40iw_copy_sg_list - copy sg list for qp
1859 * @sg_list: copied into sg_list
1860 * @sgl: copy from sgl
1861 * @num_sges: count of sg entries
1863 static void i40iw_copy_sg_list(struct i40iw_sge
*sg_list
, struct ib_sge
*sgl
, int num_sges
)
1867 for (i
= 0; (i
< num_sges
) && (i
< I40IW_MAX_WQ_FRAGMENT_COUNT
); i
++) {
1868 sg_list
[i
].tag_off
= sgl
[i
].addr
;
1869 sg_list
[i
].len
= sgl
[i
].length
;
1870 sg_list
[i
].stag
= sgl
[i
].lkey
;
1875 * i40iw_post_send - kernel application wr
1876 * @ibqp: qp ptr for wr
1877 * @ib_wr: work request ptr
1878 * @bad_wr: return of bad wr if err
1880 static int i40iw_post_send(struct ib_qp
*ibqp
,
1881 struct ib_send_wr
*ib_wr
,
1882 struct ib_send_wr
**bad_wr
)
1884 struct i40iw_qp
*iwqp
;
1885 struct i40iw_qp_uk
*ukqp
;
1886 struct i40iw_post_sq_info info
;
1887 enum i40iw_status_code ret
;
1889 unsigned long flags
;
1891 iwqp
= (struct i40iw_qp
*)ibqp
;
1892 ukqp
= &iwqp
->sc_qp
.qp_uk
;
1894 spin_lock_irqsave(&iwqp
->lock
, flags
);
1896 memset(&info
, 0, sizeof(info
));
1897 info
.wr_id
= (u64
)(ib_wr
->wr_id
);
1898 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || iwqp
->sig_all
)
1899 info
.signaled
= true;
1900 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
1901 info
.read_fence
= true;
1903 switch (ib_wr
->opcode
) {
1905 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
1906 info
.op_type
= I40IW_OP_TYPE_SEND_SOL
;
1908 info
.op_type
= I40IW_OP_TYPE_SEND
;
1910 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
1911 info
.op
.inline_send
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
1912 info
.op
.inline_send
.len
= ib_wr
->sg_list
[0].length
;
1913 ret
= ukqp
->ops
.iw_inline_send(ukqp
, &info
, rdma_wr(ib_wr
)->rkey
, false);
1915 info
.op
.send
.num_sges
= ib_wr
->num_sge
;
1916 info
.op
.send
.sg_list
= (struct i40iw_sge
*)ib_wr
->sg_list
;
1917 ret
= ukqp
->ops
.iw_send(ukqp
, &info
, rdma_wr(ib_wr
)->rkey
, false);
1923 case IB_WR_RDMA_WRITE
:
1924 info
.op_type
= I40IW_OP_TYPE_RDMA_WRITE
;
1926 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
1927 info
.op
.inline_rdma_write
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
1928 info
.op
.inline_rdma_write
.len
= ib_wr
->sg_list
[0].length
;
1929 info
.op
.inline_rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
1930 info
.op
.inline_rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
1931 info
.op
.inline_rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
1932 ret
= ukqp
->ops
.iw_inline_rdma_write(ukqp
, &info
, false);
1934 info
.op
.rdma_write
.lo_sg_list
= (void *)ib_wr
->sg_list
;
1935 info
.op
.rdma_write
.num_lo_sges
= ib_wr
->num_sge
;
1936 info
.op
.rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
1937 info
.op
.rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
1938 info
.op
.rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
1939 ret
= ukqp
->ops
.iw_rdma_write(ukqp
, &info
, false);
1945 case IB_WR_RDMA_READ
:
1946 info
.op_type
= I40IW_OP_TYPE_RDMA_READ
;
1947 info
.op
.rdma_read
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
1948 info
.op
.rdma_read
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
1949 info
.op
.rdma_read
.rem_addr
.len
= ib_wr
->sg_list
->length
;
1950 info
.op
.rdma_read
.lo_addr
.tag_off
= ib_wr
->sg_list
->addr
;
1951 info
.op
.rdma_read
.lo_addr
.stag
= ib_wr
->sg_list
->lkey
;
1952 info
.op
.rdma_read
.lo_addr
.len
= ib_wr
->sg_list
->length
;
1953 ret
= ukqp
->ops
.iw_rdma_read(ukqp
, &info
, false, false);
1959 i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
1966 ib_wr
= ib_wr
->next
;
1972 ukqp
->ops
.iw_qp_post_wr(ukqp
);
1973 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1979 * i40iw_post_recv - post receive wr for kernel application
1980 * @ibqp: ib qp pointer
1981 * @ib_wr: work request for receive
1982 * @bad_wr: bad wr caused an error
1984 static int i40iw_post_recv(struct ib_qp
*ibqp
,
1985 struct ib_recv_wr
*ib_wr
,
1986 struct ib_recv_wr
**bad_wr
)
1988 struct i40iw_qp
*iwqp
;
1989 struct i40iw_qp_uk
*ukqp
;
1990 struct i40iw_post_rq_info post_recv
;
1991 struct i40iw_sge sg_list
[I40IW_MAX_WQ_FRAGMENT_COUNT
];
1992 enum i40iw_status_code ret
= 0;
1993 unsigned long flags
;
1995 iwqp
= (struct i40iw_qp
*)ibqp
;
1996 ukqp
= &iwqp
->sc_qp
.qp_uk
;
1998 memset(&post_recv
, 0, sizeof(post_recv
));
1999 spin_lock_irqsave(&iwqp
->lock
, flags
);
2001 post_recv
.num_sges
= ib_wr
->num_sge
;
2002 post_recv
.wr_id
= ib_wr
->wr_id
;
2003 i40iw_copy_sg_list(sg_list
, ib_wr
->sg_list
, ib_wr
->num_sge
);
2004 post_recv
.sg_list
= sg_list
;
2005 ret
= ukqp
->ops
.iw_post_receive(ukqp
, &post_recv
);
2007 i40iw_pr_err(" post_recv err %d\n", ret
);
2011 ib_wr
= ib_wr
->next
;
2014 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
2021 * i40iw_poll_cq - poll cq for completion (kernel apps)
2023 * @num_entries: number of entries to poll
2024 * @entry: wr of entry completed
2026 static int i40iw_poll_cq(struct ib_cq
*ibcq
,
2028 struct ib_wc
*entry
)
2030 struct i40iw_cq
*iwcq
;
2032 struct i40iw_cq_poll_info cq_poll_info
;
2033 enum i40iw_status_code ret
;
2034 struct i40iw_cq_uk
*ukcq
;
2035 struct i40iw_sc_qp
*qp
;
2036 unsigned long flags
;
2038 iwcq
= (struct i40iw_cq
*)ibcq
;
2039 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2041 spin_lock_irqsave(&iwcq
->lock
, flags
);
2042 while (cqe_count
< num_entries
) {
2043 ret
= ukcq
->ops
.iw_cq_poll_completion(ukcq
, &cq_poll_info
, true);
2044 if (ret
== I40IW_ERR_QUEUE_EMPTY
) {
2051 entry
->wc_flags
= 0;
2052 entry
->wr_id
= cq_poll_info
.wr_id
;
2053 if (!cq_poll_info
.error
)
2054 entry
->status
= IB_WC_SUCCESS
;
2056 entry
->status
= IB_WC_WR_FLUSH_ERR
;
2058 switch (cq_poll_info
.op_type
) {
2059 case I40IW_OP_TYPE_RDMA_WRITE
:
2060 entry
->opcode
= IB_WC_RDMA_WRITE
;
2062 case I40IW_OP_TYPE_RDMA_READ_INV_STAG
:
2063 case I40IW_OP_TYPE_RDMA_READ
:
2064 entry
->opcode
= IB_WC_RDMA_READ
;
2066 case I40IW_OP_TYPE_SEND_SOL
:
2067 case I40IW_OP_TYPE_SEND_SOL_INV
:
2068 case I40IW_OP_TYPE_SEND_INV
:
2069 case I40IW_OP_TYPE_SEND
:
2070 entry
->opcode
= IB_WC_SEND
;
2072 case I40IW_OP_TYPE_REC
:
2073 entry
->opcode
= IB_WC_RECV
;
2076 entry
->opcode
= IB_WC_RECV
;
2081 cq_poll_info
.major_err
<< 16 | cq_poll_info
.minor_err
;
2082 entry
->ex
.imm_data
= 0;
2083 qp
= (struct i40iw_sc_qp
*)cq_poll_info
.qp_handle
;
2084 entry
->qp
= (struct ib_qp
*)qp
->back_qp
;
2085 entry
->src_qp
= cq_poll_info
.qp_id
;
2086 entry
->byte_len
= cq_poll_info
.bytes_xfered
;
2090 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
2095 * i40iw_req_notify_cq - arm cq kernel application
2097 * @notify_flags: notofication flags
2099 static int i40iw_req_notify_cq(struct ib_cq
*ibcq
,
2100 enum ib_cq_notify_flags notify_flags
)
2102 struct i40iw_cq
*iwcq
;
2103 struct i40iw_cq_uk
*ukcq
;
2104 enum i40iw_completion_notify cq_notify
= IW_CQ_COMPL_SOLICITED
;
2106 iwcq
= (struct i40iw_cq
*)ibcq
;
2107 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2108 if (notify_flags
== IB_CQ_NEXT_COMP
)
2109 cq_notify
= IW_CQ_COMPL_EVENT
;
2110 ukcq
->ops
.iw_cq_request_notification(ukcq
, cq_notify
);
2115 * i40iw_port_immutable - return port's immutable data
2116 * @ibdev: ib dev struct
2117 * @port_num: port number
2118 * @immutable: immutable data for the port return
2120 static int i40iw_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
2121 struct ib_port_immutable
*immutable
)
2123 struct ib_port_attr attr
;
2126 err
= i40iw_query_port(ibdev
, port_num
, &attr
);
2131 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
2132 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
2133 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
2139 * i40iw_get_protocol_stats - Populates the rdma_stats structure
2140 * @ibdev: ib dev struct
2141 * @stats: iw protocol stats struct
2143 static int i40iw_get_protocol_stats(struct ib_device
*ibdev
,
2144 union rdma_protocol_stats
*stats
)
2146 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2147 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
2148 struct i40iw_dev_pestat
*devstat
= &dev
->dev_pestat
;
2149 struct i40iw_dev_hw_stats
*hw_stats
= &devstat
->hw_stats
;
2150 struct timespec curr_time
;
2151 static struct timespec last_rd_time
= {0, 0};
2152 enum i40iw_status_code status
= 0;
2153 unsigned long flags
;
2155 curr_time
= current_kernel_time();
2156 memset(stats
, 0, sizeof(*stats
));
2159 spin_lock_irqsave(&devstat
->stats_lock
, flags
);
2160 devstat
->ops
.iw_hw_stat_read_all(devstat
,
2161 &devstat
->hw_stats
);
2162 spin_unlock_irqrestore(&devstat
->stats_lock
, flags
);
2164 if (((u64
)curr_time
.tv_sec
- (u64
)last_rd_time
.tv_sec
) > 1)
2165 status
= i40iw_vchnl_vf_get_pe_stats(dev
,
2166 &devstat
->hw_stats
);
2172 stats
->iw
.ipInReceives
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] +
2173 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
];
2174 stats
->iw
.ipInTruncatedPkts
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] +
2175 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
];
2176 stats
->iw
.ipInDiscards
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] +
2177 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
];
2178 stats
->iw
.ipOutNoRoutes
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] +
2179 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
];
2180 stats
->iw
.ipReasmReqds
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] +
2181 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
];
2182 stats
->iw
.ipFragCreates
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] +
2183 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
];
2184 stats
->iw
.ipInMcastPkts
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] +
2185 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
];
2186 stats
->iw
.ipOutMcastPkts
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] +
2187 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6TXMCPKTS
];
2188 stats
->iw
.tcpOutSegs
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
];
2189 stats
->iw
.tcpInSegs
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
];
2190 stats
->iw
.tcpRetransSegs
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
];
2192 last_rd_time
= curr_time
;
2197 * i40iw_query_gid - Query port GID
2198 * @ibdev: device pointer from stack
2199 * @port: port number
2200 * @index: Entry index
2203 static int i40iw_query_gid(struct ib_device
*ibdev
,
2208 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2210 memset(gid
->raw
, 0, sizeof(gid
->raw
));
2211 ether_addr_copy(gid
->raw
, iwdev
->netdev
->dev_addr
);
2216 * i40iw_modify_port Modify port properties
2217 * @ibdev: device pointer from stack
2218 * @port: port number
2219 * @port_modify_mask: mask for port modifications
2220 * @props: port properties
2222 static int i40iw_modify_port(struct ib_device
*ibdev
,
2224 int port_modify_mask
,
2225 struct ib_port_modify
*props
)
2231 * i40iw_query_pkey - Query partition key
2232 * @ibdev: device pointer from stack
2233 * @port: port number
2234 * @index: index of pkey
2235 * @pkey: pointer to store the pkey
2237 static int i40iw_query_pkey(struct ib_device
*ibdev
,
2247 * i40iw_create_ah - create address handle
2249 * @ah_attr: address handle attributes
2251 static struct ib_ah
*i40iw_create_ah(struct ib_pd
*ibpd
,
2252 struct ib_ah_attr
*attr
)
2254 return ERR_PTR(-ENOSYS
);
2258 * i40iw_destroy_ah - Destroy address handle
2259 * @ah: pointer to address handle
2261 static int i40iw_destroy_ah(struct ib_ah
*ah
)
2267 * i40iw_init_rdma_device - initialization of iwarp device
2268 * @iwdev: iwarp device
2270 static struct i40iw_ib_device
*i40iw_init_rdma_device(struct i40iw_device
*iwdev
)
2272 struct i40iw_ib_device
*iwibdev
;
2273 struct net_device
*netdev
= iwdev
->netdev
;
2274 struct pci_dev
*pcidev
= (struct pci_dev
*)iwdev
->hw
.dev_context
;
2276 iwibdev
= (struct i40iw_ib_device
*)ib_alloc_device(sizeof(*iwibdev
));
2278 i40iw_pr_err("iwdev == NULL\n");
2281 strlcpy(iwibdev
->ibdev
.name
, "i40iw%d", IB_DEVICE_NAME_MAX
);
2282 iwibdev
->ibdev
.owner
= THIS_MODULE
;
2283 iwdev
->iwibdev
= iwibdev
;
2284 iwibdev
->iwdev
= iwdev
;
2286 iwibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
2287 ether_addr_copy((u8
*)&iwibdev
->ibdev
.node_guid
, netdev
->dev_addr
);
2289 iwibdev
->ibdev
.uverbs_cmd_mask
=
2290 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
2291 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
2292 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
2293 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
2294 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
2295 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
2296 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
2297 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
2298 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
2299 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
2300 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
2301 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
2302 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
2303 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
2304 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
2305 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
2306 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
2307 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
2308 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
2309 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
2310 iwibdev
->ibdev
.phys_port_cnt
= 1;
2311 iwibdev
->ibdev
.num_comp_vectors
= 1;
2312 iwibdev
->ibdev
.dma_device
= &pcidev
->dev
;
2313 iwibdev
->ibdev
.dev
.parent
= &pcidev
->dev
;
2314 iwibdev
->ibdev
.query_port
= i40iw_query_port
;
2315 iwibdev
->ibdev
.modify_port
= i40iw_modify_port
;
2316 iwibdev
->ibdev
.query_pkey
= i40iw_query_pkey
;
2317 iwibdev
->ibdev
.query_gid
= i40iw_query_gid
;
2318 iwibdev
->ibdev
.alloc_ucontext
= i40iw_alloc_ucontext
;
2319 iwibdev
->ibdev
.dealloc_ucontext
= i40iw_dealloc_ucontext
;
2320 iwibdev
->ibdev
.mmap
= i40iw_mmap
;
2321 iwibdev
->ibdev
.alloc_pd
= i40iw_alloc_pd
;
2322 iwibdev
->ibdev
.dealloc_pd
= i40iw_dealloc_pd
;
2323 iwibdev
->ibdev
.create_qp
= i40iw_create_qp
;
2324 iwibdev
->ibdev
.modify_qp
= i40iw_modify_qp
;
2325 iwibdev
->ibdev
.query_qp
= i40iw_query_qp
;
2326 iwibdev
->ibdev
.destroy_qp
= i40iw_destroy_qp
;
2327 iwibdev
->ibdev
.create_cq
= i40iw_create_cq
;
2328 iwibdev
->ibdev
.destroy_cq
= i40iw_destroy_cq
;
2329 iwibdev
->ibdev
.get_dma_mr
= i40iw_get_dma_mr
;
2330 iwibdev
->ibdev
.reg_user_mr
= i40iw_reg_user_mr
;
2331 iwibdev
->ibdev
.dereg_mr
= i40iw_dereg_mr
;
2332 iwibdev
->ibdev
.get_protocol_stats
= i40iw_get_protocol_stats
;
2333 iwibdev
->ibdev
.query_device
= i40iw_query_device
;
2334 iwibdev
->ibdev
.create_ah
= i40iw_create_ah
;
2335 iwibdev
->ibdev
.destroy_ah
= i40iw_destroy_ah
;
2336 iwibdev
->ibdev
.iwcm
= kzalloc(sizeof(*iwibdev
->ibdev
.iwcm
), GFP_KERNEL
);
2337 if (!iwibdev
->ibdev
.iwcm
) {
2338 ib_dealloc_device(&iwibdev
->ibdev
);
2339 i40iw_pr_err("iwcm == NULL\n");
2343 iwibdev
->ibdev
.iwcm
->add_ref
= i40iw_add_ref
;
2344 iwibdev
->ibdev
.iwcm
->rem_ref
= i40iw_rem_ref
;
2345 iwibdev
->ibdev
.iwcm
->get_qp
= i40iw_get_qp
;
2346 iwibdev
->ibdev
.iwcm
->connect
= i40iw_connect
;
2347 iwibdev
->ibdev
.iwcm
->accept
= i40iw_accept
;
2348 iwibdev
->ibdev
.iwcm
->reject
= i40iw_reject
;
2349 iwibdev
->ibdev
.iwcm
->create_listen
= i40iw_create_listen
;
2350 iwibdev
->ibdev
.iwcm
->destroy_listen
= i40iw_destroy_listen
;
2351 memcpy(iwibdev
->ibdev
.iwcm
->ifname
, netdev
->name
,
2352 sizeof(iwibdev
->ibdev
.iwcm
->ifname
));
2353 iwibdev
->ibdev
.get_port_immutable
= i40iw_port_immutable
;
2354 iwibdev
->ibdev
.poll_cq
= i40iw_poll_cq
;
2355 iwibdev
->ibdev
.req_notify_cq
= i40iw_req_notify_cq
;
2356 iwibdev
->ibdev
.post_send
= i40iw_post_send
;
2357 iwibdev
->ibdev
.post_recv
= i40iw_post_recv
;
2363 * i40iw_port_ibevent - indicate port event
2364 * @iwdev: iwarp device
2366 void i40iw_port_ibevent(struct i40iw_device
*iwdev
)
2368 struct i40iw_ib_device
*iwibdev
= iwdev
->iwibdev
;
2369 struct ib_event event
;
2371 event
.device
= &iwibdev
->ibdev
;
2372 event
.element
.port_num
= 1;
2373 event
.event
= iwdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
2374 ib_dispatch_event(&event
);
2378 * i40iw_unregister_rdma_device - unregister of iwarp from IB
2379 * @iwibdev: rdma device ptr
2381 static void i40iw_unregister_rdma_device(struct i40iw_ib_device
*iwibdev
)
2385 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
)
2386 device_remove_file(&iwibdev
->ibdev
.dev
,
2387 i40iw_dev_attributes
[i
]);
2388 ib_unregister_device(&iwibdev
->ibdev
);
2392 * i40iw_destroy_rdma_device - destroy rdma device and free resources
2393 * @iwibdev: IB device ptr
2395 void i40iw_destroy_rdma_device(struct i40iw_ib_device
*iwibdev
)
2400 i40iw_unregister_rdma_device(iwibdev
);
2401 kfree(iwibdev
->ibdev
.iwcm
);
2402 iwibdev
->ibdev
.iwcm
= NULL
;
2403 ib_dealloc_device(&iwibdev
->ibdev
);
2407 * i40iw_register_rdma_device - register iwarp device to IB
2408 * @iwdev: iwarp device
2410 int i40iw_register_rdma_device(struct i40iw_device
*iwdev
)
2413 struct i40iw_ib_device
*iwibdev
;
2415 iwdev
->iwibdev
= i40iw_init_rdma_device(iwdev
);
2416 if (!iwdev
->iwibdev
)
2418 iwibdev
= iwdev
->iwibdev
;
2420 ret
= ib_register_device(&iwibdev
->ibdev
, NULL
);
2424 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
) {
2426 device_create_file(&iwibdev
->ibdev
.dev
,
2427 i40iw_dev_attributes
[i
]);
2431 device_remove_file(&iwibdev
->ibdev
.dev
, i40iw_dev_attributes
[i
]);
2433 ib_unregister_device(&iwibdev
->ibdev
);
2439 kfree(iwdev
->iwibdev
->ibdev
.iwcm
);
2440 iwdev
->iwibdev
->ibdev
.iwcm
= NULL
;
2441 ib_dealloc_device(&iwdev
->iwibdev
->ibdev
);