1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/random.h>
38 #include <linux/highmem.h>
39 #include <linux/time.h>
40 #include <asm/byteorder.h>
42 #include <rdma/ib_verbs.h>
43 #include <rdma/iw_cm.h>
44 #include <rdma/ib_user_verbs.h>
45 #include <rdma/ib_umem.h>
49 * i40iw_query_device - get device attributes
50 * @ibdev: device pointer from stack
51 * @props: returning device attributes
54 static int i40iw_query_device(struct ib_device
*ibdev
,
55 struct ib_device_attr
*props
,
56 struct ib_udata
*udata
)
58 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
60 if (udata
->inlen
|| udata
->outlen
)
62 memset(props
, 0, sizeof(*props
));
63 ether_addr_copy((u8
*)&props
->sys_image_guid
, iwdev
->netdev
->dev_addr
);
64 props
->fw_ver
= I40IW_FW_VERSION
;
65 props
->device_cap_flags
= iwdev
->device_cap_flags
;
66 props
->vendor_id
= iwdev
->ldev
->pcidev
->vendor
;
67 props
->vendor_part_id
= iwdev
->ldev
->pcidev
->device
;
68 props
->hw_ver
= (u32
)iwdev
->sc_dev
.hw_rev
;
69 props
->max_mr_size
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
70 props
->max_qp
= iwdev
->max_qp
;
71 props
->max_qp_wr
= (I40IW_MAX_WQ_ENTRIES
>> 2) - 1;
72 props
->max_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
73 props
->max_cq
= iwdev
->max_cq
;
74 props
->max_cqe
= iwdev
->max_cqe
;
75 props
->max_mr
= iwdev
->max_mr
;
76 props
->max_pd
= iwdev
->max_pd
;
77 props
->max_sge_rd
= 1;
78 props
->max_qp_rd_atom
= I40IW_MAX_IRD_SIZE
;
79 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
80 props
->atomic_cap
= IB_ATOMIC_NONE
;
81 props
->max_map_per_fmr
= 1;
86 * i40iw_query_port - get port attrubutes
87 * @ibdev: device pointer from stack
88 * @port: port number for query
89 * @props: returning device attributes
91 static int i40iw_query_port(struct ib_device
*ibdev
,
93 struct ib_port_attr
*props
)
95 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
96 struct net_device
*netdev
= iwdev
->netdev
;
98 memset(props
, 0, sizeof(*props
));
100 props
->max_mtu
= IB_MTU_4096
;
101 if (netdev
->mtu
>= 4096)
102 props
->active_mtu
= IB_MTU_4096
;
103 else if (netdev
->mtu
>= 2048)
104 props
->active_mtu
= IB_MTU_2048
;
105 else if (netdev
->mtu
>= 1024)
106 props
->active_mtu
= IB_MTU_1024
;
107 else if (netdev
->mtu
>= 512)
108 props
->active_mtu
= IB_MTU_512
;
110 props
->active_mtu
= IB_MTU_256
;
113 if (netif_carrier_ok(iwdev
->netdev
))
114 props
->state
= IB_PORT_ACTIVE
;
116 props
->state
= IB_PORT_DOWN
;
117 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
118 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
119 props
->gid_tbl_len
= 1;
120 props
->pkey_tbl_len
= 1;
121 props
->active_width
= IB_WIDTH_4X
;
122 props
->active_speed
= 1;
123 props
->max_msg_sz
= I40IW_MAX_OUTBOUND_MESSAGE_SIZE
;
128 * i40iw_alloc_ucontext - Allocate the user context data structure
129 * @ibdev: device pointer from stack
132 * This keeps track of all objects associated with a particular
135 static struct ib_ucontext
*i40iw_alloc_ucontext(struct ib_device
*ibdev
,
136 struct ib_udata
*udata
)
138 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
139 struct i40iw_alloc_ucontext_req req
;
140 struct i40iw_alloc_ucontext_resp uresp
;
141 struct i40iw_ucontext
*ucontext
;
143 if (ib_copy_from_udata(&req
, udata
, sizeof(req
)))
144 return ERR_PTR(-EINVAL
);
146 if (req
.userspace_ver
!= I40IW_ABI_USERSPACE_VER
) {
147 i40iw_pr_err("Invalid userspace driver version detected. Detected version %d, should be %d\n",
148 req
.userspace_ver
, I40IW_ABI_USERSPACE_VER
);
149 return ERR_PTR(-EINVAL
);
152 memset(&uresp
, 0, sizeof(uresp
));
153 uresp
.max_qps
= iwdev
->max_qp
;
154 uresp
.max_pds
= iwdev
->max_pd
;
155 uresp
.wq_size
= iwdev
->max_qp_wr
* 2;
156 uresp
.kernel_ver
= I40IW_ABI_KERNEL_VER
;
158 ucontext
= kzalloc(sizeof(*ucontext
), GFP_KERNEL
);
160 return ERR_PTR(-ENOMEM
);
162 ucontext
->iwdev
= iwdev
;
164 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
166 return ERR_PTR(-EFAULT
);
169 INIT_LIST_HEAD(&ucontext
->cq_reg_mem_list
);
170 spin_lock_init(&ucontext
->cq_reg_mem_list_lock
);
171 INIT_LIST_HEAD(&ucontext
->qp_reg_mem_list
);
172 spin_lock_init(&ucontext
->qp_reg_mem_list_lock
);
174 return &ucontext
->ibucontext
;
178 * i40iw_dealloc_ucontext - deallocate the user context data structure
179 * @context: user context created during alloc
181 static int i40iw_dealloc_ucontext(struct ib_ucontext
*context
)
183 struct i40iw_ucontext
*ucontext
= to_ucontext(context
);
186 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
187 if (!list_empty(&ucontext
->cq_reg_mem_list
)) {
188 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
191 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
192 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
193 if (!list_empty(&ucontext
->qp_reg_mem_list
)) {
194 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
197 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
204 * i40iw_mmap - user memory map
205 * @context: context created during alloc
206 * @vma: kernel info for user memory map
208 static int i40iw_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
210 struct i40iw_ucontext
*ucontext
;
214 ucontext
= to_ucontext(context
);
215 if (ucontext
->iwdev
->sc_dev
.is_pf
) {
216 db_addr_offset
= I40IW_DB_ADDR_OFFSET
;
217 push_offset
= I40IW_PUSH_OFFSET
;
219 vma
->vm_pgoff
+= I40IW_PF_FIRST_PUSH_PAGE_INDEX
- 1;
221 db_addr_offset
= I40IW_VF_DB_ADDR_OFFSET
;
222 push_offset
= I40IW_VF_PUSH_OFFSET
;
224 vma
->vm_pgoff
+= I40IW_VF_FIRST_PUSH_PAGE_INDEX
- 1;
227 vma
->vm_pgoff
+= db_addr_offset
>> PAGE_SHIFT
;
229 if (vma
->vm_pgoff
== (db_addr_offset
>> PAGE_SHIFT
)) {
230 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
231 vma
->vm_private_data
= ucontext
;
233 if ((vma
->vm_pgoff
- (push_offset
>> PAGE_SHIFT
)) % 2)
234 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
236 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
239 if (io_remap_pfn_range(vma
, vma
->vm_start
,
240 vma
->vm_pgoff
+ (pci_resource_start(ucontext
->iwdev
->ldev
->pcidev
, 0) >> PAGE_SHIFT
),
241 PAGE_SIZE
, vma
->vm_page_prot
))
248 * i40iw_alloc_push_page - allocate a push page for qp
249 * @iwdev: iwarp device
250 * @qp: hardware control qp
252 static void i40iw_alloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
254 struct i40iw_cqp_request
*cqp_request
;
255 struct cqp_commands_info
*cqp_info
;
256 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
257 enum i40iw_status_code status
;
259 if (qp
->push_idx
!= I40IW_INVALID_PUSH_PAGE_INDEX
)
262 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
266 atomic_inc(&cqp_request
->refcount
);
268 cqp_info
= &cqp_request
->info
;
269 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
270 cqp_info
->post_sq
= 1;
272 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= dev
->qs_handle
;
273 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 0;
274 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
275 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
277 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
279 qp
->push_idx
= cqp_request
->compl_info
.op_ret_val
;
281 i40iw_pr_err("CQP-OP Push page fail");
282 i40iw_put_cqp_request(&iwdev
->cqp
, cqp_request
);
286 * i40iw_dealloc_push_page - free a push page for qp
287 * @iwdev: iwarp device
288 * @qp: hardware control qp
290 static void i40iw_dealloc_push_page(struct i40iw_device
*iwdev
, struct i40iw_sc_qp
*qp
)
292 struct i40iw_cqp_request
*cqp_request
;
293 struct cqp_commands_info
*cqp_info
;
294 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
295 enum i40iw_status_code status
;
297 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
)
300 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, false);
304 cqp_info
= &cqp_request
->info
;
305 cqp_info
->cqp_cmd
= OP_MANAGE_PUSH_PAGE
;
306 cqp_info
->post_sq
= 1;
308 cqp_info
->in
.u
.manage_push_page
.info
.push_idx
= qp
->push_idx
;
309 cqp_info
->in
.u
.manage_push_page
.info
.qs_handle
= dev
->qs_handle
;
310 cqp_info
->in
.u
.manage_push_page
.info
.free_page
= 1;
311 cqp_info
->in
.u
.manage_push_page
.cqp
= &iwdev
->cqp
.sc_cqp
;
312 cqp_info
->in
.u
.manage_push_page
.scratch
= (uintptr_t)cqp_request
;
314 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
316 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
318 i40iw_pr_err("CQP-OP Push page fail");
322 * i40iw_alloc_pd - allocate protection domain
323 * @ibdev: device pointer from stack
324 * @context: user context created during alloc
327 static struct ib_pd
*i40iw_alloc_pd(struct ib_device
*ibdev
,
328 struct ib_ucontext
*context
,
329 struct ib_udata
*udata
)
331 struct i40iw_pd
*iwpd
;
332 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
333 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
334 struct i40iw_alloc_pd_resp uresp
;
335 struct i40iw_sc_pd
*sc_pd
;
339 err
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_pds
,
340 iwdev
->max_pd
, &pd_id
, &iwdev
->next_pd
);
342 i40iw_pr_err("alloc resource failed\n");
346 iwpd
= kzalloc(sizeof(*iwpd
), GFP_KERNEL
);
352 sc_pd
= &iwpd
->sc_pd
;
353 dev
->iw_pd_ops
->pd_init(dev
, sc_pd
, pd_id
);
356 memset(&uresp
, 0, sizeof(uresp
));
358 if (ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
))) {
364 i40iw_add_pdusecount(iwpd
);
369 i40iw_free_resource(iwdev
, iwdev
->allocated_pds
, pd_id
);
374 * i40iw_dealloc_pd - deallocate pd
375 * @ibpd: ptr of pd to be deallocated
377 static int i40iw_dealloc_pd(struct ib_pd
*ibpd
)
379 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
380 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
382 i40iw_rem_pdusecount(iwpd
, iwdev
);
387 * i40iw_qp_roundup - return round up qp ring size
388 * @wr_ring_size: ring size to round up
390 static int i40iw_qp_roundup(u32 wr_ring_size
)
394 if (wr_ring_size
< I40IWQP_SW_MIN_WQSIZE
)
395 wr_ring_size
= I40IWQP_SW_MIN_WQSIZE
;
397 for (wr_ring_size
--; scount
<= 16; scount
*= 2)
398 wr_ring_size
|= wr_ring_size
>> scount
;
399 return ++wr_ring_size
;
403 * i40iw_get_pbl - Retrieve pbl from a list given a virtual
405 * @va: user virtual address
406 * @pbl_list: pbl list to search in (QP's or CQ's)
408 static struct i40iw_pbl
*i40iw_get_pbl(unsigned long va
,
409 struct list_head
*pbl_list
)
411 struct i40iw_pbl
*iwpbl
;
413 list_for_each_entry(iwpbl
, pbl_list
, list
) {
414 if (iwpbl
->user_base
== va
) {
415 list_del(&iwpbl
->list
);
423 * i40iw_free_qp_resources - free up memory resources for qp
424 * @iwdev: iwarp device
425 * @iwqp: qp ptr (user or kernel)
426 * @qp_num: qp number assigned
428 void i40iw_free_qp_resources(struct i40iw_device
*iwdev
,
429 struct i40iw_qp
*iwqp
,
432 i40iw_dealloc_push_page(iwdev
, &iwqp
->sc_qp
);
434 i40iw_free_resource(iwdev
, iwdev
->allocated_qps
, qp_num
);
435 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->q2_ctx_mem
);
436 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwqp
->kqp
.dma_mem
);
437 kfree(iwqp
->kqp
.wrid_mem
);
438 iwqp
->kqp
.wrid_mem
= NULL
;
439 kfree(iwqp
->allocated_buffer
);
443 * i40iw_clean_cqes - clean cq entries for qp
444 * @iwqp: qp ptr (user or kernel)
447 static void i40iw_clean_cqes(struct i40iw_qp
*iwqp
, struct i40iw_cq
*iwcq
)
449 struct i40iw_cq_uk
*ukcq
= &iwcq
->sc_cq
.cq_uk
;
451 ukcq
->ops
.iw_cq_clean(&iwqp
->sc_qp
.qp_uk
, ukcq
);
455 * i40iw_destroy_qp - destroy qp
456 * @ibqp: qp's ib pointer also to get to device's qp address
458 static int i40iw_destroy_qp(struct ib_qp
*ibqp
)
460 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
464 if (iwqp
->ibqp_state
>= IB_QPS_INIT
&& iwqp
->ibqp_state
< IB_QPS_RTS
)
465 i40iw_next_iw_state(iwqp
, I40IW_QP_STATE_ERROR
, 0, 0, 0);
467 if (!iwqp
->user_mode
) {
469 i40iw_clean_cqes(iwqp
, iwqp
->iwscq
);
470 if (iwqp
->iwrcq
!= iwqp
->iwscq
)
471 i40iw_clean_cqes(iwqp
, iwqp
->iwrcq
);
475 i40iw_rem_ref(&iwqp
->ibqp
);
480 * i40iw_setup_virt_qp - setup for allocation of virtual qp
483 * @init_info: initialize info to return
485 static int i40iw_setup_virt_qp(struct i40iw_device
*iwdev
,
486 struct i40iw_qp
*iwqp
,
487 struct i40iw_qp_init_info
*init_info
)
489 struct i40iw_pbl
*iwpbl
= iwqp
->iwpbl
;
490 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
492 iwqp
->page
= qpmr
->sq_page
;
493 init_info
->shadow_area_pa
= cpu_to_le64(qpmr
->shadow
);
494 if (iwpbl
->pbl_allocated
) {
495 init_info
->virtual_map
= true;
496 init_info
->sq_pa
= qpmr
->sq_pbl
.idx
;
497 init_info
->rq_pa
= qpmr
->rq_pbl
.idx
;
499 init_info
->sq_pa
= qpmr
->sq_pbl
.addr
;
500 init_info
->rq_pa
= qpmr
->rq_pbl
.addr
;
506 * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
507 * @iwdev: iwarp device
508 * @iwqp: qp ptr (user or kernel)
509 * @info: initialize info to return
511 static int i40iw_setup_kmode_qp(struct i40iw_device
*iwdev
,
512 struct i40iw_qp
*iwqp
,
513 struct i40iw_qp_init_info
*info
)
515 struct i40iw_dma_mem
*mem
= &iwqp
->kqp
.dma_mem
;
516 u32 sqdepth
, rqdepth
;
517 u32 sq_size
, rq_size
;
520 enum i40iw_status_code status
;
521 struct i40iw_qp_uk_init_info
*ukinfo
= &info
->qp_uk_init_info
;
523 ukinfo
->max_sq_frag_cnt
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
525 sq_size
= i40iw_qp_roundup(ukinfo
->sq_size
+ 1);
526 rq_size
= i40iw_qp_roundup(ukinfo
->rq_size
+ 1);
528 status
= i40iw_get_wqe_shift(sq_size
, ukinfo
->max_sq_frag_cnt
, ukinfo
->max_inline_data
, &sqshift
);
530 status
= i40iw_get_wqe_shift(rq_size
, ukinfo
->max_rq_frag_cnt
, 0, &rqshift
);
535 sqdepth
= sq_size
<< sqshift
;
536 rqdepth
= rq_size
<< rqshift
;
538 size
= sqdepth
* sizeof(struct i40iw_sq_uk_wr_trk_info
) + (rqdepth
<< 3);
539 iwqp
->kqp
.wrid_mem
= kzalloc(size
, GFP_KERNEL
);
541 ukinfo
->sq_wrtrk_array
= (struct i40iw_sq_uk_wr_trk_info
*)iwqp
->kqp
.wrid_mem
;
542 if (!ukinfo
->sq_wrtrk_array
)
545 ukinfo
->rq_wrid_array
= (u64
*)&ukinfo
->sq_wrtrk_array
[sqdepth
];
547 size
= (sqdepth
+ rqdepth
) * I40IW_QP_WQE_MIN_SIZE
;
548 size
+= (I40IW_SHADOW_AREA_SIZE
<< 3);
550 status
= i40iw_allocate_dma_mem(iwdev
->sc_dev
.hw
, mem
, size
, 256);
552 kfree(ukinfo
->sq_wrtrk_array
);
553 ukinfo
->sq_wrtrk_array
= NULL
;
557 ukinfo
->sq
= mem
->va
;
558 info
->sq_pa
= mem
->pa
;
560 ukinfo
->rq
= &ukinfo
->sq
[sqdepth
];
561 info
->rq_pa
= info
->sq_pa
+ (sqdepth
* I40IW_QP_WQE_MIN_SIZE
);
563 ukinfo
->shadow_area
= ukinfo
->rq
[rqdepth
].elem
;
564 info
->shadow_area_pa
= info
->rq_pa
+ (rqdepth
* I40IW_QP_WQE_MIN_SIZE
);
566 ukinfo
->sq_size
= sq_size
;
567 ukinfo
->rq_size
= rq_size
;
568 ukinfo
->qp_id
= iwqp
->ibqp
.qp_num
;
573 * i40iw_create_qp - create qp
575 * @init_attr: attributes for qp
576 * @udata: user data for create qp
578 static struct ib_qp
*i40iw_create_qp(struct ib_pd
*ibpd
,
579 struct ib_qp_init_attr
*init_attr
,
580 struct ib_udata
*udata
)
582 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
583 struct i40iw_device
*iwdev
= to_iwdev(ibpd
->device
);
584 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
585 struct i40iw_qp
*iwqp
;
586 struct i40iw_ucontext
*ucontext
;
587 struct i40iw_create_qp_req req
;
588 struct i40iw_create_qp_resp uresp
;
591 enum i40iw_status_code ret
;
595 struct i40iw_sc_qp
*qp
;
596 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
597 struct i40iw_qp_init_info init_info
;
598 struct i40iw_create_qp_info
*qp_info
;
599 struct i40iw_cqp_request
*cqp_request
;
600 struct cqp_commands_info
*cqp_info
;
602 struct i40iw_qp_host_ctx_info
*ctx_info
;
603 struct i40iwarp_offload_info
*iwarp_info
;
606 if (init_attr
->create_flags
)
607 return ERR_PTR(-EINVAL
);
608 if (init_attr
->cap
.max_inline_data
> I40IW_MAX_INLINE_DATA_SIZE
)
609 init_attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
611 if (init_attr
->cap
.max_send_sge
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
612 init_attr
->cap
.max_send_sge
= I40IW_MAX_WQ_FRAGMENT_COUNT
;
614 memset(&init_info
, 0, sizeof(init_info
));
616 sq_size
= init_attr
->cap
.max_send_wr
;
617 rq_size
= init_attr
->cap
.max_recv_wr
;
619 init_info
.qp_uk_init_info
.sq_size
= sq_size
;
620 init_info
.qp_uk_init_info
.rq_size
= rq_size
;
621 init_info
.qp_uk_init_info
.max_sq_frag_cnt
= init_attr
->cap
.max_send_sge
;
622 init_info
.qp_uk_init_info
.max_rq_frag_cnt
= init_attr
->cap
.max_recv_sge
;
623 init_info
.qp_uk_init_info
.max_inline_data
= init_attr
->cap
.max_inline_data
;
625 mem
= kzalloc(sizeof(*iwqp
), GFP_KERNEL
);
627 return ERR_PTR(-ENOMEM
);
629 iwqp
= (struct i40iw_qp
*)mem
;
631 qp
->back_qp
= (void *)iwqp
;
632 qp
->push_idx
= I40IW_INVALID_PUSH_PAGE_INDEX
;
634 iwqp
->ctx_info
.iwarp_info
= &iwqp
->iwarp_info
;
636 if (i40iw_allocate_dma_mem(dev
->hw
,
638 I40IW_Q2_BUFFER_SIZE
+ I40IW_QP_CTX_SIZE
,
640 i40iw_pr_err("dma_mem failed\n");
645 init_info
.q2
= iwqp
->q2_ctx_mem
.va
;
646 init_info
.q2_pa
= iwqp
->q2_ctx_mem
.pa
;
648 init_info
.host_ctx
= (void *)init_info
.q2
+ I40IW_Q2_BUFFER_SIZE
;
649 init_info
.host_ctx_pa
= init_info
.q2_pa
+ I40IW_Q2_BUFFER_SIZE
;
651 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_qps
, iwdev
->max_qp
,
652 &qp_num
, &iwdev
->next_qp
);
654 i40iw_pr_err("qp resource\n");
658 iwqp
->allocated_buffer
= mem
;
661 iwqp
->ibqp
.qp_num
= qp_num
;
663 iwqp
->iwscq
= to_iwcq(init_attr
->send_cq
);
664 iwqp
->iwrcq
= to_iwcq(init_attr
->recv_cq
);
666 iwqp
->host_ctx
.va
= init_info
.host_ctx
;
667 iwqp
->host_ctx
.pa
= init_info
.host_ctx_pa
;
668 iwqp
->host_ctx
.size
= I40IW_QP_CTX_SIZE
;
670 init_info
.pd
= &iwpd
->sc_pd
;
671 init_info
.qp_uk_init_info
.qp_id
= iwqp
->ibqp
.qp_num
;
672 iwqp
->ctx_info
.qp_compl_ctx
= (uintptr_t)qp
;
674 if (init_attr
->qp_type
!= IB_QPT_RC
) {
678 if (iwdev
->push_mode
)
679 i40iw_alloc_push_page(iwdev
, qp
);
681 err_code
= ib_copy_from_udata(&req
, udata
, sizeof(req
));
683 i40iw_pr_err("ib_copy_from_data\n");
686 iwqp
->ctx_info
.qp_compl_ctx
= req
.user_compl_ctx
;
687 if (ibpd
->uobject
&& ibpd
->uobject
->context
) {
689 ucontext
= to_ucontext(ibpd
->uobject
->context
);
691 if (req
.user_wqe_buffers
) {
693 &ucontext
->qp_reg_mem_list_lock
, flags
);
694 iwqp
->iwpbl
= i40iw_get_pbl(
695 (unsigned long)req
.user_wqe_buffers
,
696 &ucontext
->qp_reg_mem_list
);
697 spin_unlock_irqrestore(
698 &ucontext
->qp_reg_mem_list_lock
, flags
);
702 i40iw_pr_err("no pbl info\n");
707 err_code
= i40iw_setup_virt_qp(iwdev
, iwqp
, &init_info
);
709 err_code
= i40iw_setup_kmode_qp(iwdev
, iwqp
, &init_info
);
713 i40iw_pr_err("setup qp failed\n");
717 init_info
.type
= I40IW_QP_TYPE_IWARP
;
718 ret
= dev
->iw_priv_qp_ops
->qp_init(qp
, &init_info
);
721 i40iw_pr_err("qp_init fail\n");
724 ctx_info
= &iwqp
->ctx_info
;
725 iwarp_info
= &iwqp
->iwarp_info
;
726 iwarp_info
->rd_enable
= true;
727 iwarp_info
->wr_rdresp_en
= true;
728 if (!iwqp
->user_mode
)
729 iwarp_info
->priv_mode_en
= true;
730 iwarp_info
->ddp_ver
= 1;
731 iwarp_info
->rdmap_ver
= 1;
733 ctx_info
->iwarp_info_valid
= true;
734 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
735 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
736 if (qp
->push_idx
== I40IW_INVALID_PUSH_PAGE_INDEX
) {
737 ctx_info
->push_mode_en
= false;
739 ctx_info
->push_mode_en
= true;
740 ctx_info
->push_idx
= qp
->push_idx
;
743 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
744 (u64
*)iwqp
->host_ctx
.va
,
746 ctx_info
->iwarp_info_valid
= false;
747 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
752 cqp_info
= &cqp_request
->info
;
753 qp_info
= &cqp_request
->info
.in
.u
.qp_create
.info
;
755 memset(qp_info
, 0, sizeof(*qp_info
));
757 qp_info
->cq_num_valid
= true;
758 qp_info
->next_iwarp_state
= I40IW_QP_STATE_IDLE
;
760 cqp_info
->cqp_cmd
= OP_QP_CREATE
;
761 cqp_info
->post_sq
= 1;
762 cqp_info
->in
.u
.qp_create
.qp
= qp
;
763 cqp_info
->in
.u
.qp_create
.scratch
= (uintptr_t)cqp_request
;
764 ret
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
766 i40iw_pr_err("CQP-OP QP create fail");
771 i40iw_add_ref(&iwqp
->ibqp
);
772 spin_lock_init(&iwqp
->lock
);
773 iwqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
774 iwdev
->qp_table
[qp_num
] = iwqp
;
775 i40iw_add_pdusecount(iwqp
->iwpd
);
776 if (ibpd
->uobject
&& udata
) {
777 memset(&uresp
, 0, sizeof(uresp
));
778 uresp
.actual_sq_size
= sq_size
;
779 uresp
.actual_rq_size
= rq_size
;
780 uresp
.qp_id
= qp_num
;
781 uresp
.push_idx
= qp
->push_idx
;
782 err_code
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
784 i40iw_pr_err("copy_to_udata failed\n");
785 i40iw_destroy_qp(&iwqp
->ibqp
);
786 /* let the completion of the qp destroy free the qp */
787 return ERR_PTR(err_code
);
793 i40iw_free_qp_resources(iwdev
, iwqp
, qp_num
);
795 return ERR_PTR(err_code
);
799 * i40iw_query - query qp attributes
801 * @attr: attributes pointer
802 * @attr_mask: Not used
803 * @init_attr: qp attributes to return
805 static int i40iw_query_qp(struct ib_qp
*ibqp
,
806 struct ib_qp_attr
*attr
,
808 struct ib_qp_init_attr
*init_attr
)
810 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
811 struct i40iw_sc_qp
*qp
= &iwqp
->sc_qp
;
813 attr
->qp_access_flags
= 0;
814 attr
->cap
.max_send_wr
= qp
->qp_uk
.sq_size
;
815 attr
->cap
.max_recv_wr
= qp
->qp_uk
.rq_size
;
816 attr
->cap
.max_recv_sge
= 1;
817 attr
->cap
.max_inline_data
= I40IW_MAX_INLINE_DATA_SIZE
;
818 init_attr
->event_handler
= iwqp
->ibqp
.event_handler
;
819 init_attr
->qp_context
= iwqp
->ibqp
.qp_context
;
820 init_attr
->send_cq
= iwqp
->ibqp
.send_cq
;
821 init_attr
->recv_cq
= iwqp
->ibqp
.recv_cq
;
822 init_attr
->srq
= iwqp
->ibqp
.srq
;
823 init_attr
->cap
= attr
->cap
;
828 * i40iw_hw_modify_qp - setup cqp for modify qp
829 * @iwdev: iwarp device
830 * @iwqp: qp ptr (user or kernel)
831 * @info: info for modify qp
832 * @wait: flag to wait or not for modify qp completion
834 void i40iw_hw_modify_qp(struct i40iw_device
*iwdev
, struct i40iw_qp
*iwqp
,
835 struct i40iw_modify_qp_info
*info
, bool wait
)
837 enum i40iw_status_code status
;
838 struct i40iw_cqp_request
*cqp_request
;
839 struct cqp_commands_info
*cqp_info
;
840 struct i40iw_modify_qp_info
*m_info
;
842 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, wait
);
846 cqp_info
= &cqp_request
->info
;
847 m_info
= &cqp_info
->in
.u
.qp_modify
.info
;
848 memcpy(m_info
, info
, sizeof(*m_info
));
849 cqp_info
->cqp_cmd
= OP_QP_MODIFY
;
850 cqp_info
->post_sq
= 1;
851 cqp_info
->in
.u
.qp_modify
.qp
= &iwqp
->sc_qp
;
852 cqp_info
->in
.u
.qp_modify
.scratch
= (uintptr_t)cqp_request
;
853 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
855 i40iw_pr_err("CQP-OP Modify QP fail");
859 * i40iw_modify_qp - modify qp request
860 * @ibqp: qp's pointer for modify
861 * @attr: access attributes
862 * @attr_mask: state mask
865 int i40iw_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
866 int attr_mask
, struct ib_udata
*udata
)
868 struct i40iw_qp
*iwqp
= to_iwqp(ibqp
);
869 struct i40iw_device
*iwdev
= iwqp
->iwdev
;
870 struct i40iw_qp_host_ctx_info
*ctx_info
;
871 struct i40iwarp_offload_info
*iwarp_info
;
872 struct i40iw_modify_qp_info info
;
873 u8 issue_modify_qp
= 0;
878 memset(&info
, 0, sizeof(info
));
879 ctx_info
= &iwqp
->ctx_info
;
880 iwarp_info
= &iwqp
->iwarp_info
;
882 spin_lock_irqsave(&iwqp
->lock
, flags
);
884 if (attr_mask
& IB_QP_STATE
) {
885 switch (attr
->qp_state
) {
888 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_IDLE
) {
892 if (iwqp
->iwarp_state
== I40IW_QP_STATE_INVALID
) {
893 info
.next_iwarp_state
= I40IW_QP_STATE_IDLE
;
898 if ((iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) ||
905 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_ESTABLISHED
;
907 info
.next_iwarp_state
= I40IW_QP_STATE_RTS
;
908 info
.tcp_ctx_valid
= true;
909 info
.ord_valid
= true;
910 info
.arp_cache_idx_valid
= true;
911 info
.cq_num_valid
= true;
914 if (iwqp
->hw_iwarp_state
> (u32
)I40IW_QP_STATE_RTS
) {
918 if ((iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_CLOSING
) ||
919 (iwqp
->iwarp_state
< (u32
)I40IW_QP_STATE_RTS
)) {
923 if (iwqp
->iwarp_state
> (u32
)I40IW_QP_STATE_CLOSING
) {
927 info
.next_iwarp_state
= I40IW_QP_STATE_CLOSING
;
931 if (iwqp
->iwarp_state
>= (u32
)I40IW_QP_STATE_TERMINATE
) {
935 info
.next_iwarp_state
= I40IW_QP_STATE_TERMINATE
;
940 if (iwqp
->iwarp_state
== (u32
)I40IW_QP_STATE_ERROR
) {
944 if (iwqp
->sc_qp
.term_flags
)
945 del_timer(&iwqp
->terminate_timer
);
946 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
947 if ((iwqp
->hw_tcp_state
> I40IW_TCP_STATE_CLOSED
) &&
949 (iwqp
->hw_tcp_state
!= I40IW_TCP_STATE_TIME_WAIT
))
950 info
.reset_tcp_conn
= true;
954 info
.next_iwarp_state
= I40IW_QP_STATE_ERROR
;
961 iwqp
->ibqp_state
= attr
->qp_state
;
964 iwqp
->iwarp_state
= info
.next_iwarp_state
;
966 info
.next_iwarp_state
= iwqp
->iwarp_state
;
968 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
969 ctx_info
->iwarp_info_valid
= true;
970 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
)
971 iwarp_info
->wr_rdresp_en
= true;
972 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
973 iwarp_info
->wr_rdresp_en
= true;
974 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
)
975 iwarp_info
->rd_enable
= true;
976 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
)
977 iwarp_info
->bind_en
= true;
979 if (iwqp
->user_mode
) {
980 iwarp_info
->rd_enable
= true;
981 iwarp_info
->wr_rdresp_en
= true;
982 iwarp_info
->priv_mode_en
= false;
986 if (ctx_info
->iwarp_info_valid
) {
987 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
990 ctx_info
->send_cq_num
= iwqp
->iwscq
->sc_cq
.cq_uk
.cq_id
;
991 ctx_info
->rcv_cq_num
= iwqp
->iwrcq
->sc_cq
.cq_uk
.cq_id
;
992 ret
= dev
->iw_priv_qp_ops
->qp_setctx(&iwqp
->sc_qp
,
993 (u64
*)iwqp
->host_ctx
.va
,
996 i40iw_pr_err("setting QP context\n");
1002 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1004 if (issue_modify_qp
)
1005 i40iw_hw_modify_qp(iwdev
, iwqp
, &info
, true);
1007 if (issue_modify_qp
&& (iwqp
->ibqp_state
> IB_QPS_RTS
)) {
1009 if (iwqp
->cm_id
&& iwqp
->hw_tcp_state
) {
1010 spin_lock_irqsave(&iwqp
->lock
, flags
);
1011 iwqp
->hw_tcp_state
= I40IW_TCP_STATE_CLOSED
;
1012 iwqp
->last_aeq
= I40IW_AE_RESET_SENT
;
1013 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1019 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1024 * cq_free_resources - free up recources for cq
1025 * @iwdev: iwarp device
1028 static void cq_free_resources(struct i40iw_device
*iwdev
, struct i40iw_cq
*iwcq
)
1030 struct i40iw_sc_cq
*cq
= &iwcq
->sc_cq
;
1032 if (!iwcq
->user_mode
)
1033 i40iw_free_dma_mem(iwdev
->sc_dev
.hw
, &iwcq
->kmem
);
1034 i40iw_free_resource(iwdev
, iwdev
->allocated_cqs
, cq
->cq_uk
.cq_id
);
1038 * cq_wq_destroy - send cq destroy cqp
1039 * @iwdev: iwarp device
1040 * @cq: hardware control cq
1042 static void cq_wq_destroy(struct i40iw_device
*iwdev
, struct i40iw_sc_cq
*cq
)
1044 enum i40iw_status_code status
;
1045 struct i40iw_cqp_request
*cqp_request
;
1046 struct cqp_commands_info
*cqp_info
;
1048 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1052 cqp_info
= &cqp_request
->info
;
1054 cqp_info
->cqp_cmd
= OP_CQ_DESTROY
;
1055 cqp_info
->post_sq
= 1;
1056 cqp_info
->in
.u
.cq_destroy
.cq
= cq
;
1057 cqp_info
->in
.u
.cq_destroy
.scratch
= (uintptr_t)cqp_request
;
1058 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1060 i40iw_pr_err("CQP-OP Destroy QP fail");
1064 * i40iw_destroy_cq - destroy cq
1065 * @ib_cq: cq pointer
1067 static int i40iw_destroy_cq(struct ib_cq
*ib_cq
)
1069 struct i40iw_cq
*iwcq
;
1070 struct i40iw_device
*iwdev
;
1071 struct i40iw_sc_cq
*cq
;
1074 i40iw_pr_err("ib_cq == NULL\n");
1078 iwcq
= to_iwcq(ib_cq
);
1079 iwdev
= to_iwdev(ib_cq
->device
);
1081 cq_wq_destroy(iwdev
, cq
);
1082 cq_free_resources(iwdev
, iwcq
);
1088 * i40iw_create_cq - create cq
1089 * @ibdev: device pointer from stack
1090 * @attr: attributes for cq
1091 * @context: user context created during alloc
1094 static struct ib_cq
*i40iw_create_cq(struct ib_device
*ibdev
,
1095 const struct ib_cq_init_attr
*attr
,
1096 struct ib_ucontext
*context
,
1097 struct ib_udata
*udata
)
1099 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
1100 struct i40iw_cq
*iwcq
;
1101 struct i40iw_pbl
*iwpbl
;
1103 struct i40iw_sc_cq
*cq
;
1104 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1105 struct i40iw_cq_init_info info
;
1106 enum i40iw_status_code status
;
1107 struct i40iw_cqp_request
*cqp_request
;
1108 struct cqp_commands_info
*cqp_info
;
1109 struct i40iw_cq_uk_init_info
*ukinfo
= &info
.cq_uk_init_info
;
1110 unsigned long flags
;
1112 int entries
= attr
->cqe
;
1114 if (entries
> iwdev
->max_cqe
)
1115 return ERR_PTR(-EINVAL
);
1117 iwcq
= kzalloc(sizeof(*iwcq
), GFP_KERNEL
);
1119 return ERR_PTR(-ENOMEM
);
1121 memset(&info
, 0, sizeof(info
));
1123 err_code
= i40iw_alloc_resource(iwdev
, iwdev
->allocated_cqs
,
1124 iwdev
->max_cq
, &cq_num
,
1130 cq
->back_cq
= (void *)iwcq
;
1131 spin_lock_init(&iwcq
->lock
);
1134 ukinfo
->cq_size
= max(entries
, 4);
1135 ukinfo
->cq_id
= cq_num
;
1136 iwcq
->ibcq
.cqe
= info
.cq_uk_init_info
.cq_size
;
1139 info
.ceq_id_valid
= true;
1141 info
.type
= I40IW_CQ_TYPE_IWARP
;
1143 struct i40iw_ucontext
*ucontext
;
1144 struct i40iw_create_cq_req req
;
1145 struct i40iw_cq_mr
*cqmr
;
1147 memset(&req
, 0, sizeof(req
));
1148 iwcq
->user_mode
= true;
1149 ucontext
= to_ucontext(context
);
1150 if (ib_copy_from_udata(&req
, udata
, sizeof(struct i40iw_create_cq_req
)))
1151 goto cq_free_resources
;
1153 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1154 iwpbl
= i40iw_get_pbl((unsigned long)req
.user_cq_buffer
,
1155 &ucontext
->cq_reg_mem_list
);
1156 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1159 goto cq_free_resources
;
1162 iwcq
->iwpbl
= iwpbl
;
1163 iwcq
->cq_mem_size
= 0;
1164 cqmr
= &iwpbl
->cq_mr
;
1165 info
.shadow_area_pa
= cpu_to_le64(cqmr
->shadow
);
1166 if (iwpbl
->pbl_allocated
) {
1167 info
.virtual_map
= true;
1168 info
.pbl_chunk_size
= 1;
1169 info
.first_pm_pbl_idx
= cqmr
->cq_pbl
.idx
;
1171 info
.cq_base_pa
= cqmr
->cq_pbl
.addr
;
1174 /* Kmode allocations */
1178 rsize
= info
.cq_uk_init_info
.cq_size
* sizeof(struct i40iw_cqe
);
1179 rsize
= round_up(rsize
, 256);
1180 shadow
= I40IW_SHADOW_AREA_SIZE
<< 3;
1181 status
= i40iw_allocate_dma_mem(dev
->hw
, &iwcq
->kmem
,
1182 rsize
+ shadow
, 256);
1185 goto cq_free_resources
;
1187 ukinfo
->cq_base
= iwcq
->kmem
.va
;
1188 info
.cq_base_pa
= iwcq
->kmem
.pa
;
1189 info
.shadow_area_pa
= info
.cq_base_pa
+ rsize
;
1190 ukinfo
->shadow_area
= iwcq
->kmem
.va
+ rsize
;
1193 if (dev
->iw_priv_cq_ops
->cq_init(cq
, &info
)) {
1194 i40iw_pr_err("init cq fail\n");
1196 goto cq_free_resources
;
1199 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1202 goto cq_free_resources
;
1205 cqp_info
= &cqp_request
->info
;
1206 cqp_info
->cqp_cmd
= OP_CQ_CREATE
;
1207 cqp_info
->post_sq
= 1;
1208 cqp_info
->in
.u
.cq_create
.cq
= cq
;
1209 cqp_info
->in
.u
.cq_create
.scratch
= (uintptr_t)cqp_request
;
1210 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1212 i40iw_pr_err("CQP-OP Create QP fail");
1214 goto cq_free_resources
;
1218 struct i40iw_create_cq_resp resp
;
1220 memset(&resp
, 0, sizeof(resp
));
1221 resp
.cq_id
= info
.cq_uk_init_info
.cq_id
;
1222 resp
.cq_size
= info
.cq_uk_init_info
.cq_size
;
1223 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
1224 i40iw_pr_err("copy to user data\n");
1230 return (struct ib_cq
*)iwcq
;
1233 cq_wq_destroy(iwdev
, cq
);
1235 cq_free_resources(iwdev
, iwcq
);
1238 return ERR_PTR(err_code
);
1242 * i40iw_get_user_access - get hw access from IB access
1243 * @acc: IB access to return hw access
1245 static inline u16
i40iw_get_user_access(int acc
)
1249 access
|= (acc
& IB_ACCESS_LOCAL_WRITE
) ? I40IW_ACCESS_FLAGS_LOCALWRITE
: 0;
1250 access
|= (acc
& IB_ACCESS_REMOTE_WRITE
) ? I40IW_ACCESS_FLAGS_REMOTEWRITE
: 0;
1251 access
|= (acc
& IB_ACCESS_REMOTE_READ
) ? I40IW_ACCESS_FLAGS_REMOTEREAD
: 0;
1252 access
|= (acc
& IB_ACCESS_MW_BIND
) ? I40IW_ACCESS_FLAGS_BIND_WINDOW
: 0;
1257 * i40iw_free_stag - free stag resource
1258 * @iwdev: iwarp device
1259 * @stag: stag to free
1261 static void i40iw_free_stag(struct i40iw_device
*iwdev
, u32 stag
)
1265 stag_idx
= (stag
& iwdev
->mr_stagmask
) >> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1266 i40iw_free_resource(iwdev
, iwdev
->allocated_mrs
, stag_idx
);
1270 * i40iw_create_stag - create random stag
1271 * @iwdev: iwarp device
1273 static u32
i40iw_create_stag(struct i40iw_device
*iwdev
)
1277 u32 next_stag_index
;
1283 get_random_bytes(&random
, sizeof(random
));
1284 consumer_key
= (u8
)random
;
1286 driver_key
= random
& ~iwdev
->mr_stagmask
;
1287 next_stag_index
= (random
& iwdev
->mr_stagmask
) >> 8;
1288 next_stag_index
%= iwdev
->max_mr
;
1290 ret
= i40iw_alloc_resource(iwdev
,
1291 iwdev
->allocated_mrs
, iwdev
->max_mr
,
1292 &stag_index
, &next_stag_index
);
1294 stag
= stag_index
<< I40IW_CQPSQ_STAG_IDX_SHIFT
;
1296 stag
+= (u32
)consumer_key
;
1302 * i40iw_next_pbl_addr - Get next pbl address
1303 * @palloc: Poiner to allocated pbles
1304 * @pbl: pointer to a pble
1305 * @pinfo: info pointer
1308 static inline u64
*i40iw_next_pbl_addr(struct i40iw_pble_alloc
*palloc
,
1310 struct i40iw_pble_info
**pinfo
,
1314 if ((!(*pinfo
)) || (*idx
!= (*pinfo
)->cnt
))
1318 return (u64
*)(*pinfo
)->addr
;
1322 * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally
1323 * @iwmr: iwmr for IB's user page addresses
1324 * @pbl: ple pointer to save 1 level or 0 level pble
1325 * @level: indicated level 0, 1 or 2
1327 static void i40iw_copy_user_pgaddrs(struct i40iw_mr
*iwmr
,
1329 enum i40iw_pble_level level
)
1331 struct ib_umem
*region
= iwmr
->region
;
1332 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1333 int chunk_pages
, entry
, pg_shift
, i
;
1334 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1335 struct i40iw_pble_info
*pinfo
;
1336 struct scatterlist
*sg
;
1339 pinfo
= (level
== I40IW_LEVEL_1
) ? NULL
: palloc
->level2
.leaf
;
1340 pg_shift
= ffs(region
->page_size
) - 1;
1341 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
1342 chunk_pages
= sg_dma_len(sg
) >> pg_shift
;
1343 if ((iwmr
->type
== IW_MEMREG_TYPE_QP
) &&
1344 !iwpbl
->qp_mr
.sq_page
)
1345 iwpbl
->qp_mr
.sq_page
= sg_page(sg
);
1346 for (i
= 0; i
< chunk_pages
; i
++) {
1347 *pbl
= cpu_to_le64(sg_dma_address(sg
) + region
->page_size
* i
);
1348 pbl
= i40iw_next_pbl_addr(palloc
, pbl
, &pinfo
, &idx
);
1354 * i40iw_setup_pbles - copy user pg address to pble's
1355 * @iwdev: iwarp device
1356 * @iwmr: mr pointer for this memory registration
1357 * @use_pbles: flag if to use pble's or memory (level 0)
1359 static int i40iw_setup_pbles(struct i40iw_device
*iwdev
,
1360 struct i40iw_mr
*iwmr
,
1363 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1364 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1365 struct i40iw_pble_info
*pinfo
;
1367 enum i40iw_status_code status
;
1368 enum i40iw_pble_level level
= I40IW_LEVEL_1
;
1370 if (!use_pbles
&& (iwmr
->page_cnt
> MAX_SAVE_PAGE_ADDRS
))
1374 mutex_lock(&iwdev
->pbl_mutex
);
1375 status
= i40iw_get_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
, palloc
, iwmr
->page_cnt
);
1376 mutex_unlock(&iwdev
->pbl_mutex
);
1380 iwpbl
->pbl_allocated
= true;
1381 level
= palloc
->level
;
1382 pinfo
= (level
== I40IW_LEVEL_1
) ? &palloc
->level1
: palloc
->level2
.leaf
;
1383 pbl
= (u64
*)pinfo
->addr
;
1385 pbl
= iwmr
->pgaddrmem
;
1388 i40iw_copy_user_pgaddrs(iwmr
, pbl
, level
);
1393 * i40iw_handle_q_mem - handle memory for qp and cq
1394 * @iwdev: iwarp device
1395 * @req: information for q memory management
1396 * @iwpbl: pble struct
1397 * @use_pbles: flag to use pble
1399 static int i40iw_handle_q_mem(struct i40iw_device
*iwdev
,
1400 struct i40iw_mem_reg_req
*req
,
1401 struct i40iw_pbl
*iwpbl
,
1404 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1405 struct i40iw_mr
*iwmr
= iwpbl
->iwmr
;
1406 struct i40iw_qp_mr
*qpmr
= &iwpbl
->qp_mr
;
1407 struct i40iw_cq_mr
*cqmr
= &iwpbl
->cq_mr
;
1408 struct i40iw_hmc_pble
*hmc_p
;
1409 u64
*arr
= iwmr
->pgaddrmem
;
1413 total
= req
->sq_pages
+ req
->rq_pages
+ req
->cq_pages
;
1415 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1418 if (use_pbles
&& (palloc
->level
!= I40IW_LEVEL_1
)) {
1419 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1420 iwpbl
->pbl_allocated
= false;
1425 arr
= (u64
*)palloc
->level1
.addr
;
1426 if (req
->reg_type
== IW_MEMREG_TYPE_QP
) {
1427 hmc_p
= &qpmr
->sq_pbl
;
1428 qpmr
->shadow
= (dma_addr_t
)arr
[total
];
1430 hmc_p
->idx
= palloc
->level1
.idx
;
1431 hmc_p
= &qpmr
->rq_pbl
;
1432 hmc_p
->idx
= palloc
->level1
.idx
+ req
->sq_pages
;
1434 hmc_p
->addr
= arr
[0];
1435 hmc_p
= &qpmr
->rq_pbl
;
1436 hmc_p
->addr
= arr
[1];
1439 hmc_p
= &cqmr
->cq_pbl
;
1440 cqmr
->shadow
= (dma_addr_t
)arr
[total
];
1442 hmc_p
->idx
= palloc
->level1
.idx
;
1444 hmc_p
->addr
= arr
[0];
1450 * i40iw_hwreg_mr - send cqp command for memory registration
1451 * @iwdev: iwarp device
1452 * @iwmr: iwarp mr pointer
1453 * @access: access for MR
1455 static int i40iw_hwreg_mr(struct i40iw_device
*iwdev
,
1456 struct i40iw_mr
*iwmr
,
1459 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1460 struct i40iw_reg_ns_stag_info
*stag_info
;
1461 struct i40iw_pd
*iwpd
= to_iwpd(iwmr
->ibmr
.pd
);
1462 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1463 enum i40iw_status_code status
;
1465 struct i40iw_cqp_request
*cqp_request
;
1466 struct cqp_commands_info
*cqp_info
;
1468 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1472 cqp_info
= &cqp_request
->info
;
1473 stag_info
= &cqp_info
->in
.u
.mr_reg_non_shared
.info
;
1474 memset(stag_info
, 0, sizeof(*stag_info
));
1475 stag_info
->va
= (void *)(unsigned long)iwpbl
->user_base
;
1476 stag_info
->stag_idx
= iwmr
->stag
>> I40IW_CQPSQ_STAG_IDX_SHIFT
;
1477 stag_info
->stag_key
= (u8
)iwmr
->stag
;
1478 stag_info
->total_len
= iwmr
->length
;
1479 stag_info
->access_rights
= access
;
1480 stag_info
->pd_id
= iwpd
->sc_pd
.pd_id
;
1481 stag_info
->addr_type
= I40IW_ADDR_TYPE_VA_BASED
;
1483 if (iwmr
->page_cnt
> 1) {
1484 if (palloc
->level
== I40IW_LEVEL_1
) {
1485 stag_info
->first_pm_pbl_index
= palloc
->level1
.idx
;
1486 stag_info
->chunk_size
= 1;
1488 stag_info
->first_pm_pbl_index
= palloc
->level2
.root
.idx
;
1489 stag_info
->chunk_size
= 3;
1492 stag_info
->reg_addr_pa
= iwmr
->pgaddrmem
[0];
1495 cqp_info
->cqp_cmd
= OP_MR_REG_NON_SHARED
;
1496 cqp_info
->post_sq
= 1;
1497 cqp_info
->in
.u
.mr_reg_non_shared
.dev
= &iwdev
->sc_dev
;
1498 cqp_info
->in
.u
.mr_reg_non_shared
.scratch
= (uintptr_t)cqp_request
;
1500 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1503 i40iw_pr_err("CQP-OP MR Reg fail");
1509 * i40iw_reg_user_mr - Register a user memory region
1511 * @start: virtual start address
1512 * @length: length of mr
1513 * @virt: virtual address
1514 * @acc: access of mr
1517 static struct ib_mr
*i40iw_reg_user_mr(struct ib_pd
*pd
,
1522 struct ib_udata
*udata
)
1524 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1525 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1526 struct i40iw_ucontext
*ucontext
;
1527 struct i40iw_pble_alloc
*palloc
;
1528 struct i40iw_pbl
*iwpbl
;
1529 struct i40iw_mr
*iwmr
;
1530 struct ib_umem
*region
;
1531 struct i40iw_mem_reg_req req
;
1536 bool use_pbles
= false;
1537 unsigned long flags
;
1540 if (length
> I40IW_MAX_MR_SIZE
)
1541 return ERR_PTR(-EINVAL
);
1542 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
1544 return (struct ib_mr
*)region
;
1546 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
1547 ib_umem_release(region
);
1548 return ERR_PTR(-EFAULT
);
1551 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1553 ib_umem_release(region
);
1554 return ERR_PTR(-ENOMEM
);
1557 iwpbl
= &iwmr
->iwpbl
;
1559 iwmr
->region
= region
;
1561 iwmr
->ibmr
.device
= pd
->device
;
1562 ucontext
= to_ucontext(pd
->uobject
->context
);
1563 region_length
= region
->length
+ (start
& 0xfff);
1564 pbl_depth
= region_length
>> 12;
1565 pbl_depth
+= (region_length
& (4096 - 1)) ? 1 : 0;
1566 iwmr
->length
= region
->length
;
1568 iwpbl
->user_base
= virt
;
1569 palloc
= &iwpbl
->pble_alloc
;
1571 iwmr
->type
= req
.reg_type
;
1572 iwmr
->page_cnt
= (u32
)pbl_depth
;
1574 switch (req
.reg_type
) {
1575 case IW_MEMREG_TYPE_QP
:
1576 use_pbles
= ((req
.sq_pages
+ req
.rq_pages
) > 2);
1577 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1580 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1581 list_add_tail(&iwpbl
->list
, &ucontext
->qp_reg_mem_list
);
1582 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1584 case IW_MEMREG_TYPE_CQ
:
1585 use_pbles
= (req
.cq_pages
> 1);
1586 err
= i40iw_handle_q_mem(iwdev
, &req
, iwpbl
, use_pbles
);
1590 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1591 list_add_tail(&iwpbl
->list
, &ucontext
->cq_reg_mem_list
);
1592 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1594 case IW_MEMREG_TYPE_MEM
:
1595 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1597 use_pbles
= (iwmr
->page_cnt
!= 1);
1598 err
= i40iw_setup_pbles(iwdev
, iwmr
, use_pbles
);
1602 access
|= i40iw_get_user_access(acc
);
1603 stag
= i40iw_create_stag(iwdev
);
1610 iwmr
->ibmr
.rkey
= stag
;
1611 iwmr
->ibmr
.lkey
= stag
;
1613 err
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1615 i40iw_free_stag(iwdev
, stag
);
1623 iwmr
->type
= req
.reg_type
;
1624 if (req
.reg_type
== IW_MEMREG_TYPE_MEM
)
1625 i40iw_add_pdusecount(iwpd
);
1629 if (palloc
->level
!= I40IW_LEVEL_0
)
1630 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1631 ib_umem_release(region
);
1633 return ERR_PTR(err
);
1637 * i40iw_reg_phys_mr - register kernel physical memory
1639 * @addr: physical address of memory to register
1640 * @size: size of memory to register
1641 * @acc: Access rights
1642 * @iova_start: start of virtual address for physical buffers
1644 struct ib_mr
*i40iw_reg_phys_mr(struct ib_pd
*pd
,
1650 struct i40iw_pd
*iwpd
= to_iwpd(pd
);
1651 struct i40iw_device
*iwdev
= to_iwdev(pd
->device
);
1652 struct i40iw_pbl
*iwpbl
;
1653 struct i40iw_mr
*iwmr
;
1654 enum i40iw_status_code status
;
1656 u16 access
= I40IW_ACCESS_FLAGS_LOCALREAD
;
1659 iwmr
= kzalloc(sizeof(*iwmr
), GFP_KERNEL
);
1661 return ERR_PTR(-ENOMEM
);
1663 iwmr
->ibmr
.device
= pd
->device
;
1664 iwpbl
= &iwmr
->iwpbl
;
1666 iwmr
->type
= IW_MEMREG_TYPE_MEM
;
1667 iwpbl
->user_base
= *iova_start
;
1668 stag
= i40iw_create_stag(iwdev
);
1673 access
|= i40iw_get_user_access(acc
);
1675 iwmr
->ibmr
.rkey
= stag
;
1676 iwmr
->ibmr
.lkey
= stag
;
1678 iwmr
->pgaddrmem
[0] = addr
;
1679 status
= i40iw_hwreg_mr(iwdev
, iwmr
, access
);
1681 i40iw_free_stag(iwdev
, stag
);
1686 i40iw_add_pdusecount(iwpd
);
1690 return ERR_PTR(ret
);
1694 * i40iw_get_dma_mr - register physical mem
1696 * @acc: access for memory
1698 static struct ib_mr
*i40iw_get_dma_mr(struct ib_pd
*pd
, int acc
)
1702 return i40iw_reg_phys_mr(pd
, 0, 0xffffffffffULL
, acc
, &kva
);
1706 * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
1707 * @iwmr: iwmr for IB's user page addresses
1708 * @ucontext: ptr to user context
1710 static void i40iw_del_memlist(struct i40iw_mr
*iwmr
,
1711 struct i40iw_ucontext
*ucontext
)
1713 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1714 unsigned long flags
;
1716 switch (iwmr
->type
) {
1717 case IW_MEMREG_TYPE_CQ
:
1718 spin_lock_irqsave(&ucontext
->cq_reg_mem_list_lock
, flags
);
1719 if (!list_empty(&ucontext
->cq_reg_mem_list
))
1720 list_del(&iwpbl
->list
);
1721 spin_unlock_irqrestore(&ucontext
->cq_reg_mem_list_lock
, flags
);
1723 case IW_MEMREG_TYPE_QP
:
1724 spin_lock_irqsave(&ucontext
->qp_reg_mem_list_lock
, flags
);
1725 if (!list_empty(&ucontext
->qp_reg_mem_list
))
1726 list_del(&iwpbl
->list
);
1727 spin_unlock_irqrestore(&ucontext
->qp_reg_mem_list_lock
, flags
);
1735 * i40iw_dereg_mr - deregister mr
1736 * @ib_mr: mr ptr for dereg
1738 static int i40iw_dereg_mr(struct ib_mr
*ib_mr
)
1740 struct ib_pd
*ibpd
= ib_mr
->pd
;
1741 struct i40iw_pd
*iwpd
= to_iwpd(ibpd
);
1742 struct i40iw_mr
*iwmr
= to_iwmr(ib_mr
);
1743 struct i40iw_device
*iwdev
= to_iwdev(ib_mr
->device
);
1744 enum i40iw_status_code status
;
1745 struct i40iw_dealloc_stag_info
*info
;
1746 struct i40iw_pbl
*iwpbl
= &iwmr
->iwpbl
;
1747 struct i40iw_pble_alloc
*palloc
= &iwpbl
->pble_alloc
;
1748 struct i40iw_cqp_request
*cqp_request
;
1749 struct cqp_commands_info
*cqp_info
;
1753 ib_umem_release(iwmr
->region
);
1755 if (iwmr
->type
!= IW_MEMREG_TYPE_MEM
) {
1756 if (ibpd
->uobject
) {
1757 struct i40iw_ucontext
*ucontext
;
1759 ucontext
= to_ucontext(ibpd
->uobject
->context
);
1760 i40iw_del_memlist(iwmr
, ucontext
);
1762 if (iwpbl
->pbl_allocated
)
1763 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1769 cqp_request
= i40iw_get_cqp_request(&iwdev
->cqp
, true);
1773 cqp_info
= &cqp_request
->info
;
1774 info
= &cqp_info
->in
.u
.dealloc_stag
.info
;
1775 memset(info
, 0, sizeof(*info
));
1777 info
->pd_id
= cpu_to_le32(iwpd
->sc_pd
.pd_id
& 0x00007fff);
1778 info
->stag_idx
= RS_64_1(ib_mr
->rkey
, I40IW_CQPSQ_STAG_IDX_SHIFT
);
1779 stag_idx
= info
->stag_idx
;
1781 if (iwpbl
->pbl_allocated
)
1782 info
->dealloc_pbl
= true;
1784 cqp_info
->cqp_cmd
= OP_DEALLOC_STAG
;
1785 cqp_info
->post_sq
= 1;
1786 cqp_info
->in
.u
.dealloc_stag
.dev
= &iwdev
->sc_dev
;
1787 cqp_info
->in
.u
.dealloc_stag
.scratch
= (uintptr_t)cqp_request
;
1788 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1790 i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx
);
1791 i40iw_rem_pdusecount(iwpd
, iwdev
);
1792 i40iw_free_stag(iwdev
, iwmr
->stag
);
1793 if (iwpbl
->pbl_allocated
)
1794 i40iw_free_pble(iwdev
->pble_rsrc
, palloc
);
1802 static ssize_t
i40iw_show_rev(struct device
*dev
,
1803 struct device_attribute
*attr
, char *buf
)
1805 struct i40iw_ib_device
*iwibdev
= container_of(dev
,
1806 struct i40iw_ib_device
,
1808 u32 hw_rev
= iwibdev
->iwdev
->sc_dev
.hw_rev
;
1810 return sprintf(buf
, "%x\n", hw_rev
);
1816 static ssize_t
i40iw_show_fw_ver(struct device
*dev
,
1817 struct device_attribute
*attr
, char *buf
)
1819 u32 firmware_version
= I40IW_FW_VERSION
;
1821 return sprintf(buf
, "%u.%u\n", firmware_version
,
1822 (firmware_version
& 0x000000ff));
1828 static ssize_t
i40iw_show_hca(struct device
*dev
,
1829 struct device_attribute
*attr
, char *buf
)
1831 return sprintf(buf
, "I40IW\n");
1837 static ssize_t
i40iw_show_board(struct device
*dev
,
1838 struct device_attribute
*attr
,
1841 return sprintf(buf
, "%.*s\n", 32, "I40IW Board ID");
1844 static DEVICE_ATTR(hw_rev
, S_IRUGO
, i40iw_show_rev
, NULL
);
1845 static DEVICE_ATTR(fw_ver
, S_IRUGO
, i40iw_show_fw_ver
, NULL
);
1846 static DEVICE_ATTR(hca_type
, S_IRUGO
, i40iw_show_hca
, NULL
);
1847 static DEVICE_ATTR(board_id
, S_IRUGO
, i40iw_show_board
, NULL
);
1849 static struct device_attribute
*i40iw_dev_attributes
[] = {
1857 * i40iw_copy_sg_list - copy sg list for qp
1858 * @sg_list: copied into sg_list
1859 * @sgl: copy from sgl
1860 * @num_sges: count of sg entries
1862 static void i40iw_copy_sg_list(struct i40iw_sge
*sg_list
, struct ib_sge
*sgl
, int num_sges
)
1866 for (i
= 0; (i
< num_sges
) && (i
< I40IW_MAX_WQ_FRAGMENT_COUNT
); i
++) {
1867 sg_list
[i
].tag_off
= sgl
[i
].addr
;
1868 sg_list
[i
].len
= sgl
[i
].length
;
1869 sg_list
[i
].stag
= sgl
[i
].lkey
;
1874 * i40iw_post_send - kernel application wr
1875 * @ibqp: qp ptr for wr
1876 * @ib_wr: work request ptr
1877 * @bad_wr: return of bad wr if err
1879 static int i40iw_post_send(struct ib_qp
*ibqp
,
1880 struct ib_send_wr
*ib_wr
,
1881 struct ib_send_wr
**bad_wr
)
1883 struct i40iw_qp
*iwqp
;
1884 struct i40iw_qp_uk
*ukqp
;
1885 struct i40iw_post_sq_info info
;
1886 enum i40iw_status_code ret
;
1888 unsigned long flags
;
1890 iwqp
= (struct i40iw_qp
*)ibqp
;
1891 ukqp
= &iwqp
->sc_qp
.qp_uk
;
1893 spin_lock_irqsave(&iwqp
->lock
, flags
);
1895 memset(&info
, 0, sizeof(info
));
1896 info
.wr_id
= (u64
)(ib_wr
->wr_id
);
1897 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || iwqp
->sig_all
)
1898 info
.signaled
= true;
1899 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
1900 info
.read_fence
= true;
1902 switch (ib_wr
->opcode
) {
1904 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
1905 info
.op_type
= I40IW_OP_TYPE_SEND_SOL
;
1907 info
.op_type
= I40IW_OP_TYPE_SEND
;
1909 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
1910 info
.op
.inline_send
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
1911 info
.op
.inline_send
.len
= ib_wr
->sg_list
[0].length
;
1912 ret
= ukqp
->ops
.iw_inline_send(ukqp
, &info
, rdma_wr(ib_wr
)->rkey
, false);
1914 info
.op
.send
.num_sges
= ib_wr
->num_sge
;
1915 info
.op
.send
.sg_list
= (struct i40iw_sge
*)ib_wr
->sg_list
;
1916 ret
= ukqp
->ops
.iw_send(ukqp
, &info
, rdma_wr(ib_wr
)->rkey
, false);
1922 case IB_WR_RDMA_WRITE
:
1923 info
.op_type
= I40IW_OP_TYPE_RDMA_WRITE
;
1925 if (ib_wr
->send_flags
& IB_SEND_INLINE
) {
1926 info
.op
.inline_rdma_write
.data
= (void *)(unsigned long)ib_wr
->sg_list
[0].addr
;
1927 info
.op
.inline_rdma_write
.len
= ib_wr
->sg_list
[0].length
;
1928 info
.op
.inline_rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
1929 info
.op
.inline_rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
1930 info
.op
.inline_rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
1931 ret
= ukqp
->ops
.iw_inline_rdma_write(ukqp
, &info
, false);
1933 info
.op
.rdma_write
.lo_sg_list
= (void *)ib_wr
->sg_list
;
1934 info
.op
.rdma_write
.num_lo_sges
= ib_wr
->num_sge
;
1935 info
.op
.rdma_write
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
1936 info
.op
.rdma_write
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
1937 info
.op
.rdma_write
.rem_addr
.len
= ib_wr
->sg_list
->length
;
1938 ret
= ukqp
->ops
.iw_rdma_write(ukqp
, &info
, false);
1944 case IB_WR_RDMA_READ
:
1945 info
.op_type
= I40IW_OP_TYPE_RDMA_READ
;
1946 info
.op
.rdma_read
.rem_addr
.tag_off
= rdma_wr(ib_wr
)->remote_addr
;
1947 info
.op
.rdma_read
.rem_addr
.stag
= rdma_wr(ib_wr
)->rkey
;
1948 info
.op
.rdma_read
.rem_addr
.len
= ib_wr
->sg_list
->length
;
1949 info
.op
.rdma_read
.lo_addr
.tag_off
= ib_wr
->sg_list
->addr
;
1950 info
.op
.rdma_read
.lo_addr
.stag
= ib_wr
->sg_list
->lkey
;
1951 info
.op
.rdma_read
.lo_addr
.len
= ib_wr
->sg_list
->length
;
1952 ret
= ukqp
->ops
.iw_rdma_read(ukqp
, &info
, false, false);
1958 i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
1965 ib_wr
= ib_wr
->next
;
1971 ukqp
->ops
.iw_qp_post_wr(ukqp
);
1972 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
1978 * i40iw_post_recv - post receive wr for kernel application
1979 * @ibqp: ib qp pointer
1980 * @ib_wr: work request for receive
1981 * @bad_wr: bad wr caused an error
1983 static int i40iw_post_recv(struct ib_qp
*ibqp
,
1984 struct ib_recv_wr
*ib_wr
,
1985 struct ib_recv_wr
**bad_wr
)
1987 struct i40iw_qp
*iwqp
;
1988 struct i40iw_qp_uk
*ukqp
;
1989 struct i40iw_post_rq_info post_recv
;
1990 struct i40iw_sge sg_list
[I40IW_MAX_WQ_FRAGMENT_COUNT
];
1991 enum i40iw_status_code ret
= 0;
1992 unsigned long flags
;
1994 iwqp
= (struct i40iw_qp
*)ibqp
;
1995 ukqp
= &iwqp
->sc_qp
.qp_uk
;
1997 memset(&post_recv
, 0, sizeof(post_recv
));
1998 spin_lock_irqsave(&iwqp
->lock
, flags
);
2000 post_recv
.num_sges
= ib_wr
->num_sge
;
2001 post_recv
.wr_id
= ib_wr
->wr_id
;
2002 i40iw_copy_sg_list(sg_list
, ib_wr
->sg_list
, ib_wr
->num_sge
);
2003 post_recv
.sg_list
= sg_list
;
2004 ret
= ukqp
->ops
.iw_post_receive(ukqp
, &post_recv
);
2006 i40iw_pr_err(" post_recv err %d\n", ret
);
2010 ib_wr
= ib_wr
->next
;
2013 spin_unlock_irqrestore(&iwqp
->lock
, flags
);
2020 * i40iw_poll_cq - poll cq for completion (kernel apps)
2022 * @num_entries: number of entries to poll
2023 * @entry: wr of entry completed
2025 static int i40iw_poll_cq(struct ib_cq
*ibcq
,
2027 struct ib_wc
*entry
)
2029 struct i40iw_cq
*iwcq
;
2031 struct i40iw_cq_poll_info cq_poll_info
;
2032 enum i40iw_status_code ret
;
2033 struct i40iw_cq_uk
*ukcq
;
2034 struct i40iw_sc_qp
*qp
;
2035 unsigned long flags
;
2037 iwcq
= (struct i40iw_cq
*)ibcq
;
2038 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2040 spin_lock_irqsave(&iwcq
->lock
, flags
);
2041 while (cqe_count
< num_entries
) {
2042 ret
= ukcq
->ops
.iw_cq_poll_completion(ukcq
, &cq_poll_info
, true);
2043 if (ret
== I40IW_ERR_QUEUE_EMPTY
) {
2050 entry
->wc_flags
= 0;
2051 entry
->wr_id
= cq_poll_info
.wr_id
;
2052 if (cq_poll_info
.error
) {
2053 entry
->status
= IB_WC_WR_FLUSH_ERR
;
2054 entry
->vendor_err
= cq_poll_info
.major_err
<< 16 | cq_poll_info
.minor_err
;
2056 entry
->status
= IB_WC_SUCCESS
;
2059 switch (cq_poll_info
.op_type
) {
2060 case I40IW_OP_TYPE_RDMA_WRITE
:
2061 entry
->opcode
= IB_WC_RDMA_WRITE
;
2063 case I40IW_OP_TYPE_RDMA_READ_INV_STAG
:
2064 case I40IW_OP_TYPE_RDMA_READ
:
2065 entry
->opcode
= IB_WC_RDMA_READ
;
2067 case I40IW_OP_TYPE_SEND_SOL
:
2068 case I40IW_OP_TYPE_SEND_SOL_INV
:
2069 case I40IW_OP_TYPE_SEND_INV
:
2070 case I40IW_OP_TYPE_SEND
:
2071 entry
->opcode
= IB_WC_SEND
;
2073 case I40IW_OP_TYPE_REC
:
2074 entry
->opcode
= IB_WC_RECV
;
2077 entry
->opcode
= IB_WC_RECV
;
2081 entry
->ex
.imm_data
= 0;
2082 qp
= (struct i40iw_sc_qp
*)cq_poll_info
.qp_handle
;
2083 entry
->qp
= (struct ib_qp
*)qp
->back_qp
;
2084 entry
->src_qp
= cq_poll_info
.qp_id
;
2085 entry
->byte_len
= cq_poll_info
.bytes_xfered
;
2089 spin_unlock_irqrestore(&iwcq
->lock
, flags
);
2094 * i40iw_req_notify_cq - arm cq kernel application
2096 * @notify_flags: notofication flags
2098 static int i40iw_req_notify_cq(struct ib_cq
*ibcq
,
2099 enum ib_cq_notify_flags notify_flags
)
2101 struct i40iw_cq
*iwcq
;
2102 struct i40iw_cq_uk
*ukcq
;
2103 enum i40iw_completion_notify cq_notify
= IW_CQ_COMPL_SOLICITED
;
2105 iwcq
= (struct i40iw_cq
*)ibcq
;
2106 ukcq
= &iwcq
->sc_cq
.cq_uk
;
2107 if (notify_flags
== IB_CQ_NEXT_COMP
)
2108 cq_notify
= IW_CQ_COMPL_EVENT
;
2109 ukcq
->ops
.iw_cq_request_notification(ukcq
, cq_notify
);
2114 * i40iw_port_immutable - return port's immutable data
2115 * @ibdev: ib dev struct
2116 * @port_num: port number
2117 * @immutable: immutable data for the port return
2119 static int i40iw_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
2120 struct ib_port_immutable
*immutable
)
2122 struct ib_port_attr attr
;
2125 err
= i40iw_query_port(ibdev
, port_num
, &attr
);
2130 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
2131 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
2132 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
2138 * i40iw_get_protocol_stats - Populates the rdma_stats structure
2139 * @ibdev: ib dev struct
2140 * @stats: iw protocol stats struct
2142 static int i40iw_get_protocol_stats(struct ib_device
*ibdev
,
2143 union rdma_protocol_stats
*stats
)
2145 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2146 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
2147 struct i40iw_dev_pestat
*devstat
= &dev
->dev_pestat
;
2148 struct i40iw_dev_hw_stats
*hw_stats
= &devstat
->hw_stats
;
2149 struct timespec curr_time
;
2150 static struct timespec last_rd_time
= {0, 0};
2151 unsigned long flags
;
2153 curr_time
= current_kernel_time();
2154 memset(stats
, 0, sizeof(*stats
));
2157 spin_lock_irqsave(&devstat
->stats_lock
, flags
);
2158 devstat
->ops
.iw_hw_stat_read_all(devstat
,
2159 &devstat
->hw_stats
);
2160 spin_unlock_irqrestore(&devstat
->stats_lock
, flags
);
2162 if (((u64
)curr_time
.tv_sec
- (u64
)last_rd_time
.tv_sec
) > 1)
2163 if (i40iw_vchnl_vf_get_pe_stats(dev
, &devstat
->hw_stats
))
2167 stats
->iw
.ipInReceives
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] +
2168 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
];
2169 stats
->iw
.ipInTruncatedPkts
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] +
2170 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
];
2171 stats
->iw
.ipInDiscards
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] +
2172 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
];
2173 stats
->iw
.ipOutNoRoutes
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] +
2174 hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
];
2175 stats
->iw
.ipReasmReqds
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] +
2176 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
];
2177 stats
->iw
.ipFragCreates
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] +
2178 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
];
2179 stats
->iw
.ipInMcastPkts
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] +
2180 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
];
2181 stats
->iw
.ipOutMcastPkts
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] +
2182 hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_IP6TXMCPKTS
];
2183 stats
->iw
.tcpOutSegs
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
];
2184 stats
->iw
.tcpInSegs
= hw_stats
->stat_value_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
];
2185 stats
->iw
.tcpRetransSegs
= hw_stats
->stat_value_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
];
2187 last_rd_time
= curr_time
;
2192 * i40iw_query_gid - Query port GID
2193 * @ibdev: device pointer from stack
2194 * @port: port number
2195 * @index: Entry index
2198 static int i40iw_query_gid(struct ib_device
*ibdev
,
2203 struct i40iw_device
*iwdev
= to_iwdev(ibdev
);
2205 memset(gid
->raw
, 0, sizeof(gid
->raw
));
2206 ether_addr_copy(gid
->raw
, iwdev
->netdev
->dev_addr
);
2211 * i40iw_modify_port Modify port properties
2212 * @ibdev: device pointer from stack
2213 * @port: port number
2214 * @port_modify_mask: mask for port modifications
2215 * @props: port properties
2217 static int i40iw_modify_port(struct ib_device
*ibdev
,
2219 int port_modify_mask
,
2220 struct ib_port_modify
*props
)
2226 * i40iw_query_pkey - Query partition key
2227 * @ibdev: device pointer from stack
2228 * @port: port number
2229 * @index: index of pkey
2230 * @pkey: pointer to store the pkey
2232 static int i40iw_query_pkey(struct ib_device
*ibdev
,
2242 * i40iw_create_ah - create address handle
2244 * @ah_attr: address handle attributes
2246 static struct ib_ah
*i40iw_create_ah(struct ib_pd
*ibpd
,
2247 struct ib_ah_attr
*attr
)
2249 return ERR_PTR(-ENOSYS
);
2253 * i40iw_destroy_ah - Destroy address handle
2254 * @ah: pointer to address handle
2256 static int i40iw_destroy_ah(struct ib_ah
*ah
)
2262 * i40iw_init_rdma_device - initialization of iwarp device
2263 * @iwdev: iwarp device
2265 static struct i40iw_ib_device
*i40iw_init_rdma_device(struct i40iw_device
*iwdev
)
2267 struct i40iw_ib_device
*iwibdev
;
2268 struct net_device
*netdev
= iwdev
->netdev
;
2269 struct pci_dev
*pcidev
= (struct pci_dev
*)iwdev
->hw
.dev_context
;
2271 iwibdev
= (struct i40iw_ib_device
*)ib_alloc_device(sizeof(*iwibdev
));
2273 i40iw_pr_err("iwdev == NULL\n");
2276 strlcpy(iwibdev
->ibdev
.name
, "i40iw%d", IB_DEVICE_NAME_MAX
);
2277 iwibdev
->ibdev
.owner
= THIS_MODULE
;
2278 iwdev
->iwibdev
= iwibdev
;
2279 iwibdev
->iwdev
= iwdev
;
2281 iwibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
2282 ether_addr_copy((u8
*)&iwibdev
->ibdev
.node_guid
, netdev
->dev_addr
);
2284 iwibdev
->ibdev
.uverbs_cmd_mask
=
2285 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
2286 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
2287 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
2288 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
2289 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
2290 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
2291 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
2292 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
2293 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
2294 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
2295 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
2296 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
2297 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
2298 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
2299 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
2300 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
2301 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
2302 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
2303 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
2304 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
2305 iwibdev
->ibdev
.phys_port_cnt
= 1;
2306 iwibdev
->ibdev
.num_comp_vectors
= 1;
2307 iwibdev
->ibdev
.dma_device
= &pcidev
->dev
;
2308 iwibdev
->ibdev
.dev
.parent
= &pcidev
->dev
;
2309 iwibdev
->ibdev
.query_port
= i40iw_query_port
;
2310 iwibdev
->ibdev
.modify_port
= i40iw_modify_port
;
2311 iwibdev
->ibdev
.query_pkey
= i40iw_query_pkey
;
2312 iwibdev
->ibdev
.query_gid
= i40iw_query_gid
;
2313 iwibdev
->ibdev
.alloc_ucontext
= i40iw_alloc_ucontext
;
2314 iwibdev
->ibdev
.dealloc_ucontext
= i40iw_dealloc_ucontext
;
2315 iwibdev
->ibdev
.mmap
= i40iw_mmap
;
2316 iwibdev
->ibdev
.alloc_pd
= i40iw_alloc_pd
;
2317 iwibdev
->ibdev
.dealloc_pd
= i40iw_dealloc_pd
;
2318 iwibdev
->ibdev
.create_qp
= i40iw_create_qp
;
2319 iwibdev
->ibdev
.modify_qp
= i40iw_modify_qp
;
2320 iwibdev
->ibdev
.query_qp
= i40iw_query_qp
;
2321 iwibdev
->ibdev
.destroy_qp
= i40iw_destroy_qp
;
2322 iwibdev
->ibdev
.create_cq
= i40iw_create_cq
;
2323 iwibdev
->ibdev
.destroy_cq
= i40iw_destroy_cq
;
2324 iwibdev
->ibdev
.get_dma_mr
= i40iw_get_dma_mr
;
2325 iwibdev
->ibdev
.reg_user_mr
= i40iw_reg_user_mr
;
2326 iwibdev
->ibdev
.dereg_mr
= i40iw_dereg_mr
;
2327 iwibdev
->ibdev
.get_protocol_stats
= i40iw_get_protocol_stats
;
2328 iwibdev
->ibdev
.query_device
= i40iw_query_device
;
2329 iwibdev
->ibdev
.create_ah
= i40iw_create_ah
;
2330 iwibdev
->ibdev
.destroy_ah
= i40iw_destroy_ah
;
2331 iwibdev
->ibdev
.iwcm
= kzalloc(sizeof(*iwibdev
->ibdev
.iwcm
), GFP_KERNEL
);
2332 if (!iwibdev
->ibdev
.iwcm
) {
2333 ib_dealloc_device(&iwibdev
->ibdev
);
2334 i40iw_pr_err("iwcm == NULL\n");
2338 iwibdev
->ibdev
.iwcm
->add_ref
= i40iw_add_ref
;
2339 iwibdev
->ibdev
.iwcm
->rem_ref
= i40iw_rem_ref
;
2340 iwibdev
->ibdev
.iwcm
->get_qp
= i40iw_get_qp
;
2341 iwibdev
->ibdev
.iwcm
->connect
= i40iw_connect
;
2342 iwibdev
->ibdev
.iwcm
->accept
= i40iw_accept
;
2343 iwibdev
->ibdev
.iwcm
->reject
= i40iw_reject
;
2344 iwibdev
->ibdev
.iwcm
->create_listen
= i40iw_create_listen
;
2345 iwibdev
->ibdev
.iwcm
->destroy_listen
= i40iw_destroy_listen
;
2346 memcpy(iwibdev
->ibdev
.iwcm
->ifname
, netdev
->name
,
2347 sizeof(iwibdev
->ibdev
.iwcm
->ifname
));
2348 iwibdev
->ibdev
.get_port_immutable
= i40iw_port_immutable
;
2349 iwibdev
->ibdev
.poll_cq
= i40iw_poll_cq
;
2350 iwibdev
->ibdev
.req_notify_cq
= i40iw_req_notify_cq
;
2351 iwibdev
->ibdev
.post_send
= i40iw_post_send
;
2352 iwibdev
->ibdev
.post_recv
= i40iw_post_recv
;
2358 * i40iw_port_ibevent - indicate port event
2359 * @iwdev: iwarp device
2361 void i40iw_port_ibevent(struct i40iw_device
*iwdev
)
2363 struct i40iw_ib_device
*iwibdev
= iwdev
->iwibdev
;
2364 struct ib_event event
;
2366 event
.device
= &iwibdev
->ibdev
;
2367 event
.element
.port_num
= 1;
2368 event
.event
= iwdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
2369 ib_dispatch_event(&event
);
2373 * i40iw_unregister_rdma_device - unregister of iwarp from IB
2374 * @iwibdev: rdma device ptr
2376 static void i40iw_unregister_rdma_device(struct i40iw_ib_device
*iwibdev
)
2380 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
)
2381 device_remove_file(&iwibdev
->ibdev
.dev
,
2382 i40iw_dev_attributes
[i
]);
2383 ib_unregister_device(&iwibdev
->ibdev
);
2387 * i40iw_destroy_rdma_device - destroy rdma device and free resources
2388 * @iwibdev: IB device ptr
2390 void i40iw_destroy_rdma_device(struct i40iw_ib_device
*iwibdev
)
2395 i40iw_unregister_rdma_device(iwibdev
);
2396 kfree(iwibdev
->ibdev
.iwcm
);
2397 iwibdev
->ibdev
.iwcm
= NULL
;
2398 ib_dealloc_device(&iwibdev
->ibdev
);
2402 * i40iw_register_rdma_device - register iwarp device to IB
2403 * @iwdev: iwarp device
2405 int i40iw_register_rdma_device(struct i40iw_device
*iwdev
)
2408 struct i40iw_ib_device
*iwibdev
;
2410 iwdev
->iwibdev
= i40iw_init_rdma_device(iwdev
);
2411 if (!iwdev
->iwibdev
)
2413 iwibdev
= iwdev
->iwibdev
;
2415 ret
= ib_register_device(&iwibdev
->ibdev
, NULL
);
2419 for (i
= 0; i
< ARRAY_SIZE(i40iw_dev_attributes
); ++i
) {
2421 device_create_file(&iwibdev
->ibdev
.dev
,
2422 i40iw_dev_attributes
[i
]);
2426 device_remove_file(&iwibdev
->ibdev
.dev
, i40iw_dev_attributes
[i
]);
2428 ib_unregister_device(&iwibdev
->ibdev
);
2434 kfree(iwdev
->iwibdev
->ibdev
.iwcm
);
2435 iwdev
->iwibdev
->ibdev
.iwcm
= NULL
;
2436 ib_dealloc_device(&iwdev
->iwibdev
->ibdev
);