1 /*******************************************************************
2 * This file is part of the Emulex RoCE Device Driver for *
3 * RoCE (RDMA over Converged Ethernet) adapters. *
4 * Copyright (C) 2008-2012 Emulex. All rights reserved. *
5 * EMULEX and SLI are trademarks of Emulex. *
8 * This program is free software; you can redistribute it and/or *
9 * modify it under the terms of version 2 of the GNU General *
10 * Public License as published by the Free Software Foundation. *
11 * This program is distributed in the hope that it will be useful. *
12 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
13 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
14 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
15 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
16 * TO BE LEGALLY INVALID. See the GNU General Public License for *
17 * more details, a copy of which can be found in the file COPYING *
18 * included with this package. *
20 * Contact Information:
21 * linux-drivers@emulex.com
25 * Costa Mesa, CA 92626
26 *******************************************************************/
28 #include <linux/dma-mapping.h>
29 #include <rdma/ib_verbs.h>
30 #include <rdma/ib_user_verbs.h>
31 #include <rdma/iw_cm.h>
32 #include <rdma/ib_umem.h>
33 #include <rdma/ib_addr.h>
36 #include "ocrdma_hw.h"
37 #include "ocrdma_verbs.h"
38 #include "ocrdma_abi.h"
40 int ocrdma_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
, u16
*pkey
)
49 int ocrdma_query_gid(struct ib_device
*ibdev
, u8 port
,
50 int index
, union ib_gid
*sgid
)
52 struct ocrdma_dev
*dev
;
54 dev
= get_ocrdma_dev(ibdev
);
55 memset(sgid
, 0, sizeof(*sgid
));
56 if (index
>= OCRDMA_MAX_SGID
)
59 memcpy(sgid
, &dev
->sgid_tbl
[index
], sizeof(*sgid
));
64 int ocrdma_query_device(struct ib_device
*ibdev
, struct ib_device_attr
*attr
)
66 struct ocrdma_dev
*dev
= get_ocrdma_dev(ibdev
);
68 memset(attr
, 0, sizeof *attr
);
69 memcpy(&attr
->fw_ver
, &dev
->attr
.fw_ver
[0],
70 min(sizeof(dev
->attr
.fw_ver
), sizeof(attr
->fw_ver
)));
71 ocrdma_get_guid(dev
, (u8
*)&attr
->sys_image_guid
);
72 attr
->max_mr_size
= ~0ull;
73 attr
->page_size_cap
= 0xffff000;
74 attr
->vendor_id
= dev
->nic_info
.pdev
->vendor
;
75 attr
->vendor_part_id
= dev
->nic_info
.pdev
->device
;
77 attr
->max_qp
= dev
->attr
.max_qp
;
78 attr
->max_ah
= dev
->attr
.max_qp
;
79 attr
->max_qp_wr
= dev
->attr
.max_wqe
;
81 attr
->device_cap_flags
= IB_DEVICE_CURR_QP_STATE_MOD
|
82 IB_DEVICE_RC_RNR_NAK_GEN
|
83 IB_DEVICE_SHUTDOWN_PORT
|
84 IB_DEVICE_SYS_IMAGE_GUID
|
85 IB_DEVICE_LOCAL_DMA_LKEY
;
86 attr
->max_sge
= min(dev
->attr
.max_send_sge
, dev
->attr
.max_srq_sge
);
88 attr
->max_cq
= dev
->attr
.max_cq
;
89 attr
->max_cqe
= dev
->attr
.max_cqe
;
90 attr
->max_mr
= dev
->attr
.max_mr
;
92 attr
->max_pd
= dev
->attr
.max_pd
;
95 attr
->max_map_per_fmr
= 0;
96 attr
->max_qp_rd_atom
=
97 min(dev
->attr
.max_ord_per_qp
, dev
->attr
.max_ird_per_qp
);
98 attr
->max_qp_init_rd_atom
= dev
->attr
.max_ord_per_qp
;
99 attr
->max_srq
= (dev
->attr
.max_qp
- 1);
100 attr
->max_srq_sge
= attr
->max_srq_sge
;
101 attr
->max_srq_wr
= dev
->attr
.max_rqe
;
102 attr
->local_ca_ack_delay
= dev
->attr
.local_ca_ack_delay
;
103 attr
->max_fast_reg_page_list_len
= 0;
108 int ocrdma_query_port(struct ib_device
*ibdev
,
109 u8 port
, struct ib_port_attr
*props
)
111 enum ib_port_state port_state
;
112 struct ocrdma_dev
*dev
;
113 struct net_device
*netdev
;
115 dev
= get_ocrdma_dev(ibdev
);
117 ocrdma_err("%s(%d) invalid_port=0x%x\n", __func__
,
121 netdev
= dev
->nic_info
.netdev
;
122 if (netif_running(netdev
) && netif_oper_up(netdev
)) {
123 port_state
= IB_PORT_ACTIVE
;
124 props
->phys_state
= 5;
126 port_state
= IB_PORT_DOWN
;
127 props
->phys_state
= 3;
129 props
->max_mtu
= IB_MTU_4096
;
130 props
->active_mtu
= iboe_get_mtu(netdev
->mtu
);
135 props
->state
= port_state
;
136 props
->port_cap_flags
=
139 IB_PORT_DEVICE_MGMT_SUP
| IB_PORT_VENDOR_CLASS_SUP
;
140 props
->gid_tbl_len
= OCRDMA_MAX_SGID
;
141 props
->pkey_tbl_len
= 1;
142 props
->bad_pkey_cntr
= 0;
143 props
->qkey_viol_cntr
= 0;
144 props
->active_width
= IB_WIDTH_1X
;
145 props
->active_speed
= 4;
146 props
->max_msg_sz
= 0x80000000;
147 props
->max_vl_num
= 4;
151 int ocrdma_modify_port(struct ib_device
*ibdev
, u8 port
, int mask
,
152 struct ib_port_modify
*props
)
154 struct ocrdma_dev
*dev
;
156 dev
= get_ocrdma_dev(ibdev
);
158 ocrdma_err("%s(%d) invalid_port=0x%x\n", __func__
,
165 static int ocrdma_add_mmap(struct ocrdma_ucontext
*uctx
, u64 phy_addr
,
168 struct ocrdma_mm
*mm
;
170 mm
= kzalloc(sizeof(*mm
), GFP_KERNEL
);
173 mm
->key
.phy_addr
= phy_addr
;
175 INIT_LIST_HEAD(&mm
->entry
);
177 mutex_lock(&uctx
->mm_list_lock
);
178 list_add_tail(&mm
->entry
, &uctx
->mm_head
);
179 mutex_unlock(&uctx
->mm_list_lock
);
183 static void ocrdma_del_mmap(struct ocrdma_ucontext
*uctx
, u64 phy_addr
,
186 struct ocrdma_mm
*mm
, *tmp
;
188 mutex_lock(&uctx
->mm_list_lock
);
189 list_for_each_entry_safe(mm
, tmp
, &uctx
->mm_head
, entry
) {
190 if (len
!= mm
->key
.len
|| phy_addr
!= mm
->key
.phy_addr
)
193 list_del(&mm
->entry
);
197 mutex_unlock(&uctx
->mm_list_lock
);
200 static bool ocrdma_search_mmap(struct ocrdma_ucontext
*uctx
, u64 phy_addr
,
204 struct ocrdma_mm
*mm
;
206 mutex_lock(&uctx
->mm_list_lock
);
207 list_for_each_entry(mm
, &uctx
->mm_head
, entry
) {
208 if (len
!= mm
->key
.len
|| phy_addr
!= mm
->key
.phy_addr
)
214 mutex_unlock(&uctx
->mm_list_lock
);
218 struct ib_ucontext
*ocrdma_alloc_ucontext(struct ib_device
*ibdev
,
219 struct ib_udata
*udata
)
222 struct ocrdma_ucontext
*ctx
;
223 struct ocrdma_alloc_ucontext_resp resp
;
224 struct ocrdma_dev
*dev
= get_ocrdma_dev(ibdev
);
225 struct pci_dev
*pdev
= dev
->nic_info
.pdev
;
226 u32 map_len
= roundup(sizeof(u32
) * 2048, PAGE_SIZE
);
229 return ERR_PTR(-EFAULT
);
230 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
232 return ERR_PTR(-ENOMEM
);
234 INIT_LIST_HEAD(&ctx
->mm_head
);
235 mutex_init(&ctx
->mm_list_lock
);
237 ctx
->ah_tbl
.va
= dma_alloc_coherent(&pdev
->dev
, map_len
,
238 &ctx
->ah_tbl
.pa
, GFP_KERNEL
);
239 if (!ctx
->ah_tbl
.va
) {
241 return ERR_PTR(-ENOMEM
);
243 memset(ctx
->ah_tbl
.va
, 0, map_len
);
244 ctx
->ah_tbl
.len
= map_len
;
246 resp
.ah_tbl_len
= ctx
->ah_tbl
.len
;
247 resp
.ah_tbl_page
= ctx
->ah_tbl
.pa
;
249 status
= ocrdma_add_mmap(ctx
, resp
.ah_tbl_page
, resp
.ah_tbl_len
);
252 resp
.dev_id
= dev
->id
;
253 resp
.max_inline_data
= dev
->attr
.max_inline_data
;
254 resp
.wqe_size
= dev
->attr
.wqe_size
;
255 resp
.rqe_size
= dev
->attr
.rqe_size
;
256 resp
.dpp_wqe_size
= dev
->attr
.wqe_size
;
259 memcpy(resp
.fw_ver
, dev
->attr
.fw_ver
, sizeof(resp
.fw_ver
));
260 status
= ib_copy_to_udata(udata
, &resp
, sizeof(resp
));
263 return &ctx
->ibucontext
;
266 ocrdma_del_mmap(ctx
, ctx
->ah_tbl
.pa
, ctx
->ah_tbl
.len
);
268 dma_free_coherent(&pdev
->dev
, ctx
->ah_tbl
.len
, ctx
->ah_tbl
.va
,
271 return ERR_PTR(status
);
274 int ocrdma_dealloc_ucontext(struct ib_ucontext
*ibctx
)
276 struct ocrdma_mm
*mm
, *tmp
;
277 struct ocrdma_ucontext
*uctx
= get_ocrdma_ucontext(ibctx
);
278 struct pci_dev
*pdev
= uctx
->dev
->nic_info
.pdev
;
280 ocrdma_del_mmap(uctx
, uctx
->ah_tbl
.pa
, uctx
->ah_tbl
.len
);
281 dma_free_coherent(&pdev
->dev
, uctx
->ah_tbl
.len
, uctx
->ah_tbl
.va
,
284 list_for_each_entry_safe(mm
, tmp
, &uctx
->mm_head
, entry
) {
285 list_del(&mm
->entry
);
292 int ocrdma_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
294 struct ocrdma_ucontext
*ucontext
= get_ocrdma_ucontext(context
);
295 struct ocrdma_dev
*dev
= ucontext
->dev
;
296 unsigned long vm_page
= vma
->vm_pgoff
<< PAGE_SHIFT
;
297 u64 unmapped_db
= (u64
) dev
->nic_info
.unmapped_db
;
298 unsigned long len
= (vma
->vm_end
- vma
->vm_start
);
302 if (vma
->vm_start
& (PAGE_SIZE
- 1))
304 found
= ocrdma_search_mmap(ucontext
, vma
->vm_pgoff
<< PAGE_SHIFT
, len
);
308 if ((vm_page
>= unmapped_db
) && (vm_page
<= (unmapped_db
+
309 dev
->nic_info
.db_total_size
)) &&
310 (len
<= dev
->nic_info
.db_page_size
)) {
311 /* doorbell mapping */
312 status
= io_remap_pfn_range(vma
, vma
->vm_start
, vma
->vm_pgoff
,
313 len
, vma
->vm_page_prot
);
314 } else if (dev
->nic_info
.dpp_unmapped_len
&&
315 (vm_page
>= (u64
) dev
->nic_info
.dpp_unmapped_addr
) &&
316 (vm_page
<= (u64
) (dev
->nic_info
.dpp_unmapped_addr
+
317 dev
->nic_info
.dpp_unmapped_len
)) &&
318 (len
<= dev
->nic_info
.dpp_unmapped_len
)) {
319 /* dpp area mapping */
320 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
321 status
= io_remap_pfn_range(vma
, vma
->vm_start
, vma
->vm_pgoff
,
322 len
, vma
->vm_page_prot
);
324 /* queue memory mapping */
325 status
= remap_pfn_range(vma
, vma
->vm_start
,
326 vma
->vm_pgoff
, len
, vma
->vm_page_prot
);
331 static int ocrdma_copy_pd_uresp(struct ocrdma_pd
*pd
,
332 struct ib_ucontext
*ib_ctx
,
333 struct ib_udata
*udata
)
337 u64 dpp_page_addr
= 0;
339 struct ocrdma_alloc_pd_uresp rsp
;
340 struct ocrdma_ucontext
*uctx
= get_ocrdma_ucontext(ib_ctx
);
343 rsp
.dpp_enabled
= pd
->dpp_enabled
;
344 db_page_addr
= pd
->dev
->nic_info
.unmapped_db
+
345 (pd
->id
* pd
->dev
->nic_info
.db_page_size
);
346 db_page_size
= pd
->dev
->nic_info
.db_page_size
;
348 status
= ocrdma_add_mmap(uctx
, db_page_addr
, db_page_size
);
352 if (pd
->dpp_enabled
) {
353 dpp_page_addr
= pd
->dev
->nic_info
.dpp_unmapped_addr
+
354 (pd
->id
* OCRDMA_DPP_PAGE_SIZE
);
355 status
= ocrdma_add_mmap(uctx
, dpp_page_addr
,
356 OCRDMA_DPP_PAGE_SIZE
);
359 rsp
.dpp_page_addr_hi
= upper_32_bits(dpp_page_addr
);
360 rsp
.dpp_page_addr_lo
= dpp_page_addr
;
363 status
= ib_copy_to_udata(udata
, &rsp
, sizeof(rsp
));
372 ocrdma_del_mmap(pd
->uctx
, dpp_page_addr
, OCRDMA_DPP_PAGE_SIZE
);
374 ocrdma_del_mmap(pd
->uctx
, db_page_addr
, db_page_size
);
378 struct ib_pd
*ocrdma_alloc_pd(struct ib_device
*ibdev
,
379 struct ib_ucontext
*context
,
380 struct ib_udata
*udata
)
382 struct ocrdma_dev
*dev
= get_ocrdma_dev(ibdev
);
383 struct ocrdma_pd
*pd
;
386 pd
= kzalloc(sizeof(*pd
), GFP_KERNEL
);
388 return ERR_PTR(-ENOMEM
);
390 if (udata
&& context
) {
391 pd
->dpp_enabled
= (dev
->nic_info
.dev_family
==
392 OCRDMA_GEN2_FAMILY
) ? true : false;
394 pd
->dpp_enabled
? OCRDMA_PD_MAX_DPP_ENABLED_QP
: 0;
396 status
= ocrdma_mbx_alloc_pd(dev
, pd
);
399 return ERR_PTR(status
);
401 atomic_set(&pd
->use_cnt
, 0);
403 if (udata
&& context
) {
404 status
= ocrdma_copy_pd_uresp(pd
, context
, udata
);
411 ocrdma_dealloc_pd(&pd
->ibpd
);
412 return ERR_PTR(status
);
415 int ocrdma_dealloc_pd(struct ib_pd
*ibpd
)
417 struct ocrdma_pd
*pd
= get_ocrdma_pd(ibpd
);
418 struct ocrdma_dev
*dev
= pd
->dev
;
422 if (atomic_read(&pd
->use_cnt
)) {
423 ocrdma_err("%s(%d) pd=0x%x is in use.\n",
424 __func__
, dev
->id
, pd
->id
);
428 status
= ocrdma_mbx_dealloc_pd(dev
, pd
);
430 u64 dpp_db
= dev
->nic_info
.dpp_unmapped_addr
+
431 (pd
->id
* OCRDMA_DPP_PAGE_SIZE
);
433 ocrdma_del_mmap(pd
->uctx
, dpp_db
, OCRDMA_DPP_PAGE_SIZE
);
434 usr_db
= dev
->nic_info
.unmapped_db
+
435 (pd
->id
* dev
->nic_info
.db_page_size
);
436 ocrdma_del_mmap(pd
->uctx
, usr_db
, dev
->nic_info
.db_page_size
);
443 static struct ocrdma_mr
*ocrdma_alloc_lkey(struct ib_pd
*ibpd
,
444 int acc
, u32 num_pbls
,
448 struct ocrdma_mr
*mr
;
449 struct ocrdma_pd
*pd
= get_ocrdma_pd(ibpd
);
450 struct ocrdma_dev
*dev
= pd
->dev
;
452 if (acc
& IB_ACCESS_REMOTE_WRITE
&& !(acc
& IB_ACCESS_LOCAL_WRITE
)) {
453 ocrdma_err("%s(%d) leaving err, invalid access rights\n",
455 return ERR_PTR(-EINVAL
);
458 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
460 return ERR_PTR(-ENOMEM
);
463 mr
->hwmr
.local_rd
= 1;
464 mr
->hwmr
.remote_rd
= (acc
& IB_ACCESS_REMOTE_READ
) ? 1 : 0;
465 mr
->hwmr
.remote_wr
= (acc
& IB_ACCESS_REMOTE_WRITE
) ? 1 : 0;
466 mr
->hwmr
.local_wr
= (acc
& IB_ACCESS_LOCAL_WRITE
) ? 1 : 0;
467 mr
->hwmr
.mw_bind
= (acc
& IB_ACCESS_MW_BIND
) ? 1 : 0;
468 mr
->hwmr
.remote_atomic
= (acc
& IB_ACCESS_REMOTE_ATOMIC
) ? 1 : 0;
469 mr
->hwmr
.num_pbls
= num_pbls
;
471 status
= ocrdma_mbx_alloc_lkey(dev
, &mr
->hwmr
, pd
->id
, addr_check
);
474 return ERR_PTR(-ENOMEM
);
477 atomic_inc(&pd
->use_cnt
);
478 mr
->ibmr
.lkey
= mr
->hwmr
.lkey
;
479 if (mr
->hwmr
.remote_wr
|| mr
->hwmr
.remote_rd
)
480 mr
->ibmr
.rkey
= mr
->hwmr
.lkey
;
484 struct ib_mr
*ocrdma_get_dma_mr(struct ib_pd
*ibpd
, int acc
)
486 struct ocrdma_mr
*mr
;
488 mr
= ocrdma_alloc_lkey(ibpd
, acc
, 0, OCRDMA_ADDR_CHECK_DISABLE
);
495 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev
*dev
,
496 struct ocrdma_hw_mr
*mr
)
498 struct pci_dev
*pdev
= dev
->nic_info
.pdev
;
502 for (i
= 0; i
< mr
->num_pbls
; i
++) {
503 if (!mr
->pbl_table
[i
].va
)
505 dma_free_coherent(&pdev
->dev
, mr
->pbl_size
,
507 mr
->pbl_table
[i
].pa
);
509 kfree(mr
->pbl_table
);
510 mr
->pbl_table
= NULL
;
514 static int ocrdma_get_pbl_info(struct ocrdma_mr
*mr
, u32 num_pbes
)
522 pbl_size
= OCRDMA_MIN_HPAGE_SIZE
* (1 << idx
);
523 if (pbl_size
> MAX_OCRDMA_PBL_SIZE
) {
527 num_pbls
= roundup(num_pbes
, (pbl_size
/ sizeof(u64
)));
528 num_pbls
= num_pbls
/ (pbl_size
/ sizeof(u64
));
530 } while (num_pbls
>= mr
->hwmr
.dev
->attr
.max_num_mr_pbl
);
532 mr
->hwmr
.num_pbes
= num_pbes
;
533 mr
->hwmr
.num_pbls
= num_pbls
;
534 mr
->hwmr
.pbl_size
= pbl_size
;
538 static int ocrdma_build_pbl_tbl(struct ocrdma_dev
*dev
, struct ocrdma_hw_mr
*mr
)
542 u32 dma_len
= mr
->pbl_size
;
543 struct pci_dev
*pdev
= dev
->nic_info
.pdev
;
547 mr
->pbl_table
= kzalloc(sizeof(struct ocrdma_pbl
) *
548 mr
->num_pbls
, GFP_KERNEL
);
553 for (i
= 0; i
< mr
->num_pbls
; i
++) {
554 va
= dma_alloc_coherent(&pdev
->dev
, dma_len
, &pa
, GFP_KERNEL
);
556 ocrdma_free_mr_pbl_tbl(dev
, mr
);
560 memset(va
, 0, dma_len
);
561 mr
->pbl_table
[i
].va
= va
;
562 mr
->pbl_table
[i
].pa
= pa
;
567 static void build_user_pbes(struct ocrdma_dev
*dev
, struct ocrdma_mr
*mr
,
570 struct ocrdma_pbe
*pbe
;
571 struct ib_umem_chunk
*chunk
;
572 struct ocrdma_pbl
*pbl_tbl
= mr
->hwmr
.pbl_table
;
573 struct ib_umem
*umem
= mr
->umem
;
574 int i
, shift
, pg_cnt
, pages
, pbe_cnt
, total_num_pbes
= 0;
576 if (!mr
->hwmr
.num_pbes
)
579 pbe
= (struct ocrdma_pbe
*)pbl_tbl
->va
;
582 shift
= ilog2(umem
->page_size
);
584 list_for_each_entry(chunk
, &umem
->chunk_list
, list
) {
585 /* get all the dma regions from the chunk. */
586 for (i
= 0; i
< chunk
->nmap
; i
++) {
587 pages
= sg_dma_len(&chunk
->page_list
[i
]) >> shift
;
588 for (pg_cnt
= 0; pg_cnt
< pages
; pg_cnt
++) {
589 /* store the page address in pbe */
591 cpu_to_le32(sg_dma_address
592 (&chunk
->page_list
[i
]) +
593 (umem
->page_size
* pg_cnt
));
595 cpu_to_le32(upper_32_bits
597 (&chunk
->page_list
[i
]) +
598 umem
->page_size
* pg_cnt
)));
603 /* if done building pbes, issue the mbx cmd. */
604 if (total_num_pbes
== num_pbes
)
607 /* if the given pbl is full storing the pbes,
611 (mr
->hwmr
.pbl_size
/ sizeof(u64
))) {
613 pbe
= (struct ocrdma_pbe
*)pbl_tbl
->va
;
621 struct ib_mr
*ocrdma_reg_user_mr(struct ib_pd
*ibpd
, u64 start
, u64 len
,
622 u64 usr_addr
, int acc
, struct ib_udata
*udata
)
624 int status
= -ENOMEM
;
625 struct ocrdma_dev
*dev
;
626 struct ocrdma_mr
*mr
;
627 struct ocrdma_pd
*pd
;
630 pd
= get_ocrdma_pd(ibpd
);
633 if (acc
& IB_ACCESS_REMOTE_WRITE
&& !(acc
& IB_ACCESS_LOCAL_WRITE
))
634 return ERR_PTR(-EINVAL
);
636 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
638 return ERR_PTR(status
);
640 mr
->umem
= ib_umem_get(ibpd
->uobject
->context
, start
, len
, acc
, 0);
641 if (IS_ERR(mr
->umem
)) {
645 num_pbes
= ib_umem_page_count(mr
->umem
);
646 status
= ocrdma_get_pbl_info(mr
, num_pbes
);
650 mr
->hwmr
.pbe_size
= mr
->umem
->page_size
;
651 mr
->hwmr
.fbo
= mr
->umem
->offset
;
652 mr
->hwmr
.va
= usr_addr
;
654 mr
->hwmr
.remote_wr
= (acc
& IB_ACCESS_REMOTE_WRITE
) ? 1 : 0;
655 mr
->hwmr
.remote_rd
= (acc
& IB_ACCESS_REMOTE_READ
) ? 1 : 0;
656 mr
->hwmr
.local_wr
= (acc
& IB_ACCESS_LOCAL_WRITE
) ? 1 : 0;
657 mr
->hwmr
.local_rd
= 1;
658 mr
->hwmr
.remote_atomic
= (acc
& IB_ACCESS_REMOTE_ATOMIC
) ? 1 : 0;
659 status
= ocrdma_build_pbl_tbl(dev
, &mr
->hwmr
);
662 build_user_pbes(dev
, mr
, num_pbes
);
663 status
= ocrdma_reg_mr(dev
, &mr
->hwmr
, pd
->id
, acc
);
667 atomic_inc(&pd
->use_cnt
);
668 mr
->ibmr
.lkey
= mr
->hwmr
.lkey
;
669 if (mr
->hwmr
.remote_wr
|| mr
->hwmr
.remote_rd
)
670 mr
->ibmr
.rkey
= mr
->hwmr
.lkey
;
675 ocrdma_free_mr_pbl_tbl(dev
, &mr
->hwmr
);
678 return ERR_PTR(status
);
681 int ocrdma_dereg_mr(struct ib_mr
*ib_mr
)
683 struct ocrdma_mr
*mr
= get_ocrdma_mr(ib_mr
);
684 struct ocrdma_dev
*dev
= mr
->hwmr
.dev
;
687 status
= ocrdma_mbx_dealloc_lkey(dev
, mr
->hwmr
.fr_mr
, mr
->hwmr
.lkey
);
689 if (mr
->hwmr
.fr_mr
== 0)
690 ocrdma_free_mr_pbl_tbl(dev
, &mr
->hwmr
);
692 atomic_dec(&mr
->pd
->use_cnt
);
693 /* it could be user registered memory. */
695 ib_umem_release(mr
->umem
);
700 static int ocrdma_copy_cq_uresp(struct ocrdma_cq
*cq
, struct ib_udata
*udata
,
701 struct ib_ucontext
*ib_ctx
)
704 struct ocrdma_ucontext
*uctx
;
705 struct ocrdma_create_cq_uresp uresp
;
707 uresp
.cq_id
= cq
->id
;
708 uresp
.page_size
= cq
->len
;
710 uresp
.max_hw_cqe
= cq
->max_hw_cqe
;
711 uresp
.page_addr
[0] = cq
->pa
;
712 uresp
.db_page_addr
= cq
->dev
->nic_info
.unmapped_db
;
713 uresp
.db_page_size
= cq
->dev
->nic_info
.db_page_size
;
714 uresp
.phase_change
= cq
->phase_change
? 1 : 0;
715 status
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
717 ocrdma_err("%s(%d) copy error cqid=0x%x.\n",
718 __func__
, cq
->dev
->id
, cq
->id
);
721 uctx
= get_ocrdma_ucontext(ib_ctx
);
722 status
= ocrdma_add_mmap(uctx
, uresp
.db_page_addr
, uresp
.db_page_size
);
725 status
= ocrdma_add_mmap(uctx
, uresp
.page_addr
[0], uresp
.page_size
);
727 ocrdma_del_mmap(uctx
, uresp
.db_page_addr
, uresp
.db_page_size
);
735 struct ib_cq
*ocrdma_create_cq(struct ib_device
*ibdev
, int entries
, int vector
,
736 struct ib_ucontext
*ib_ctx
,
737 struct ib_udata
*udata
)
739 struct ocrdma_cq
*cq
;
740 struct ocrdma_dev
*dev
= get_ocrdma_dev(ibdev
);
742 struct ocrdma_create_cq_ureq ureq
;
745 if (ib_copy_from_udata(&ureq
, udata
, sizeof(ureq
)))
746 return ERR_PTR(-EFAULT
);
749 cq
= kzalloc(sizeof(*cq
), GFP_KERNEL
);
751 return ERR_PTR(-ENOMEM
);
753 spin_lock_init(&cq
->cq_lock
);
754 spin_lock_init(&cq
->comp_handler_lock
);
755 atomic_set(&cq
->use_cnt
, 0);
756 INIT_LIST_HEAD(&cq
->sq_head
);
757 INIT_LIST_HEAD(&cq
->rq_head
);
760 status
= ocrdma_mbx_create_cq(dev
, cq
, entries
, ureq
.dpp_cq
);
763 return ERR_PTR(status
);
766 status
= ocrdma_copy_cq_uresp(cq
, udata
, ib_ctx
);
770 cq
->phase
= OCRDMA_CQE_VALID
;
771 cq
->arm_needed
= true;
772 dev
->cq_tbl
[cq
->id
] = cq
;
777 ocrdma_mbx_destroy_cq(dev
, cq
);
779 return ERR_PTR(status
);
782 int ocrdma_resize_cq(struct ib_cq
*ibcq
, int new_cnt
,
783 struct ib_udata
*udata
)
786 struct ocrdma_cq
*cq
= get_ocrdma_cq(ibcq
);
788 if (new_cnt
< 1 || new_cnt
> cq
->max_hw_cqe
) {
796 int ocrdma_destroy_cq(struct ib_cq
*ibcq
)
799 struct ocrdma_cq
*cq
= get_ocrdma_cq(ibcq
);
800 struct ocrdma_dev
*dev
= cq
->dev
;
802 if (atomic_read(&cq
->use_cnt
))
805 status
= ocrdma_mbx_destroy_cq(dev
, cq
);
808 ocrdma_del_mmap(cq
->ucontext
, (u64
) cq
->pa
, cq
->len
);
809 ocrdma_del_mmap(cq
->ucontext
, dev
->nic_info
.unmapped_db
,
810 dev
->nic_info
.db_page_size
);
812 dev
->cq_tbl
[cq
->id
] = NULL
;
818 static int ocrdma_add_qpn_map(struct ocrdma_dev
*dev
, struct ocrdma_qp
*qp
)
820 int status
= -EINVAL
;
822 if (qp
->id
< OCRDMA_MAX_QP
&& dev
->qp_tbl
[qp
->id
] == NULL
) {
823 dev
->qp_tbl
[qp
->id
] = qp
;
829 static void ocrdma_del_qpn_map(struct ocrdma_dev
*dev
, struct ocrdma_qp
*qp
)
831 dev
->qp_tbl
[qp
->id
] = NULL
;
834 static int ocrdma_check_qp_params(struct ib_pd
*ibpd
, struct ocrdma_dev
*dev
,
835 struct ib_qp_init_attr
*attrs
)
837 if (attrs
->qp_type
!= IB_QPT_GSI
&&
838 attrs
->qp_type
!= IB_QPT_RC
&&
839 attrs
->qp_type
!= IB_QPT_UD
) {
840 ocrdma_err("%s(%d) unsupported qp type=0x%x requested\n",
841 __func__
, dev
->id
, attrs
->qp_type
);
844 if (attrs
->cap
.max_send_wr
> dev
->attr
.max_wqe
) {
845 ocrdma_err("%s(%d) unsupported send_wr=0x%x requested\n",
846 __func__
, dev
->id
, attrs
->cap
.max_send_wr
);
847 ocrdma_err("%s(%d) supported send_wr=0x%x\n",
848 __func__
, dev
->id
, dev
->attr
.max_wqe
);
851 if (!attrs
->srq
&& (attrs
->cap
.max_recv_wr
> dev
->attr
.max_rqe
)) {
852 ocrdma_err("%s(%d) unsupported recv_wr=0x%x requested\n",
853 __func__
, dev
->id
, attrs
->cap
.max_recv_wr
);
854 ocrdma_err("%s(%d) supported recv_wr=0x%x\n",
855 __func__
, dev
->id
, dev
->attr
.max_rqe
);
858 if (attrs
->cap
.max_inline_data
> dev
->attr
.max_inline_data
) {
859 ocrdma_err("%s(%d) unsupported inline data size=0x%x"
860 " requested\n", __func__
, dev
->id
,
861 attrs
->cap
.max_inline_data
);
862 ocrdma_err("%s(%d) supported inline data size=0x%x\n",
863 __func__
, dev
->id
, dev
->attr
.max_inline_data
);
866 if (attrs
->cap
.max_send_sge
> dev
->attr
.max_send_sge
) {
867 ocrdma_err("%s(%d) unsupported send_sge=0x%x requested\n",
868 __func__
, dev
->id
, attrs
->cap
.max_send_sge
);
869 ocrdma_err("%s(%d) supported send_sge=0x%x\n",
870 __func__
, dev
->id
, dev
->attr
.max_send_sge
);
873 if (attrs
->cap
.max_recv_sge
> dev
->attr
.max_recv_sge
) {
874 ocrdma_err("%s(%d) unsupported recv_sge=0x%x requested\n",
875 __func__
, dev
->id
, attrs
->cap
.max_recv_sge
);
876 ocrdma_err("%s(%d) supported recv_sge=0x%x\n",
877 __func__
, dev
->id
, dev
->attr
.max_recv_sge
);
880 /* unprivileged user space cannot create special QP */
881 if (ibpd
->uobject
&& attrs
->qp_type
== IB_QPT_GSI
) {
883 ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
884 __func__
, dev
->id
, attrs
->qp_type
);
887 /* allow creating only one GSI type of QP */
888 if (attrs
->qp_type
== IB_QPT_GSI
&& dev
->gsi_qp_created
) {
889 ocrdma_err("%s(%d) GSI special QPs already created.\n",
893 /* verify consumer QPs are not trying to use GSI QP's CQ */
894 if ((attrs
->qp_type
!= IB_QPT_GSI
) && (dev
->gsi_qp_created
)) {
895 if ((dev
->gsi_sqcq
== get_ocrdma_cq(attrs
->send_cq
)) ||
896 (dev
->gsi_sqcq
== get_ocrdma_cq(attrs
->send_cq
))) {
897 ocrdma_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
905 static int ocrdma_copy_qp_uresp(struct ocrdma_qp
*qp
,
906 struct ib_udata
*udata
, int dpp_offset
,
907 int dpp_credit_lmt
, int srq
)
911 struct ocrdma_create_qp_uresp uresp
;
912 struct ocrdma_dev
*dev
= qp
->dev
;
913 struct ocrdma_pd
*pd
= qp
->pd
;
915 memset(&uresp
, 0, sizeof(uresp
));
916 usr_db
= dev
->nic_info
.unmapped_db
+
917 (pd
->id
* dev
->nic_info
.db_page_size
);
918 uresp
.qp_id
= qp
->id
;
919 uresp
.sq_dbid
= qp
->sq
.dbid
;
920 uresp
.num_sq_pages
= 1;
921 uresp
.sq_page_size
= qp
->sq
.len
;
922 uresp
.sq_page_addr
[0] = qp
->sq
.pa
;
923 uresp
.num_wqe_allocated
= qp
->sq
.max_cnt
;
925 uresp
.rq_dbid
= qp
->rq
.dbid
;
926 uresp
.num_rq_pages
= 1;
927 uresp
.rq_page_size
= qp
->rq
.len
;
928 uresp
.rq_page_addr
[0] = qp
->rq
.pa
;
929 uresp
.num_rqe_allocated
= qp
->rq
.max_cnt
;
931 uresp
.db_page_addr
= usr_db
;
932 uresp
.db_page_size
= dev
->nic_info
.db_page_size
;
933 if (dev
->nic_info
.dev_family
== OCRDMA_GEN2_FAMILY
) {
934 uresp
.db_sq_offset
= OCRDMA_DB_GEN2_SQ_OFFSET
;
935 uresp
.db_rq_offset
= ((qp
->id
& 0xFFFF) < 128) ?
936 OCRDMA_DB_GEN2_RQ1_OFFSET
: OCRDMA_DB_GEN2_RQ2_OFFSET
;
937 uresp
.db_shift
= (qp
->id
< 128) ? 24 : 16;
939 uresp
.db_sq_offset
= OCRDMA_DB_SQ_OFFSET
;
940 uresp
.db_rq_offset
= OCRDMA_DB_RQ_OFFSET
;
944 if (qp
->dpp_enabled
) {
945 uresp
.dpp_credit
= dpp_credit_lmt
;
946 uresp
.dpp_offset
= dpp_offset
;
948 status
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
950 ocrdma_err("%s(%d) user copy error.\n", __func__
, dev
->id
);
953 status
= ocrdma_add_mmap(pd
->uctx
, uresp
.sq_page_addr
[0],
959 status
= ocrdma_add_mmap(pd
->uctx
, uresp
.rq_page_addr
[0],
966 ocrdma_del_mmap(pd
->uctx
, uresp
.sq_page_addr
[0], uresp
.sq_page_size
);
971 static void ocrdma_set_qp_db(struct ocrdma_dev
*dev
, struct ocrdma_qp
*qp
,
972 struct ocrdma_pd
*pd
)
974 if (dev
->nic_info
.dev_family
== OCRDMA_GEN2_FAMILY
) {
975 qp
->sq_db
= dev
->nic_info
.db
+
976 (pd
->id
* dev
->nic_info
.db_page_size
) +
977 OCRDMA_DB_GEN2_SQ_OFFSET
;
978 qp
->rq_db
= dev
->nic_info
.db
+
979 (pd
->id
* dev
->nic_info
.db_page_size
) +
981 OCRDMA_DB_GEN2_RQ1_OFFSET
: OCRDMA_DB_GEN2_RQ2_OFFSET
);
983 qp
->sq_db
= dev
->nic_info
.db
+
984 (pd
->id
* dev
->nic_info
.db_page_size
) +
986 qp
->rq_db
= dev
->nic_info
.db
+
987 (pd
->id
* dev
->nic_info
.db_page_size
) +
992 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp
*qp
)
995 kzalloc(sizeof(*(qp
->wqe_wr_id_tbl
)) * qp
->sq
.max_cnt
,
997 if (qp
->wqe_wr_id_tbl
== NULL
)
1000 kzalloc(sizeof(u64
) * qp
->rq
.max_cnt
, GFP_KERNEL
);
1001 if (qp
->rqe_wr_id_tbl
== NULL
)
1007 static void ocrdma_set_qp_init_params(struct ocrdma_qp
*qp
,
1008 struct ocrdma_pd
*pd
,
1009 struct ib_qp_init_attr
*attrs
)
1012 spin_lock_init(&qp
->q_lock
);
1013 INIT_LIST_HEAD(&qp
->sq_entry
);
1014 INIT_LIST_HEAD(&qp
->rq_entry
);
1016 qp
->qp_type
= attrs
->qp_type
;
1017 qp
->cap_flags
= OCRDMA_QP_INB_RD
| OCRDMA_QP_INB_WR
;
1018 qp
->max_inline_data
= attrs
->cap
.max_inline_data
;
1019 qp
->sq
.max_sges
= attrs
->cap
.max_send_sge
;
1020 qp
->rq
.max_sges
= attrs
->cap
.max_recv_sge
;
1021 qp
->state
= OCRDMA_QPS_RST
;
1024 static void ocrdma_set_qp_use_cnt(struct ocrdma_qp
*qp
, struct ocrdma_pd
*pd
)
1026 atomic_inc(&pd
->use_cnt
);
1027 atomic_inc(&qp
->sq_cq
->use_cnt
);
1028 atomic_inc(&qp
->rq_cq
->use_cnt
);
1030 atomic_inc(&qp
->srq
->use_cnt
);
1031 qp
->ibqp
.qp_num
= qp
->id
;
1034 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev
*dev
,
1035 struct ib_qp_init_attr
*attrs
)
1037 if (attrs
->qp_type
== IB_QPT_GSI
) {
1038 dev
->gsi_qp_created
= 1;
1039 dev
->gsi_sqcq
= get_ocrdma_cq(attrs
->send_cq
);
1040 dev
->gsi_rqcq
= get_ocrdma_cq(attrs
->recv_cq
);
1044 struct ib_qp
*ocrdma_create_qp(struct ib_pd
*ibpd
,
1045 struct ib_qp_init_attr
*attrs
,
1046 struct ib_udata
*udata
)
1049 struct ocrdma_pd
*pd
= get_ocrdma_pd(ibpd
);
1050 struct ocrdma_qp
*qp
;
1051 struct ocrdma_dev
*dev
= pd
->dev
;
1052 struct ocrdma_create_qp_ureq ureq
;
1053 u16 dpp_credit_lmt
, dpp_offset
;
1055 status
= ocrdma_check_qp_params(ibpd
, dev
, attrs
);
1059 memset(&ureq
, 0, sizeof(ureq
));
1061 if (ib_copy_from_udata(&ureq
, udata
, sizeof(ureq
)))
1062 return ERR_PTR(-EFAULT
);
1064 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
1070 ocrdma_set_qp_init_params(qp
, pd
, attrs
);
1072 mutex_lock(&dev
->dev_lock
);
1073 status
= ocrdma_mbx_create_qp(qp
, attrs
, ureq
.enable_dpp_cq
,
1075 &dpp_offset
, &dpp_credit_lmt
);
1079 /* user space QP's wr_id table are managed in library */
1080 if (udata
== NULL
) {
1081 qp
->cap_flags
|= (OCRDMA_QP_MW_BIND
| OCRDMA_QP_LKEY0
|
1082 OCRDMA_QP_FAST_REG
);
1083 status
= ocrdma_alloc_wr_id_tbl(qp
);
1088 status
= ocrdma_add_qpn_map(dev
, qp
);
1091 ocrdma_set_qp_db(dev
, qp
, pd
);
1093 status
= ocrdma_copy_qp_uresp(qp
, udata
, dpp_offset
,
1095 (attrs
->srq
!= NULL
));
1099 ocrdma_store_gsi_qp_cq(dev
, attrs
);
1100 ocrdma_set_qp_use_cnt(qp
, pd
);
1101 mutex_unlock(&dev
->dev_lock
);
1105 ocrdma_del_qpn_map(dev
, qp
);
1107 ocrdma_mbx_destroy_qp(dev
, qp
);
1109 mutex_unlock(&dev
->dev_lock
);
1110 kfree(qp
->wqe_wr_id_tbl
);
1111 kfree(qp
->rqe_wr_id_tbl
);
1113 ocrdma_err("%s(%d) error=%d\n", __func__
, dev
->id
, status
);
1115 return ERR_PTR(status
);
1118 int _ocrdma_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
1122 struct ocrdma_qp
*qp
;
1123 struct ocrdma_dev
*dev
;
1124 enum ib_qp_state old_qps
;
1126 qp
= get_ocrdma_qp(ibqp
);
1128 if (attr_mask
& IB_QP_STATE
)
1129 status
= ocrdma_qp_state_machine(qp
, attr
->qp_state
, &old_qps
);
1130 /* if new and previous states are same hw doesn't need to
1135 status
= ocrdma_mbx_modify_qp(dev
, qp
, attr
, attr_mask
, old_qps
);
1139 int ocrdma_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
1140 int attr_mask
, struct ib_udata
*udata
)
1142 unsigned long flags
;
1143 int status
= -EINVAL
;
1144 struct ocrdma_qp
*qp
;
1145 struct ocrdma_dev
*dev
;
1146 enum ib_qp_state old_qps
, new_qps
;
1148 qp
= get_ocrdma_qp(ibqp
);
1151 /* syncronize with multiple context trying to change, retrive qps */
1152 mutex_lock(&dev
->dev_lock
);
1153 /* syncronize with wqe, rqe posting and cqe processing contexts */
1154 spin_lock_irqsave(&qp
->q_lock
, flags
);
1155 old_qps
= get_ibqp_state(qp
->state
);
1156 if (attr_mask
& IB_QP_STATE
)
1157 new_qps
= attr
->qp_state
;
1160 spin_unlock_irqrestore(&qp
->q_lock
, flags
);
1162 if (!ib_modify_qp_is_ok(old_qps
, new_qps
, ibqp
->qp_type
, attr_mask
)) {
1163 ocrdma_err("%s(%d) invalid attribute mask=0x%x specified for "
1164 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1165 __func__
, dev
->id
, attr_mask
, qp
->id
, ibqp
->qp_type
,
1170 status
= _ocrdma_modify_qp(ibqp
, attr
, attr_mask
);
1174 mutex_unlock(&dev
->dev_lock
);
1178 static enum ib_mtu
ocrdma_mtu_int_to_enum(u16 mtu
)
1196 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags
)
1198 int ib_qp_acc_flags
= 0;
1200 if (qp_cap_flags
& OCRDMA_QP_INB_WR
)
1201 ib_qp_acc_flags
|= IB_ACCESS_REMOTE_WRITE
;
1202 if (qp_cap_flags
& OCRDMA_QP_INB_RD
)
1203 ib_qp_acc_flags
|= IB_ACCESS_LOCAL_WRITE
;
1204 return ib_qp_acc_flags
;
1207 int ocrdma_query_qp(struct ib_qp
*ibqp
,
1208 struct ib_qp_attr
*qp_attr
,
1209 int attr_mask
, struct ib_qp_init_attr
*qp_init_attr
)
1213 struct ocrdma_qp_params params
;
1214 struct ocrdma_qp
*qp
= get_ocrdma_qp(ibqp
);
1215 struct ocrdma_dev
*dev
= qp
->dev
;
1217 memset(¶ms
, 0, sizeof(params
));
1218 mutex_lock(&dev
->dev_lock
);
1219 status
= ocrdma_mbx_query_qp(dev
, qp
, ¶ms
);
1220 mutex_unlock(&dev
->dev_lock
);
1223 qp_attr
->qp_state
= get_ibqp_state(IB_QPS_INIT
);
1224 qp_attr
->cur_qp_state
= get_ibqp_state(IB_QPS_INIT
);
1226 ocrdma_mtu_int_to_enum(params
.path_mtu_pkey_indx
&
1227 OCRDMA_QP_PARAMS_PATH_MTU_MASK
) >>
1228 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT
;
1229 qp_attr
->path_mig_state
= IB_MIG_MIGRATED
;
1230 qp_attr
->rq_psn
= params
.hop_lmt_rq_psn
& OCRDMA_QP_PARAMS_RQ_PSN_MASK
;
1231 qp_attr
->sq_psn
= params
.tclass_sq_psn
& OCRDMA_QP_PARAMS_SQ_PSN_MASK
;
1232 qp_attr
->dest_qp_num
=
1233 params
.ack_to_rnr_rtc_dest_qpn
& OCRDMA_QP_PARAMS_DEST_QPN_MASK
;
1235 qp_attr
->qp_access_flags
= ocrdma_to_ib_qp_acc_flags(qp
->cap_flags
);
1236 qp_attr
->cap
.max_send_wr
= qp
->sq
.max_cnt
- 1;
1237 qp_attr
->cap
.max_recv_wr
= qp
->rq
.max_cnt
- 1;
1238 qp_attr
->cap
.max_send_sge
= qp
->sq
.max_sges
;
1239 qp_attr
->cap
.max_recv_sge
= qp
->rq
.max_sges
;
1240 qp_attr
->cap
.max_inline_data
= dev
->attr
.max_inline_data
;
1241 qp_init_attr
->cap
= qp_attr
->cap
;
1242 memcpy(&qp_attr
->ah_attr
.grh
.dgid
, ¶ms
.dgid
[0],
1243 sizeof(params
.dgid
));
1244 qp_attr
->ah_attr
.grh
.flow_label
= params
.rnt_rc_sl_fl
&
1245 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK
;
1246 qp_attr
->ah_attr
.grh
.sgid_index
= qp
->sgid_idx
;
1247 qp_attr
->ah_attr
.grh
.hop_limit
= (params
.hop_lmt_rq_psn
&
1248 OCRDMA_QP_PARAMS_HOP_LMT_MASK
) >>
1249 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT
;
1250 qp_attr
->ah_attr
.grh
.traffic_class
= (params
.tclass_sq_psn
&
1251 OCRDMA_QP_PARAMS_SQ_PSN_MASK
) >>
1252 OCRDMA_QP_PARAMS_TCLASS_SHIFT
;
1254 qp_attr
->ah_attr
.ah_flags
= IB_AH_GRH
;
1255 qp_attr
->ah_attr
.port_num
= 1;
1256 qp_attr
->ah_attr
.sl
= (params
.rnt_rc_sl_fl
&
1257 OCRDMA_QP_PARAMS_SL_MASK
) >>
1258 OCRDMA_QP_PARAMS_SL_SHIFT
;
1259 qp_attr
->timeout
= (params
.ack_to_rnr_rtc_dest_qpn
&
1260 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK
) >>
1261 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT
;
1262 qp_attr
->rnr_retry
= (params
.ack_to_rnr_rtc_dest_qpn
&
1263 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK
) >>
1264 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT
;
1265 qp_attr
->retry_cnt
=
1266 (params
.rnt_rc_sl_fl
& OCRDMA_QP_PARAMS_RETRY_CNT_MASK
) >>
1267 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT
;
1268 qp_attr
->min_rnr_timer
= 0;
1269 qp_attr
->pkey_index
= 0;
1270 qp_attr
->port_num
= 1;
1271 qp_attr
->ah_attr
.src_path_bits
= 0;
1272 qp_attr
->ah_attr
.static_rate
= 0;
1273 qp_attr
->alt_pkey_index
= 0;
1274 qp_attr
->alt_port_num
= 0;
1275 qp_attr
->alt_timeout
= 0;
1276 memset(&qp_attr
->alt_ah_attr
, 0, sizeof(qp_attr
->alt_ah_attr
));
1277 qp_state
= (params
.max_sge_recv_flags
& OCRDMA_QP_PARAMS_STATE_MASK
) >>
1278 OCRDMA_QP_PARAMS_STATE_SHIFT
;
1279 qp_attr
->sq_draining
= (qp_state
== OCRDMA_QPS_SQ_DRAINING
) ? 1 : 0;
1280 qp_attr
->max_dest_rd_atomic
=
1281 params
.max_ord_ird
>> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT
;
1282 qp_attr
->max_rd_atomic
=
1283 params
.max_ord_ird
& OCRDMA_QP_PARAMS_MAX_IRD_MASK
;
1284 qp_attr
->en_sqd_async_notify
= (params
.max_sge_recv_flags
&
1285 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC
) ? 1 : 0;
1290 static void ocrdma_srq_toggle_bit(struct ocrdma_srq
*srq
, int idx
)
1293 unsigned int mask
= (1 << (idx
% 32));
1295 if (srq
->idx_bit_fields
[i
] & mask
)
1296 srq
->idx_bit_fields
[i
] &= ~mask
;
1298 srq
->idx_bit_fields
[i
] |= mask
;
1301 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info
*q
)
1304 if (q
->head
>= q
->tail
)
1305 free_cnt
= (q
->max_cnt
- q
->head
) + q
->tail
;
1307 free_cnt
= q
->tail
- q
->head
;
1311 static int is_hw_sq_empty(struct ocrdma_qp
*qp
)
1313 return (qp
->sq
.tail
== qp
->sq
.head
&&
1314 ocrdma_hwq_free_cnt(&qp
->sq
) ? 1 : 0);
1317 static int is_hw_rq_empty(struct ocrdma_qp
*qp
)
1319 return (qp
->rq
.tail
== qp
->rq
.head
) ? 1 : 0;
1322 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info
*q
)
1324 return q
->va
+ (q
->head
* q
->entry_size
);
1327 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info
*q
,
1330 return q
->va
+ (idx
* q
->entry_size
);
1333 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info
*q
)
1335 q
->head
= (q
->head
+ 1) & q
->max_wqe_idx
;
1338 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info
*q
)
1340 q
->tail
= (q
->tail
+ 1) & q
->max_wqe_idx
;
1343 /* discard the cqe for a given QP */
1344 static void ocrdma_discard_cqes(struct ocrdma_qp
*qp
, struct ocrdma_cq
*cq
)
1346 unsigned long cq_flags
;
1347 unsigned long flags
;
1348 int discard_cnt
= 0;
1349 u32 cur_getp
, stop_getp
;
1350 struct ocrdma_cqe
*cqe
;
1353 spin_lock_irqsave(&cq
->cq_lock
, cq_flags
);
1355 /* traverse through the CQEs in the hw CQ,
1356 * find the matching CQE for a given qp,
1357 * mark the matching one discarded by clearing qpn.
1358 * ring the doorbell in the poll_cq() as
1359 * we don't complete out of order cqe.
1362 cur_getp
= cq
->getp
;
1363 /* find upto when do we reap the cq. */
1364 stop_getp
= cur_getp
;
1366 if (is_hw_sq_empty(qp
) && (!qp
->srq
&& is_hw_rq_empty(qp
)))
1369 cqe
= cq
->va
+ cur_getp
;
1370 /* if (a) done reaping whole hw cq, or
1371 * (b) qp_xq becomes empty.
1374 qpn
= cqe
->cmn
.qpn
& OCRDMA_CQE_QPN_MASK
;
1375 /* if previously discarded cqe found, skip that too. */
1376 /* check for matching qp */
1377 if (qpn
== 0 || qpn
!= qp
->id
)
1380 /* mark cqe discarded so that it is not picked up later
1385 if (is_cqe_for_sq(cqe
))
1386 ocrdma_hwq_inc_tail(&qp
->sq
);
1389 spin_lock_irqsave(&qp
->srq
->q_lock
, flags
);
1390 ocrdma_hwq_inc_tail(&qp
->srq
->rq
);
1391 ocrdma_srq_toggle_bit(qp
->srq
, cur_getp
);
1392 spin_unlock_irqrestore(&qp
->srq
->q_lock
, flags
);
1395 ocrdma_hwq_inc_tail(&qp
->rq
);
1398 cur_getp
= (cur_getp
+ 1) % cq
->max_hw_cqe
;
1399 } while (cur_getp
!= stop_getp
);
1400 spin_unlock_irqrestore(&cq
->cq_lock
, cq_flags
);
1403 static void ocrdma_del_flush_qp(struct ocrdma_qp
*qp
)
1406 unsigned long flags
;
1407 struct ocrdma_dev
*dev
= qp
->dev
;
1408 /* sync with any active CQ poll */
1410 spin_lock_irqsave(&dev
->flush_q_lock
, flags
);
1411 found
= ocrdma_is_qp_in_sq_flushlist(qp
->sq_cq
, qp
);
1413 list_del(&qp
->sq_entry
);
1415 found
= ocrdma_is_qp_in_rq_flushlist(qp
->rq_cq
, qp
);
1417 list_del(&qp
->rq_entry
);
1419 spin_unlock_irqrestore(&dev
->flush_q_lock
, flags
);
1422 int ocrdma_destroy_qp(struct ib_qp
*ibqp
)
1425 struct ocrdma_pd
*pd
;
1426 struct ocrdma_qp
*qp
;
1427 struct ocrdma_dev
*dev
;
1428 struct ib_qp_attr attrs
;
1429 int attr_mask
= IB_QP_STATE
;
1430 unsigned long flags
;
1432 qp
= get_ocrdma_qp(ibqp
);
1435 attrs
.qp_state
= IB_QPS_ERR
;
1438 /* change the QP state to ERROR */
1439 _ocrdma_modify_qp(ibqp
, &attrs
, attr_mask
);
1441 /* ensure that CQEs for newly created QP (whose id may be same with
1442 * one which just getting destroyed are same), dont get
1443 * discarded until the old CQEs are discarded.
1445 mutex_lock(&dev
->dev_lock
);
1446 status
= ocrdma_mbx_destroy_qp(dev
, qp
);
1449 * acquire CQ lock while destroy is in progress, in order to
1450 * protect against proessing in-flight CQEs for this QP.
1452 spin_lock_irqsave(&qp
->sq_cq
->cq_lock
, flags
);
1453 if (qp
->rq_cq
&& (qp
->rq_cq
!= qp
->sq_cq
))
1454 spin_lock(&qp
->rq_cq
->cq_lock
);
1456 ocrdma_del_qpn_map(dev
, qp
);
1458 if (qp
->rq_cq
&& (qp
->rq_cq
!= qp
->sq_cq
))
1459 spin_unlock(&qp
->rq_cq
->cq_lock
);
1460 spin_unlock_irqrestore(&qp
->sq_cq
->cq_lock
, flags
);
1463 ocrdma_discard_cqes(qp
, qp
->sq_cq
);
1464 ocrdma_discard_cqes(qp
, qp
->rq_cq
);
1466 mutex_unlock(&dev
->dev_lock
);
1469 ocrdma_del_mmap(pd
->uctx
, (u64
) qp
->sq
.pa
, qp
->sq
.len
);
1471 ocrdma_del_mmap(pd
->uctx
, (u64
) qp
->rq
.pa
, qp
->rq
.len
);
1474 ocrdma_del_flush_qp(qp
);
1476 atomic_dec(&qp
->pd
->use_cnt
);
1477 atomic_dec(&qp
->sq_cq
->use_cnt
);
1478 atomic_dec(&qp
->rq_cq
->use_cnt
);
1480 atomic_dec(&qp
->srq
->use_cnt
);
1481 kfree(qp
->wqe_wr_id_tbl
);
1482 kfree(qp
->rqe_wr_id_tbl
);
1487 static int ocrdma_copy_srq_uresp(struct ocrdma_srq
*srq
, struct ib_udata
*udata
)
1490 struct ocrdma_create_srq_uresp uresp
;
1492 uresp
.rq_dbid
= srq
->rq
.dbid
;
1493 uresp
.num_rq_pages
= 1;
1494 uresp
.rq_page_addr
[0] = srq
->rq
.pa
;
1495 uresp
.rq_page_size
= srq
->rq
.len
;
1496 uresp
.db_page_addr
= srq
->dev
->nic_info
.unmapped_db
+
1497 (srq
->pd
->id
* srq
->dev
->nic_info
.db_page_size
);
1498 uresp
.db_page_size
= srq
->dev
->nic_info
.db_page_size
;
1499 uresp
.num_rqe_allocated
= srq
->rq
.max_cnt
;
1500 if (srq
->dev
->nic_info
.dev_family
== OCRDMA_GEN2_FAMILY
) {
1501 uresp
.db_rq_offset
= OCRDMA_DB_GEN2_RQ1_OFFSET
;
1502 uresp
.db_shift
= 24;
1504 uresp
.db_rq_offset
= OCRDMA_DB_RQ_OFFSET
;
1505 uresp
.db_shift
= 16;
1508 status
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
1511 status
= ocrdma_add_mmap(srq
->pd
->uctx
, uresp
.rq_page_addr
[0],
1512 uresp
.rq_page_size
);
1518 struct ib_srq
*ocrdma_create_srq(struct ib_pd
*ibpd
,
1519 struct ib_srq_init_attr
*init_attr
,
1520 struct ib_udata
*udata
)
1522 int status
= -ENOMEM
;
1523 struct ocrdma_pd
*pd
= get_ocrdma_pd(ibpd
);
1524 struct ocrdma_dev
*dev
= pd
->dev
;
1525 struct ocrdma_srq
*srq
;
1527 if (init_attr
->attr
.max_sge
> dev
->attr
.max_recv_sge
)
1528 return ERR_PTR(-EINVAL
);
1529 if (init_attr
->attr
.max_wr
> dev
->attr
.max_rqe
)
1530 return ERR_PTR(-EINVAL
);
1532 srq
= kzalloc(sizeof(*srq
), GFP_KERNEL
);
1534 return ERR_PTR(status
);
1536 spin_lock_init(&srq
->q_lock
);
1539 srq
->db
= dev
->nic_info
.db
+ (pd
->id
* dev
->nic_info
.db_page_size
);
1540 status
= ocrdma_mbx_create_srq(srq
, init_attr
, pd
);
1544 if (udata
== NULL
) {
1545 srq
->rqe_wr_id_tbl
= kzalloc(sizeof(u64
) * srq
->rq
.max_cnt
,
1547 if (srq
->rqe_wr_id_tbl
== NULL
)
1550 srq
->bit_fields_len
= (srq
->rq
.max_cnt
/ 32) +
1551 (srq
->rq
.max_cnt
% 32 ? 1 : 0);
1552 srq
->idx_bit_fields
=
1553 kmalloc(srq
->bit_fields_len
* sizeof(u32
), GFP_KERNEL
);
1554 if (srq
->idx_bit_fields
== NULL
)
1556 memset(srq
->idx_bit_fields
, 0xff,
1557 srq
->bit_fields_len
* sizeof(u32
));
1560 if (init_attr
->attr
.srq_limit
) {
1561 status
= ocrdma_mbx_modify_srq(srq
, &init_attr
->attr
);
1566 atomic_set(&srq
->use_cnt
, 0);
1568 status
= ocrdma_copy_srq_uresp(srq
, udata
);
1573 atomic_inc(&pd
->use_cnt
);
1577 ocrdma_mbx_destroy_srq(dev
, srq
);
1579 kfree(srq
->rqe_wr_id_tbl
);
1580 kfree(srq
->idx_bit_fields
);
1582 return ERR_PTR(status
);
1585 int ocrdma_modify_srq(struct ib_srq
*ibsrq
,
1586 struct ib_srq_attr
*srq_attr
,
1587 enum ib_srq_attr_mask srq_attr_mask
,
1588 struct ib_udata
*udata
)
1591 struct ocrdma_srq
*srq
;
1593 srq
= get_ocrdma_srq(ibsrq
);
1594 if (srq_attr_mask
& IB_SRQ_MAX_WR
)
1597 status
= ocrdma_mbx_modify_srq(srq
, srq_attr
);
1601 int ocrdma_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
)
1604 struct ocrdma_srq
*srq
;
1606 srq
= get_ocrdma_srq(ibsrq
);
1607 status
= ocrdma_mbx_query_srq(srq
, srq_attr
);
1611 int ocrdma_destroy_srq(struct ib_srq
*ibsrq
)
1614 struct ocrdma_srq
*srq
;
1615 struct ocrdma_dev
*dev
;
1617 srq
= get_ocrdma_srq(ibsrq
);
1619 if (atomic_read(&srq
->use_cnt
)) {
1620 ocrdma_err("%s(%d) err, srq=0x%x in use\n",
1621 __func__
, dev
->id
, srq
->id
);
1625 status
= ocrdma_mbx_destroy_srq(dev
, srq
);
1628 ocrdma_del_mmap(srq
->pd
->uctx
, (u64
) srq
->rq
.pa
, srq
->rq
.len
);
1630 atomic_dec(&srq
->pd
->use_cnt
);
1631 kfree(srq
->idx_bit_fields
);
1632 kfree(srq
->rqe_wr_id_tbl
);
1637 /* unprivileged verbs and their support functions. */
1638 static void ocrdma_build_ud_hdr(struct ocrdma_qp
*qp
,
1639 struct ocrdma_hdr_wqe
*hdr
,
1640 struct ib_send_wr
*wr
)
1642 struct ocrdma_ewqe_ud_hdr
*ud_hdr
=
1643 (struct ocrdma_ewqe_ud_hdr
*)(hdr
+ 1);
1644 struct ocrdma_ah
*ah
= get_ocrdma_ah(wr
->wr
.ud
.ah
);
1646 ud_hdr
->rsvd_dest_qpn
= wr
->wr
.ud
.remote_qpn
;
1647 if (qp
->qp_type
== IB_QPT_GSI
)
1648 ud_hdr
->qkey
= qp
->qkey
;
1650 ud_hdr
->qkey
= wr
->wr
.ud
.remote_qkey
;
1651 ud_hdr
->rsvd_ahid
= ah
->id
;
1654 static void ocrdma_build_sges(struct ocrdma_hdr_wqe
*hdr
,
1655 struct ocrdma_sge
*sge
, int num_sge
,
1656 struct ib_sge
*sg_list
)
1660 for (i
= 0; i
< num_sge
; i
++) {
1661 sge
[i
].lrkey
= sg_list
[i
].lkey
;
1662 sge
[i
].addr_lo
= sg_list
[i
].addr
;
1663 sge
[i
].addr_hi
= upper_32_bits(sg_list
[i
].addr
);
1664 sge
[i
].len
= sg_list
[i
].length
;
1665 hdr
->total_len
+= sg_list
[i
].length
;
1668 memset(sge
, 0, sizeof(*sge
));
1671 static int ocrdma_build_inline_sges(struct ocrdma_qp
*qp
,
1672 struct ocrdma_hdr_wqe
*hdr
,
1673 struct ocrdma_sge
*sge
,
1674 struct ib_send_wr
*wr
, u32 wqe_size
)
1676 if (wr
->send_flags
& IB_SEND_INLINE
) {
1677 if (wr
->sg_list
[0].length
> qp
->max_inline_data
) {
1678 ocrdma_err("%s() supported_len=0x%x,"
1679 " unspported len req=0x%x\n", __func__
,
1680 qp
->max_inline_data
, wr
->sg_list
[0].length
);
1684 (void *)(unsigned long)wr
->sg_list
[0].addr
,
1685 wr
->sg_list
[0].length
);
1686 hdr
->total_len
= wr
->sg_list
[0].length
;
1687 wqe_size
+= roundup(hdr
->total_len
, OCRDMA_WQE_ALIGN_BYTES
);
1688 hdr
->cw
|= (OCRDMA_TYPE_INLINE
<< OCRDMA_WQE_TYPE_SHIFT
);
1690 ocrdma_build_sges(hdr
, sge
, wr
->num_sge
, wr
->sg_list
);
1692 wqe_size
+= (wr
->num_sge
* sizeof(struct ocrdma_sge
));
1694 wqe_size
+= sizeof(struct ocrdma_sge
);
1695 hdr
->cw
|= (OCRDMA_TYPE_LKEY
<< OCRDMA_WQE_TYPE_SHIFT
);
1697 hdr
->cw
|= ((wqe_size
/ OCRDMA_WQE_STRIDE
) << OCRDMA_WQE_SIZE_SHIFT
);
1701 static int ocrdma_build_send(struct ocrdma_qp
*qp
, struct ocrdma_hdr_wqe
*hdr
,
1702 struct ib_send_wr
*wr
)
1705 struct ocrdma_sge
*sge
;
1706 u32 wqe_size
= sizeof(*hdr
);
1708 if (qp
->qp_type
== IB_QPT_UD
|| qp
->qp_type
== IB_QPT_GSI
) {
1709 ocrdma_build_ud_hdr(qp
, hdr
, wr
);
1710 sge
= (struct ocrdma_sge
*)(hdr
+ 2);
1711 wqe_size
+= sizeof(struct ocrdma_ewqe_ud_hdr
);
1713 sge
= (struct ocrdma_sge
*)(hdr
+ 1);
1715 status
= ocrdma_build_inline_sges(qp
, hdr
, sge
, wr
, wqe_size
);
1719 static int ocrdma_build_write(struct ocrdma_qp
*qp
, struct ocrdma_hdr_wqe
*hdr
,
1720 struct ib_send_wr
*wr
)
1723 struct ocrdma_sge
*ext_rw
= (struct ocrdma_sge
*)(hdr
+ 1);
1724 struct ocrdma_sge
*sge
= ext_rw
+ 1;
1725 u32 wqe_size
= sizeof(*hdr
) + sizeof(*ext_rw
);
1727 status
= ocrdma_build_inline_sges(qp
, hdr
, sge
, wr
, wqe_size
);
1730 ext_rw
->addr_lo
= wr
->wr
.rdma
.remote_addr
;
1731 ext_rw
->addr_hi
= upper_32_bits(wr
->wr
.rdma
.remote_addr
);
1732 ext_rw
->lrkey
= wr
->wr
.rdma
.rkey
;
1733 ext_rw
->len
= hdr
->total_len
;
1737 static void ocrdma_build_read(struct ocrdma_qp
*qp
, struct ocrdma_hdr_wqe
*hdr
,
1738 struct ib_send_wr
*wr
)
1740 struct ocrdma_sge
*ext_rw
= (struct ocrdma_sge
*)(hdr
+ 1);
1741 struct ocrdma_sge
*sge
= ext_rw
+ 1;
1742 u32 wqe_size
= ((wr
->num_sge
+ 1) * sizeof(struct ocrdma_sge
)) +
1743 sizeof(struct ocrdma_hdr_wqe
);
1745 ocrdma_build_sges(hdr
, sge
, wr
->num_sge
, wr
->sg_list
);
1746 hdr
->cw
|= ((wqe_size
/ OCRDMA_WQE_STRIDE
) << OCRDMA_WQE_SIZE_SHIFT
);
1747 hdr
->cw
|= (OCRDMA_READ
<< OCRDMA_WQE_OPCODE_SHIFT
);
1748 hdr
->cw
|= (OCRDMA_TYPE_LKEY
<< OCRDMA_WQE_TYPE_SHIFT
);
1750 ext_rw
->addr_lo
= wr
->wr
.rdma
.remote_addr
;
1751 ext_rw
->addr_hi
= upper_32_bits(wr
->wr
.rdma
.remote_addr
);
1752 ext_rw
->lrkey
= wr
->wr
.rdma
.rkey
;
1753 ext_rw
->len
= hdr
->total_len
;
1756 static void ocrdma_ring_sq_db(struct ocrdma_qp
*qp
)
1758 u32 val
= qp
->sq
.dbid
| (1 << 16);
1760 iowrite32(val
, qp
->sq_db
);
1763 int ocrdma_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
1764 struct ib_send_wr
**bad_wr
)
1767 struct ocrdma_qp
*qp
= get_ocrdma_qp(ibqp
);
1768 struct ocrdma_hdr_wqe
*hdr
;
1769 unsigned long flags
;
1771 spin_lock_irqsave(&qp
->q_lock
, flags
);
1772 if (qp
->state
!= OCRDMA_QPS_RTS
&& qp
->state
!= OCRDMA_QPS_SQD
) {
1773 spin_unlock_irqrestore(&qp
->q_lock
, flags
);
1778 if (ocrdma_hwq_free_cnt(&qp
->sq
) == 0 ||
1779 wr
->num_sge
> qp
->sq
.max_sges
) {
1783 hdr
= ocrdma_hwq_head(&qp
->sq
);
1785 if (wr
->send_flags
& IB_SEND_SIGNALED
)
1786 hdr
->cw
|= (OCRDMA_FLAG_SIG
<< OCRDMA_WQE_FLAGS_SHIFT
);
1787 if (wr
->send_flags
& IB_SEND_FENCE
)
1789 (OCRDMA_FLAG_FENCE_L
<< OCRDMA_WQE_FLAGS_SHIFT
);
1790 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1792 (OCRDMA_FLAG_SOLICIT
<< OCRDMA_WQE_FLAGS_SHIFT
);
1794 switch (wr
->opcode
) {
1795 case IB_WR_SEND_WITH_IMM
:
1796 hdr
->cw
|= (OCRDMA_FLAG_IMM
<< OCRDMA_WQE_FLAGS_SHIFT
);
1797 hdr
->immdt
= ntohl(wr
->ex
.imm_data
);
1799 hdr
->cw
|= (OCRDMA_SEND
<< OCRDMA_WQE_OPCODE_SHIFT
);
1800 ocrdma_build_send(qp
, hdr
, wr
);
1802 case IB_WR_SEND_WITH_INV
:
1803 hdr
->cw
|= (OCRDMA_FLAG_INV
<< OCRDMA_WQE_FLAGS_SHIFT
);
1804 hdr
->cw
|= (OCRDMA_SEND
<< OCRDMA_WQE_OPCODE_SHIFT
);
1805 hdr
->lkey
= wr
->ex
.invalidate_rkey
;
1806 status
= ocrdma_build_send(qp
, hdr
, wr
);
1808 case IB_WR_RDMA_WRITE_WITH_IMM
:
1809 hdr
->cw
|= (OCRDMA_FLAG_IMM
<< OCRDMA_WQE_FLAGS_SHIFT
);
1810 hdr
->immdt
= ntohl(wr
->ex
.imm_data
);
1811 case IB_WR_RDMA_WRITE
:
1812 hdr
->cw
|= (OCRDMA_WRITE
<< OCRDMA_WQE_OPCODE_SHIFT
);
1813 status
= ocrdma_build_write(qp
, hdr
, wr
);
1815 case IB_WR_RDMA_READ_WITH_INV
:
1816 hdr
->cw
|= (OCRDMA_FLAG_INV
<< OCRDMA_WQE_FLAGS_SHIFT
);
1817 case IB_WR_RDMA_READ
:
1818 ocrdma_build_read(qp
, hdr
, wr
);
1820 case IB_WR_LOCAL_INV
:
1822 (OCRDMA_LKEY_INV
<< OCRDMA_WQE_OPCODE_SHIFT
);
1823 hdr
->cw
|= (sizeof(struct ocrdma_hdr_wqe
) /
1824 OCRDMA_WQE_STRIDE
) << OCRDMA_WQE_SIZE_SHIFT
;
1825 hdr
->lkey
= wr
->ex
.invalidate_rkey
;
1835 if (wr
->send_flags
& IB_SEND_SIGNALED
)
1836 qp
->wqe_wr_id_tbl
[qp
->sq
.head
].signaled
= 1;
1838 qp
->wqe_wr_id_tbl
[qp
->sq
.head
].signaled
= 0;
1839 qp
->wqe_wr_id_tbl
[qp
->sq
.head
].wrid
= wr
->wr_id
;
1840 ocrdma_cpu_to_le32(hdr
, ((hdr
->cw
>> OCRDMA_WQE_SIZE_SHIFT
) &
1841 OCRDMA_WQE_SIZE_MASK
) * OCRDMA_WQE_STRIDE
);
1842 /* make sure wqe is written before adapter can access it */
1844 /* inform hw to start processing it */
1845 ocrdma_ring_sq_db(qp
);
1847 /* update pointer, counter for next wr */
1848 ocrdma_hwq_inc_head(&qp
->sq
);
1851 spin_unlock_irqrestore(&qp
->q_lock
, flags
);
1855 static void ocrdma_ring_rq_db(struct ocrdma_qp
*qp
)
1857 u32 val
= qp
->rq
.dbid
| (1 << OCRDMA_GET_NUM_POSTED_SHIFT_VAL(qp
));
1859 iowrite32(val
, qp
->rq_db
);
1862 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe
*rqe
, struct ib_recv_wr
*wr
,
1866 struct ocrdma_sge
*sge
;
1868 wqe_size
= (wr
->num_sge
* sizeof(*sge
)) + sizeof(*rqe
);
1870 wqe_size
= sizeof(*sge
) + sizeof(*rqe
);
1872 rqe
->cw
= ((wqe_size
/ OCRDMA_WQE_STRIDE
) <<
1873 OCRDMA_WQE_SIZE_SHIFT
);
1874 rqe
->cw
|= (OCRDMA_FLAG_SIG
<< OCRDMA_WQE_FLAGS_SHIFT
);
1875 rqe
->cw
|= (OCRDMA_TYPE_LKEY
<< OCRDMA_WQE_TYPE_SHIFT
);
1877 rqe
->rsvd_tag
= tag
;
1878 sge
= (struct ocrdma_sge
*)(rqe
+ 1);
1879 ocrdma_build_sges(rqe
, sge
, wr
->num_sge
, wr
->sg_list
);
1880 ocrdma_cpu_to_le32(rqe
, wqe_size
);
1883 int ocrdma_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
1884 struct ib_recv_wr
**bad_wr
)
1887 unsigned long flags
;
1888 struct ocrdma_qp
*qp
= get_ocrdma_qp(ibqp
);
1889 struct ocrdma_hdr_wqe
*rqe
;
1891 spin_lock_irqsave(&qp
->q_lock
, flags
);
1892 if (qp
->state
== OCRDMA_QPS_RST
|| qp
->state
== OCRDMA_QPS_ERR
) {
1893 spin_unlock_irqrestore(&qp
->q_lock
, flags
);
1898 if (ocrdma_hwq_free_cnt(&qp
->rq
) == 0 ||
1899 wr
->num_sge
> qp
->rq
.max_sges
) {
1904 rqe
= ocrdma_hwq_head(&qp
->rq
);
1905 ocrdma_build_rqe(rqe
, wr
, 0);
1907 qp
->rqe_wr_id_tbl
[qp
->rq
.head
] = wr
->wr_id
;
1908 /* make sure rqe is written before adapter can access it */
1911 /* inform hw to start processing it */
1912 ocrdma_ring_rq_db(qp
);
1914 /* update pointer, counter for next wr */
1915 ocrdma_hwq_inc_head(&qp
->rq
);
1918 spin_unlock_irqrestore(&qp
->q_lock
, flags
);
1922 /* cqe for srq's rqe can potentially arrive out of order.
1923 * index gives the entry in the shadow table where to store
1924 * the wr_id. tag/index is returned in cqe to reference back
1927 static int ocrdma_srq_get_idx(struct ocrdma_srq
*srq
)
1932 for (row
= 0; row
< srq
->bit_fields_len
; row
++) {
1933 if (srq
->idx_bit_fields
[row
]) {
1934 indx
= ffs(srq
->idx_bit_fields
[row
]);
1935 indx
= (row
* 32) + (indx
- 1);
1936 if (indx
>= srq
->rq
.max_cnt
)
1938 ocrdma_srq_toggle_bit(srq
, indx
);
1943 if (row
== srq
->bit_fields_len
)
1948 static void ocrdma_ring_srq_db(struct ocrdma_srq
*srq
)
1950 u32 val
= srq
->rq
.dbid
| (1 << 16);
1952 iowrite32(val
, srq
->db
+ OCRDMA_DB_GEN2_SRQ_OFFSET
);
1955 int ocrdma_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
1956 struct ib_recv_wr
**bad_wr
)
1959 unsigned long flags
;
1960 struct ocrdma_srq
*srq
;
1961 struct ocrdma_hdr_wqe
*rqe
;
1964 srq
= get_ocrdma_srq(ibsrq
);
1966 spin_lock_irqsave(&srq
->q_lock
, flags
);
1968 if (ocrdma_hwq_free_cnt(&srq
->rq
) == 0 ||
1969 wr
->num_sge
> srq
->rq
.max_sges
) {
1974 tag
= ocrdma_srq_get_idx(srq
);
1975 rqe
= ocrdma_hwq_head(&srq
->rq
);
1976 ocrdma_build_rqe(rqe
, wr
, tag
);
1978 srq
->rqe_wr_id_tbl
[tag
] = wr
->wr_id
;
1979 /* make sure rqe is written before adapter can perform DMA */
1981 /* inform hw to start processing it */
1982 ocrdma_ring_srq_db(srq
);
1983 /* update pointer, counter for next wr */
1984 ocrdma_hwq_inc_head(&srq
->rq
);
1987 spin_unlock_irqrestore(&srq
->q_lock
, flags
);
1991 static enum ib_wc_status
ocrdma_to_ibwc_err(u16 status
)
1993 enum ib_wc_status ibwc_status
= IB_WC_GENERAL_ERR
;
1996 case OCRDMA_CQE_GENERAL_ERR
:
1997 ibwc_status
= IB_WC_GENERAL_ERR
;
1999 case OCRDMA_CQE_LOC_LEN_ERR
:
2000 ibwc_status
= IB_WC_LOC_LEN_ERR
;
2002 case OCRDMA_CQE_LOC_QP_OP_ERR
:
2003 ibwc_status
= IB_WC_LOC_QP_OP_ERR
;
2005 case OCRDMA_CQE_LOC_EEC_OP_ERR
:
2006 ibwc_status
= IB_WC_LOC_EEC_OP_ERR
;
2008 case OCRDMA_CQE_LOC_PROT_ERR
:
2009 ibwc_status
= IB_WC_LOC_PROT_ERR
;
2011 case OCRDMA_CQE_WR_FLUSH_ERR
:
2012 ibwc_status
= IB_WC_WR_FLUSH_ERR
;
2014 case OCRDMA_CQE_MW_BIND_ERR
:
2015 ibwc_status
= IB_WC_MW_BIND_ERR
;
2017 case OCRDMA_CQE_BAD_RESP_ERR
:
2018 ibwc_status
= IB_WC_BAD_RESP_ERR
;
2020 case OCRDMA_CQE_LOC_ACCESS_ERR
:
2021 ibwc_status
= IB_WC_LOC_ACCESS_ERR
;
2023 case OCRDMA_CQE_REM_INV_REQ_ERR
:
2024 ibwc_status
= IB_WC_REM_INV_REQ_ERR
;
2026 case OCRDMA_CQE_REM_ACCESS_ERR
:
2027 ibwc_status
= IB_WC_REM_ACCESS_ERR
;
2029 case OCRDMA_CQE_REM_OP_ERR
:
2030 ibwc_status
= IB_WC_REM_OP_ERR
;
2032 case OCRDMA_CQE_RETRY_EXC_ERR
:
2033 ibwc_status
= IB_WC_RETRY_EXC_ERR
;
2035 case OCRDMA_CQE_RNR_RETRY_EXC_ERR
:
2036 ibwc_status
= IB_WC_RNR_RETRY_EXC_ERR
;
2038 case OCRDMA_CQE_LOC_RDD_VIOL_ERR
:
2039 ibwc_status
= IB_WC_LOC_RDD_VIOL_ERR
;
2041 case OCRDMA_CQE_REM_INV_RD_REQ_ERR
:
2042 ibwc_status
= IB_WC_REM_INV_RD_REQ_ERR
;
2044 case OCRDMA_CQE_REM_ABORT_ERR
:
2045 ibwc_status
= IB_WC_REM_ABORT_ERR
;
2047 case OCRDMA_CQE_INV_EECN_ERR
:
2048 ibwc_status
= IB_WC_INV_EECN_ERR
;
2050 case OCRDMA_CQE_INV_EEC_STATE_ERR
:
2051 ibwc_status
= IB_WC_INV_EEC_STATE_ERR
;
2053 case OCRDMA_CQE_FATAL_ERR
:
2054 ibwc_status
= IB_WC_FATAL_ERR
;
2056 case OCRDMA_CQE_RESP_TIMEOUT_ERR
:
2057 ibwc_status
= IB_WC_RESP_TIMEOUT_ERR
;
2060 ibwc_status
= IB_WC_GENERAL_ERR
;
2066 static void ocrdma_update_wc(struct ocrdma_qp
*qp
, struct ib_wc
*ibwc
,
2069 struct ocrdma_hdr_wqe
*hdr
;
2070 struct ocrdma_sge
*rw
;
2073 hdr
= ocrdma_hwq_head_from_idx(&qp
->sq
, wqe_idx
);
2075 ibwc
->wr_id
= qp
->wqe_wr_id_tbl
[wqe_idx
].wrid
;
2076 /* Undo the hdr->cw swap */
2077 opcode
= le32_to_cpu(hdr
->cw
) & OCRDMA_WQE_OPCODE_MASK
;
2080 ibwc
->opcode
= IB_WC_RDMA_WRITE
;
2083 rw
= (struct ocrdma_sge
*)(hdr
+ 1);
2084 ibwc
->opcode
= IB_WC_RDMA_READ
;
2085 ibwc
->byte_len
= rw
->len
;
2088 ibwc
->opcode
= IB_WC_SEND
;
2090 case OCRDMA_LKEY_INV
:
2091 ibwc
->opcode
= IB_WC_LOCAL_INV
;
2094 ibwc
->status
= IB_WC_GENERAL_ERR
;
2095 ocrdma_err("%s() invalid opcode received = 0x%x\n",
2096 __func__
, hdr
->cw
& OCRDMA_WQE_OPCODE_MASK
);
2101 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp
*qp
,
2102 struct ocrdma_cqe
*cqe
)
2104 if (is_cqe_for_sq(cqe
)) {
2105 cqe
->flags_status_srcqpn
= cpu_to_le32(le32_to_cpu(
2106 cqe
->flags_status_srcqpn
) &
2107 ~OCRDMA_CQE_STATUS_MASK
);
2108 cqe
->flags_status_srcqpn
= cpu_to_le32(le32_to_cpu(
2109 cqe
->flags_status_srcqpn
) |
2110 (OCRDMA_CQE_WR_FLUSH_ERR
<<
2111 OCRDMA_CQE_STATUS_SHIFT
));
2113 if (qp
->qp_type
== IB_QPT_UD
|| qp
->qp_type
== IB_QPT_GSI
) {
2114 cqe
->flags_status_srcqpn
= cpu_to_le32(le32_to_cpu(
2115 cqe
->flags_status_srcqpn
) &
2116 ~OCRDMA_CQE_UD_STATUS_MASK
);
2117 cqe
->flags_status_srcqpn
= cpu_to_le32(le32_to_cpu(
2118 cqe
->flags_status_srcqpn
) |
2119 (OCRDMA_CQE_WR_FLUSH_ERR
<<
2120 OCRDMA_CQE_UD_STATUS_SHIFT
));
2122 cqe
->flags_status_srcqpn
= cpu_to_le32(le32_to_cpu(
2123 cqe
->flags_status_srcqpn
) &
2124 ~OCRDMA_CQE_STATUS_MASK
);
2125 cqe
->flags_status_srcqpn
= cpu_to_le32(le32_to_cpu(
2126 cqe
->flags_status_srcqpn
) |
2127 (OCRDMA_CQE_WR_FLUSH_ERR
<<
2128 OCRDMA_CQE_STATUS_SHIFT
));
2133 static bool ocrdma_update_err_cqe(struct ib_wc
*ibwc
, struct ocrdma_cqe
*cqe
,
2134 struct ocrdma_qp
*qp
, int status
)
2136 bool expand
= false;
2139 ibwc
->qp
= &qp
->ibqp
;
2140 ibwc
->status
= ocrdma_to_ibwc_err(status
);
2142 ocrdma_flush_qp(qp
);
2143 ocrdma_qp_state_machine(qp
, IB_QPS_ERR
, NULL
);
2145 /* if wqe/rqe pending for which cqe needs to be returned,
2146 * trigger inflating it.
2148 if (!is_hw_rq_empty(qp
) || !is_hw_sq_empty(qp
)) {
2150 ocrdma_set_cqe_status_flushed(qp
, cqe
);
2155 static int ocrdma_update_err_rcqe(struct ib_wc
*ibwc
, struct ocrdma_cqe
*cqe
,
2156 struct ocrdma_qp
*qp
, int status
)
2158 ibwc
->opcode
= IB_WC_RECV
;
2159 ibwc
->wr_id
= qp
->rqe_wr_id_tbl
[qp
->rq
.tail
];
2160 ocrdma_hwq_inc_tail(&qp
->rq
);
2162 return ocrdma_update_err_cqe(ibwc
, cqe
, qp
, status
);
2165 static int ocrdma_update_err_scqe(struct ib_wc
*ibwc
, struct ocrdma_cqe
*cqe
,
2166 struct ocrdma_qp
*qp
, int status
)
2168 ocrdma_update_wc(qp
, ibwc
, qp
->sq
.tail
);
2169 ocrdma_hwq_inc_tail(&qp
->sq
);
2171 return ocrdma_update_err_cqe(ibwc
, cqe
, qp
, status
);
2175 static bool ocrdma_poll_err_scqe(struct ocrdma_qp
*qp
,
2176 struct ocrdma_cqe
*cqe
, struct ib_wc
*ibwc
,
2177 bool *polled
, bool *stop
)
2180 int status
= (le32_to_cpu(cqe
->flags_status_srcqpn
) &
2181 OCRDMA_CQE_STATUS_MASK
) >> OCRDMA_CQE_STATUS_SHIFT
;
2183 /* when hw sq is empty, but rq is not empty, so we continue
2184 * to keep the cqe in order to get the cq event again.
2186 if (is_hw_sq_empty(qp
) && !is_hw_rq_empty(qp
)) {
2187 /* when cq for rq and sq is same, it is safe to return
2188 * flush cqe for RQEs.
2190 if (!qp
->srq
&& (qp
->sq_cq
== qp
->rq_cq
)) {
2192 status
= OCRDMA_CQE_WR_FLUSH_ERR
;
2193 expand
= ocrdma_update_err_rcqe(ibwc
, cqe
, qp
, status
);
2195 /* stop processing further cqe as this cqe is used for
2196 * triggering cq event on buddy cq of RQ.
2197 * When QP is destroyed, this cqe will be removed
2198 * from the cq's hardware q.
2206 expand
= ocrdma_update_err_scqe(ibwc
, cqe
, qp
, status
);
2211 static bool ocrdma_poll_success_scqe(struct ocrdma_qp
*qp
,
2212 struct ocrdma_cqe
*cqe
,
2213 struct ib_wc
*ibwc
, bool *polled
)
2215 bool expand
= false;
2216 int tail
= qp
->sq
.tail
;
2219 if (!qp
->wqe_wr_id_tbl
[tail
].signaled
) {
2220 expand
= true; /* CQE cannot be consumed yet */
2221 *polled
= false; /* WC cannot be consumed yet */
2223 ibwc
->status
= IB_WC_SUCCESS
;
2225 ibwc
->qp
= &qp
->ibqp
;
2226 ocrdma_update_wc(qp
, ibwc
, tail
);
2228 wqe_idx
= le32_to_cpu(cqe
->wq
.wqeidx
) & OCRDMA_CQE_WQEIDX_MASK
;
2229 if (tail
!= wqe_idx
)
2230 expand
= true; /* Coalesced CQE can't be consumed yet */
2232 ocrdma_hwq_inc_tail(&qp
->sq
);
2236 static bool ocrdma_poll_scqe(struct ocrdma_qp
*qp
, struct ocrdma_cqe
*cqe
,
2237 struct ib_wc
*ibwc
, bool *polled
, bool *stop
)
2242 status
= (le32_to_cpu(cqe
->flags_status_srcqpn
) &
2243 OCRDMA_CQE_STATUS_MASK
) >> OCRDMA_CQE_STATUS_SHIFT
;
2245 if (status
== OCRDMA_CQE_SUCCESS
)
2246 expand
= ocrdma_poll_success_scqe(qp
, cqe
, ibwc
, polled
);
2248 expand
= ocrdma_poll_err_scqe(qp
, cqe
, ibwc
, polled
, stop
);
2252 static int ocrdma_update_ud_rcqe(struct ib_wc
*ibwc
, struct ocrdma_cqe
*cqe
)
2256 status
= (le32_to_cpu(cqe
->flags_status_srcqpn
) &
2257 OCRDMA_CQE_UD_STATUS_MASK
) >> OCRDMA_CQE_UD_STATUS_SHIFT
;
2258 ibwc
->src_qp
= le32_to_cpu(cqe
->flags_status_srcqpn
) &
2259 OCRDMA_CQE_SRCQP_MASK
;
2260 ibwc
->pkey_index
= le32_to_cpu(cqe
->ud
.rxlen_pkey
) &
2261 OCRDMA_CQE_PKEY_MASK
;
2262 ibwc
->wc_flags
= IB_WC_GRH
;
2263 ibwc
->byte_len
= (le32_to_cpu(cqe
->ud
.rxlen_pkey
) >>
2264 OCRDMA_CQE_UD_XFER_LEN_SHIFT
);
2268 static void ocrdma_update_free_srq_cqe(struct ib_wc
*ibwc
,
2269 struct ocrdma_cqe
*cqe
,
2270 struct ocrdma_qp
*qp
)
2272 unsigned long flags
;
2273 struct ocrdma_srq
*srq
;
2276 srq
= get_ocrdma_srq(qp
->ibqp
.srq
);
2277 wqe_idx
= le32_to_cpu(cqe
->rq
.buftag_qpn
) >> OCRDMA_CQE_BUFTAG_SHIFT
;
2278 ibwc
->wr_id
= srq
->rqe_wr_id_tbl
[wqe_idx
];
2279 spin_lock_irqsave(&srq
->q_lock
, flags
);
2280 ocrdma_srq_toggle_bit(srq
, wqe_idx
);
2281 spin_unlock_irqrestore(&srq
->q_lock
, flags
);
2282 ocrdma_hwq_inc_tail(&srq
->rq
);
2285 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp
*qp
, struct ocrdma_cqe
*cqe
,
2286 struct ib_wc
*ibwc
, bool *polled
, bool *stop
,
2291 /* when hw_rq is empty, but wq is not empty, so continue
2292 * to keep the cqe to get the cq event again.
2294 if (is_hw_rq_empty(qp
) && !is_hw_sq_empty(qp
)) {
2295 if (!qp
->srq
&& (qp
->sq_cq
== qp
->rq_cq
)) {
2297 status
= OCRDMA_CQE_WR_FLUSH_ERR
;
2298 expand
= ocrdma_update_err_scqe(ibwc
, cqe
, qp
, status
);
2306 expand
= ocrdma_update_err_rcqe(ibwc
, cqe
, qp
, status
);
2311 static void ocrdma_poll_success_rcqe(struct ocrdma_qp
*qp
,
2312 struct ocrdma_cqe
*cqe
, struct ib_wc
*ibwc
)
2314 ibwc
->opcode
= IB_WC_RECV
;
2315 ibwc
->qp
= &qp
->ibqp
;
2316 ibwc
->status
= IB_WC_SUCCESS
;
2318 if (qp
->qp_type
== IB_QPT_UD
|| qp
->qp_type
== IB_QPT_GSI
)
2319 ocrdma_update_ud_rcqe(ibwc
, cqe
);
2321 ibwc
->byte_len
= le32_to_cpu(cqe
->rq
.rxlen
);
2323 if (is_cqe_imm(cqe
)) {
2324 ibwc
->ex
.imm_data
= htonl(le32_to_cpu(cqe
->rq
.lkey_immdt
));
2325 ibwc
->wc_flags
|= IB_WC_WITH_IMM
;
2326 } else if (is_cqe_wr_imm(cqe
)) {
2327 ibwc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
2328 ibwc
->ex
.imm_data
= htonl(le32_to_cpu(cqe
->rq
.lkey_immdt
));
2329 ibwc
->wc_flags
|= IB_WC_WITH_IMM
;
2330 } else if (is_cqe_invalidated(cqe
)) {
2331 ibwc
->ex
.invalidate_rkey
= le32_to_cpu(cqe
->rq
.lkey_immdt
);
2332 ibwc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
2335 ocrdma_update_free_srq_cqe(ibwc
, cqe
, qp
);
2337 ibwc
->wr_id
= qp
->rqe_wr_id_tbl
[qp
->rq
.tail
];
2338 ocrdma_hwq_inc_tail(&qp
->rq
);
2342 static bool ocrdma_poll_rcqe(struct ocrdma_qp
*qp
, struct ocrdma_cqe
*cqe
,
2343 struct ib_wc
*ibwc
, bool *polled
, bool *stop
)
2346 bool expand
= false;
2349 if (qp
->qp_type
== IB_QPT_UD
|| qp
->qp_type
== IB_QPT_GSI
)
2350 status
= (le32_to_cpu(cqe
->flags_status_srcqpn
) &
2351 OCRDMA_CQE_UD_STATUS_MASK
) >>
2352 OCRDMA_CQE_UD_STATUS_SHIFT
;
2354 status
= (le32_to_cpu(cqe
->flags_status_srcqpn
) &
2355 OCRDMA_CQE_STATUS_MASK
) >> OCRDMA_CQE_STATUS_SHIFT
;
2357 if (status
== OCRDMA_CQE_SUCCESS
) {
2359 ocrdma_poll_success_rcqe(qp
, cqe
, ibwc
);
2361 expand
= ocrdma_poll_err_rcqe(qp
, cqe
, ibwc
, polled
, stop
,
2367 static void ocrdma_change_cq_phase(struct ocrdma_cq
*cq
, struct ocrdma_cqe
*cqe
,
2370 if (cq
->phase_change
) {
2372 cq
->phase
= (~cq
->phase
& OCRDMA_CQE_VALID
);
2374 /* clear valid bit */
2375 cqe
->flags_status_srcqpn
= 0;
2378 static int ocrdma_poll_hwcq(struct ocrdma_cq
*cq
, int num_entries
,
2383 bool expand
= false;
2384 int polled_hw_cqes
= 0;
2385 struct ocrdma_qp
*qp
= NULL
;
2386 struct ocrdma_dev
*dev
= cq
->dev
;
2387 struct ocrdma_cqe
*cqe
;
2388 u16 cur_getp
; bool polled
= false; bool stop
= false;
2390 cur_getp
= cq
->getp
;
2391 while (num_entries
) {
2392 cqe
= cq
->va
+ cur_getp
;
2393 /* check whether valid cqe or not */
2394 if (!is_cqe_valid(cq
, cqe
))
2396 qpn
= (le32_to_cpu(cqe
->cmn
.qpn
) & OCRDMA_CQE_QPN_MASK
);
2397 /* ignore discarded cqe */
2400 qp
= dev
->qp_tbl
[qpn
];
2403 if (is_cqe_for_sq(cqe
)) {
2404 expand
= ocrdma_poll_scqe(qp
, cqe
, ibwc
, &polled
,
2407 expand
= ocrdma_poll_rcqe(qp
, cqe
, ibwc
, &polled
,
2414 /* clear qpn to avoid duplicate processing by discard_cqe() */
2417 polled_hw_cqes
+= 1;
2418 cur_getp
= (cur_getp
+ 1) % cq
->max_hw_cqe
;
2419 ocrdma_change_cq_phase(cq
, cqe
, cur_getp
);
2429 cq
->getp
= cur_getp
;
2430 if (polled_hw_cqes
|| expand
|| stop
) {
2431 ocrdma_ring_cq_db(dev
, cq
->id
, cq
->armed
, cq
->solicited
,
2437 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2438 static int ocrdma_add_err_cqe(struct ocrdma_cq
*cq
, int num_entries
,
2439 struct ocrdma_qp
*qp
, struct ib_wc
*ibwc
)
2443 while (num_entries
) {
2444 if (is_hw_sq_empty(qp
) && is_hw_rq_empty(qp
))
2446 if (!is_hw_sq_empty(qp
) && qp
->sq_cq
== cq
) {
2447 ocrdma_update_wc(qp
, ibwc
, qp
->sq
.tail
);
2448 ocrdma_hwq_inc_tail(&qp
->sq
);
2449 } else if (!is_hw_rq_empty(qp
) && qp
->rq_cq
== cq
) {
2450 ibwc
->wr_id
= qp
->rqe_wr_id_tbl
[qp
->rq
.tail
];
2451 ocrdma_hwq_inc_tail(&qp
->rq
);
2455 ibwc
->status
= IB_WC_WR_FLUSH_ERR
;
2463 int ocrdma_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
2465 int cqes_to_poll
= num_entries
;
2466 struct ocrdma_cq
*cq
= NULL
;
2467 unsigned long flags
;
2468 struct ocrdma_dev
*dev
;
2469 int num_os_cqe
= 0, err_cqes
= 0;
2470 struct ocrdma_qp
*qp
;
2472 cq
= get_ocrdma_cq(ibcq
);
2475 /* poll cqes from adapter CQ */
2476 spin_lock_irqsave(&cq
->cq_lock
, flags
);
2477 num_os_cqe
= ocrdma_poll_hwcq(cq
, cqes_to_poll
, wc
);
2478 spin_unlock_irqrestore(&cq
->cq_lock
, flags
);
2479 cqes_to_poll
-= num_os_cqe
;
2482 wc
= wc
+ num_os_cqe
;
2483 /* adapter returns single error cqe when qp moves to
2484 * error state. So insert error cqes with wc_status as
2485 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2486 * respectively which uses this CQ.
2488 spin_lock_irqsave(&dev
->flush_q_lock
, flags
);
2489 list_for_each_entry(qp
, &cq
->sq_head
, sq_entry
) {
2490 if (cqes_to_poll
== 0)
2492 err_cqes
= ocrdma_add_err_cqe(cq
, cqes_to_poll
, qp
, wc
);
2493 cqes_to_poll
-= err_cqes
;
2494 num_os_cqe
+= err_cqes
;
2497 spin_unlock_irqrestore(&dev
->flush_q_lock
, flags
);
2502 int ocrdma_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags cq_flags
)
2504 struct ocrdma_cq
*cq
;
2505 unsigned long flags
;
2506 struct ocrdma_dev
*dev
;
2509 struct ocrdma_cqe
*cqe
;
2511 cq
= get_ocrdma_cq(ibcq
);
2515 spin_lock_irqsave(&cq
->cq_lock
, flags
);
2516 if (cq_flags
& IB_CQ_NEXT_COMP
|| cq_flags
& IB_CQ_SOLICITED
)
2518 if (cq_flags
& IB_CQ_SOLICITED
)
2519 cq
->solicited
= true;
2521 cur_getp
= cq
->getp
;
2522 cqe
= cq
->va
+ cur_getp
;
2524 /* check whether any valid cqe exist or not, if not then safe to
2525 * arm. If cqe is not yet consumed, then let it get consumed and then
2526 * we arm it to avoid false interrupts.
2528 if (!is_cqe_valid(cq
, cqe
) || cq
->arm_needed
) {
2529 cq
->arm_needed
= false;
2530 ocrdma_ring_cq_db(dev
, cq_id
, cq
->armed
, cq
->solicited
, 0);
2532 spin_unlock_irqrestore(&cq
->cq_lock
, flags
);