projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/ocrdma: Don't allow zero/invalid sgid usage
[deliverable/linux.git] / drivers / infiniband / hw / ocrdma / ocrdma_verbs.c
CommitLineData
fe2caefc
PP		 1/*******************************************************************
2 * This file is part of the Emulex RoCE Device Driver for *
3 * RoCE (RDMA over Converged Ethernet) adapters. *
4 * Copyright (C) 2008-2012 Emulex. All rights reserved. *
5 * EMULEX and SLI are trademarks of Emulex. *
6 * www.emulex.com *
7 * *
8 * This program is free software; you can redistribute it and/or *
9 * modify it under the terms of version 2 of the GNU General *
10 * Public License as published by the Free Software Foundation. *
11 * This program is distributed in the hope that it will be useful. *
12 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
13 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
14 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
15 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
16 * TO BE LEGALLY INVALID. See the GNU General Public License for *
17 * more details, a copy of which can be found in the file COPYING *
18 * included with this package. *
19 *
20 * Contact Information:
21 * linux-drivers@emulex.com
22 *
23 * Emulex
24 * 3333 Susan Street
25 * Costa Mesa, CA 92626
26 *******************************************************************/
27
28#include <linux/dma-mapping.h>
29#include <rdma/ib_verbs.h>
30#include <rdma/ib_user_verbs.h>
31#include <rdma/iw_cm.h>
32#include <rdma/ib_umem.h>
33#include <rdma/ib_addr.h>
34
35#include "ocrdma.h"
36#include "ocrdma_hw.h"
37#include "ocrdma_verbs.h"
38#include "ocrdma_abi.h"
39
40int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
41{
42 if (index > 1)
43 return -EINVAL;
44
45 *pkey = 0xffff;
46 return 0;
47}
48
49int ocrdma_query_gid(struct ib_device *ibdev, u8 port,
50 int index, union ib_gid *sgid)
51{
52 struct ocrdma_dev *dev;
53
54 dev = get_ocrdma_dev(ibdev);
55 memset(sgid, 0, sizeof(*sgid));
7b33dc2b 56 if (index >= OCRDMA_MAX_SGID)
fe2caefc
PP		 57 return -EINVAL;
58
59 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
60
61 return 0;
62}
63
64int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
65{
66 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
67
68 memset(attr, 0, sizeof *attr);
69 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
70 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
71 ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid);
72 attr->max_mr_size = ~0ull;
73 attr->page_size_cap = 0xffff000;
74 attr->vendor_id = dev->nic_info.pdev->vendor;
75 attr->vendor_part_id = dev->nic_info.pdev->device;
76 attr->hw_ver = 0;
77 attr->max_qp = dev->attr.max_qp;
78 attr->max_ah = dev->attr.max_qp;
79 attr->max_qp_wr = dev->attr.max_wqe;
80
81 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
82 IB_DEVICE_RC_RNR_NAK_GEN |
83 IB_DEVICE_SHUTDOWN_PORT |
84 IB_DEVICE_SYS_IMAGE_GUID |
85 IB_DEVICE_LOCAL_DMA_LKEY;
634c5796
MV		 86 attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_srq_sge);
87 attr->max_sge_rd = 0;
fe2caefc
PP		 88 attr->max_cq = dev->attr.max_cq;
89 attr->max_cqe = dev->attr.max_cqe;
90 attr->max_mr = dev->attr.max_mr;
91 attr->max_mw = 0;
92 attr->max_pd = dev->attr.max_pd;
93 attr->atomic_cap = 0;
94 attr->max_fmr = 0;
95 attr->max_map_per_fmr = 0;
96 attr->max_qp_rd_atom =
97 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
98 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
99 attr->max_srq = (dev->attr.max_qp - 1);
d1e09ebf 100 attr->max_srq_sge = dev->attr.max_srq_sge;
fe2caefc
PP		 101 attr->max_srq_wr = dev->attr.max_rqe;
102 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
103 attr->max_fast_reg_page_list_len = 0;
104 attr->max_pkeys = 1;
105 return 0;
106}
107
108int ocrdma_query_port(struct ib_device *ibdev,
109 u8 port, struct ib_port_attr *props)
110{
111 enum ib_port_state port_state;
112 struct ocrdma_dev *dev;
113 struct net_device *netdev;
114
115 dev = get_ocrdma_dev(ibdev);
116 if (port > 1) {
ef99c4c2
NG		 117 pr_err("%s(%d) invalid_port=0x%x\n", __func__,
118 dev->id, port);
fe2caefc
PP		 119 return -EINVAL;
120 }
121 netdev = dev->nic_info.netdev;
122 if (netif_running(netdev) && netif_oper_up(netdev)) {
123 port_state = IB_PORT_ACTIVE;
124 props->phys_state = 5;
125 } else {
126 port_state = IB_PORT_DOWN;
127 props->phys_state = 3;
128 }
129 props->max_mtu = IB_MTU_4096;
130 props->active_mtu = iboe_get_mtu(netdev->mtu);
131 props->lid = 0;
132 props->lmc = 0;
133 props->sm_lid = 0;
134 props->sm_sl = 0;
135 props->state = port_state;
136 props->port_cap_flags =
137 IB_PORT_CM_SUP |
138 IB_PORT_REINIT_SUP |
139 IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP;
140 props->gid_tbl_len = OCRDMA_MAX_SGID;
141 props->pkey_tbl_len = 1;
142 props->bad_pkey_cntr = 0;
143 props->qkey_viol_cntr = 0;
144 props->active_width = IB_WIDTH_1X;
145 props->active_speed = 4;
146 props->max_msg_sz = 0x80000000;
147 props->max_vl_num = 4;
148 return 0;
149}
150
151int ocrdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
152 struct ib_port_modify *props)
153{
154 struct ocrdma_dev *dev;
155
156 dev = get_ocrdma_dev(ibdev);
157 if (port > 1) {
ef99c4c2 158 pr_err("%s(%d) invalid_port=0x%x\n", __func__, dev->id, port);
fe2caefc
PP		 159 return -EINVAL;
160 }
161 return 0;
162}
163
164static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
165 unsigned long len)
166{
167 struct ocrdma_mm *mm;
168
169 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
170 if (mm == NULL)
171 return -ENOMEM;
172 mm->key.phy_addr = phy_addr;
173 mm->key.len = len;
174 INIT_LIST_HEAD(&mm->entry);
175
176 mutex_lock(&uctx->mm_list_lock);
177 list_add_tail(&mm->entry, &uctx->mm_head);
178 mutex_unlock(&uctx->mm_list_lock);
179 return 0;
180}
181
182static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
183 unsigned long len)
184{
185 struct ocrdma_mm *mm, *tmp;
186
187 mutex_lock(&uctx->mm_list_lock);
188 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
189 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
190 continue;
191
192 list_del(&mm->entry);
193 kfree(mm);
194 break;
195 }
196 mutex_unlock(&uctx->mm_list_lock);
197}
198
199static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
200 unsigned long len)
201{
202 bool found = false;
203 struct ocrdma_mm *mm;
204
205 mutex_lock(&uctx->mm_list_lock);
206 list_for_each_entry(mm, &uctx->mm_head, entry) {
207 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
208 continue;
209
210 found = true;
211 break;
212 }
213 mutex_unlock(&uctx->mm_list_lock);
214 return found;
215}
216
217struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
218 struct ib_udata *udata)
219{
220 int status;
221 struct ocrdma_ucontext *ctx;
222 struct ocrdma_alloc_ucontext_resp resp;
223 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
224 struct pci_dev *pdev = dev->nic_info.pdev;
225 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
226
227 if (!udata)
228 return ERR_PTR(-EFAULT);
229 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
230 if (!ctx)
231 return ERR_PTR(-ENOMEM);
fe2caefc
PP		 232 INIT_LIST_HEAD(&ctx->mm_head);
233 mutex_init(&ctx->mm_list_lock);
234
235 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
236 &ctx->ah_tbl.pa, GFP_KERNEL);
237 if (!ctx->ah_tbl.va) {
238 kfree(ctx);
239 return ERR_PTR(-ENOMEM);
240 }
241 memset(ctx->ah_tbl.va, 0, map_len);
242 ctx->ah_tbl.len = map_len;
243
63ea3749 244 memset(&resp, 0, sizeof(resp));
fe2caefc
PP		 245 resp.ah_tbl_len = ctx->ah_tbl.len;
246 resp.ah_tbl_page = ctx->ah_tbl.pa;
247
248 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
249 if (status)
250 goto map_err;
251 resp.dev_id = dev->id;
252 resp.max_inline_data = dev->attr.max_inline_data;
253 resp.wqe_size = dev->attr.wqe_size;
254 resp.rqe_size = dev->attr.rqe_size;
255 resp.dpp_wqe_size = dev->attr.wqe_size;
fe2caefc
PP		 256
257 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
258 status = ib_copy_to_udata(udata, &resp, sizeof(resp));
259 if (status)
260 goto cpy_err;
261 return &ctx->ibucontext;
262
263cpy_err:
264 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
265map_err:
266 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
267 ctx->ah_tbl.pa);
268 kfree(ctx);
269 return ERR_PTR(status);
270}
271
272int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
273{
274 struct ocrdma_mm *mm, *tmp;
275 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
1afc0454
NG		 276 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);
277 struct pci_dev *pdev = dev->nic_info.pdev;
fe2caefc
PP		 278
279 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
280 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
281 uctx->ah_tbl.pa);
282
283 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
284 list_del(&mm->entry);
285 kfree(mm);
286 }
287 kfree(uctx);
288 return 0;
289}
290
291int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
292{
293 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
1afc0454 294 struct ocrdma_dev *dev = get_ocrdma_dev(context->device);
fe2caefc
PP		 295 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
296 u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
297 unsigned long len = (vma->vm_end - vma->vm_start);
298 int status = 0;
299 bool found;
300
301 if (vma->vm_start & (PAGE_SIZE - 1))
302 return -EINVAL;
303 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);
304 if (!found)
305 return -EINVAL;
306
307 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
308 dev->nic_info.db_total_size)) &&
309 (len <= dev->nic_info.db_page_size)) {
310 /* doorbell mapping */
311 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
312 len, vma->vm_page_prot);
313 } else if (dev->nic_info.dpp_unmapped_len &&
314 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
315 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
316 dev->nic_info.dpp_unmapped_len)) &&
317 (len <= dev->nic_info.dpp_unmapped_len)) {
318 /* dpp area mapping */
319 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
320 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
321 len, vma->vm_page_prot);
322 } else {
323 /* queue memory mapping */
324 status = remap_pfn_range(vma, vma->vm_start,
325 vma->vm_pgoff, len, vma->vm_page_prot);
326 }
327 return status;
328}
329
330static int ocrdma_copy_pd_uresp(struct ocrdma_pd *pd,
331 struct ib_ucontext *ib_ctx,
332 struct ib_udata *udata)
333{
334 int status;
335 u64 db_page_addr;
da496438 336 u64 dpp_page_addr = 0;
fe2caefc
PP		 337 u32 db_page_size;
338 struct ocrdma_alloc_pd_uresp rsp;
339 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
f99b1649 340 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
fe2caefc 341
63ea3749 342 memset(&rsp, 0, sizeof(rsp));
fe2caefc
PP		 343 rsp.id = pd->id;
344 rsp.dpp_enabled = pd->dpp_enabled;
f99b1649
NG		 345 db_page_addr = dev->nic_info.unmapped_db +
346 (pd->id * dev->nic_info.db_page_size);
347 db_page_size = dev->nic_info.db_page_size;
fe2caefc
PP		 348
349 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
350 if (status)
351 return status;
352
353 if (pd->dpp_enabled) {
f99b1649 354 dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
fe2caefc
PP		 355 (pd->id * OCRDMA_DPP_PAGE_SIZE);
356 status = ocrdma_add_mmap(uctx, dpp_page_addr,
357 OCRDMA_DPP_PAGE_SIZE);
358 if (status)
359 goto dpp_map_err;
360 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
361 rsp.dpp_page_addr_lo = dpp_page_addr;
362 }
363
364 status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
365 if (status)
366 goto ucopy_err;
367
368 pd->uctx = uctx;
369 return 0;
370
371ucopy_err:
da496438
RD		 372 if (pd->dpp_enabled)
373 ocrdma_del_mmap(pd->uctx, dpp_page_addr, OCRDMA_DPP_PAGE_SIZE);
fe2caefc
PP		 374dpp_map_err:
375 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
376 return status;
377}
378
379struct ib_pd *ocrdma_alloc_pd(struct ib_device *ibdev,
380 struct ib_ucontext *context,
381 struct ib_udata *udata)
382{
383 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
384 struct ocrdma_pd *pd;
385 int status;
386
387 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
388 if (!pd)
389 return ERR_PTR(-ENOMEM);
fe2caefc 390 if (udata && context) {
f99b1649
NG		 391 pd->dpp_enabled =
392 (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY);
fe2caefc
PP		 393 pd->num_dpp_qp =
394 pd->dpp_enabled ? OCRDMA_PD_MAX_DPP_ENABLED_QP : 0;
395 }
396 status = ocrdma_mbx_alloc_pd(dev, pd);
397 if (status) {
398 kfree(pd);
399 return ERR_PTR(status);
400 }
fe2caefc
PP		 401
402 if (udata && context) {
403 status = ocrdma_copy_pd_uresp(pd, context, udata);
404 if (status)
405 goto err;
406 }
407 return &pd->ibpd;
408
409err:
410 ocrdma_dealloc_pd(&pd->ibpd);
411 return ERR_PTR(status);
412}
413
414int ocrdma_dealloc_pd(struct ib_pd *ibpd)
415{
416 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
f99b1649 417 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
fe2caefc
PP		 418 int status;
419 u64 usr_db;
420
fe2caefc
PP		 421 status = ocrdma_mbx_dealloc_pd(dev, pd);
422 if (pd->uctx) {
423 u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
424 (pd->id * OCRDMA_DPP_PAGE_SIZE);
425 if (pd->dpp_enabled)
426 ocrdma_del_mmap(pd->uctx, dpp_db, OCRDMA_DPP_PAGE_SIZE);
427 usr_db = dev->nic_info.unmapped_db +
428 (pd->id * dev->nic_info.db_page_size);
429 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
430 }
431 kfree(pd);
fe2caefc
PP		 432 return status;
433}
434
1afc0454
NG		 435static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
436 u32 pdid, int acc, u32 num_pbls, u32 addr_check)
fe2caefc
PP		 437{
438 int status;
fe2caefc 439
fe2caefc
PP		 440 mr->hwmr.fr_mr = 0;
441 mr->hwmr.local_rd = 1;
442 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
443 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
444 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
445 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
446 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
447 mr->hwmr.num_pbls = num_pbls;
448
f99b1649
NG		 449 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);
450 if (status)
451 return status;
452
fe2caefc
PP		 453 mr->ibmr.lkey = mr->hwmr.lkey;
454 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
455 mr->ibmr.rkey = mr->hwmr.lkey;
f99b1649 456 return 0;
fe2caefc
PP		 457}
458
459struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
460{
f99b1649 461 int status;
fe2caefc 462 struct ocrdma_mr *mr;
f99b1649
NG		 463 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
464 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
465
466 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
467 pr_err("%s err, invalid access rights\n", __func__);
468 return ERR_PTR(-EINVAL);
469 }
fe2caefc 470
f99b1649
NG		 471 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
472 if (!mr)
473 return ERR_PTR(-ENOMEM);
474
1afc0454 475 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,
f99b1649
NG		 476 OCRDMA_ADDR_CHECK_DISABLE);
477 if (status) {
478 kfree(mr);
479 return ERR_PTR(status);
480 }
fe2caefc
PP		 481
482 return &mr->ibmr;
483}
484
485static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
486 struct ocrdma_hw_mr *mr)
487{
488 struct pci_dev *pdev = dev->nic_info.pdev;
489 int i = 0;
490
491 if (mr->pbl_table) {
492 for (i = 0; i < mr->num_pbls; i++) {
493 if (!mr->pbl_table[i].va)
494 continue;
495 dma_free_coherent(&pdev->dev, mr->pbl_size,
496 mr->pbl_table[i].va,
497 mr->pbl_table[i].pa);
498 }
499 kfree(mr->pbl_table);
500 mr->pbl_table = NULL;
501 }
502}
503
1afc0454
NG		 504static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
505 u32 num_pbes)
fe2caefc
PP		 506{
507 u32 num_pbls = 0;
508 u32 idx = 0;
509 int status = 0;
510 u32 pbl_size;
511
512 do {
513 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
514 if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
515 status = -EFAULT;
516 break;
517 }
518 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
519 num_pbls = num_pbls / (pbl_size / sizeof(u64));
520 idx++;
1afc0454 521 } while (num_pbls >= dev->attr.max_num_mr_pbl);
fe2caefc
PP		 522
523 mr->hwmr.num_pbes = num_pbes;
524 mr->hwmr.num_pbls = num_pbls;
525 mr->hwmr.pbl_size = pbl_size;
526 return status;
527}
528
529static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
530{
531 int status = 0;
532 int i;
533 u32 dma_len = mr->pbl_size;
534 struct pci_dev *pdev = dev->nic_info.pdev;
535 void *va;
536 dma_addr_t pa;
537
538 mr->pbl_table = kzalloc(sizeof(struct ocrdma_pbl) *
539 mr->num_pbls, GFP_KERNEL);
540
541 if (!mr->pbl_table)
542 return -ENOMEM;
543
544 for (i = 0; i < mr->num_pbls; i++) {
545 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
546 if (!va) {
547 ocrdma_free_mr_pbl_tbl(dev, mr);
548 status = -ENOMEM;
549 break;
550 }
551 memset(va, 0, dma_len);
552 mr->pbl_table[i].va = va;
553 mr->pbl_table[i].pa = pa;
554 }
555 return status;
556}
557
558static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
559 u32 num_pbes)
560{
561 struct ocrdma_pbe *pbe;
562 struct ib_umem_chunk *chunk;
563 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
564 struct ib_umem *umem = mr->umem;
565 int i, shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
566
567 if (!mr->hwmr.num_pbes)
568 return;
569
570 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
571 pbe_cnt = 0;
572
573 shift = ilog2(umem->page_size);
574
575 list_for_each_entry(chunk, &umem->chunk_list, list) {
576 /* get all the dma regions from the chunk. */
577 for (i = 0; i < chunk->nmap; i++) {
578 pages = sg_dma_len(&chunk->page_list[i]) >> shift;
579 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
580 /* store the page address in pbe */
581 pbe->pa_lo =
582 cpu_to_le32(sg_dma_address
583 (&chunk->page_list[i]) +
584 (umem->page_size * pg_cnt));
585 pbe->pa_hi =
586 cpu_to_le32(upper_32_bits
587 ((sg_dma_address
588 (&chunk->page_list[i]) +
589 umem->page_size * pg_cnt)));
590 pbe_cnt += 1;
591 total_num_pbes += 1;
592 pbe++;
593
594 /* if done building pbes, issue the mbx cmd. */
595 if (total_num_pbes == num_pbes)
596 return;
597
598 /* if the given pbl is full storing the pbes,
599 * move to next pbl.
600 */
601 if (pbe_cnt ==
602 (mr->hwmr.pbl_size / sizeof(u64))) {
603 pbl_tbl++;
604 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
605 pbe_cnt = 0;
606 }
607 }
608 }
609 }
610}
611
612struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
613 u64 usr_addr, int acc, struct ib_udata *udata)
614{
615 int status = -ENOMEM;
f99b1649 616 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
fe2caefc
PP		 617 struct ocrdma_mr *mr;
618 struct ocrdma_pd *pd;
fe2caefc
PP		 619 u32 num_pbes;
620
621 pd = get_ocrdma_pd(ibpd);
fe2caefc
PP		 622
623 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
624 return ERR_PTR(-EINVAL);
625
626 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
627 if (!mr)
628 return ERR_PTR(status);
fe2caefc
PP		 629 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
630 if (IS_ERR(mr->umem)) {
631 status = -EFAULT;
632 goto umem_err;
633 }
634 num_pbes = ib_umem_page_count(mr->umem);
1afc0454 635 status = ocrdma_get_pbl_info(dev, mr, num_pbes);
fe2caefc
PP		 636 if (status)
637 goto umem_err;
638
639 mr->hwmr.pbe_size = mr->umem->page_size;
640 mr->hwmr.fbo = mr->umem->offset;
641 mr->hwmr.va = usr_addr;
642 mr->hwmr.len = len;
643 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
644 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
645 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
646 mr->hwmr.local_rd = 1;
647 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
648 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
649 if (status)
650 goto umem_err;
651 build_user_pbes(dev, mr, num_pbes);
652 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
653 if (status)
654 goto mbx_err;
fe2caefc
PP		 655 mr->ibmr.lkey = mr->hwmr.lkey;
656 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
657 mr->ibmr.rkey = mr->hwmr.lkey;
658
659 return &mr->ibmr;
660
661mbx_err:
662 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
663umem_err:
664 kfree(mr);
665 return ERR_PTR(status);
666}
667
668int ocrdma_dereg_mr(struct ib_mr *ib_mr)
669{
670 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
1afc0454 671 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
fe2caefc
PP		 672 int status;
673
674 status = ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
675
676 if (mr->hwmr.fr_mr == 0)
677 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
678
fe2caefc
PP		 679 /* it could be user registered memory. */
680 if (mr->umem)
681 ib_umem_release(mr->umem);
682 kfree(mr);
683 return status;
684}
685
1afc0454
NG		 686static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
687 struct ib_udata *udata,
fe2caefc
PP		 688 struct ib_ucontext *ib_ctx)
689{
690 int status;
691 struct ocrdma_ucontext *uctx;
692 struct ocrdma_create_cq_uresp uresp;
693
63ea3749 694 memset(&uresp, 0, sizeof(uresp));
fe2caefc
PP		 695 uresp.cq_id = cq->id;
696 uresp.page_size = cq->len;
697 uresp.num_pages = 1;
698 uresp.max_hw_cqe = cq->max_hw_cqe;
699 uresp.page_addr[0] = cq->pa;
1afc0454
NG		 700 uresp.db_page_addr = dev->nic_info.unmapped_db;
701 uresp.db_page_size = dev->nic_info.db_page_size;
fe2caefc
PP		 702 uresp.phase_change = cq->phase_change ? 1 : 0;
703 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
704 if (status) {
ef99c4c2 705 pr_err("%s(%d) copy error cqid=0x%x.\n",
1afc0454 706 __func__, dev->id, cq->id);
fe2caefc
PP		 707 goto err;
708 }
709 uctx = get_ocrdma_ucontext(ib_ctx);
710 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
711 if (status)
712 goto err;
713 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);
714 if (status) {
715 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
716 goto err;
717 }
718 cq->ucontext = uctx;
719err:
720 return status;
721}
722
723struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, int entries, int vector,
724 struct ib_ucontext *ib_ctx,
725 struct ib_udata *udata)
726{
727 struct ocrdma_cq *cq;
728 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
729 int status;
730 struct ocrdma_create_cq_ureq ureq;
731
732 if (udata) {
733 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
734 return ERR_PTR(-EFAULT);
735 } else
736 ureq.dpp_cq = 0;
737 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
738 if (!cq)
739 return ERR_PTR(-ENOMEM);
740
741 spin_lock_init(&cq->cq_lock);
742 spin_lock_init(&cq->comp_handler_lock);
fe2caefc
PP		 743 INIT_LIST_HEAD(&cq->sq_head);
744 INIT_LIST_HEAD(&cq->rq_head);
fe2caefc
PP		 745
746 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq);
747 if (status) {
748 kfree(cq);
749 return ERR_PTR(status);
750 }
751 if (ib_ctx) {
1afc0454 752 status = ocrdma_copy_cq_uresp(dev, cq, udata, ib_ctx);
fe2caefc
PP		 753 if (status)
754 goto ctx_err;
755 }
756 cq->phase = OCRDMA_CQE_VALID;
757 cq->arm_needed = true;
758 dev->cq_tbl[cq->id] = cq;
759
760 return &cq->ibcq;
761
762ctx_err:
763 ocrdma_mbx_destroy_cq(dev, cq);
764 kfree(cq);
765 return ERR_PTR(status);
766}
767
768int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
769 struct ib_udata *udata)
770{
771 int status = 0;
772 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
773
774 if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
775 status = -EINVAL;
776 return status;
777 }
778 ibcq->cqe = new_cnt;
779 return status;
780}
781
782int ocrdma_destroy_cq(struct ib_cq *ibcq)
783{
784 int status;
785 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1afc0454 786 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
fe2caefc 787
fe2caefc
PP		 788 status = ocrdma_mbx_destroy_cq(dev, cq);
789
790 if (cq->ucontext) {
791 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, cq->len);
792 ocrdma_del_mmap(cq->ucontext, dev->nic_info.unmapped_db,
793 dev->nic_info.db_page_size);
794 }
795 dev->cq_tbl[cq->id] = NULL;
796
797 kfree(cq);
798 return status;
799}
800
801static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
802{
803 int status = -EINVAL;
804
805 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
806 dev->qp_tbl[qp->id] = qp;
807 status = 0;
808 }
809 return status;
810}
811
812static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
813{
814 dev->qp_tbl[qp->id] = NULL;
815}
816
817static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
818 struct ib_qp_init_attr *attrs)
819{
820 if (attrs->qp_type != IB_QPT_GSI &&
821 attrs->qp_type != IB_QPT_RC &&
822 attrs->qp_type != IB_QPT_UD) {
ef99c4c2
NG		 823 pr_err("%s(%d) unsupported qp type=0x%x requested\n",
824 __func__, dev->id, attrs->qp_type);
fe2caefc
PP		 825 return -EINVAL;
826 }
827 if (attrs->cap.max_send_wr > dev->attr.max_wqe) {
ef99c4c2
NG		 828 pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
829 __func__, dev->id, attrs->cap.max_send_wr);
830 pr_err("%s(%d) supported send_wr=0x%x\n",
831 __func__, dev->id, dev->attr.max_wqe);
fe2caefc
PP		 832 return -EINVAL;
833 }
834 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
ef99c4c2
NG		 835 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
836 __func__, dev->id, attrs->cap.max_recv_wr);
837 pr_err("%s(%d) supported recv_wr=0x%x\n",
838 __func__, dev->id, dev->attr.max_rqe);
fe2caefc
PP		 839 return -EINVAL;
840 }
841 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
ef99c4c2
NG		 842 pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
843 __func__, dev->id, attrs->cap.max_inline_data);
844 pr_err("%s(%d) supported inline data size=0x%x\n",
845 __func__, dev->id, dev->attr.max_inline_data);
fe2caefc
PP		 846 return -EINVAL;
847 }
848 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
ef99c4c2
NG		 849 pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
850 __func__, dev->id, attrs->cap.max_send_sge);
851 pr_err("%s(%d) supported send_sge=0x%x\n",
852 __func__, dev->id, dev->attr.max_send_sge);
fe2caefc
PP		 853 return -EINVAL;
854 }
855 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
ef99c4c2
NG		 856 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
857 __func__, dev->id, attrs->cap.max_recv_sge);
858 pr_err("%s(%d) supported recv_sge=0x%x\n",
859 __func__, dev->id, dev->attr.max_recv_sge);
fe2caefc
PP		 860 return -EINVAL;
861 }
862 /* unprivileged user space cannot create special QP */
863 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
ef99c4c2 864 pr_err
fe2caefc
PP		 865 ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
866 __func__, dev->id, attrs->qp_type);
867 return -EINVAL;
868 }
869 /* allow creating only one GSI type of QP */
870 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
ef99c4c2
NG		 871 pr_err("%s(%d) GSI special QPs already created.\n",
872 __func__, dev->id);
fe2caefc
PP		 873 return -EINVAL;
874 }
875 /* verify consumer QPs are not trying to use GSI QP's CQ */
876 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
877 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
9e8fa040
RD		 878 (dev->gsi_sqcq == get_ocrdma_cq(attrs->recv_cq)) ||
879 (dev->gsi_rqcq == get_ocrdma_cq(attrs->send_cq)) ||
880 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
ef99c4c2
NG		 881 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
882 __func__, dev->id);
fe2caefc
PP		 883 return -EINVAL;
884 }
885 }
886 return 0;
887}
888
889static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
890 struct ib_udata *udata, int dpp_offset,
891 int dpp_credit_lmt, int srq)
892{
893 int status = 0;
894 u64 usr_db;
895 struct ocrdma_create_qp_uresp uresp;
896 struct ocrdma_dev *dev = qp->dev;
897 struct ocrdma_pd *pd = qp->pd;
898
899 memset(&uresp, 0, sizeof(uresp));
900 usr_db = dev->nic_info.unmapped_db +
901 (pd->id * dev->nic_info.db_page_size);
902 uresp.qp_id = qp->id;
903 uresp.sq_dbid = qp->sq.dbid;
904 uresp.num_sq_pages = 1;
905 uresp.sq_page_size = qp->sq.len;
906 uresp.sq_page_addr[0] = qp->sq.pa;
907 uresp.num_wqe_allocated = qp->sq.max_cnt;
908 if (!srq) {
909 uresp.rq_dbid = qp->rq.dbid;
910 uresp.num_rq_pages = 1;
911 uresp.rq_page_size = qp->rq.len;
912 uresp.rq_page_addr[0] = qp->rq.pa;
913 uresp.num_rqe_allocated = qp->rq.max_cnt;
914 }
915 uresp.db_page_addr = usr_db;
916 uresp.db_page_size = dev->nic_info.db_page_size;
917 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
918 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
919 uresp.db_rq_offset = ((qp->id & 0xFFFF) < 128) ?
920 OCRDMA_DB_GEN2_RQ1_OFFSET : OCRDMA_DB_GEN2_RQ2_OFFSET;
921 uresp.db_shift = (qp->id < 128) ? 24 : 16;
922 } else {
923 uresp.db_sq_offset = OCRDMA_DB_SQ_OFFSET;
924 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
925 uresp.db_shift = 16;
926 }
fe2caefc
PP		 927
928 if (qp->dpp_enabled) {
929 uresp.dpp_credit = dpp_credit_lmt;
930 uresp.dpp_offset = dpp_offset;
931 }
932 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
933 if (status) {
ef99c4c2 934 pr_err("%s(%d) user copy error.\n", __func__, dev->id);
fe2caefc
PP		 935 goto err;
936 }
937 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
938 uresp.sq_page_size);
939 if (status)
940 goto err;
941
942 if (!srq) {
943 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],
944 uresp.rq_page_size);
945 if (status)
946 goto rq_map_err;
947 }
948 return status;
949rq_map_err:
950 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);
951err:
952 return status;
953}
954
955static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
956 struct ocrdma_pd *pd)
957{
958 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
959 qp->sq_db = dev->nic_info.db +
960 (pd->id * dev->nic_info.db_page_size) +
961 OCRDMA_DB_GEN2_SQ_OFFSET;
962 qp->rq_db = dev->nic_info.db +
963 (pd->id * dev->nic_info.db_page_size) +
964 ((qp->id < 128) ?
965 OCRDMA_DB_GEN2_RQ1_OFFSET : OCRDMA_DB_GEN2_RQ2_OFFSET);
966 } else {
967 qp->sq_db = dev->nic_info.db +
968 (pd->id * dev->nic_info.db_page_size) +
969 OCRDMA_DB_SQ_OFFSET;
970 qp->rq_db = dev->nic_info.db +
971 (pd->id * dev->nic_info.db_page_size) +
972 OCRDMA_DB_RQ_OFFSET;
973 }
974}
975
976static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
977{
978 qp->wqe_wr_id_tbl =
979 kzalloc(sizeof(*(qp->wqe_wr_id_tbl)) * qp->sq.max_cnt,
980 GFP_KERNEL);
981 if (qp->wqe_wr_id_tbl == NULL)
982 return -ENOMEM;
983 qp->rqe_wr_id_tbl =
984 kzalloc(sizeof(u64) * qp->rq.max_cnt, GFP_KERNEL);
985 if (qp->rqe_wr_id_tbl == NULL)
986 return -ENOMEM;
987
988 return 0;
989}
990
991static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
992 struct ocrdma_pd *pd,
993 struct ib_qp_init_attr *attrs)
994{
995 qp->pd = pd;
996 spin_lock_init(&qp->q_lock);
997 INIT_LIST_HEAD(&qp->sq_entry);
998 INIT_LIST_HEAD(&qp->rq_entry);
999
1000 qp->qp_type = attrs->qp_type;
1001 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1002 qp->max_inline_data = attrs->cap.max_inline_data;
1003 qp->sq.max_sges = attrs->cap.max_send_sge;
1004 qp->rq.max_sges = attrs->cap.max_recv_sge;
1005 qp->state = OCRDMA_QPS_RST;
1006}
1007
fe2caefc
PP		 1008
1009static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1010 struct ib_qp_init_attr *attrs)
1011{
1012 if (attrs->qp_type == IB_QPT_GSI) {
1013 dev->gsi_qp_created = 1;
1014 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);
1015 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);
1016 }
1017}
1018
1019struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1020 struct ib_qp_init_attr *attrs,
1021 struct ib_udata *udata)
1022{
1023 int status;
1024 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1025 struct ocrdma_qp *qp;
f99b1649 1026 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
fe2caefc
PP		 1027 struct ocrdma_create_qp_ureq ureq;
1028 u16 dpp_credit_lmt, dpp_offset;
1029
1030 status = ocrdma_check_qp_params(ibpd, dev, attrs);
1031 if (status)
1032 goto gen_err;
1033
1034 memset(&ureq, 0, sizeof(ureq));
1035 if (udata) {
1036 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1037 return ERR_PTR(-EFAULT);
1038 }
1039 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1040 if (!qp) {
1041 status = -ENOMEM;
1042 goto gen_err;
1043 }
1044 qp->dev = dev;
1045 ocrdma_set_qp_init_params(qp, pd, attrs);
1046
1047 mutex_lock(&dev->dev_lock);
1048 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1049 ureq.dpp_cq_id,
1050 &dpp_offset, &dpp_credit_lmt);
1051 if (status)
1052 goto mbx_err;
1053
1054 /* user space QP's wr_id table are managed in library */
1055 if (udata == NULL) {
1056 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1057 OCRDMA_QP_FAST_REG);
1058 status = ocrdma_alloc_wr_id_tbl(qp);
1059 if (status)
1060 goto map_err;
1061 }
1062
1063 status = ocrdma_add_qpn_map(dev, qp);
1064 if (status)
1065 goto map_err;
1066 ocrdma_set_qp_db(dev, qp, pd);
1067 if (udata) {
1068 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1069 dpp_credit_lmt,
1070 (attrs->srq != NULL));
1071 if (status)
1072 goto cpy_err;
1073 }
1074 ocrdma_store_gsi_qp_cq(dev, attrs);
27159f50 1075 qp->ibqp.qp_num = qp->id;
fe2caefc
PP		 1076 mutex_unlock(&dev->dev_lock);
1077 return &qp->ibqp;
1078
1079cpy_err:
1080 ocrdma_del_qpn_map(dev, qp);
1081map_err:
1082 ocrdma_mbx_destroy_qp(dev, qp);
1083mbx_err:
1084 mutex_unlock(&dev->dev_lock);
1085 kfree(qp->wqe_wr_id_tbl);
1086 kfree(qp->rqe_wr_id_tbl);
1087 kfree(qp);
ef99c4c2 1088 pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
fe2caefc
PP		 1089gen_err:
1090 return ERR_PTR(status);
1091}
1092
1093int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1094 int attr_mask)
1095{
1096 int status = 0;
1097 struct ocrdma_qp *qp;
1098 struct ocrdma_dev *dev;
1099 enum ib_qp_state old_qps;
1100
1101 qp = get_ocrdma_qp(ibqp);
1102 dev = qp->dev;
1103 if (attr_mask & IB_QP_STATE)
1104 status = ocrdma_qp_state_machine(qp, attr->qp_state, &old_qps);
1105 /* if new and previous states are same hw doesn't need to
1106 * know about it.
1107 */
1108 if (status < 0)
1109 return status;
1110 status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask, old_qps);
1111 return status;
1112}
1113
1114int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1115 int attr_mask, struct ib_udata *udata)
1116{
1117 unsigned long flags;
1118 int status = -EINVAL;
1119 struct ocrdma_qp *qp;
1120 struct ocrdma_dev *dev;
1121 enum ib_qp_state old_qps, new_qps;
1122
1123 qp = get_ocrdma_qp(ibqp);
1124 dev = qp->dev;
1125
1126 /* syncronize with multiple context trying to change, retrive qps */
1127 mutex_lock(&dev->dev_lock);
1128 /* syncronize with wqe, rqe posting and cqe processing contexts */
1129 spin_lock_irqsave(&qp->q_lock, flags);
1130 old_qps = get_ibqp_state(qp->state);
1131 if (attr_mask & IB_QP_STATE)
1132 new_qps = attr->qp_state;
1133 else
1134 new_qps = old_qps;
1135 spin_unlock_irqrestore(&qp->q_lock, flags);
1136
1137 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) {
ef99c4c2
NG		 1138 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1139 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1140 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1141 old_qps, new_qps);
fe2caefc
PP		 1142 goto param_err;
1143 }
1144
1145 status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1146 if (status > 0)
1147 status = 0;
1148param_err:
1149 mutex_unlock(&dev->dev_lock);
1150 return status;
1151}
1152
1153static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1154{
1155 switch (mtu) {
1156 case 256:
1157 return IB_MTU_256;
1158 case 512:
1159 return IB_MTU_512;
1160 case 1024:
1161 return IB_MTU_1024;
1162 case 2048:
1163 return IB_MTU_2048;
1164 case 4096:
1165 return IB_MTU_4096;
1166 default:
1167 return IB_MTU_1024;
1168 }
1169}
1170
1171static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1172{
1173 int ib_qp_acc_flags = 0;
1174
1175 if (qp_cap_flags & OCRDMA_QP_INB_WR)
1176 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1177 if (qp_cap_flags & OCRDMA_QP_INB_RD)
1178 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1179 return ib_qp_acc_flags;
1180}
1181
1182int ocrdma_query_qp(struct ib_qp *ibqp,
1183 struct ib_qp_attr *qp_attr,
1184 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1185{
1186 int status;
1187 u32 qp_state;
1188 struct ocrdma_qp_params params;
1189 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1190 struct ocrdma_dev *dev = qp->dev;
1191
1192 memset(&params, 0, sizeof(params));
1193 mutex_lock(&dev->dev_lock);
1194 status = ocrdma_mbx_query_qp(dev, qp, &params);
1195 mutex_unlock(&dev->dev_lock);
1196 if (status)
1197 goto mbx_err;
1198 qp_attr->qp_state = get_ibqp_state(IB_QPS_INIT);
1199 qp_attr->cur_qp_state = get_ibqp_state(IB_QPS_INIT);
1200 qp_attr->path_mtu =
1201 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &
1202 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1203 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1204 qp_attr->path_mig_state = IB_MIG_MIGRATED;
1205 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1206 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1207 qp_attr->dest_qp_num =
1208 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1209
1210 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1211 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1212 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1213 qp_attr->cap.max_send_sge = qp->sq.max_sges;
1214 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1215 qp_attr->cap.max_inline_data = dev->attr.max_inline_data;
1216 qp_init_attr->cap = qp_attr->cap;
1217 memcpy(&qp_attr->ah_attr.grh.dgid, &params.dgid[0],
1218 sizeof(params.dgid));
1219 qp_attr->ah_attr.grh.flow_label = params.rnt_rc_sl_fl &
1220 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK;
1221 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
1222 qp_attr->ah_attr.grh.hop_limit = (params.hop_lmt_rq_psn &
1223 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1224 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT;
1225 qp_attr->ah_attr.grh.traffic_class = (params.tclass_sq_psn &
1226 OCRDMA_QP_PARAMS_SQ_PSN_MASK) >>
1227 OCRDMA_QP_PARAMS_TCLASS_SHIFT;
1228
1229 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
1230 qp_attr->ah_attr.port_num = 1;
1231 qp_attr->ah_attr.sl = (params.rnt_rc_sl_fl &
1232 OCRDMA_QP_PARAMS_SL_MASK) >>
1233 OCRDMA_QP_PARAMS_SL_SHIFT;
1234 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1235 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1236 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1237 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1238 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1239 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1240 qp_attr->retry_cnt =
1241 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1242 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1243 qp_attr->min_rnr_timer = 0;
1244 qp_attr->pkey_index = 0;
1245 qp_attr->port_num = 1;
1246 qp_attr->ah_attr.src_path_bits = 0;
1247 qp_attr->ah_attr.static_rate = 0;
1248 qp_attr->alt_pkey_index = 0;
1249 qp_attr->alt_port_num = 0;
1250 qp_attr->alt_timeout = 0;
1251 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1252 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1253 OCRDMA_QP_PARAMS_STATE_SHIFT;
1254 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1255 qp_attr->max_dest_rd_atomic =
1256 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1257 qp_attr->max_rd_atomic =
1258 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1259 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1260 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1261mbx_err:
1262 return status;
1263}
1264
1265static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, int idx)
1266{
1267 int i = idx / 32;
1268 unsigned int mask = (1 << (idx % 32));
1269
1270 if (srq->idx_bit_fields[i] & mask)
1271 srq->idx_bit_fields[i] &= ~mask;
1272 else
1273 srq->idx_bit_fields[i] |= mask;
1274}
1275
1276static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1277{
1278 int free_cnt;
1279 if (q->head >= q->tail)
1280 free_cnt = (q->max_cnt - q->head) + q->tail;
1281 else
1282 free_cnt = q->tail - q->head;
fe2caefc
PP		 1283 return free_cnt;
1284}
1285
1286static int is_hw_sq_empty(struct ocrdma_qp *qp)
1287{
1288 return (qp->sq.tail == qp->sq.head &&
1289 ocrdma_hwq_free_cnt(&qp->sq) ? 1 : 0);
1290}
1291
1292static int is_hw_rq_empty(struct ocrdma_qp *qp)
1293{
1294 return (qp->rq.tail == qp->rq.head) ? 1 : 0;
1295}
1296
1297static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1298{
1299 return q->va + (q->head * q->entry_size);
1300}
1301
1302static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1303 u32 idx)
1304{
1305 return q->va + (idx * q->entry_size);
1306}
1307
1308static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1309{
1310 q->head = (q->head + 1) & q->max_wqe_idx;
1311}
1312
1313static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1314{
1315 q->tail = (q->tail + 1) & q->max_wqe_idx;
1316}
1317
1318/* discard the cqe for a given QP */
1319static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1320{
1321 unsigned long cq_flags;
1322 unsigned long flags;
1323 int discard_cnt = 0;
1324 u32 cur_getp, stop_getp;
1325 struct ocrdma_cqe *cqe;
1326 u32 qpn = 0;
1327
1328 spin_lock_irqsave(&cq->cq_lock, cq_flags);
1329
1330 /* traverse through the CQEs in the hw CQ,
1331 * find the matching CQE for a given qp,
1332 * mark the matching one discarded by clearing qpn.
1333 * ring the doorbell in the poll_cq() as
1334 * we don't complete out of order cqe.
1335 */
1336
1337 cur_getp = cq->getp;
1338 /* find upto when do we reap the cq. */
1339 stop_getp = cur_getp;
1340 do {
1341 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1342 break;
1343
1344 cqe = cq->va + cur_getp;
1345 /* if (a) done reaping whole hw cq, or
1346 * (b) qp_xq becomes empty.
1347 * then exit
1348 */
1349 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1350 /* if previously discarded cqe found, skip that too. */
1351 /* check for matching qp */
1352 if (qpn == 0 || qpn != qp->id)
1353 goto skip_cqe;
1354
1355 /* mark cqe discarded so that it is not picked up later
1356 * in the poll_cq().
1357 */
1358 discard_cnt += 1;
1359 cqe->cmn.qpn = 0;
f99b1649 1360 if (is_cqe_for_sq(cqe)) {
fe2caefc 1361 ocrdma_hwq_inc_tail(&qp->sq);
f99b1649 1362 } else {
fe2caefc
PP		 1363 if (qp->srq) {
1364 spin_lock_irqsave(&qp->srq->q_lock, flags);
1365 ocrdma_hwq_inc_tail(&qp->srq->rq);
1366 ocrdma_srq_toggle_bit(qp->srq, cur_getp);
1367 spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1368
f99b1649 1369 } else {
fe2caefc 1370 ocrdma_hwq_inc_tail(&qp->rq);
f99b1649 1371 }
fe2caefc
PP		 1372 }
1373skip_cqe:
1374 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1375 } while (cur_getp != stop_getp);
1376 spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1377}
1378
1379static void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1380{
1381 int found = false;
1382 unsigned long flags;
1383 struct ocrdma_dev *dev = qp->dev;
1384 /* sync with any active CQ poll */
1385
1386 spin_lock_irqsave(&dev->flush_q_lock, flags);
1387 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1388 if (found)
1389 list_del(&qp->sq_entry);
1390 if (!qp->srq) {
1391 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1392 if (found)
1393 list_del(&qp->rq_entry);
1394 }
1395 spin_unlock_irqrestore(&dev->flush_q_lock, flags);
1396}
1397
1398int ocrdma_destroy_qp(struct ib_qp *ibqp)
1399{
1400 int status;
1401 struct ocrdma_pd *pd;
1402 struct ocrdma_qp *qp;
1403 struct ocrdma_dev *dev;
1404 struct ib_qp_attr attrs;
1405 int attr_mask = IB_QP_STATE;
d19081e0 1406 unsigned long flags;
fe2caefc
PP		 1407
1408 qp = get_ocrdma_qp(ibqp);
1409 dev = qp->dev;
1410
1411 attrs.qp_state = IB_QPS_ERR;
1412 pd = qp->pd;
1413
1414 /* change the QP state to ERROR */
1415 _ocrdma_modify_qp(ibqp, &attrs, attr_mask);
1416
1417 /* ensure that CQEs for newly created QP (whose id may be same with
1418 * one which just getting destroyed are same), dont get
1419 * discarded until the old CQEs are discarded.
1420 */
1421 mutex_lock(&dev->dev_lock);
1422 status = ocrdma_mbx_destroy_qp(dev, qp);
1423
1424 /*
1425 * acquire CQ lock while destroy is in progress, in order to
1426 * protect against proessing in-flight CQEs for this QP.
1427 */
d19081e0 1428 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
fe2caefc 1429 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
d19081e0 1430 spin_lock(&qp->rq_cq->cq_lock);
fe2caefc
PP		 1431
1432 ocrdma_del_qpn_map(dev, qp);
1433
1434 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
d19081e0
DC		 1435 spin_unlock(&qp->rq_cq->cq_lock);
1436 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
fe2caefc
PP		 1437
1438 if (!pd->uctx) {
1439 ocrdma_discard_cqes(qp, qp->sq_cq);
1440 ocrdma_discard_cqes(qp, qp->rq_cq);
1441 }
1442 mutex_unlock(&dev->dev_lock);
1443
1444 if (pd->uctx) {
1445 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, qp->sq.len);
1446 if (!qp->srq)
1447 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, qp->rq.len);
1448 }
1449
1450 ocrdma_del_flush_qp(qp);
1451
fe2caefc
PP		 1452 kfree(qp->wqe_wr_id_tbl);
1453 kfree(qp->rqe_wr_id_tbl);
1454 kfree(qp);
1455 return status;
1456}
1457
1afc0454
NG		 1458static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1459 struct ib_udata *udata)
fe2caefc
PP		 1460{
1461 int status;
1462 struct ocrdma_create_srq_uresp uresp;
1463
63ea3749 1464 memset(&uresp, 0, sizeof(uresp));
fe2caefc
PP		 1465 uresp.rq_dbid = srq->rq.dbid;
1466 uresp.num_rq_pages = 1;
1467 uresp.rq_page_addr[0] = srq->rq.pa;
1468 uresp.rq_page_size = srq->rq.len;
1afc0454
NG		 1469 uresp.db_page_addr = dev->nic_info.unmapped_db +
1470 (srq->pd->id * dev->nic_info.db_page_size);
1471 uresp.db_page_size = dev->nic_info.db_page_size;
fe2caefc 1472 uresp.num_rqe_allocated = srq->rq.max_cnt;
1afc0454 1473 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
fe2caefc
PP		 1474 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ1_OFFSET;
1475 uresp.db_shift = 24;
1476 } else {
1477 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1478 uresp.db_shift = 16;
1479 }
1480
1481 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1482 if (status)
1483 return status;
1484 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
1485 uresp.rq_page_size);
1486 if (status)
1487 return status;
1488 return status;
1489}
1490
1491struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
1492 struct ib_srq_init_attr *init_attr,
1493 struct ib_udata *udata)
1494{
1495 int status = -ENOMEM;
1496 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
f99b1649 1497 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
fe2caefc
PP		 1498 struct ocrdma_srq *srq;
1499
1500 if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1501 return ERR_PTR(-EINVAL);
1502 if (init_attr->attr.max_wr > dev->attr.max_rqe)
1503 return ERR_PTR(-EINVAL);
1504
1505 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1506 if (!srq)
1507 return ERR_PTR(status);
1508
1509 spin_lock_init(&srq->q_lock);
fe2caefc
PP		 1510 srq->pd = pd;
1511 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1afc0454 1512 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
fe2caefc
PP		 1513 if (status)
1514 goto err;
1515
1516 if (udata == NULL) {
1517 srq->rqe_wr_id_tbl = kzalloc(sizeof(u64) * srq->rq.max_cnt,
1518 GFP_KERNEL);
1519 if (srq->rqe_wr_id_tbl == NULL)
1520 goto arm_err;
1521
1522 srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1523 (srq->rq.max_cnt % 32 ? 1 : 0);
1524 srq->idx_bit_fields =
1525 kmalloc(srq->bit_fields_len * sizeof(u32), GFP_KERNEL);
1526 if (srq->idx_bit_fields == NULL)
1527 goto arm_err;
1528 memset(srq->idx_bit_fields, 0xff,
1529 srq->bit_fields_len * sizeof(u32));
1530 }
1531
1532 if (init_attr->attr.srq_limit) {
1533 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1534 if (status)
1535 goto arm_err;
1536 }
1537
fe2caefc 1538 if (udata) {
1afc0454 1539 status = ocrdma_copy_srq_uresp(dev, srq, udata);
fe2caefc
PP		 1540 if (status)
1541 goto arm_err;
1542 }
1543
fe2caefc
PP		 1544 return &srq->ibsrq;
1545
1546arm_err:
1547 ocrdma_mbx_destroy_srq(dev, srq);
1548err:
1549 kfree(srq->rqe_wr_id_tbl);
1550 kfree(srq->idx_bit_fields);
1551 kfree(srq);
1552 return ERR_PTR(status);
1553}
1554
1555int ocrdma_modify_srq(struct ib_srq *ibsrq,
1556 struct ib_srq_attr *srq_attr,
1557 enum ib_srq_attr_mask srq_attr_mask,
1558 struct ib_udata *udata)
1559{
1560 int status = 0;
1561 struct ocrdma_srq *srq;
fe2caefc
PP		 1562
1563 srq = get_ocrdma_srq(ibsrq);
fe2caefc
PP		 1564 if (srq_attr_mask & IB_SRQ_MAX_WR)
1565 status = -EINVAL;
1566 else
1567 status = ocrdma_mbx_modify_srq(srq, srq_attr);
1568 return status;
1569}
1570
1571int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1572{
1573 int status;
1574 struct ocrdma_srq *srq;
fe2caefc
PP		 1575
1576 srq = get_ocrdma_srq(ibsrq);
fe2caefc
PP		 1577 status = ocrdma_mbx_query_srq(srq, srq_attr);
1578 return status;
1579}
1580
1581int ocrdma_destroy_srq(struct ib_srq *ibsrq)
1582{
1583 int status;
1584 struct ocrdma_srq *srq;
1afc0454 1585 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
fe2caefc
PP		 1586
1587 srq = get_ocrdma_srq(ibsrq);
fe2caefc
PP		 1588
1589 status = ocrdma_mbx_destroy_srq(dev, srq);
1590
1591 if (srq->pd->uctx)
1592 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, srq->rq.len);
1593
fe2caefc
PP		 1594 kfree(srq->idx_bit_fields);
1595 kfree(srq->rqe_wr_id_tbl);
1596 kfree(srq);
1597 return status;
1598}
1599
1600/* unprivileged verbs and their support functions. */
1601static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1602 struct ocrdma_hdr_wqe *hdr,
1603 struct ib_send_wr *wr)
1604{
1605 struct ocrdma_ewqe_ud_hdr *ud_hdr =
1606 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1607 struct ocrdma_ah *ah = get_ocrdma_ah(wr->wr.ud.ah);
1608
1609 ud_hdr->rsvd_dest_qpn = wr->wr.ud.remote_qpn;
1610 if (qp->qp_type == IB_QPT_GSI)
1611 ud_hdr->qkey = qp->qkey;
1612 else
1613 ud_hdr->qkey = wr->wr.ud.remote_qkey;
1614 ud_hdr->rsvd_ahid = ah->id;
1615}
1616
1617static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1618 struct ocrdma_sge *sge, int num_sge,
1619 struct ib_sge *sg_list)
1620{
1621 int i;
1622
1623 for (i = 0; i < num_sge; i++) {
1624 sge[i].lrkey = sg_list[i].lkey;
1625 sge[i].addr_lo = sg_list[i].addr;
1626 sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1627 sge[i].len = sg_list[i].length;
1628 hdr->total_len += sg_list[i].length;
1629 }
1630 if (num_sge == 0)
1631 memset(sge, 0, sizeof(*sge));
1632}
1633
1634static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1635 struct ocrdma_hdr_wqe *hdr,
1636 struct ocrdma_sge *sge,
1637 struct ib_send_wr *wr, u32 wqe_size)
1638{
1639 if (wr->send_flags & IB_SEND_INLINE) {
1640 if (wr->sg_list[0].length > qp->max_inline_data) {
ef99c4c2
NG		 1641 pr_err("%s() supported_len=0x%x,\n"
1642 " unspported len req=0x%x\n", __func__,
1643 qp->max_inline_data, wr->sg_list[0].length);
fe2caefc
PP		 1644 return -EINVAL;
1645 }
1646 memcpy(sge,
1647 (void *)(unsigned long)wr->sg_list[0].addr,
1648 wr->sg_list[0].length);
1649 hdr->total_len = wr->sg_list[0].length;
1650 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1651 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1652 } else {
1653 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1654 if (wr->num_sge)
1655 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1656 else
1657 wqe_size += sizeof(struct ocrdma_sge);
1658 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1659 }
1660 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1661 return 0;
1662}
1663
1664static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1665 struct ib_send_wr *wr)
1666{
1667 int status;
1668 struct ocrdma_sge *sge;
1669 u32 wqe_size = sizeof(*hdr);
1670
1671 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1672 ocrdma_build_ud_hdr(qp, hdr, wr);
1673 sge = (struct ocrdma_sge *)(hdr + 2);
1674 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
f99b1649 1675 } else {
fe2caefc 1676 sge = (struct ocrdma_sge *)(hdr + 1);
f99b1649 1677 }
fe2caefc
PP		 1678
1679 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1680 return status;
1681}
1682
1683static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1684 struct ib_send_wr *wr)
1685{
1686 int status;
1687 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1688 struct ocrdma_sge *sge = ext_rw + 1;
1689 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1690
1691 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1692 if (status)
1693 return status;
1694 ext_rw->addr_lo = wr->wr.rdma.remote_addr;
1695 ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);
1696 ext_rw->lrkey = wr->wr.rdma.rkey;
1697 ext_rw->len = hdr->total_len;
1698 return 0;
1699}
1700
1701static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1702 struct ib_send_wr *wr)
1703{
1704 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1705 struct ocrdma_sge *sge = ext_rw + 1;
1706 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
1707 sizeof(struct ocrdma_hdr_wqe);
1708
1709 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1710 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1711 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
1712 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1713
1714 ext_rw->addr_lo = wr->wr.rdma.remote_addr;
1715 ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);
1716 ext_rw->lrkey = wr->wr.rdma.rkey;
1717 ext_rw->len = hdr->total_len;
1718}
1719
1720static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
1721{
1722 u32 val = qp->sq.dbid | (1 << 16);
1723
1724 iowrite32(val, qp->sq_db);
1725}
1726
1727int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1728 struct ib_send_wr **bad_wr)
1729{
1730 int status = 0;
1731 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1732 struct ocrdma_hdr_wqe *hdr;
1733 unsigned long flags;
1734
1735 spin_lock_irqsave(&qp->q_lock, flags);
1736 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
1737 spin_unlock_irqrestore(&qp->q_lock, flags);
f6ddcf71 1738 *bad_wr = wr;
fe2caefc
PP		 1739 return -EINVAL;
1740 }
1741
1742 while (wr) {
1743 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
1744 wr->num_sge > qp->sq.max_sges) {
f6ddcf71 1745 *bad_wr = wr;
fe2caefc
PP		 1746 status = -ENOMEM;
1747 break;
1748 }
1749 hdr = ocrdma_hwq_head(&qp->sq);
1750 hdr->cw = 0;
1751 if (wr->send_flags & IB_SEND_SIGNALED)
1752 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
1753 if (wr->send_flags & IB_SEND_FENCE)
1754 hdr->cw |=
1755 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
1756 if (wr->send_flags & IB_SEND_SOLICITED)
1757 hdr->cw |=
1758 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
1759 hdr->total_len = 0;
1760 switch (wr->opcode) {
1761 case IB_WR_SEND_WITH_IMM:
1762 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
1763 hdr->immdt = ntohl(wr->ex.imm_data);
1764 case IB_WR_SEND:
1765 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
1766 ocrdma_build_send(qp, hdr, wr);
1767 break;
1768 case IB_WR_SEND_WITH_INV:
1769 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
1770 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
1771 hdr->lkey = wr->ex.invalidate_rkey;
1772 status = ocrdma_build_send(qp, hdr, wr);
1773 break;
1774 case IB_WR_RDMA_WRITE_WITH_IMM:
1775 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
1776 hdr->immdt = ntohl(wr->ex.imm_data);
1777 case IB_WR_RDMA_WRITE:
1778 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
1779 status = ocrdma_build_write(qp, hdr, wr);
1780 break;
1781 case IB_WR_RDMA_READ_WITH_INV:
1782 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
1783 case IB_WR_RDMA_READ:
1784 ocrdma_build_read(qp, hdr, wr);
1785 break;
1786 case IB_WR_LOCAL_INV:
1787 hdr->cw |=
1788 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
1789 hdr->cw |= (sizeof(struct ocrdma_hdr_wqe) /
1790 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
1791 hdr->lkey = wr->ex.invalidate_rkey;
1792 break;
1793 default:
1794 status = -EINVAL;
1795 break;
1796 }
1797 if (status) {
1798 *bad_wr = wr;
1799 break;
1800 }
1801 if (wr->send_flags & IB_SEND_SIGNALED)
1802 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
1803 else
1804 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
1805 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
1806 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
1807 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
1808 /* make sure wqe is written before adapter can access it */
1809 wmb();
1810 /* inform hw to start processing it */
1811 ocrdma_ring_sq_db(qp);
1812
1813 /* update pointer, counter for next wr */
1814 ocrdma_hwq_inc_head(&qp->sq);
1815 wr = wr->next;
1816 }
1817 spin_unlock_irqrestore(&qp->q_lock, flags);
1818 return status;
1819}
1820
1821static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
1822{
df176ea0 1823 u32 val = qp->rq.dbid | (1 << ocrdma_get_num_posted_shift(qp));
fe2caefc
PP		 1824
1825 iowrite32(val, qp->rq_db);
1826}
1827
1828static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr,
1829 u16 tag)
1830{
1831 u32 wqe_size = 0;
1832 struct ocrdma_sge *sge;
1833 if (wr->num_sge)
1834 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
1835 else
1836 wqe_size = sizeof(*sge) + sizeof(*rqe);
1837
1838 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
1839 OCRDMA_WQE_SIZE_SHIFT);
1840 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
1841 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1842 rqe->total_len = 0;
1843 rqe->rsvd_tag = tag;
1844 sge = (struct ocrdma_sge *)(rqe + 1);
1845 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
1846 ocrdma_cpu_to_le32(rqe, wqe_size);
1847}
1848
1849int ocrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1850 struct ib_recv_wr **bad_wr)
1851{
1852 int status = 0;
1853 unsigned long flags;
1854 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1855 struct ocrdma_hdr_wqe *rqe;
1856
1857 spin_lock_irqsave(&qp->q_lock, flags);
1858 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
1859 spin_unlock_irqrestore(&qp->q_lock, flags);
1860 *bad_wr = wr;
1861 return -EINVAL;
1862 }
1863 while (wr) {
1864 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
1865 wr->num_sge > qp->rq.max_sges) {
1866 *bad_wr = wr;
1867 status = -ENOMEM;
1868 break;
1869 }
1870 rqe = ocrdma_hwq_head(&qp->rq);
1871 ocrdma_build_rqe(rqe, wr, 0);
1872
1873 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
1874 /* make sure rqe is written before adapter can access it */
1875 wmb();
1876
1877 /* inform hw to start processing it */
1878 ocrdma_ring_rq_db(qp);
1879
1880 /* update pointer, counter for next wr */
1881 ocrdma_hwq_inc_head(&qp->rq);
1882 wr = wr->next;
1883 }
1884 spin_unlock_irqrestore(&qp->q_lock, flags);
1885 return status;
1886}
1887
1888/* cqe for srq's rqe can potentially arrive out of order.
1889 * index gives the entry in the shadow table where to store
1890 * the wr_id. tag/index is returned in cqe to reference back
1891 * for a given rqe.
1892 */
1893static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
1894{
1895 int row = 0;
1896 int indx = 0;
1897
1898 for (row = 0; row < srq->bit_fields_len; row++) {
1899 if (srq->idx_bit_fields[row]) {
1900 indx = ffs(srq->idx_bit_fields[row]);
1901 indx = (row * 32) + (indx - 1);
1902 if (indx >= srq->rq.max_cnt)
1903 BUG();
1904 ocrdma_srq_toggle_bit(srq, indx);
1905 break;
1906 }
1907 }
1908
1909 if (row == srq->bit_fields_len)
1910 BUG();
1911 return indx;
1912}
1913
1914static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
1915{
1916 u32 val = srq->rq.dbid | (1 << 16);
1917
1918 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
1919}
1920
1921int ocrdma_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
1922 struct ib_recv_wr **bad_wr)
1923{
1924 int status = 0;
1925 unsigned long flags;
1926 struct ocrdma_srq *srq;
1927 struct ocrdma_hdr_wqe *rqe;
1928 u16 tag;
1929
1930 srq = get_ocrdma_srq(ibsrq);
1931
1932 spin_lock_irqsave(&srq->q_lock, flags);
1933 while (wr) {
1934 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
1935 wr->num_sge > srq->rq.max_sges) {
1936 status = -ENOMEM;
1937 *bad_wr = wr;
1938 break;
1939 }
1940 tag = ocrdma_srq_get_idx(srq);
1941 rqe = ocrdma_hwq_head(&srq->rq);
1942 ocrdma_build_rqe(rqe, wr, tag);
1943
1944 srq->rqe_wr_id_tbl[tag] = wr->wr_id;
1945 /* make sure rqe is written before adapter can perform DMA */
1946 wmb();
1947 /* inform hw to start processing it */
1948 ocrdma_ring_srq_db(srq);
1949 /* update pointer, counter for next wr */
1950 ocrdma_hwq_inc_head(&srq->rq);
1951 wr = wr->next;
1952 }
1953 spin_unlock_irqrestore(&srq->q_lock, flags);
1954 return status;
1955}
1956
1957static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
1958{
f99b1649 1959 enum ib_wc_status ibwc_status;
fe2caefc
PP		 1960
1961 switch (status) {
1962 case OCRDMA_CQE_GENERAL_ERR:
1963 ibwc_status = IB_WC_GENERAL_ERR;
1964 break;
1965 case OCRDMA_CQE_LOC_LEN_ERR:
1966 ibwc_status = IB_WC_LOC_LEN_ERR;
1967 break;
1968 case OCRDMA_CQE_LOC_QP_OP_ERR:
1969 ibwc_status = IB_WC_LOC_QP_OP_ERR;
1970 break;
1971 case OCRDMA_CQE_LOC_EEC_OP_ERR:
1972 ibwc_status = IB_WC_LOC_EEC_OP_ERR;
1973 break;
1974 case OCRDMA_CQE_LOC_PROT_ERR:
1975 ibwc_status = IB_WC_LOC_PROT_ERR;
1976 break;
1977 case OCRDMA_CQE_WR_FLUSH_ERR:
1978 ibwc_status = IB_WC_WR_FLUSH_ERR;
1979 break;
1980 case OCRDMA_CQE_MW_BIND_ERR:
1981 ibwc_status = IB_WC_MW_BIND_ERR;
1982 break;
1983 case OCRDMA_CQE_BAD_RESP_ERR:
1984 ibwc_status = IB_WC_BAD_RESP_ERR;
1985 break;
1986 case OCRDMA_CQE_LOC_ACCESS_ERR:
1987 ibwc_status = IB_WC_LOC_ACCESS_ERR;
1988 break;
1989 case OCRDMA_CQE_REM_INV_REQ_ERR:
1990 ibwc_status = IB_WC_REM_INV_REQ_ERR;
1991 break;
1992 case OCRDMA_CQE_REM_ACCESS_ERR:
1993 ibwc_status = IB_WC_REM_ACCESS_ERR;
1994 break;
1995 case OCRDMA_CQE_REM_OP_ERR:
1996 ibwc_status = IB_WC_REM_OP_ERR;
1997 break;
1998 case OCRDMA_CQE_RETRY_EXC_ERR:
1999 ibwc_status = IB_WC_RETRY_EXC_ERR;
2000 break;
2001 case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2002 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2003 break;
2004 case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2005 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2006 break;
2007 case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2008 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2009 break;
2010 case OCRDMA_CQE_REM_ABORT_ERR:
2011 ibwc_status = IB_WC_REM_ABORT_ERR;
2012 break;
2013 case OCRDMA_CQE_INV_EECN_ERR:
2014 ibwc_status = IB_WC_INV_EECN_ERR;
2015 break;
2016 case OCRDMA_CQE_INV_EEC_STATE_ERR:
2017 ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2018 break;
2019 case OCRDMA_CQE_FATAL_ERR:
2020 ibwc_status = IB_WC_FATAL_ERR;
2021 break;
2022 case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2023 ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2024 break;
2025 default:
2026 ibwc_status = IB_WC_GENERAL_ERR;
2027 break;
2028 };
2029 return ibwc_status;
2030}
2031
2032static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2033 u32 wqe_idx)
2034{
2035 struct ocrdma_hdr_wqe *hdr;
2036 struct ocrdma_sge *rw;
2037 int opcode;
2038
2039 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2040
2041 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2042 /* Undo the hdr->cw swap */
2043 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2044 switch (opcode) {
2045 case OCRDMA_WRITE:
2046 ibwc->opcode = IB_WC_RDMA_WRITE;
2047 break;
2048 case OCRDMA_READ:
2049 rw = (struct ocrdma_sge *)(hdr + 1);
2050 ibwc->opcode = IB_WC_RDMA_READ;
2051 ibwc->byte_len = rw->len;
2052 break;
2053 case OCRDMA_SEND:
2054 ibwc->opcode = IB_WC_SEND;
2055 break;
2056 case OCRDMA_LKEY_INV:
2057 ibwc->opcode = IB_WC_LOCAL_INV;
2058 break;
2059 default:
2060 ibwc->status = IB_WC_GENERAL_ERR;
ef99c4c2
NG		 2061 pr_err("%s() invalid opcode received = 0x%x\n",
2062 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
fe2caefc
PP		 2063 break;
2064 };
2065}
2066
2067static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2068 struct ocrdma_cqe *cqe)
2069{
2070 if (is_cqe_for_sq(cqe)) {
2071 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2072 cqe->flags_status_srcqpn) &
2073 ~OCRDMA_CQE_STATUS_MASK);
2074 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2075 cqe->flags_status_srcqpn) |
2076 (OCRDMA_CQE_WR_FLUSH_ERR <<
2077 OCRDMA_CQE_STATUS_SHIFT));
2078 } else {
2079 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2080 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2081 cqe->flags_status_srcqpn) &
2082 ~OCRDMA_CQE_UD_STATUS_MASK);
2083 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2084 cqe->flags_status_srcqpn) |
2085 (OCRDMA_CQE_WR_FLUSH_ERR <<
2086 OCRDMA_CQE_UD_STATUS_SHIFT));
2087 } else {
2088 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2089 cqe->flags_status_srcqpn) &
2090 ~OCRDMA_CQE_STATUS_MASK);
2091 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2092 cqe->flags_status_srcqpn) |
2093 (OCRDMA_CQE_WR_FLUSH_ERR <<
2094 OCRDMA_CQE_STATUS_SHIFT));
2095 }
2096 }
2097}
2098
2099static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2100 struct ocrdma_qp *qp, int status)
2101{
2102 bool expand = false;
2103
2104 ibwc->byte_len = 0;
2105 ibwc->qp = &qp->ibqp;
2106 ibwc->status = ocrdma_to_ibwc_err(status);
2107
2108 ocrdma_flush_qp(qp);
2109 ocrdma_qp_state_machine(qp, IB_QPS_ERR, NULL);
2110
2111 /* if wqe/rqe pending for which cqe needs to be returned,
2112 * trigger inflating it.
2113 */
2114 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2115 expand = true;
2116 ocrdma_set_cqe_status_flushed(qp, cqe);
2117 }
2118 return expand;
2119}
2120
2121static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2122 struct ocrdma_qp *qp, int status)
2123{
2124 ibwc->opcode = IB_WC_RECV;
2125 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2126 ocrdma_hwq_inc_tail(&qp->rq);
2127
2128 return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2129}
2130
2131static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2132 struct ocrdma_qp *qp, int status)
2133{
2134 ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2135 ocrdma_hwq_inc_tail(&qp->sq);
2136
2137 return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2138}
2139
2140
2141static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2142 struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2143 bool *polled, bool *stop)
2144{
2145 bool expand;
2146 int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2147 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2148
2149 /* when hw sq is empty, but rq is not empty, so we continue
2150 * to keep the cqe in order to get the cq event again.
2151 */
2152 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2153 /* when cq for rq and sq is same, it is safe to return
2154 * flush cqe for RQEs.
2155 */
2156 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2157 *polled = true;
2158 status = OCRDMA_CQE_WR_FLUSH_ERR;
2159 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2160 } else {
2161 /* stop processing further cqe as this cqe is used for
2162 * triggering cq event on buddy cq of RQ.
2163 * When QP is destroyed, this cqe will be removed
2164 * from the cq's hardware q.
2165 */
2166 *polled = false;
2167 *stop = true;
2168 expand = false;
2169 }
2170 } else {
2171 *polled = true;
2172 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2173 }
2174 return expand;
2175}
2176
2177static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2178 struct ocrdma_cqe *cqe,
2179 struct ib_wc *ibwc, bool *polled)
2180{
2181 bool expand = false;
2182 int tail = qp->sq.tail;
2183 u32 wqe_idx;
2184
2185 if (!qp->wqe_wr_id_tbl[tail].signaled) {
fe2caefc
PP		 2186 *polled = false; /* WC cannot be consumed yet */
2187 } else {
2188 ibwc->status = IB_WC_SUCCESS;
2189 ibwc->wc_flags = 0;
2190 ibwc->qp = &qp->ibqp;
2191 ocrdma_update_wc(qp, ibwc, tail);
2192 *polled = true;
fe2caefc 2193 }
ae3bca90
PP		 2194 wqe_idx = le32_to_cpu(cqe->wq.wqeidx) & OCRDMA_CQE_WQEIDX_MASK;
2195 if (tail != wqe_idx)
2196 expand = true; /* Coalesced CQE can't be consumed yet */
2197
fe2caefc
PP		 2198 ocrdma_hwq_inc_tail(&qp->sq);
2199 return expand;
2200}
2201
2202static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2203 struct ib_wc *ibwc, bool *polled, bool *stop)
2204{
2205 int status;
2206 bool expand;
2207
2208 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2209 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2210
2211 if (status == OCRDMA_CQE_SUCCESS)
2212 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2213 else
2214 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2215 return expand;
2216}
2217
2218static int ocrdma_update_ud_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe)
2219{
2220 int status;
2221
2222 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2223 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2224 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2225 OCRDMA_CQE_SRCQP_MASK;
2226 ibwc->pkey_index = le32_to_cpu(cqe->ud.rxlen_pkey) &
2227 OCRDMA_CQE_PKEY_MASK;
2228 ibwc->wc_flags = IB_WC_GRH;
2229 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2230 OCRDMA_CQE_UD_XFER_LEN_SHIFT);
2231 return status;
2232}
2233
2234static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2235 struct ocrdma_cqe *cqe,
2236 struct ocrdma_qp *qp)
2237{
2238 unsigned long flags;
2239 struct ocrdma_srq *srq;
2240 u32 wqe_idx;
2241
2242 srq = get_ocrdma_srq(qp->ibqp.srq);
2243 wqe_idx = le32_to_cpu(cqe->rq.buftag_qpn) >> OCRDMA_CQE_BUFTAG_SHIFT;
2244 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2245 spin_lock_irqsave(&srq->q_lock, flags);
2246 ocrdma_srq_toggle_bit(srq, wqe_idx);
2247 spin_unlock_irqrestore(&srq->q_lock, flags);
2248 ocrdma_hwq_inc_tail(&srq->rq);
2249}
2250
2251static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2252 struct ib_wc *ibwc, bool *polled, bool *stop,
2253 int status)
2254{
2255 bool expand;
2256
2257 /* when hw_rq is empty, but wq is not empty, so continue
2258 * to keep the cqe to get the cq event again.
2259 */
2260 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2261 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2262 *polled = true;
2263 status = OCRDMA_CQE_WR_FLUSH_ERR;
2264 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2265 } else {
2266 *polled = false;
2267 *stop = true;
2268 expand = false;
2269 }
a3698a9b
PP		 2270 } else {
2271 *polled = true;
fe2caefc 2272 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
a3698a9b 2273 }
fe2caefc
PP		 2274 return expand;
2275}
2276
2277static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2278 struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2279{
2280 ibwc->opcode = IB_WC_RECV;
2281 ibwc->qp = &qp->ibqp;
2282 ibwc->status = IB_WC_SUCCESS;
2283
2284 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2285 ocrdma_update_ud_rcqe(ibwc, cqe);
2286 else
2287 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2288
2289 if (is_cqe_imm(cqe)) {
2290 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2291 ibwc->wc_flags |= IB_WC_WITH_IMM;
2292 } else if (is_cqe_wr_imm(cqe)) {
2293 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2294 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2295 ibwc->wc_flags |= IB_WC_WITH_IMM;
2296 } else if (is_cqe_invalidated(cqe)) {
2297 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2298 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2299 }
f99b1649 2300 if (qp->ibqp.srq) {
fe2caefc 2301 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
f99b1649 2302 } else {
fe2caefc
PP		 2303 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2304 ocrdma_hwq_inc_tail(&qp->rq);
2305 }
2306}
2307
2308static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2309 struct ib_wc *ibwc, bool *polled, bool *stop)
2310{
2311 int status;
2312 bool expand = false;
2313
2314 ibwc->wc_flags = 0;
f99b1649 2315 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
fe2caefc
PP		 2316 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2317 OCRDMA_CQE_UD_STATUS_MASK) >>
2318 OCRDMA_CQE_UD_STATUS_SHIFT;
f99b1649 2319 } else {
fe2caefc
PP		 2320 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2321 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
f99b1649 2322 }
fe2caefc
PP		 2323
2324 if (status == OCRDMA_CQE_SUCCESS) {
2325 *polled = true;
2326 ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2327 } else {
2328 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2329 status);
2330 }
2331 return expand;
2332}
2333
2334static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2335 u16 cur_getp)
2336{
2337 if (cq->phase_change) {
2338 if (cur_getp == 0)
2339 cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
f99b1649 2340 } else {
fe2caefc
PP		 2341 /* clear valid bit */
2342 cqe->flags_status_srcqpn = 0;
f99b1649 2343 }
fe2caefc
PP		 2344}
2345
2346static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2347 struct ib_wc *ibwc)
2348{
2349 u16 qpn = 0;
2350 int i = 0;
2351 bool expand = false;
2352 int polled_hw_cqes = 0;
2353 struct ocrdma_qp *qp = NULL;
1afc0454 2354 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
fe2caefc
PP		 2355 struct ocrdma_cqe *cqe;
2356 u16 cur_getp; bool polled = false; bool stop = false;
2357
2358 cur_getp = cq->getp;
2359 while (num_entries) {
2360 cqe = cq->va + cur_getp;
2361 /* check whether valid cqe or not */
2362 if (!is_cqe_valid(cq, cqe))
2363 break;
2364 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2365 /* ignore discarded cqe */
2366 if (qpn == 0)
2367 goto skip_cqe;
2368 qp = dev->qp_tbl[qpn];
2369 BUG_ON(qp == NULL);
2370
2371 if (is_cqe_for_sq(cqe)) {
2372 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2373 &stop);
2374 } else {
2375 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2376 &stop);
2377 }
2378 if (expand)
2379 goto expand_cqe;
2380 if (stop)
2381 goto stop_cqe;
2382 /* clear qpn to avoid duplicate processing by discard_cqe() */
2383 cqe->cmn.qpn = 0;
2384skip_cqe:
2385 polled_hw_cqes += 1;
2386 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2387 ocrdma_change_cq_phase(cq, cqe, cur_getp);
2388expand_cqe:
2389 if (polled) {
2390 num_entries -= 1;
2391 i += 1;
2392 ibwc = ibwc + 1;
2393 polled = false;
2394 }
2395 }
2396stop_cqe:
2397 cq->getp = cur_getp;
2398 if (polled_hw_cqes || expand || stop) {
2399 ocrdma_ring_cq_db(dev, cq->id, cq->armed, cq->solicited,
2400 polled_hw_cqes);
2401 }
2402 return i;
2403}
2404
2405/* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2406static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2407 struct ocrdma_qp *qp, struct ib_wc *ibwc)
2408{
2409 int err_cqes = 0;
2410
2411 while (num_entries) {
2412 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2413 break;
2414 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2415 ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2416 ocrdma_hwq_inc_tail(&qp->sq);
2417 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2418 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2419 ocrdma_hwq_inc_tail(&qp->rq);
f99b1649 2420 } else {
fe2caefc 2421 return err_cqes;
f99b1649 2422 }
fe2caefc
PP		 2423 ibwc->byte_len = 0;
2424 ibwc->status = IB_WC_WR_FLUSH_ERR;
2425 ibwc = ibwc + 1;
2426 err_cqes += 1;
2427 num_entries -= 1;
2428 }
2429 return err_cqes;
2430}
2431
2432int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2433{
2434 int cqes_to_poll = num_entries;
1afc0454
NG		 2435 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2436 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
fe2caefc
PP		 2437 int num_os_cqe = 0, err_cqes = 0;
2438 struct ocrdma_qp *qp;
1afc0454 2439 unsigned long flags;
fe2caefc
PP		 2440
2441 /* poll cqes from adapter CQ */
2442 spin_lock_irqsave(&cq->cq_lock, flags);
2443 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);
2444 spin_unlock_irqrestore(&cq->cq_lock, flags);
2445 cqes_to_poll -= num_os_cqe;
2446
2447 if (cqes_to_poll) {
2448 wc = wc + num_os_cqe;
2449 /* adapter returns single error cqe when qp moves to
2450 * error state. So insert error cqes with wc_status as
2451 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2452 * respectively which uses this CQ.
2453 */
2454 spin_lock_irqsave(&dev->flush_q_lock, flags);
2455 list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2456 if (cqes_to_poll == 0)
2457 break;
2458 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
2459 cqes_to_poll -= err_cqes;
2460 num_os_cqe += err_cqes;
2461 wc = wc + err_cqes;
2462 }
2463 spin_unlock_irqrestore(&dev->flush_q_lock, flags);
2464 }
2465 return num_os_cqe;
2466}
2467
2468int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2469{
1afc0454
NG		 2470 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2471 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
fe2caefc
PP		 2472 u16 cq_id;
2473 u16 cur_getp;
2474 struct ocrdma_cqe *cqe;
1afc0454 2475 unsigned long flags;
fe2caefc 2476
fe2caefc 2477 cq_id = cq->id;
fe2caefc
PP		 2478
2479 spin_lock_irqsave(&cq->cq_lock, flags);
2480 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2481 cq->armed = true;
2482 if (cq_flags & IB_CQ_SOLICITED)
2483 cq->solicited = true;
2484
2485 cur_getp = cq->getp;
2486 cqe = cq->va + cur_getp;
2487
2488 /* check whether any valid cqe exist or not, if not then safe to
2489 * arm. If cqe is not yet consumed, then let it get consumed and then
2490 * we arm it to avoid false interrupts.
2491 */
2492 if (!is_cqe_valid(cq, cqe) || cq->arm_needed) {
2493 cq->arm_needed = false;
2494 ocrdma_ring_cq_db(dev, cq_id, cq->armed, cq->solicited, 0);
2495 }
2496 spin_unlock_irqrestore(&cq->cq_lock, flags);
2497 return 0;
2498}
Linux kernel - Deliverables
This page took 0.179756 seconds and 5 git commands to generate.