2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/random.h>
37 #include <linux/highmem.h>
38 #include <linux/slab.h>
39 #include <asm/byteorder.h>
41 #include <rdma/ib_verbs.h>
42 #include <rdma/iw_cm.h>
43 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
49 atomic_t mod_qp_timouts
;
51 atomic_t sw_qps_destroyed
;
53 static void nes_unregister_ofa_device(struct nes_ib_device
*nesibdev
);
54 static int nes_dereg_mr(struct ib_mr
*ib_mr
);
59 static struct ib_mw
*nes_alloc_mw(struct ib_pd
*ibpd
, enum ib_mw_type type
)
61 struct nes_pd
*nespd
= to_nespd(ibpd
);
62 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
63 struct nes_device
*nesdev
= nesvnic
->nesdev
;
64 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
65 struct nes_cqp_request
*cqp_request
;
68 struct nes_hw_cqp_wqe
*cqp_wqe
;
72 u32 next_stag_index
= 0;
76 if (type
!= IB_MW_TYPE_1
)
77 return ERR_PTR(-EINVAL
);
79 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
80 stag_key
= (u8
)next_stag_index
;
84 next_stag_index
>>= 8;
85 next_stag_index
%= nesadapter
->max_mr
;
87 ret
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
88 nesadapter
->max_mr
, &stag_index
, &next_stag_index
, NES_RESOURCE_MW
);
93 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
95 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
96 return ERR_PTR(-ENOMEM
);
99 stag
= stag_index
<< 8;
101 stag
+= (u32
)stag_key
;
103 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, index = 0x%08X\n",
106 /* Register the region with the adapter */
107 cqp_request
= nes_get_cqp_request(nesdev
);
108 if (cqp_request
== NULL
) {
110 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
111 return ERR_PTR(-ENOMEM
);
114 cqp_request
->waiting
= 1;
115 cqp_wqe
= &cqp_request
->cqp_wqe
;
117 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] =
118 cpu_to_le32( NES_CQP_ALLOCATE_STAG
| NES_CQP_STAG_RIGHTS_REMOTE_READ
|
119 NES_CQP_STAG_RIGHTS_REMOTE_WRITE
| NES_CQP_STAG_VA_TO
|
120 NES_CQP_STAG_REM_ACC_EN
);
122 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
123 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
, (nespd
->pd_id
& 0x00007fff));
124 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
126 atomic_set(&cqp_request
->refcount
, 2);
127 nes_post_cqp_request(nesdev
, cqp_request
);
130 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
132 nes_debug(NES_DBG_MR
, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
133 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
134 stag
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
135 if ((!ret
) || (cqp_request
->major_code
)) {
136 nes_put_cqp_request(nesdev
, cqp_request
);
138 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
140 return ERR_PTR(-ETIME
);
142 return ERR_PTR(-ENOMEM
);
145 nes_put_cqp_request(nesdev
, cqp_request
);
147 nesmr
->ibmw
.rkey
= stag
;
148 nesmr
->mode
= IWNES_MEMREG_TYPE_MW
;
151 nesmr
->pbls_used
= 0;
160 static int nes_dealloc_mw(struct ib_mw
*ibmw
)
162 struct nes_mr
*nesmr
= to_nesmw(ibmw
);
163 struct nes_vnic
*nesvnic
= to_nesvnic(ibmw
->device
);
164 struct nes_device
*nesdev
= nesvnic
->nesdev
;
165 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
166 struct nes_hw_cqp_wqe
*cqp_wqe
;
167 struct nes_cqp_request
*cqp_request
;
171 /* Deallocate the window with the adapter */
172 cqp_request
= nes_get_cqp_request(nesdev
);
173 if (cqp_request
== NULL
) {
174 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
177 cqp_request
->waiting
= 1;
178 cqp_wqe
= &cqp_request
->cqp_wqe
;
179 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
180 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, NES_CQP_DEALLOCATE_STAG
);
181 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, ibmw
->rkey
);
183 atomic_set(&cqp_request
->refcount
, 2);
184 nes_post_cqp_request(nesdev
, cqp_request
);
187 nes_debug(NES_DBG_MR
, "Waiting for deallocate STag 0x%08X to complete.\n",
189 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
191 nes_debug(NES_DBG_MR
, "Deallocate STag completed, wait_event_timeout ret = %u,"
192 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
193 ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
196 else if (cqp_request
->major_code
)
199 nes_put_cqp_request(nesdev
, cqp_request
);
201 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
202 (ibmw
->rkey
& 0x0fffff00) >> 8);
212 static int nes_bind_mw(struct ib_qp
*ibqp
, struct ib_mw
*ibmw
,
213 struct ib_mw_bind
*ibmw_bind
)
216 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
217 struct nes_device
*nesdev
= nesvnic
->nesdev
;
218 /* struct nes_mr *nesmr = to_nesmw(ibmw); */
219 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
220 struct nes_hw_qp_wqe
*wqe
;
221 unsigned long flags
= 0;
226 if (nesqp
->ibqp_state
> IB_QPS_RTS
)
229 spin_lock_irqsave(&nesqp
->lock
, flags
);
231 head
= nesqp
->hwqp
.sq_head
;
232 qsize
= nesqp
->hwqp
.sq_tail
;
234 /* Check for SQ overflow */
235 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.sq_tail
) % qsize
) == (qsize
- 1)) {
236 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
240 wqe
= &nesqp
->hwqp
.sq_vbase
[head
];
241 /* nes_debug(NES_DBG_MR, "processing sq wqe at %p, head = %u.\n", wqe, head); */
242 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
243 u64temp
= ibmw_bind
->wr_id
;
244 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
, u64temp
);
245 wqe_misc
= NES_IWARP_SQ_OP_BIND
;
247 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
249 if (ibmw_bind
->send_flags
& IB_SEND_SIGNALED
)
250 wqe_misc
|= NES_IWARP_SQ_WQE_SIGNALED_COMPL
;
252 if (ibmw_bind
->bind_info
.mw_access_flags
& IB_ACCESS_REMOTE_WRITE
)
253 wqe_misc
|= NES_CQP_STAG_RIGHTS_REMOTE_WRITE
;
254 if (ibmw_bind
->bind_info
.mw_access_flags
& IB_ACCESS_REMOTE_READ
)
255 wqe_misc
|= NES_CQP_STAG_RIGHTS_REMOTE_READ
;
257 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_MISC_IDX
, wqe_misc
);
258 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_MR_IDX
,
259 ibmw_bind
->bind_info
.mr
->lkey
);
260 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_MW_IDX
, ibmw
->rkey
);
261 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX
,
262 ibmw_bind
->bind_info
.length
);
263 wqe
->wqe_words
[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX
] = 0;
264 u64temp
= (u64
)ibmw_bind
->bind_info
.addr
;
265 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX
, u64temp
);
271 nesqp
->hwqp
.sq_head
= head
;
274 nes_write32(nesdev
->regs
+NES_WQE_ALLOC
,
275 (1 << 24) | 0x00800000 | nesqp
->hwqp
.qp_id
);
277 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
286 static int alloc_fast_reg_mr(struct nes_device
*nesdev
, struct nes_pd
*nespd
,
287 u32 stag
, u32 page_count
)
289 struct nes_hw_cqp_wqe
*cqp_wqe
;
290 struct nes_cqp_request
*cqp_request
;
293 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
296 u64 region_length
= page_count
* PAGE_SIZE
;
299 cqp_request
= nes_get_cqp_request(nesdev
);
300 if (cqp_request
== NULL
) {
301 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
304 nes_debug(NES_DBG_MR
, "alloc_fast_reg_mr: page_count = %d, "
305 "region_length = %llu\n",
306 page_count
, region_length
);
307 cqp_request
->waiting
= 1;
308 cqp_wqe
= &cqp_request
->cqp_wqe
;
310 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
311 if (nesadapter
->free_4kpbl
> 0) {
312 nesadapter
->free_4kpbl
--;
313 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
315 /* No 4kpbl's available: */
316 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
317 nes_debug(NES_DBG_MR
, "Out of Pbls\n");
318 nes_free_cqp_request(nesdev
, cqp_request
);
322 opcode
= NES_CQP_ALLOCATE_STAG
| NES_CQP_STAG_MR
|
323 NES_CQP_STAG_PBL_BLK_SIZE
| NES_CQP_STAG_VA_TO
|
324 NES_CQP_STAG_REM_ACC_EN
;
326 * The current OFED API does not support the zero based TO option.
327 * If added then need to changed the NES_CQP_STAG_VA* option. Also,
328 * the API does not support that ability to have the MR set for local
329 * access only when created and not allow the SQ op to override. Given
330 * this the remote enable must be set here.
333 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
334 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
335 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX
, 1);
337 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] =
338 cpu_to_le32((u32
)(region_length
>> 8) & 0xff000000);
339 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] |=
340 cpu_to_le32(nespd
->pd_id
& 0x00007fff);
342 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
343 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_VA_LOW_IDX
, 0);
344 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_LOW_IDX
, 0);
345 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, 0);
346 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_LEN_IDX
, (page_count
* 8));
347 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE
);
350 atomic_set(&cqp_request
->refcount
, 2);
351 nes_post_cqp_request(nesdev
, cqp_request
);
354 ret
= wait_event_timeout(cqp_request
->waitq
,
355 (0 != cqp_request
->request_done
),
358 nes_debug(NES_DBG_MR
, "Allocate STag 0x%08X completed, "
359 "wait_event_timeout ret = %u, CQP Major:Minor codes = "
360 "0x%04X:0x%04X.\n", stag
, ret
, cqp_request
->major_code
,
361 cqp_request
->minor_code
);
362 major_code
= cqp_request
->major_code
;
363 nes_put_cqp_request(nesdev
, cqp_request
);
365 if (!ret
|| major_code
) {
366 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
367 nesadapter
->free_4kpbl
++;
368 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
381 static struct ib_mr
*nes_alloc_mr(struct ib_pd
*ibpd
,
382 enum ib_mr_type mr_type
,
385 struct nes_pd
*nespd
= to_nespd(ibpd
);
386 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
387 struct nes_device
*nesdev
= nesvnic
->nesdev
;
388 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
395 struct nes_mr
*nesmr
;
400 if (mr_type
!= IB_MR_TYPE_MEM_REG
)
401 return ERR_PTR(-EINVAL
);
403 if (max_num_sg
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
404 return ERR_PTR(-E2BIG
);
407 * Note: Set to always use a fixed length single page entry PBL. This is to allow
408 * for the fast_reg_mr operation to always know the size of the PBL.
410 if (max_num_sg
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
411 return ERR_PTR(-E2BIG
);
413 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
414 stag_key
= (u8
)next_stag_index
;
415 next_stag_index
>>= 8;
416 next_stag_index
%= nesadapter
->max_mr
;
418 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
419 nesadapter
->max_mr
, &stag_index
,
420 &next_stag_index
, NES_RESOURCE_FAST_MR
);
424 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
426 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
427 return ERR_PTR(-ENOMEM
);
430 stag
= stag_index
<< 8;
432 stag
+= (u32
)stag_key
;
434 nes_debug(NES_DBG_MR
, "Allocating STag 0x%08X index = 0x%08X\n",
437 ret
= alloc_fast_reg_mr(nesdev
, nespd
, stag
, max_num_sg
);
440 nesmr
->ibmr
.rkey
= stag
;
441 nesmr
->ibmr
.lkey
= stag
;
442 nesmr
->mode
= IWNES_MEMREG_TYPE_FMEM
;
446 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
447 return ERR_PTR(-ENOMEM
);
450 nesmr
->pages
= pci_alloc_consistent(nesdev
->pcidev
,
451 max_num_sg
* sizeof(u64
),
456 nesmr
->max_pages
= max_num_sg
;
463 return ERR_PTR(-ENOMEM
);
466 static int nes_set_page(struct ib_mr
*ibmr
, u64 addr
)
468 struct nes_mr
*nesmr
= to_nesmr(ibmr
);
470 if (unlikely(nesmr
->npages
== nesmr
->max_pages
))
473 nesmr
->pages
[nesmr
->npages
++] = cpu_to_le64(addr
);
478 static int nes_map_mr_sg(struct ib_mr
*ibmr
,
479 struct scatterlist
*sg
,
482 struct nes_mr
*nesmr
= to_nesmr(ibmr
);
486 return ib_sg_to_pages(ibmr
, sg
, sg_nents
, nes_set_page
);
490 * nes_alloc_fast_reg_page_list
492 static struct ib_fast_reg_page_list
*nes_alloc_fast_reg_page_list(
493 struct ib_device
*ibdev
,
496 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
497 struct nes_device
*nesdev
= nesvnic
->nesdev
;
498 struct ib_fast_reg_page_list
*pifrpl
;
499 struct nes_ib_fast_reg_page_list
*pnesfrpl
;
501 if (page_list_len
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
502 return ERR_PTR(-E2BIG
);
504 * Allocate the ib_fast_reg_page_list structure, the
505 * nes_fast_bpl structure, and the PLB table.
507 pnesfrpl
= kmalloc(sizeof(struct nes_ib_fast_reg_page_list
) +
508 page_list_len
* sizeof(u64
), GFP_KERNEL
);
511 return ERR_PTR(-ENOMEM
);
513 pifrpl
= &pnesfrpl
->ibfrpl
;
514 pifrpl
->page_list
= &pnesfrpl
->pbl
;
515 pifrpl
->max_page_list_len
= page_list_len
;
517 * Allocate the WQE PBL
519 pnesfrpl
->nes_wqe_pbl
.kva
= pci_alloc_consistent(nesdev
->pcidev
,
520 page_list_len
* sizeof(u64
),
521 &pnesfrpl
->nes_wqe_pbl
.paddr
);
523 if (!pnesfrpl
->nes_wqe_pbl
.kva
) {
525 return ERR_PTR(-ENOMEM
);
527 nes_debug(NES_DBG_MR
, "nes_alloc_fast_reg_pbl: nes_frpl = %p, "
528 "ibfrpl = %p, ibfrpl.page_list = %p, pbl.kva = %p, "
529 "pbl.paddr = %llx\n", pnesfrpl
, &pnesfrpl
->ibfrpl
,
530 pnesfrpl
->ibfrpl
.page_list
, pnesfrpl
->nes_wqe_pbl
.kva
,
531 (unsigned long long) pnesfrpl
->nes_wqe_pbl
.paddr
);
537 * nes_free_fast_reg_page_list
539 static void nes_free_fast_reg_page_list(struct ib_fast_reg_page_list
*pifrpl
)
541 struct nes_vnic
*nesvnic
= to_nesvnic(pifrpl
->device
);
542 struct nes_device
*nesdev
= nesvnic
->nesdev
;
543 struct nes_ib_fast_reg_page_list
*pnesfrpl
;
545 pnesfrpl
= container_of(pifrpl
, struct nes_ib_fast_reg_page_list
, ibfrpl
);
549 pci_free_consistent(nesdev
->pcidev
,
550 pifrpl
->max_page_list_len
* sizeof(u64
),
551 pnesfrpl
->nes_wqe_pbl
.kva
,
552 pnesfrpl
->nes_wqe_pbl
.paddr
);
554 * Free the PBL structure
562 static int nes_query_device(struct ib_device
*ibdev
, struct ib_device_attr
*props
,
563 struct ib_udata
*uhw
)
565 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
566 struct nes_device
*nesdev
= nesvnic
->nesdev
;
567 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
569 if (uhw
->inlen
|| uhw
->outlen
)
572 memset(props
, 0, sizeof(*props
));
573 memcpy(&props
->sys_image_guid
, nesvnic
->netdev
->dev_addr
, 6);
575 props
->fw_ver
= nesdev
->nesadapter
->firmware_version
;
576 props
->device_cap_flags
= nesdev
->nesadapter
->device_cap_flags
;
577 props
->vendor_id
= nesdev
->nesadapter
->vendor_id
;
578 props
->vendor_part_id
= nesdev
->nesadapter
->vendor_part_id
;
579 props
->hw_ver
= nesdev
->nesadapter
->hw_rev
;
580 props
->max_mr_size
= 0x80000000;
581 props
->max_qp
= nesibdev
->max_qp
;
582 props
->max_qp_wr
= nesdev
->nesadapter
->max_qp_wr
- 2;
583 props
->max_sge
= nesdev
->nesadapter
->max_sge
;
584 props
->max_cq
= nesibdev
->max_cq
;
585 props
->max_cqe
= nesdev
->nesadapter
->max_cqe
;
586 props
->max_mr
= nesibdev
->max_mr
;
587 props
->max_mw
= nesibdev
->max_mr
;
588 props
->max_pd
= nesibdev
->max_pd
;
589 props
->max_sge_rd
= 1;
590 switch (nesdev
->nesadapter
->max_irrq_wr
) {
592 props
->max_qp_rd_atom
= 2;
595 props
->max_qp_rd_atom
= 8;
598 props
->max_qp_rd_atom
= 32;
601 props
->max_qp_rd_atom
= 64;
604 props
->max_qp_rd_atom
= 0;
606 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
607 props
->atomic_cap
= IB_ATOMIC_NONE
;
608 props
->max_map_per_fmr
= 1;
617 static int nes_query_port(struct ib_device
*ibdev
, u8 port
, struct ib_port_attr
*props
)
619 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
620 struct net_device
*netdev
= nesvnic
->netdev
;
622 memset(props
, 0, sizeof(*props
));
624 props
->max_mtu
= IB_MTU_4096
;
626 if (netdev
->mtu
>= 4096)
627 props
->active_mtu
= IB_MTU_4096
;
628 else if (netdev
->mtu
>= 2048)
629 props
->active_mtu
= IB_MTU_2048
;
630 else if (netdev
->mtu
>= 1024)
631 props
->active_mtu
= IB_MTU_1024
;
632 else if (netdev
->mtu
>= 512)
633 props
->active_mtu
= IB_MTU_512
;
635 props
->active_mtu
= IB_MTU_256
;
641 if (netif_queue_stopped(netdev
))
642 props
->state
= IB_PORT_DOWN
;
643 else if (nesvnic
->linkup
)
644 props
->state
= IB_PORT_ACTIVE
;
646 props
->state
= IB_PORT_DOWN
;
647 props
->phys_state
= 0;
648 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
649 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
650 props
->gid_tbl_len
= 1;
651 props
->pkey_tbl_len
= 1;
652 props
->qkey_viol_cntr
= 0;
653 props
->active_width
= IB_WIDTH_4X
;
654 props
->active_speed
= IB_SPEED_SDR
;
655 props
->max_msg_sz
= 0x80000000;
663 static int nes_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
, u16
*pkey
)
673 static int nes_query_gid(struct ib_device
*ibdev
, u8 port
,
674 int index
, union ib_gid
*gid
)
676 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
678 memset(&(gid
->raw
[0]), 0, sizeof(gid
->raw
));
679 memcpy(&(gid
->raw
[0]), nesvnic
->netdev
->dev_addr
, 6);
686 * nes_alloc_ucontext - Allocate the user context data structure. This keeps track
687 * of all objects associated with a particular user-mode client.
689 static struct ib_ucontext
*nes_alloc_ucontext(struct ib_device
*ibdev
,
690 struct ib_udata
*udata
)
692 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
693 struct nes_device
*nesdev
= nesvnic
->nesdev
;
694 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
695 struct nes_alloc_ucontext_req req
;
696 struct nes_alloc_ucontext_resp uresp
;
697 struct nes_ucontext
*nes_ucontext
;
698 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
701 if (ib_copy_from_udata(&req
, udata
, sizeof(struct nes_alloc_ucontext_req
))) {
702 printk(KERN_ERR PFX
"Invalid structure size on allocate user context.\n");
703 return ERR_PTR(-EINVAL
);
706 if (req
.userspace_ver
!= NES_ABI_USERSPACE_VER
) {
707 printk(KERN_ERR PFX
"Invalid userspace driver version detected. Detected version %d, should be %d\n",
708 req
.userspace_ver
, NES_ABI_USERSPACE_VER
);
709 return ERR_PTR(-EINVAL
);
713 memset(&uresp
, 0, sizeof uresp
);
715 uresp
.max_qps
= nesibdev
->max_qp
;
716 uresp
.max_pds
= nesibdev
->max_pd
;
717 uresp
.wq_size
= nesdev
->nesadapter
->max_qp_wr
* 2;
718 uresp
.virtwq
= nesadapter
->virtwq
;
719 uresp
.kernel_ver
= NES_ABI_KERNEL_VER
;
721 nes_ucontext
= kzalloc(sizeof *nes_ucontext
, GFP_KERNEL
);
723 return ERR_PTR(-ENOMEM
);
725 nes_ucontext
->nesdev
= nesdev
;
726 nes_ucontext
->mmap_wq_offset
= uresp
.max_pds
;
727 nes_ucontext
->mmap_cq_offset
= nes_ucontext
->mmap_wq_offset
+
728 ((sizeof(struct nes_hw_qp_wqe
) * uresp
.max_qps
* 2) + PAGE_SIZE
-1) /
732 if (ib_copy_to_udata(udata
, &uresp
, sizeof uresp
)) {
734 return ERR_PTR(-EFAULT
);
737 INIT_LIST_HEAD(&nes_ucontext
->cq_reg_mem_list
);
738 INIT_LIST_HEAD(&nes_ucontext
->qp_reg_mem_list
);
739 atomic_set(&nes_ucontext
->usecnt
, 1);
740 return &nes_ucontext
->ibucontext
;
745 * nes_dealloc_ucontext
747 static int nes_dealloc_ucontext(struct ib_ucontext
*context
)
749 /* struct nes_vnic *nesvnic = to_nesvnic(context->device); */
750 /* struct nes_device *nesdev = nesvnic->nesdev; */
751 struct nes_ucontext
*nes_ucontext
= to_nesucontext(context
);
753 if (!atomic_dec_and_test(&nes_ucontext
->usecnt
))
763 static int nes_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
766 struct nes_vnic
*nesvnic
= to_nesvnic(context
->device
);
767 struct nes_device
*nesdev
= nesvnic
->nesdev
;
768 /* struct nes_adapter *nesadapter = nesdev->nesadapter; */
769 struct nes_ucontext
*nes_ucontext
;
770 struct nes_qp
*nesqp
;
772 nes_ucontext
= to_nesucontext(context
);
775 if (vma
->vm_pgoff
>= nes_ucontext
->mmap_wq_offset
) {
776 index
= (vma
->vm_pgoff
- nes_ucontext
->mmap_wq_offset
) * PAGE_SIZE
;
777 index
/= ((sizeof(struct nes_hw_qp_wqe
) * nesdev
->nesadapter
->max_qp_wr
* 2) +
778 PAGE_SIZE
-1) & (~(PAGE_SIZE
-1));
779 if (!test_bit(index
, nes_ucontext
->allocated_wqs
)) {
780 nes_debug(NES_DBG_MMAP
, "wq %lu not allocated\n", index
);
783 nesqp
= nes_ucontext
->mmap_nesqp
[index
];
785 nes_debug(NES_DBG_MMAP
, "wq %lu has a NULL QP base.\n", index
);
788 if (remap_pfn_range(vma
, vma
->vm_start
,
789 virt_to_phys(nesqp
->hwqp
.sq_vbase
) >> PAGE_SHIFT
,
790 vma
->vm_end
- vma
->vm_start
,
791 vma
->vm_page_prot
)) {
792 nes_debug(NES_DBG_MMAP
, "remap_pfn_range failed.\n");
795 vma
->vm_private_data
= nesqp
;
798 index
= vma
->vm_pgoff
;
799 if (!test_bit(index
, nes_ucontext
->allocated_doorbells
))
802 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
803 if (io_remap_pfn_range(vma
, vma
->vm_start
,
804 (nesdev
->doorbell_start
+
805 ((nes_ucontext
->mmap_db_index
[index
] - nesdev
->base_doorbell_index
) * 4096))
806 >> PAGE_SHIFT
, PAGE_SIZE
, vma
->vm_page_prot
))
808 vma
->vm_private_data
= nes_ucontext
;
819 static struct ib_pd
*nes_alloc_pd(struct ib_device
*ibdev
,
820 struct ib_ucontext
*context
, struct ib_udata
*udata
)
822 struct nes_pd
*nespd
;
823 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
824 struct nes_device
*nesdev
= nesvnic
->nesdev
;
825 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
826 struct nes_ucontext
*nesucontext
;
827 struct nes_alloc_pd_resp uresp
;
831 nes_debug(NES_DBG_PD
, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
832 nesvnic
, nesdev
->netdev
[0], nesdev
->netdev
[0]->name
, ibdev
, context
,
833 netdev_refcnt_read(nesvnic
->netdev
));
835 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_pds
,
836 nesadapter
->max_pd
, &pd_num
, &nesadapter
->next_pd
, NES_RESOURCE_PD
);
841 nespd
= kzalloc(sizeof (struct nes_pd
), GFP_KERNEL
);
843 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
844 return ERR_PTR(-ENOMEM
);
847 nes_debug(NES_DBG_PD
, "Allocating PD (%p) for ib device %s\n",
848 nespd
, nesvnic
->nesibdev
->ibdev
.name
);
850 nespd
->pd_id
= (pd_num
<< (PAGE_SHIFT
-12)) + nesadapter
->base_pd
;
853 nesucontext
= to_nesucontext(context
);
854 nespd
->mmap_db_index
= find_next_zero_bit(nesucontext
->allocated_doorbells
,
855 NES_MAX_USER_DB_REGIONS
, nesucontext
->first_free_db
);
856 nes_debug(NES_DBG_PD
, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
857 nespd
->mmap_db_index
, nespd
->pd_id
);
858 if (nespd
->mmap_db_index
>= NES_MAX_USER_DB_REGIONS
) {
859 nes_debug(NES_DBG_PD
, "mmap_db_index > MAX\n");
860 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
862 return ERR_PTR(-ENOMEM
);
865 uresp
.pd_id
= nespd
->pd_id
;
866 uresp
.mmap_db_index
= nespd
->mmap_db_index
;
867 if (ib_copy_to_udata(udata
, &uresp
, sizeof (struct nes_alloc_pd_resp
))) {
868 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
870 return ERR_PTR(-EFAULT
);
873 set_bit(nespd
->mmap_db_index
, nesucontext
->allocated_doorbells
);
874 nesucontext
->mmap_db_index
[nespd
->mmap_db_index
] = nespd
->pd_id
;
875 nesucontext
->first_free_db
= nespd
->mmap_db_index
+ 1;
878 nes_debug(NES_DBG_PD
, "PD%u structure located @%p.\n", nespd
->pd_id
, nespd
);
886 static int nes_dealloc_pd(struct ib_pd
*ibpd
)
888 struct nes_ucontext
*nesucontext
;
889 struct nes_pd
*nespd
= to_nespd(ibpd
);
890 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
891 struct nes_device
*nesdev
= nesvnic
->nesdev
;
892 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
894 if ((ibpd
->uobject
) && (ibpd
->uobject
->context
)) {
895 nesucontext
= to_nesucontext(ibpd
->uobject
->context
);
896 nes_debug(NES_DBG_PD
, "Clearing bit %u from allocated doorbells\n",
897 nespd
->mmap_db_index
);
898 clear_bit(nespd
->mmap_db_index
, nesucontext
->allocated_doorbells
);
899 nesucontext
->mmap_db_index
[nespd
->mmap_db_index
] = 0;
900 if (nesucontext
->first_free_db
> nespd
->mmap_db_index
) {
901 nesucontext
->first_free_db
= nespd
->mmap_db_index
;
905 nes_debug(NES_DBG_PD
, "Deallocating PD%u structure located @%p.\n",
906 nespd
->pd_id
, nespd
);
907 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
,
908 (nespd
->pd_id
-nesadapter
->base_pd
)>>(PAGE_SHIFT
-12));
918 static struct ib_ah
*nes_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
)
920 return ERR_PTR(-ENOSYS
);
927 static int nes_destroy_ah(struct ib_ah
*ah
)
934 * nes_get_encoded_size
936 static inline u8
nes_get_encoded_size(int *size
)
942 } else if (*size
<= 128) {
945 } else if (*size
<= 512) {
949 return (encoded_size
);
957 static int nes_setup_virt_qp(struct nes_qp
*nesqp
, struct nes_pbl
*nespbl
,
958 struct nes_vnic
*nesvnic
, int sq_size
, int rq_size
)
965 struct nes_device
*nesdev
= nesvnic
->nesdev
;
966 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
971 pbl_entries
= nespbl
->pbl_size
>> 3;
972 nes_debug(NES_DBG_QP
, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
973 nespbl
->pbl_size
, pbl_entries
,
974 (void *)nespbl
->pbl_vbase
,
975 (unsigned long) nespbl
->pbl_pbase
);
976 pbl
= (__le64
*) nespbl
->pbl_vbase
; /* points to first pbl entry */
977 /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
978 /* the first pbl to be fro the rq_vbase... */
979 rq_pbl_entries
= (rq_size
* sizeof(struct nes_hw_qp_wqe
)) >> 12;
980 sq_pbl_entries
= (sq_size
* sizeof(struct nes_hw_qp_wqe
)) >> 12;
981 nesqp
->hwqp
.sq_pbase
= (le32_to_cpu(((__le32
*)pbl
)[0])) | ((u64
)((le32_to_cpu(((__le32
*)pbl
)[1]))) << 32);
983 nes_debug(NES_DBG_QP
, "QP nespbl->page is NULL \n");
988 nesqp
->hwqp
.sq_vbase
= kmap(nespbl
->page
);
989 nesqp
->page
= nespbl
->page
;
990 if (!nesqp
->hwqp
.sq_vbase
) {
991 nes_debug(NES_DBG_QP
, "QP sq_vbase kmap failed\n");
996 /* Now to get to sq.. we need to calculate how many */
997 /* PBL entries were used by the rq.. */
998 pbl
+= sq_pbl_entries
;
999 nesqp
->hwqp
.rq_pbase
= (le32_to_cpu(((__le32
*)pbl
)[0])) | ((u64
)((le32_to_cpu(((__le32
*)pbl
)[1]))) << 32);
1000 /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
1001 /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
1003 nes_debug(NES_DBG_QP
, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
1004 nesqp
->hwqp
.sq_vbase
, (unsigned long) nesqp
->hwqp
.sq_pbase
,
1005 nesqp
->hwqp
.rq_vbase
, (unsigned long) nesqp
->hwqp
.rq_pbase
);
1006 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1007 if (!nesadapter
->free_256pbl
) {
1008 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1010 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1011 kunmap(nesqp
->page
);
1015 nesadapter
->free_256pbl
--;
1016 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1018 nesqp
->pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 256, &nesqp
->pbl_pbase
);
1019 pblbuffer
= nesqp
->pbl_vbase
;
1020 if (!nesqp
->pbl_vbase
) {
1021 /* memory allocated during nes_reg_user_mr() */
1022 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1025 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1026 nesadapter
->free_256pbl
++;
1027 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1028 kunmap(nesqp
->page
);
1031 memset(nesqp
->pbl_vbase
, 0, 256);
1032 /* fill in the page address in the pbl buffer.. */
1033 tpbl
= pblbuffer
+ 16;
1034 pbl
= (__le64
*)nespbl
->pbl_vbase
;
1035 while (sq_pbl_entries
--)
1038 while (rq_pbl_entries
--)
1041 /* done with memory allocated during nes_reg_user_mr() */
1042 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1046 nesqp
->qp_mem_size
=
1047 max((u32
)sizeof(struct nes_qp_context
), ((u32
)256)) + 256; /* this is Q2 */
1048 /* Round up to a multiple of a page */
1049 nesqp
->qp_mem_size
+= PAGE_SIZE
- 1;
1050 nesqp
->qp_mem_size
&= ~(PAGE_SIZE
- 1);
1052 mem
= pci_alloc_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
1053 &nesqp
->hwqp
.q2_pbase
);
1056 pci_free_consistent(nesdev
->pcidev
, 256, nesqp
->pbl_vbase
, nesqp
->pbl_pbase
);
1057 nesqp
->pbl_vbase
= NULL
;
1058 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1059 nesadapter
->free_256pbl
++;
1060 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1061 kunmap(nesqp
->page
);
1064 nesqp
->sq_kmapped
= 1;
1065 nesqp
->hwqp
.q2_vbase
= mem
;
1067 memset(nesqp
->hwqp
.q2_vbase
, 0, 256);
1068 nesqp
->nesqp_context
= mem
;
1069 memset(nesqp
->nesqp_context
, 0, sizeof(*nesqp
->nesqp_context
));
1070 nesqp
->nesqp_context_pbase
= nesqp
->hwqp
.q2_pbase
+ 256;
1079 static int nes_setup_mmap_qp(struct nes_qp
*nesqp
, struct nes_vnic
*nesvnic
,
1080 int sq_size
, int rq_size
)
1083 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1085 nesqp
->qp_mem_size
= (sizeof(struct nes_hw_qp_wqe
) * sq_size
) +
1086 (sizeof(struct nes_hw_qp_wqe
) * rq_size
) +
1087 max((u32
)sizeof(struct nes_qp_context
), ((u32
)256)) +
1088 256; /* this is Q2 */
1089 /* Round up to a multiple of a page */
1090 nesqp
->qp_mem_size
+= PAGE_SIZE
- 1;
1091 nesqp
->qp_mem_size
&= ~(PAGE_SIZE
- 1);
1093 mem
= pci_alloc_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
1094 &nesqp
->hwqp
.sq_pbase
);
1097 nes_debug(NES_DBG_QP
, "PCI consistent memory for "
1098 "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n",
1099 mem
, (unsigned long)nesqp
->hwqp
.sq_pbase
, nesqp
->qp_mem_size
);
1101 memset(mem
, 0, nesqp
->qp_mem_size
);
1103 nesqp
->hwqp
.sq_vbase
= mem
;
1104 mem
+= sizeof(struct nes_hw_qp_wqe
) * sq_size
;
1106 nesqp
->hwqp
.rq_vbase
= mem
;
1107 nesqp
->hwqp
.rq_pbase
= nesqp
->hwqp
.sq_pbase
+
1108 sizeof(struct nes_hw_qp_wqe
) * sq_size
;
1109 mem
+= sizeof(struct nes_hw_qp_wqe
) * rq_size
;
1111 nesqp
->hwqp
.q2_vbase
= mem
;
1112 nesqp
->hwqp
.q2_pbase
= nesqp
->hwqp
.rq_pbase
+
1113 sizeof(struct nes_hw_qp_wqe
) * rq_size
;
1115 memset(nesqp
->hwqp
.q2_vbase
, 0, 256);
1117 nesqp
->nesqp_context
= mem
;
1118 nesqp
->nesqp_context_pbase
= nesqp
->hwqp
.q2_pbase
+ 256;
1119 memset(nesqp
->nesqp_context
, 0, sizeof(*nesqp
->nesqp_context
));
1125 * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory.
1127 static inline void nes_free_qp_mem(struct nes_device
*nesdev
,
1128 struct nes_qp
*nesqp
, int virt_wqs
)
1130 unsigned long flags
;
1131 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1133 pci_free_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
1134 nesqp
->hwqp
.sq_vbase
, nesqp
->hwqp
.sq_pbase
);
1136 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1137 nesadapter
->free_256pbl
++;
1138 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1139 pci_free_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
, nesqp
->hwqp
.q2_vbase
, nesqp
->hwqp
.q2_pbase
);
1140 pci_free_consistent(nesdev
->pcidev
, 256, nesqp
->pbl_vbase
, nesqp
->pbl_pbase
);
1141 nesqp
->pbl_vbase
= NULL
;
1142 if (nesqp
->sq_kmapped
) {
1143 nesqp
->sq_kmapped
= 0;
1144 kunmap(nesqp
->page
);
1153 static struct ib_qp
*nes_create_qp(struct ib_pd
*ibpd
,
1154 struct ib_qp_init_attr
*init_attr
, struct ib_udata
*udata
)
1158 struct nes_pd
*nespd
= to_nespd(ibpd
);
1159 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
1160 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1161 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1162 struct nes_qp
*nesqp
;
1163 struct nes_cq
*nescq
;
1164 struct nes_ucontext
*nes_ucontext
;
1165 struct nes_hw_cqp_wqe
*cqp_wqe
;
1166 struct nes_cqp_request
*cqp_request
;
1167 struct nes_create_qp_req req
;
1168 struct nes_create_qp_resp uresp
;
1169 struct nes_pbl
*nespbl
= NULL
;
1172 /* u32 counter = 0; */
1174 unsigned long flags
;
1184 if (init_attr
->create_flags
)
1185 return ERR_PTR(-EINVAL
);
1187 atomic_inc(&qps_created
);
1188 switch (init_attr
->qp_type
) {
1190 if (nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) {
1191 init_attr
->cap
.max_inline_data
= 0;
1193 init_attr
->cap
.max_inline_data
= 64;
1195 sq_size
= init_attr
->cap
.max_send_wr
;
1196 rq_size
= init_attr
->cap
.max_recv_wr
;
1198 /* check if the encoded sizes are OK or not... */
1199 sq_encoded_size
= nes_get_encoded_size(&sq_size
);
1200 rq_encoded_size
= nes_get_encoded_size(&rq_size
);
1202 if ((!sq_encoded_size
) || (!rq_encoded_size
)) {
1203 nes_debug(NES_DBG_QP
, "ERROR bad rq (%u) or sq (%u) size\n",
1205 return ERR_PTR(-EINVAL
);
1208 init_attr
->cap
.max_send_wr
= sq_size
-2;
1209 init_attr
->cap
.max_recv_wr
= rq_size
-1;
1210 nes_debug(NES_DBG_QP
, "RQ size=%u, SQ Size=%u\n", rq_size
, sq_size
);
1212 ret
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_qps
,
1213 nesadapter
->max_qp
, &qp_num
, &nesadapter
->next_qp
, NES_RESOURCE_QP
);
1215 return ERR_PTR(ret
);
1218 /* Need 512 (actually now 1024) byte alignment on this structure */
1219 mem
= kzalloc(sizeof(*nesqp
)+NES_SW_CONTEXT_ALIGN
-1, GFP_KERNEL
);
1221 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1222 nes_debug(NES_DBG_QP
, "Unable to allocate QP\n");
1223 return ERR_PTR(-ENOMEM
);
1225 u64nesqp
= (unsigned long)mem
;
1226 u64nesqp
+= ((u64
)NES_SW_CONTEXT_ALIGN
) - 1;
1227 u64temp
= ((u64
)NES_SW_CONTEXT_ALIGN
) - 1;
1228 u64nesqp
&= ~u64temp
;
1229 nesqp
= (struct nes_qp
*)(unsigned long)u64nesqp
;
1230 /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n",
1231 nesqp, mem, NES_SW_CONTEXT_ALIGN); */
1232 nesqp
->allocated_buffer
= mem
;
1235 if (ib_copy_from_udata(&req
, udata
, sizeof(struct nes_create_qp_req
))) {
1236 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1237 kfree(nesqp
->allocated_buffer
);
1238 nes_debug(NES_DBG_QP
, "ib_copy_from_udata() Failed \n");
1239 return ERR_PTR(-EFAULT
);
1241 if (req
.user_wqe_buffers
) {
1244 if (req
.user_qp_buffer
)
1245 nesqp
->nesuqp_addr
= req
.user_qp_buffer
;
1246 if ((ibpd
->uobject
) && (ibpd
->uobject
->context
)) {
1247 nesqp
->user_mode
= 1;
1248 nes_ucontext
= to_nesucontext(ibpd
->uobject
->context
);
1251 list_for_each_entry(nespbl
, &nes_ucontext
->qp_reg_mem_list
, list
) {
1252 if (nespbl
->user_base
== (unsigned long )req
.user_wqe_buffers
) {
1253 list_del(&nespbl
->list
);
1255 nes_debug(NES_DBG_QP
, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
1256 nespbl
, nespbl
->user_base
);
1261 nes_debug(NES_DBG_QP
, "Didn't Find PBL for virtual QP. address = %llx.\n",
1262 (long long unsigned int)req
.user_wqe_buffers
);
1263 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1264 kfree(nesqp
->allocated_buffer
);
1265 return ERR_PTR(-EFAULT
);
1269 nes_ucontext
= to_nesucontext(ibpd
->uobject
->context
);
1270 nesqp
->mmap_sq_db_index
=
1271 find_next_zero_bit(nes_ucontext
->allocated_wqs
,
1272 NES_MAX_USER_WQ_REGIONS
, nes_ucontext
->first_free_wq
);
1273 /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
1274 nespd->mmap_db_index); */
1275 if (nesqp
->mmap_sq_db_index
>= NES_MAX_USER_WQ_REGIONS
) {
1276 nes_debug(NES_DBG_QP
,
1277 "db index > max user regions, failing create QP\n");
1278 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1280 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1284 kfree(nesqp
->allocated_buffer
);
1285 return ERR_PTR(-ENOMEM
);
1287 set_bit(nesqp
->mmap_sq_db_index
, nes_ucontext
->allocated_wqs
);
1288 nes_ucontext
->mmap_nesqp
[nesqp
->mmap_sq_db_index
] = nesqp
;
1289 nes_ucontext
->first_free_wq
= nesqp
->mmap_sq_db_index
+ 1;
1291 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1292 kfree(nesqp
->allocated_buffer
);
1293 return ERR_PTR(-EFAULT
);
1296 err
= (!virt_wqs
) ? nes_setup_mmap_qp(nesqp
, nesvnic
, sq_size
, rq_size
) :
1297 nes_setup_virt_qp(nesqp
, nespbl
, nesvnic
, sq_size
, rq_size
);
1299 nes_debug(NES_DBG_QP
,
1300 "error geting qp mem code = %d\n", err
);
1301 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1302 kfree(nesqp
->allocated_buffer
);
1303 return ERR_PTR(-ENOMEM
);
1306 nesqp
->hwqp
.sq_size
= sq_size
;
1307 nesqp
->hwqp
.sq_encoded_size
= sq_encoded_size
;
1308 nesqp
->hwqp
.sq_head
= 1;
1309 nesqp
->hwqp
.rq_size
= rq_size
;
1310 nesqp
->hwqp
.rq_encoded_size
= rq_encoded_size
;
1311 /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n",
1312 (void *)nesqp->nesqp_context_pbase);
1314 nesqp
->hwqp
.qp_id
= qp_num
;
1315 nesqp
->ibqp
.qp_num
= nesqp
->hwqp
.qp_id
;
1316 nesqp
->nespd
= nespd
;
1318 nescq
= to_nescq(init_attr
->send_cq
);
1319 nesqp
->nesscq
= nescq
;
1320 nescq
= to_nescq(init_attr
->recv_cq
);
1321 nesqp
->nesrcq
= nescq
;
1323 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)PCI_FUNC(nesdev
->pcidev
->devfn
) <<
1324 NES_QPCONTEXT_MISC_PCI_FCN_SHIFT
);
1325 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)nesqp
->hwqp
.rq_encoded_size
<<
1326 NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT
);
1327 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)nesqp
->hwqp
.sq_encoded_size
<<
1328 NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT
);
1330 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN
);
1331 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN
);
1333 nesqp
->nesqp_context
->cqs
= cpu_to_le32(nesqp
->nesscq
->hw_cq
.cq_number
+
1334 ((u32
)nesqp
->nesrcq
->hw_cq
.cq_number
<< 16));
1335 u64temp
= (u64
)nesqp
->hwqp
.sq_pbase
;
1336 nesqp
->nesqp_context
->sq_addr_low
= cpu_to_le32((u32
)u64temp
);
1337 nesqp
->nesqp_context
->sq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1341 u64temp
= (u64
)nesqp
->hwqp
.sq_pbase
;
1342 nesqp
->nesqp_context
->sq_addr_low
= cpu_to_le32((u32
)u64temp
);
1343 nesqp
->nesqp_context
->sq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1344 u64temp
= (u64
)nesqp
->hwqp
.rq_pbase
;
1345 nesqp
->nesqp_context
->rq_addr_low
= cpu_to_le32((u32
)u64temp
);
1346 nesqp
->nesqp_context
->rq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1348 u64temp
= (u64
)nesqp
->pbl_pbase
;
1349 nesqp
->nesqp_context
->rq_addr_low
= cpu_to_le32((u32
)u64temp
);
1350 nesqp
->nesqp_context
->rq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1353 /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n",
1354 nesvnic->next_qp_nic_index,
1355 nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */
1356 spin_lock_irqsave(&nesdev
->cqp
.lock
, flags
);
1357 nesqp
->nesqp_context
->misc2
|= cpu_to_le32(
1358 (u32
)nesvnic
->qp_nic_index
[nesvnic
->next_qp_nic_index
] <<
1359 NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT
);
1360 nesvnic
->next_qp_nic_index
++;
1361 if ((nesvnic
->next_qp_nic_index
> 3) ||
1362 (nesvnic
->qp_nic_index
[nesvnic
->next_qp_nic_index
] == 0xf)) {
1363 nesvnic
->next_qp_nic_index
= 0;
1365 spin_unlock_irqrestore(&nesdev
->cqp
.lock
, flags
);
1367 nesqp
->nesqp_context
->pd_index_wscale
|= cpu_to_le32((u32
)nesqp
->nespd
->pd_id
<< 16);
1368 u64temp
= (u64
)nesqp
->hwqp
.q2_pbase
;
1369 nesqp
->nesqp_context
->q2_addr_low
= cpu_to_le32((u32
)u64temp
);
1370 nesqp
->nesqp_context
->q2_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1371 nesqp
->nesqp_context
->aeq_token_low
= cpu_to_le32((u32
)((unsigned long)(nesqp
)));
1372 nesqp
->nesqp_context
->aeq_token_high
= cpu_to_le32((u32
)(upper_32_bits((unsigned long)(nesqp
))));
1373 nesqp
->nesqp_context
->ird_ord_sizes
= cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM
|
1374 NES_QPCONTEXT_ORDIRD_AAH
|
1375 ((((u32
)nesadapter
->max_irrq_wr
) <<
1376 NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT
) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK
));
1377 if (disable_mpa_crc
) {
1378 nes_debug(NES_DBG_QP
, "Disabling MPA crc checking due to module option.\n");
1379 nesqp
->nesqp_context
->ird_ord_sizes
|= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC
);
1384 cqp_request
= nes_get_cqp_request(nesdev
);
1385 if (cqp_request
== NULL
) {
1386 nes_debug(NES_DBG_QP
, "Failed to get a cqp_request\n");
1387 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1388 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1389 kfree(nesqp
->allocated_buffer
);
1390 return ERR_PTR(-ENOMEM
);
1392 cqp_request
->waiting
= 1;
1393 cqp_wqe
= &cqp_request
->cqp_wqe
;
1396 opcode
= NES_CQP_CREATE_QP
| NES_CQP_QP_TYPE_IWARP
|
1397 NES_CQP_QP_IWARP_STATE_IDLE
;
1399 opcode
= NES_CQP_CREATE_QP
| NES_CQP_QP_TYPE_IWARP
| NES_CQP_QP_VIRT_WQS
|
1400 NES_CQP_QP_IWARP_STATE_IDLE
;
1402 opcode
|= NES_CQP_QP_CQS_VALID
;
1403 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1404 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1405 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
, nesqp
->hwqp
.qp_id
);
1407 u64temp
= (u64
)nesqp
->nesqp_context_pbase
;
1408 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_CONTEXT_LOW_IDX
, u64temp
);
1410 atomic_set(&cqp_request
->refcount
, 2);
1411 nes_post_cqp_request(nesdev
, cqp_request
);
1414 nes_debug(NES_DBG_QP
, "Waiting for create iWARP QP%u to complete.\n",
1416 ret
= wait_event_timeout(cqp_request
->waitq
,
1417 (cqp_request
->request_done
!= 0), NES_EVENT_TIMEOUT
);
1418 nes_debug(NES_DBG_QP
, "Create iwarp QP%u completed, wait_event_timeout ret=%u,"
1419 " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u,"
1420 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1421 nesqp
->hwqp
.qp_id
, ret
, nesdev
->cqp
.sq_head
, nesdev
->cqp
.sq_tail
,
1422 cqp_request
->major_code
, cqp_request
->minor_code
);
1423 if ((!ret
) || (cqp_request
->major_code
)) {
1424 nes_put_cqp_request(nesdev
, cqp_request
);
1425 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1426 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1427 kfree(nesqp
->allocated_buffer
);
1429 return ERR_PTR(-ETIME
);
1431 return ERR_PTR(-EIO
);
1435 nes_put_cqp_request(nesdev
, cqp_request
);
1437 if (ibpd
->uobject
) {
1438 uresp
.mmap_sq_db_index
= nesqp
->mmap_sq_db_index
;
1439 uresp
.mmap_rq_db_index
= 0;
1440 uresp
.actual_sq_size
= sq_size
;
1441 uresp
.actual_rq_size
= rq_size
;
1442 uresp
.qp_id
= nesqp
->hwqp
.qp_id
;
1443 uresp
.nes_drv_opt
= nes_drv_opt
;
1444 if (ib_copy_to_udata(udata
, &uresp
, sizeof uresp
)) {
1445 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1446 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1447 kfree(nesqp
->allocated_buffer
);
1448 return ERR_PTR(-EFAULT
);
1452 nes_debug(NES_DBG_QP
, "QP%u structure located @%p.Size = %u.\n",
1453 nesqp
->hwqp
.qp_id
, nesqp
, (u32
)sizeof(*nesqp
));
1454 spin_lock_init(&nesqp
->lock
);
1455 nes_add_ref(&nesqp
->ibqp
);
1458 nes_debug(NES_DBG_QP
, "Invalid QP type: %d\n", init_attr
->qp_type
);
1459 return ERR_PTR(-EINVAL
);
1462 nesqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
);
1463 init_timer(&nesqp
->terminate_timer
);
1464 nesqp
->terminate_timer
.function
= nes_terminate_timeout
;
1465 nesqp
->terminate_timer
.data
= (unsigned long)nesqp
;
1467 /* update the QP table */
1468 nesdev
->nesadapter
->qp_table
[nesqp
->hwqp
.qp_id
-NES_FIRST_QPN
] = nesqp
;
1469 nes_debug(NES_DBG_QP
, "netdev refcnt=%u\n",
1470 netdev_refcnt_read(nesvnic
->netdev
));
1472 return &nesqp
->ibqp
;
1478 static void nes_clean_cq(struct nes_qp
*nesqp
, struct nes_cq
*nescq
)
1484 unsigned long flags
= 0;
1486 spin_lock_irqsave(&nescq
->lock
, flags
);
1488 cq_head
= nescq
->hw_cq
.cq_head
;
1489 while (le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_OPCODE_IDX
]) & NES_CQE_VALID
) {
1491 lo
= le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
1492 hi
= le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
]);
1493 u64temp
= (((u64
)hi
) << 32) | ((u64
)lo
);
1494 u64temp
&= ~(NES_SW_CONTEXT_ALIGN
-1);
1495 if (u64temp
== (u64
)(unsigned long)nesqp
) {
1496 /* Zero the context value so cqe will be ignored */
1497 nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
] = 0;
1498 nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
] = 0;
1501 if (++cq_head
>= nescq
->hw_cq
.cq_size
)
1505 spin_unlock_irqrestore(&nescq
->lock
, flags
);
1512 static int nes_destroy_qp(struct ib_qp
*ibqp
)
1514 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
1515 struct nes_ucontext
*nes_ucontext
;
1516 struct ib_qp_attr attr
;
1517 struct iw_cm_id
*cm_id
;
1518 struct iw_cm_event cm_event
;
1521 atomic_inc(&sw_qps_destroyed
);
1522 nesqp
->destroyed
= 1;
1524 /* Blow away the connection if it exists. */
1525 if (nesqp
->ibqp_state
>= IB_QPS_INIT
&& nesqp
->ibqp_state
<= IB_QPS_RTS
) {
1526 /* if (nesqp->ibqp_state == IB_QPS_RTS) { */
1527 attr
.qp_state
= IB_QPS_ERR
;
1528 nes_modify_qp(&nesqp
->ibqp
, &attr
, IB_QP_STATE
, NULL
);
1531 if (((nesqp
->ibqp_state
== IB_QPS_INIT
) ||
1532 (nesqp
->ibqp_state
== IB_QPS_RTR
)) && (nesqp
->cm_id
)) {
1533 cm_id
= nesqp
->cm_id
;
1534 cm_event
.event
= IW_CM_EVENT_CONNECT_REPLY
;
1535 cm_event
.status
= -ETIMEDOUT
;
1536 cm_event
.local_addr
= cm_id
->local_addr
;
1537 cm_event
.remote_addr
= cm_id
->remote_addr
;
1538 cm_event
.private_data
= NULL
;
1539 cm_event
.private_data_len
= 0;
1541 nes_debug(NES_DBG_QP
, "Generating a CM Timeout Event for "
1542 "QP%u. cm_id = %p, refcount = %u. \n",
1543 nesqp
->hwqp
.qp_id
, cm_id
, atomic_read(&nesqp
->refcount
));
1545 cm_id
->rem_ref(cm_id
);
1546 ret
= cm_id
->event_handler(cm_id
, &cm_event
);
1548 nes_debug(NES_DBG_QP
, "OFA CM event_handler returned, ret=%d\n", ret
);
1551 if (nesqp
->user_mode
) {
1552 if ((ibqp
->uobject
)&&(ibqp
->uobject
->context
)) {
1553 nes_ucontext
= to_nesucontext(ibqp
->uobject
->context
);
1554 clear_bit(nesqp
->mmap_sq_db_index
, nes_ucontext
->allocated_wqs
);
1555 nes_ucontext
->mmap_nesqp
[nesqp
->mmap_sq_db_index
] = NULL
;
1556 if (nes_ucontext
->first_free_wq
> nesqp
->mmap_sq_db_index
) {
1557 nes_ucontext
->first_free_wq
= nesqp
->mmap_sq_db_index
;
1560 if (nesqp
->pbl_pbase
&& nesqp
->sq_kmapped
) {
1561 nesqp
->sq_kmapped
= 0;
1562 kunmap(nesqp
->page
);
1565 /* Clean any pending completions from the cq(s) */
1567 nes_clean_cq(nesqp
, nesqp
->nesscq
);
1569 if ((nesqp
->nesrcq
) && (nesqp
->nesrcq
!= nesqp
->nesscq
))
1570 nes_clean_cq(nesqp
, nesqp
->nesrcq
);
1572 nes_rem_ref(&nesqp
->ibqp
);
1580 static struct ib_cq
*nes_create_cq(struct ib_device
*ibdev
,
1581 const struct ib_cq_init_attr
*attr
,
1582 struct ib_ucontext
*context
,
1583 struct ib_udata
*udata
)
1585 int entries
= attr
->cqe
;
1587 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
1588 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1589 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1590 struct nes_cq
*nescq
;
1591 struct nes_ucontext
*nes_ucontext
= NULL
;
1592 struct nes_cqp_request
*cqp_request
;
1594 struct nes_hw_cqp_wqe
*cqp_wqe
;
1595 struct nes_pbl
*nespbl
= NULL
;
1596 struct nes_create_cq_req req
;
1597 struct nes_create_cq_resp resp
;
1600 u32 pbl_entries
= 1;
1602 unsigned long flags
;
1606 return ERR_PTR(-EINVAL
);
1608 if (entries
> nesadapter
->max_cqe
)
1609 return ERR_PTR(-EINVAL
);
1611 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_cqs
,
1612 nesadapter
->max_cq
, &cq_num
, &nesadapter
->next_cq
, NES_RESOURCE_CQ
);
1614 return ERR_PTR(err
);
1617 nescq
= kzalloc(sizeof(struct nes_cq
), GFP_KERNEL
);
1619 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1620 nes_debug(NES_DBG_CQ
, "Unable to allocate nes_cq struct\n");
1621 return ERR_PTR(-ENOMEM
);
1624 nescq
->hw_cq
.cq_size
= max(entries
+ 1, 5);
1625 nescq
->hw_cq
.cq_number
= cq_num
;
1626 nescq
->ibcq
.cqe
= nescq
->hw_cq
.cq_size
- 1;
1630 nes_ucontext
= to_nesucontext(context
);
1631 if (ib_copy_from_udata(&req
, udata
, sizeof (struct nes_create_cq_req
))) {
1632 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1634 return ERR_PTR(-EFAULT
);
1636 nesvnic
->mcrq_ucontext
= nes_ucontext
;
1637 nes_ucontext
->mcrqf
= req
.mcrqf
;
1638 if (nes_ucontext
->mcrqf
) {
1639 if (nes_ucontext
->mcrqf
& 0x80000000)
1640 nescq
->hw_cq
.cq_number
= nesvnic
->nic
.qp_id
+ 28 + 2 * ((nes_ucontext
->mcrqf
& 0xf) - 1);
1641 else if (nes_ucontext
->mcrqf
& 0x40000000)
1642 nescq
->hw_cq
.cq_number
= nes_ucontext
->mcrqf
& 0xffff;
1644 nescq
->hw_cq
.cq_number
= nesvnic
->mcrq_qp_id
+ nes_ucontext
->mcrqf
-1;
1645 nescq
->mcrqf
= nes_ucontext
->mcrqf
;
1646 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1648 nes_debug(NES_DBG_CQ
, "CQ Virtual Address = %08lX, size = %u.\n",
1649 (unsigned long)req
.user_cq_buffer
, entries
);
1651 list_for_each_entry(nespbl
, &nes_ucontext
->cq_reg_mem_list
, list
) {
1652 if (nespbl
->user_base
== (unsigned long )req
.user_cq_buffer
) {
1653 list_del(&nespbl
->list
);
1655 nes_debug(NES_DBG_CQ
, "Found PBL for virtual CQ. nespbl=%p.\n",
1661 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1663 return ERR_PTR(-EFAULT
);
1666 pbl_entries
= nespbl
->pbl_size
>> 3;
1667 nescq
->cq_mem_size
= 0;
1669 nescq
->cq_mem_size
= nescq
->hw_cq
.cq_size
* sizeof(struct nes_hw_cqe
);
1670 nes_debug(NES_DBG_CQ
, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n",
1671 entries
, nescq
->cq_mem_size
, nescq
->hw_cq
.cq_number
);
1673 /* allocate the physical buffer space */
1674 mem
= pci_zalloc_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
,
1675 &nescq
->hw_cq
.cq_pbase
);
1677 printk(KERN_ERR PFX
"Unable to allocate pci memory for cq\n");
1678 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1680 return ERR_PTR(-ENOMEM
);
1683 nescq
->hw_cq
.cq_vbase
= mem
;
1684 nescq
->hw_cq
.cq_head
= 0;
1685 nes_debug(NES_DBG_CQ
, "CQ%u virtual address @ %p, phys = 0x%08X\n",
1686 nescq
->hw_cq
.cq_number
, nescq
->hw_cq
.cq_vbase
,
1687 (u32
)nescq
->hw_cq
.cq_pbase
);
1690 nescq
->hw_cq
.ce_handler
= nes_iwarp_ce_handler
;
1691 spin_lock_init(&nescq
->lock
);
1693 /* send CreateCQ request to CQP */
1694 cqp_request
= nes_get_cqp_request(nesdev
);
1695 if (cqp_request
== NULL
) {
1696 nes_debug(NES_DBG_CQ
, "Failed to get a cqp_request.\n");
1698 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1699 nescq
->hw_cq
.cq_pbase
);
1701 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1702 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1706 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1708 return ERR_PTR(-ENOMEM
);
1710 cqp_request
->waiting
= 1;
1711 cqp_wqe
= &cqp_request
->cqp_wqe
;
1713 opcode
= NES_CQP_CREATE_CQ
| NES_CQP_CQ_CEQ_VALID
|
1714 NES_CQP_CQ_CHK_OVERFLOW
|
1715 NES_CQP_CQ_CEQE_MASK
| ((u32
)nescq
->hw_cq
.cq_size
<< 16);
1717 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1719 if (pbl_entries
!= 1) {
1720 if (pbl_entries
> 32) {
1722 nes_debug(NES_DBG_CQ
, "pbl_entries=%u, use a 4k PBL\n", pbl_entries
);
1723 if (nesadapter
->free_4kpbl
== 0) {
1724 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1725 nes_free_cqp_request(nesdev
, cqp_request
);
1727 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1728 nescq
->hw_cq
.cq_pbase
);
1730 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1731 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1734 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1736 return ERR_PTR(-ENOMEM
);
1738 opcode
|= (NES_CQP_CQ_VIRT
| NES_CQP_CQ_4KB_CHUNK
);
1739 nescq
->virtual_cq
= 2;
1740 nesadapter
->free_4kpbl
--;
1743 /* use 256 byte pbl */
1744 nes_debug(NES_DBG_CQ
, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries
);
1745 if (nesadapter
->free_256pbl
== 0) {
1746 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1747 nes_free_cqp_request(nesdev
, cqp_request
);
1749 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1750 nescq
->hw_cq
.cq_pbase
);
1752 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1753 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1756 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1758 return ERR_PTR(-ENOMEM
);
1760 opcode
|= NES_CQP_CQ_VIRT
;
1761 nescq
->virtual_cq
= 1;
1762 nesadapter
->free_256pbl
--;
1767 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1769 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1770 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1771 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
,
1772 (nescq
->hw_cq
.cq_number
| ((u32
)nesdev
->ceq_index
<< 16)));
1775 if (pbl_entries
!= 1)
1776 u64temp
= (u64
)nespbl
->pbl_pbase
;
1778 u64temp
= le64_to_cpu(nespbl
->pbl_vbase
[0]);
1779 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX
,
1780 nes_ucontext
->mmap_db_index
[0]);
1782 u64temp
= (u64
)nescq
->hw_cq
.cq_pbase
;
1783 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX
] = 0;
1785 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_CQ_WQE_PBL_LOW_IDX
, u64temp
);
1786 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX
] = 0;
1787 u64temp
= (u64
)(unsigned long)&nescq
->hw_cq
;
1788 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX
] =
1789 cpu_to_le32((u32
)(u64temp
>> 1));
1790 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX
] =
1791 cpu_to_le32(((u32
)((u64temp
) >> 33)) & 0x7FFFFFFF);
1793 atomic_set(&cqp_request
->refcount
, 2);
1794 nes_post_cqp_request(nesdev
, cqp_request
);
1797 nes_debug(NES_DBG_CQ
, "Waiting for create iWARP CQ%u to complete.\n",
1798 nescq
->hw_cq
.cq_number
);
1799 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1800 NES_EVENT_TIMEOUT
* 2);
1801 nes_debug(NES_DBG_CQ
, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n",
1802 nescq
->hw_cq
.cq_number
, ret
);
1803 if ((!ret
) || (cqp_request
->major_code
)) {
1804 nes_put_cqp_request(nesdev
, cqp_request
);
1806 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1807 nescq
->hw_cq
.cq_pbase
);
1809 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1810 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1813 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1815 return ERR_PTR(-EIO
);
1817 nes_put_cqp_request(nesdev
, cqp_request
);
1820 /* free the nespbl */
1821 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1824 resp
.cq_id
= nescq
->hw_cq
.cq_number
;
1825 resp
.cq_size
= nescq
->hw_cq
.cq_size
;
1826 resp
.mmap_db_index
= 0;
1827 if (ib_copy_to_udata(udata
, &resp
, sizeof resp
- sizeof resp
.reserved
)) {
1828 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1830 return ERR_PTR(-EFAULT
);
1834 return &nescq
->ibcq
;
1841 static int nes_destroy_cq(struct ib_cq
*ib_cq
)
1843 struct nes_cq
*nescq
;
1844 struct nes_device
*nesdev
;
1845 struct nes_vnic
*nesvnic
;
1846 struct nes_adapter
*nesadapter
;
1847 struct nes_hw_cqp_wqe
*cqp_wqe
;
1848 struct nes_cqp_request
*cqp_request
;
1849 unsigned long flags
;
1856 nescq
= to_nescq(ib_cq
);
1857 nesvnic
= to_nesvnic(ib_cq
->device
);
1858 nesdev
= nesvnic
->nesdev
;
1859 nesadapter
= nesdev
->nesadapter
;
1861 nes_debug(NES_DBG_CQ
, "Destroy CQ%u\n", nescq
->hw_cq
.cq_number
);
1863 /* Send DestroyCQ request to CQP */
1864 cqp_request
= nes_get_cqp_request(nesdev
);
1865 if (cqp_request
== NULL
) {
1866 nes_debug(NES_DBG_CQ
, "Failed to get a cqp_request.\n");
1869 cqp_request
->waiting
= 1;
1870 cqp_wqe
= &cqp_request
->cqp_wqe
;
1871 opcode
= NES_CQP_DESTROY_CQ
| (nescq
->hw_cq
.cq_size
<< 16);
1872 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1873 if (nescq
->virtual_cq
== 1) {
1874 nesadapter
->free_256pbl
++;
1875 if (nesadapter
->free_256pbl
> nesadapter
->max_256pbl
) {
1876 printk(KERN_ERR PFX
"%s: free 256B PBLs(%u) has exceeded the max(%u)\n",
1877 __func__
, nesadapter
->free_256pbl
, nesadapter
->max_256pbl
);
1879 } else if (nescq
->virtual_cq
== 2) {
1880 nesadapter
->free_4kpbl
++;
1881 if (nesadapter
->free_4kpbl
> nesadapter
->max_4kpbl
) {
1882 printk(KERN_ERR PFX
"%s: free 4K PBLs(%u) has exceeded the max(%u)\n",
1883 __func__
, nesadapter
->free_4kpbl
, nesadapter
->max_4kpbl
);
1885 opcode
|= NES_CQP_CQ_4KB_CHUNK
;
1888 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1890 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1891 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1892 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
,
1893 (nescq
->hw_cq
.cq_number
| ((u32
)PCI_FUNC(nesdev
->pcidev
->devfn
) << 16)));
1895 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, nescq
->hw_cq
.cq_number
);
1897 atomic_set(&cqp_request
->refcount
, 2);
1898 nes_post_cqp_request(nesdev
, cqp_request
);
1901 nes_debug(NES_DBG_CQ
, "Waiting for destroy iWARP CQ%u to complete.\n",
1902 nescq
->hw_cq
.cq_number
);
1903 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1905 nes_debug(NES_DBG_CQ
, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u,"
1906 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1907 nescq
->hw_cq
.cq_number
, ret
, cqp_request
->major_code
,
1908 cqp_request
->minor_code
);
1910 nes_debug(NES_DBG_CQ
, "iWARP CQ%u destroy timeout expired\n",
1911 nescq
->hw_cq
.cq_number
);
1913 } else if (cqp_request
->major_code
) {
1914 nes_debug(NES_DBG_CQ
, "iWARP CQ%u destroy failed\n",
1915 nescq
->hw_cq
.cq_number
);
1920 nes_put_cqp_request(nesdev
, cqp_request
);
1922 if (nescq
->cq_mem_size
)
1923 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
,
1924 nescq
->hw_cq
.cq_vbase
, nescq
->hw_cq
.cq_pbase
);
1933 static u32
root_256(struct nes_device
*nesdev
,
1934 struct nes_root_vpbl
*root_vpbl
,
1935 struct nes_root_vpbl
*new_root
,
1941 if (pbl_count_4k
== 1) {
1942 new_root
->pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
,
1943 512, &new_root
->pbl_pbase
);
1945 if (new_root
->pbl_vbase
== NULL
)
1948 leaf_pbl
= (u64
)root_vpbl
->pbl_pbase
;
1949 for (i
= 0; i
< 16; i
++) {
1950 new_root
->pbl_vbase
[i
].pa_low
=
1951 cpu_to_le32((u32
)leaf_pbl
);
1952 new_root
->pbl_vbase
[i
].pa_high
=
1953 cpu_to_le32((u32
)((((u64
)leaf_pbl
) >> 32)));
1957 for (i
= 3; i
>= 0; i
--) {
1959 root_vpbl
->pbl_vbase
[j
] = root_vpbl
->pbl_vbase
[i
];
1960 leaf_pbl
= le32_to_cpu(root_vpbl
->pbl_vbase
[j
].pa_low
) +
1961 (((u64
)le32_to_cpu(root_vpbl
->pbl_vbase
[j
].pa_high
))
1963 for (k
= 1; k
< 16; k
++) {
1965 root_vpbl
->pbl_vbase
[j
+ k
].pa_low
=
1966 cpu_to_le32((u32
)leaf_pbl
);
1967 root_vpbl
->pbl_vbase
[j
+ k
].pa_high
=
1968 cpu_to_le32((u32
)((((u64
)leaf_pbl
) >> 32)));
1980 static int nes_reg_mr(struct nes_device
*nesdev
, struct nes_pd
*nespd
,
1981 u32 stag
, u64 region_length
, struct nes_root_vpbl
*root_vpbl
,
1982 dma_addr_t single_buffer
, u16 pbl_count_4k
,
1983 u16 residual_page_count_4k
, int acc
, u64
*iova_start
,
1984 u16
*actual_pbl_cnt
, u8
*used_4k_pbls
)
1986 struct nes_hw_cqp_wqe
*cqp_wqe
;
1987 struct nes_cqp_request
*cqp_request
;
1988 unsigned long flags
;
1990 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1992 u16 pbl_count_256
= 0;
1994 u8 use_256_pbls
= 0;
1996 u16 use_two_level
= (pbl_count_4k
> 1) ? 1 : 0;
1997 struct nes_root_vpbl new_root
= { 0, NULL
, NULL
};
2001 /* Register the region with the adapter */
2002 cqp_request
= nes_get_cqp_request(nesdev
);
2003 if (cqp_request
== NULL
) {
2004 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
2007 cqp_request
->waiting
= 1;
2008 cqp_wqe
= &cqp_request
->cqp_wqe
;
2011 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2013 pg_cnt
= ((pbl_count_4k
- 1) * 512) + residual_page_count_4k
;
2014 pbl_count_256
= (pg_cnt
+ 31) / 32;
2016 if (pbl_count_256
<= nesadapter
->free_256pbl
)
2018 else if (pbl_count_4k
<= nesadapter
->free_4kpbl
)
2020 } else if (pg_cnt
<= 2048) {
2021 if (((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) &&
2022 (nesadapter
->free_4kpbl
> (nesadapter
->max_4kpbl
>> 1))) {
2024 } else if ((pbl_count_256
+ 1) <= nesadapter
->free_256pbl
) {
2027 } else if ((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) {
2031 if ((pbl_count_4k
+ 1) <= nesadapter
->free_4kpbl
)
2036 pbl_count
= pbl_count_256
;
2037 nesadapter
->free_256pbl
-= pbl_count
+ use_two_level
;
2038 } else if (use_4k_pbls
) {
2039 pbl_count
= pbl_count_4k
;
2040 nesadapter
->free_4kpbl
-= pbl_count
+ use_two_level
;
2042 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2043 nes_debug(NES_DBG_MR
, "Out of Pbls\n");
2044 nes_free_cqp_request(nesdev
, cqp_request
);
2048 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2051 if (use_256_pbls
&& use_two_level
) {
2052 if (root_256(nesdev
, root_vpbl
, &new_root
, pbl_count_4k
) == 1) {
2053 if (new_root
.pbl_pbase
!= 0)
2054 root_vpbl
= &new_root
;
2056 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2057 nesadapter
->free_256pbl
+= pbl_count_256
+ use_two_level
;
2060 if (pbl_count_4k
== 1)
2062 pbl_count
= pbl_count_4k
;
2064 if ((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) {
2065 nesadapter
->free_4kpbl
-= pbl_count
+ use_two_level
;
2068 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2070 if (use_4k_pbls
== 0)
2075 opcode
= NES_CQP_REGISTER_STAG
| NES_CQP_STAG_RIGHTS_LOCAL_READ
|
2076 NES_CQP_STAG_VA_TO
| NES_CQP_STAG_MR
;
2077 if (acc
& IB_ACCESS_LOCAL_WRITE
)
2078 opcode
|= NES_CQP_STAG_RIGHTS_LOCAL_WRITE
;
2079 if (acc
& IB_ACCESS_REMOTE_WRITE
)
2080 opcode
|= NES_CQP_STAG_RIGHTS_REMOTE_WRITE
| NES_CQP_STAG_REM_ACC_EN
;
2081 if (acc
& IB_ACCESS_REMOTE_READ
)
2082 opcode
|= NES_CQP_STAG_RIGHTS_REMOTE_READ
| NES_CQP_STAG_REM_ACC_EN
;
2083 if (acc
& IB_ACCESS_MW_BIND
)
2084 opcode
|= NES_CQP_STAG_RIGHTS_WINDOW_BIND
| NES_CQP_STAG_REM_ACC_EN
;
2086 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2087 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
2088 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_VA_LOW_IDX
, *iova_start
);
2089 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_LOW_IDX
, region_length
);
2091 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] =
2092 cpu_to_le32((u32
)(region_length
>> 8) & 0xff000000);
2093 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] |=
2094 cpu_to_le32(nespd
->pd_id
& 0x00007fff);
2095 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
2097 if (pbl_count
== 0) {
2098 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, single_buffer
);
2100 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, root_vpbl
->pbl_pbase
);
2101 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX
, pbl_count
);
2102 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_LEN_IDX
, (pg_cnt
* 8));
2105 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE
);
2109 atomic_set(&cqp_request
->refcount
, 2);
2110 nes_post_cqp_request(nesdev
, cqp_request
);
2113 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
2115 nes_debug(NES_DBG_MR
, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
2116 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2117 stag
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2118 major_code
= cqp_request
->major_code
;
2119 nes_put_cqp_request(nesdev
, cqp_request
);
2121 if ((!ret
|| major_code
) && pbl_count
!= 0) {
2122 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2124 nesadapter
->free_256pbl
+= pbl_count
+ use_two_level
;
2125 else if (use_4k_pbls
)
2126 nesadapter
->free_4kpbl
+= pbl_count
+ use_two_level
;
2127 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2129 if (new_root
.pbl_pbase
)
2130 pci_free_consistent(nesdev
->pcidev
, 512, new_root
.pbl_vbase
,
2131 new_root
.pbl_pbase
);
2135 else if (major_code
)
2138 *actual_pbl_cnt
= pbl_count
+ use_two_level
;
2139 *used_4k_pbls
= use_4k_pbls
;
2147 static struct ib_mr
*nes_reg_phys_mr(struct ib_pd
*ib_pd
,
2148 struct ib_phys_buf
*buffer_list
, int num_phys_buf
, int acc
,
2152 struct nes_pd
*nespd
= to_nespd(ib_pd
);
2153 struct nes_vnic
*nesvnic
= to_nesvnic(ib_pd
->device
);
2154 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2155 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2156 struct nes_mr
*nesmr
;
2158 struct nes_vpbl vpbl
;
2159 struct nes_root_vpbl root_vpbl
;
2164 u32 next_stag_index
= 0;
2166 u32 root_pbl_index
= 0;
2167 u32 cur_pbl_index
= 0;
2175 vpbl
.pbl_vbase
= NULL
;
2176 root_vpbl
.pbl_vbase
= NULL
;
2177 root_vpbl
.pbl_pbase
= 0;
2179 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
2180 stag_key
= (u8
)next_stag_index
;
2184 next_stag_index
>>= 8;
2185 next_stag_index
%= nesadapter
->max_mr
;
2186 if (num_phys_buf
> (1024*512)) {
2187 return ERR_PTR(-E2BIG
);
2190 if ((buffer_list
[0].addr
^ *iova_start
) & ~PAGE_MASK
)
2191 return ERR_PTR(-EINVAL
);
2193 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
, nesadapter
->max_mr
,
2194 &stag_index
, &next_stag_index
, NES_RESOURCE_PHYS_MR
);
2196 return ERR_PTR(err
);
2199 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2201 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2202 return ERR_PTR(-ENOMEM
);
2205 for (i
= 0; i
< num_phys_buf
; i
++) {
2207 if ((i
& 0x01FF) == 0) {
2208 if (root_pbl_index
== 1) {
2209 /* Allocate the root PBL */
2210 root_vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 8192,
2211 &root_vpbl
.pbl_pbase
);
2212 nes_debug(NES_DBG_MR
, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2213 root_vpbl
.pbl_vbase
, (unsigned int)root_vpbl
.pbl_pbase
);
2214 if (!root_vpbl
.pbl_vbase
) {
2215 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2217 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2219 return ERR_PTR(-ENOMEM
);
2221 root_vpbl
.leaf_vpbl
= kzalloc(sizeof(*root_vpbl
.leaf_vpbl
)*1024, GFP_KERNEL
);
2222 if (!root_vpbl
.leaf_vpbl
) {
2223 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2224 root_vpbl
.pbl_pbase
);
2225 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2227 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2229 return ERR_PTR(-ENOMEM
);
2231 root_vpbl
.pbl_vbase
[0].pa_low
= cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2232 root_vpbl
.pbl_vbase
[0].pa_high
=
2233 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2234 root_vpbl
.leaf_vpbl
[0] = vpbl
;
2236 /* Allocate a 4K buffer for the PBL */
2237 vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 4096,
2239 nes_debug(NES_DBG_MR
, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
2240 vpbl
.pbl_vbase
, (unsigned long)vpbl
.pbl_pbase
);
2241 if (!vpbl
.pbl_vbase
) {
2242 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2243 ibmr
= ERR_PTR(-ENOMEM
);
2247 /* Fill in the root table */
2248 if (1 <= root_pbl_index
) {
2249 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_low
=
2250 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2251 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_high
=
2252 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2253 root_vpbl
.leaf_vpbl
[root_pbl_index
] = vpbl
;
2259 mask
= !buffer_list
[i
].size
;
2261 mask
|= buffer_list
[i
].addr
;
2262 if (i
!= num_phys_buf
- 1)
2263 mask
|= buffer_list
[i
].addr
+ buffer_list
[i
].size
;
2265 if (mask
& ~PAGE_MASK
) {
2266 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2267 nes_debug(NES_DBG_MR
, "Invalid buffer addr or size\n");
2268 ibmr
= ERR_PTR(-EINVAL
);
2273 region_length
+= buffer_list
[i
].size
;
2274 if ((i
!= 0) && (single_page
)) {
2275 if ((buffer_list
[i
-1].addr
+PAGE_SIZE
) != buffer_list
[i
].addr
)
2278 vpbl
.pbl_vbase
[cur_pbl_index
].pa_low
= cpu_to_le32((u32
)buffer_list
[i
].addr
& PAGE_MASK
);
2279 vpbl
.pbl_vbase
[cur_pbl_index
++].pa_high
=
2280 cpu_to_le32((u32
)((((u64
)buffer_list
[i
].addr
) >> 32)));
2283 stag
= stag_index
<< 8;
2285 stag
+= (u32
)stag_key
;
2287 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, VA = 0x%016lX,"
2288 " length = 0x%016lX, index = 0x%08X\n",
2289 stag
, (unsigned long)*iova_start
, (unsigned long)region_length
, stag_index
);
2291 /* Make the leaf PBL the root if only one PBL */
2292 if (root_pbl_index
== 1) {
2293 root_vpbl
.pbl_pbase
= vpbl
.pbl_pbase
;
2299 pbl_count
= root_pbl_index
;
2301 ret
= nes_reg_mr(nesdev
, nespd
, stag
, region_length
, &root_vpbl
,
2302 buffer_list
[0].addr
, pbl_count
, (u16
)cur_pbl_index
, acc
, iova_start
,
2303 &nesmr
->pbls_used
, &nesmr
->pbl_4k
);
2306 nesmr
->ibmr
.rkey
= stag
;
2307 nesmr
->ibmr
.lkey
= stag
;
2308 nesmr
->mode
= IWNES_MEMREG_TYPE_MEM
;
2309 ibmr
= &nesmr
->ibmr
;
2312 ibmr
= ERR_PTR(-ENOMEM
);
2316 /* free the resources */
2317 if (root_pbl_index
== 1) {
2318 /* single PBL case */
2319 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
, vpbl
.pbl_pbase
);
2321 for (i
=0; i
<root_pbl_index
; i
++) {
2322 pci_free_consistent(nesdev
->pcidev
, 4096, root_vpbl
.leaf_vpbl
[i
].pbl_vbase
,
2323 root_vpbl
.leaf_vpbl
[i
].pbl_pbase
);
2325 kfree(root_vpbl
.leaf_vpbl
);
2326 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2327 root_vpbl
.pbl_pbase
);
2337 static struct ib_mr
*nes_get_dma_mr(struct ib_pd
*pd
, int acc
)
2339 struct ib_phys_buf bl
;
2342 nes_debug(NES_DBG_MR
, "\n");
2344 bl
.size
= (u64
)0xffffffffffULL
;
2346 return nes_reg_phys_mr(pd
, &bl
, 1, acc
, &kva
);
2353 static struct ib_mr
*nes_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
2354 u64 virt
, int acc
, struct ib_udata
*udata
)
2359 dma_addr_t last_dma_addr
= 0;
2360 dma_addr_t first_dma_addr
= 0;
2361 struct nes_pd
*nespd
= to_nespd(pd
);
2362 struct nes_vnic
*nesvnic
= to_nesvnic(pd
->device
);
2363 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2364 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2365 struct ib_mr
*ibmr
= ERR_PTR(-EINVAL
);
2366 struct scatterlist
*sg
;
2367 struct nes_ucontext
*nes_ucontext
;
2368 struct nes_pbl
*nespbl
;
2369 struct nes_mr
*nesmr
;
2370 struct ib_umem
*region
;
2371 struct nes_mem_reg_req req
;
2372 struct nes_vpbl vpbl
;
2373 struct nes_root_vpbl root_vpbl
;
2374 int entry
, page_index
;
2376 int err
, pbl_depth
= 0;
2381 u32 next_stag_index
;
2383 u32 root_pbl_index
= 0;
2384 u32 cur_pbl_index
= 0;
2391 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
2392 if (IS_ERR(region
)) {
2393 return (struct ib_mr
*)region
;
2396 nes_debug(NES_DBG_MR
, "User base = 0x%lX, Virt base = 0x%lX, length = %u,"
2397 " offset = %u, page size = %u.\n",
2398 (unsigned long int)start
, (unsigned long int)virt
, (u32
)length
,
2399 ib_umem_offset(region
), region
->page_size
);
2401 skip_pages
= ((u32
)ib_umem_offset(region
)) >> 12;
2403 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
2404 ib_umem_release(region
);
2405 return ERR_PTR(-EFAULT
);
2407 nes_debug(NES_DBG_MR
, "Memory Registration type = %08X.\n", req
.reg_type
);
2409 switch (req
.reg_type
) {
2410 case IWNES_MEMREG_TYPE_MEM
:
2413 vpbl
.pbl_vbase
= NULL
;
2414 root_vpbl
.pbl_vbase
= NULL
;
2415 root_vpbl
.pbl_pbase
= 0;
2417 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
2418 stag_key
= (u8
)next_stag_index
;
2420 driver_key
= next_stag_index
& 0x70000000;
2422 next_stag_index
>>= 8;
2423 next_stag_index
%= nesadapter
->max_mr
;
2425 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
2426 nesadapter
->max_mr
, &stag_index
, &next_stag_index
, NES_RESOURCE_USER_MR
);
2428 ib_umem_release(region
);
2429 return ERR_PTR(err
);
2432 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2434 ib_umem_release(region
);
2435 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2436 return ERR_PTR(-ENOMEM
);
2438 nesmr
->region
= region
;
2440 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
2441 if (sg_dma_address(sg
) & ~PAGE_MASK
) {
2442 ib_umem_release(region
);
2443 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2444 nes_debug(NES_DBG_MR
, "Unaligned Memory Buffer: 0x%x\n",
2445 (unsigned int) sg_dma_address(sg
));
2446 ibmr
= ERR_PTR(-EINVAL
);
2448 goto reg_user_mr_err
;
2451 if (!sg_dma_len(sg
)) {
2452 ib_umem_release(region
);
2453 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2455 nes_debug(NES_DBG_MR
, "Invalid Buffer Size\n");
2456 ibmr
= ERR_PTR(-EINVAL
);
2458 goto reg_user_mr_err
;
2461 region_length
+= sg_dma_len(sg
);
2462 chunk_pages
= sg_dma_len(sg
) >> 12;
2463 region_length
-= skip_pages
<< 12;
2464 for (page_index
= skip_pages
; page_index
< chunk_pages
; page_index
++) {
2466 if ((page_count
!= 0) && (page_count
<< 12) - (ib_umem_offset(region
) & (4096 - 1)) >= region
->length
)
2468 if ((page_count
&0x01FF) == 0) {
2469 if (page_count
>= 1024 * 512) {
2470 ib_umem_release(region
);
2471 nes_free_resource(nesadapter
,
2472 nesadapter
->allocated_mrs
, stag_index
);
2474 ibmr
= ERR_PTR(-E2BIG
);
2475 goto reg_user_mr_err
;
2477 if (root_pbl_index
== 1) {
2478 root_vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
,
2479 8192, &root_vpbl
.pbl_pbase
);
2480 nes_debug(NES_DBG_MR
, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2481 root_vpbl
.pbl_vbase
, (unsigned int)root_vpbl
.pbl_pbase
);
2482 if (!root_vpbl
.pbl_vbase
) {
2483 ib_umem_release(region
);
2484 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2486 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2489 ibmr
= ERR_PTR(-ENOMEM
);
2490 goto reg_user_mr_err
;
2492 root_vpbl
.leaf_vpbl
= kzalloc(sizeof(*root_vpbl
.leaf_vpbl
)*1024,
2494 if (!root_vpbl
.leaf_vpbl
) {
2495 ib_umem_release(region
);
2496 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2497 root_vpbl
.pbl_pbase
);
2498 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2500 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2503 ibmr
= ERR_PTR(-ENOMEM
);
2504 goto reg_user_mr_err
;
2506 root_vpbl
.pbl_vbase
[0].pa_low
=
2507 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2508 root_vpbl
.pbl_vbase
[0].pa_high
=
2509 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2510 root_vpbl
.leaf_vpbl
[0] = vpbl
;
2512 vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 4096,
2514 nes_debug(NES_DBG_MR
, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2515 vpbl
.pbl_vbase
, (unsigned int)vpbl
.pbl_pbase
);
2516 if (!vpbl
.pbl_vbase
) {
2517 ib_umem_release(region
);
2518 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2519 ibmr
= ERR_PTR(-ENOMEM
);
2521 goto reg_user_mr_err
;
2523 if (1 <= root_pbl_index
) {
2524 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_low
=
2525 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2526 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_high
=
2527 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
)>>32)));
2528 root_vpbl
.leaf_vpbl
[root_pbl_index
] = vpbl
;
2534 if (page_count
!= 0) {
2535 if ((last_dma_addr
+4096) !=
2536 (sg_dma_address(sg
)+
2539 last_dma_addr
= sg_dma_address(sg
)+
2542 first_dma_addr
= sg_dma_address(sg
)+
2544 last_dma_addr
= first_dma_addr
;
2548 vpbl
.pbl_vbase
[cur_pbl_index
].pa_low
=
2549 cpu_to_le32((u32
)(sg_dma_address(sg
)+
2550 (page_index
*4096)));
2551 vpbl
.pbl_vbase
[cur_pbl_index
].pa_high
=
2552 cpu_to_le32((u32
)((((u64
)(sg_dma_address(sg
)+
2553 (page_index
*4096))) >> 32)));
2560 nes_debug(NES_DBG_MR
, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
2561 " stag_key=0x%08x\n",
2562 stag_index
, driver_key
, stag_key
);
2563 stag
= stag_index
<< 8;
2565 stag
+= (u32
)stag_key
;
2568 /* Make the leaf PBL the root if only one PBL */
2569 if (root_pbl_index
== 1) {
2570 root_vpbl
.pbl_pbase
= vpbl
.pbl_pbase
;
2576 pbl_count
= root_pbl_index
;
2579 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X,"
2580 " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n",
2581 stag
, (unsigned int)iova_start
,
2582 (unsigned int)region_length
, stag_index
,
2583 (unsigned long long)region
->length
, pbl_count
);
2584 ret
= nes_reg_mr(nesdev
, nespd
, stag
, region
->length
, &root_vpbl
,
2585 first_dma_addr
, pbl_count
, (u16
)cur_pbl_index
, acc
,
2586 &iova_start
, &nesmr
->pbls_used
, &nesmr
->pbl_4k
);
2588 nes_debug(NES_DBG_MR
, "ret=%d\n", ret
);
2591 nesmr
->ibmr
.rkey
= stag
;
2592 nesmr
->ibmr
.lkey
= stag
;
2593 nesmr
->mode
= IWNES_MEMREG_TYPE_MEM
;
2594 ibmr
= &nesmr
->ibmr
;
2596 ib_umem_release(region
);
2598 ibmr
= ERR_PTR(-ENOMEM
);
2602 /* free the resources */
2603 if (root_pbl_index
== 1) {
2604 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2607 for (page_index
=0; page_index
<root_pbl_index
; page_index
++) {
2608 pci_free_consistent(nesdev
->pcidev
, 4096,
2609 root_vpbl
.leaf_vpbl
[page_index
].pbl_vbase
,
2610 root_vpbl
.leaf_vpbl
[page_index
].pbl_pbase
);
2612 kfree(root_vpbl
.leaf_vpbl
);
2613 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2614 root_vpbl
.pbl_pbase
);
2617 nes_debug(NES_DBG_MR
, "Leaving, ibmr=%p", ibmr
);
2620 case IWNES_MEMREG_TYPE_QP
:
2621 case IWNES_MEMREG_TYPE_CQ
:
2622 if (!region
->length
) {
2623 nes_debug(NES_DBG_MR
, "Unable to register zero length region for CQ\n");
2624 ib_umem_release(region
);
2625 return ERR_PTR(-EINVAL
);
2627 nespbl
= kzalloc(sizeof(*nespbl
), GFP_KERNEL
);
2629 nes_debug(NES_DBG_MR
, "Unable to allocate PBL\n");
2630 ib_umem_release(region
);
2631 return ERR_PTR(-ENOMEM
);
2633 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2635 ib_umem_release(region
);
2637 nes_debug(NES_DBG_MR
, "Unable to allocate nesmr\n");
2638 return ERR_PTR(-ENOMEM
);
2640 nesmr
->region
= region
;
2641 nes_ucontext
= to_nesucontext(pd
->uobject
->context
);
2642 pbl_depth
= region
->length
>> 12;
2643 pbl_depth
+= (region
->length
& (4096-1)) ? 1 : 0;
2644 nespbl
->pbl_size
= pbl_depth
*sizeof(u64
);
2645 if (req
.reg_type
== IWNES_MEMREG_TYPE_QP
) {
2646 nes_debug(NES_DBG_MR
, "Attempting to allocate QP PBL memory");
2648 nes_debug(NES_DBG_MR
, "Attempting to allocate CP PBL memory");
2651 nes_debug(NES_DBG_MR
, " %u bytes, %u entries.\n",
2652 nespbl
->pbl_size
, pbl_depth
);
2653 pbl
= pci_alloc_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
2654 &nespbl
->pbl_pbase
);
2656 ib_umem_release(region
);
2659 nes_debug(NES_DBG_MR
, "Unable to allocate PBL memory\n");
2660 return ERR_PTR(-ENOMEM
);
2663 nespbl
->pbl_vbase
= (u64
*)pbl
;
2664 nespbl
->user_base
= start
;
2665 nes_debug(NES_DBG_MR
, "Allocated PBL memory, %u bytes, pbl_pbase=%lx,"
2666 " pbl_vbase=%p user_base=0x%lx\n",
2667 nespbl
->pbl_size
, (unsigned long) nespbl
->pbl_pbase
,
2668 (void *) nespbl
->pbl_vbase
, nespbl
->user_base
);
2670 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
2671 chunk_pages
= sg_dma_len(sg
) >> 12;
2672 chunk_pages
+= (sg_dma_len(sg
) & (4096-1)) ? 1 : 0;
2674 nespbl
->page
= sg_page(sg
);
2678 for (page_index
= 0; page_index
< chunk_pages
; page_index
++) {
2679 ((__le32
*)pbl
)[0] = cpu_to_le32((u32
)
2680 (sg_dma_address(sg
)+
2681 (page_index
*4096)));
2682 ((__le32
*)pbl
)[1] = cpu_to_le32(((u64
)
2683 (sg_dma_address(sg
)+
2684 (page_index
*4096)))>>32);
2685 nes_debug(NES_DBG_MR
, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl
,
2686 (unsigned long long)*pbl
,
2687 le32_to_cpu(((__le32
*)pbl
)[1]), le32_to_cpu(((__le32
*)pbl
)[0]));
2692 if (req
.reg_type
== IWNES_MEMREG_TYPE_QP
) {
2693 list_add_tail(&nespbl
->list
, &nes_ucontext
->qp_reg_mem_list
);
2695 list_add_tail(&nespbl
->list
, &nes_ucontext
->cq_reg_mem_list
);
2697 nesmr
->ibmr
.rkey
= -1;
2698 nesmr
->ibmr
.lkey
= -1;
2699 nesmr
->mode
= req
.reg_type
;
2700 return &nesmr
->ibmr
;
2703 ib_umem_release(region
);
2704 return ERR_PTR(-ENOSYS
);
2711 static int nes_dereg_mr(struct ib_mr
*ib_mr
)
2713 struct nes_mr
*nesmr
= to_nesmr(ib_mr
);
2714 struct nes_vnic
*nesvnic
= to_nesvnic(ib_mr
->device
);
2715 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2716 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2717 struct nes_hw_cqp_wqe
*cqp_wqe
;
2718 struct nes_cqp_request
*cqp_request
;
2719 unsigned long flags
;
2726 pci_free_consistent(nesdev
->pcidev
,
2727 nesmr
->max_pages
* sizeof(u64
),
2731 if (nesmr
->region
) {
2732 ib_umem_release(nesmr
->region
);
2734 if (nesmr
->mode
!= IWNES_MEMREG_TYPE_MEM
) {
2739 /* Deallocate the region with the adapter */
2741 cqp_request
= nes_get_cqp_request(nesdev
);
2742 if (cqp_request
== NULL
) {
2743 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
2746 cqp_request
->waiting
= 1;
2747 cqp_wqe
= &cqp_request
->cqp_wqe
;
2749 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2750 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
,
2751 NES_CQP_DEALLOCATE_STAG
| NES_CQP_STAG_VA_TO
|
2752 NES_CQP_STAG_DEALLOC_PBLS
| NES_CQP_STAG_MR
);
2753 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, ib_mr
->rkey
);
2755 atomic_set(&cqp_request
->refcount
, 2);
2756 nes_post_cqp_request(nesdev
, cqp_request
);
2759 nes_debug(NES_DBG_MR
, "Waiting for deallocate STag 0x%08X completed\n", ib_mr
->rkey
);
2760 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
2762 nes_debug(NES_DBG_MR
, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u,"
2763 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
2764 ib_mr
->rkey
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2766 major_code
= cqp_request
->major_code
;
2767 minor_code
= cqp_request
->minor_code
;
2769 nes_put_cqp_request(nesdev
, cqp_request
);
2772 nes_debug(NES_DBG_MR
, "Timeout waiting to destroy STag,"
2773 " ib_mr=%p, rkey = 0x%08X\n",
2774 ib_mr
, ib_mr
->rkey
);
2776 } else if (major_code
) {
2777 nes_debug(NES_DBG_MR
, "Error (0x%04X:0x%04X) while attempting"
2778 " to destroy STag, ib_mr=%p, rkey = 0x%08X\n",
2779 major_code
, minor_code
, ib_mr
, ib_mr
->rkey
);
2783 if (nesmr
->pbls_used
!= 0) {
2784 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2785 if (nesmr
->pbl_4k
) {
2786 nesadapter
->free_4kpbl
+= nesmr
->pbls_used
;
2787 if (nesadapter
->free_4kpbl
> nesadapter
->max_4kpbl
)
2788 printk(KERN_ERR PFX
"free 4KB PBLs(%u) has "
2789 "exceeded the max(%u)\n",
2790 nesadapter
->free_4kpbl
,
2791 nesadapter
->max_4kpbl
);
2793 nesadapter
->free_256pbl
+= nesmr
->pbls_used
;
2794 if (nesadapter
->free_256pbl
> nesadapter
->max_256pbl
)
2795 printk(KERN_ERR PFX
"free 256B PBLs(%u) has "
2796 "exceeded the max(%u)\n",
2797 nesadapter
->free_256pbl
,
2798 nesadapter
->max_256pbl
);
2800 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2802 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2803 (ib_mr
->rkey
& 0x0fffff00) >> 8);
2814 static ssize_t
show_rev(struct device
*dev
, struct device_attribute
*attr
,
2817 struct nes_ib_device
*nesibdev
=
2818 container_of(dev
, struct nes_ib_device
, ibdev
.dev
);
2819 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
2821 nes_debug(NES_DBG_INIT
, "\n");
2822 return sprintf(buf
, "%x\n", nesvnic
->nesdev
->nesadapter
->hw_rev
);
2829 static ssize_t
show_fw_ver(struct device
*dev
, struct device_attribute
*attr
,
2832 struct nes_ib_device
*nesibdev
=
2833 container_of(dev
, struct nes_ib_device
, ibdev
.dev
);
2834 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
2836 nes_debug(NES_DBG_INIT
, "\n");
2837 return sprintf(buf
, "%u.%u\n",
2838 (nesvnic
->nesdev
->nesadapter
->firmware_version
>> 16),
2839 (nesvnic
->nesdev
->nesadapter
->firmware_version
& 0x000000ff));
2846 static ssize_t
show_hca(struct device
*dev
, struct device_attribute
*attr
,
2849 nes_debug(NES_DBG_INIT
, "\n");
2850 return sprintf(buf
, "NES020\n");
2857 static ssize_t
show_board(struct device
*dev
, struct device_attribute
*attr
,
2860 nes_debug(NES_DBG_INIT
, "\n");
2861 return sprintf(buf
, "%.*s\n", 32, "NES020 Board ID");
2865 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
2866 static DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
2867 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
2868 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
2870 static struct device_attribute
*nes_dev_attributes
[] = {
2881 static int nes_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2882 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
2884 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
2886 nes_debug(NES_DBG_QP
, "\n");
2888 attr
->qp_access_flags
= 0;
2889 attr
->cap
.max_send_wr
= nesqp
->hwqp
.sq_size
;
2890 attr
->cap
.max_recv_wr
= nesqp
->hwqp
.rq_size
;
2891 attr
->cap
.max_recv_sge
= 1;
2892 if (nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
)
2893 attr
->cap
.max_inline_data
= 0;
2895 attr
->cap
.max_inline_data
= 64;
2897 init_attr
->event_handler
= nesqp
->ibqp
.event_handler
;
2898 init_attr
->qp_context
= nesqp
->ibqp
.qp_context
;
2899 init_attr
->send_cq
= nesqp
->ibqp
.send_cq
;
2900 init_attr
->recv_cq
= nesqp
->ibqp
.recv_cq
;
2901 init_attr
->srq
= nesqp
->ibqp
.srq
;
2902 init_attr
->cap
= attr
->cap
;
2911 int nes_hw_modify_qp(struct nes_device
*nesdev
, struct nes_qp
*nesqp
,
2912 u32 next_iwarp_state
, u32 termlen
, u32 wait_completion
)
2914 struct nes_hw_cqp_wqe
*cqp_wqe
;
2915 /* struct iw_cm_id *cm_id = nesqp->cm_id; */
2916 /* struct iw_cm_event cm_event; */
2917 struct nes_cqp_request
*cqp_request
;
2921 nes_debug(NES_DBG_MOD_QP
, "QP%u, refcount=%d\n",
2922 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
2924 cqp_request
= nes_get_cqp_request(nesdev
);
2925 if (cqp_request
== NULL
) {
2926 nes_debug(NES_DBG_MOD_QP
, "Failed to get a cqp_request.\n");
2929 if (wait_completion
) {
2930 cqp_request
->waiting
= 1;
2932 cqp_request
->waiting
= 0;
2934 cqp_wqe
= &cqp_request
->cqp_wqe
;
2936 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
,
2937 NES_CQP_MODIFY_QP
| NES_CQP_QP_TYPE_IWARP
| next_iwarp_state
);
2938 nes_debug(NES_DBG_MOD_QP
, "using next_iwarp_state=%08x, wqe_words=%08x\n",
2939 next_iwarp_state
, le32_to_cpu(cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
]));
2940 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2941 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
, nesqp
->hwqp
.qp_id
);
2942 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_CONTEXT_LOW_IDX
, (u64
)nesqp
->nesqp_context_pbase
);
2944 /* If sending a terminate message, fill in the length (in words) */
2945 if (((next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
) == NES_CQP_QP_IWARP_STATE_TERMINATE
) &&
2946 !(next_iwarp_state
& NES_CQP_QP_TERM_DONT_SEND_TERM_MSG
)) {
2947 termlen
= ((termlen
+ 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT
;
2948 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_NEW_MSS_IDX
, termlen
);
2951 atomic_set(&cqp_request
->refcount
, 2);
2952 nes_post_cqp_request(nesdev
, cqp_request
);
2955 if (wait_completion
) {
2956 /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n",
2957 nesqp->hwqp.qp_id); */
2958 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
2960 nes_debug(NES_DBG_MOD_QP
, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, "
2961 "CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2962 nesqp
->hwqp
.qp_id
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2963 major_code
= cqp_request
->major_code
;
2965 nes_debug(NES_DBG_MOD_QP
, "Modify iwarp QP%u failed"
2966 "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n",
2967 nesqp
->hwqp
.qp_id
, cqp_request
->major_code
,
2968 cqp_request
->minor_code
, next_iwarp_state
);
2971 nes_put_cqp_request(nesdev
, cqp_request
);
2975 else if (major_code
)
2988 int nes_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2989 int attr_mask
, struct ib_udata
*udata
)
2991 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
2992 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
2993 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2996 u32 next_iwarp_state
= 0;
2998 unsigned long qplockflags
;
3000 u16 original_last_aeq
;
3001 u8 issue_modify_qp
= 0;
3004 nes_debug(NES_DBG_MOD_QP
, "QP%u: QP State=%u, cur QP State=%u,"
3005 " iwarp_state=0x%X, refcount=%d\n",
3006 nesqp
->hwqp
.qp_id
, attr
->qp_state
, nesqp
->ibqp_state
,
3007 nesqp
->iwarp_state
, atomic_read(&nesqp
->refcount
));
3009 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
3011 nes_debug(NES_DBG_MOD_QP
, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X,"
3012 " QP Access Flags=0x%X, attr_mask = 0x%0x\n",
3013 nesqp
->hwqp
.qp_id
, nesqp
->hw_iwarp_state
,
3014 nesqp
->hw_tcp_state
, attr
->qp_access_flags
, attr_mask
);
3016 if (attr_mask
& IB_QP_STATE
) {
3017 switch (attr
->qp_state
) {
3019 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = init\n",
3021 if (nesqp
->iwarp_state
> (u32
)NES_CQP_QP_IWARP_STATE_IDLE
) {
3022 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3025 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_IDLE
;
3026 issue_modify_qp
= 1;
3029 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = rtr\n",
3031 if (nesqp
->iwarp_state
>(u32
)NES_CQP_QP_IWARP_STATE_IDLE
) {
3032 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3035 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_IDLE
;
3036 issue_modify_qp
= 1;
3039 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = rts\n",
3041 if (nesqp
->iwarp_state
>(u32
)NES_CQP_QP_IWARP_STATE_RTS
) {
3042 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3045 if (nesqp
->cm_id
== NULL
) {
3046 nes_debug(NES_DBG_MOD_QP
, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n",
3047 nesqp
->hwqp
.qp_id
);
3048 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3051 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_RTS
;
3052 if (nesqp
->iwarp_state
!= NES_CQP_QP_IWARP_STATE_RTS
)
3053 next_iwarp_state
|= NES_CQP_QP_CONTEXT_VALID
|
3054 NES_CQP_QP_ARP_VALID
| NES_CQP_QP_ORD_VALID
;
3055 issue_modify_qp
= 1;
3056 nesqp
->hw_tcp_state
= NES_AEQE_TCP_STATE_ESTABLISHED
;
3057 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_RTS
;
3058 nesqp
->hte_added
= 1;
3061 issue_modify_qp
= 1;
3062 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n",
3063 nesqp
->hwqp
.qp_id
, nesqp
->hwqp
.sq_head
, nesqp
->hwqp
.sq_tail
);
3064 if (nesqp
->iwarp_state
== (u32
)NES_CQP_QP_IWARP_STATE_CLOSING
) {
3065 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3068 if (nesqp
->iwarp_state
> (u32
)NES_CQP_QP_IWARP_STATE_CLOSING
) {
3069 nes_debug(NES_DBG_MOD_QP
, "QP%u: State change to closing"
3070 " ignored due to current iWARP state\n",
3072 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3075 if (nesqp
->hw_iwarp_state
!= NES_AEQE_IWARP_STATE_RTS
) {
3076 nes_debug(NES_DBG_MOD_QP
, "QP%u: State change to closing"
3077 " already done based on hw state.\n",
3079 issue_modify_qp
= 0;
3081 switch (nesqp
->hw_iwarp_state
) {
3082 case NES_AEQE_IWARP_STATE_CLOSING
:
3083 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_CLOSING
;
3085 case NES_AEQE_IWARP_STATE_TERMINATE
:
3086 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_TERMINATE
;
3088 case NES_AEQE_IWARP_STATE_ERROR
:
3089 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_ERROR
;
3092 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_CLOSING
;
3093 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_CLOSING
;
3099 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = terminate\n",
3101 if (nesqp
->iwarp_state
>=(u32
)NES_CQP_QP_IWARP_STATE_TERMINATE
) {
3102 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3105 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
3106 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_TERMINATE
;
3107 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_TERMINATE
;
3108 issue_modify_qp
= 1;
3112 if (nesqp
->iwarp_state
== (u32
)NES_CQP_QP_IWARP_STATE_ERROR
) {
3113 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3116 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = error\n",
3118 if (nesqp
->term_flags
)
3119 del_timer(&nesqp
->terminate_timer
);
3121 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_ERROR
;
3122 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
3123 if (nesqp
->hte_added
) {
3124 nes_debug(NES_DBG_MOD_QP
, "set CQP_QP_DEL_HTE\n");
3125 next_iwarp_state
|= NES_CQP_QP_DEL_HTE
;
3126 nesqp
->hte_added
= 0;
3128 if ((nesqp
->hw_tcp_state
> NES_AEQE_TCP_STATE_CLOSED
) &&
3129 (nesdev
->iw_status
) &&
3130 (nesqp
->hw_tcp_state
!= NES_AEQE_TCP_STATE_TIME_WAIT
)) {
3131 next_iwarp_state
|= NES_CQP_QP_RESET
;
3133 nes_debug(NES_DBG_MOD_QP
, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
3134 nesqp
->hwqp
.qp_id
, nesqp
->hw_tcp_state
);
3137 issue_modify_qp
= 1;
3138 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_ERROR
;
3141 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3146 nesqp
->ibqp_state
= attr
->qp_state
;
3147 nesqp
->iwarp_state
= next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
;
3148 nes_debug(NES_DBG_MOD_QP
, "Change nesqp->iwarp_state=%08x\n",
3149 nesqp
->iwarp_state
);
3152 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
3153 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
) {
3154 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
|
3155 NES_QPCONTEXT_MISC_RDMA_READ_EN
);
3156 issue_modify_qp
= 1;
3158 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
) {
3159 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
);
3160 issue_modify_qp
= 1;
3162 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
) {
3163 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN
);
3164 issue_modify_qp
= 1;
3166 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
) {
3167 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN
);
3168 issue_modify_qp
= 1;
3171 if (nesqp
->user_mode
) {
3172 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
|
3173 NES_QPCONTEXT_MISC_RDMA_READ_EN
);
3174 issue_modify_qp
= 1;
3178 original_last_aeq
= nesqp
->last_aeq
;
3179 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3181 nes_debug(NES_DBG_MOD_QP
, "issue_modify_qp=%u\n", issue_modify_qp
);
3186 if (issue_modify_qp
) {
3187 nes_debug(NES_DBG_MOD_QP
, "call nes_hw_modify_qp\n");
3188 ret
= nes_hw_modify_qp(nesdev
, nesqp
, next_iwarp_state
, 0, 1);
3190 nes_debug(NES_DBG_MOD_QP
, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)"
3191 " failed for QP%u.\n",
3192 next_iwarp_state
, nesqp
->hwqp
.qp_id
);
3196 if ((issue_modify_qp
) && (nesqp
->ibqp_state
> IB_QPS_RTS
)) {
3197 nes_debug(NES_DBG_MOD_QP
, "QP%u Issued ModifyQP refcount (%d),"
3198 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3199 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3200 original_last_aeq
, nesqp
->last_aeq
);
3201 if (!ret
|| original_last_aeq
!= NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE
) {
3203 if (nesqp
->cm_id
&& nesqp
->hw_tcp_state
!= 0) {
3204 nes_debug(NES_DBG_MOD_QP
, "QP%u Queuing fake disconnect for QP refcount (%d),"
3205 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3206 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3207 original_last_aeq
, nesqp
->last_aeq
);
3208 /* this one is for the cm_disconnect thread */
3209 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
3210 nesqp
->hw_tcp_state
= NES_AEQE_TCP_STATE_CLOSED
;
3211 nesqp
->last_aeq
= NES_AEQE_AEID_RESET_SENT
;
3212 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3213 nes_cm_disconn(nesqp
);
3215 nes_debug(NES_DBG_MOD_QP
, "QP%u No fake disconnect, QP refcount=%d\n",
3216 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
3219 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
3221 /* These two are for the timer thread */
3222 if (atomic_inc_return(&nesqp
->close_timer_started
) == 1) {
3223 nesqp
->cm_id
->add_ref(nesqp
->cm_id
);
3224 nes_debug(NES_DBG_MOD_QP
, "QP%u Not decrementing QP refcount (%d),"
3225 " need ae to finish up, original_last_aeq = 0x%04X."
3226 " last_aeq = 0x%04X, scheduling timer.\n",
3227 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3228 original_last_aeq
, nesqp
->last_aeq
);
3229 schedule_nes_timer(nesqp
->cm_node
, (struct sk_buff
*) nesqp
, NES_TIMER_TYPE_CLOSE
, 1, 0);
3231 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3233 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3234 nes_debug(NES_DBG_MOD_QP
, "QP%u Not decrementing QP refcount (%d),"
3235 " need ae to finish up, original_last_aeq = 0x%04X."
3236 " last_aeq = 0x%04X.\n",
3237 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3238 original_last_aeq
, nesqp
->last_aeq
);
3242 nes_debug(NES_DBG_MOD_QP
, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
3243 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3244 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3245 original_last_aeq
, nesqp
->last_aeq
);
3248 nes_debug(NES_DBG_MOD_QP
, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
3249 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3250 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3251 original_last_aeq
, nesqp
->last_aeq
);
3256 nes_debug(NES_DBG_MOD_QP
, "QP%u Leaving, refcount=%d\n",
3257 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
3264 * nes_muticast_attach
3266 static int nes_multicast_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
3268 nes_debug(NES_DBG_INIT
, "\n");
3274 * nes_multicast_detach
3276 static int nes_multicast_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
3278 nes_debug(NES_DBG_INIT
, "\n");
3286 static int nes_process_mad(struct ib_device
*ibdev
, int mad_flags
,
3287 u8 port_num
, const struct ib_wc
*in_wc
, const struct ib_grh
*in_grh
,
3288 const struct ib_mad_hdr
*in
, size_t in_mad_size
,
3289 struct ib_mad_hdr
*out
, size_t *out_mad_size
,
3290 u16
*out_mad_pkey_index
)
3292 nes_debug(NES_DBG_INIT
, "\n");
3297 fill_wqe_sg_send(struct nes_hw_qp_wqe
*wqe
, struct ib_send_wr
*ib_wr
, u32 uselkey
)
3300 int total_payload_length
= 0;
3301 for (sge_index
= 0; sge_index
< ib_wr
->num_sge
; sge_index
++) {
3302 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX
+(sge_index
*4),
3303 ib_wr
->sg_list
[sge_index
].addr
);
3304 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_LENGTH0_IDX
+ (sge_index
*4),
3305 ib_wr
->sg_list
[sge_index
].length
);
3307 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
+ (sge_index
*4),
3308 (ib_wr
->sg_list
[sge_index
].lkey
));
3310 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
+ (sge_index
*4), 0);
3312 total_payload_length
+= ib_wr
->sg_list
[sge_index
].length
;
3314 nes_debug(NES_DBG_IW_TX
, "UC UC UC, sending total_payload_length=%u \n",
3315 total_payload_length
);
3316 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3317 total_payload_length
);
3323 static int nes_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*ib_wr
,
3324 struct ib_send_wr
**bad_wr
)
3327 unsigned long flags
= 0;
3328 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
3329 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3330 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3331 struct nes_hw_qp_wqe
*wqe
;
3333 u32 qsize
= nesqp
->hwqp
.sq_size
;
3339 if (nesqp
->ibqp_state
> IB_QPS_RTS
) {
3344 spin_lock_irqsave(&nesqp
->lock
, flags
);
3346 head
= nesqp
->hwqp
.sq_head
;
3349 /* Check for QP error */
3350 if (nesqp
->term_flags
) {
3355 /* Check for SQ overflow */
3356 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.sq_tail
) % qsize
) == (qsize
- 1)) {
3361 wqe
= &nesqp
->hwqp
.sq_vbase
[head
];
3362 /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n",
3363 nesqp->hwqp.qp_id, wqe, head); */
3364 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
3365 u64temp
= (u64
)(ib_wr
->wr_id
);
3366 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
,
3368 switch (ib_wr
->opcode
) {
3370 case IB_WR_SEND_WITH_INV
:
3371 if (IB_WR_SEND
== ib_wr
->opcode
) {
3372 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3373 wqe_misc
= NES_IWARP_SQ_OP_SENDSE
;
3375 wqe_misc
= NES_IWARP_SQ_OP_SEND
;
3377 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3378 wqe_misc
= NES_IWARP_SQ_OP_SENDSEINV
;
3380 wqe_misc
= NES_IWARP_SQ_OP_SENDINV
;
3382 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX
,
3383 ib_wr
->ex
.invalidate_rkey
);
3386 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3391 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3392 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
3394 if ((ib_wr
->send_flags
& IB_SEND_INLINE
) &&
3395 ((nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) == 0) &&
3396 (ib_wr
->sg_list
[0].length
<= 64)) {
3397 memcpy(&wqe
->wqe_words
[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX
],
3398 (void *)(unsigned long)ib_wr
->sg_list
[0].addr
, ib_wr
->sg_list
[0].length
);
3399 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3400 ib_wr
->sg_list
[0].length
);
3401 wqe_misc
|= NES_IWARP_SQ_WQE_IMM_DATA
;
3403 fill_wqe_sg_send(wqe
, ib_wr
, 1);
3407 case IB_WR_RDMA_WRITE
:
3408 wqe_misc
= NES_IWARP_SQ_OP_RDMAW
;
3409 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3410 nes_debug(NES_DBG_IW_TX
, "Exceeded max sge, ib_wr=%u, max=%u\n",
3411 ib_wr
->num_sge
, nesdev
->nesadapter
->max_sge
);
3416 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3417 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
3419 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_STAG_IDX
,
3420 rdma_wr(ib_wr
)->rkey
);
3421 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX
,
3422 rdma_wr(ib_wr
)->remote_addr
);
3424 if ((ib_wr
->send_flags
& IB_SEND_INLINE
) &&
3425 ((nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) == 0) &&
3426 (ib_wr
->sg_list
[0].length
<= 64)) {
3427 memcpy(&wqe
->wqe_words
[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX
],
3428 (void *)(unsigned long)ib_wr
->sg_list
[0].addr
, ib_wr
->sg_list
[0].length
);
3429 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3430 ib_wr
->sg_list
[0].length
);
3431 wqe_misc
|= NES_IWARP_SQ_WQE_IMM_DATA
;
3433 fill_wqe_sg_send(wqe
, ib_wr
, 1);
3436 wqe
->wqe_words
[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
] =
3437 wqe
->wqe_words
[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
];
3439 case IB_WR_RDMA_READ
:
3440 case IB_WR_RDMA_READ_WITH_INV
:
3441 /* iWARP only supports 1 sge for RDMA reads */
3442 if (ib_wr
->num_sge
> 1) {
3443 nes_debug(NES_DBG_IW_TX
, "Exceeded max sge, ib_wr=%u, max=1\n",
3448 if (ib_wr
->opcode
== IB_WR_RDMA_READ
) {
3449 wqe_misc
= NES_IWARP_SQ_OP_RDMAR
;
3451 wqe_misc
= NES_IWARP_SQ_OP_RDMAR_LOCINV
;
3452 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX
,
3453 ib_wr
->ex
.invalidate_rkey
);
3456 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX
,
3457 rdma_wr(ib_wr
)->remote_addr
);
3458 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_STAG_IDX
,
3459 rdma_wr(ib_wr
)->rkey
);
3460 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
,
3461 ib_wr
->sg_list
->length
);
3462 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX
,
3463 ib_wr
->sg_list
->addr
);
3464 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
,
3465 ib_wr
->sg_list
->lkey
);
3467 case IB_WR_LOCAL_INV
:
3468 wqe_misc
= NES_IWARP_SQ_OP_LOCINV
;
3469 set_wqe_32bit_value(wqe
->wqe_words
,
3470 NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX
,
3471 ib_wr
->ex
.invalidate_rkey
);
3473 case IB_WR_FAST_REG_MR
:
3476 struct ib_fast_reg_wr
*fwr
= fast_reg_wr(ib_wr
);
3477 int flags
= fwr
->access_flags
;
3478 struct nes_ib_fast_reg_page_list
*pnesfrpl
=
3479 container_of(fwr
->page_list
,
3480 struct nes_ib_fast_reg_page_list
,
3482 u64
*src_page_list
= pnesfrpl
->ibfrpl
.page_list
;
3483 u64
*dst_page_list
= pnesfrpl
->nes_wqe_pbl
.kva
;
3485 if (fwr
->page_list_len
>
3486 (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
))) {
3487 nes_debug(NES_DBG_IW_TX
, "SQ_FMR: bad page_list_len\n");
3491 wqe_misc
= NES_IWARP_SQ_OP_FAST_REG
;
3492 set_wqe_64bit_value(wqe
->wqe_words
,
3493 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX
,
3495 set_wqe_32bit_value(wqe
->wqe_words
,
3496 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX
,
3498 set_wqe_32bit_value(wqe
->wqe_words
,
3499 NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX
, 0);
3500 set_wqe_32bit_value(wqe
->wqe_words
,
3501 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX
,
3503 /* Set page size: */
3504 if (fwr
->page_shift
== 12) {
3505 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K
;
3506 } else if (fwr
->page_shift
== 21) {
3507 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M
;
3509 nes_debug(NES_DBG_IW_TX
, "Invalid page shift,"
3510 " ib_wr=%u, max=1\n", ib_wr
->num_sge
);
3514 /* Set access_flags */
3515 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ
;
3516 if (flags
& IB_ACCESS_LOCAL_WRITE
)
3517 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE
;
3519 if (flags
& IB_ACCESS_REMOTE_WRITE
)
3520 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE
;
3522 if (flags
& IB_ACCESS_REMOTE_READ
)
3523 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ
;
3525 if (flags
& IB_ACCESS_MW_BIND
)
3526 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND
;
3528 /* Fill in PBL info: */
3529 if (fwr
->page_list_len
>
3530 pnesfrpl
->ibfrpl
.max_page_list_len
) {
3531 nes_debug(NES_DBG_IW_TX
, "Invalid page list length,"
3532 " ib_wr=%p, value=%u, max=%u\n",
3533 ib_wr
, fwr
->page_list_len
,
3534 pnesfrpl
->ibfrpl
.max_page_list_len
);
3539 set_wqe_64bit_value(wqe
->wqe_words
,
3540 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX
,
3541 pnesfrpl
->nes_wqe_pbl
.paddr
);
3543 set_wqe_32bit_value(wqe
->wqe_words
,
3544 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX
,
3545 fwr
->page_list_len
* 8);
3547 for (i
= 0; i
< fwr
->page_list_len
; i
++)
3548 dst_page_list
[i
] = cpu_to_le64(src_page_list
[i
]);
3550 nes_debug(NES_DBG_IW_TX
, "SQ_FMR: iova_start: %llx, "
3551 "length: %d, rkey: %0x, pgl_paddr: %llx, "
3552 "page_list_len: %u, wqe_misc: %x\n",
3553 (unsigned long long) fwr
->iova_start
,
3556 (unsigned long long) pnesfrpl
->nes_wqe_pbl
.paddr
,
3563 struct nes_mr
*mr
= to_nesmr(reg_wr(ib_wr
)->mr
);
3564 int page_shift
= ilog2(reg_wr(ib_wr
)->mr
->page_size
);
3565 int flags
= reg_wr(ib_wr
)->access
;
3567 if (mr
->npages
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
))) {
3568 nes_debug(NES_DBG_IW_TX
, "SQ_FMR: bad page_list_len\n");
3572 wqe_misc
= NES_IWARP_SQ_OP_FAST_REG
;
3573 set_wqe_64bit_value(wqe
->wqe_words
,
3574 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX
,
3576 set_wqe_32bit_value(wqe
->wqe_words
,
3577 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX
,
3579 set_wqe_32bit_value(wqe
->wqe_words
,
3580 NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX
, 0);
3581 set_wqe_32bit_value(wqe
->wqe_words
,
3582 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX
,
3583 reg_wr(ib_wr
)->key
);
3585 if (page_shift
== 12) {
3586 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K
;
3587 } else if (page_shift
== 21) {
3588 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M
;
3590 nes_debug(NES_DBG_IW_TX
, "Invalid page shift,"
3591 " ib_wr=%u, max=1\n", ib_wr
->num_sge
);
3596 /* Set access_flags */
3597 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ
;
3598 if (flags
& IB_ACCESS_LOCAL_WRITE
)
3599 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE
;
3601 if (flags
& IB_ACCESS_REMOTE_WRITE
)
3602 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE
;
3604 if (flags
& IB_ACCESS_REMOTE_READ
)
3605 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ
;
3607 if (flags
& IB_ACCESS_MW_BIND
)
3608 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND
;
3610 /* Fill in PBL info: */
3611 set_wqe_64bit_value(wqe
->wqe_words
,
3612 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX
,
3615 set_wqe_32bit_value(wqe
->wqe_words
,
3616 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX
,
3619 nes_debug(NES_DBG_IW_TX
, "SQ_REG_MR: iova_start: %llx, "
3620 "length: %d, rkey: %0x, pgl_paddr: %llx, "
3621 "page_list_len: %u, wqe_misc: %x\n",
3622 (unsigned long long) mr
->ibmr
.iova
,
3625 (unsigned long long) mr
->paddr
,
3639 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || nesqp
->sig_all
)
3640 wqe_misc
|= NES_IWARP_SQ_WQE_SIGNALED_COMPL
;
3642 wqe
->wqe_words
[NES_IWARP_SQ_WQE_MISC_IDX
] = cpu_to_le32(wqe_misc
);
3644 ib_wr
= ib_wr
->next
;
3652 nesqp
->hwqp
.sq_head
= head
;
3655 counter
= min(wqe_count
, ((u32
)255));
3656 wqe_count
-= counter
;
3657 nes_write32(nesdev
->regs
+ NES_WQE_ALLOC
,
3658 (counter
<< 24) | 0x00800000 | nesqp
->hwqp
.qp_id
);
3661 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
3673 static int nes_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*ib_wr
,
3674 struct ib_recv_wr
**bad_wr
)
3677 unsigned long flags
= 0;
3678 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
3679 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3680 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3681 struct nes_hw_qp_wqe
*wqe
;
3684 u32 qsize
= nesqp
->hwqp
.rq_size
;
3688 u32 total_payload_length
;
3690 if (nesqp
->ibqp_state
> IB_QPS_RTS
) {
3695 spin_lock_irqsave(&nesqp
->lock
, flags
);
3697 head
= nesqp
->hwqp
.rq_head
;
3700 /* Check for QP error */
3701 if (nesqp
->term_flags
) {
3706 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3710 /* Check for RQ overflow */
3711 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.rq_tail
) % qsize
) == (qsize
- 1)) {
3716 nes_debug(NES_DBG_IW_RX
, "ibwr sge count = %u.\n", ib_wr
->num_sge
);
3717 wqe
= &nesqp
->hwqp
.rq_vbase
[head
];
3719 /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n",
3720 nesqp->hwqp.qp_id, wqe, head); */
3721 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
3722 u64temp
= (u64
)(ib_wr
->wr_id
);
3723 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
,
3725 total_payload_length
= 0;
3726 for (sge_index
=0; sge_index
< ib_wr
->num_sge
; sge_index
++) {
3727 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX
+(sge_index
*4),
3728 ib_wr
->sg_list
[sge_index
].addr
);
3729 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_LENGTH0_IDX
+(sge_index
*4),
3730 ib_wr
->sg_list
[sge_index
].length
);
3731 set_wqe_32bit_value(wqe
->wqe_words
,NES_IWARP_RQ_WQE_STAG0_IDX
+(sge_index
*4),
3732 ib_wr
->sg_list
[sge_index
].lkey
);
3734 total_payload_length
+= ib_wr
->sg_list
[sge_index
].length
;
3736 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX
,
3737 total_payload_length
);
3739 ib_wr
= ib_wr
->next
;
3746 nesqp
->hwqp
.rq_head
= head
;
3749 counter
= min(wqe_count
, ((u32
)255));
3750 wqe_count
-= counter
;
3751 nes_write32(nesdev
->regs
+NES_WQE_ALLOC
, (counter
<<24) | nesqp
->hwqp
.qp_id
);
3754 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
3766 static int nes_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*entry
)
3770 unsigned long flags
= 0;
3771 struct nes_vnic
*nesvnic
= to_nesvnic(ibcq
->device
);
3772 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3773 struct nes_cq
*nescq
= to_nescq(ibcq
);
3774 struct nes_qp
*nesqp
;
3775 struct nes_hw_cqe cqe
;
3782 u32 move_cq_head
= 1;
3785 nes_debug(NES_DBG_CQ
, "\n");
3787 spin_lock_irqsave(&nescq
->lock
, flags
);
3789 head
= nescq
->hw_cq
.cq_head
;
3790 cq_size
= nescq
->hw_cq
.cq_size
;
3792 while (cqe_count
< num_entries
) {
3793 if ((le32_to_cpu(nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_OPCODE_IDX
]) &
3794 NES_CQE_VALID
) == 0)
3798 * Make sure we read CQ entry contents *after*
3799 * we've checked the valid bit.
3803 cqe
= nescq
->hw_cq
.cq_vbase
[head
];
3804 u32temp
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
3805 wqe_index
= u32temp
& (nesdev
->nesadapter
->max_qp_wr
- 1);
3806 u32temp
&= ~(NES_SW_CONTEXT_ALIGN
-1);
3807 /* parse CQE, get completion context from WQE (either rq or sq) */
3808 u64temp
= (((u64
)(le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
])))<<32) |
3812 nesqp
= (struct nes_qp
*)(unsigned long)u64temp
;
3813 memset(entry
, 0, sizeof *entry
);
3814 if (cqe
.cqe_words
[NES_CQE_ERROR_CODE_IDX
] == 0) {
3815 entry
->status
= IB_WC_SUCCESS
;
3817 err_code
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_ERROR_CODE_IDX
]);
3818 if (NES_IWARP_CQE_MAJOR_DRV
== (err_code
>> 16)) {
3819 entry
->status
= err_code
& 0x0000ffff;
3821 /* The rest of the cqe's will be marked as flushed */
3822 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_ERROR_CODE_IDX
] =
3823 cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH
<< 16) |
3824 NES_IWARP_CQE_MINOR_FLUSH
);
3826 entry
->status
= IB_WC_WR_FLUSH_ERR
;
3829 entry
->qp
= &nesqp
->ibqp
;
3830 entry
->src_qp
= nesqp
->hwqp
.qp_id
;
3832 if (le32_to_cpu(cqe
.cqe_words
[NES_CQE_OPCODE_IDX
]) & NES_CQE_SQ
) {
3833 if (nesqp
->skip_lsmm
) {
3834 nesqp
->skip_lsmm
= 0;
3835 nesqp
->hwqp
.sq_tail
++;
3838 /* Working on a SQ Completion*/
3839 wrid
= (((u64
)(cpu_to_le32((u32
)nesqp
->hwqp
.sq_vbase
[wqe_index
].
3840 wqe_words
[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX
]))) << 32) |
3841 ((u64
)(cpu_to_le32((u32
)nesqp
->hwqp
.sq_vbase
[wqe_index
].
3842 wqe_words
[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
])));
3843 entry
->byte_len
= le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3844 wqe_words
[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
]);
3846 switch (le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3847 wqe_words
[NES_IWARP_SQ_WQE_MISC_IDX
]) & 0x3f) {
3848 case NES_IWARP_SQ_OP_RDMAW
:
3849 nes_debug(NES_DBG_CQ
, "Operation = RDMA WRITE.\n");
3850 entry
->opcode
= IB_WC_RDMA_WRITE
;
3852 case NES_IWARP_SQ_OP_RDMAR
:
3853 nes_debug(NES_DBG_CQ
, "Operation = RDMA READ.\n");
3854 entry
->opcode
= IB_WC_RDMA_READ
;
3855 entry
->byte_len
= le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3856 wqe_words
[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
]);
3858 case NES_IWARP_SQ_OP_SENDINV
:
3859 case NES_IWARP_SQ_OP_SENDSEINV
:
3860 case NES_IWARP_SQ_OP_SEND
:
3861 case NES_IWARP_SQ_OP_SENDSE
:
3862 nes_debug(NES_DBG_CQ
, "Operation = Send.\n");
3863 entry
->opcode
= IB_WC_SEND
;
3865 case NES_IWARP_SQ_OP_LOCINV
:
3866 entry
->opcode
= IB_WC_LOCAL_INV
;
3868 case NES_IWARP_SQ_OP_FAST_REG
:
3869 entry
->opcode
= IB_WC_FAST_REG_MR
;
3873 nesqp
->hwqp
.sq_tail
= (wqe_index
+1)&(nesqp
->hwqp
.sq_size
- 1);
3874 if ((entry
->status
!= IB_WC_SUCCESS
) && (nesqp
->hwqp
.sq_tail
!= nesqp
->hwqp
.sq_head
)) {
3876 wq_tail
= nesqp
->hwqp
.sq_tail
;
3879 /* Working on a RQ Completion*/
3880 entry
->byte_len
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_PAYLOAD_LENGTH_IDX
]);
3881 wrid
= ((u64
)(le32_to_cpu(nesqp
->hwqp
.rq_vbase
[wqe_index
].wqe_words
[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX
]))) |
3882 ((u64
)(le32_to_cpu(nesqp
->hwqp
.rq_vbase
[wqe_index
].wqe_words
[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX
]))<<32);
3883 entry
->opcode
= IB_WC_RECV
;
3885 nesqp
->hwqp
.rq_tail
= (wqe_index
+1)&(nesqp
->hwqp
.rq_size
- 1);
3886 if ((entry
->status
!= IB_WC_SUCCESS
) && (nesqp
->hwqp
.rq_tail
!= nesqp
->hwqp
.rq_head
)) {
3888 wq_tail
= nesqp
->hwqp
.rq_tail
;
3892 entry
->wr_id
= wrid
;
3898 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_OPCODE_IDX
] = 0;
3899 if (++head
>= cq_size
)
3901 nescq
->polled_completions
++;
3903 if ((nescq
->polled_completions
> (cq_size
/ 2)) ||
3904 (nescq
->polled_completions
== 255)) {
3905 nes_debug(NES_DBG_CQ
, "CQ%u Issuing CQE Allocate since more than half of cqes"
3906 " are pending %u of %u.\n",
3907 nescq
->hw_cq
.cq_number
, nescq
->polled_completions
, cq_size
);
3908 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
,
3909 nescq
->hw_cq
.cq_number
| (nescq
->polled_completions
<< 16));
3910 nescq
->polled_completions
= 0;
3913 /* Update the wqe index and set status to flush */
3914 wqe_index
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
3915 wqe_index
= (wqe_index
& (~(nesdev
->nesadapter
->max_qp_wr
- 1))) | wq_tail
;
3916 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
] =
3917 cpu_to_le32(wqe_index
);
3918 move_cq_head
= 1; /* ready for next pass */
3922 if (nescq
->polled_completions
) {
3923 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
,
3924 nescq
->hw_cq
.cq_number
| (nescq
->polled_completions
<< 16));
3925 nescq
->polled_completions
= 0;
3928 nescq
->hw_cq
.cq_head
= head
;
3929 nes_debug(NES_DBG_CQ
, "Reporting %u completions for CQ%u.\n",
3930 cqe_count
, nescq
->hw_cq
.cq_number
);
3932 spin_unlock_irqrestore(&nescq
->lock
, flags
);
3941 static int nes_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags notify_flags
)
3943 struct nes_vnic
*nesvnic
= to_nesvnic(ibcq
->device
);
3944 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3945 struct nes_cq
*nescq
= to_nescq(ibcq
);
3948 nes_debug(NES_DBG_CQ
, "Requesting notification for CQ%u.\n",
3949 nescq
->hw_cq
.cq_number
);
3951 cq_arm
= nescq
->hw_cq
.cq_number
;
3952 if ((notify_flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_NEXT_COMP
)
3953 cq_arm
|= NES_CQE_ALLOC_NOTIFY_NEXT
;
3954 else if ((notify_flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
)
3955 cq_arm
|= NES_CQE_ALLOC_NOTIFY_SE
;
3959 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
, cq_arm
);
3960 nes_read32(nesdev
->regs
+NES_CQE_ALLOC
);
3965 static int nes_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
3966 struct ib_port_immutable
*immutable
)
3968 struct ib_port_attr attr
;
3971 err
= nes_query_port(ibdev
, port_num
, &attr
);
3975 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
3976 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
3977 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
3983 * nes_init_ofa_device
3985 struct nes_ib_device
*nes_init_ofa_device(struct net_device
*netdev
)
3987 struct nes_ib_device
*nesibdev
;
3988 struct nes_vnic
*nesvnic
= netdev_priv(netdev
);
3989 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3991 nesibdev
= (struct nes_ib_device
*)ib_alloc_device(sizeof(struct nes_ib_device
));
3992 if (nesibdev
== NULL
) {
3995 strlcpy(nesibdev
->ibdev
.name
, "nes%d", IB_DEVICE_NAME_MAX
);
3996 nesibdev
->ibdev
.owner
= THIS_MODULE
;
3998 nesibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
3999 memset(&nesibdev
->ibdev
.node_guid
, 0, sizeof(nesibdev
->ibdev
.node_guid
));
4000 memcpy(&nesibdev
->ibdev
.node_guid
, netdev
->dev_addr
, 6);
4002 nesibdev
->ibdev
.uverbs_cmd_mask
=
4003 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
4004 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
4005 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
4006 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
4007 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
4008 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
4009 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
4010 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
4011 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
4012 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
4013 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
4014 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
4015 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
4016 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
4017 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
4018 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
4019 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
4020 (1ull << IB_USER_VERBS_CMD_ALLOC_MW
) |
4021 (1ull << IB_USER_VERBS_CMD_BIND_MW
) |
4022 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW
) |
4023 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
4024 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
4026 nesibdev
->ibdev
.phys_port_cnt
= 1;
4027 nesibdev
->ibdev
.num_comp_vectors
= 1;
4028 nesibdev
->ibdev
.dma_device
= &nesdev
->pcidev
->dev
;
4029 nesibdev
->ibdev
.dev
.parent
= &nesdev
->pcidev
->dev
;
4030 nesibdev
->ibdev
.query_device
= nes_query_device
;
4031 nesibdev
->ibdev
.query_port
= nes_query_port
;
4032 nesibdev
->ibdev
.query_pkey
= nes_query_pkey
;
4033 nesibdev
->ibdev
.query_gid
= nes_query_gid
;
4034 nesibdev
->ibdev
.alloc_ucontext
= nes_alloc_ucontext
;
4035 nesibdev
->ibdev
.dealloc_ucontext
= nes_dealloc_ucontext
;
4036 nesibdev
->ibdev
.mmap
= nes_mmap
;
4037 nesibdev
->ibdev
.alloc_pd
= nes_alloc_pd
;
4038 nesibdev
->ibdev
.dealloc_pd
= nes_dealloc_pd
;
4039 nesibdev
->ibdev
.create_ah
= nes_create_ah
;
4040 nesibdev
->ibdev
.destroy_ah
= nes_destroy_ah
;
4041 nesibdev
->ibdev
.create_qp
= nes_create_qp
;
4042 nesibdev
->ibdev
.modify_qp
= nes_modify_qp
;
4043 nesibdev
->ibdev
.query_qp
= nes_query_qp
;
4044 nesibdev
->ibdev
.destroy_qp
= nes_destroy_qp
;
4045 nesibdev
->ibdev
.create_cq
= nes_create_cq
;
4046 nesibdev
->ibdev
.destroy_cq
= nes_destroy_cq
;
4047 nesibdev
->ibdev
.poll_cq
= nes_poll_cq
;
4048 nesibdev
->ibdev
.get_dma_mr
= nes_get_dma_mr
;
4049 nesibdev
->ibdev
.reg_phys_mr
= nes_reg_phys_mr
;
4050 nesibdev
->ibdev
.reg_user_mr
= nes_reg_user_mr
;
4051 nesibdev
->ibdev
.dereg_mr
= nes_dereg_mr
;
4052 nesibdev
->ibdev
.alloc_mw
= nes_alloc_mw
;
4053 nesibdev
->ibdev
.dealloc_mw
= nes_dealloc_mw
;
4054 nesibdev
->ibdev
.bind_mw
= nes_bind_mw
;
4056 nesibdev
->ibdev
.alloc_mr
= nes_alloc_mr
;
4057 nesibdev
->ibdev
.map_mr_sg
= nes_map_mr_sg
;
4058 nesibdev
->ibdev
.alloc_fast_reg_page_list
= nes_alloc_fast_reg_page_list
;
4059 nesibdev
->ibdev
.free_fast_reg_page_list
= nes_free_fast_reg_page_list
;
4061 nesibdev
->ibdev
.attach_mcast
= nes_multicast_attach
;
4062 nesibdev
->ibdev
.detach_mcast
= nes_multicast_detach
;
4063 nesibdev
->ibdev
.process_mad
= nes_process_mad
;
4065 nesibdev
->ibdev
.req_notify_cq
= nes_req_notify_cq
;
4066 nesibdev
->ibdev
.post_send
= nes_post_send
;
4067 nesibdev
->ibdev
.post_recv
= nes_post_recv
;
4069 nesibdev
->ibdev
.iwcm
= kzalloc(sizeof(*nesibdev
->ibdev
.iwcm
), GFP_KERNEL
);
4070 if (nesibdev
->ibdev
.iwcm
== NULL
) {
4071 ib_dealloc_device(&nesibdev
->ibdev
);
4074 nesibdev
->ibdev
.iwcm
->add_ref
= nes_add_ref
;
4075 nesibdev
->ibdev
.iwcm
->rem_ref
= nes_rem_ref
;
4076 nesibdev
->ibdev
.iwcm
->get_qp
= nes_get_qp
;
4077 nesibdev
->ibdev
.iwcm
->connect
= nes_connect
;
4078 nesibdev
->ibdev
.iwcm
->accept
= nes_accept
;
4079 nesibdev
->ibdev
.iwcm
->reject
= nes_reject
;
4080 nesibdev
->ibdev
.iwcm
->create_listen
= nes_create_listen
;
4081 nesibdev
->ibdev
.iwcm
->destroy_listen
= nes_destroy_listen
;
4082 nesibdev
->ibdev
.get_port_immutable
= nes_port_immutable
;
4089 * nes_handle_delayed_event
4091 static void nes_handle_delayed_event(unsigned long data
)
4093 struct nes_vnic
*nesvnic
= (void *) data
;
4095 if (nesvnic
->delayed_event
!= nesvnic
->last_dispatched_event
) {
4096 struct ib_event event
;
4098 event
.device
= &nesvnic
->nesibdev
->ibdev
;
4101 event
.event
= nesvnic
->delayed_event
;
4102 event
.element
.port_num
= nesvnic
->logical_port
+ 1;
4103 ib_dispatch_event(&event
);
4107 nesvnic
->event_timer
.function
= NULL
;
4111 void nes_port_ibevent(struct nes_vnic
*nesvnic
)
4113 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
4114 struct nes_device
*nesdev
= nesvnic
->nesdev
;
4115 struct ib_event event
;
4116 event
.device
= &nesibdev
->ibdev
;
4117 event
.element
.port_num
= nesvnic
->logical_port
+ 1;
4118 event
.event
= nesdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
4120 if (!nesvnic
->event_timer
.function
) {
4121 ib_dispatch_event(&event
);
4122 nesvnic
->last_dispatched_event
= event
.event
;
4123 nesvnic
->event_timer
.function
= nes_handle_delayed_event
;
4124 nesvnic
->event_timer
.data
= (unsigned long) nesvnic
;
4125 nesvnic
->event_timer
.expires
= jiffies
+ NES_EVENT_DELAY
;
4126 add_timer(&nesvnic
->event_timer
);
4128 mod_timer(&nesvnic
->event_timer
, jiffies
+ NES_EVENT_DELAY
);
4130 nesvnic
->delayed_event
= event
.event
;
4135 * nes_destroy_ofa_device
4137 void nes_destroy_ofa_device(struct nes_ib_device
*nesibdev
)
4139 if (nesibdev
== NULL
)
4142 nes_unregister_ofa_device(nesibdev
);
4144 kfree(nesibdev
->ibdev
.iwcm
);
4145 ib_dealloc_device(&nesibdev
->ibdev
);
4150 * nes_register_ofa_device
4152 int nes_register_ofa_device(struct nes_ib_device
*nesibdev
)
4154 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
4155 struct nes_device
*nesdev
= nesvnic
->nesdev
;
4156 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
4159 ret
= ib_register_device(&nesvnic
->nesibdev
->ibdev
, NULL
);
4164 /* Get the resources allocated to this device */
4165 nesibdev
->max_cq
= (nesadapter
->max_cq
-NES_FIRST_QPN
) / nesadapter
->port_count
;
4166 nesibdev
->max_mr
= nesadapter
->max_mr
/ nesadapter
->port_count
;
4167 nesibdev
->max_qp
= (nesadapter
->max_qp
-NES_FIRST_QPN
) / nesadapter
->port_count
;
4168 nesibdev
->max_pd
= nesadapter
->max_pd
/ nesadapter
->port_count
;
4170 for (i
= 0; i
< ARRAY_SIZE(nes_dev_attributes
); ++i
) {
4171 ret
= device_create_file(&nesibdev
->ibdev
.dev
, nes_dev_attributes
[i
]);
4175 device_remove_file(&nesibdev
->ibdev
.dev
,
4176 nes_dev_attributes
[i
]);
4178 ib_unregister_device(&nesibdev
->ibdev
);
4183 nesvnic
->of_device_registered
= 1;
4190 * nes_unregister_ofa_device
4192 static void nes_unregister_ofa_device(struct nes_ib_device
*nesibdev
)
4194 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
4197 for (i
= 0; i
< ARRAY_SIZE(nes_dev_attributes
); ++i
) {
4198 device_remove_file(&nesibdev
->ibdev
.dev
, nes_dev_attributes
[i
]);
4201 if (nesvnic
->of_device_registered
) {
4202 ib_unregister_device(&nesibdev
->ibdev
);
4205 nesvnic
->of_device_registered
= 0;