2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 static int destroy_cq(struct c4iw_rdev
*rdev
, struct t4_cq
*cq
,
36 struct c4iw_dev_ucontext
*uctx
)
38 struct fw_ri_res_wr
*res_wr
;
39 struct fw_ri_res
*res
;
41 struct c4iw_wr_wait wr_wait
;
45 wr_len
= sizeof *res_wr
+ sizeof *res
;
46 skb
= alloc_skb(wr_len
, GFP_KERNEL
);
49 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
51 res_wr
= (struct fw_ri_res_wr
*)__skb_put(skb
, wr_len
);
52 memset(res_wr
, 0, wr_len
);
53 res_wr
->op_nres
= cpu_to_be32(
54 FW_WR_OP_V(FW_RI_RES_WR
) |
55 FW_RI_RES_WR_NRES_V(1) |
57 res_wr
->len16_pkd
= cpu_to_be32(DIV_ROUND_UP(wr_len
, 16));
58 res_wr
->cookie
= (uintptr_t)&wr_wait
;
60 res
->u
.cq
.restype
= FW_RI_RES_TYPE_CQ
;
61 res
->u
.cq
.op
= FW_RI_RES_OP_RESET
;
62 res
->u
.cq
.iqid
= cpu_to_be32(cq
->cqid
);
64 c4iw_init_wr_wait(&wr_wait
);
65 ret
= c4iw_ofld_send(rdev
, skb
);
67 ret
= c4iw_wait_for_reply(rdev
, &wr_wait
, 0, 0, __func__
);
71 dma_free_coherent(&(rdev
->lldi
.pdev
->dev
),
72 cq
->memsize
, cq
->queue
,
73 dma_unmap_addr(cq
, mapping
));
74 c4iw_put_cqid(rdev
, cq
->cqid
, uctx
);
78 static int create_cq(struct c4iw_rdev
*rdev
, struct t4_cq
*cq
,
79 struct c4iw_dev_ucontext
*uctx
)
81 struct fw_ri_res_wr
*res_wr
;
82 struct fw_ri_res
*res
;
84 int user
= (uctx
!= &rdev
->uctx
);
85 struct c4iw_wr_wait wr_wait
;
89 cq
->cqid
= c4iw_get_cqid(rdev
, uctx
);
96 cq
->sw_queue
= kzalloc(cq
->memsize
, GFP_KERNEL
);
102 cq
->queue
= dma_alloc_coherent(&rdev
->lldi
.pdev
->dev
, cq
->memsize
,
103 &cq
->dma_addr
, GFP_KERNEL
);
108 dma_unmap_addr_set(cq
, mapping
, cq
->dma_addr
);
109 memset(cq
->queue
, 0, cq
->memsize
);
111 /* build fw_ri_res_wr */
112 wr_len
= sizeof *res_wr
+ sizeof *res
;
114 skb
= alloc_skb(wr_len
, GFP_KERNEL
);
119 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
121 res_wr
= (struct fw_ri_res_wr
*)__skb_put(skb
, wr_len
);
122 memset(res_wr
, 0, wr_len
);
123 res_wr
->op_nres
= cpu_to_be32(
124 FW_WR_OP_V(FW_RI_RES_WR
) |
125 FW_RI_RES_WR_NRES_V(1) |
127 res_wr
->len16_pkd
= cpu_to_be32(DIV_ROUND_UP(wr_len
, 16));
128 res_wr
->cookie
= (uintptr_t)&wr_wait
;
130 res
->u
.cq
.restype
= FW_RI_RES_TYPE_CQ
;
131 res
->u
.cq
.op
= FW_RI_RES_OP_WRITE
;
132 res
->u
.cq
.iqid
= cpu_to_be32(cq
->cqid
);
133 res
->u
.cq
.iqandst_to_iqandstindex
= cpu_to_be32(
134 FW_RI_RES_WR_IQANUS_V(0) |
135 FW_RI_RES_WR_IQANUD_V(1) |
136 FW_RI_RES_WR_IQANDST_F
|
137 FW_RI_RES_WR_IQANDSTINDEX_V(
138 rdev
->lldi
.ciq_ids
[cq
->vector
]));
139 res
->u
.cq
.iqdroprss_to_iqesize
= cpu_to_be16(
140 FW_RI_RES_WR_IQDROPRSS_F
|
141 FW_RI_RES_WR_IQPCIECH_V(2) |
142 FW_RI_RES_WR_IQINTCNTTHRESH_V(0) |
144 FW_RI_RES_WR_IQESIZE_V(1));
145 res
->u
.cq
.iqsize
= cpu_to_be16(cq
->size
);
146 res
->u
.cq
.iqaddr
= cpu_to_be64(cq
->dma_addr
);
148 c4iw_init_wr_wait(&wr_wait
);
150 ret
= c4iw_ofld_send(rdev
, skb
);
153 PDBG("%s wait_event wr_wait %p\n", __func__
, &wr_wait
);
154 ret
= c4iw_wait_for_reply(rdev
, &wr_wait
, 0, 0, __func__
);
161 u32 off
= (cq
->cqid
<< rdev
->cqshift
) & PAGE_MASK
;
163 cq
->ugts
= (u64
)rdev
->bar2_pa
+ off
;
164 } else if (is_t4(rdev
->lldi
.adapter_type
)) {
165 cq
->gts
= rdev
->lldi
.gts_reg
;
168 u32 off
= ((cq
->cqid
<< rdev
->cqshift
) & PAGE_MASK
) + 12;
170 cq
->gts
= rdev
->bar2_kva
+ off
;
171 cq
->qid_mask
= rdev
->qpmask
;
175 dma_free_coherent(&rdev
->lldi
.pdev
->dev
, cq
->memsize
, cq
->queue
,
176 dma_unmap_addr(cq
, mapping
));
180 c4iw_put_cqid(rdev
, cq
->cqid
, uctx
);
185 static void insert_recv_cqe(struct t4_wq
*wq
, struct t4_cq
*cq
)
189 PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__
,
190 wq
, cq
, cq
->sw_cidx
, cq
->sw_pidx
);
191 memset(&cqe
, 0, sizeof(cqe
));
192 cqe
.header
= cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH
) |
193 CQE_OPCODE_V(FW_RI_SEND
) |
196 CQE_QPID_V(wq
->sq
.qid
));
197 cqe
.bits_type_ts
= cpu_to_be64(CQE_GENBIT_V((u64
)cq
->gen
));
198 cq
->sw_queue
[cq
->sw_pidx
] = cqe
;
202 int c4iw_flush_rq(struct t4_wq
*wq
, struct t4_cq
*cq
, int count
)
205 int in_use
= wq
->rq
.in_use
- count
;
208 PDBG("%s wq %p cq %p rq.in_use %u skip count %u\n", __func__
,
209 wq
, cq
, wq
->rq
.in_use
, count
);
211 insert_recv_cqe(wq
, cq
);
217 static void insert_sq_cqe(struct t4_wq
*wq
, struct t4_cq
*cq
,
218 struct t4_swsqe
*swcqe
)
222 PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__
,
223 wq
, cq
, cq
->sw_cidx
, cq
->sw_pidx
);
224 memset(&cqe
, 0, sizeof(cqe
));
225 cqe
.header
= cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH
) |
226 CQE_OPCODE_V(swcqe
->opcode
) |
229 CQE_QPID_V(wq
->sq
.qid
));
230 CQE_WRID_SQ_IDX(&cqe
) = swcqe
->idx
;
231 cqe
.bits_type_ts
= cpu_to_be64(CQE_GENBIT_V((u64
)cq
->gen
));
232 cq
->sw_queue
[cq
->sw_pidx
] = cqe
;
236 static void advance_oldest_read(struct t4_wq
*wq
);
238 int c4iw_flush_sq(struct c4iw_qp
*qhp
)
241 struct t4_wq
*wq
= &qhp
->wq
;
242 struct c4iw_cq
*chp
= to_c4iw_cq(qhp
->ibqp
.send_cq
);
243 struct t4_cq
*cq
= &chp
->cq
;
245 struct t4_swsqe
*swsqe
;
247 if (wq
->sq
.flush_cidx
== -1)
248 wq
->sq
.flush_cidx
= wq
->sq
.cidx
;
249 idx
= wq
->sq
.flush_cidx
;
250 BUG_ON(idx
>= wq
->sq
.size
);
251 while (idx
!= wq
->sq
.pidx
) {
252 swsqe
= &wq
->sq
.sw_sq
[idx
];
253 BUG_ON(swsqe
->flushed
);
255 insert_sq_cqe(wq
, cq
, swsqe
);
256 if (wq
->sq
.oldest_read
== swsqe
) {
257 BUG_ON(swsqe
->opcode
!= FW_RI_READ_REQ
);
258 advance_oldest_read(wq
);
261 if (++idx
== wq
->sq
.size
)
264 wq
->sq
.flush_cidx
+= flushed
;
265 if (wq
->sq
.flush_cidx
>= wq
->sq
.size
)
266 wq
->sq
.flush_cidx
-= wq
->sq
.size
;
270 static void flush_completed_wrs(struct t4_wq
*wq
, struct t4_cq
*cq
)
272 struct t4_swsqe
*swsqe
;
275 if (wq
->sq
.flush_cidx
== -1)
276 wq
->sq
.flush_cidx
= wq
->sq
.cidx
;
277 cidx
= wq
->sq
.flush_cidx
;
278 BUG_ON(cidx
> wq
->sq
.size
);
280 while (cidx
!= wq
->sq
.pidx
) {
281 swsqe
= &wq
->sq
.sw_sq
[cidx
];
282 if (!swsqe
->signaled
) {
283 if (++cidx
== wq
->sq
.size
)
285 } else if (swsqe
->complete
) {
287 BUG_ON(swsqe
->flushed
);
290 * Insert this completed cqe into the swcq.
292 PDBG("%s moving cqe into swcq sq idx %u cq idx %u\n",
293 __func__
, cidx
, cq
->sw_pidx
);
294 swsqe
->cqe
.header
|= htonl(CQE_SWCQE_V(1));
295 cq
->sw_queue
[cq
->sw_pidx
] = swsqe
->cqe
;
298 if (++cidx
== wq
->sq
.size
)
300 wq
->sq
.flush_cidx
= cidx
;
306 static void create_read_req_cqe(struct t4_wq
*wq
, struct t4_cqe
*hw_cqe
,
307 struct t4_cqe
*read_cqe
)
309 read_cqe
->u
.scqe
.cidx
= wq
->sq
.oldest_read
->idx
;
310 read_cqe
->len
= htonl(wq
->sq
.oldest_read
->read_len
);
311 read_cqe
->header
= htonl(CQE_QPID_V(CQE_QPID(hw_cqe
)) |
312 CQE_SWCQE_V(SW_CQE(hw_cqe
)) |
313 CQE_OPCODE_V(FW_RI_READ_REQ
) |
315 read_cqe
->bits_type_ts
= hw_cqe
->bits_type_ts
;
318 static void advance_oldest_read(struct t4_wq
*wq
)
321 u32 rptr
= wq
->sq
.oldest_read
- wq
->sq
.sw_sq
+ 1;
323 if (rptr
== wq
->sq
.size
)
325 while (rptr
!= wq
->sq
.pidx
) {
326 wq
->sq
.oldest_read
= &wq
->sq
.sw_sq
[rptr
];
328 if (wq
->sq
.oldest_read
->opcode
== FW_RI_READ_REQ
)
330 if (++rptr
== wq
->sq
.size
)
333 wq
->sq
.oldest_read
= NULL
;
337 * Move all CQEs from the HWCQ into the SWCQ.
338 * Deal with out-of-order and/or completions that complete
339 * prior unsignalled WRs.
341 void c4iw_flush_hw_cq(struct c4iw_cq
*chp
)
343 struct t4_cqe
*hw_cqe
, *swcqe
, read_cqe
;
345 struct t4_swsqe
*swsqe
;
348 PDBG("%s cqid 0x%x\n", __func__
, chp
->cq
.cqid
);
349 ret
= t4_next_hw_cqe(&chp
->cq
, &hw_cqe
);
352 * This logic is similar to poll_cq(), but not quite the same
353 * unfortunately. Need to move pertinent HW CQEs to the SW CQ but
354 * also do any translation magic that poll_cq() normally does.
357 qhp
= get_qhp(chp
->rhp
, CQE_QPID(hw_cqe
));
360 * drop CQEs with no associated QP
365 if (CQE_OPCODE(hw_cqe
) == FW_RI_TERMINATE
)
368 if (CQE_OPCODE(hw_cqe
) == FW_RI_READ_RESP
) {
370 /* If we have reached here because of async
371 * event or other error, and have egress error
374 if (CQE_TYPE(hw_cqe
) == 1)
377 /* drop peer2peer RTR reads.
379 if (CQE_WRID_STAG(hw_cqe
) == 1)
383 * Eat completions for unsignaled read WRs.
385 if (!qhp
->wq
.sq
.oldest_read
->signaled
) {
386 advance_oldest_read(&qhp
->wq
);
391 * Don't write to the HWCQ, create a new read req CQE
392 * in local memory and move it into the swcq.
394 create_read_req_cqe(&qhp
->wq
, hw_cqe
, &read_cqe
);
396 advance_oldest_read(&qhp
->wq
);
399 /* if its a SQ completion, then do the magic to move all the
400 * unsignaled and now in-order completions into the swcq.
402 if (SQ_TYPE(hw_cqe
)) {
403 swsqe
= &qhp
->wq
.sq
.sw_sq
[CQE_WRID_SQ_IDX(hw_cqe
)];
404 swsqe
->cqe
= *hw_cqe
;
406 flush_completed_wrs(&qhp
->wq
, &chp
->cq
);
408 swcqe
= &chp
->cq
.sw_queue
[chp
->cq
.sw_pidx
];
410 swcqe
->header
|= cpu_to_be32(CQE_SWCQE_V(1));
411 t4_swcq_produce(&chp
->cq
);
414 t4_hwcq_consume(&chp
->cq
);
415 ret
= t4_next_hw_cqe(&chp
->cq
, &hw_cqe
);
419 static int cqe_completes_wr(struct t4_cqe
*cqe
, struct t4_wq
*wq
)
421 if (CQE_OPCODE(cqe
) == FW_RI_TERMINATE
)
424 if ((CQE_OPCODE(cqe
) == FW_RI_RDMA_WRITE
) && RQ_TYPE(cqe
))
427 if ((CQE_OPCODE(cqe
) == FW_RI_READ_RESP
) && SQ_TYPE(cqe
))
430 if (CQE_SEND_OPCODE(cqe
) && RQ_TYPE(cqe
) && t4_rq_empty(wq
))
435 void c4iw_count_rcqes(struct t4_cq
*cq
, struct t4_wq
*wq
, int *count
)
441 PDBG("%s count zero %d\n", __func__
, *count
);
443 while (ptr
!= cq
->sw_pidx
) {
444 cqe
= &cq
->sw_queue
[ptr
];
445 if (RQ_TYPE(cqe
) && (CQE_OPCODE(cqe
) != FW_RI_READ_RESP
) &&
446 (CQE_QPID(cqe
) == wq
->sq
.qid
) && cqe_completes_wr(cqe
, wq
))
448 if (++ptr
== cq
->size
)
451 PDBG("%s cq %p count %d\n", __func__
, cq
, *count
);
458 * check the validity of the first CQE,
459 * supply the wq assicated with the qpid.
461 * credit: cq credit to return to sge.
462 * cqe_flushed: 1 iff the CQE is flushed.
463 * cqe: copy of the polled CQE.
467 * -EAGAIN CQE skipped, try again.
468 * -EOVERFLOW CQ overflow detected.
470 static int poll_cq(struct t4_wq
*wq
, struct t4_cq
*cq
, struct t4_cqe
*cqe
,
471 u8
*cqe_flushed
, u64
*cookie
, u32
*credit
)
474 struct t4_cqe
*hw_cqe
, read_cqe
;
478 ret
= t4_next_cqe(cq
, &hw_cqe
);
482 PDBG("%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x"
483 " opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
484 __func__
, CQE_OVFBIT(hw_cqe
), CQE_QPID(hw_cqe
),
485 CQE_GENBIT(hw_cqe
), CQE_TYPE(hw_cqe
), CQE_STATUS(hw_cqe
),
486 CQE_OPCODE(hw_cqe
), CQE_LEN(hw_cqe
), CQE_WRID_HI(hw_cqe
),
487 CQE_WRID_LOW(hw_cqe
));
490 * skip cqe's not affiliated with a QP.
498 * skip hw cqe's if the wq is flushed.
500 if (wq
->flushed
&& !SW_CQE(hw_cqe
)) {
506 * skip TERMINATE cqes...
508 if (CQE_OPCODE(hw_cqe
) == FW_RI_TERMINATE
) {
514 * Gotta tweak READ completions:
515 * 1) the cqe doesn't contain the sq_wptr from the wr.
516 * 2) opcode not reflected from the wr.
517 * 3) read_len not reflected from the wr.
518 * 4) cq_type is RQ_TYPE not SQ_TYPE.
520 if (RQ_TYPE(hw_cqe
) && (CQE_OPCODE(hw_cqe
) == FW_RI_READ_RESP
)) {
522 /* If we have reached here because of async
523 * event or other error, and have egress error
526 if (CQE_TYPE(hw_cqe
) == 1) {
527 if (CQE_STATUS(hw_cqe
))
528 t4_set_wq_in_error(wq
);
533 /* If this is an unsolicited read response, then the read
534 * was generated by the kernel driver as part of peer-2-peer
535 * connection setup. So ignore the completion.
537 if (CQE_WRID_STAG(hw_cqe
) == 1) {
538 if (CQE_STATUS(hw_cqe
))
539 t4_set_wq_in_error(wq
);
545 * Eat completions for unsignaled read WRs.
547 if (!wq
->sq
.oldest_read
->signaled
) {
548 advance_oldest_read(wq
);
554 * Don't write to the HWCQ, so create a new read req CQE
557 create_read_req_cqe(wq
, hw_cqe
, &read_cqe
);
559 advance_oldest_read(wq
);
562 if (CQE_STATUS(hw_cqe
) || t4_wq_in_error(wq
)) {
563 *cqe_flushed
= (CQE_STATUS(hw_cqe
) == T4_ERR_SWFLUSH
);
564 t4_set_wq_in_error(wq
);
570 if (RQ_TYPE(hw_cqe
)) {
573 * HW only validates 4 bits of MSN. So we must validate that
574 * the MSN in the SEND is the next expected MSN. If its not,
575 * then we complete this with T4_ERR_MSN and mark the wq in
579 if (t4_rq_empty(wq
)) {
580 t4_set_wq_in_error(wq
);
584 if (unlikely((CQE_WRID_MSN(hw_cqe
) != (wq
->rq
.msn
)))) {
585 t4_set_wq_in_error(wq
);
586 hw_cqe
->header
|= htonl(CQE_STATUS_V(T4_ERR_MSN
));
593 * If we get here its a send completion.
595 * Handle out of order completion. These get stuffed
596 * in the SW SQ. Then the SW SQ is walked to move any
597 * now in-order completions into the SW CQ. This handles
599 * 1) reaping unsignaled WRs when the first subsequent
600 * signaled WR is completed.
601 * 2) out of order read completions.
603 if (!SW_CQE(hw_cqe
) && (CQE_WRID_SQ_IDX(hw_cqe
) != wq
->sq
.cidx
)) {
604 struct t4_swsqe
*swsqe
;
606 PDBG("%s out of order completion going in sw_sq at idx %u\n",
607 __func__
, CQE_WRID_SQ_IDX(hw_cqe
));
608 swsqe
= &wq
->sq
.sw_sq
[CQE_WRID_SQ_IDX(hw_cqe
)];
609 swsqe
->cqe
= *hw_cqe
;
619 * Reap the associated WR(s) that are freed up with this
622 if (SQ_TYPE(hw_cqe
)) {
623 int idx
= CQE_WRID_SQ_IDX(hw_cqe
);
624 BUG_ON(idx
>= wq
->sq
.size
);
627 * Account for any unsignaled completions completed by
628 * this signaled completion. In this case, cidx points
629 * to the first unsignaled one, and idx points to the
630 * signaled one. So adjust in_use based on this delta.
631 * if this is not completing any unsigned wrs, then the
632 * delta will be 0. Handle wrapping also!
634 if (idx
< wq
->sq
.cidx
)
635 wq
->sq
.in_use
-= wq
->sq
.size
+ idx
- wq
->sq
.cidx
;
637 wq
->sq
.in_use
-= idx
- wq
->sq
.cidx
;
638 BUG_ON(wq
->sq
.in_use
<= 0 && wq
->sq
.in_use
>= wq
->sq
.size
);
640 wq
->sq
.cidx
= (uint16_t)idx
;
641 PDBG("%s completing sq idx %u\n", __func__
, wq
->sq
.cidx
);
642 *cookie
= wq
->sq
.sw_sq
[wq
->sq
.cidx
].wr_id
;
644 c4iw_log_wr_stats(wq
, hw_cqe
);
647 PDBG("%s completing rq idx %u\n", __func__
, wq
->rq
.cidx
);
648 *cookie
= wq
->rq
.sw_rq
[wq
->rq
.cidx
].wr_id
;
649 BUG_ON(t4_rq_empty(wq
));
651 c4iw_log_wr_stats(wq
, hw_cqe
);
658 * Flush any completed cqes that are now in-order.
660 flush_completed_wrs(wq
, cq
);
663 if (SW_CQE(hw_cqe
)) {
664 PDBG("%s cq %p cqid 0x%x skip sw cqe cidx %u\n",
665 __func__
, cq
, cq
->cqid
, cq
->sw_cidx
);
668 PDBG("%s cq %p cqid 0x%x skip hw cqe cidx %u\n",
669 __func__
, cq
, cq
->cqid
, cq
->cidx
);
676 * Get one cq entry from c4iw and map it to openib.
681 * -EAGAIN caller must try again
682 * any other -errno fatal error
684 static int c4iw_poll_cq_one(struct c4iw_cq
*chp
, struct ib_wc
*wc
)
686 struct c4iw_qp
*qhp
= NULL
;
687 struct t4_cqe
uninitialized_var(cqe
), *rd_cqe
;
694 ret
= t4_next_cqe(&chp
->cq
, &rd_cqe
);
699 qhp
= get_qhp(chp
->rhp
, CQE_QPID(rd_cqe
));
703 spin_lock(&qhp
->lock
);
706 ret
= poll_cq(wq
, &(chp
->cq
), &cqe
, &cqe_flushed
, &cookie
, &credit
);
712 wc
->vendor_err
= CQE_STATUS(&cqe
);
715 PDBG("%s qpid 0x%x type %d opcode %d status 0x%x len %u wrid hi 0x%x "
716 "lo 0x%x cookie 0x%llx\n", __func__
, CQE_QPID(&cqe
),
717 CQE_TYPE(&cqe
), CQE_OPCODE(&cqe
), CQE_STATUS(&cqe
), CQE_LEN(&cqe
),
718 CQE_WRID_HI(&cqe
), CQE_WRID_LOW(&cqe
), (unsigned long long)cookie
);
720 if (CQE_TYPE(&cqe
) == 0) {
721 if (!CQE_STATUS(&cqe
))
722 wc
->byte_len
= CQE_LEN(&cqe
);
725 wc
->opcode
= IB_WC_RECV
;
726 if (CQE_OPCODE(&cqe
) == FW_RI_SEND_WITH_INV
||
727 CQE_OPCODE(&cqe
) == FW_RI_SEND_WITH_SE_INV
) {
728 wc
->ex
.invalidate_rkey
= CQE_WRID_STAG(&cqe
);
729 wc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
732 switch (CQE_OPCODE(&cqe
)) {
733 case FW_RI_RDMA_WRITE
:
734 wc
->opcode
= IB_WC_RDMA_WRITE
;
737 wc
->opcode
= IB_WC_RDMA_READ
;
738 wc
->byte_len
= CQE_LEN(&cqe
);
740 case FW_RI_SEND_WITH_INV
:
741 case FW_RI_SEND_WITH_SE_INV
:
742 wc
->opcode
= IB_WC_SEND
;
743 wc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
746 case FW_RI_SEND_WITH_SE
:
747 wc
->opcode
= IB_WC_SEND
;
750 wc
->opcode
= IB_WC_BIND_MW
;
753 case FW_RI_LOCAL_INV
:
754 wc
->opcode
= IB_WC_LOCAL_INV
;
756 case FW_RI_FAST_REGISTER
:
757 wc
->opcode
= IB_WC_FAST_REG_MR
;
760 printk(KERN_ERR MOD
"Unexpected opcode %d "
761 "in the CQE received for QPID=0x%0x\n",
762 CQE_OPCODE(&cqe
), CQE_QPID(&cqe
));
769 wc
->status
= IB_WC_WR_FLUSH_ERR
;
772 switch (CQE_STATUS(&cqe
)) {
774 wc
->status
= IB_WC_SUCCESS
;
777 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
780 wc
->status
= IB_WC_LOC_PROT_ERR
;
784 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
787 wc
->status
= IB_WC_GENERAL_ERR
;
790 wc
->status
= IB_WC_LOC_LEN_ERR
;
792 case T4_ERR_INVALIDATE_SHARED_MR
:
793 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND
:
794 wc
->status
= IB_WC_MW_BIND_ERR
;
798 case T4_ERR_PDU_LEN_ERR
:
799 case T4_ERR_OUT_OF_RQE
:
800 case T4_ERR_DDP_VERSION
:
801 case T4_ERR_RDMA_VERSION
:
802 case T4_ERR_DDP_QUEUE_NUM
:
806 case T4_ERR_MSN_RANGE
:
807 case T4_ERR_IRD_OVERFLOW
:
809 case T4_ERR_INTERNAL_ERR
:
810 wc
->status
= IB_WC_FATAL_ERR
;
813 wc
->status
= IB_WC_WR_FLUSH_ERR
;
817 "Unexpected cqe_status 0x%x for QPID=0x%0x\n",
818 CQE_STATUS(&cqe
), CQE_QPID(&cqe
));
824 spin_unlock(&qhp
->lock
);
828 int c4iw_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
835 chp
= to_c4iw_cq(ibcq
);
837 spin_lock_irqsave(&chp
->lock
, flags
);
838 for (npolled
= 0; npolled
< num_entries
; ++npolled
) {
840 err
= c4iw_poll_cq_one(chp
, wc
+ npolled
);
841 } while (err
== -EAGAIN
);
845 spin_unlock_irqrestore(&chp
->lock
, flags
);
846 return !err
|| err
== -ENODATA
? npolled
: err
;
849 int c4iw_destroy_cq(struct ib_cq
*ib_cq
)
852 struct c4iw_ucontext
*ucontext
;
854 PDBG("%s ib_cq %p\n", __func__
, ib_cq
);
855 chp
= to_c4iw_cq(ib_cq
);
857 remove_handle(chp
->rhp
, &chp
->rhp
->cqidr
, chp
->cq
.cqid
);
858 atomic_dec(&chp
->refcnt
);
859 wait_event(chp
->wait
, !atomic_read(&chp
->refcnt
));
861 ucontext
= ib_cq
->uobject
? to_c4iw_ucontext(ib_cq
->uobject
->context
)
863 destroy_cq(&chp
->rhp
->rdev
, &chp
->cq
,
864 ucontext
? &ucontext
->uctx
: &chp
->cq
.rdev
->uctx
);
869 struct ib_cq
*c4iw_create_cq(struct ib_device
*ibdev
, int entries
,
870 int vector
, struct ib_ucontext
*ib_context
,
871 struct ib_udata
*udata
)
873 struct c4iw_dev
*rhp
;
875 struct c4iw_create_cq_resp uresp
;
876 struct c4iw_ucontext
*ucontext
= NULL
;
878 size_t memsize
, hwentries
;
879 struct c4iw_mm_entry
*mm
, *mm2
;
881 PDBG("%s ib_dev %p entries %d\n", __func__
, ibdev
, entries
);
883 rhp
= to_c4iw_dev(ibdev
);
885 if (vector
>= rhp
->rdev
.lldi
.nciq
)
886 return ERR_PTR(-EINVAL
);
888 chp
= kzalloc(sizeof(*chp
), GFP_KERNEL
);
890 return ERR_PTR(-ENOMEM
);
893 ucontext
= to_c4iw_ucontext(ib_context
);
895 /* account for the status page. */
898 /* IQ needs one extra entry to differentiate full vs empty. */
902 * entries must be multiple of 16 for HW.
904 entries
= roundup(entries
, 16);
907 * Make actual HW queue 2x to avoid cdix_inc overflows.
909 hwentries
= min(entries
* 2, rhp
->rdev
.hw_queue
.t4_max_iq_size
);
912 * Make HW queue at least 64 entries so GTS updates aren't too
918 memsize
= hwentries
* sizeof *chp
->cq
.queue
;
921 * memsize must be a multiple of the page size if its a user cq.
924 memsize
= roundup(memsize
, PAGE_SIZE
);
925 chp
->cq
.size
= hwentries
;
926 chp
->cq
.memsize
= memsize
;
927 chp
->cq
.vector
= vector
;
929 ret
= create_cq(&rhp
->rdev
, &chp
->cq
,
930 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
);
935 chp
->cq
.size
--; /* status page */
936 chp
->ibcq
.cqe
= entries
- 2;
937 spin_lock_init(&chp
->lock
);
938 spin_lock_init(&chp
->comp_handler_lock
);
939 atomic_set(&chp
->refcnt
, 1);
940 init_waitqueue_head(&chp
->wait
);
941 ret
= insert_handle(rhp
, &rhp
->cqidr
, chp
, chp
->cq
.cqid
);
946 mm
= kmalloc(sizeof *mm
, GFP_KERNEL
);
949 mm2
= kmalloc(sizeof *mm2
, GFP_KERNEL
);
953 uresp
.qid_mask
= rhp
->rdev
.cqmask
;
954 uresp
.cqid
= chp
->cq
.cqid
;
955 uresp
.size
= chp
->cq
.size
;
956 uresp
.memsize
= chp
->cq
.memsize
;
957 spin_lock(&ucontext
->mmap_lock
);
958 uresp
.key
= ucontext
->key
;
959 ucontext
->key
+= PAGE_SIZE
;
960 uresp
.gts_key
= ucontext
->key
;
961 ucontext
->key
+= PAGE_SIZE
;
962 spin_unlock(&ucontext
->mmap_lock
);
963 ret
= ib_copy_to_udata(udata
, &uresp
,
964 sizeof(uresp
) - sizeof(uresp
.reserved
));
969 mm
->addr
= virt_to_phys(chp
->cq
.queue
);
970 mm
->len
= chp
->cq
.memsize
;
971 insert_mmap(ucontext
, mm
);
973 mm2
->key
= uresp
.gts_key
;
974 mm2
->addr
= chp
->cq
.ugts
;
975 mm2
->len
= PAGE_SIZE
;
976 insert_mmap(ucontext
, mm2
);
978 PDBG("%s cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n",
979 __func__
, chp
->cq
.cqid
, chp
, chp
->cq
.size
,
980 chp
->cq
.memsize
, (unsigned long long) chp
->cq
.dma_addr
);
987 remove_handle(rhp
, &rhp
->cqidr
, chp
->cq
.cqid
);
989 destroy_cq(&chp
->rhp
->rdev
, &chp
->cq
,
990 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
);
996 int c4iw_resize_cq(struct ib_cq
*cq
, int cqe
, struct ib_udata
*udata
)
1001 int c4iw_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
1003 struct c4iw_cq
*chp
;
1007 chp
= to_c4iw_cq(ibcq
);
1008 spin_lock_irqsave(&chp
->lock
, flag
);
1009 ret
= t4_arm_cq(&chp
->cq
,
1010 (flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
);
1011 spin_unlock_irqrestore(&chp
->lock
, flag
);
1012 if (ret
&& !(flags
& IB_CQ_REPORT_MISSED_EVENTS
))