2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kref.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_user_verbs.h>
36 #include <rdma/ib_cache.h>
40 static void mlx5_ib_cq_comp(struct mlx5_core_cq
*cq
)
42 struct ib_cq
*ibcq
= &to_mibcq(cq
)->ibcq
;
44 ibcq
->comp_handler(ibcq
, ibcq
->cq_context
);
47 static void mlx5_ib_cq_event(struct mlx5_core_cq
*mcq
, enum mlx5_event type
)
49 struct mlx5_ib_cq
*cq
= container_of(mcq
, struct mlx5_ib_cq
, mcq
);
50 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
51 struct ib_cq
*ibcq
= &cq
->ibcq
;
52 struct ib_event event
;
54 if (type
!= MLX5_EVENT_TYPE_CQ_ERROR
) {
55 mlx5_ib_warn(dev
, "Unexpected event type %d on CQ %06x\n",
60 if (ibcq
->event_handler
) {
61 event
.device
= &dev
->ib_dev
;
62 event
.event
= IB_EVENT_CQ_ERR
;
63 event
.element
.cq
= ibcq
;
64 ibcq
->event_handler(&event
, ibcq
->cq_context
);
68 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf
*buf
, int n
, int size
)
70 return mlx5_buf_offset(&buf
->buf
, n
* size
);
73 static void *get_cqe(struct mlx5_ib_cq
*cq
, int n
)
75 return get_cqe_from_buf(&cq
->buf
, n
, cq
->mcq
.cqe_sz
);
78 static u8
sw_ownership_bit(int n
, int nent
)
80 return (n
& nent
) ? 1 : 0;
83 static void *get_sw_cqe(struct mlx5_ib_cq
*cq
, int n
)
85 void *cqe
= get_cqe(cq
, n
& cq
->ibcq
.cqe
);
86 struct mlx5_cqe64
*cqe64
;
88 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
90 if (likely((cqe64
->op_own
) >> 4 != MLX5_CQE_INVALID
) &&
91 !((cqe64
->op_own
& MLX5_CQE_OWNER_MASK
) ^ !!(n
& (cq
->ibcq
.cqe
+ 1)))) {
98 static void *next_cqe_sw(struct mlx5_ib_cq
*cq
)
100 return get_sw_cqe(cq
, cq
->mcq
.cons_index
);
103 static enum ib_wc_opcode
get_umr_comp(struct mlx5_ib_wq
*wq
, int idx
)
105 switch (wq
->wr_data
[idx
]) {
109 case IB_WR_LOCAL_INV
:
110 return IB_WC_LOCAL_INV
;
116 pr_warn("unknown completion status\n");
121 static void handle_good_req(struct ib_wc
*wc
, struct mlx5_cqe64
*cqe
,
122 struct mlx5_ib_wq
*wq
, int idx
)
125 switch (be32_to_cpu(cqe
->sop_drop_qpn
) >> 24) {
126 case MLX5_OPCODE_RDMA_WRITE_IMM
:
127 wc
->wc_flags
|= IB_WC_WITH_IMM
;
128 case MLX5_OPCODE_RDMA_WRITE
:
129 wc
->opcode
= IB_WC_RDMA_WRITE
;
131 case MLX5_OPCODE_SEND_IMM
:
132 wc
->wc_flags
|= IB_WC_WITH_IMM
;
133 case MLX5_OPCODE_SEND
:
134 case MLX5_OPCODE_SEND_INVAL
:
135 wc
->opcode
= IB_WC_SEND
;
137 case MLX5_OPCODE_RDMA_READ
:
138 wc
->opcode
= IB_WC_RDMA_READ
;
139 wc
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
141 case MLX5_OPCODE_ATOMIC_CS
:
142 wc
->opcode
= IB_WC_COMP_SWAP
;
145 case MLX5_OPCODE_ATOMIC_FA
:
146 wc
->opcode
= IB_WC_FETCH_ADD
;
149 case MLX5_OPCODE_ATOMIC_MASKED_CS
:
150 wc
->opcode
= IB_WC_MASKED_COMP_SWAP
;
153 case MLX5_OPCODE_ATOMIC_MASKED_FA
:
154 wc
->opcode
= IB_WC_MASKED_FETCH_ADD
;
157 case MLX5_OPCODE_UMR
:
158 wc
->opcode
= get_umr_comp(wq
, idx
);
164 MLX5_GRH_IN_BUFFER
= 1,
168 static void handle_responder(struct ib_wc
*wc
, struct mlx5_cqe64
*cqe
,
169 struct mlx5_ib_qp
*qp
)
171 enum rdma_link_layer ll
= rdma_port_get_link_layer(qp
->ibqp
.device
, 1);
172 struct mlx5_ib_dev
*dev
= to_mdev(qp
->ibqp
.device
);
173 struct mlx5_ib_srq
*srq
;
174 struct mlx5_ib_wq
*wq
;
178 if (qp
->ibqp
.srq
|| qp
->ibqp
.xrcd
) {
179 struct mlx5_core_srq
*msrq
= NULL
;
182 msrq
= mlx5_core_get_srq(dev
->mdev
,
183 be32_to_cpu(cqe
->srqn
));
184 srq
= to_mibsrq(msrq
);
186 srq
= to_msrq(qp
->ibqp
.srq
);
189 wqe_ctr
= be16_to_cpu(cqe
->wqe_counter
);
190 wc
->wr_id
= srq
->wrid
[wqe_ctr
];
191 mlx5_ib_free_srq_wqe(srq
, wqe_ctr
);
192 if (msrq
&& atomic_dec_and_test(&msrq
->refcount
))
193 complete(&msrq
->free
);
197 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
200 wc
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
202 switch (cqe
->op_own
>> 4) {
203 case MLX5_CQE_RESP_WR_IMM
:
204 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
205 wc
->wc_flags
= IB_WC_WITH_IMM
;
206 wc
->ex
.imm_data
= cqe
->imm_inval_pkey
;
208 case MLX5_CQE_RESP_SEND
:
209 wc
->opcode
= IB_WC_RECV
;
210 wc
->wc_flags
= IB_WC_IP_CSUM_OK
;
211 if (unlikely(!((cqe
->hds_ip_ext
& CQE_L3_OK
) &&
212 (cqe
->hds_ip_ext
& CQE_L4_OK
))))
215 case MLX5_CQE_RESP_SEND_IMM
:
216 wc
->opcode
= IB_WC_RECV
;
217 wc
->wc_flags
= IB_WC_WITH_IMM
;
218 wc
->ex
.imm_data
= cqe
->imm_inval_pkey
;
220 case MLX5_CQE_RESP_SEND_INV
:
221 wc
->opcode
= IB_WC_RECV
;
222 wc
->wc_flags
= IB_WC_WITH_INVALIDATE
;
223 wc
->ex
.invalidate_rkey
= be32_to_cpu(cqe
->imm_inval_pkey
);
226 wc
->slid
= be16_to_cpu(cqe
->slid
);
227 wc
->sl
= (be32_to_cpu(cqe
->flags_rqpn
) >> 24) & 0xf;
228 wc
->src_qp
= be32_to_cpu(cqe
->flags_rqpn
) & 0xffffff;
229 wc
->dlid_path_bits
= cqe
->ml_path
;
230 g
= (be32_to_cpu(cqe
->flags_rqpn
) >> 28) & 3;
231 wc
->wc_flags
|= g
? IB_WC_GRH
: 0;
232 if (unlikely(is_qp1(qp
->ibqp
.qp_type
))) {
233 u16 pkey
= be32_to_cpu(cqe
->imm_inval_pkey
) & 0xffff;
235 ib_find_cached_pkey(&dev
->ib_dev
, qp
->port
, pkey
,
241 if (ll
!= IB_LINK_LAYER_ETHERNET
)
244 switch (wc
->sl
& 0x3) {
245 case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH
:
246 wc
->network_hdr_type
= RDMA_NETWORK_IB
;
248 case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6
:
249 wc
->network_hdr_type
= RDMA_NETWORK_IPV6
;
251 case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV4
:
252 wc
->network_hdr_type
= RDMA_NETWORK_IPV4
;
255 wc
->wc_flags
|= IB_WC_WITH_NETWORK_HDR_TYPE
;
258 static void dump_cqe(struct mlx5_ib_dev
*dev
, struct mlx5_err_cqe
*cqe
)
260 __be32
*p
= (__be32
*)cqe
;
263 mlx5_ib_warn(dev
, "dump error cqe\n");
264 for (i
= 0; i
< sizeof(*cqe
) / 16; i
++, p
+= 4)
265 pr_info("%08x %08x %08x %08x\n", be32_to_cpu(p
[0]),
266 be32_to_cpu(p
[1]), be32_to_cpu(p
[2]),
270 static void mlx5_handle_error_cqe(struct mlx5_ib_dev
*dev
,
271 struct mlx5_err_cqe
*cqe
,
276 switch (cqe
->syndrome
) {
277 case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR
:
278 wc
->status
= IB_WC_LOC_LEN_ERR
;
280 case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR
:
281 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
283 case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR
:
284 wc
->status
= IB_WC_LOC_PROT_ERR
;
286 case MLX5_CQE_SYNDROME_WR_FLUSH_ERR
:
288 wc
->status
= IB_WC_WR_FLUSH_ERR
;
290 case MLX5_CQE_SYNDROME_MW_BIND_ERR
:
291 wc
->status
= IB_WC_MW_BIND_ERR
;
293 case MLX5_CQE_SYNDROME_BAD_RESP_ERR
:
294 wc
->status
= IB_WC_BAD_RESP_ERR
;
296 case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR
:
297 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
299 case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR
:
300 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
302 case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR
:
303 wc
->status
= IB_WC_REM_ACCESS_ERR
;
305 case MLX5_CQE_SYNDROME_REMOTE_OP_ERR
:
306 wc
->status
= IB_WC_REM_OP_ERR
;
308 case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR
:
309 wc
->status
= IB_WC_RETRY_EXC_ERR
;
312 case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR
:
313 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
316 case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR
:
317 wc
->status
= IB_WC_REM_ABORT_ERR
;
320 wc
->status
= IB_WC_GENERAL_ERR
;
324 wc
->vendor_err
= cqe
->vendor_err_synd
;
329 static int is_atomic_response(struct mlx5_ib_qp
*qp
, uint16_t idx
)
331 /* TBD: waiting decision
336 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp
*qp
, uint16_t idx
)
338 struct mlx5_wqe_data_seg
*dpseg
;
341 dpseg
= mlx5_get_send_wqe(qp
, idx
) + sizeof(struct mlx5_wqe_ctrl_seg
) +
342 sizeof(struct mlx5_wqe_raddr_seg
) +
343 sizeof(struct mlx5_wqe_atomic_seg
);
344 addr
= (void *)(unsigned long)be64_to_cpu(dpseg
->addr
);
348 static void handle_atomic(struct mlx5_ib_qp
*qp
, struct mlx5_cqe64
*cqe64
,
355 if (!is_atomic_response(qp
, idx
))
358 byte_count
= be32_to_cpu(cqe64
->byte_cnt
);
359 addr
= mlx5_get_atomic_laddr(qp
, idx
);
361 if (byte_count
== 4) {
362 *(uint32_t *)addr
= be32_to_cpu(*((__be32
*)addr
));
364 for (i
= 0; i
< byte_count
; i
+= 8) {
365 *(uint64_t *)addr
= be64_to_cpu(*((__be64
*)addr
));
373 static void handle_atomics(struct mlx5_ib_qp
*qp
, struct mlx5_cqe64
*cqe64
,
379 idx
= tail
& (qp
->sq
.wqe_cnt
- 1);
380 handle_atomic(qp
, cqe64
, idx
);
384 tail
= qp
->sq
.w_list
[idx
].next
;
386 tail
= qp
->sq
.w_list
[idx
].next
;
387 qp
->sq
.last_poll
= tail
;
390 static void free_cq_buf(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq_buf
*buf
)
392 mlx5_buf_free(dev
->mdev
, &buf
->buf
);
395 static void get_sig_err_item(struct mlx5_sig_err_cqe
*cqe
,
396 struct ib_sig_err
*item
)
398 u16 syndrome
= be16_to_cpu(cqe
->syndrome
);
400 #define GUARD_ERR (1 << 13)
401 #define APPTAG_ERR (1 << 12)
402 #define REFTAG_ERR (1 << 11)
404 if (syndrome
& GUARD_ERR
) {
405 item
->err_type
= IB_SIG_BAD_GUARD
;
406 item
->expected
= be32_to_cpu(cqe
->expected_trans_sig
) >> 16;
407 item
->actual
= be32_to_cpu(cqe
->actual_trans_sig
) >> 16;
409 if (syndrome
& REFTAG_ERR
) {
410 item
->err_type
= IB_SIG_BAD_REFTAG
;
411 item
->expected
= be32_to_cpu(cqe
->expected_reftag
);
412 item
->actual
= be32_to_cpu(cqe
->actual_reftag
);
414 if (syndrome
& APPTAG_ERR
) {
415 item
->err_type
= IB_SIG_BAD_APPTAG
;
416 item
->expected
= be32_to_cpu(cqe
->expected_trans_sig
) & 0xffff;
417 item
->actual
= be32_to_cpu(cqe
->actual_trans_sig
) & 0xffff;
419 pr_err("Got signature completion error with bad syndrome %04x\n",
423 item
->sig_err_offset
= be64_to_cpu(cqe
->err_offset
);
424 item
->key
= be32_to_cpu(cqe
->mkey
);
427 static void sw_send_comp(struct mlx5_ib_qp
*qp
, int num_entries
,
428 struct ib_wc
*wc
, int *npolled
)
430 struct mlx5_ib_wq
*wq
;
437 cur
= wq
->head
- wq
->tail
;
443 for (i
= 0; i
< cur
&& np
< num_entries
; i
++) {
444 idx
= wq
->last_poll
& (wq
->wqe_cnt
- 1);
445 wc
->wr_id
= wq
->wrid
[idx
];
446 wc
->status
= IB_WC_WR_FLUSH_ERR
;
447 wc
->vendor_err
= MLX5_CQE_SYNDROME_WR_FLUSH_ERR
;
452 wq
->last_poll
= wq
->w_list
[idx
].next
;
457 static void sw_recv_comp(struct mlx5_ib_qp
*qp
, int num_entries
,
458 struct ib_wc
*wc
, int *npolled
)
460 struct mlx5_ib_wq
*wq
;
466 cur
= wq
->head
- wq
->tail
;
472 for (i
= 0; i
< cur
&& np
< num_entries
; i
++) {
473 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
474 wc
->status
= IB_WC_WR_FLUSH_ERR
;
475 wc
->vendor_err
= MLX5_CQE_SYNDROME_WR_FLUSH_ERR
;
484 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq
*cq
, int num_entries
,
485 struct ib_wc
*wc
, int *npolled
)
487 struct mlx5_ib_qp
*qp
;
490 /* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
491 list_for_each_entry(qp
, &cq
->list_send_qp
, cq_send_list
) {
492 sw_send_comp(qp
, num_entries
, wc
+ *npolled
, npolled
);
493 if (*npolled
>= num_entries
)
497 list_for_each_entry(qp
, &cq
->list_recv_qp
, cq_recv_list
) {
498 sw_recv_comp(qp
, num_entries
, wc
+ *npolled
, npolled
);
499 if (*npolled
>= num_entries
)
504 static int mlx5_poll_one(struct mlx5_ib_cq
*cq
,
505 struct mlx5_ib_qp
**cur_qp
,
508 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
509 struct mlx5_err_cqe
*err_cqe
;
510 struct mlx5_cqe64
*cqe64
;
511 struct mlx5_core_qp
*mqp
;
512 struct mlx5_ib_wq
*wq
;
513 struct mlx5_sig_err_cqe
*sig_err_cqe
;
514 struct mlx5_core_mkey
*mmkey
;
515 struct mlx5_ib_mr
*mr
;
523 cqe
= next_cqe_sw(cq
);
527 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
529 ++cq
->mcq
.cons_index
;
531 /* Make sure we read CQ entry contents after we've checked the
536 opcode
= cqe64
->op_own
>> 4;
537 if (unlikely(opcode
== MLX5_CQE_RESIZE_CQ
)) {
538 if (likely(cq
->resize_buf
)) {
539 free_cq_buf(dev
, &cq
->buf
);
540 cq
->buf
= *cq
->resize_buf
;
541 kfree(cq
->resize_buf
);
542 cq
->resize_buf
= NULL
;
545 mlx5_ib_warn(dev
, "unexpected resize cqe\n");
549 qpn
= ntohl(cqe64
->sop_drop_qpn
) & 0xffffff;
550 if (!*cur_qp
|| (qpn
!= (*cur_qp
)->ibqp
.qp_num
)) {
551 /* We do not have to take the QP table lock here,
552 * because CQs will be locked while QPs are removed
555 mqp
= __mlx5_qp_lookup(dev
->mdev
, qpn
);
556 if (unlikely(!mqp
)) {
557 mlx5_ib_warn(dev
, "CQE@CQ %06x for unknown QPN %6x\n",
562 *cur_qp
= to_mibqp(mqp
);
565 wc
->qp
= &(*cur_qp
)->ibqp
;
569 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
570 idx
= wqe_ctr
& (wq
->wqe_cnt
- 1);
571 handle_good_req(wc
, cqe64
, wq
, idx
);
572 handle_atomics(*cur_qp
, cqe64
, wq
->last_poll
, idx
);
573 wc
->wr_id
= wq
->wrid
[idx
];
574 wq
->tail
= wq
->wqe_head
[idx
] + 1;
575 wc
->status
= IB_WC_SUCCESS
;
577 case MLX5_CQE_RESP_WR_IMM
:
578 case MLX5_CQE_RESP_SEND
:
579 case MLX5_CQE_RESP_SEND_IMM
:
580 case MLX5_CQE_RESP_SEND_INV
:
581 handle_responder(wc
, cqe64
, *cur_qp
);
582 wc
->status
= IB_WC_SUCCESS
;
584 case MLX5_CQE_RESIZE_CQ
:
586 case MLX5_CQE_REQ_ERR
:
587 case MLX5_CQE_RESP_ERR
:
588 err_cqe
= (struct mlx5_err_cqe
*)cqe64
;
589 mlx5_handle_error_cqe(dev
, err_cqe
, wc
);
590 mlx5_ib_dbg(dev
, "%s error cqe on cqn 0x%x:\n",
591 opcode
== MLX5_CQE_REQ_ERR
?
592 "Requestor" : "Responder", cq
->mcq
.cqn
);
593 mlx5_ib_dbg(dev
, "syndrome 0x%x, vendor syndrome 0x%x\n",
594 err_cqe
->syndrome
, err_cqe
->vendor_err_synd
);
595 if (opcode
== MLX5_CQE_REQ_ERR
) {
597 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
598 idx
= wqe_ctr
& (wq
->wqe_cnt
- 1);
599 wc
->wr_id
= wq
->wrid
[idx
];
600 wq
->tail
= wq
->wqe_head
[idx
] + 1;
602 struct mlx5_ib_srq
*srq
;
604 if ((*cur_qp
)->ibqp
.srq
) {
605 srq
= to_msrq((*cur_qp
)->ibqp
.srq
);
606 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
607 wc
->wr_id
= srq
->wrid
[wqe_ctr
];
608 mlx5_ib_free_srq_wqe(srq
, wqe_ctr
);
611 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
616 case MLX5_CQE_SIG_ERR
:
617 sig_err_cqe
= (struct mlx5_sig_err_cqe
*)cqe64
;
619 read_lock(&dev
->mdev
->priv
.mkey_table
.lock
);
620 mmkey
= __mlx5_mr_lookup(dev
->mdev
,
621 mlx5_base_mkey(be32_to_cpu(sig_err_cqe
->mkey
)));
622 if (unlikely(!mmkey
)) {
623 read_unlock(&dev
->mdev
->priv
.mkey_table
.lock
);
624 mlx5_ib_warn(dev
, "CQE@CQ %06x for unknown MR %6x\n",
625 cq
->mcq
.cqn
, be32_to_cpu(sig_err_cqe
->mkey
));
629 mr
= to_mibmr(mmkey
);
630 get_sig_err_item(sig_err_cqe
, &mr
->sig
->err_item
);
631 mr
->sig
->sig_err_exists
= true;
632 mr
->sig
->sigerr_count
++;
634 mlx5_ib_warn(dev
, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",
635 cq
->mcq
.cqn
, mr
->sig
->err_item
.key
,
636 mr
->sig
->err_item
.err_type
,
637 mr
->sig
->err_item
.sig_err_offset
,
638 mr
->sig
->err_item
.expected
,
639 mr
->sig
->err_item
.actual
);
641 read_unlock(&dev
->mdev
->priv
.mkey_table
.lock
);
648 static int poll_soft_wc(struct mlx5_ib_cq
*cq
, int num_entries
,
651 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
652 struct mlx5_ib_wc
*soft_wc
, *next
;
655 list_for_each_entry_safe(soft_wc
, next
, &cq
->wc_list
, list
) {
656 if (npolled
>= num_entries
)
659 mlx5_ib_dbg(dev
, "polled software generated completion on CQ 0x%x\n",
662 wc
[npolled
++] = soft_wc
->wc
;
663 list_del(&soft_wc
->list
);
670 int mlx5_ib_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
672 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
673 struct mlx5_ib_qp
*cur_qp
= NULL
;
674 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
675 struct mlx5_core_dev
*mdev
= dev
->mdev
;
681 spin_lock_irqsave(&cq
->lock
, flags
);
682 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
) {
683 mlx5_ib_poll_sw_comp(cq
, num_entries
, wc
, &npolled
);
687 if (unlikely(!list_empty(&cq
->wc_list
)))
688 soft_polled
= poll_soft_wc(cq
, num_entries
, wc
);
690 for (npolled
= 0; npolled
< num_entries
- soft_polled
; npolled
++) {
691 err
= mlx5_poll_one(cq
, &cur_qp
, wc
+ soft_polled
+ npolled
);
697 mlx5_cq_set_ci(&cq
->mcq
);
699 spin_unlock_irqrestore(&cq
->lock
, flags
);
701 if (err
== 0 || err
== -EAGAIN
)
702 return soft_polled
+ npolled
;
707 int mlx5_ib_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
709 struct mlx5_core_dev
*mdev
= to_mdev(ibcq
->device
)->mdev
;
710 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
711 void __iomem
*uar_page
= mdev
->priv
.uuari
.uars
[0].map
;
712 unsigned long irq_flags
;
715 spin_lock_irqsave(&cq
->lock
, irq_flags
);
716 if (cq
->notify_flags
!= IB_CQ_NEXT_COMP
)
717 cq
->notify_flags
= flags
& IB_CQ_SOLICITED_MASK
;
719 if ((flags
& IB_CQ_REPORT_MISSED_EVENTS
) && !list_empty(&cq
->wc_list
))
721 spin_unlock_irqrestore(&cq
->lock
, irq_flags
);
723 mlx5_cq_arm(&cq
->mcq
,
724 (flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
?
725 MLX5_CQ_DB_REQ_NOT_SOL
: MLX5_CQ_DB_REQ_NOT
,
727 MLX5_GET_DOORBELL_LOCK(&mdev
->priv
.cq_uar_lock
),
728 to_mcq(ibcq
)->mcq
.cons_index
);
733 static int alloc_cq_buf(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq_buf
*buf
,
734 int nent
, int cqe_size
)
738 err
= mlx5_buf_alloc(dev
->mdev
, nent
* cqe_size
, &buf
->buf
);
742 buf
->cqe_size
= cqe_size
;
748 static int create_cq_user(struct mlx5_ib_dev
*dev
, struct ib_udata
*udata
,
749 struct ib_ucontext
*context
, struct mlx5_ib_cq
*cq
,
750 int entries
, struct mlx5_create_cq_mbox_in
**cqb
,
751 int *cqe_size
, int *index
, int *inlen
)
753 struct mlx5_ib_create_cq ucmd
;
761 (udata
->inlen
- sizeof(struct ib_uverbs_cmd_hdr
) <
762 sizeof(ucmd
)) ? (sizeof(ucmd
) -
763 sizeof(ucmd
.reserved
)) : sizeof(ucmd
);
765 if (ib_copy_from_udata(&ucmd
, udata
, ucmdlen
))
768 if (ucmdlen
== sizeof(ucmd
) &&
772 if (ucmd
.cqe_size
!= 64 && ucmd
.cqe_size
!= 128)
775 *cqe_size
= ucmd
.cqe_size
;
777 cq
->buf
.umem
= ib_umem_get(context
, ucmd
.buf_addr
,
778 entries
* ucmd
.cqe_size
,
779 IB_ACCESS_LOCAL_WRITE
, 1);
780 if (IS_ERR(cq
->buf
.umem
)) {
781 err
= PTR_ERR(cq
->buf
.umem
);
785 err
= mlx5_ib_db_map_user(to_mucontext(context
), ucmd
.db_addr
,
790 mlx5_ib_cont_pages(cq
->buf
.umem
, ucmd
.buf_addr
, &npages
, &page_shift
,
792 mlx5_ib_dbg(dev
, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
793 ucmd
.buf_addr
, entries
* ucmd
.cqe_size
, npages
, page_shift
, ncont
);
795 *inlen
= sizeof(**cqb
) + sizeof(*(*cqb
)->pas
) * ncont
;
796 *cqb
= mlx5_vzalloc(*inlen
);
801 mlx5_ib_populate_pas(dev
, cq
->buf
.umem
, page_shift
, (*cqb
)->pas
, 0);
802 (*cqb
)->ctx
.log_pg_sz
= page_shift
- MLX5_ADAPTER_PAGE_SHIFT
;
804 *index
= to_mucontext(context
)->uuari
.uars
[0].index
;
809 mlx5_ib_db_unmap_user(to_mucontext(context
), &cq
->db
);
812 ib_umem_release(cq
->buf
.umem
);
816 static void destroy_cq_user(struct mlx5_ib_cq
*cq
, struct ib_ucontext
*context
)
818 mlx5_ib_db_unmap_user(to_mucontext(context
), &cq
->db
);
819 ib_umem_release(cq
->buf
.umem
);
822 static void init_cq_buf(struct mlx5_ib_cq
*cq
, struct mlx5_ib_cq_buf
*buf
)
826 struct mlx5_cqe64
*cqe64
;
828 for (i
= 0; i
< buf
->nent
; i
++) {
829 cqe
= get_cqe_from_buf(buf
, i
, buf
->cqe_size
);
830 cqe64
= buf
->cqe_size
== 64 ? cqe
: cqe
+ 64;
831 cqe64
->op_own
= MLX5_CQE_INVALID
<< 4;
835 static int create_cq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
836 int entries
, int cqe_size
,
837 struct mlx5_create_cq_mbox_in
**cqb
,
838 int *index
, int *inlen
)
842 err
= mlx5_db_alloc(dev
->mdev
, &cq
->db
);
846 cq
->mcq
.set_ci_db
= cq
->db
.db
;
847 cq
->mcq
.arm_db
= cq
->db
.db
+ 1;
848 cq
->mcq
.cqe_sz
= cqe_size
;
850 err
= alloc_cq_buf(dev
, &cq
->buf
, entries
, cqe_size
);
854 init_cq_buf(cq
, &cq
->buf
);
856 *inlen
= sizeof(**cqb
) + sizeof(*(*cqb
)->pas
) * cq
->buf
.buf
.npages
;
857 *cqb
= mlx5_vzalloc(*inlen
);
862 mlx5_fill_page_array(&cq
->buf
.buf
, (*cqb
)->pas
);
864 (*cqb
)->ctx
.log_pg_sz
= cq
->buf
.buf
.page_shift
- MLX5_ADAPTER_PAGE_SHIFT
;
865 *index
= dev
->mdev
->priv
.uuari
.uars
[0].index
;
870 free_cq_buf(dev
, &cq
->buf
);
873 mlx5_db_free(dev
->mdev
, &cq
->db
);
877 static void destroy_cq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
)
879 free_cq_buf(dev
, &cq
->buf
);
880 mlx5_db_free(dev
->mdev
, &cq
->db
);
883 static void notify_soft_wc_handler(struct work_struct
*work
)
885 struct mlx5_ib_cq
*cq
= container_of(work
, struct mlx5_ib_cq
,
888 cq
->ibcq
.comp_handler(&cq
->ibcq
, cq
->ibcq
.cq_context
);
891 struct ib_cq
*mlx5_ib_create_cq(struct ib_device
*ibdev
,
892 const struct ib_cq_init_attr
*attr
,
893 struct ib_ucontext
*context
,
894 struct ib_udata
*udata
)
896 int entries
= attr
->cqe
;
897 int vector
= attr
->comp_vector
;
898 struct mlx5_create_cq_mbox_in
*cqb
= NULL
;
899 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
900 struct mlx5_ib_cq
*cq
;
901 int uninitialized_var(index
);
902 int uninitialized_var(inlen
);
909 (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
))))
910 return ERR_PTR(-EINVAL
);
912 if (check_cq_create_flags(attr
->flags
))
913 return ERR_PTR(-EOPNOTSUPP
);
915 entries
= roundup_pow_of_two(entries
+ 1);
916 if (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
)))
917 return ERR_PTR(-EINVAL
);
919 cq
= kzalloc(sizeof(*cq
), GFP_KERNEL
);
921 return ERR_PTR(-ENOMEM
);
923 cq
->ibcq
.cqe
= entries
- 1;
924 mutex_init(&cq
->resize_mutex
);
925 spin_lock_init(&cq
->lock
);
926 cq
->resize_buf
= NULL
;
927 cq
->resize_umem
= NULL
;
928 cq
->create_flags
= attr
->flags
;
929 INIT_LIST_HEAD(&cq
->list_send_qp
);
930 INIT_LIST_HEAD(&cq
->list_recv_qp
);
933 err
= create_cq_user(dev
, udata
, context
, cq
, entries
,
934 &cqb
, &cqe_size
, &index
, &inlen
);
938 /* for now choose 64 bytes till we have a proper interface */
940 err
= create_cq_kernel(dev
, cq
, entries
, cqe_size
, &cqb
,
945 INIT_WORK(&cq
->notify_work
, notify_soft_wc_handler
);
948 cq
->cqe_size
= cqe_size
;
949 cqb
->ctx
.cqe_sz_flags
= cqe_sz_to_mlx_sz(cqe_size
) << 5;
951 if (cq
->create_flags
& IB_CQ_FLAGS_IGNORE_OVERRUN
)
952 cqb
->ctx
.cqe_sz_flags
|= (1 << 1);
954 cqb
->ctx
.log_sz_usr_page
= cpu_to_be32((ilog2(entries
) << 24) | index
);
955 err
= mlx5_vector2eqn(dev
->mdev
, vector
, &eqn
, &irqn
);
959 cqb
->ctx
.c_eqn
= cpu_to_be16(eqn
);
960 cqb
->ctx
.db_record_addr
= cpu_to_be64(cq
->db
.dma
);
962 err
= mlx5_core_create_cq(dev
->mdev
, &cq
->mcq
, cqb
, inlen
);
966 mlx5_ib_dbg(dev
, "cqn 0x%x\n", cq
->mcq
.cqn
);
969 cq
->mcq
.tasklet_ctx
.comp
= mlx5_ib_cq_comp
;
971 cq
->mcq
.comp
= mlx5_ib_cq_comp
;
972 cq
->mcq
.event
= mlx5_ib_cq_event
;
974 INIT_LIST_HEAD(&cq
->wc_list
);
977 if (ib_copy_to_udata(udata
, &cq
->mcq
.cqn
, sizeof(__u32
))) {
987 mlx5_core_destroy_cq(dev
->mdev
, &cq
->mcq
);
992 destroy_cq_user(cq
, context
);
994 destroy_cq_kernel(dev
, cq
);
1003 int mlx5_ib_destroy_cq(struct ib_cq
*cq
)
1005 struct mlx5_ib_dev
*dev
= to_mdev(cq
->device
);
1006 struct mlx5_ib_cq
*mcq
= to_mcq(cq
);
1007 struct ib_ucontext
*context
= NULL
;
1010 context
= cq
->uobject
->context
;
1012 mlx5_core_destroy_cq(dev
->mdev
, &mcq
->mcq
);
1014 destroy_cq_user(mcq
, context
);
1016 destroy_cq_kernel(dev
, mcq
);
1023 static int is_equal_rsn(struct mlx5_cqe64
*cqe64
, u32 rsn
)
1025 return rsn
== (ntohl(cqe64
->sop_drop_qpn
) & 0xffffff);
1028 void __mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 rsn
, struct mlx5_ib_srq
*srq
)
1030 struct mlx5_cqe64
*cqe64
, *dest64
;
1039 /* First we need to find the current producer index, so we
1040 * know where to start cleaning from. It doesn't matter if HW
1041 * adds new entries after this loop -- the QP we're worried
1042 * about is already in RESET, so the new entries won't come
1043 * from our QP and therefore don't need to be checked.
1045 for (prod_index
= cq
->mcq
.cons_index
; get_sw_cqe(cq
, prod_index
); prod_index
++)
1046 if (prod_index
== cq
->mcq
.cons_index
+ cq
->ibcq
.cqe
)
1049 /* Now sweep backwards through the CQ, removing CQ entries
1050 * that match our QP by copying older entries on top of them.
1052 while ((int) --prod_index
- (int) cq
->mcq
.cons_index
>= 0) {
1053 cqe
= get_cqe(cq
, prod_index
& cq
->ibcq
.cqe
);
1054 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
1055 if (is_equal_rsn(cqe64
, rsn
)) {
1056 if (srq
&& (ntohl(cqe64
->srqn
) & 0xffffff))
1057 mlx5_ib_free_srq_wqe(srq
, be16_to_cpu(cqe64
->wqe_counter
));
1059 } else if (nfreed
) {
1060 dest
= get_cqe(cq
, (prod_index
+ nfreed
) & cq
->ibcq
.cqe
);
1061 dest64
= (cq
->mcq
.cqe_sz
== 64) ? dest
: dest
+ 64;
1062 owner_bit
= dest64
->op_own
& MLX5_CQE_OWNER_MASK
;
1063 memcpy(dest
, cqe
, cq
->mcq
.cqe_sz
);
1064 dest64
->op_own
= owner_bit
|
1065 (dest64
->op_own
& ~MLX5_CQE_OWNER_MASK
);
1070 cq
->mcq
.cons_index
+= nfreed
;
1071 /* Make sure update of buffer contents is done before
1072 * updating consumer index.
1075 mlx5_cq_set_ci(&cq
->mcq
);
1079 void mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 qpn
, struct mlx5_ib_srq
*srq
)
1084 spin_lock_irq(&cq
->lock
);
1085 __mlx5_ib_cq_clean(cq
, qpn
, srq
);
1086 spin_unlock_irq(&cq
->lock
);
1089 int mlx5_ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
1091 struct mlx5_modify_cq_mbox_in
*in
;
1092 struct mlx5_ib_dev
*dev
= to_mdev(cq
->device
);
1093 struct mlx5_ib_cq
*mcq
= to_mcq(cq
);
1097 if (!MLX5_CAP_GEN(dev
->mdev
, cq_moderation
))
1100 in
= kzalloc(sizeof(*in
), GFP_KERNEL
);
1104 in
->cqn
= cpu_to_be32(mcq
->mcq
.cqn
);
1105 fsel
= (MLX5_CQ_MODIFY_PERIOD
| MLX5_CQ_MODIFY_COUNT
);
1106 in
->ctx
.cq_period
= cpu_to_be16(cq_period
);
1107 in
->ctx
.cq_max_count
= cpu_to_be16(cq_count
);
1108 in
->field_select
= cpu_to_be32(fsel
);
1109 err
= mlx5_core_modify_cq(dev
->mdev
, &mcq
->mcq
, in
, sizeof(*in
));
1113 mlx5_ib_warn(dev
, "modify cq 0x%x failed\n", mcq
->mcq
.cqn
);
1118 static int resize_user(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
1119 int entries
, struct ib_udata
*udata
, int *npas
,
1120 int *page_shift
, int *cqe_size
)
1122 struct mlx5_ib_resize_cq ucmd
;
1123 struct ib_umem
*umem
;
1126 struct ib_ucontext
*context
= cq
->buf
.umem
->context
;
1128 err
= ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
));
1132 if (ucmd
.reserved0
|| ucmd
.reserved1
)
1135 umem
= ib_umem_get(context
, ucmd
.buf_addr
, entries
* ucmd
.cqe_size
,
1136 IB_ACCESS_LOCAL_WRITE
, 1);
1138 err
= PTR_ERR(umem
);
1142 mlx5_ib_cont_pages(umem
, ucmd
.buf_addr
, &npages
, page_shift
,
1145 cq
->resize_umem
= umem
;
1146 *cqe_size
= ucmd
.cqe_size
;
1151 static void un_resize_user(struct mlx5_ib_cq
*cq
)
1153 ib_umem_release(cq
->resize_umem
);
1156 static int resize_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
1157 int entries
, int cqe_size
)
1161 cq
->resize_buf
= kzalloc(sizeof(*cq
->resize_buf
), GFP_KERNEL
);
1162 if (!cq
->resize_buf
)
1165 err
= alloc_cq_buf(dev
, cq
->resize_buf
, entries
, cqe_size
);
1169 init_cq_buf(cq
, cq
->resize_buf
);
1174 kfree(cq
->resize_buf
);
1178 static void un_resize_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
)
1180 free_cq_buf(dev
, cq
->resize_buf
);
1181 cq
->resize_buf
= NULL
;
1184 static int copy_resize_cqes(struct mlx5_ib_cq
*cq
)
1186 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
1187 struct mlx5_cqe64
*scqe64
;
1188 struct mlx5_cqe64
*dcqe64
;
1197 ssize
= cq
->buf
.cqe_size
;
1198 dsize
= cq
->resize_buf
->cqe_size
;
1199 if (ssize
!= dsize
) {
1200 mlx5_ib_warn(dev
, "resize from different cqe size is not supported\n");
1204 i
= cq
->mcq
.cons_index
;
1205 scqe
= get_sw_cqe(cq
, i
);
1206 scqe64
= ssize
== 64 ? scqe
: scqe
+ 64;
1209 mlx5_ib_warn(dev
, "expected cqe in sw ownership\n");
1213 while ((scqe64
->op_own
>> 4) != MLX5_CQE_RESIZE_CQ
) {
1214 dcqe
= get_cqe_from_buf(cq
->resize_buf
,
1215 (i
+ 1) & (cq
->resize_buf
->nent
),
1217 dcqe64
= dsize
== 64 ? dcqe
: dcqe
+ 64;
1218 sw_own
= sw_ownership_bit(i
+ 1, cq
->resize_buf
->nent
);
1219 memcpy(dcqe
, scqe
, dsize
);
1220 dcqe64
->op_own
= (dcqe64
->op_own
& ~MLX5_CQE_OWNER_MASK
) | sw_own
;
1223 scqe
= get_sw_cqe(cq
, i
);
1224 scqe64
= ssize
== 64 ? scqe
: scqe
+ 64;
1226 mlx5_ib_warn(dev
, "expected cqe in sw ownership\n");
1230 if (scqe
== start_cqe
) {
1231 pr_warn("resize CQ failed to get resize CQE, CQN 0x%x\n",
1236 ++cq
->mcq
.cons_index
;
1240 int mlx5_ib_resize_cq(struct ib_cq
*ibcq
, int entries
, struct ib_udata
*udata
)
1242 struct mlx5_ib_dev
*dev
= to_mdev(ibcq
->device
);
1243 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
1244 struct mlx5_modify_cq_mbox_in
*in
;
1249 int uninitialized_var(cqe_size
);
1250 unsigned long flags
;
1252 if (!MLX5_CAP_GEN(dev
->mdev
, cq_resize
)) {
1253 pr_info("Firmware does not support resize CQ\n");
1258 entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
))) {
1259 mlx5_ib_warn(dev
, "wrong entries number %d, max %d\n",
1261 1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
));
1265 entries
= roundup_pow_of_two(entries
+ 1);
1266 if (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
)) + 1)
1269 if (entries
== ibcq
->cqe
+ 1)
1272 mutex_lock(&cq
->resize_mutex
);
1274 err
= resize_user(dev
, cq
, entries
, udata
, &npas
, &page_shift
,
1278 err
= resize_kernel(dev
, cq
, entries
, cqe_size
);
1280 npas
= cq
->resize_buf
->buf
.npages
;
1281 page_shift
= cq
->resize_buf
->buf
.page_shift
;
1288 inlen
= sizeof(*in
) + npas
* sizeof(in
->pas
[0]);
1289 in
= mlx5_vzalloc(inlen
);
1296 mlx5_ib_populate_pas(dev
, cq
->resize_umem
, page_shift
,
1299 mlx5_fill_page_array(&cq
->resize_buf
->buf
, in
->pas
);
1301 in
->field_select
= cpu_to_be32(MLX5_MODIFY_CQ_MASK_LOG_SIZE
|
1302 MLX5_MODIFY_CQ_MASK_PG_OFFSET
|
1303 MLX5_MODIFY_CQ_MASK_PG_SIZE
);
1304 in
->ctx
.log_pg_sz
= page_shift
- MLX5_ADAPTER_PAGE_SHIFT
;
1305 in
->ctx
.cqe_sz_flags
= cqe_sz_to_mlx_sz(cqe_size
) << 5;
1306 in
->ctx
.page_offset
= 0;
1307 in
->ctx
.log_sz_usr_page
= cpu_to_be32(ilog2(entries
) << 24);
1308 in
->hdr
.opmod
= cpu_to_be16(MLX5_CQ_OPMOD_RESIZE
);
1309 in
->cqn
= cpu_to_be32(cq
->mcq
.cqn
);
1311 err
= mlx5_core_modify_cq(dev
->mdev
, &cq
->mcq
, in
, inlen
);
1316 cq
->ibcq
.cqe
= entries
- 1;
1317 ib_umem_release(cq
->buf
.umem
);
1318 cq
->buf
.umem
= cq
->resize_umem
;
1319 cq
->resize_umem
= NULL
;
1321 struct mlx5_ib_cq_buf tbuf
;
1324 spin_lock_irqsave(&cq
->lock
, flags
);
1325 if (cq
->resize_buf
) {
1326 err
= copy_resize_cqes(cq
);
1329 cq
->buf
= *cq
->resize_buf
;
1330 kfree(cq
->resize_buf
);
1331 cq
->resize_buf
= NULL
;
1335 cq
->ibcq
.cqe
= entries
- 1;
1336 spin_unlock_irqrestore(&cq
->lock
, flags
);
1338 free_cq_buf(dev
, &tbuf
);
1340 mutex_unlock(&cq
->resize_mutex
);
1352 un_resize_kernel(dev
, cq
);
1354 mutex_unlock(&cq
->resize_mutex
);
1358 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev
*dev
, struct ib_cq
*ibcq
)
1360 struct mlx5_ib_cq
*cq
;
1366 return cq
->cqe_size
;
1369 /* Called from atomic context */
1370 int mlx5_ib_generate_wc(struct ib_cq
*ibcq
, struct ib_wc
*wc
)
1372 struct mlx5_ib_wc
*soft_wc
;
1373 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
1374 unsigned long flags
;
1376 soft_wc
= kmalloc(sizeof(*soft_wc
), GFP_ATOMIC
);
1381 spin_lock_irqsave(&cq
->lock
, flags
);
1382 list_add_tail(&soft_wc
->list
, &cq
->wc_list
);
1383 if (cq
->notify_flags
== IB_CQ_NEXT_COMP
||
1384 wc
->status
!= IB_WC_SUCCESS
) {
1385 cq
->notify_flags
= 0;
1386 schedule_work(&cq
->notify_work
);
1388 spin_unlock_irqrestore(&cq
->lock
, flags
);