Commit | Line | Data |
---|---|---|
e126ba97 EC |
1 | /* |
2 | * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | ||
33 | #include <linux/kref.h> | |
34 | #include <rdma/ib_umem.h> | |
35 | #include "mlx5_ib.h" | |
36 | #include "user.h" | |
37 | ||
38 | static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq) | |
39 | { | |
40 | struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; | |
41 | ||
42 | ibcq->comp_handler(ibcq, ibcq->cq_context); | |
43 | } | |
44 | ||
45 | static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type) | |
46 | { | |
47 | struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq); | |
48 | struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); | |
49 | struct ib_cq *ibcq = &cq->ibcq; | |
50 | struct ib_event event; | |
51 | ||
52 | if (type != MLX5_EVENT_TYPE_CQ_ERROR) { | |
53 | mlx5_ib_warn(dev, "Unexpected event type %d on CQ %06x\n", | |
54 | type, mcq->cqn); | |
55 | return; | |
56 | } | |
57 | ||
58 | if (ibcq->event_handler) { | |
59 | event.device = &dev->ib_dev; | |
60 | event.event = IB_EVENT_CQ_ERR; | |
61 | event.element.cq = ibcq; | |
62 | ibcq->event_handler(&event, ibcq->cq_context); | |
63 | } | |
64 | } | |
65 | ||
66 | static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size) | |
67 | { | |
68 | return mlx5_buf_offset(&buf->buf, n * size); | |
69 | } | |
70 | ||
71 | static void *get_cqe(struct mlx5_ib_cq *cq, int n) | |
72 | { | |
73 | return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz); | |
74 | } | |
75 | ||
76 | static void *get_sw_cqe(struct mlx5_ib_cq *cq, int n) | |
77 | { | |
78 | void *cqe = get_cqe(cq, n & cq->ibcq.cqe); | |
79 | struct mlx5_cqe64 *cqe64; | |
80 | ||
81 | cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; | |
82 | return ((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ | |
83 | !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; | |
84 | } | |
85 | ||
86 | static void *next_cqe_sw(struct mlx5_ib_cq *cq) | |
87 | { | |
88 | return get_sw_cqe(cq, cq->mcq.cons_index); | |
89 | } | |
90 | ||
91 | static enum ib_wc_opcode get_umr_comp(struct mlx5_ib_wq *wq, int idx) | |
92 | { | |
93 | switch (wq->wr_data[idx]) { | |
94 | case MLX5_IB_WR_UMR: | |
95 | return 0; | |
96 | ||
97 | case IB_WR_LOCAL_INV: | |
98 | return IB_WC_LOCAL_INV; | |
99 | ||
100 | case IB_WR_FAST_REG_MR: | |
101 | return IB_WC_FAST_REG_MR; | |
102 | ||
103 | default: | |
104 | pr_warn("unknown completion status\n"); | |
105 | return 0; | |
106 | } | |
107 | } | |
108 | ||
109 | static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe, | |
110 | struct mlx5_ib_wq *wq, int idx) | |
111 | { | |
112 | wc->wc_flags = 0; | |
113 | switch (be32_to_cpu(cqe->sop_drop_qpn) >> 24) { | |
114 | case MLX5_OPCODE_RDMA_WRITE_IMM: | |
115 | wc->wc_flags |= IB_WC_WITH_IMM; | |
116 | case MLX5_OPCODE_RDMA_WRITE: | |
117 | wc->opcode = IB_WC_RDMA_WRITE; | |
118 | break; | |
119 | case MLX5_OPCODE_SEND_IMM: | |
120 | wc->wc_flags |= IB_WC_WITH_IMM; | |
121 | case MLX5_OPCODE_SEND: | |
122 | case MLX5_OPCODE_SEND_INVAL: | |
123 | wc->opcode = IB_WC_SEND; | |
124 | break; | |
125 | case MLX5_OPCODE_RDMA_READ: | |
126 | wc->opcode = IB_WC_RDMA_READ; | |
127 | wc->byte_len = be32_to_cpu(cqe->byte_cnt); | |
128 | break; | |
129 | case MLX5_OPCODE_ATOMIC_CS: | |
130 | wc->opcode = IB_WC_COMP_SWAP; | |
131 | wc->byte_len = 8; | |
132 | break; | |
133 | case MLX5_OPCODE_ATOMIC_FA: | |
134 | wc->opcode = IB_WC_FETCH_ADD; | |
135 | wc->byte_len = 8; | |
136 | break; | |
137 | case MLX5_OPCODE_ATOMIC_MASKED_CS: | |
138 | wc->opcode = IB_WC_MASKED_COMP_SWAP; | |
139 | wc->byte_len = 8; | |
140 | break; | |
141 | case MLX5_OPCODE_ATOMIC_MASKED_FA: | |
142 | wc->opcode = IB_WC_MASKED_FETCH_ADD; | |
143 | wc->byte_len = 8; | |
144 | break; | |
145 | case MLX5_OPCODE_BIND_MW: | |
146 | wc->opcode = IB_WC_BIND_MW; | |
147 | break; | |
148 | case MLX5_OPCODE_UMR: | |
149 | wc->opcode = get_umr_comp(wq, idx); | |
150 | break; | |
151 | } | |
152 | } | |
153 | ||
154 | enum { | |
155 | MLX5_GRH_IN_BUFFER = 1, | |
156 | MLX5_GRH_IN_CQE = 2, | |
157 | }; | |
158 | ||
159 | static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, | |
160 | struct mlx5_ib_qp *qp) | |
161 | { | |
162 | struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); | |
163 | struct mlx5_ib_srq *srq; | |
164 | struct mlx5_ib_wq *wq; | |
165 | u16 wqe_ctr; | |
166 | u8 g; | |
167 | ||
168 | if (qp->ibqp.srq || qp->ibqp.xrcd) { | |
169 | struct mlx5_core_srq *msrq = NULL; | |
170 | ||
171 | if (qp->ibqp.xrcd) { | |
172 | msrq = mlx5_core_get_srq(&dev->mdev, | |
173 | be32_to_cpu(cqe->srqn)); | |
174 | srq = to_mibsrq(msrq); | |
175 | } else { | |
176 | srq = to_msrq(qp->ibqp.srq); | |
177 | } | |
178 | if (srq) { | |
179 | wqe_ctr = be16_to_cpu(cqe->wqe_counter); | |
180 | wc->wr_id = srq->wrid[wqe_ctr]; | |
181 | mlx5_ib_free_srq_wqe(srq, wqe_ctr); | |
182 | if (msrq && atomic_dec_and_test(&msrq->refcount)) | |
183 | complete(&msrq->free); | |
184 | } | |
185 | } else { | |
186 | wq = &qp->rq; | |
187 | wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; | |
188 | ++wq->tail; | |
189 | } | |
190 | wc->byte_len = be32_to_cpu(cqe->byte_cnt); | |
191 | ||
192 | switch (cqe->op_own >> 4) { | |
193 | case MLX5_CQE_RESP_WR_IMM: | |
194 | wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; | |
195 | wc->wc_flags = IB_WC_WITH_IMM; | |
196 | wc->ex.imm_data = cqe->imm_inval_pkey; | |
197 | break; | |
198 | case MLX5_CQE_RESP_SEND: | |
199 | wc->opcode = IB_WC_RECV; | |
200 | wc->wc_flags = 0; | |
201 | break; | |
202 | case MLX5_CQE_RESP_SEND_IMM: | |
203 | wc->opcode = IB_WC_RECV; | |
204 | wc->wc_flags = IB_WC_WITH_IMM; | |
205 | wc->ex.imm_data = cqe->imm_inval_pkey; | |
206 | break; | |
207 | case MLX5_CQE_RESP_SEND_INV: | |
208 | wc->opcode = IB_WC_RECV; | |
209 | wc->wc_flags = IB_WC_WITH_INVALIDATE; | |
210 | wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey); | |
211 | break; | |
212 | } | |
213 | wc->slid = be16_to_cpu(cqe->slid); | |
214 | wc->sl = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0xf; | |
215 | wc->src_qp = be32_to_cpu(cqe->flags_rqpn) & 0xffffff; | |
216 | wc->dlid_path_bits = cqe->ml_path; | |
217 | g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3; | |
218 | wc->wc_flags |= g ? IB_WC_GRH : 0; | |
219 | wc->pkey_index = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff; | |
220 | } | |
221 | ||
222 | static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe) | |
223 | { | |
224 | __be32 *p = (__be32 *)cqe; | |
225 | int i; | |
226 | ||
227 | mlx5_ib_warn(dev, "dump error cqe\n"); | |
228 | for (i = 0; i < sizeof(*cqe) / 16; i++, p += 4) | |
229 | pr_info("%08x %08x %08x %08x\n", be32_to_cpu(p[0]), | |
230 | be32_to_cpu(p[1]), be32_to_cpu(p[2]), | |
231 | be32_to_cpu(p[3])); | |
232 | } | |
233 | ||
234 | static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, | |
235 | struct mlx5_err_cqe *cqe, | |
236 | struct ib_wc *wc) | |
237 | { | |
238 | int dump = 1; | |
239 | ||
240 | switch (cqe->syndrome) { | |
241 | case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR: | |
242 | wc->status = IB_WC_LOC_LEN_ERR; | |
243 | break; | |
244 | case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR: | |
245 | wc->status = IB_WC_LOC_QP_OP_ERR; | |
246 | break; | |
247 | case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR: | |
248 | wc->status = IB_WC_LOC_PROT_ERR; | |
249 | break; | |
250 | case MLX5_CQE_SYNDROME_WR_FLUSH_ERR: | |
251 | dump = 0; | |
252 | wc->status = IB_WC_WR_FLUSH_ERR; | |
253 | break; | |
254 | case MLX5_CQE_SYNDROME_MW_BIND_ERR: | |
255 | wc->status = IB_WC_MW_BIND_ERR; | |
256 | break; | |
257 | case MLX5_CQE_SYNDROME_BAD_RESP_ERR: | |
258 | wc->status = IB_WC_BAD_RESP_ERR; | |
259 | break; | |
260 | case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR: | |
261 | wc->status = IB_WC_LOC_ACCESS_ERR; | |
262 | break; | |
263 | case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: | |
264 | wc->status = IB_WC_REM_INV_REQ_ERR; | |
265 | break; | |
266 | case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR: | |
267 | wc->status = IB_WC_REM_ACCESS_ERR; | |
268 | break; | |
269 | case MLX5_CQE_SYNDROME_REMOTE_OP_ERR: | |
270 | wc->status = IB_WC_REM_OP_ERR; | |
271 | break; | |
272 | case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: | |
273 | wc->status = IB_WC_RETRY_EXC_ERR; | |
274 | dump = 0; | |
275 | break; | |
276 | case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR: | |
277 | wc->status = IB_WC_RNR_RETRY_EXC_ERR; | |
278 | dump = 0; | |
279 | break; | |
280 | case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR: | |
281 | wc->status = IB_WC_REM_ABORT_ERR; | |
282 | break; | |
283 | default: | |
284 | wc->status = IB_WC_GENERAL_ERR; | |
285 | break; | |
286 | } | |
287 | ||
288 | wc->vendor_err = cqe->vendor_err_synd; | |
289 | if (dump) | |
290 | dump_cqe(dev, cqe); | |
291 | } | |
292 | ||
293 | static int is_atomic_response(struct mlx5_ib_qp *qp, uint16_t idx) | |
294 | { | |
295 | /* TBD: waiting decision | |
296 | */ | |
297 | return 0; | |
298 | } | |
299 | ||
300 | static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp *qp, uint16_t idx) | |
301 | { | |
302 | struct mlx5_wqe_data_seg *dpseg; | |
303 | void *addr; | |
304 | ||
305 | dpseg = mlx5_get_send_wqe(qp, idx) + sizeof(struct mlx5_wqe_ctrl_seg) + | |
306 | sizeof(struct mlx5_wqe_raddr_seg) + | |
307 | sizeof(struct mlx5_wqe_atomic_seg); | |
308 | addr = (void *)(unsigned long)be64_to_cpu(dpseg->addr); | |
309 | return addr; | |
310 | } | |
311 | ||
312 | static void handle_atomic(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, | |
313 | uint16_t idx) | |
314 | { | |
315 | void *addr; | |
316 | int byte_count; | |
317 | int i; | |
318 | ||
319 | if (!is_atomic_response(qp, idx)) | |
320 | return; | |
321 | ||
322 | byte_count = be32_to_cpu(cqe64->byte_cnt); | |
323 | addr = mlx5_get_atomic_laddr(qp, idx); | |
324 | ||
325 | if (byte_count == 4) { | |
326 | *(uint32_t *)addr = be32_to_cpu(*((__be32 *)addr)); | |
327 | } else { | |
328 | for (i = 0; i < byte_count; i += 8) { | |
329 | *(uint64_t *)addr = be64_to_cpu(*((__be64 *)addr)); | |
330 | addr += 8; | |
331 | } | |
332 | } | |
333 | ||
334 | return; | |
335 | } | |
336 | ||
337 | static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, | |
338 | u16 tail, u16 head) | |
339 | { | |
340 | int idx; | |
341 | ||
342 | do { | |
343 | idx = tail & (qp->sq.wqe_cnt - 1); | |
344 | handle_atomic(qp, cqe64, idx); | |
345 | if (idx == head) | |
346 | break; | |
347 | ||
348 | tail = qp->sq.w_list[idx].next; | |
349 | } while (1); | |
350 | tail = qp->sq.w_list[idx].next; | |
351 | qp->sq.last_poll = tail; | |
352 | } | |
353 | ||
354 | static int mlx5_poll_one(struct mlx5_ib_cq *cq, | |
355 | struct mlx5_ib_qp **cur_qp, | |
356 | struct ib_wc *wc) | |
357 | { | |
358 | struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); | |
359 | struct mlx5_err_cqe *err_cqe; | |
360 | struct mlx5_cqe64 *cqe64; | |
361 | struct mlx5_core_qp *mqp; | |
362 | struct mlx5_ib_wq *wq; | |
363 | uint8_t opcode; | |
364 | uint32_t qpn; | |
365 | u16 wqe_ctr; | |
366 | void *cqe; | |
367 | int idx; | |
368 | ||
369 | cqe = next_cqe_sw(cq); | |
370 | if (!cqe) | |
371 | return -EAGAIN; | |
372 | ||
373 | cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; | |
374 | ||
375 | ++cq->mcq.cons_index; | |
376 | ||
377 | /* Make sure we read CQ entry contents after we've checked the | |
378 | * ownership bit. | |
379 | */ | |
380 | rmb(); | |
381 | ||
382 | /* TBD: resize CQ */ | |
383 | ||
384 | qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff; | |
385 | if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) { | |
386 | /* We do not have to take the QP table lock here, | |
387 | * because CQs will be locked while QPs are removed | |
388 | * from the table. | |
389 | */ | |
390 | mqp = __mlx5_qp_lookup(&dev->mdev, qpn); | |
391 | if (unlikely(!mqp)) { | |
392 | mlx5_ib_warn(dev, "CQE@CQ %06x for unknown QPN %6x\n", | |
393 | cq->mcq.cqn, qpn); | |
394 | return -EINVAL; | |
395 | } | |
396 | ||
397 | *cur_qp = to_mibqp(mqp); | |
398 | } | |
399 | ||
400 | wc->qp = &(*cur_qp)->ibqp; | |
401 | opcode = cqe64->op_own >> 4; | |
402 | switch (opcode) { | |
403 | case MLX5_CQE_REQ: | |
404 | wq = &(*cur_qp)->sq; | |
405 | wqe_ctr = be16_to_cpu(cqe64->wqe_counter); | |
406 | idx = wqe_ctr & (wq->wqe_cnt - 1); | |
407 | handle_good_req(wc, cqe64, wq, idx); | |
408 | handle_atomics(*cur_qp, cqe64, wq->last_poll, idx); | |
409 | wc->wr_id = wq->wrid[idx]; | |
410 | wq->tail = wq->wqe_head[idx] + 1; | |
411 | wc->status = IB_WC_SUCCESS; | |
412 | break; | |
413 | case MLX5_CQE_RESP_WR_IMM: | |
414 | case MLX5_CQE_RESP_SEND: | |
415 | case MLX5_CQE_RESP_SEND_IMM: | |
416 | case MLX5_CQE_RESP_SEND_INV: | |
417 | handle_responder(wc, cqe64, *cur_qp); | |
418 | wc->status = IB_WC_SUCCESS; | |
419 | break; | |
420 | case MLX5_CQE_RESIZE_CQ: | |
421 | break; | |
422 | case MLX5_CQE_REQ_ERR: | |
423 | case MLX5_CQE_RESP_ERR: | |
424 | err_cqe = (struct mlx5_err_cqe *)cqe64; | |
425 | mlx5_handle_error_cqe(dev, err_cqe, wc); | |
426 | mlx5_ib_dbg(dev, "%s error cqe on cqn 0x%x:\n", | |
427 | opcode == MLX5_CQE_REQ_ERR ? | |
428 | "Requestor" : "Responder", cq->mcq.cqn); | |
429 | mlx5_ib_dbg(dev, "syndrome 0x%x, vendor syndrome 0x%x\n", | |
430 | err_cqe->syndrome, err_cqe->vendor_err_synd); | |
431 | if (opcode == MLX5_CQE_REQ_ERR) { | |
432 | wq = &(*cur_qp)->sq; | |
433 | wqe_ctr = be16_to_cpu(cqe64->wqe_counter); | |
434 | idx = wqe_ctr & (wq->wqe_cnt - 1); | |
435 | wc->wr_id = wq->wrid[idx]; | |
436 | wq->tail = wq->wqe_head[idx] + 1; | |
437 | } else { | |
438 | struct mlx5_ib_srq *srq; | |
439 | ||
440 | if ((*cur_qp)->ibqp.srq) { | |
441 | srq = to_msrq((*cur_qp)->ibqp.srq); | |
442 | wqe_ctr = be16_to_cpu(cqe64->wqe_counter); | |
443 | wc->wr_id = srq->wrid[wqe_ctr]; | |
444 | mlx5_ib_free_srq_wqe(srq, wqe_ctr); | |
445 | } else { | |
446 | wq = &(*cur_qp)->rq; | |
447 | wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; | |
448 | ++wq->tail; | |
449 | } | |
450 | } | |
451 | break; | |
452 | } | |
453 | ||
454 | return 0; | |
455 | } | |
456 | ||
457 | int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) | |
458 | { | |
459 | struct mlx5_ib_cq *cq = to_mcq(ibcq); | |
460 | struct mlx5_ib_qp *cur_qp = NULL; | |
461 | unsigned long flags; | |
462 | int npolled; | |
463 | int err = 0; | |
464 | ||
465 | spin_lock_irqsave(&cq->lock, flags); | |
466 | ||
467 | for (npolled = 0; npolled < num_entries; npolled++) { | |
468 | err = mlx5_poll_one(cq, &cur_qp, wc + npolled); | |
469 | if (err) | |
470 | break; | |
471 | } | |
472 | ||
473 | if (npolled) | |
474 | mlx5_cq_set_ci(&cq->mcq); | |
475 | ||
476 | spin_unlock_irqrestore(&cq->lock, flags); | |
477 | ||
478 | if (err == 0 || err == -EAGAIN) | |
479 | return npolled; | |
480 | else | |
481 | return err; | |
482 | } | |
483 | ||
484 | int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) | |
485 | { | |
486 | mlx5_cq_arm(&to_mcq(ibcq)->mcq, | |
487 | (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? | |
488 | MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT, | |
489 | to_mdev(ibcq->device)->mdev.priv.uuari.uars[0].map, | |
490 | MLX5_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->mdev.priv.cq_uar_lock)); | |
491 | ||
492 | return 0; | |
493 | } | |
494 | ||
495 | static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf, | |
496 | int nent, int cqe_size) | |
497 | { | |
498 | int err; | |
499 | ||
500 | err = mlx5_buf_alloc(&dev->mdev, nent * cqe_size, | |
501 | PAGE_SIZE * 2, &buf->buf); | |
502 | if (err) | |
503 | return err; | |
504 | ||
505 | buf->cqe_size = cqe_size; | |
506 | ||
507 | return 0; | |
508 | } | |
509 | ||
510 | static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf) | |
511 | { | |
512 | mlx5_buf_free(&dev->mdev, &buf->buf); | |
513 | } | |
514 | ||
515 | static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, | |
516 | struct ib_ucontext *context, struct mlx5_ib_cq *cq, | |
517 | int entries, struct mlx5_create_cq_mbox_in **cqb, | |
518 | int *cqe_size, int *index, int *inlen) | |
519 | { | |
520 | struct mlx5_ib_create_cq ucmd; | |
521 | int page_shift; | |
522 | int npages; | |
523 | int ncont; | |
524 | int err; | |
525 | ||
526 | if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) | |
527 | return -EFAULT; | |
528 | ||
529 | if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128) | |
530 | return -EINVAL; | |
531 | ||
532 | *cqe_size = ucmd.cqe_size; | |
533 | ||
534 | cq->buf.umem = ib_umem_get(context, ucmd.buf_addr, | |
535 | entries * ucmd.cqe_size, | |
536 | IB_ACCESS_LOCAL_WRITE, 1); | |
537 | if (IS_ERR(cq->buf.umem)) { | |
538 | err = PTR_ERR(cq->buf.umem); | |
539 | return err; | |
540 | } | |
541 | ||
542 | err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr, | |
543 | &cq->db); | |
544 | if (err) | |
545 | goto err_umem; | |
546 | ||
547 | mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift, | |
548 | &ncont, NULL); | |
549 | mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n", | |
550 | ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont); | |
551 | ||
552 | *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * ncont; | |
553 | *cqb = mlx5_vzalloc(*inlen); | |
554 | if (!*cqb) { | |
555 | err = -ENOMEM; | |
556 | goto err_db; | |
557 | } | |
558 | mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, 0); | |
cf1c5e1f | 559 | (*cqb)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT; |
e126ba97 EC |
560 | |
561 | *index = to_mucontext(context)->uuari.uars[0].index; | |
562 | ||
563 | return 0; | |
564 | ||
565 | err_db: | |
566 | mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db); | |
567 | ||
568 | err_umem: | |
569 | ib_umem_release(cq->buf.umem); | |
570 | return err; | |
571 | } | |
572 | ||
573 | static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context) | |
574 | { | |
575 | mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db); | |
576 | ib_umem_release(cq->buf.umem); | |
577 | } | |
578 | ||
579 | static void init_cq_buf(struct mlx5_ib_cq *cq, int nent) | |
580 | { | |
581 | int i; | |
582 | void *cqe; | |
583 | struct mlx5_cqe64 *cqe64; | |
584 | ||
585 | for (i = 0; i < nent; i++) { | |
586 | cqe = get_cqe(cq, i); | |
587 | cqe64 = (cq->buf.cqe_size == 64) ? cqe : cqe + 64; | |
588 | cqe64->op_own = 0xf1; | |
589 | } | |
590 | } | |
591 | ||
592 | static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, | |
593 | int entries, int cqe_size, | |
594 | struct mlx5_create_cq_mbox_in **cqb, | |
595 | int *index, int *inlen) | |
596 | { | |
597 | int err; | |
598 | ||
599 | err = mlx5_db_alloc(&dev->mdev, &cq->db); | |
600 | if (err) | |
601 | return err; | |
602 | ||
603 | cq->mcq.set_ci_db = cq->db.db; | |
604 | cq->mcq.arm_db = cq->db.db + 1; | |
605 | *cq->mcq.set_ci_db = 0; | |
606 | *cq->mcq.arm_db = 0; | |
607 | cq->mcq.cqe_sz = cqe_size; | |
608 | ||
609 | err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size); | |
610 | if (err) | |
611 | goto err_db; | |
612 | ||
613 | init_cq_buf(cq, entries); | |
614 | ||
615 | *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * cq->buf.buf.npages; | |
616 | *cqb = mlx5_vzalloc(*inlen); | |
617 | if (!*cqb) { | |
618 | err = -ENOMEM; | |
619 | goto err_buf; | |
620 | } | |
621 | mlx5_fill_page_array(&cq->buf.buf, (*cqb)->pas); | |
622 | ||
1b77d2bd | 623 | (*cqb)->ctx.log_pg_sz = cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; |
e126ba97 EC |
624 | *index = dev->mdev.priv.uuari.uars[0].index; |
625 | ||
626 | return 0; | |
627 | ||
628 | err_buf: | |
629 | free_cq_buf(dev, &cq->buf); | |
630 | ||
631 | err_db: | |
632 | mlx5_db_free(&dev->mdev, &cq->db); | |
633 | return err; | |
634 | } | |
635 | ||
636 | static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) | |
637 | { | |
638 | free_cq_buf(dev, &cq->buf); | |
639 | mlx5_db_free(&dev->mdev, &cq->db); | |
640 | } | |
641 | ||
642 | struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, int entries, | |
643 | int vector, struct ib_ucontext *context, | |
644 | struct ib_udata *udata) | |
645 | { | |
646 | struct mlx5_create_cq_mbox_in *cqb = NULL; | |
647 | struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
648 | struct mlx5_ib_cq *cq; | |
649 | int uninitialized_var(index); | |
650 | int uninitialized_var(inlen); | |
651 | int cqe_size; | |
652 | int irqn; | |
653 | int eqn; | |
654 | int err; | |
655 | ||
51ee86a4 EC |
656 | if (entries < 0) |
657 | return ERR_PTR(-EINVAL); | |
658 | ||
e126ba97 | 659 | entries = roundup_pow_of_two(entries + 1); |
51ee86a4 | 660 | if (entries > dev->mdev.caps.max_cqes) |
e126ba97 EC |
661 | return ERR_PTR(-EINVAL); |
662 | ||
663 | cq = kzalloc(sizeof(*cq), GFP_KERNEL); | |
664 | if (!cq) | |
665 | return ERR_PTR(-ENOMEM); | |
666 | ||
667 | cq->ibcq.cqe = entries - 1; | |
668 | mutex_init(&cq->resize_mutex); | |
669 | spin_lock_init(&cq->lock); | |
670 | cq->resize_buf = NULL; | |
671 | cq->resize_umem = NULL; | |
672 | ||
673 | if (context) { | |
674 | err = create_cq_user(dev, udata, context, cq, entries, | |
675 | &cqb, &cqe_size, &index, &inlen); | |
676 | if (err) | |
677 | goto err_create; | |
678 | } else { | |
679 | /* for now choose 64 bytes till we have a proper interface */ | |
680 | cqe_size = 64; | |
681 | err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, | |
682 | &index, &inlen); | |
683 | if (err) | |
684 | goto err_create; | |
685 | } | |
686 | ||
687 | cq->cqe_size = cqe_size; | |
688 | cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5; | |
689 | cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index); | |
690 | err = mlx5_vector2eqn(dev, vector, &eqn, &irqn); | |
691 | if (err) | |
692 | goto err_cqb; | |
693 | ||
694 | cqb->ctx.c_eqn = cpu_to_be16(eqn); | |
695 | cqb->ctx.db_record_addr = cpu_to_be64(cq->db.dma); | |
696 | ||
697 | err = mlx5_core_create_cq(&dev->mdev, &cq->mcq, cqb, inlen); | |
698 | if (err) | |
699 | goto err_cqb; | |
700 | ||
701 | mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn); | |
702 | cq->mcq.irqn = irqn; | |
703 | cq->mcq.comp = mlx5_ib_cq_comp; | |
704 | cq->mcq.event = mlx5_ib_cq_event; | |
705 | ||
706 | if (context) | |
707 | if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { | |
708 | err = -EFAULT; | |
709 | goto err_cmd; | |
710 | } | |
711 | ||
712 | ||
713 | mlx5_vfree(cqb); | |
714 | return &cq->ibcq; | |
715 | ||
716 | err_cmd: | |
717 | mlx5_core_destroy_cq(&dev->mdev, &cq->mcq); | |
718 | ||
719 | err_cqb: | |
720 | mlx5_vfree(cqb); | |
721 | if (context) | |
722 | destroy_cq_user(cq, context); | |
723 | else | |
724 | destroy_cq_kernel(dev, cq); | |
725 | ||
726 | err_create: | |
727 | kfree(cq); | |
728 | ||
729 | return ERR_PTR(err); | |
730 | } | |
731 | ||
732 | ||
733 | int mlx5_ib_destroy_cq(struct ib_cq *cq) | |
734 | { | |
735 | struct mlx5_ib_dev *dev = to_mdev(cq->device); | |
736 | struct mlx5_ib_cq *mcq = to_mcq(cq); | |
737 | struct ib_ucontext *context = NULL; | |
738 | ||
739 | if (cq->uobject) | |
740 | context = cq->uobject->context; | |
741 | ||
742 | mlx5_core_destroy_cq(&dev->mdev, &mcq->mcq); | |
743 | if (context) | |
744 | destroy_cq_user(mcq, context); | |
745 | else | |
746 | destroy_cq_kernel(dev, mcq); | |
747 | ||
748 | kfree(mcq); | |
749 | ||
750 | return 0; | |
751 | } | |
752 | ||
cfd8f1d4 | 753 | static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn) |
e126ba97 | 754 | { |
cfd8f1d4 | 755 | return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff); |
e126ba97 EC |
756 | } |
757 | ||
758 | void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq) | |
759 | { | |
760 | struct mlx5_cqe64 *cqe64, *dest64; | |
761 | void *cqe, *dest; | |
762 | u32 prod_index; | |
763 | int nfreed = 0; | |
764 | u8 owner_bit; | |
765 | ||
766 | if (!cq) | |
767 | return; | |
768 | ||
769 | /* First we need to find the current producer index, so we | |
770 | * know where to start cleaning from. It doesn't matter if HW | |
771 | * adds new entries after this loop -- the QP we're worried | |
772 | * about is already in RESET, so the new entries won't come | |
773 | * from our QP and therefore don't need to be checked. | |
774 | */ | |
775 | for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++) | |
776 | if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) | |
777 | break; | |
778 | ||
779 | /* Now sweep backwards through the CQ, removing CQ entries | |
780 | * that match our QP by copying older entries on top of them. | |
781 | */ | |
782 | while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { | |
783 | cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); | |
784 | cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; | |
cfd8f1d4 ML |
785 | if (is_equal_rsn(cqe64, rsn)) { |
786 | if (srq && (ntohl(cqe64->srqn) & 0xffffff)) | |
e126ba97 EC |
787 | mlx5_ib_free_srq_wqe(srq, be16_to_cpu(cqe64->wqe_counter)); |
788 | ++nfreed; | |
789 | } else if (nfreed) { | |
790 | dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); | |
791 | dest64 = (cq->mcq.cqe_sz == 64) ? dest : dest + 64; | |
792 | owner_bit = dest64->op_own & MLX5_CQE_OWNER_MASK; | |
793 | memcpy(dest, cqe, cq->mcq.cqe_sz); | |
794 | dest64->op_own = owner_bit | | |
795 | (dest64->op_own & ~MLX5_CQE_OWNER_MASK); | |
796 | } | |
797 | } | |
798 | ||
799 | if (nfreed) { | |
800 | cq->mcq.cons_index += nfreed; | |
801 | /* Make sure update of buffer contents is done before | |
802 | * updating consumer index. | |
803 | */ | |
804 | wmb(); | |
805 | mlx5_cq_set_ci(&cq->mcq); | |
806 | } | |
807 | } | |
808 | ||
809 | void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq) | |
810 | { | |
811 | if (!cq) | |
812 | return; | |
813 | ||
814 | spin_lock_irq(&cq->lock); | |
815 | __mlx5_ib_cq_clean(cq, qpn, srq); | |
816 | spin_unlock_irq(&cq->lock); | |
817 | } | |
818 | ||
819 | int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) | |
820 | { | |
3bdb31f6 EC |
821 | struct mlx5_modify_cq_mbox_in *in; |
822 | struct mlx5_ib_dev *dev = to_mdev(cq->device); | |
823 | struct mlx5_ib_cq *mcq = to_mcq(cq); | |
824 | int err; | |
825 | u32 fsel; | |
826 | ||
827 | if (!(dev->mdev.caps.flags & MLX5_DEV_CAP_FLAG_CQ_MODER)) | |
828 | return -ENOSYS; | |
829 | ||
830 | in = kzalloc(sizeof(*in), GFP_KERNEL); | |
831 | if (!in) | |
832 | return -ENOMEM; | |
833 | ||
834 | in->cqn = cpu_to_be32(mcq->mcq.cqn); | |
835 | fsel = (MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT); | |
836 | in->ctx.cq_period = cpu_to_be16(cq_period); | |
837 | in->ctx.cq_max_count = cpu_to_be16(cq_count); | |
838 | in->field_select = cpu_to_be32(fsel); | |
839 | err = mlx5_core_modify_cq(&dev->mdev, &mcq->mcq, in); | |
840 | kfree(in); | |
841 | ||
842 | if (err) | |
843 | mlx5_ib_warn(dev, "modify cq 0x%x failed\n", mcq->mcq.cqn); | |
844 | ||
845 | return err; | |
e126ba97 EC |
846 | } |
847 | ||
848 | int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) | |
849 | { | |
850 | return -ENOSYS; | |
851 | } | |
852 | ||
853 | int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq) | |
854 | { | |
855 | struct mlx5_ib_cq *cq; | |
856 | ||
857 | if (!ibcq) | |
858 | return 128; | |
859 | ||
860 | cq = to_mcq(ibcq); | |
861 | return cq->cqe_size; | |
862 | } |