Commit | Line | Data |
---|---|---|
b038ced7 SW |
1 | /* |
2 | * Copyright (c) 2006 Chelsio, Inc. All rights reserved. | |
b038ced7 SW |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | #include "iwch_provider.h" | |
33 | #include "iwch.h" | |
34 | #include "iwch_cm.h" | |
35 | #include "cxio_hal.h" | |
36 | ||
37 | #define NO_SUPPORT -1 | |
38 | ||
2b540355 AB |
39 | static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, |
40 | u8 * flit_cnt) | |
b038ced7 SW |
41 | { |
42 | int i; | |
43 | u32 plen; | |
44 | ||
45 | switch (wr->opcode) { | |
46 | case IB_WR_SEND: | |
47 | case IB_WR_SEND_WITH_IMM: | |
48 | if (wr->send_flags & IB_SEND_SOLICITED) | |
49 | wqe->send.rdmaop = T3_SEND_WITH_SE; | |
50 | else | |
51 | wqe->send.rdmaop = T3_SEND; | |
52 | wqe->send.rem_stag = 0; | |
53 | break; | |
54 | #if 0 /* Not currently supported */ | |
55 | case TYPE_SEND_INVALIDATE: | |
56 | case TYPE_SEND_INVALIDATE_IMMEDIATE: | |
57 | wqe->send.rdmaop = T3_SEND_WITH_INV; | |
58 | wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
59 | break; | |
60 | case TYPE_SEND_SE_INVALIDATE: | |
61 | wqe->send.rdmaop = T3_SEND_WITH_SE_INV; | |
62 | wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
63 | break; | |
64 | #endif | |
65 | default: | |
66 | break; | |
67 | } | |
68 | if (wr->num_sge > T3_MAX_SGE) | |
69 | return -EINVAL; | |
70 | wqe->send.reserved[0] = 0; | |
71 | wqe->send.reserved[1] = 0; | |
72 | wqe->send.reserved[2] = 0; | |
73 | if (wr->opcode == IB_WR_SEND_WITH_IMM) { | |
74 | plen = 4; | |
75 | wqe->send.sgl[0].stag = wr->imm_data; | |
76 | wqe->send.sgl[0].len = __constant_cpu_to_be32(0); | |
77 | wqe->send.num_sgle = __constant_cpu_to_be32(0); | |
78 | *flit_cnt = 5; | |
79 | } else { | |
80 | plen = 0; | |
81 | for (i = 0; i < wr->num_sge; i++) { | |
82 | if ((plen + wr->sg_list[i].length) < plen) { | |
83 | return -EMSGSIZE; | |
84 | } | |
85 | plen += wr->sg_list[i].length; | |
86 | wqe->send.sgl[i].stag = | |
87 | cpu_to_be32(wr->sg_list[i].lkey); | |
88 | wqe->send.sgl[i].len = | |
89 | cpu_to_be32(wr->sg_list[i].length); | |
90 | wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); | |
91 | } | |
92 | wqe->send.num_sgle = cpu_to_be32(wr->num_sge); | |
93 | *flit_cnt = 4 + ((wr->num_sge) << 1); | |
94 | } | |
95 | wqe->send.plen = cpu_to_be32(plen); | |
96 | return 0; | |
97 | } | |
98 | ||
2b540355 AB |
99 | static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, |
100 | u8 *flit_cnt) | |
b038ced7 SW |
101 | { |
102 | int i; | |
103 | u32 plen; | |
104 | if (wr->num_sge > T3_MAX_SGE) | |
105 | return -EINVAL; | |
106 | wqe->write.rdmaop = T3_RDMA_WRITE; | |
107 | wqe->write.reserved[0] = 0; | |
108 | wqe->write.reserved[1] = 0; | |
109 | wqe->write.reserved[2] = 0; | |
110 | wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); | |
111 | wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); | |
112 | ||
113 | if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { | |
114 | plen = 4; | |
115 | wqe->write.sgl[0].stag = wr->imm_data; | |
116 | wqe->write.sgl[0].len = __constant_cpu_to_be32(0); | |
117 | wqe->write.num_sgle = __constant_cpu_to_be32(0); | |
118 | *flit_cnt = 6; | |
119 | } else { | |
120 | plen = 0; | |
121 | for (i = 0; i < wr->num_sge; i++) { | |
122 | if ((plen + wr->sg_list[i].length) < plen) { | |
123 | return -EMSGSIZE; | |
124 | } | |
125 | plen += wr->sg_list[i].length; | |
126 | wqe->write.sgl[i].stag = | |
127 | cpu_to_be32(wr->sg_list[i].lkey); | |
128 | wqe->write.sgl[i].len = | |
129 | cpu_to_be32(wr->sg_list[i].length); | |
130 | wqe->write.sgl[i].to = | |
131 | cpu_to_be64(wr->sg_list[i].addr); | |
132 | } | |
133 | wqe->write.num_sgle = cpu_to_be32(wr->num_sge); | |
134 | *flit_cnt = 5 + ((wr->num_sge) << 1); | |
135 | } | |
136 | wqe->write.plen = cpu_to_be32(plen); | |
137 | return 0; | |
138 | } | |
139 | ||
2b540355 AB |
140 | static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, |
141 | u8 *flit_cnt) | |
b038ced7 SW |
142 | { |
143 | if (wr->num_sge > 1) | |
144 | return -EINVAL; | |
145 | wqe->read.rdmaop = T3_READ_REQ; | |
146 | wqe->read.reserved[0] = 0; | |
147 | wqe->read.reserved[1] = 0; | |
148 | wqe->read.reserved[2] = 0; | |
149 | wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
150 | wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); | |
151 | wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); | |
152 | wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); | |
153 | wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); | |
154 | *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; | |
155 | return 0; | |
156 | } | |
157 | ||
158 | /* | |
159 | * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now. | |
160 | */ | |
2b540355 AB |
161 | static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, |
162 | u32 num_sgle, u32 * pbl_addr, u8 * page_size) | |
b038ced7 SW |
163 | { |
164 | int i; | |
165 | struct iwch_mr *mhp; | |
166 | u32 offset; | |
167 | for (i = 0; i < num_sgle; i++) { | |
168 | ||
169 | mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); | |
170 | if (!mhp) { | |
171 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
172 | return -EIO; | |
173 | } | |
174 | if (!mhp->attr.state) { | |
175 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
176 | return -EIO; | |
177 | } | |
178 | if (mhp->attr.zbva) { | |
179 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
180 | return -EIO; | |
181 | } | |
182 | ||
183 | if (sg_list[i].addr < mhp->attr.va_fbo) { | |
184 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
185 | return -EINVAL; | |
186 | } | |
187 | if (sg_list[i].addr + ((u64) sg_list[i].length) < | |
188 | sg_list[i].addr) { | |
189 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
190 | return -EINVAL; | |
191 | } | |
192 | if (sg_list[i].addr + ((u64) sg_list[i].length) > | |
193 | mhp->attr.va_fbo + ((u64) mhp->attr.len)) { | |
194 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
195 | return -EINVAL; | |
196 | } | |
197 | offset = sg_list[i].addr - mhp->attr.va_fbo; | |
198 | offset += ((u32) mhp->attr.va_fbo) % | |
199 | (1UL << (12 + mhp->attr.page_size)); | |
200 | pbl_addr[i] = ((mhp->attr.pbl_addr - | |
201 | rhp->rdev.rnic_info.pbl_base) >> 3) + | |
202 | (offset >> (12 + mhp->attr.page_size)); | |
203 | page_size[i] = mhp->attr.page_size; | |
204 | } | |
205 | return 0; | |
206 | } | |
207 | ||
2b540355 AB |
208 | static int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe, |
209 | struct ib_recv_wr *wr) | |
b038ced7 SW |
210 | { |
211 | int i, err = 0; | |
212 | u32 pbl_addr[4]; | |
213 | u8 page_size[4]; | |
214 | if (wr->num_sge > T3_MAX_SGE) | |
215 | return -EINVAL; | |
216 | err = iwch_sgl2pbl_map(rhp, wr->sg_list, wr->num_sge, pbl_addr, | |
217 | page_size); | |
218 | if (err) | |
219 | return err; | |
220 | wqe->recv.pagesz[0] = page_size[0]; | |
221 | wqe->recv.pagesz[1] = page_size[1]; | |
222 | wqe->recv.pagesz[2] = page_size[2]; | |
223 | wqe->recv.pagesz[3] = page_size[3]; | |
224 | wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); | |
225 | for (i = 0; i < wr->num_sge; i++) { | |
226 | wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); | |
227 | wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); | |
228 | ||
229 | /* to in the WQE == the offset into the page */ | |
230 | wqe->recv.sgl[i].to = cpu_to_be64(((u32) wr->sg_list[i].addr) % | |
231 | (1UL << (12 + page_size[i]))); | |
232 | ||
233 | /* pbl_addr is the adapters address in the PBL */ | |
234 | wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); | |
235 | } | |
236 | for (; i < T3_MAX_SGE; i++) { | |
237 | wqe->recv.sgl[i].stag = 0; | |
238 | wqe->recv.sgl[i].len = 0; | |
239 | wqe->recv.sgl[i].to = 0; | |
240 | wqe->recv.pbl_addr[i] = 0; | |
241 | } | |
242 | return 0; | |
243 | } | |
244 | ||
245 | int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, | |
246 | struct ib_send_wr **bad_wr) | |
247 | { | |
248 | int err = 0; | |
249 | u8 t3_wr_flit_cnt; | |
250 | enum t3_wr_opcode t3_wr_opcode = 0; | |
251 | enum t3_wr_flags t3_wr_flags; | |
252 | struct iwch_qp *qhp; | |
253 | u32 idx; | |
254 | union t3_wr *wqe; | |
255 | u32 num_wrs; | |
256 | unsigned long flag; | |
257 | struct t3_swsq *sqp; | |
258 | ||
259 | qhp = to_iwch_qp(ibqp); | |
260 | spin_lock_irqsave(&qhp->lock, flag); | |
261 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
262 | spin_unlock_irqrestore(&qhp->lock, flag); | |
263 | return -EINVAL; | |
264 | } | |
265 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
266 | qhp->wq.sq_size_log2); | |
267 | if (num_wrs <= 0) { | |
268 | spin_unlock_irqrestore(&qhp->lock, flag); | |
269 | return -ENOMEM; | |
270 | } | |
271 | while (wr) { | |
272 | if (num_wrs == 0) { | |
273 | err = -ENOMEM; | |
274 | *bad_wr = wr; | |
275 | break; | |
276 | } | |
277 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
278 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
279 | t3_wr_flags = 0; | |
280 | if (wr->send_flags & IB_SEND_SOLICITED) | |
281 | t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; | |
282 | if (wr->send_flags & IB_SEND_FENCE) | |
283 | t3_wr_flags |= T3_READ_FENCE_FLAG; | |
284 | if (wr->send_flags & IB_SEND_SIGNALED) | |
285 | t3_wr_flags |= T3_COMPLETION_FLAG; | |
286 | sqp = qhp->wq.sq + | |
287 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
288 | switch (wr->opcode) { | |
289 | case IB_WR_SEND: | |
290 | case IB_WR_SEND_WITH_IMM: | |
291 | t3_wr_opcode = T3_WR_SEND; | |
292 | err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt); | |
293 | break; | |
294 | case IB_WR_RDMA_WRITE: | |
295 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
296 | t3_wr_opcode = T3_WR_WRITE; | |
297 | err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt); | |
298 | break; | |
299 | case IB_WR_RDMA_READ: | |
300 | t3_wr_opcode = T3_WR_READ; | |
301 | t3_wr_flags = 0; /* T3 reads are always signaled */ | |
302 | err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt); | |
303 | if (err) | |
304 | break; | |
305 | sqp->read_len = wqe->read.local_len; | |
306 | if (!qhp->wq.oldest_read) | |
307 | qhp->wq.oldest_read = sqp; | |
308 | break; | |
309 | default: | |
310 | PDBG("%s post of type=%d TBD!\n", __FUNCTION__, | |
311 | wr->opcode); | |
312 | err = -EINVAL; | |
313 | } | |
314 | if (err) { | |
315 | *bad_wr = wr; | |
316 | break; | |
317 | } | |
318 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
319 | sqp->wr_id = wr->wr_id; | |
320 | sqp->opcode = wr2opcode(t3_wr_opcode); | |
321 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
322 | sqp->complete = 0; | |
323 | sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); | |
324 | ||
325 | build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, | |
326 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
327 | 0, t3_wr_flit_cnt); | |
328 | PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", | |
329 | __FUNCTION__, (unsigned long long) wr->wr_id, idx, | |
330 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), | |
331 | sqp->opcode); | |
332 | wr = wr->next; | |
333 | num_wrs--; | |
334 | ++(qhp->wq.wptr); | |
335 | ++(qhp->wq.sq_wptr); | |
336 | } | |
337 | spin_unlock_irqrestore(&qhp->lock, flag); | |
338 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
339 | return err; | |
340 | } | |
341 | ||
342 | int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, | |
343 | struct ib_recv_wr **bad_wr) | |
344 | { | |
345 | int err = 0; | |
346 | struct iwch_qp *qhp; | |
347 | u32 idx; | |
348 | union t3_wr *wqe; | |
349 | u32 num_wrs; | |
350 | unsigned long flag; | |
351 | ||
352 | qhp = to_iwch_qp(ibqp); | |
353 | spin_lock_irqsave(&qhp->lock, flag); | |
354 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
355 | spin_unlock_irqrestore(&qhp->lock, flag); | |
356 | return -EINVAL; | |
357 | } | |
358 | num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, | |
359 | qhp->wq.rq_size_log2) - 1; | |
360 | if (!wr) { | |
361 | spin_unlock_irqrestore(&qhp->lock, flag); | |
362 | return -EINVAL; | |
363 | } | |
364 | while (wr) { | |
365 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
366 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
367 | if (num_wrs) | |
368 | err = iwch_build_rdma_recv(qhp->rhp, wqe, wr); | |
369 | else | |
370 | err = -ENOMEM; | |
371 | if (err) { | |
372 | *bad_wr = wr; | |
373 | break; | |
374 | } | |
375 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = | |
376 | wr->wr_id; | |
377 | build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, | |
378 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
379 | 0, sizeof(struct t3_receive_wr) >> 3); | |
380 | PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " | |
381 | "wqe %p \n", __FUNCTION__, (unsigned long long) wr->wr_id, | |
382 | idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); | |
383 | ++(qhp->wq.rq_wptr); | |
384 | ++(qhp->wq.wptr); | |
385 | wr = wr->next; | |
386 | num_wrs--; | |
387 | } | |
388 | spin_unlock_irqrestore(&qhp->lock, flag); | |
389 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
390 | return err; | |
391 | } | |
392 | ||
393 | int iwch_bind_mw(struct ib_qp *qp, | |
394 | struct ib_mw *mw, | |
395 | struct ib_mw_bind *mw_bind) | |
396 | { | |
397 | struct iwch_dev *rhp; | |
398 | struct iwch_mw *mhp; | |
399 | struct iwch_qp *qhp; | |
400 | union t3_wr *wqe; | |
401 | u32 pbl_addr; | |
402 | u8 page_size; | |
403 | u32 num_wrs; | |
404 | unsigned long flag; | |
405 | struct ib_sge sgl; | |
406 | int err=0; | |
407 | enum t3_wr_flags t3_wr_flags; | |
408 | u32 idx; | |
409 | struct t3_swsq *sqp; | |
410 | ||
411 | qhp = to_iwch_qp(qp); | |
412 | mhp = to_iwch_mw(mw); | |
413 | rhp = qhp->rhp; | |
414 | ||
415 | spin_lock_irqsave(&qhp->lock, flag); | |
416 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
417 | spin_unlock_irqrestore(&qhp->lock, flag); | |
418 | return -EINVAL; | |
419 | } | |
420 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
421 | qhp->wq.sq_size_log2); | |
422 | if ((num_wrs) <= 0) { | |
423 | spin_unlock_irqrestore(&qhp->lock, flag); | |
424 | return -ENOMEM; | |
425 | } | |
426 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
427 | PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __FUNCTION__, idx, | |
428 | mw, mw_bind); | |
429 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
430 | ||
431 | t3_wr_flags = 0; | |
432 | if (mw_bind->send_flags & IB_SEND_SIGNALED) | |
433 | t3_wr_flags = T3_COMPLETION_FLAG; | |
434 | ||
435 | sgl.addr = mw_bind->addr; | |
436 | sgl.lkey = mw_bind->mr->lkey; | |
437 | sgl.length = mw_bind->length; | |
438 | wqe->bind.reserved = 0; | |
439 | wqe->bind.type = T3_VA_BASED_TO; | |
440 | ||
441 | /* TBD: check perms */ | |
442 | wqe->bind.perms = iwch_convert_access(mw_bind->mw_access_flags); | |
443 | wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); | |
444 | wqe->bind.mw_stag = cpu_to_be32(mw->rkey); | |
445 | wqe->bind.mw_len = cpu_to_be32(mw_bind->length); | |
446 | wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); | |
447 | err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); | |
448 | if (err) { | |
449 | spin_unlock_irqrestore(&qhp->lock, flag); | |
450 | return err; | |
451 | } | |
452 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
453 | sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
454 | sqp->wr_id = mw_bind->wr_id; | |
455 | sqp->opcode = T3_BIND_MW; | |
456 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
457 | sqp->complete = 0; | |
458 | sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); | |
459 | wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); | |
460 | wqe->bind.mr_pagesz = page_size; | |
461 | wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; | |
462 | build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, | |
463 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, | |
464 | sizeof(struct t3_bind_mw_wr) >> 3); | |
465 | ++(qhp->wq.wptr); | |
466 | ++(qhp->wq.sq_wptr); | |
467 | spin_unlock_irqrestore(&qhp->lock, flag); | |
468 | ||
469 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
470 | ||
471 | return err; | |
472 | } | |
473 | ||
2b540355 | 474 | static void build_term_codes(int t3err, u8 *layer_type, u8 *ecode, int tagged) |
b038ced7 SW |
475 | { |
476 | switch (t3err) { | |
477 | case TPT_ERR_STAG: | |
478 | if (tagged == 1) { | |
479 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
480 | *ecode = DDPT_INV_STAG; | |
481 | } else if (tagged == 2) { | |
482 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
483 | *ecode = RDMAP_INV_STAG; | |
484 | } | |
485 | break; | |
486 | case TPT_ERR_PDID: | |
487 | case TPT_ERR_QPID: | |
488 | case TPT_ERR_ACCESS: | |
489 | if (tagged == 1) { | |
490 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
491 | *ecode = DDPT_STAG_NOT_ASSOC; | |
492 | } else if (tagged == 2) { | |
493 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
494 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
495 | } | |
496 | break; | |
497 | case TPT_ERR_WRAP: | |
498 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
499 | *ecode = RDMAP_TO_WRAP; | |
500 | break; | |
501 | case TPT_ERR_BOUND: | |
502 | if (tagged == 1) { | |
503 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
504 | *ecode = DDPT_BASE_BOUNDS; | |
505 | } else if (tagged == 2) { | |
506 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
507 | *ecode = RDMAP_BASE_BOUNDS; | |
508 | } else { | |
509 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
510 | *ecode = DDPU_MSG_TOOBIG; | |
511 | } | |
512 | break; | |
513 | case TPT_ERR_INVALIDATE_SHARED_MR: | |
514 | case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: | |
515 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
516 | *ecode = RDMAP_CANT_INV_STAG; | |
517 | break; | |
518 | case TPT_ERR_ECC: | |
519 | case TPT_ERR_ECC_PSTAG: | |
520 | case TPT_ERR_INTERNAL_ERR: | |
521 | *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; | |
522 | *ecode = 0; | |
523 | break; | |
524 | case TPT_ERR_OUT_OF_RQE: | |
525 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
526 | *ecode = DDPU_INV_MSN_NOBUF; | |
527 | break; | |
528 | case TPT_ERR_PBL_ADDR_BOUND: | |
529 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
530 | *ecode = DDPT_BASE_BOUNDS; | |
531 | break; | |
532 | case TPT_ERR_CRC: | |
533 | *layer_type = LAYER_MPA|DDP_LLP; | |
534 | *ecode = MPA_CRC_ERR; | |
535 | break; | |
536 | case TPT_ERR_MARKER: | |
537 | *layer_type = LAYER_MPA|DDP_LLP; | |
538 | *ecode = MPA_MARKER_ERR; | |
539 | break; | |
540 | case TPT_ERR_PDU_LEN_ERR: | |
541 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
542 | *ecode = DDPU_MSG_TOOBIG; | |
543 | break; | |
544 | case TPT_ERR_DDP_VERSION: | |
545 | if (tagged) { | |
546 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
547 | *ecode = DDPT_INV_VERS; | |
548 | } else { | |
549 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
550 | *ecode = DDPU_INV_VERS; | |
551 | } | |
552 | break; | |
553 | case TPT_ERR_RDMA_VERSION: | |
554 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
555 | *ecode = RDMAP_INV_VERS; | |
556 | break; | |
557 | case TPT_ERR_OPCODE: | |
558 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
559 | *ecode = RDMAP_INV_OPCODE; | |
560 | break; | |
561 | case TPT_ERR_DDP_QUEUE_NUM: | |
562 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
563 | *ecode = DDPU_INV_QN; | |
564 | break; | |
565 | case TPT_ERR_MSN: | |
566 | case TPT_ERR_MSN_GAP: | |
567 | case TPT_ERR_MSN_RANGE: | |
568 | case TPT_ERR_IRD_OVERFLOW: | |
569 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
570 | *ecode = DDPU_INV_MSN_RANGE; | |
571 | break; | |
572 | case TPT_ERR_TBIT: | |
573 | *layer_type = LAYER_DDP|DDP_LOCAL_CATA; | |
574 | *ecode = 0; | |
575 | break; | |
576 | case TPT_ERR_MO: | |
577 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
578 | *ecode = DDPU_INV_MO; | |
579 | break; | |
580 | default: | |
581 | *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; | |
582 | *ecode = 0; | |
583 | break; | |
584 | } | |
585 | } | |
586 | ||
587 | /* | |
588 | * This posts a TERMINATE with layer=RDMA, type=catastrophic. | |
589 | */ | |
590 | int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) | |
591 | { | |
592 | union t3_wr *wqe; | |
593 | struct terminate_message *term; | |
594 | int status; | |
595 | int tagged = 0; | |
596 | struct sk_buff *skb; | |
597 | ||
598 | PDBG("%s %d\n", __FUNCTION__, __LINE__); | |
599 | skb = alloc_skb(40, GFP_ATOMIC); | |
600 | if (!skb) { | |
601 | printk(KERN_ERR "%s cannot send TERMINATE!\n", __FUNCTION__); | |
602 | return -ENOMEM; | |
603 | } | |
604 | wqe = (union t3_wr *)skb_put(skb, 40); | |
605 | memset(wqe, 0, 40); | |
606 | wqe->send.rdmaop = T3_TERMINATE; | |
607 | ||
608 | /* immediate data length */ | |
609 | wqe->send.plen = htonl(4); | |
610 | ||
611 | /* immediate data starts here. */ | |
612 | term = (struct terminate_message *)wqe->send.sgl; | |
613 | if (rsp_msg) { | |
614 | status = CQE_STATUS(rsp_msg->cqe); | |
615 | if (CQE_OPCODE(rsp_msg->cqe) == T3_RDMA_WRITE) | |
616 | tagged = 1; | |
617 | if ((CQE_OPCODE(rsp_msg->cqe) == T3_READ_REQ) || | |
618 | (CQE_OPCODE(rsp_msg->cqe) == T3_READ_RESP)) | |
619 | tagged = 2; | |
620 | } else { | |
621 | status = TPT_ERR_INTERNAL_ERR; | |
622 | } | |
623 | build_term_codes(status, &term->layer_etype, &term->ecode, tagged); | |
624 | build_fw_riwrh((void *)wqe, T3_WR_SEND, | |
625 | T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 1, | |
626 | qhp->ep->hwtid, 5); | |
627 | skb->priority = CPL_PRIORITY_DATA; | |
628 | return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); | |
629 | } | |
630 | ||
631 | /* | |
632 | * Assumes qhp lock is held. | |
633 | */ | |
634 | static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) | |
635 | { | |
636 | struct iwch_cq *rchp, *schp; | |
637 | int count; | |
638 | ||
639 | rchp = get_chp(qhp->rhp, qhp->attr.rcq); | |
640 | schp = get_chp(qhp->rhp, qhp->attr.scq); | |
641 | ||
642 | PDBG("%s qhp %p rchp %p schp %p\n", __FUNCTION__, qhp, rchp, schp); | |
643 | /* take a ref on the qhp since we must release the lock */ | |
644 | atomic_inc(&qhp->refcnt); | |
645 | spin_unlock_irqrestore(&qhp->lock, *flag); | |
646 | ||
647 | /* locking heirarchy: cq lock first, then qp lock. */ | |
648 | spin_lock_irqsave(&rchp->lock, *flag); | |
649 | spin_lock(&qhp->lock); | |
650 | cxio_flush_hw_cq(&rchp->cq); | |
651 | cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); | |
652 | cxio_flush_rq(&qhp->wq, &rchp->cq, count); | |
653 | spin_unlock(&qhp->lock); | |
654 | spin_unlock_irqrestore(&rchp->lock, *flag); | |
655 | ||
656 | /* locking heirarchy: cq lock first, then qp lock. */ | |
657 | spin_lock_irqsave(&schp->lock, *flag); | |
658 | spin_lock(&qhp->lock); | |
659 | cxio_flush_hw_cq(&schp->cq); | |
660 | cxio_count_scqes(&schp->cq, &qhp->wq, &count); | |
661 | cxio_flush_sq(&qhp->wq, &schp->cq, count); | |
662 | spin_unlock(&qhp->lock); | |
663 | spin_unlock_irqrestore(&schp->lock, *flag); | |
664 | ||
665 | /* deref */ | |
666 | if (atomic_dec_and_test(&qhp->refcnt)) | |
667 | wake_up(&qhp->wait); | |
668 | ||
669 | spin_lock_irqsave(&qhp->lock, *flag); | |
670 | } | |
671 | ||
2b540355 | 672 | static void flush_qp(struct iwch_qp *qhp, unsigned long *flag) |
b038ced7 SW |
673 | { |
674 | if (t3b_device(qhp->rhp)) | |
675 | cxio_set_wq_in_error(&qhp->wq); | |
676 | else | |
677 | __flush_qp(qhp, flag); | |
678 | } | |
679 | ||
680 | ||
681 | /* | |
682 | * Return non zero if at least one RECV was pre-posted. | |
683 | */ | |
2b540355 | 684 | static int rqes_posted(struct iwch_qp *qhp) |
b038ced7 SW |
685 | { |
686 | return fw_riwrh_opcode((struct fw_riwrh *)qhp->wq.queue) == T3_WR_RCV; | |
687 | } | |
688 | ||
689 | static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
690 | enum iwch_qp_attr_mask mask, | |
691 | struct iwch_qp_attributes *attrs) | |
692 | { | |
693 | struct t3_rdma_init_attr init_attr; | |
694 | int ret; | |
695 | ||
696 | init_attr.tid = qhp->ep->hwtid; | |
697 | init_attr.qpid = qhp->wq.qpid; | |
698 | init_attr.pdid = qhp->attr.pd; | |
699 | init_attr.scqid = qhp->attr.scq; | |
700 | init_attr.rcqid = qhp->attr.rcq; | |
701 | init_attr.rq_addr = qhp->wq.rq_addr; | |
702 | init_attr.rq_size = 1 << qhp->wq.rq_size_log2; | |
703 | init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | | |
704 | qhp->attr.mpa_attr.recv_marker_enabled | | |
705 | (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | | |
706 | (qhp->attr.mpa_attr.crc_enabled << 2); | |
707 | ||
708 | /* | |
709 | * XXX - The IWCM doesn't quite handle getting these | |
710 | * attrs set before going into RTS. For now, just turn | |
711 | * them on always... | |
712 | */ | |
713 | #if 0 | |
714 | init_attr.qpcaps = qhp->attr.enableRdmaRead | | |
715 | (qhp->attr.enableRdmaWrite << 1) | | |
716 | (qhp->attr.enableBind << 2) | | |
717 | (qhp->attr.enable_stag0_fastreg << 3) | | |
718 | (qhp->attr.enable_stag0_fastreg << 4); | |
719 | #else | |
720 | init_attr.qpcaps = 0x1f; | |
721 | #endif | |
722 | init_attr.tcp_emss = qhp->ep->emss; | |
723 | init_attr.ord = qhp->attr.max_ord; | |
724 | init_attr.ird = qhp->attr.max_ird; | |
725 | init_attr.qp_dma_addr = qhp->wq.dma_addr; | |
726 | init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); | |
727 | init_attr.flags = rqes_posted(qhp) ? RECVS_POSTED : 0; | |
728 | PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " | |
729 | "flags 0x%x qpcaps 0x%x\n", __FUNCTION__, | |
730 | init_attr.rq_addr, init_attr.rq_size, | |
731 | init_attr.flags, init_attr.qpcaps); | |
732 | ret = cxio_rdma_init(&rhp->rdev, &init_attr); | |
733 | PDBG("%s ret %d\n", __FUNCTION__, ret); | |
734 | return ret; | |
735 | } | |
736 | ||
737 | int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
738 | enum iwch_qp_attr_mask mask, | |
739 | struct iwch_qp_attributes *attrs, | |
740 | int internal) | |
741 | { | |
742 | int ret = 0; | |
743 | struct iwch_qp_attributes newattr = qhp->attr; | |
744 | unsigned long flag; | |
745 | int disconnect = 0; | |
746 | int terminate = 0; | |
747 | int abort = 0; | |
748 | int free = 0; | |
749 | struct iwch_ep *ep = NULL; | |
750 | ||
751 | PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __FUNCTION__, | |
752 | qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, | |
753 | (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); | |
754 | ||
755 | spin_lock_irqsave(&qhp->lock, flag); | |
756 | ||
757 | /* Process attr changes if in IDLE */ | |
758 | if (mask & IWCH_QP_ATTR_VALID_MODIFY) { | |
759 | if (qhp->attr.state != IWCH_QP_STATE_IDLE) { | |
760 | ret = -EIO; | |
761 | goto out; | |
762 | } | |
763 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) | |
764 | newattr.enable_rdma_read = attrs->enable_rdma_read; | |
765 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) | |
766 | newattr.enable_rdma_write = attrs->enable_rdma_write; | |
767 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) | |
768 | newattr.enable_bind = attrs->enable_bind; | |
769 | if (mask & IWCH_QP_ATTR_MAX_ORD) { | |
770 | if (attrs->max_ord > | |
771 | rhp->attr.max_rdma_read_qp_depth) { | |
772 | ret = -EINVAL; | |
773 | goto out; | |
774 | } | |
775 | newattr.max_ord = attrs->max_ord; | |
776 | } | |
777 | if (mask & IWCH_QP_ATTR_MAX_IRD) { | |
778 | if (attrs->max_ird > | |
779 | rhp->attr.max_rdma_reads_per_qp) { | |
780 | ret = -EINVAL; | |
781 | goto out; | |
782 | } | |
783 | newattr.max_ird = attrs->max_ird; | |
784 | } | |
785 | qhp->attr = newattr; | |
786 | } | |
787 | ||
788 | if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) | |
789 | goto out; | |
790 | if (qhp->attr.state == attrs->next_state) | |
791 | goto out; | |
792 | ||
793 | switch (qhp->attr.state) { | |
794 | case IWCH_QP_STATE_IDLE: | |
795 | switch (attrs->next_state) { | |
796 | case IWCH_QP_STATE_RTS: | |
797 | if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { | |
798 | ret = -EINVAL; | |
799 | goto out; | |
800 | } | |
801 | if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { | |
802 | ret = -EINVAL; | |
803 | goto out; | |
804 | } | |
805 | qhp->attr.mpa_attr = attrs->mpa_attr; | |
806 | qhp->attr.llp_stream_handle = attrs->llp_stream_handle; | |
807 | qhp->ep = qhp->attr.llp_stream_handle; | |
808 | qhp->attr.state = IWCH_QP_STATE_RTS; | |
809 | ||
810 | /* | |
811 | * Ref the endpoint here and deref when we | |
812 | * disassociate the endpoint from the QP. This | |
813 | * happens in CLOSING->IDLE transition or *->ERROR | |
814 | * transition. | |
815 | */ | |
816 | get_ep(&qhp->ep->com); | |
817 | spin_unlock_irqrestore(&qhp->lock, flag); | |
818 | ret = rdma_init(rhp, qhp, mask, attrs); | |
819 | spin_lock_irqsave(&qhp->lock, flag); | |
820 | if (ret) | |
821 | goto err; | |
822 | break; | |
823 | case IWCH_QP_STATE_ERROR: | |
824 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
825 | flush_qp(qhp, &flag); | |
826 | break; | |
827 | default: | |
828 | ret = -EINVAL; | |
829 | goto out; | |
830 | } | |
831 | break; | |
832 | case IWCH_QP_STATE_RTS: | |
833 | switch (attrs->next_state) { | |
834 | case IWCH_QP_STATE_CLOSING: | |
835 | BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); | |
836 | qhp->attr.state = IWCH_QP_STATE_CLOSING; | |
837 | if (!internal) { | |
838 | abort=0; | |
839 | disconnect = 1; | |
840 | ep = qhp->ep; | |
841 | } | |
842 | break; | |
843 | case IWCH_QP_STATE_TERMINATE: | |
844 | qhp->attr.state = IWCH_QP_STATE_TERMINATE; | |
a1a75052 SW |
845 | if (t3b_device(qhp->rhp)) |
846 | cxio_set_wq_in_error(&qhp->wq); | |
b038ced7 SW |
847 | if (!internal) |
848 | terminate = 1; | |
849 | break; | |
850 | case IWCH_QP_STATE_ERROR: | |
851 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
852 | if (!internal) { | |
853 | abort=1; | |
854 | disconnect = 1; | |
855 | ep = qhp->ep; | |
856 | } | |
857 | goto err; | |
858 | break; | |
859 | default: | |
860 | ret = -EINVAL; | |
861 | goto out; | |
862 | } | |
863 | break; | |
864 | case IWCH_QP_STATE_CLOSING: | |
865 | if (!internal) { | |
866 | ret = -EINVAL; | |
867 | goto out; | |
868 | } | |
869 | switch (attrs->next_state) { | |
870 | case IWCH_QP_STATE_IDLE: | |
871 | qhp->attr.state = IWCH_QP_STATE_IDLE; | |
872 | qhp->attr.llp_stream_handle = NULL; | |
873 | put_ep(&qhp->ep->com); | |
874 | qhp->ep = NULL; | |
875 | wake_up(&qhp->wait); | |
876 | break; | |
877 | case IWCH_QP_STATE_ERROR: | |
878 | goto err; | |
879 | default: | |
880 | ret = -EINVAL; | |
881 | goto err; | |
882 | } | |
883 | break; | |
884 | case IWCH_QP_STATE_ERROR: | |
885 | if (attrs->next_state != IWCH_QP_STATE_IDLE) { | |
886 | ret = -EINVAL; | |
887 | goto out; | |
888 | } | |
889 | ||
890 | if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || | |
891 | !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { | |
892 | ret = -EINVAL; | |
893 | goto out; | |
894 | } | |
895 | qhp->attr.state = IWCH_QP_STATE_IDLE; | |
896 | memset(&qhp->attr, 0, sizeof(qhp->attr)); | |
897 | break; | |
898 | case IWCH_QP_STATE_TERMINATE: | |
899 | if (!internal) { | |
900 | ret = -EINVAL; | |
901 | goto out; | |
902 | } | |
903 | goto err; | |
904 | break; | |
905 | default: | |
906 | printk(KERN_ERR "%s in a bad state %d\n", | |
907 | __FUNCTION__, qhp->attr.state); | |
908 | ret = -EINVAL; | |
909 | goto err; | |
910 | break; | |
911 | } | |
912 | goto out; | |
913 | err: | |
914 | PDBG("%s disassociating ep %p qpid 0x%x\n", __FUNCTION__, qhp->ep, | |
915 | qhp->wq.qpid); | |
916 | ||
917 | /* disassociate the LLP connection */ | |
918 | qhp->attr.llp_stream_handle = NULL; | |
919 | ep = qhp->ep; | |
920 | qhp->ep = NULL; | |
921 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
922 | free=1; | |
923 | wake_up(&qhp->wait); | |
924 | BUG_ON(!ep); | |
925 | flush_qp(qhp, &flag); | |
926 | out: | |
927 | spin_unlock_irqrestore(&qhp->lock, flag); | |
928 | ||
929 | if (terminate) | |
930 | iwch_post_terminate(qhp, NULL); | |
931 | ||
932 | /* | |
933 | * If disconnect is 1, then we need to initiate a disconnect | |
934 | * on the EP. This can be a normal close (RTS->CLOSING) or | |
935 | * an abnormal close (RTS/CLOSING->ERROR). | |
936 | */ | |
937 | if (disconnect) | |
938 | iwch_ep_disconnect(ep, abort, GFP_KERNEL); | |
939 | ||
940 | /* | |
941 | * If free is 1, then we've disassociated the EP from the QP | |
942 | * and we need to dereference the EP. | |
943 | */ | |
944 | if (free) | |
945 | put_ep(&ep->com); | |
946 | ||
947 | PDBG("%s exit state %d\n", __FUNCTION__, qhp->attr.state); | |
948 | return ret; | |
949 | } | |
950 | ||
951 | static int quiesce_qp(struct iwch_qp *qhp) | |
952 | { | |
953 | spin_lock_irq(&qhp->lock); | |
954 | iwch_quiesce_tid(qhp->ep); | |
955 | qhp->flags |= QP_QUIESCED; | |
956 | spin_unlock_irq(&qhp->lock); | |
957 | return 0; | |
958 | } | |
959 | ||
960 | static int resume_qp(struct iwch_qp *qhp) | |
961 | { | |
962 | spin_lock_irq(&qhp->lock); | |
963 | iwch_resume_tid(qhp->ep); | |
964 | qhp->flags &= ~QP_QUIESCED; | |
965 | spin_unlock_irq(&qhp->lock); | |
966 | return 0; | |
967 | } | |
968 | ||
969 | int iwch_quiesce_qps(struct iwch_cq *chp) | |
970 | { | |
971 | int i; | |
972 | struct iwch_qp *qhp; | |
973 | ||
974 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
975 | qhp = get_qhp(chp->rhp, i); | |
976 | if (!qhp) | |
977 | continue; | |
978 | if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { | |
979 | quiesce_qp(qhp); | |
980 | continue; | |
981 | } | |
982 | if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) | |
983 | quiesce_qp(qhp); | |
984 | } | |
985 | return 0; | |
986 | } | |
987 | ||
988 | int iwch_resume_qps(struct iwch_cq *chp) | |
989 | { | |
990 | int i; | |
991 | struct iwch_qp *qhp; | |
992 | ||
993 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
994 | qhp = get_qhp(chp->rhp, i); | |
995 | if (!qhp) | |
996 | continue; | |
997 | if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { | |
998 | resume_qp(qhp); | |
999 | continue; | |
1000 | } | |
1001 | if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) | |
1002 | resume_qp(qhp); | |
1003 | } | |
1004 | return 0; | |
1005 | } |