projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/qib: Remove ibport and use rdmavt version
[deliverable/linux.git] / drivers / infiniband / hw / qib / qib_rc.c
CommitLineData
f931551b
RC		 1/*
2 * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/io.h>
35
36#include "qib.h"
37
38/* cut down ridiculously long IB macro names */
39#define OP(x) IB_OPCODE_RC_##x
40
41static void rc_timeout(unsigned long arg);
42
7c2e11fe 43static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
f931551b
RC		 44 u32 psn, u32 pmtu)
45{
46 u32 len;
47
48 len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
49 ss->sge = wqe->sg_list[0];
50 ss->sg_list = wqe->sg_list + 1;
51 ss->num_sge = wqe->wr.num_sge;
52 ss->total_len = wqe->length;
53 qib_skip_sge(ss, len, 0);
54 return wqe->length - len;
55}
56
7c2e11fe 57static void start_timer(struct rvt_qp *qp)
f931551b
RC		 58{
59 qp->s_flags |= QIB_S_TIMER;
60 qp->s_timer.function = rc_timeout;
61 /* 4.096 usec. * (1 << qp->timeout) */
d0f2faf7 62 qp->s_timer.expires = jiffies + qp->timeout_jiffies;
f931551b
RC		 63 add_timer(&qp->s_timer);
64}
65
66/**
67 * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read)
68 * @dev: the device for this QP
69 * @qp: a pointer to the QP
70 * @ohdr: a pointer to the IB header being constructed
71 * @pmtu: the path MTU
72 *
73 * Return 1 if constructed; otherwise, return 0.
74 * Note that we are in the responder's side of the QP context.
75 * Note the QP s_lock must be held.
76 */
7c2e11fe 77static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
f931551b
RC		 78 struct qib_other_headers *ohdr, u32 pmtu)
79{
7c2e11fe 80 struct rvt_ack_entry *e;
f931551b
RC		 81 u32 hwords;
82 u32 len;
83 u32 bth0;
84 u32 bth2;
85
86 /* Don't send an ACK if we aren't supposed to. */
87 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK))
88 goto bail;
89
90 /* header size in 32-bit words LRH+BTH = (8+12)/4. */
91 hwords = 5;
92
93 switch (qp->s_ack_state) {
94 case OP(RDMA_READ_RESPONSE_LAST):
95 case OP(RDMA_READ_RESPONSE_ONLY):
96 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
97 if (e->rdma_sge.mr) {
7c2e11fe 98 rvt_put_mr(e->rdma_sge.mr);
f931551b
RC		 99 e->rdma_sge.mr = NULL;
100 }
101 /* FALLTHROUGH */
102 case OP(ATOMIC_ACKNOWLEDGE):
103 /*
104 * We can increment the tail pointer now that the last
105 * response has been sent instead of only being
106 * constructed.
107 */
108 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC)
109 qp->s_tail_ack_queue = 0;
110 /* FALLTHROUGH */
111 case OP(SEND_ONLY):
112 case OP(ACKNOWLEDGE):
113 /* Check for no next entry in the queue. */
114 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
115 if (qp->s_flags & QIB_S_ACK_PENDING)
116 goto normal;
117 goto bail;
118 }
119
120 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
121 if (e->opcode == OP(RDMA_READ_REQUEST)) {
122 /*
123 * If a RDMA read response is being resent and
124 * we haven't seen the duplicate request yet,
125 * then stop sending the remaining responses the
126 * responder has seen until the requester resends it.
127 */
128 len = e->rdma_sge.sge_length;
129 if (len && !e->rdma_sge.mr) {
130 qp->s_tail_ack_queue = qp->r_head_ack_queue;
131 goto bail;
132 }
133 /* Copy SGE state in case we need to resend */
134 qp->s_rdma_mr = e->rdma_sge.mr;
135 if (qp->s_rdma_mr)
7c2e11fe 136 rvt_get_mr(qp->s_rdma_mr);
f931551b
RC		 137 qp->s_ack_rdma_sge.sge = e->rdma_sge;
138 qp->s_ack_rdma_sge.num_sge = 1;
139 qp->s_cur_sge = &qp->s_ack_rdma_sge;
140 if (len > pmtu) {
141 len = pmtu;
142 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
143 } else {
144 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
145 e->sent = 1;
146 }
147 ohdr->u.aeth = qib_compute_aeth(qp);
148 hwords++;
149 qp->s_ack_rdma_psn = e->psn;
150 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
151 } else {
152 /* COMPARE_SWAP or FETCH_ADD */
153 qp->s_cur_sge = NULL;
154 len = 0;
155 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
156 ohdr->u.at.aeth = qib_compute_aeth(qp);
157 ohdr->u.at.atomic_ack_eth[0] =
158 cpu_to_be32(e->atomic_data >> 32);
159 ohdr->u.at.atomic_ack_eth[1] =
160 cpu_to_be32(e->atomic_data);
161 hwords += sizeof(ohdr->u.at) / sizeof(u32);
162 bth2 = e->psn & QIB_PSN_MASK;
163 e->sent = 1;
164 }
165 bth0 = qp->s_ack_state << 24;
166 break;
167
168 case OP(RDMA_READ_RESPONSE_FIRST):
169 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
170 /* FALLTHROUGH */
171 case OP(RDMA_READ_RESPONSE_MIDDLE):
172 qp->s_cur_sge = &qp->s_ack_rdma_sge;
173 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr;
174 if (qp->s_rdma_mr)
7c2e11fe 175 rvt_get_mr(qp->s_rdma_mr);
f931551b
RC		 176 len = qp->s_ack_rdma_sge.sge.sge_length;
177 if (len > pmtu)
178 len = pmtu;
179 else {
180 ohdr->u.aeth = qib_compute_aeth(qp);
181 hwords++;
182 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
183 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
184 e->sent = 1;
185 }
186 bth0 = qp->s_ack_state << 24;
187 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
188 break;
189
190 default:
191normal:
192 /*
193 * Send a regular ACK.
194 * Set the s_ack_state so we wait until after sending
195 * the ACK before setting s_ack_state to ACKNOWLEDGE
196 * (see above).
197 */
198 qp->s_ack_state = OP(SEND_ONLY);
199 qp->s_flags &= ~QIB_S_ACK_PENDING;
200 qp->s_cur_sge = NULL;
201 if (qp->s_nak_state)
202 ohdr->u.aeth =
203 cpu_to_be32((qp->r_msn & QIB_MSN_MASK) |
204 (qp->s_nak_state <<
205 QIB_AETH_CREDIT_SHIFT));
206 else
207 ohdr->u.aeth = qib_compute_aeth(qp);
208 hwords++;
209 len = 0;
210 bth0 = OP(ACKNOWLEDGE) << 24;
211 bth2 = qp->s_ack_psn & QIB_PSN_MASK;
212 }
213 qp->s_rdma_ack_cnt++;
214 qp->s_hdrwords = hwords;
215 qp->s_cur_size = len;
216 qib_make_ruc_header(qp, ohdr, bth0, bth2);
217 return 1;
218
219bail:
220 qp->s_ack_state = OP(ACKNOWLEDGE);
221 qp->s_flags &= ~(QIB_S_RESP_PENDING | QIB_S_ACK_PENDING);
222 return 0;
223}
224
225/**
226 * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC)
227 * @qp: a pointer to the QP
228 *
229 * Return 1 if constructed; otherwise, return 0.
230 */
7c2e11fe 231int qib_make_rc_req(struct rvt_qp *qp)
f931551b 232{
ffc26907 233 struct qib_qp_priv *priv = qp->priv;
f931551b
RC		 234 struct qib_ibdev *dev = to_idev(qp->ibqp.device);
235 struct qib_other_headers *ohdr;
7c2e11fe
DD		 236 struct rvt_sge_state *ss;
237 struct rvt_swqe *wqe;
f931551b
RC		 238 u32 hwords;
239 u32 len;
240 u32 bth0;
241 u32 bth2;
cc6ea138 242 u32 pmtu = qp->pmtu;
f931551b
RC		 243 char newreq;
244 unsigned long flags;
245 int ret = 0;
246 int delta;
247
ffc26907 248 ohdr = &priv->s_hdr->u.oth;
f931551b 249 if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
ffc26907 250 ohdr = &priv->s_hdr->u.l.oth;
f931551b
RC		 251
252 /*
253 * The lock is needed to synchronize between the sending tasklet,
254 * the receive interrupt handler, and timeout resends.
255 */
256 spin_lock_irqsave(&qp->s_lock, flags);
257
258 /* Sending responses has higher priority over sending requests. */
259 if ((qp->s_flags & QIB_S_RESP_PENDING) &&
260 qib_make_rc_ack(dev, qp, ohdr, pmtu))
261 goto done;
262
263 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_SEND_OK)) {
264 if (!(ib_qib_state_ops[qp->state] & QIB_FLUSH_SEND))
265 goto bail;
266 /* We are in the error state, flush the work request. */
267 if (qp->s_last == qp->s_head)
268 goto bail;
269 /* If DMAs are in progress, we can't flush immediately. */
ffc26907 270 if (atomic_read(&priv->s_dma_busy)) {
f931551b
RC		 271 qp->s_flags |= QIB_S_WAIT_DMA;
272 goto bail;
273 }
274 wqe = get_swqe_ptr(qp, qp->s_last);
30ab7e23
MM		 275 qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
276 IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
277 /* will get called again */
f931551b
RC		 278 goto done;
279 }
280
281 if (qp->s_flags & (QIB_S_WAIT_RNR | QIB_S_WAIT_ACK))
282 goto bail;
283
284 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) {
285 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
286 qp->s_flags |= QIB_S_WAIT_PSN;
287 goto bail;
288 }
289 qp->s_sending_psn = qp->s_psn;
290 qp->s_sending_hpsn = qp->s_psn - 1;
291 }
292
293 /* header size in 32-bit words LRH+BTH = (8+12)/4. */
294 hwords = 5;
295 bth0 = 0;
296
297 /* Send a request. */
298 wqe = get_swqe_ptr(qp, qp->s_cur);
299 switch (qp->s_state) {
300 default:
301 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_NEXT_SEND_OK))
302 goto bail;
303 /*
304 * Resend an old request or start a new one.
305 *
306 * We keep track of the current SWQE so that
307 * we don't reset the "furthest progress" state
308 * if we need to back up.
309 */
310 newreq = 0;
311 if (qp->s_cur == qp->s_tail) {
312 /* Check if send work queue is empty. */
313 if (qp->s_tail == qp->s_head)
314 goto bail;
315 /*
316 * If a fence is requested, wait for previous
317 * RDMA read and atomic operations to finish.
318 */
319 if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
320 qp->s_num_rd_atomic) {
321 qp->s_flags |= QIB_S_WAIT_FENCE;
322 goto bail;
323 }
324 wqe->psn = qp->s_next_psn;
325 newreq = 1;
326 }
327 /*
328 * Note that we have to be careful not to modify the
329 * original work request since we may need to resend
330 * it.
331 */
332 len = wqe->length;
333 ss = &qp->s_sge;
334 bth2 = qp->s_psn & QIB_PSN_MASK;
335 switch (wqe->wr.opcode) {
336 case IB_WR_SEND:
337 case IB_WR_SEND_WITH_IMM:
338 /* If no credit, return. */
339 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) &&
340 qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
341 qp->s_flags |= QIB_S_WAIT_SSN_CREDIT;
342 goto bail;
343 }
344 wqe->lpsn = wqe->psn;
345 if (len > pmtu) {
346 wqe->lpsn += (len - 1) / pmtu;
347 qp->s_state = OP(SEND_FIRST);
348 len = pmtu;
349 break;
350 }
351 if (wqe->wr.opcode == IB_WR_SEND)
352 qp->s_state = OP(SEND_ONLY);
353 else {
354 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
355 /* Immediate data comes after the BTH */
356 ohdr->u.imm_data = wqe->wr.ex.imm_data;
357 hwords += 1;
358 }
359 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
360 bth0 |= IB_BTH_SOLICITED;
361 bth2 |= IB_BTH_REQ_ACK;
362 if (++qp->s_cur == qp->s_size)
363 qp->s_cur = 0;
364 break;
365
366 case IB_WR_RDMA_WRITE:
367 if (newreq && !(qp->s_flags & QIB_S_UNLIMITED_CREDIT))
368 qp->s_lsn++;
369 /* FALLTHROUGH */
370 case IB_WR_RDMA_WRITE_WITH_IMM:
371 /* If no credit, return. */
372 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) &&
373 qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
374 qp->s_flags |= QIB_S_WAIT_SSN_CREDIT;
375 goto bail;
376 }
e622f2f4 377
f931551b 378 ohdr->u.rc.reth.vaddr =
e622f2f4 379 cpu_to_be64(wqe->rdma_wr.remote_addr);
f931551b 380 ohdr->u.rc.reth.rkey =
e622f2f4 381 cpu_to_be32(wqe->rdma_wr.rkey);
f931551b
RC		 382 ohdr->u.rc.reth.length = cpu_to_be32(len);
383 hwords += sizeof(struct ib_reth) / sizeof(u32);
384 wqe->lpsn = wqe->psn;
385 if (len > pmtu) {
386 wqe->lpsn += (len - 1) / pmtu;
387 qp->s_state = OP(RDMA_WRITE_FIRST);
388 len = pmtu;
389 break;
390 }
e622f2f4 391 if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE)
f931551b
RC		 392 qp->s_state = OP(RDMA_WRITE_ONLY);
393 else {
e622f2f4 394 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
f931551b 395 /* Immediate data comes after RETH */
e622f2f4
CH		 396 ohdr->u.rc.imm_data =
397 wqe->rdma_wr.wr.ex.imm_data;
f931551b 398 hwords += 1;
e622f2f4 399 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED)
f931551b
RC		 400 bth0 |= IB_BTH_SOLICITED;
401 }
402 bth2 |= IB_BTH_REQ_ACK;
403 if (++qp->s_cur == qp->s_size)
404 qp->s_cur = 0;
405 break;
406
407 case IB_WR_RDMA_READ:
408 /*
409 * Don't allow more operations to be started
410 * than the QP limits allow.
411 */
412 if (newreq) {
413 if (qp->s_num_rd_atomic >=
414 qp->s_max_rd_atomic) {
415 qp->s_flags |= QIB_S_WAIT_RDMAR;
416 goto bail;
417 }
418 qp->s_num_rd_atomic++;
419 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT))
420 qp->s_lsn++;
421 /*
422 * Adjust s_next_psn to count the
423 * expected number of responses.
424 */
425 if (len > pmtu)
426 qp->s_next_psn += (len - 1) / pmtu;
427 wqe->lpsn = qp->s_next_psn++;
428 }
e622f2f4 429
f931551b 430 ohdr->u.rc.reth.vaddr =
e622f2f4 431 cpu_to_be64(wqe->rdma_wr.remote_addr);
f931551b 432 ohdr->u.rc.reth.rkey =
e622f2f4 433 cpu_to_be32(wqe->rdma_wr.rkey);
f931551b
RC		 434 ohdr->u.rc.reth.length = cpu_to_be32(len);
435 qp->s_state = OP(RDMA_READ_REQUEST);
436 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
437 ss = NULL;
438 len = 0;
439 bth2 |= IB_BTH_REQ_ACK;
440 if (++qp->s_cur == qp->s_size)
441 qp->s_cur = 0;
442 break;
443
444 case IB_WR_ATOMIC_CMP_AND_SWP:
445 case IB_WR_ATOMIC_FETCH_AND_ADD:
446 /*
447 * Don't allow more operations to be started
448 * than the QP limits allow.
449 */
450 if (newreq) {
451 if (qp->s_num_rd_atomic >=
452 qp->s_max_rd_atomic) {
453 qp->s_flags |= QIB_S_WAIT_RDMAR;
454 goto bail;
455 }
456 qp->s_num_rd_atomic++;
457 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT))
458 qp->s_lsn++;
459 wqe->lpsn = wqe->psn;
460 }
e622f2f4 461 if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
f931551b
RC		 462 qp->s_state = OP(COMPARE_SWAP);
463 ohdr->u.atomic_eth.swap_data = cpu_to_be64(
e622f2f4 464 wqe->atomic_wr.swap);
f931551b 465 ohdr->u.atomic_eth.compare_data = cpu_to_be64(
e622f2f4 466 wqe->atomic_wr.compare_add);
f931551b
RC		 467 } else {
468 qp->s_state = OP(FETCH_ADD);
469 ohdr->u.atomic_eth.swap_data = cpu_to_be64(
e622f2f4 470 wqe->atomic_wr.compare_add);
f931551b
RC		 471 ohdr->u.atomic_eth.compare_data = 0;
472 }
473 ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32(
e622f2f4 474 wqe->atomic_wr.remote_addr >> 32);
f931551b 475 ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32(
e622f2f4 476 wqe->atomic_wr.remote_addr);
f931551b 477 ohdr->u.atomic_eth.rkey = cpu_to_be32(
e622f2f4 478 wqe->atomic_wr.rkey);
f931551b
RC		 479 hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
480 ss = NULL;
481 len = 0;
482 bth2 |= IB_BTH_REQ_ACK;
483 if (++qp->s_cur == qp->s_size)
484 qp->s_cur = 0;
485 break;
486
487 default:
488 goto bail;
489 }
490 qp->s_sge.sge = wqe->sg_list[0];
491 qp->s_sge.sg_list = wqe->sg_list + 1;
492 qp->s_sge.num_sge = wqe->wr.num_sge;
493 qp->s_sge.total_len = wqe->length;
494 qp->s_len = wqe->length;
495 if (newreq) {
496 qp->s_tail++;
497 if (qp->s_tail >= qp->s_size)
498 qp->s_tail = 0;
499 }
500 if (wqe->wr.opcode == IB_WR_RDMA_READ)
501 qp->s_psn = wqe->lpsn + 1;
502 else {
503 qp->s_psn++;
504 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0)
505 qp->s_next_psn = qp->s_psn;
506 }
507 break;
508
509 case OP(RDMA_READ_RESPONSE_FIRST):
510 /*
511 * qp->s_state is normally set to the opcode of the
512 * last packet constructed for new requests and therefore
513 * is never set to RDMA read response.
514 * RDMA_READ_RESPONSE_FIRST is used by the ACK processing
515 * thread to indicate a SEND needs to be restarted from an
516 * earlier PSN without interferring with the sending thread.
517 * See qib_restart_rc().
518 */
519 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
520 /* FALLTHROUGH */
521 case OP(SEND_FIRST):
522 qp->s_state = OP(SEND_MIDDLE);
523 /* FALLTHROUGH */
524 case OP(SEND_MIDDLE):
525 bth2 = qp->s_psn++ & QIB_PSN_MASK;
526 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0)
527 qp->s_next_psn = qp->s_psn;
528 ss = &qp->s_sge;
529 len = qp->s_len;
530 if (len > pmtu) {
531 len = pmtu;
532 break;
533 }
534 if (wqe->wr.opcode == IB_WR_SEND)
535 qp->s_state = OP(SEND_LAST);
536 else {
537 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
538 /* Immediate data comes after the BTH */
539 ohdr->u.imm_data = wqe->wr.ex.imm_data;
540 hwords += 1;
541 }
542 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
543 bth0 |= IB_BTH_SOLICITED;
544 bth2 |= IB_BTH_REQ_ACK;
545 qp->s_cur++;
546 if (qp->s_cur >= qp->s_size)
547 qp->s_cur = 0;
548 break;
549
550 case OP(RDMA_READ_RESPONSE_LAST):
551 /*
552 * qp->s_state is normally set to the opcode of the
553 * last packet constructed for new requests and therefore
554 * is never set to RDMA read response.
555 * RDMA_READ_RESPONSE_LAST is used by the ACK processing
556 * thread to indicate a RDMA write needs to be restarted from
557 * an earlier PSN without interferring with the sending thread.
558 * See qib_restart_rc().
559 */
560 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
561 /* FALLTHROUGH */
562 case OP(RDMA_WRITE_FIRST):
563 qp->s_state = OP(RDMA_WRITE_MIDDLE);
564 /* FALLTHROUGH */
565 case OP(RDMA_WRITE_MIDDLE):
566 bth2 = qp->s_psn++ & QIB_PSN_MASK;
567 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0)
568 qp->s_next_psn = qp->s_psn;
569 ss = &qp->s_sge;
570 len = qp->s_len;
571 if (len > pmtu) {
572 len = pmtu;
573 break;
574 }
575 if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
576 qp->s_state = OP(RDMA_WRITE_LAST);
577 else {
578 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
579 /* Immediate data comes after the BTH */
580 ohdr->u.imm_data = wqe->wr.ex.imm_data;
581 hwords += 1;
582 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
583 bth0 |= IB_BTH_SOLICITED;
584 }
585 bth2 |= IB_BTH_REQ_ACK;
586 qp->s_cur++;
587 if (qp->s_cur >= qp->s_size)
588 qp->s_cur = 0;
589 break;
590
591 case OP(RDMA_READ_RESPONSE_MIDDLE):
592 /*
593 * qp->s_state is normally set to the opcode of the
594 * last packet constructed for new requests and therefore
595 * is never set to RDMA read response.
596 * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing
597 * thread to indicate a RDMA read needs to be restarted from
598 * an earlier PSN without interferring with the sending thread.
599 * See qib_restart_rc().
600 */
601 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
602 ohdr->u.rc.reth.vaddr =
e622f2f4 603 cpu_to_be64(wqe->rdma_wr.remote_addr + len);
f931551b 604 ohdr->u.rc.reth.rkey =
e622f2f4 605 cpu_to_be32(wqe->rdma_wr.rkey);
f931551b
RC		 606 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
607 qp->s_state = OP(RDMA_READ_REQUEST);
608 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
609 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK;
610 qp->s_psn = wqe->lpsn + 1;
611 ss = NULL;
612 len = 0;
613 qp->s_cur++;
614 if (qp->s_cur == qp->s_size)
615 qp->s_cur = 0;
616 break;
617 }
618 qp->s_sending_hpsn = bth2;
619 delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8;
620 if (delta && delta % QIB_PSN_CREDIT == 0)
621 bth2 |= IB_BTH_REQ_ACK;
622 if (qp->s_flags & QIB_S_SEND_ONE) {
623 qp->s_flags &= ~QIB_S_SEND_ONE;
624 qp->s_flags |= QIB_S_WAIT_ACK;
625 bth2 |= IB_BTH_REQ_ACK;
626 }
627 qp->s_len -= len;
628 qp->s_hdrwords = hwords;
629 qp->s_cur_sge = ss;
630 qp->s_cur_size = len;
631 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2);
632done:
633 ret = 1;
634 goto unlock;
635
636bail:
637 qp->s_flags &= ~QIB_S_BUSY;
638unlock:
639 spin_unlock_irqrestore(&qp->s_lock, flags);
640 return ret;
641}
642
643/**
644 * qib_send_rc_ack - Construct an ACK packet and send it
645 * @qp: a pointer to the QP
646 *
647 * This is called from qib_rc_rcv() and qib_kreceive().
648 * Note that RDMA reads and atomics are handled in the
649 * send side QP state and tasklet.
650 */
7c2e11fe 651void qib_send_rc_ack(struct rvt_qp *qp)
f931551b
RC		 652{
653 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
654 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
655 struct qib_pportdata *ppd = ppd_from_ibp(ibp);
656 u64 pbc;
657 u16 lrh0;
658 u32 bth0;
659 u32 hwords;
660 u32 pbufn;
661 u32 __iomem *piobuf;
662 struct qib_ib_header hdr;
663 struct qib_other_headers *ohdr;
664 u32 control;
665 unsigned long flags;
666
667 spin_lock_irqsave(&qp->s_lock, flags);
668
669 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK))
670 goto unlock;
671
672 /* Don't send ACK or NAK if a RDMA read or atomic is pending. */
673 if ((qp->s_flags & QIB_S_RESP_PENDING) || qp->s_rdma_ack_cnt)
674 goto queue_ack;
675
676 /* Construct the header with s_lock held so APM doesn't change it. */
677 ohdr = &hdr.u.oth;
678 lrh0 = QIB_LRH_BTH;
679 /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */
680 hwords = 6;
681 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
682 hwords += qib_make_grh(ibp, &hdr.u.l.grh,
683 &qp->remote_ah_attr.grh, hwords, 0);
684 ohdr = &hdr.u.l.oth;
685 lrh0 = QIB_LRH_GRH;
686 }
687 /* read pkey_index w/o lock (its atomic) */
688 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
689 if (qp->s_mig_state == IB_MIG_MIGRATED)
690 bth0 |= IB_BTH_MIG_REQ;
691 if (qp->r_nak_state)
692 ohdr->u.aeth = cpu_to_be32((qp->r_msn & QIB_MSN_MASK) |
693 (qp->r_nak_state <<
694 QIB_AETH_CREDIT_SHIFT));
695 else
696 ohdr->u.aeth = qib_compute_aeth(qp);
697 lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 |
698 qp->remote_ah_attr.sl << 4;
699 hdr.lrh[0] = cpu_to_be16(lrh0);
700 hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
701 hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
702 hdr.lrh[3] = cpu_to_be16(ppd->lid | qp->remote_ah_attr.src_path_bits);
703 ohdr->bth[0] = cpu_to_be32(bth0);
704 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
705 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK);
706
707 spin_unlock_irqrestore(&qp->s_lock, flags);
708
709 /* Don't try to send ACKs if the link isn't ACTIVE */
710 if (!(ppd->lflags & QIBL_LINKACTIVE))
711 goto done;
712
713 control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC,
714 qp->s_srate, lrh0 >> 12);
715 /* length is + 1 for the control dword */
716 pbc = ((u64) control << 32) | (hwords + 1);
717
718 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn);
719 if (!piobuf) {
720 /*
721 * We are out of PIO buffers at the moment.
722 * Pass responsibility for sending the ACK to the
723 * send tasklet so that when a PIO buffer becomes
724 * available, the ACK is sent ahead of other outgoing
725 * packets.
726 */
727 spin_lock_irqsave(&qp->s_lock, flags);
728 goto queue_ack;
729 }
730
731 /*
732 * Write the pbc.
733 * We have to flush after the PBC for correctness
734 * on some cpus or WC buffer can be written out of order.
735 */
736 writeq(pbc, piobuf);
737
738 if (dd->flags & QIB_PIO_FLUSH_WC) {
739 u32 *hdrp = (u32 *) &hdr;
740
741 qib_flush_wc();
742 qib_pio_copy(piobuf + 2, hdrp, hwords - 1);
743 qib_flush_wc();
744 __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1);
745 } else
746 qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords);
747
748 if (dd->flags & QIB_USE_SPCL_TRIG) {
749 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023;
750
751 qib_flush_wc();
752 __raw_writel(0xaebecede, piobuf + spcl_off);
753 }
754
755 qib_flush_wc();
756 qib_sendbuf_done(dd, pbufn);
757
7d7632ad 758 this_cpu_inc(ibp->pmastats->n_unicast_xmit);
f931551b
RC		 759 goto done;
760
761queue_ack:
762 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) {
f24a6d48 763 this_cpu_inc(*ibp->rvp.rc_qacks);
f931551b
RC		 764 qp->s_flags |= QIB_S_ACK_PENDING | QIB_S_RESP_PENDING;
765 qp->s_nak_state = qp->r_nak_state;
766 qp->s_ack_psn = qp->r_ack_psn;
767
768 /* Schedule the send tasklet. */
769 qib_schedule_send(qp);
770 }
771unlock:
772 spin_unlock_irqrestore(&qp->s_lock, flags);
773done:
774 return;
775}
776
777/**
778 * reset_psn - reset the QP state to send starting from PSN
779 * @qp: the QP
780 * @psn: the packet sequence number to restart at
781 *
782 * This is called from qib_rc_rcv() to process an incoming RC ACK
783 * for the given QP.
784 * Called at interrupt level with the QP s_lock held.
785 */
7c2e11fe 786static void reset_psn(struct rvt_qp *qp, u32 psn)
f931551b
RC		 787{
788 u32 n = qp->s_acked;
7c2e11fe 789 struct rvt_swqe *wqe = get_swqe_ptr(qp, n);
f931551b
RC		 790 u32 opcode;
791
792 qp->s_cur = n;
793
794 /*
795 * If we are starting the request from the beginning,
796 * let the normal send code handle initialization.
797 */
798 if (qib_cmp24(psn, wqe->psn) <= 0) {
799 qp->s_state = OP(SEND_LAST);
800 goto done;
801 }
802
803 /* Find the work request opcode corresponding to the given PSN. */
804 opcode = wqe->wr.opcode;
805 for (;;) {
806 int diff;
807
808 if (++n == qp->s_size)
809 n = 0;
810 if (n == qp->s_tail)
811 break;
812 wqe = get_swqe_ptr(qp, n);
813 diff = qib_cmp24(psn, wqe->psn);
814 if (diff < 0)
815 break;
816 qp->s_cur = n;
817 /*
818 * If we are starting the request from the beginning,
819 * let the normal send code handle initialization.
820 */
821 if (diff == 0) {
822 qp->s_state = OP(SEND_LAST);
823 goto done;
824 }
825 opcode = wqe->wr.opcode;
826 }
827
828 /*
829 * Set the state to restart in the middle of a request.
830 * Don't change the s_sge, s_cur_sge, or s_cur_size.
831 * See qib_make_rc_req().
832 */
833 switch (opcode) {
834 case IB_WR_SEND:
835 case IB_WR_SEND_WITH_IMM:
836 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
837 break;
838
839 case IB_WR_RDMA_WRITE:
840 case IB_WR_RDMA_WRITE_WITH_IMM:
841 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
842 break;
843
844 case IB_WR_RDMA_READ:
845 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
846 break;
847
848 default:
849 /*
850 * This case shouldn't happen since its only
851 * one PSN per req.
852 */
853 qp->s_state = OP(SEND_LAST);
854 }
855done:
856 qp->s_psn = psn;
857 /*
858 * Set QIB_S_WAIT_PSN as qib_rc_complete() may start the timer
859 * asynchronously before the send tasklet can get scheduled.
860 * Doing it in qib_make_rc_req() is too late.
861 */
862 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
863 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
864 qp->s_flags |= QIB_S_WAIT_PSN;
865}
866
867/*
868 * Back up requester to resend the last un-ACKed request.
a5210c12 869 * The QP r_lock and s_lock should be held and interrupts disabled.
f931551b 870 */
7c2e11fe 871static void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
f931551b 872{
7c2e11fe 873 struct rvt_swqe *wqe = get_swqe_ptr(qp, qp->s_acked);
f931551b
RC		 874 struct qib_ibport *ibp;
875
876 if (qp->s_retry == 0) {
877 if (qp->s_mig_state == IB_MIG_ARMED) {
878 qib_migrate_qp(qp);
879 qp->s_retry = qp->s_retry_cnt;
880 } else if (qp->s_last == qp->s_acked) {
881 qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
882 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR);
883 return;
884 } else /* XXX need to handle delayed completion */
885 return;
886 } else
887 qp->s_retry--;
888
889 ibp = to_iport(qp->ibqp.device, qp->port_num);
890 if (wqe->wr.opcode == IB_WR_RDMA_READ)
f24a6d48 891 ibp->rvp.n_rc_resends++;
f931551b 892 else
f24a6d48 893 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
f931551b
RC		 894
895 qp->s_flags &= ~(QIB_S_WAIT_FENCE | QIB_S_WAIT_RDMAR |
896 QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_PSN |
897 QIB_S_WAIT_ACK);
898 if (wait)
899 qp->s_flags |= QIB_S_SEND_ONE;
900 reset_psn(qp, psn);
901}
902
903/*
904 * This is called from s_timer for missing responses.
905 */
906static void rc_timeout(unsigned long arg)
907{
7c2e11fe 908 struct rvt_qp *qp = (struct rvt_qp *)arg;
f931551b
RC		 909 struct qib_ibport *ibp;
910 unsigned long flags;
911
a5210c12
RC		 912 spin_lock_irqsave(&qp->r_lock, flags);
913 spin_lock(&qp->s_lock);
f931551b
RC		 914 if (qp->s_flags & QIB_S_TIMER) {
915 ibp = to_iport(qp->ibqp.device, qp->port_num);
f24a6d48 916 ibp->rvp.n_rc_timeouts++;
f931551b
RC		 917 qp->s_flags &= ~QIB_S_TIMER;
918 del_timer(&qp->s_timer);
919 qib_restart_rc(qp, qp->s_last_psn + 1, 1);
920 qib_schedule_send(qp);
921 }
a5210c12
RC		 922 spin_unlock(&qp->s_lock);
923 spin_unlock_irqrestore(&qp->r_lock, flags);
f931551b
RC		 924}
925
926/*
927 * This is called from s_timer for RNR timeouts.
928 */
929void qib_rc_rnr_retry(unsigned long arg)
930{
7c2e11fe 931 struct rvt_qp *qp = (struct rvt_qp *)arg;
f931551b
RC		 932 unsigned long flags;
933
934 spin_lock_irqsave(&qp->s_lock, flags);
935 if (qp->s_flags & QIB_S_WAIT_RNR) {
936 qp->s_flags &= ~QIB_S_WAIT_RNR;
937 del_timer(&qp->s_timer);
938 qib_schedule_send(qp);
939 }
940 spin_unlock_irqrestore(&qp->s_lock, flags);
941}
942
943/*
944 * Set qp->s_sending_psn to the next PSN after the given one.
945 * This would be psn+1 except when RDMA reads are present.
946 */
7c2e11fe 947static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
f931551b 948{
7c2e11fe 949 struct rvt_swqe *wqe;
f931551b
RC		 950 u32 n = qp->s_last;
951
952 /* Find the work request corresponding to the given PSN. */
953 for (;;) {
954 wqe = get_swqe_ptr(qp, n);
955 if (qib_cmp24(psn, wqe->lpsn) <= 0) {
956 if (wqe->wr.opcode == IB_WR_RDMA_READ)
957 qp->s_sending_psn = wqe->lpsn + 1;
958 else
959 qp->s_sending_psn = psn + 1;
960 break;
961 }
962 if (++n == qp->s_size)
963 n = 0;
964 if (n == qp->s_tail)
965 break;
966 }
967}
968
969/*
970 * This should be called with the QP s_lock held and interrupts disabled.
971 */
7c2e11fe 972void qib_rc_send_complete(struct rvt_qp *qp, struct qib_ib_header *hdr)
f931551b
RC		 973{
974 struct qib_other_headers *ohdr;
7c2e11fe 975 struct rvt_swqe *wqe;
f931551b
RC		 976 struct ib_wc wc;
977 unsigned i;
978 u32 opcode;
979 u32 psn;
980
981 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND))
982 return;
983
984 /* Find out where the BTH is */
985 if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH)
986 ohdr = &hdr->u.oth;
987 else
988 ohdr = &hdr->u.l.oth;
989
990 opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
991 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
992 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
993 WARN_ON(!qp->s_rdma_ack_cnt);
994 qp->s_rdma_ack_cnt--;
995 return;
996 }
997
998 psn = be32_to_cpu(ohdr->bth[2]);
999 reset_sending_psn(qp, psn);
1000
1001 /*
1002 * Start timer after a packet requesting an ACK has been sent and
1003 * there are still requests that haven't been acked.
1004 */
1005 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail &&
c0af2c05
MM		 1006 !(qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR | QIB_S_WAIT_PSN)) &&
1007 (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK))
f931551b
RC		 1008 start_timer(qp);
1009
1010 while (qp->s_last != qp->s_acked) {
1011 wqe = get_swqe_ptr(qp, qp->s_last);
1012 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
1013 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
1014 break;
1015 for (i = 0; i < wqe->wr.num_sge; i++) {
7c2e11fe 1016 struct rvt_sge *sge = &wqe->sg_list[i];
f931551b 1017
7c2e11fe 1018 rvt_put_mr(sge->mr);
f931551b
RC		 1019 }
1020 /* Post a send completion queue entry if requested. */
1021 if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) ||
1022 (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
041af0bb 1023 memset(&wc, 0, sizeof(wc));
f931551b
RC		 1024 wc.wr_id = wqe->wr.wr_id;
1025 wc.status = IB_WC_SUCCESS;
1026 wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];
1027 wc.byte_len = wqe->length;
1028 wc.qp = &qp->ibqp;
1029 qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
1030 }
1031 if (++qp->s_last >= qp->s_size)
1032 qp->s_last = 0;
1033 }
1034 /*
1035 * If we were waiting for sends to complete before resending,
1036 * and they are now complete, restart sending.
1037 */
1038 if (qp->s_flags & QIB_S_WAIT_PSN &&
1039 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1040 qp->s_flags &= ~QIB_S_WAIT_PSN;
1041 qp->s_sending_psn = qp->s_psn;
1042 qp->s_sending_hpsn = qp->s_psn - 1;
1043 qib_schedule_send(qp);
1044 }
1045}
1046
7c2e11fe 1047static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
f931551b
RC		 1048{
1049 qp->s_last_psn = psn;
1050}
1051
1052/*
1053 * Generate a SWQE completion.
1054 * This is similar to qib_send_complete but has to check to be sure
1055 * that the SGEs are not being referenced if the SWQE is being resent.
1056 */
7c2e11fe
DD		 1057static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
1058 struct rvt_swqe *wqe,
f931551b
RC		 1059 struct qib_ibport *ibp)
1060{
1061 struct ib_wc wc;
1062 unsigned i;
1063
1064 /*
1065 * Don't decrement refcount and don't generate a
1066 * completion if the SWQE is being resent until the send
1067 * is finished.
1068 */
1069 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
1070 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1071 for (i = 0; i < wqe->wr.num_sge; i++) {
7c2e11fe 1072 struct rvt_sge *sge = &wqe->sg_list[i];
f931551b 1073
7c2e11fe 1074 rvt_put_mr(sge->mr);
f931551b
RC		 1075 }
1076 /* Post a send completion queue entry if requested. */
1077 if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) ||
1078 (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
041af0bb 1079 memset(&wc, 0, sizeof(wc));
f931551b
RC		 1080 wc.wr_id = wqe->wr.wr_id;
1081 wc.status = IB_WC_SUCCESS;
1082 wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];
1083 wc.byte_len = wqe->length;
1084 wc.qp = &qp->ibqp;
1085 qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
1086 }
1087 if (++qp->s_last >= qp->s_size)
1088 qp->s_last = 0;
1089 } else
f24a6d48 1090 this_cpu_inc(*ibp->rvp.rc_delayed_comp);
f931551b
RC		 1091
1092 qp->s_retry = qp->s_retry_cnt;
1093 update_last_psn(qp, wqe->lpsn);
1094
1095 /*
1096 * If we are completing a request which is in the process of
1097 * being resent, we can stop resending it since we know the
1098 * responder has already seen it.
1099 */
1100 if (qp->s_acked == qp->s_cur) {
1101 if (++qp->s_cur >= qp->s_size)
1102 qp->s_cur = 0;
1103 qp->s_acked = qp->s_cur;
1104 wqe = get_swqe_ptr(qp, qp->s_cur);
1105 if (qp->s_acked != qp->s_tail) {
1106 qp->s_state = OP(SEND_LAST);
1107 qp->s_psn = wqe->psn;
1108 }
1109 } else {
1110 if (++qp->s_acked >= qp->s_size)
1111 qp->s_acked = 0;
1112 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
1113 qp->s_draining = 0;
1114 wqe = get_swqe_ptr(qp, qp->s_acked);
1115 }
1116 return wqe;
1117}
1118
1119/**
1120 * do_rc_ack - process an incoming RC ACK
1121 * @qp: the QP the ACK came in on
1122 * @psn: the packet sequence number of the ACK
1123 * @opcode: the opcode of the request that resulted in the ACK
1124 *
1125 * This is called from qib_rc_rcv_resp() to process an incoming RC ACK
1126 * for the given QP.
1127 * Called at interrupt level with the QP s_lock held.
1128 * Returns 1 if OK, 0 if current operation should be aborted (NAK).
1129 */
7c2e11fe 1130static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
f931551b
RC		 1131 u64 val, struct qib_ctxtdata *rcd)
1132{
1133 struct qib_ibport *ibp;
1134 enum ib_wc_status status;
7c2e11fe 1135 struct rvt_swqe *wqe;
f931551b
RC		 1136 int ret = 0;
1137 u32 ack_psn;
1138 int diff;
1139
1140 /* Remove QP from retry timer */
1141 if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) {
1142 qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR);
1143 del_timer(&qp->s_timer);
1144 }
1145
1146 /*
1147 * Note that NAKs implicitly ACK outstanding SEND and RDMA write
1148 * requests and implicitly NAK RDMA read and atomic requests issued
1149 * before the NAK'ed request. The MSN won't include the NAK'ed
1150 * request but will include an ACK'ed request(s).
1151 */
1152 ack_psn = psn;
1153 if (aeth >> 29)
1154 ack_psn--;
1155 wqe = get_swqe_ptr(qp, qp->s_acked);
1156 ibp = to_iport(qp->ibqp.device, qp->port_num);
1157
1158 /*
1159 * The MSN might be for a later WQE than the PSN indicates so
1160 * only complete WQEs that the PSN finishes.
1161 */
1162 while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) {
1163 /*
1164 * RDMA_READ_RESPONSE_ONLY is a special case since
1165 * we want to generate completion events for everything
1166 * before the RDMA read, copy the data, then generate
1167 * the completion for the read.
1168 */
1169 if (wqe->wr.opcode == IB_WR_RDMA_READ &&
1170 opcode == OP(RDMA_READ_RESPONSE_ONLY) &&
1171 diff == 0) {
1172 ret = 1;
1173 goto bail;
1174 }
1175 /*
1176 * If this request is a RDMA read or atomic, and the ACK is
1177 * for a later operation, this ACK NAKs the RDMA read or
1178 * atomic. In other words, only a RDMA_READ_LAST or ONLY
1179 * can ACK a RDMA read and likewise for atomic ops. Note
1180 * that the NAK case can only happen if relaxed ordering is
1181 * used and requests are sent after an RDMA read or atomic
1182 * is sent but before the response is received.
1183 */
1184 if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
1185 (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) ||
1186 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1187 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
1188 (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) {
1189 /* Retry this request. */
1190 if (!(qp->r_flags & QIB_R_RDMAR_SEQ)) {
1191 qp->r_flags |= QIB_R_RDMAR_SEQ;
1192 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1193 if (list_empty(&qp->rspwait)) {
1194 qp->r_flags |= QIB_R_RSP_SEND;
1195 atomic_inc(&qp->refcount);
1196 list_add_tail(&qp->rspwait,
1197 &rcd->qp_wait_list);
1198 }
1199 }
1200 /*
1201 * No need to process the ACK/NAK since we are
1202 * restarting an earlier request.
1203 */
1204 goto bail;
1205 }
1206 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1207 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
1208 u64 *vaddr = wqe->sg_list[0].vaddr;
1209 *vaddr = val;
1210 }
1211 if (qp->s_num_rd_atomic &&
1212 (wqe->wr.opcode == IB_WR_RDMA_READ ||
1213 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1214 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
1215 qp->s_num_rd_atomic--;
1216 /* Restart sending task if fence is complete */
1217 if ((qp->s_flags & QIB_S_WAIT_FENCE) &&
1218 !qp->s_num_rd_atomic) {
1219 qp->s_flags &= ~(QIB_S_WAIT_FENCE |
1220 QIB_S_WAIT_ACK);
1221 qib_schedule_send(qp);
1222 } else if (qp->s_flags & QIB_S_WAIT_RDMAR) {
1223 qp->s_flags &= ~(QIB_S_WAIT_RDMAR |
1224 QIB_S_WAIT_ACK);
1225 qib_schedule_send(qp);
1226 }
1227 }
1228 wqe = do_rc_completion(qp, wqe, ibp);
1229 if (qp->s_acked == qp->s_tail)
1230 break;
1231 }
1232
1233 switch (aeth >> 29) {
1234 case 0: /* ACK */
f24a6d48 1235 this_cpu_inc(*ibp->rvp.rc_acks);
f931551b
RC		 1236 if (qp->s_acked != qp->s_tail) {
1237 /*
1238 * We are expecting more ACKs so
1239 * reset the retransmit timer.
1240 */
1241 start_timer(qp);
1242 /*
1243 * We can stop resending the earlier packets and
1244 * continue with the next packet the receiver wants.
1245 */
1246 if (qib_cmp24(qp->s_psn, psn) <= 0)
1247 reset_psn(qp, psn + 1);
1248 } else if (qib_cmp24(qp->s_psn, psn) <= 0) {
1249 qp->s_state = OP(SEND_LAST);
1250 qp->s_psn = psn + 1;
1251 }
1252 if (qp->s_flags & QIB_S_WAIT_ACK) {
1253 qp->s_flags &= ~QIB_S_WAIT_ACK;
1254 qib_schedule_send(qp);
1255 }
1256 qib_get_credit(qp, aeth);
1257 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1258 qp->s_retry = qp->s_retry_cnt;
1259 update_last_psn(qp, psn);
1260 ret = 1;
1261 goto bail;
1262
1263 case 1: /* RNR NAK */
f24a6d48 1264 ibp->rvp.n_rnr_naks++;
f931551b
RC		 1265 if (qp->s_acked == qp->s_tail)
1266 goto bail;
1267 if (qp->s_flags & QIB_S_WAIT_RNR)
1268 goto bail;
1269 if (qp->s_rnr_retry == 0) {
1270 status = IB_WC_RNR_RETRY_EXC_ERR;
1271 goto class_b;
1272 }
1273 if (qp->s_rnr_retry_cnt < 7)
1274 qp->s_rnr_retry--;
1275
1276 /* The last valid PSN is the previous PSN. */
1277 update_last_psn(qp, psn - 1);
1278
f24a6d48 1279 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
f931551b
RC		 1280
1281 reset_psn(qp, psn);
1282
1283 qp->s_flags &= ~(QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_ACK);
1284 qp->s_flags |= QIB_S_WAIT_RNR;
1285 qp->s_timer.function = qib_rc_rnr_retry;
1286 qp->s_timer.expires = jiffies + usecs_to_jiffies(
1287 ib_qib_rnr_table[(aeth >> QIB_AETH_CREDIT_SHIFT) &
1288 QIB_AETH_CREDIT_MASK]);
1289 add_timer(&qp->s_timer);
1290 goto bail;
1291
1292 case 3: /* NAK */
1293 if (qp->s_acked == qp->s_tail)
1294 goto bail;
1295 /* The last valid PSN is the previous PSN. */
1296 update_last_psn(qp, psn - 1);
1297 switch ((aeth >> QIB_AETH_CREDIT_SHIFT) &
1298 QIB_AETH_CREDIT_MASK) {
1299 case 0: /* PSN sequence error */
f24a6d48 1300 ibp->rvp.n_seq_naks++;
f931551b
RC		 1301 /*
1302 * Back up to the responder's expected PSN.
1303 * Note that we might get a NAK in the middle of an
1304 * RDMA READ response which terminates the RDMA
1305 * READ.
1306 */
1307 qib_restart_rc(qp, psn, 0);
1308 qib_schedule_send(qp);
1309 break;
1310
1311 case 1: /* Invalid Request */
1312 status = IB_WC_REM_INV_REQ_ERR;
f24a6d48 1313 ibp->rvp.n_other_naks++;
f931551b
RC		 1314 goto class_b;
1315
1316 case 2: /* Remote Access Error */
1317 status = IB_WC_REM_ACCESS_ERR;
f24a6d48 1318 ibp->rvp.n_other_naks++;
f931551b
RC		 1319 goto class_b;
1320
1321 case 3: /* Remote Operation Error */
1322 status = IB_WC_REM_OP_ERR;
f24a6d48 1323 ibp->rvp.n_other_naks++;
f931551b
RC		 1324class_b:
1325 if (qp->s_last == qp->s_acked) {
1326 qib_send_complete(qp, wqe, status);
1327 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1328 }
1329 break;
1330
1331 default:
1332 /* Ignore other reserved NAK error codes */
1333 goto reserved;
1334 }
1335 qp->s_retry = qp->s_retry_cnt;
1336 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1337 goto bail;
1338
1339 default: /* 2: reserved */
1340reserved:
1341 /* Ignore reserved NAK codes. */
1342 goto bail;
1343 }
1344
1345bail:
1346 return ret;
1347}
1348
1349/*
1350 * We have seen an out of sequence RDMA read middle or last packet.
1351 * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE.
1352 */
7c2e11fe 1353static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
f931551b
RC		 1354 struct qib_ctxtdata *rcd)
1355{
7c2e11fe 1356 struct rvt_swqe *wqe;
f931551b
RC		 1357
1358 /* Remove QP from retry timer */
1359 if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) {
1360 qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR);
1361 del_timer(&qp->s_timer);
1362 }
1363
1364 wqe = get_swqe_ptr(qp, qp->s_acked);
1365
1366 while (qib_cmp24(psn, wqe->lpsn) > 0) {
1367 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1368 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1369 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1370 break;
1371 wqe = do_rc_completion(qp, wqe, ibp);
1372 }
1373
f24a6d48 1374 ibp->rvp.n_rdma_seq++;
f931551b
RC		 1375 qp->r_flags |= QIB_R_RDMAR_SEQ;
1376 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1377 if (list_empty(&qp->rspwait)) {
1378 qp->r_flags |= QIB_R_RSP_SEND;
1379 atomic_inc(&qp->refcount);
1380 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1381 }
1382}
1383
1384/**
1385 * qib_rc_rcv_resp - process an incoming RC response packet
1386 * @ibp: the port this packet came in on
1387 * @ohdr: the other headers for this packet
1388 * @data: the packet data
1389 * @tlen: the packet length
1390 * @qp: the QP for this packet
1391 * @opcode: the opcode for this packet
1392 * @psn: the packet sequence number for this packet
1393 * @hdrsize: the header length
1394 * @pmtu: the path MTU
1395 *
1396 * This is called from qib_rc_rcv() to process an incoming RC response
1397 * packet for the given QP.
1398 * Called at interrupt level.
1399 */
1400static void qib_rc_rcv_resp(struct qib_ibport *ibp,
1401 struct qib_other_headers *ohdr,
1402 void *data, u32 tlen,
7c2e11fe 1403 struct rvt_qp *qp,
f931551b
RC		 1404 u32 opcode,
1405 u32 psn, u32 hdrsize, u32 pmtu,
1406 struct qib_ctxtdata *rcd)
1407{
7c2e11fe 1408 struct rvt_swqe *wqe;
dd04e43d 1409 struct qib_pportdata *ppd = ppd_from_ibp(ibp);
f931551b
RC		 1410 enum ib_wc_status status;
1411 unsigned long flags;
1412 int diff;
1413 u32 pad;
1414 u32 aeth;
1415 u64 val;
1416
dd04e43d
MM		 1417 if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) {
1418 /*
1419 * If ACK'd PSN on SDMA busy list try to make progress to
1420 * reclaim SDMA credits.
1421 */
1422 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) &&
1423 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) {
1424
1425 /*
1426 * If send tasklet not running attempt to progress
1427 * SDMA queue.
1428 */
1429 if (!(qp->s_flags & QIB_S_BUSY)) {
1430 /* Acquire SDMA Lock */
1431 spin_lock_irqsave(&ppd->sdma_lock, flags);
1432 /* Invoke sdma make progress */
1433 qib_sdma_make_progress(ppd);
1434 /* Release SDMA Lock */
1435 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1436 }
1437 }
1438 }
1439
f931551b 1440 spin_lock_irqsave(&qp->s_lock, flags);
414ed90c
MM		 1441 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK))
1442 goto ack_done;
f931551b 1443
f931551b
RC		 1444 /* Ignore invalid responses. */
1445 if (qib_cmp24(psn, qp->s_next_psn) >= 0)
1446 goto ack_done;
1447
1448 /* Ignore duplicate responses. */
1449 diff = qib_cmp24(psn, qp->s_last_psn);
1450 if (unlikely(diff <= 0)) {
1451 /* Update credits for "ghost" ACKs */
1452 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) {
1453 aeth = be32_to_cpu(ohdr->u.aeth);
1454 if ((aeth >> 29) == 0)
1455 qib_get_credit(qp, aeth);
1456 }
1457 goto ack_done;
1458 }
1459
1460 /*
1461 * Skip everything other than the PSN we expect, if we are waiting
1462 * for a reply to a restarted RDMA read or atomic op.
1463 */
1464 if (qp->r_flags & QIB_R_RDMAR_SEQ) {
1465 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0)
1466 goto ack_done;
1467 qp->r_flags &= ~QIB_R_RDMAR_SEQ;
1468 }
1469
1470 if (unlikely(qp->s_acked == qp->s_tail))
1471 goto ack_done;
1472 wqe = get_swqe_ptr(qp, qp->s_acked);
1473 status = IB_WC_SUCCESS;
1474
1475 switch (opcode) {
1476 case OP(ACKNOWLEDGE):
1477 case OP(ATOMIC_ACKNOWLEDGE):
1478 case OP(RDMA_READ_RESPONSE_FIRST):
1479 aeth = be32_to_cpu(ohdr->u.aeth);
1480 if (opcode == OP(ATOMIC_ACKNOWLEDGE)) {
1481 __be32 *p = ohdr->u.at.atomic_ack_eth;
1482
1483 val = ((u64) be32_to_cpu(p[0]) << 32) |
1484 be32_to_cpu(p[1]);
1485 } else
1486 val = 0;
1487 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
1488 opcode != OP(RDMA_READ_RESPONSE_FIRST))
1489 goto ack_done;
1490 hdrsize += 4;
1491 wqe = get_swqe_ptr(qp, qp->s_acked);
1492 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1493 goto ack_op_err;
1494 /*
1495 * If this is a response to a resent RDMA read, we
1496 * have to be careful to copy the data to the right
1497 * location.
1498 */
1499 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1500 wqe, psn, pmtu);
1501 goto read_middle;
1502
1503 case OP(RDMA_READ_RESPONSE_MIDDLE):
1504 /* no AETH, no ACK */
1505 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1506 goto ack_seq_err;
1507 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1508 goto ack_op_err;
1509read_middle:
1510 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1511 goto ack_len_err;
1512 if (unlikely(pmtu >= qp->s_rdma_read_len))
1513 goto ack_len_err;
1514
1515 /*
1516 * We got a response so update the timeout.
1517 * 4.096 usec. * (1 << qp->timeout)
1518 */
1519 qp->s_flags |= QIB_S_TIMER;
d0f2faf7 1520 mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies);
f931551b
RC		 1521 if (qp->s_flags & QIB_S_WAIT_ACK) {
1522 qp->s_flags &= ~QIB_S_WAIT_ACK;
1523 qib_schedule_send(qp);
1524 }
1525
1526 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE))
1527 qp->s_retry = qp->s_retry_cnt;
1528
1529 /*
1530 * Update the RDMA receive state but do the copy w/o
1531 * holding the locks and blocking interrupts.
1532 */
1533 qp->s_rdma_read_len -= pmtu;
1534 update_last_psn(qp, psn);
1535 spin_unlock_irqrestore(&qp->s_lock, flags);
1536 qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0);
1537 goto bail;
1538
1539 case OP(RDMA_READ_RESPONSE_ONLY):
1540 aeth = be32_to_cpu(ohdr->u.aeth);
1541 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
1542 goto ack_done;
1543 /* Get the number of bytes the message was padded by. */
1544 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1545 /*
1546 * Check that the data size is >= 0 && <= pmtu.
1547 * Remember to account for the AETH header (4) and
1548 * ICRC (4).
1549 */
1550 if (unlikely(tlen < (hdrsize + pad + 8)))
1551 goto ack_len_err;
1552 /*
1553 * If this is a response to a resent RDMA read, we
1554 * have to be careful to copy the data to the right
1555 * location.
1556 */
1557 wqe = get_swqe_ptr(qp, qp->s_acked);
1558 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1559 wqe, psn, pmtu);
1560 goto read_last;
1561
1562 case OP(RDMA_READ_RESPONSE_LAST):
1563 /* ACKs READ req. */
1564 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1565 goto ack_seq_err;
1566 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1567 goto ack_op_err;
1568 /* Get the number of bytes the message was padded by. */
1569 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1570 /*
1571 * Check that the data size is >= 1 && <= pmtu.
1572 * Remember to account for the AETH header (4) and
1573 * ICRC (4).
1574 */
1575 if (unlikely(tlen <= (hdrsize + pad + 8)))
1576 goto ack_len_err;
1577read_last:
1578 tlen -= hdrsize + pad + 8;
1579 if (unlikely(tlen != qp->s_rdma_read_len))
1580 goto ack_len_err;
1581 aeth = be32_to_cpu(ohdr->u.aeth);
1582 qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0);
1583 WARN_ON(qp->s_rdma_read_sge.num_sge);
1584 (void) do_rc_ack(qp, aeth, psn,
1585 OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
1586 goto ack_done;
1587 }
1588
1589ack_op_err:
1590 status = IB_WC_LOC_QP_OP_ERR;
1591 goto ack_err;
1592
1593ack_seq_err:
1594 rdma_seq_err(qp, ibp, psn, rcd);
1595 goto ack_done;
1596
1597ack_len_err:
1598 status = IB_WC_LOC_LEN_ERR;
1599ack_err:
1600 if (qp->s_last == qp->s_acked) {
1601 qib_send_complete(qp, wqe, status);
1602 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1603 }
1604ack_done:
1605 spin_unlock_irqrestore(&qp->s_lock, flags);
1606bail:
1607 return;
1608}
1609
1610/**
1611 * qib_rc_rcv_error - process an incoming duplicate or error RC packet
1612 * @ohdr: the other headers for this packet
1613 * @data: the packet data
1614 * @qp: the QP for this packet
1615 * @opcode: the opcode for this packet
1616 * @psn: the packet sequence number for this packet
1617 * @diff: the difference between the PSN and the expected PSN
1618 *
1619 * This is called from qib_rc_rcv() to process an unexpected
1620 * incoming RC packet for the given QP.
1621 * Called at interrupt level.
1622 * Return 1 if no more processing is needed; otherwise return 0 to
1623 * schedule a response to be sent.
1624 */
1625static int qib_rc_rcv_error(struct qib_other_headers *ohdr,
1626 void *data,
7c2e11fe 1627 struct rvt_qp *qp,
f931551b
RC		 1628 u32 opcode,
1629 u32 psn,
1630 int diff,
1631 struct qib_ctxtdata *rcd)
1632{
1633 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
7c2e11fe 1634 struct rvt_ack_entry *e;
f931551b
RC		 1635 unsigned long flags;
1636 u8 i, prev;
1637 int old_req;
1638
1639 if (diff > 0) {
1640 /*
1641 * Packet sequence error.
1642 * A NAK will ACK earlier sends and RDMA writes.
1643 * Don't queue the NAK if we already sent one.
1644 */
1645 if (!qp->r_nak_state) {
f24a6d48 1646 ibp->rvp.n_rc_seqnak++;
f931551b
RC		 1647 qp->r_nak_state = IB_NAK_PSN_ERROR;
1648 /* Use the expected PSN. */
1649 qp->r_ack_psn = qp->r_psn;
1650 /*
1651 * Wait to send the sequence NAK until all packets
1652 * in the receive queue have been processed.
1653 * Otherwise, we end up propagating congestion.
1654 */
1655 if (list_empty(&qp->rspwait)) {
1656 qp->r_flags |= QIB_R_RSP_NAK;
1657 atomic_inc(&qp->refcount);
1658 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1659 }
1660 }
1661 goto done;
1662 }
1663
1664 /*
1665 * Handle a duplicate request. Don't re-execute SEND, RDMA
1666 * write or atomic op. Don't NAK errors, just silently drop
1667 * the duplicate request. Note that r_sge, r_len, and
1668 * r_rcv_len may be in use so don't modify them.
1669 *
1670 * We are supposed to ACK the earliest duplicate PSN but we
1671 * can coalesce an outstanding duplicate ACK. We have to
1672 * send the earliest so that RDMA reads can be restarted at
1673 * the requester's expected PSN.
1674 *
1675 * First, find where this duplicate PSN falls within the
1676 * ACKs previously sent.
1677 * old_req is true if there is an older response that is scheduled
1678 * to be sent before sending this one.
1679 */
1680 e = NULL;
1681 old_req = 1;
f24a6d48 1682 ibp->rvp.n_rc_dupreq++;
f931551b
RC		 1683
1684 spin_lock_irqsave(&qp->s_lock, flags);
f931551b
RC		 1685
1686 for (i = qp->r_head_ack_queue; ; i = prev) {
1687 if (i == qp->s_tail_ack_queue)
1688 old_req = 0;
1689 if (i)
1690 prev = i - 1;
1691 else
1692 prev = QIB_MAX_RDMA_ATOMIC;
1693 if (prev == qp->r_head_ack_queue) {
1694 e = NULL;
1695 break;
1696 }
1697 e = &qp->s_ack_queue[prev];
1698 if (!e->opcode) {
1699 e = NULL;
1700 break;
1701 }
1702 if (qib_cmp24(psn, e->psn) >= 0) {
1703 if (prev == qp->s_tail_ack_queue &&
1704 qib_cmp24(psn, e->lpsn) <= 0)
1705 old_req = 0;
1706 break;
1707 }
1708 }
1709 switch (opcode) {
1710 case OP(RDMA_READ_REQUEST): {
1711 struct ib_reth *reth;
1712 u32 offset;
1713 u32 len;
1714
1715 /*
1716 * If we didn't find the RDMA read request in the ack queue,
1717 * we can ignore this request.
1718 */
1719 if (!e || e->opcode != OP(RDMA_READ_REQUEST))
1720 goto unlock_done;
1721 /* RETH comes after BTH */
1722 reth = &ohdr->u.rc.reth;
1723 /*
1724 * Address range must be a subset of the original
1725 * request and start on pmtu boundaries.
1726 * We reuse the old ack_queue slot since the requester
1727 * should not back up and request an earlier PSN for the
1728 * same request.
1729 */
1730 offset = ((psn - e->psn) & QIB_PSN_MASK) *
cc6ea138 1731 qp->pmtu;
f931551b
RC		 1732 len = be32_to_cpu(reth->length);
1733 if (unlikely(offset + len != e->rdma_sge.sge_length))
1734 goto unlock_done;
1735 if (e->rdma_sge.mr) {
7c2e11fe 1736 rvt_put_mr(e->rdma_sge.mr);
f931551b
RC		 1737 e->rdma_sge.mr = NULL;
1738 }
1739 if (len != 0) {
1740 u32 rkey = be32_to_cpu(reth->rkey);
1741 u64 vaddr = be64_to_cpu(reth->vaddr);
1742 int ok;
1743
7c2e11fe 1744 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
f931551b
RC		 1745 IB_ACCESS_REMOTE_READ);
1746 if (unlikely(!ok))
1747 goto unlock_done;
1748 } else {
1749 e->rdma_sge.vaddr = NULL;
1750 e->rdma_sge.length = 0;
1751 e->rdma_sge.sge_length = 0;
1752 }
1753 e->psn = psn;
1754 if (old_req)
1755 goto unlock_done;
1756 qp->s_tail_ack_queue = prev;
1757 break;
1758 }
1759
1760 case OP(COMPARE_SWAP):
1761 case OP(FETCH_ADD): {
1762 /*
1763 * If we didn't find the atomic request in the ack queue
1764 * or the send tasklet is already backed up to send an
1765 * earlier entry, we can ignore this request.
1766 */
1767 if (!e || e->opcode != (u8) opcode || old_req)
1768 goto unlock_done;
1769 qp->s_tail_ack_queue = prev;
1770 break;
1771 }
1772
1773 default:
1774 /*
1775 * Ignore this operation if it doesn't request an ACK
1776 * or an earlier RDMA read or atomic is going to be resent.
1777 */
1778 if (!(psn & IB_BTH_REQ_ACK) || old_req)
1779 goto unlock_done;
1780 /*
1781 * Resend the most recent ACK if this request is
1782 * after all the previous RDMA reads and atomics.
1783 */
1784 if (i == qp->r_head_ack_queue) {
1785 spin_unlock_irqrestore(&qp->s_lock, flags);
1786 qp->r_nak_state = 0;
1787 qp->r_ack_psn = qp->r_psn - 1;
1788 goto send_ack;
1789 }
1790 /*
1791 * Try to send a simple ACK to work around a Mellanox bug
1792 * which doesn't accept a RDMA read response or atomic
1793 * response as an ACK for earlier SENDs or RDMA writes.
1794 */
1795 if (!(qp->s_flags & QIB_S_RESP_PENDING)) {
1796 spin_unlock_irqrestore(&qp->s_lock, flags);
1797 qp->r_nak_state = 0;
1798 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1;
1799 goto send_ack;
1800 }
1801 /*
1802 * Resend the RDMA read or atomic op which
1803 * ACKs this duplicate request.
1804 */
1805 qp->s_tail_ack_queue = i;
1806 break;
1807 }
1808 qp->s_ack_state = OP(ACKNOWLEDGE);
1809 qp->s_flags |= QIB_S_RESP_PENDING;
1810 qp->r_nak_state = 0;
1811 qib_schedule_send(qp);
1812
1813unlock_done:
1814 spin_unlock_irqrestore(&qp->s_lock, flags);
1815done:
1816 return 1;
1817
1818send_ack:
1819 return 0;
1820}
1821
7c2e11fe 1822void qib_rc_error(struct rvt_qp *qp, enum ib_wc_status err)
f931551b
RC		 1823{
1824 unsigned long flags;
1825 int lastwqe;
1826
1827 spin_lock_irqsave(&qp->s_lock, flags);
1828 lastwqe = qib_error_qp(qp, err);
1829 spin_unlock_irqrestore(&qp->s_lock, flags);
1830
1831 if (lastwqe) {
1832 struct ib_event ev;
1833
1834 ev.device = qp->ibqp.device;
1835 ev.element.qp = &qp->ibqp;
1836 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
1837 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
1838 }
1839}
1840
7c2e11fe 1841static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n)
f931551b
RC		 1842{
1843 unsigned next;
1844
1845 next = n + 1;
1846 if (next > QIB_MAX_RDMA_ATOMIC)
1847 next = 0;
1848 qp->s_tail_ack_queue = next;
1849 qp->s_ack_state = OP(ACKNOWLEDGE);
1850}
1851
1852/**
1853 * qib_rc_rcv - process an incoming RC packet
1854 * @rcd: the context pointer
1855 * @hdr: the header of this packet
1856 * @has_grh: true if the header has a GRH
1857 * @data: the packet data
1858 * @tlen: the packet length
1859 * @qp: the QP for this packet
1860 *
1861 * This is called from qib_qp_rcv() to process an incoming RC packet
1862 * for the given QP.
1863 * Called at interrupt level.
1864 */
1865void qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr,
7c2e11fe 1866 int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
f931551b
RC		 1867{
1868 struct qib_ibport *ibp = &rcd->ppd->ibport_data;
1869 struct qib_other_headers *ohdr;
1870 u32 opcode;
1871 u32 hdrsize;
1872 u32 psn;
1873 u32 pad;
1874 struct ib_wc wc;
cc6ea138 1875 u32 pmtu = qp->pmtu;
f931551b
RC		 1876 int diff;
1877 struct ib_reth *reth;
1878 unsigned long flags;
1879 int ret;
1880
1881 /* Check for GRH */
1882 if (!has_grh) {
1883 ohdr = &hdr->u.oth;
1884 hdrsize = 8 + 12; /* LRH + BTH */
1885 } else {
1886 ohdr = &hdr->u.l.oth;
1887 hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */
1888 }
1889
1890 opcode = be32_to_cpu(ohdr->bth[0]);
f931551b 1891 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode))
9fd5473d 1892 return;
f931551b
RC		 1893
1894 psn = be32_to_cpu(ohdr->bth[2]);
1895 opcode >>= 24;
1896
f931551b
RC		 1897 /*
1898 * Process responses (ACKs) before anything else. Note that the
1899 * packet sequence number will be for something in the send work
1900 * queue rather than the expected receive packet sequence number.
1901 * In other words, this QP is the requester.
1902 */
1903 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
1904 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
1905 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
1906 hdrsize, pmtu, rcd);
a5210c12 1907 return;
f931551b
RC		 1908 }
1909
1910 /* Compute 24 bits worth of difference. */
1911 diff = qib_cmp24(psn, qp->r_psn);
1912 if (unlikely(diff)) {
1913 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
a5210c12 1914 return;
f931551b
RC		 1915 goto send_ack;
1916 }
1917
1918 /* Check for opcode sequence errors. */
1919 switch (qp->r_state) {
1920 case OP(SEND_FIRST):
1921 case OP(SEND_MIDDLE):
1922 if (opcode == OP(SEND_MIDDLE) ||
1923 opcode == OP(SEND_LAST) ||
1924 opcode == OP(SEND_LAST_WITH_IMMEDIATE))
1925 break;
1926 goto nack_inv;
1927
1928 case OP(RDMA_WRITE_FIRST):
1929 case OP(RDMA_WRITE_MIDDLE):
1930 if (opcode == OP(RDMA_WRITE_MIDDLE) ||
1931 opcode == OP(RDMA_WRITE_LAST) ||
1932 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
1933 break;
1934 goto nack_inv;
1935
1936 default:
1937 if (opcode == OP(SEND_MIDDLE) ||
1938 opcode == OP(SEND_LAST) ||
1939 opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
1940 opcode == OP(RDMA_WRITE_MIDDLE) ||
1941 opcode == OP(RDMA_WRITE_LAST) ||
1942 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
1943 goto nack_inv;
1944 /*
1945 * Note that it is up to the requester to not send a new
1946 * RDMA read or atomic operation before receiving an ACK
1947 * for the previous operation.
1948 */
1949 break;
1950 }
1951
f931551b
RC		 1952 if (qp->state == IB_QPS_RTR && !(qp->r_flags & QIB_R_COMM_EST)) {
1953 qp->r_flags |= QIB_R_COMM_EST;
1954 if (qp->ibqp.event_handler) {
1955 struct ib_event ev;
1956
1957 ev.device = qp->ibqp.device;
1958 ev.element.qp = &qp->ibqp;
1959 ev.event = IB_EVENT_COMM_EST;
1960 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
1961 }
1962 }
1963
1964 /* OK, process the packet. */
1965 switch (opcode) {
1966 case OP(SEND_FIRST):
1967 ret = qib_get_rwqe(qp, 0);
1968 if (ret < 0)
1969 goto nack_op_err;
1970 if (!ret)
1971 goto rnr_nak;
1972 qp->r_rcv_len = 0;
1973 /* FALLTHROUGH */
1974 case OP(SEND_MIDDLE):
1975 case OP(RDMA_WRITE_MIDDLE):
1976send_middle:
1977 /* Check for invalid length PMTU or posted rwqe len. */
1978 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1979 goto nack_inv;
1980 qp->r_rcv_len += pmtu;
1981 if (unlikely(qp->r_rcv_len > qp->r_len))
1982 goto nack_inv;
1983 qib_copy_sge(&qp->r_sge, data, pmtu, 1);
1984 break;
1985
1986 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
1987 /* consume RWQE */
1988 ret = qib_get_rwqe(qp, 1);
1989 if (ret < 0)
1990 goto nack_op_err;
1991 if (!ret)
1992 goto rnr_nak;
1993 goto send_last_imm;
1994
1995 case OP(SEND_ONLY):
1996 case OP(SEND_ONLY_WITH_IMMEDIATE):
1997 ret = qib_get_rwqe(qp, 0);
1998 if (ret < 0)
1999 goto nack_op_err;
2000 if (!ret)
2001 goto rnr_nak;
2002 qp->r_rcv_len = 0;
2003 if (opcode == OP(SEND_ONLY))
2fc109c8
MM		 2004 goto no_immediate_data;
2005 /* FALLTHROUGH for SEND_ONLY_WITH_IMMEDIATE */
f931551b
RC		 2006 case OP(SEND_LAST_WITH_IMMEDIATE):
2007send_last_imm:
2008 wc.ex.imm_data = ohdr->u.imm_data;
2009 hdrsize += 4;
2010 wc.wc_flags = IB_WC_WITH_IMM;
2fc109c8 2011 goto send_last;
f931551b
RC		 2012 case OP(SEND_LAST):
2013 case OP(RDMA_WRITE_LAST):
2fc109c8
MM		 2014no_immediate_data:
2015 wc.wc_flags = 0;
2016 wc.ex.imm_data = 0;
f931551b
RC		 2017send_last:
2018 /* Get the number of bytes the message was padded by. */
2019 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
2020 /* Check for invalid length. */
2021 /* XXX LAST len should be >= 1 */
2022 if (unlikely(tlen < (hdrsize + pad + 4)))
2023 goto nack_inv;
2024 /* Don't count the CRC. */
2025 tlen -= (hdrsize + pad + 4);
2026 wc.byte_len = tlen + qp->r_rcv_len;
2027 if (unlikely(wc.byte_len > qp->r_len))
2028 goto nack_inv;
2029 qib_copy_sge(&qp->r_sge, data, tlen, 1);
6a82649f 2030 qib_put_ss(&qp->r_sge);
f931551b
RC		 2031 qp->r_msn++;
2032 if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags))
2033 break;
2034 wc.wr_id = qp->r_wr_id;
2035 wc.status = IB_WC_SUCCESS;
2036 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
2037 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
2038 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
2039 else
2040 wc.opcode = IB_WC_RECV;
2041 wc.qp = &qp->ibqp;
2042 wc.src_qp = qp->remote_qpn;
2043 wc.slid = qp->remote_ah_attr.dlid;
2044 wc.sl = qp->remote_ah_attr.sl;
2fc109c8
MM		 2045 /* zero fields that are N/A */
2046 wc.vendor_err = 0;
2047 wc.pkey_index = 0;
2048 wc.dlid_path_bits = 0;
2049 wc.port_num = 0;
f931551b
RC		 2050 /* Signal completion event if the solicited bit is set. */
2051 qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
2052 (ohdr->bth[0] &
2053 cpu_to_be32(IB_BTH_SOLICITED)) != 0);
2054 break;
2055
2056 case OP(RDMA_WRITE_FIRST):
2057 case OP(RDMA_WRITE_ONLY):
2058 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
2059 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
2060 goto nack_inv;
2061 /* consume RWQE */
2062 reth = &ohdr->u.rc.reth;
2063 hdrsize += sizeof(*reth);
2064 qp->r_len = be32_to_cpu(reth->length);
2065 qp->r_rcv_len = 0;
2066 qp->r_sge.sg_list = NULL;
2067 if (qp->r_len != 0) {
2068 u32 rkey = be32_to_cpu(reth->rkey);
2069 u64 vaddr = be64_to_cpu(reth->vaddr);
2070 int ok;
2071
2072 /* Check rkey & NAK */
7c2e11fe 2073 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
f931551b
RC		 2074 rkey, IB_ACCESS_REMOTE_WRITE);
2075 if (unlikely(!ok))
2076 goto nack_acc;
2077 qp->r_sge.num_sge = 1;
2078 } else {
2079 qp->r_sge.num_sge = 0;
2080 qp->r_sge.sge.mr = NULL;
2081 qp->r_sge.sge.vaddr = NULL;
2082 qp->r_sge.sge.length = 0;
2083 qp->r_sge.sge.sge_length = 0;
2084 }
2085 if (opcode == OP(RDMA_WRITE_FIRST))
2086 goto send_middle;
2087 else if (opcode == OP(RDMA_WRITE_ONLY))
2fc109c8 2088 goto no_immediate_data;
f931551b
RC		 2089 ret = qib_get_rwqe(qp, 1);
2090 if (ret < 0)
2091 goto nack_op_err;
2092 if (!ret)
2093 goto rnr_nak;
5715f5d4
JG		 2094 wc.ex.imm_data = ohdr->u.rc.imm_data;
2095 hdrsize += 4;
2096 wc.wc_flags = IB_WC_WITH_IMM;
2097 goto send_last;
f931551b
RC		 2098
2099 case OP(RDMA_READ_REQUEST): {
7c2e11fe 2100 struct rvt_ack_entry *e;
f931551b
RC		 2101 u32 len;
2102 u8 next;
2103
2104 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
2105 goto nack_inv;
2106 next = qp->r_head_ack_queue + 1;
2107 /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */
2108 if (next > QIB_MAX_RDMA_ATOMIC)
2109 next = 0;
2110 spin_lock_irqsave(&qp->s_lock, flags);
f931551b
RC		 2111 if (unlikely(next == qp->s_tail_ack_queue)) {
2112 if (!qp->s_ack_queue[next].sent)
2113 goto nack_inv_unlck;
2114 qib_update_ack_queue(qp, next);
2115 }
2116 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2117 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
7c2e11fe 2118 rvt_put_mr(e->rdma_sge.mr);
f931551b
RC		 2119 e->rdma_sge.mr = NULL;
2120 }
2121 reth = &ohdr->u.rc.reth;
2122 len = be32_to_cpu(reth->length);
2123 if (len) {
2124 u32 rkey = be32_to_cpu(reth->rkey);
2125 u64 vaddr = be64_to_cpu(reth->vaddr);
2126 int ok;
2127
2128 /* Check rkey & NAK */
7c2e11fe 2129 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
f931551b
RC		 2130 rkey, IB_ACCESS_REMOTE_READ);
2131 if (unlikely(!ok))
2132 goto nack_acc_unlck;
2133 /*
2134 * Update the next expected PSN. We add 1 later
2135 * below, so only add the remainder here.
2136 */
2137 if (len > pmtu)
2138 qp->r_psn += (len - 1) / pmtu;
2139 } else {
2140 e->rdma_sge.mr = NULL;
2141 e->rdma_sge.vaddr = NULL;
2142 e->rdma_sge.length = 0;
2143 e->rdma_sge.sge_length = 0;
2144 }
2145 e->opcode = opcode;
2146 e->sent = 0;
2147 e->psn = psn;
2148 e->lpsn = qp->r_psn;
2149 /*
2150 * We need to increment the MSN here instead of when we
2151 * finish sending the result since a duplicate request would
2152 * increment it more than once.
2153 */
2154 qp->r_msn++;
2155 qp->r_psn++;
2156 qp->r_state = opcode;
2157 qp->r_nak_state = 0;
2158 qp->r_head_ack_queue = next;
2159
2160 /* Schedule the send tasklet. */
2161 qp->s_flags |= QIB_S_RESP_PENDING;
2162 qib_schedule_send(qp);
2163
a5210c12 2164 goto sunlock;
f931551b
RC		 2165 }
2166
2167 case OP(COMPARE_SWAP):
2168 case OP(FETCH_ADD): {
2169 struct ib_atomic_eth *ateth;
7c2e11fe 2170 struct rvt_ack_entry *e;
f931551b
RC		 2171 u64 vaddr;
2172 atomic64_t *maddr;
2173 u64 sdata;
2174 u32 rkey;
2175 u8 next;
2176
2177 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
2178 goto nack_inv;
2179 next = qp->r_head_ack_queue + 1;
2180 if (next > QIB_MAX_RDMA_ATOMIC)
2181 next = 0;
2182 spin_lock_irqsave(&qp->s_lock, flags);
f931551b
RC		 2183 if (unlikely(next == qp->s_tail_ack_queue)) {
2184 if (!qp->s_ack_queue[next].sent)
2185 goto nack_inv_unlck;
2186 qib_update_ack_queue(qp, next);
2187 }
2188 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2189 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
7c2e11fe 2190 rvt_put_mr(e->rdma_sge.mr);
f931551b
RC		 2191 e->rdma_sge.mr = NULL;
2192 }
2193 ateth = &ohdr->u.atomic_eth;
2194 vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) |
2195 be32_to_cpu(ateth->vaddr[1]);
2196 if (unlikely(vaddr & (sizeof(u64) - 1)))
2197 goto nack_inv_unlck;
2198 rkey = be32_to_cpu(ateth->rkey);
2199 /* Check rkey & NAK */
7c2e11fe 2200 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
f931551b
RC		 2201 vaddr, rkey,
2202 IB_ACCESS_REMOTE_ATOMIC)))
2203 goto nack_acc_unlck;
2204 /* Perform atomic OP and save result. */
2205 maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
2206 sdata = be64_to_cpu(ateth->swap_data);
2207 e->atomic_data = (opcode == OP(FETCH_ADD)) ?
2208 (u64) atomic64_add_return(sdata, maddr) - sdata :
2209 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
2210 be64_to_cpu(ateth->compare_data),
2211 sdata);
7c2e11fe 2212 rvt_put_mr(qp->r_sge.sge.mr);
f931551b
RC		 2213 qp->r_sge.num_sge = 0;
2214 e->opcode = opcode;
2215 e->sent = 0;
2216 e->psn = psn;
2217 e->lpsn = psn;
2218 qp->r_msn++;
2219 qp->r_psn++;
2220 qp->r_state = opcode;
2221 qp->r_nak_state = 0;
2222 qp->r_head_ack_queue = next;
2223
2224 /* Schedule the send tasklet. */
2225 qp->s_flags |= QIB_S_RESP_PENDING;
2226 qib_schedule_send(qp);
2227
a5210c12 2228 goto sunlock;
f931551b
RC		 2229 }
2230
2231 default:
2232 /* NAK unknown opcodes. */
2233 goto nack_inv;
2234 }
2235 qp->r_psn++;
2236 qp->r_state = opcode;
2237 qp->r_ack_psn = psn;
2238 qp->r_nak_state = 0;
2239 /* Send an ACK if requested or required. */
2240 if (psn & (1 << 31))
2241 goto send_ack;
a5210c12 2242 return;
f931551b
RC		 2243
2244rnr_nak:
2245 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
2246 qp->r_ack_psn = qp->r_psn;
2247 /* Queue RNR NAK for later */
2248 if (list_empty(&qp->rspwait)) {
2249 qp->r_flags |= QIB_R_RSP_NAK;
2250 atomic_inc(&qp->refcount);
2251 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2252 }
a5210c12 2253 return;
f931551b
RC		 2254
2255nack_op_err:
2256 qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2257 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
2258 qp->r_ack_psn = qp->r_psn;
2259 /* Queue NAK for later */
2260 if (list_empty(&qp->rspwait)) {
2261 qp->r_flags |= QIB_R_RSP_NAK;
2262 atomic_inc(&qp->refcount);
2263 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2264 }
a5210c12 2265 return;
f931551b
RC		 2266
2267nack_inv_unlck:
2268 spin_unlock_irqrestore(&qp->s_lock, flags);
2269nack_inv:
2270 qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2271 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
2272 qp->r_ack_psn = qp->r_psn;
2273 /* Queue NAK for later */
2274 if (list_empty(&qp->rspwait)) {
2275 qp->r_flags |= QIB_R_RSP_NAK;
2276 atomic_inc(&qp->refcount);
2277 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2278 }
a5210c12 2279 return;
f931551b
RC		 2280
2281nack_acc_unlck:
2282 spin_unlock_irqrestore(&qp->s_lock, flags);
2283nack_acc:
2284 qib_rc_error(qp, IB_WC_LOC_PROT_ERR);
2285 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
2286 qp->r_ack_psn = qp->r_psn;
2287send_ack:
2288 qib_send_rc_ack(qp);
f931551b
RC		 2289 return;
2290
2291sunlock:
2292 spin_unlock_irqrestore(&qp->s_lock, flags);
2293}
Linux kernel - Deliverables
This page took 0.597162 seconds and 5 git commands to generate.