2 * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <rdma/ib_smi.h>
35 #include <rdma/ib_verbs.h>
41 * qib_ud_loopback - handle send on loopback QPs
42 * @sqp: the sending QP
43 * @swqe: the send work request
45 * This is called from qib_make_ud_req() to forward a WQE addressed
47 * Note that the receive interrupt handler may be calling qib_ud_rcv()
48 * while this is being called.
50 static void qib_ud_loopback(struct rvt_qp
*sqp
, struct rvt_swqe
*swqe
)
52 struct qib_ibport
*ibp
= to_iport(sqp
->ibqp
.device
, sqp
->port_num
);
53 struct qib_pportdata
*ppd
;
55 struct ib_ah_attr
*ah_attr
;
57 struct rvt_sge_state ssge
;
61 enum ib_qp_type sqptype
, dqptype
;
63 qp
= qib_lookup_qpn(ibp
, swqe
->ud_wr
.remote_qpn
);
65 ibp
->rvp
.n_pkt_drops
++;
69 sqptype
= sqp
->ibqp
.qp_type
== IB_QPT_GSI
?
70 IB_QPT_UD
: sqp
->ibqp
.qp_type
;
71 dqptype
= qp
->ibqp
.qp_type
== IB_QPT_GSI
?
72 IB_QPT_UD
: qp
->ibqp
.qp_type
;
74 if (dqptype
!= sqptype
||
75 !(ib_qib_state_ops
[qp
->state
] & QIB_PROCESS_RECV_OK
)) {
76 ibp
->rvp
.n_pkt_drops
++;
80 ah_attr
= &ibah_to_rvtah(swqe
->ud_wr
.ah
)->attr
;
81 ppd
= ppd_from_ibp(ibp
);
83 if (qp
->ibqp
.qp_num
> 1) {
88 pkey1
= qib_get_pkey(ibp
, sqp
->s_pkey_index
);
89 pkey2
= qib_get_pkey(ibp
, qp
->s_pkey_index
);
90 if (unlikely(!qib_pkey_ok(pkey1
, pkey2
))) {
91 lid
= ppd
->lid
| (ah_attr
->src_path_bits
&
92 ((1 << ppd
->lmc
) - 1));
93 qib_bad_pqkey(ibp
, IB_NOTICE_TRAP_BAD_PKEY
, pkey1
,
95 sqp
->ibqp
.qp_num
, qp
->ibqp
.qp_num
,
97 cpu_to_be16(ah_attr
->dlid
));
103 * Check that the qkey matches (except for QP0, see 9.6.1.4.1).
104 * Qkeys with the high order bit set mean use the
105 * qkey from the QP context instead of the WR (see 10.2.5).
107 if (qp
->ibqp
.qp_num
) {
110 qkey
= (int)swqe
->ud_wr
.remote_qkey
< 0 ?
111 sqp
->qkey
: swqe
->ud_wr
.remote_qkey
;
112 if (unlikely(qkey
!= qp
->qkey
)) {
115 lid
= ppd
->lid
| (ah_attr
->src_path_bits
&
116 ((1 << ppd
->lmc
) - 1));
117 qib_bad_pqkey(ibp
, IB_NOTICE_TRAP_BAD_QKEY
, qkey
,
119 sqp
->ibqp
.qp_num
, qp
->ibqp
.qp_num
,
121 cpu_to_be16(ah_attr
->dlid
));
127 * A GRH is expected to precede the data even if not
128 * present on the wire.
130 length
= swqe
->length
;
131 memset(&wc
, 0, sizeof(wc
));
132 wc
.byte_len
= length
+ sizeof(struct ib_grh
);
134 if (swqe
->wr
.opcode
== IB_WR_SEND_WITH_IMM
) {
135 wc
.wc_flags
= IB_WC_WITH_IMM
;
136 wc
.ex
.imm_data
= swqe
->wr
.ex
.imm_data
;
139 spin_lock_irqsave(&qp
->r_lock
, flags
);
142 * Get the next work request entry to find where to put the data.
144 if (qp
->r_flags
& QIB_R_REUSE_SGE
)
145 qp
->r_flags
&= ~QIB_R_REUSE_SGE
;
149 ret
= qib_get_rwqe(qp
, 0);
151 qib_rc_error(qp
, IB_WC_LOC_QP_OP_ERR
);
155 if (qp
->ibqp
.qp_num
== 0)
156 ibp
->rvp
.n_vl15_dropped
++;
160 /* Silently drop packets which are too big. */
161 if (unlikely(wc
.byte_len
> qp
->r_len
)) {
162 qp
->r_flags
|= QIB_R_REUSE_SGE
;
163 ibp
->rvp
.n_pkt_drops
++;
167 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
168 qib_copy_sge(&qp
->r_sge
, &ah_attr
->grh
,
169 sizeof(struct ib_grh
), 1);
170 wc
.wc_flags
|= IB_WC_GRH
;
172 qib_skip_sge(&qp
->r_sge
, sizeof(struct ib_grh
), 1);
173 ssge
.sg_list
= swqe
->sg_list
+ 1;
174 ssge
.sge
= *swqe
->sg_list
;
175 ssge
.num_sge
= swqe
->wr
.num_sge
;
178 u32 len
= sge
->length
;
182 if (len
> sge
->sge_length
)
183 len
= sge
->sge_length
;
185 qib_copy_sge(&qp
->r_sge
, sge
->vaddr
, len
, 1);
188 sge
->sge_length
-= len
;
189 if (sge
->sge_length
== 0) {
191 *sge
= *ssge
.sg_list
++;
192 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
193 if (++sge
->n
>= RVT_SEGSZ
) {
194 if (++sge
->m
>= sge
->mr
->mapsz
)
199 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
201 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
205 qib_put_ss(&qp
->r_sge
);
206 if (!test_and_clear_bit(QIB_R_WRID_VALID
, &qp
->r_aflags
))
208 wc
.wr_id
= qp
->r_wr_id
;
209 wc
.status
= IB_WC_SUCCESS
;
210 wc
.opcode
= IB_WC_RECV
;
212 wc
.src_qp
= sqp
->ibqp
.qp_num
;
213 wc
.pkey_index
= qp
->ibqp
.qp_type
== IB_QPT_GSI
?
214 swqe
->ud_wr
.pkey_index
: 0;
215 wc
.slid
= ppd
->lid
| (ah_attr
->src_path_bits
& ((1 << ppd
->lmc
) - 1));
217 wc
.dlid_path_bits
= ah_attr
->dlid
& ((1 << ppd
->lmc
) - 1);
218 wc
.port_num
= qp
->port_num
;
219 /* Signal completion event if the solicited bit is set. */
220 qib_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
,
221 swqe
->wr
.send_flags
& IB_SEND_SOLICITED
);
222 ibp
->rvp
.n_loop_pkts
++;
224 spin_unlock_irqrestore(&qp
->r_lock
, flags
);
226 if (atomic_dec_and_test(&qp
->refcount
))
231 * qib_make_ud_req - construct a UD request packet
234 * Return 1 if constructed; otherwise, return 0.
236 int qib_make_ud_req(struct rvt_qp
*qp
)
238 struct qib_qp_priv
*priv
= qp
->priv
;
239 struct qib_other_headers
*ohdr
;
240 struct ib_ah_attr
*ah_attr
;
241 struct qib_pportdata
*ppd
;
242 struct qib_ibport
*ibp
;
243 struct rvt_swqe
*wqe
;
253 spin_lock_irqsave(&qp
->s_lock
, flags
);
255 if (!(ib_qib_state_ops
[qp
->state
] & QIB_PROCESS_NEXT_SEND_OK
)) {
256 if (!(ib_qib_state_ops
[qp
->state
] & QIB_FLUSH_SEND
))
258 /* We are in the error state, flush the work request. */
259 if (qp
->s_last
== qp
->s_head
)
261 /* If DMAs are in progress, we can't flush immediately. */
262 if (atomic_read(&priv
->s_dma_busy
)) {
263 qp
->s_flags
|= QIB_S_WAIT_DMA
;
266 wqe
= get_swqe_ptr(qp
, qp
->s_last
);
267 qib_send_complete(qp
, wqe
, IB_WC_WR_FLUSH_ERR
);
271 if (qp
->s_cur
== qp
->s_head
)
274 wqe
= get_swqe_ptr(qp
, qp
->s_cur
);
275 next_cur
= qp
->s_cur
+ 1;
276 if (next_cur
>= qp
->s_size
)
279 /* Construct the header. */
280 ibp
= to_iport(qp
->ibqp
.device
, qp
->port_num
);
281 ppd
= ppd_from_ibp(ibp
);
282 ah_attr
= &ibah_to_rvtah(wqe
->ud_wr
.ah
)->attr
;
283 if (ah_attr
->dlid
>= be16_to_cpu(IB_MULTICAST_LID_BASE
)) {
284 if (ah_attr
->dlid
!= be16_to_cpu(IB_LID_PERMISSIVE
))
285 this_cpu_inc(ibp
->pmastats
->n_multicast_xmit
);
287 this_cpu_inc(ibp
->pmastats
->n_unicast_xmit
);
289 this_cpu_inc(ibp
->pmastats
->n_unicast_xmit
);
290 lid
= ah_attr
->dlid
& ~((1 << ppd
->lmc
) - 1);
291 if (unlikely(lid
== ppd
->lid
)) {
293 * If DMAs are in progress, we can't generate
294 * a completion for the loopback packet since
295 * it would be out of order.
296 * XXX Instead of waiting, we could queue a
297 * zero length descriptor so we get a callback.
299 if (atomic_read(&priv
->s_dma_busy
)) {
300 qp
->s_flags
|= QIB_S_WAIT_DMA
;
303 qp
->s_cur
= next_cur
;
304 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
305 qib_ud_loopback(qp
, wqe
);
306 spin_lock_irqsave(&qp
->s_lock
, flags
);
307 qib_send_complete(qp
, wqe
, IB_WC_SUCCESS
);
312 qp
->s_cur
= next_cur
;
313 extra_bytes
= -wqe
->length
& 3;
314 nwords
= (wqe
->length
+ extra_bytes
) >> 2;
316 /* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
318 qp
->s_cur_size
= wqe
->length
;
319 qp
->s_cur_sge
= &qp
->s_sge
;
320 qp
->s_srate
= ah_attr
->static_rate
;
322 qp
->s_sge
.sge
= wqe
->sg_list
[0];
323 qp
->s_sge
.sg_list
= wqe
->sg_list
+ 1;
324 qp
->s_sge
.num_sge
= wqe
->wr
.num_sge
;
325 qp
->s_sge
.total_len
= wqe
->length
;
327 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
328 /* Header size in 32-bit words. */
329 qp
->s_hdrwords
+= qib_make_grh(ibp
, &priv
->s_hdr
->u
.l
.grh
,
331 qp
->s_hdrwords
, nwords
);
333 ohdr
= &priv
->s_hdr
->u
.l
.oth
;
335 * Don't worry about sending to locally attached multicast
336 * QPs. It is unspecified by the spec. what happens.
339 /* Header size in 32-bit words. */
341 ohdr
= &priv
->s_hdr
->u
.oth
;
343 if (wqe
->wr
.opcode
== IB_WR_SEND_WITH_IMM
) {
345 ohdr
->u
.ud
.imm_data
= wqe
->wr
.ex
.imm_data
;
346 bth0
= IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
<< 24;
348 bth0
= IB_OPCODE_UD_SEND_ONLY
<< 24;
349 lrh0
|= ah_attr
->sl
<< 4;
350 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
)
351 lrh0
|= 0xF000; /* Set VL (see ch. 13.5.3.1) */
353 lrh0
|= ibp
->sl_to_vl
[ah_attr
->sl
] << 12;
354 priv
->s_hdr
->lrh
[0] = cpu_to_be16(lrh0
);
355 priv
->s_hdr
->lrh
[1] = cpu_to_be16(ah_attr
->dlid
); /* DEST LID */
356 priv
->s_hdr
->lrh
[2] =
357 cpu_to_be16(qp
->s_hdrwords
+ nwords
+ SIZE_OF_CRC
);
360 lid
|= ah_attr
->src_path_bits
& ((1 << ppd
->lmc
) - 1);
361 priv
->s_hdr
->lrh
[3] = cpu_to_be16(lid
);
363 priv
->s_hdr
->lrh
[3] = IB_LID_PERMISSIVE
;
364 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
365 bth0
|= IB_BTH_SOLICITED
;
366 bth0
|= extra_bytes
<< 20;
367 bth0
|= qp
->ibqp
.qp_type
== IB_QPT_SMI
? QIB_DEFAULT_P_KEY
:
368 qib_get_pkey(ibp
, qp
->ibqp
.qp_type
== IB_QPT_GSI
?
369 wqe
->ud_wr
.pkey_index
: qp
->s_pkey_index
);
370 ohdr
->bth
[0] = cpu_to_be32(bth0
);
372 * Use the multicast QP if the destination LID is a multicast LID.
374 ohdr
->bth
[1] = ah_attr
->dlid
>= be16_to_cpu(IB_MULTICAST_LID_BASE
) &&
375 ah_attr
->dlid
!= be16_to_cpu(IB_LID_PERMISSIVE
) ?
376 cpu_to_be32(QIB_MULTICAST_QPN
) :
377 cpu_to_be32(wqe
->ud_wr
.remote_qpn
);
378 ohdr
->bth
[2] = cpu_to_be32(qp
->s_next_psn
++ & QIB_PSN_MASK
);
380 * Qkeys with the high order bit set mean use the
381 * qkey from the QP context instead of the WR (see 10.2.5).
383 ohdr
->u
.ud
.deth
[0] = cpu_to_be32((int)wqe
->ud_wr
.remote_qkey
< 0 ?
384 qp
->qkey
: wqe
->ud_wr
.remote_qkey
);
385 ohdr
->u
.ud
.deth
[1] = cpu_to_be32(qp
->ibqp
.qp_num
);
392 qp
->s_flags
&= ~QIB_S_BUSY
;
394 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
398 static unsigned qib_lookup_pkey(struct qib_ibport
*ibp
, u16 pkey
)
400 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
401 struct qib_devdata
*dd
= ppd
->dd
;
402 unsigned ctxt
= ppd
->hw_pidx
;
405 pkey
&= 0x7fff; /* remove limited/full membership bit */
407 for (i
= 0; i
< ARRAY_SIZE(dd
->rcd
[ctxt
]->pkeys
); ++i
)
408 if ((dd
->rcd
[ctxt
]->pkeys
[i
] & 0x7fff) == pkey
)
412 * Should not get here, this means hardware failed to validate pkeys.
413 * Punt and return index 0.
419 * qib_ud_rcv - receive an incoming UD packet
420 * @ibp: the port the packet came in on
421 * @hdr: the packet header
422 * @has_grh: true if the packet has a GRH
423 * @data: the packet data
424 * @tlen: the packet length
425 * @qp: the QP the packet came on
427 * This is called from qib_qp_rcv() to process an incoming UD packet
429 * Called at interrupt level.
431 void qib_ud_rcv(struct qib_ibport
*ibp
, struct qib_ib_header
*hdr
,
432 int has_grh
, void *data
, u32 tlen
, struct rvt_qp
*qp
)
434 struct qib_other_headers
*ohdr
;
446 hdrsize
= 8 + 12 + 8; /* LRH + BTH + DETH */
448 ohdr
= &hdr
->u
.l
.oth
;
449 hdrsize
= 8 + 40 + 12 + 8; /* LRH + GRH + BTH + DETH */
451 qkey
= be32_to_cpu(ohdr
->u
.ud
.deth
[0]);
452 src_qp
= be32_to_cpu(ohdr
->u
.ud
.deth
[1]) & QIB_QPN_MASK
;
455 * Get the number of bytes the message was padded by
456 * and drop incomplete packets.
458 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
459 if (unlikely(tlen
< (hdrsize
+ pad
+ 4)))
462 tlen
-= hdrsize
+ pad
+ 4;
465 * Check that the permissive LID is only used on QP0
466 * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
468 if (qp
->ibqp
.qp_num
) {
469 if (unlikely(hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
470 hdr
->lrh
[3] == IB_LID_PERMISSIVE
))
472 if (qp
->ibqp
.qp_num
> 1) {
475 pkey1
= be32_to_cpu(ohdr
->bth
[0]);
476 pkey2
= qib_get_pkey(ibp
, qp
->s_pkey_index
);
477 if (unlikely(!qib_pkey_ok(pkey1
, pkey2
))) {
478 qib_bad_pqkey(ibp
, IB_NOTICE_TRAP_BAD_PKEY
,
480 (be16_to_cpu(hdr
->lrh
[0]) >> 4) &
482 src_qp
, qp
->ibqp
.qp_num
,
483 hdr
->lrh
[3], hdr
->lrh
[1]);
487 if (unlikely(qkey
!= qp
->qkey
)) {
488 qib_bad_pqkey(ibp
, IB_NOTICE_TRAP_BAD_QKEY
, qkey
,
489 (be16_to_cpu(hdr
->lrh
[0]) >> 4) & 0xF,
490 src_qp
, qp
->ibqp
.qp_num
,
491 hdr
->lrh
[3], hdr
->lrh
[1]);
494 /* Drop invalid MAD packets (see 13.5.3.1). */
495 if (unlikely(qp
->ibqp
.qp_num
== 1 &&
497 (be16_to_cpu(hdr
->lrh
[0]) >> 12) == 15)))
502 /* Drop invalid MAD packets (see 13.5.3.1). */
503 if (tlen
!= 256 || (be16_to_cpu(hdr
->lrh
[0]) >> 12) != 15)
505 smp
= (struct ib_smp
*) data
;
506 if ((hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
507 hdr
->lrh
[3] == IB_LID_PERMISSIVE
) &&
508 smp
->mgmt_class
!= IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
)
513 * The opcode is in the low byte when its in network order
514 * (top byte when in host order).
516 opcode
= be32_to_cpu(ohdr
->bth
[0]) >> 24;
517 if (qp
->ibqp
.qp_num
> 1 &&
518 opcode
== IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
) {
519 wc
.ex
.imm_data
= ohdr
->u
.ud
.imm_data
;
520 wc
.wc_flags
= IB_WC_WITH_IMM
;
522 } else if (opcode
== IB_OPCODE_UD_SEND_ONLY
) {
529 * A GRH is expected to precede the data even if not
530 * present on the wire.
532 wc
.byte_len
= tlen
+ sizeof(struct ib_grh
);
535 * Get the next work request entry to find where to put the data.
537 if (qp
->r_flags
& QIB_R_REUSE_SGE
)
538 qp
->r_flags
&= ~QIB_R_REUSE_SGE
;
542 ret
= qib_get_rwqe(qp
, 0);
544 qib_rc_error(qp
, IB_WC_LOC_QP_OP_ERR
);
548 if (qp
->ibqp
.qp_num
== 0)
549 ibp
->rvp
.n_vl15_dropped
++;
553 /* Silently drop packets which are too big. */
554 if (unlikely(wc
.byte_len
> qp
->r_len
)) {
555 qp
->r_flags
|= QIB_R_REUSE_SGE
;
559 qib_copy_sge(&qp
->r_sge
, &hdr
->u
.l
.grh
,
560 sizeof(struct ib_grh
), 1);
561 wc
.wc_flags
|= IB_WC_GRH
;
563 qib_skip_sge(&qp
->r_sge
, sizeof(struct ib_grh
), 1);
564 qib_copy_sge(&qp
->r_sge
, data
, wc
.byte_len
- sizeof(struct ib_grh
), 1);
565 qib_put_ss(&qp
->r_sge
);
566 if (!test_and_clear_bit(QIB_R_WRID_VALID
, &qp
->r_aflags
))
568 wc
.wr_id
= qp
->r_wr_id
;
569 wc
.status
= IB_WC_SUCCESS
;
570 wc
.opcode
= IB_WC_RECV
;
574 wc
.pkey_index
= qp
->ibqp
.qp_type
== IB_QPT_GSI
?
575 qib_lookup_pkey(ibp
, be32_to_cpu(ohdr
->bth
[0])) : 0;
576 wc
.slid
= be16_to_cpu(hdr
->lrh
[3]);
577 wc
.sl
= (be16_to_cpu(hdr
->lrh
[0]) >> 4) & 0xF;
578 dlid
= be16_to_cpu(hdr
->lrh
[1]);
580 * Save the LMC lower bits if the destination LID is a unicast LID.
582 wc
.dlid_path_bits
= dlid
>= be16_to_cpu(IB_MULTICAST_LID_BASE
) ? 0 :
583 dlid
& ((1 << ppd_from_ibp(ibp
)->lmc
) - 1);
584 wc
.port_num
= qp
->port_num
;
585 /* Signal completion event if the solicited bit is set. */
586 qib_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
,
588 cpu_to_be32(IB_BTH_SOLICITED
)) != 0);
592 ibp
->rvp
.n_pkt_drops
++;