2 * Copyright(c) 2015, 2016 Intel Corporation.
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 #include <linux/err.h>
49 #include <linux/vmalloc.h>
50 #include <linux/hash.h>
51 #include <linux/module.h>
52 #include <linux/random.h>
53 #include <linux/seq_file.h>
54 #include <rdma/rdma_vt.h>
55 #include <rdma/rdmavt_qp.h>
60 #include "verbs_txreq.h"
62 unsigned int hfi1_qp_table_size
= 256;
63 module_param_named(qp_table_size
, hfi1_qp_table_size
, uint
, S_IRUGO
);
64 MODULE_PARM_DESC(qp_table_size
, "QP table size");
66 static void flush_tx_list(struct rvt_qp
*qp
);
67 static int iowait_sleep(
68 struct sdma_engine
*sde
,
70 struct sdma_txreq
*stx
,
72 static void iowait_wakeup(struct iowait
*wait
, int reason
);
73 static void iowait_sdma_drained(struct iowait
*wait
);
74 static void qp_pio_drain(struct rvt_qp
*qp
);
76 static inline unsigned mk_qpn(struct rvt_qpn_table
*qpt
,
77 struct rvt_qpn_map
*map
, unsigned off
)
79 return (map
- qpt
->map
) * RVT_BITS_PER_PAGE
+ off
;
83 * Convert the AETH credit code into the number of credits.
85 static const u16 credit_table
[31] = {
119 static void flush_tx_list(struct rvt_qp
*qp
)
121 struct hfi1_qp_priv
*priv
= qp
->priv
;
123 while (!list_empty(&priv
->s_iowait
.tx_head
)) {
124 struct sdma_txreq
*tx
;
126 tx
= list_first_entry(
127 &priv
->s_iowait
.tx_head
,
130 list_del_init(&tx
->list
);
132 container_of(tx
, struct verbs_txreq
, txreq
));
136 static void flush_iowait(struct rvt_qp
*qp
)
138 struct hfi1_qp_priv
*priv
= qp
->priv
;
139 struct hfi1_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
142 write_seqlock_irqsave(&dev
->iowait_lock
, flags
);
143 if (!list_empty(&priv
->s_iowait
.list
)) {
144 list_del_init(&priv
->s_iowait
.list
);
145 if (atomic_dec_and_test(&qp
->refcount
))
148 write_sequnlock_irqrestore(&dev
->iowait_lock
, flags
);
151 static inline int opa_mtu_enum_to_int(int mtu
)
154 case OPA_MTU_8192
: return 8192;
155 case OPA_MTU_10240
: return 10240;
161 * This function is what we would push to the core layer if we wanted to be a
162 * "first class citizen". Instead we hide this here and rely on Verbs ULPs
163 * to blindly pass the MTU enum value from the PathRecord to us.
165 * The actual flag used to determine "8k MTU" will change and is currently
168 static inline int verbs_mtu_enum_to_int(struct ib_device
*dev
, enum ib_mtu mtu
)
170 int val
= opa_mtu_enum_to_int((int)mtu
);
174 return ib_mtu_enum_to_int(mtu
);
177 int hfi1_check_modify_qp(struct rvt_qp
*qp
, struct ib_qp_attr
*attr
,
178 int attr_mask
, struct ib_udata
*udata
)
180 struct ib_qp
*ibqp
= &qp
->ibqp
;
181 struct hfi1_ibdev
*dev
= to_idev(ibqp
->device
);
182 struct hfi1_devdata
*dd
= dd_from_dev(dev
);
185 if (attr_mask
& IB_QP_AV
) {
186 sc
= ah_to_sc(ibqp
->device
, &attr
->ah_attr
);
190 if (!qp_to_sdma_engine(qp
, sc
) &&
191 dd
->flags
& HFI1_HAS_SEND_DMA
)
194 if (!qp_to_send_context(qp
, sc
))
198 if (attr_mask
& IB_QP_ALT_PATH
) {
199 sc
= ah_to_sc(ibqp
->device
, &attr
->alt_ah_attr
);
203 if (!qp_to_sdma_engine(qp
, sc
) &&
204 dd
->flags
& HFI1_HAS_SEND_DMA
)
207 if (!qp_to_send_context(qp
, sc
))
214 void hfi1_modify_qp(struct rvt_qp
*qp
, struct ib_qp_attr
*attr
,
215 int attr_mask
, struct ib_udata
*udata
)
217 struct ib_qp
*ibqp
= &qp
->ibqp
;
218 struct hfi1_qp_priv
*priv
= qp
->priv
;
220 if (attr_mask
& IB_QP_AV
) {
221 priv
->s_sc
= ah_to_sc(ibqp
->device
, &qp
->remote_ah_attr
);
222 priv
->s_sde
= qp_to_sdma_engine(qp
, priv
->s_sc
);
223 priv
->s_sendcontext
= qp_to_send_context(qp
, priv
->s_sc
);
226 if (attr_mask
& IB_QP_PATH_MIG_STATE
&&
227 attr
->path_mig_state
== IB_MIG_MIGRATED
&&
228 qp
->s_mig_state
== IB_MIG_ARMED
) {
229 qp
->s_flags
|= RVT_S_AHG_CLEAR
;
230 priv
->s_sc
= ah_to_sc(ibqp
->device
, &qp
->remote_ah_attr
);
231 priv
->s_sde
= qp_to_sdma_engine(qp
, priv
->s_sc
);
232 priv
->s_sendcontext
= qp_to_send_context(qp
, priv
->s_sc
);
237 * hfi1_check_send_wqe - validate wqe
239 * @wqe - The built wqe
241 * validate wqe. This is called
242 * prior to inserting the wqe into
243 * the ring but after the wqe has been
246 * Returns 0 on success, -EINVAL on failure
249 int hfi1_check_send_wqe(struct rvt_qp
*qp
,
250 struct rvt_swqe
*wqe
)
252 struct hfi1_ibport
*ibp
= to_iport(qp
->ibqp
.device
, qp
->port_num
);
255 switch (qp
->ibqp
.qp_type
) {
258 if (wqe
->length
> 0x80000000U
)
262 ah
= ibah_to_rvtah(wqe
->ud_wr
.ah
);
263 if (wqe
->length
> (1 << ah
->log_pmtu
))
268 ah
= ibah_to_rvtah(wqe
->ud_wr
.ah
);
269 if (wqe
->length
> (1 << ah
->log_pmtu
))
271 if (ibp
->sl_to_sc
[ah
->attr
.sl
] == 0xf)
276 return wqe
->length
<= piothreshold
;
280 * hfi1_compute_aeth - compute the AETH (syndrome + MSN)
281 * @qp: the queue pair to compute the AETH for
285 __be32
hfi1_compute_aeth(struct rvt_qp
*qp
)
287 u32 aeth
= qp
->r_msn
& HFI1_MSN_MASK
;
291 * Shared receive queues don't generate credits.
292 * Set the credit field to the invalid value.
294 aeth
|= HFI1_AETH_CREDIT_INVAL
<< HFI1_AETH_CREDIT_SHIFT
;
298 struct rvt_rwq
*wq
= qp
->r_rq
.wq
;
302 /* sanity check pointers before trusting them */
304 if (head
>= qp
->r_rq
.size
)
307 if (tail
>= qp
->r_rq
.size
)
310 * Compute the number of credits available (RWQEs).
311 * There is a small chance that the pair of reads are
312 * not atomic, which is OK, since the fuzziness is
313 * resolved as further ACKs go out.
315 credits
= head
- tail
;
316 if ((int)credits
< 0)
317 credits
+= qp
->r_rq
.size
;
319 * Binary search the credit table to find the code to
326 if (credit_table
[x
] == credits
)
328 if (credit_table
[x
] > credits
) {
336 aeth
|= x
<< HFI1_AETH_CREDIT_SHIFT
;
338 return cpu_to_be32(aeth
);
342 * _hfi1_schedule_send - schedule progress
345 * This schedules qp progress w/o regard to the s_flags.
347 * It is only used in the post send, which doesn't hold
350 void _hfi1_schedule_send(struct rvt_qp
*qp
)
352 struct hfi1_qp_priv
*priv
= qp
->priv
;
353 struct hfi1_ibport
*ibp
=
354 to_iport(qp
->ibqp
.device
, qp
->port_num
);
355 struct hfi1_pportdata
*ppd
= ppd_from_ibp(ibp
);
356 struct hfi1_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
358 iowait_schedule(&priv
->s_iowait
, ppd
->hfi1_wq
,
361 cpumask_first(cpumask_of_node(dd
->node
)));
364 static void qp_pio_drain(struct rvt_qp
*qp
)
366 struct hfi1_ibdev
*dev
;
367 struct hfi1_qp_priv
*priv
= qp
->priv
;
369 if (!priv
->s_sendcontext
)
371 dev
= to_idev(qp
->ibqp
.device
);
372 while (iowait_pio_pending(&priv
->s_iowait
)) {
373 write_seqlock_irq(&dev
->iowait_lock
);
374 hfi1_sc_wantpiobuf_intr(priv
->s_sendcontext
, 1);
375 write_sequnlock_irq(&dev
->iowait_lock
);
376 iowait_pio_drain(&priv
->s_iowait
);
377 write_seqlock_irq(&dev
->iowait_lock
);
378 hfi1_sc_wantpiobuf_intr(priv
->s_sendcontext
, 0);
379 write_sequnlock_irq(&dev
->iowait_lock
);
384 * hfi1_schedule_send - schedule progress
387 * This schedules qp progress and caller should hold
390 void hfi1_schedule_send(struct rvt_qp
*qp
)
392 if (hfi1_send_ok(qp
))
393 _hfi1_schedule_send(qp
);
397 * hfi1_get_credit - flush the send work queue of a QP
398 * @qp: the qp who's send work queue to flush
399 * @aeth: the Acknowledge Extended Transport Header
401 * The QP s_lock should be held.
403 void hfi1_get_credit(struct rvt_qp
*qp
, u32 aeth
)
405 u32 credit
= (aeth
>> HFI1_AETH_CREDIT_SHIFT
) & HFI1_AETH_CREDIT_MASK
;
408 * If the credit is invalid, we can send
409 * as many packets as we like. Otherwise, we have to
410 * honor the credit field.
412 if (credit
== HFI1_AETH_CREDIT_INVAL
) {
413 if (!(qp
->s_flags
& RVT_S_UNLIMITED_CREDIT
)) {
414 qp
->s_flags
|= RVT_S_UNLIMITED_CREDIT
;
415 if (qp
->s_flags
& RVT_S_WAIT_SSN_CREDIT
) {
416 qp
->s_flags
&= ~RVT_S_WAIT_SSN_CREDIT
;
417 hfi1_schedule_send(qp
);
420 } else if (!(qp
->s_flags
& RVT_S_UNLIMITED_CREDIT
)) {
421 /* Compute new LSN (i.e., MSN + credit) */
422 credit
= (aeth
+ credit_table
[credit
]) & HFI1_MSN_MASK
;
423 if (cmp_msn(credit
, qp
->s_lsn
) > 0) {
425 if (qp
->s_flags
& RVT_S_WAIT_SSN_CREDIT
) {
426 qp
->s_flags
&= ~RVT_S_WAIT_SSN_CREDIT
;
427 hfi1_schedule_send(qp
);
433 void hfi1_qp_wakeup(struct rvt_qp
*qp
, u32 flag
)
437 spin_lock_irqsave(&qp
->s_lock
, flags
);
438 if (qp
->s_flags
& flag
) {
439 qp
->s_flags
&= ~flag
;
440 trace_hfi1_qpwakeup(qp
, flag
);
441 hfi1_schedule_send(qp
);
443 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
444 /* Notify hfi1_destroy_qp() if it is waiting. */
445 if (atomic_dec_and_test(&qp
->refcount
))
449 static int iowait_sleep(
450 struct sdma_engine
*sde
,
452 struct sdma_txreq
*stx
,
455 struct verbs_txreq
*tx
= container_of(stx
, struct verbs_txreq
, txreq
);
457 struct hfi1_qp_priv
*priv
;
460 struct hfi1_ibdev
*dev
;
465 spin_lock_irqsave(&qp
->s_lock
, flags
);
466 if (ib_rvt_state_ops
[qp
->state
] & RVT_PROCESS_RECV_OK
) {
468 * If we couldn't queue the DMA request, save the info
469 * and try again later rather than destroying the
470 * buffer and undoing the side effects of the copy.
472 /* Make a common routine? */
473 dev
= &sde
->dd
->verbs_dev
;
474 list_add_tail(&stx
->list
, &wait
->tx_head
);
475 write_seqlock(&dev
->iowait_lock
);
476 if (sdma_progress(sde
, seq
, stx
))
478 if (list_empty(&priv
->s_iowait
.list
)) {
479 struct hfi1_ibport
*ibp
=
480 to_iport(qp
->ibqp
.device
, qp
->port_num
);
482 ibp
->rvp
.n_dmawait
++;
483 qp
->s_flags
|= RVT_S_WAIT_DMA_DESC
;
484 list_add_tail(&priv
->s_iowait
.list
, &sde
->dmawait
);
485 trace_hfi1_qpsleep(qp
, RVT_S_WAIT_DMA_DESC
);
486 atomic_inc(&qp
->refcount
);
488 write_sequnlock(&dev
->iowait_lock
);
489 qp
->s_flags
&= ~RVT_S_BUSY
;
490 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
493 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
498 write_sequnlock(&dev
->iowait_lock
);
499 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
500 list_del_init(&stx
->list
);
504 static void iowait_wakeup(struct iowait
*wait
, int reason
)
506 struct rvt_qp
*qp
= iowait_to_qp(wait
);
508 WARN_ON(reason
!= SDMA_AVAIL_REASON
);
509 hfi1_qp_wakeup(qp
, RVT_S_WAIT_DMA_DESC
);
512 static void iowait_sdma_drained(struct iowait
*wait
)
514 struct rvt_qp
*qp
= iowait_to_qp(wait
);
517 * This happens when the send engine notes
518 * a QP in the error state and cannot
519 * do the flush work until that QP's
520 * sdma work has finished.
522 if (qp
->s_flags
& RVT_S_WAIT_DMA
) {
523 qp
->s_flags
&= ~RVT_S_WAIT_DMA
;
524 hfi1_schedule_send(qp
);
530 * qp_to_sdma_engine - map a qp to a send engine
535 * A send engine for the qp or NULL for SMI type qp.
537 struct sdma_engine
*qp_to_sdma_engine(struct rvt_qp
*qp
, u8 sc5
)
539 struct hfi1_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
540 struct sdma_engine
*sde
;
542 if (!(dd
->flags
& HFI1_HAS_SEND_DMA
))
544 switch (qp
->ibqp
.qp_type
) {
550 sde
= sdma_select_engine_sc(dd
, qp
->ibqp
.qp_num
>> dd
->qos_shift
, sc5
);
555 * qp_to_send_context - map a qp to a send context
560 * A send context for the qp
562 struct send_context
*qp_to_send_context(struct rvt_qp
*qp
, u8 sc5
)
564 struct hfi1_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
566 switch (qp
->ibqp
.qp_type
) {
568 /* SMA packets to VL15 */
569 return dd
->vld
[15].sc
;
574 return pio_select_send_context_sc(dd
, qp
->ibqp
.qp_num
>> dd
->qos_shift
,
579 struct hfi1_ibdev
*dev
;
585 struct qp_iter
*qp_iter_init(struct hfi1_ibdev
*dev
)
587 struct qp_iter
*iter
;
589 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
594 iter
->specials
= dev
->rdi
.ibdev
.phys_port_cnt
* 2;
595 if (qp_iter_next(iter
)) {
603 int qp_iter_next(struct qp_iter
*iter
)
605 struct hfi1_ibdev
*dev
= iter
->dev
;
608 struct rvt_qp
*pqp
= iter
->qp
;
612 * The approach is to consider the special qps
613 * as an additional table entries before the
614 * real hash table. Since the qp code sets
615 * the qp->next hash link to NULL, this works just fine.
617 * iter->specials is 2 * # ports
619 * n = 0..iter->specials is the special qp indices
621 * n = iter->specials..dev->rdi.qp_dev->qp_table_size+iter->specials are
622 * the potential hash bucket entries
625 for (; n
< dev
->rdi
.qp_dev
->qp_table_size
+ iter
->specials
; n
++) {
627 qp
= rcu_dereference(pqp
->next
);
629 if (n
< iter
->specials
) {
630 struct hfi1_pportdata
*ppd
;
631 struct hfi1_ibport
*ibp
;
634 pidx
= n
% dev
->rdi
.ibdev
.phys_port_cnt
;
635 ppd
= &dd_from_dev(dev
)->pport
[pidx
];
636 ibp
= &ppd
->ibport_data
;
639 qp
= rcu_dereference(ibp
->rvp
.qp
[0]);
641 qp
= rcu_dereference(ibp
->rvp
.qp
[1]);
643 qp
= rcu_dereference(
644 dev
->rdi
.qp_dev
->qp_table
[
645 (n
- iter
->specials
)]);
658 static const char * const qp_type_str
[] = {
659 "SMI", "GSI", "RC", "UC", "UD",
662 static int qp_idle(struct rvt_qp
*qp
)
665 qp
->s_last
== qp
->s_acked
&&
666 qp
->s_acked
== qp
->s_cur
&&
667 qp
->s_cur
== qp
->s_tail
&&
668 qp
->s_tail
== qp
->s_head
;
671 void qp_iter_print(struct seq_file
*s
, struct qp_iter
*iter
)
673 struct rvt_swqe
*wqe
;
674 struct rvt_qp
*qp
= iter
->qp
;
675 struct hfi1_qp_priv
*priv
= qp
->priv
;
676 struct sdma_engine
*sde
;
677 struct send_context
*send_context
;
679 sde
= qp_to_sdma_engine(qp
, priv
->s_sc
);
680 wqe
= rvt_get_swqe_ptr(qp
, qp
->s_last
);
681 send_context
= qp_to_send_context(qp
, priv
->s_sc
);
683 "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u PSN %x %x %x %x %x (%u %u %u %u %u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d\n",
685 qp_idle(qp
) ? "I" : "B",
687 atomic_read(&qp
->refcount
),
688 qp_type_str
[qp
->ibqp
.qp_type
],
690 wqe
? wqe
->wr
.opcode
: 0,
693 iowait_sdma_pending(&priv
->s_iowait
),
694 iowait_pio_pending(&priv
->s_iowait
),
695 !list_empty(&priv
->s_iowait
.list
),
700 qp
->s_psn
, qp
->s_next_psn
,
701 qp
->s_sending_psn
, qp
->s_sending_hpsn
,
702 qp
->s_last
, qp
->s_acked
, qp
->s_cur
,
703 qp
->s_tail
, qp
->s_head
, qp
->s_size
,
706 qp
->remote_ah_attr
.dlid
,
707 qp
->remote_ah_attr
.sl
,
713 sde
? sde
->this_idx
: 0,
715 send_context
? send_context
->sw_index
: 0,
716 ibcq_to_rvtcq(qp
->ibqp
.send_cq
)->queue
->head
,
717 ibcq_to_rvtcq(qp
->ibqp
.send_cq
)->queue
->tail
,
721 void qp_comm_est(struct rvt_qp
*qp
)
723 qp
->r_flags
|= RVT_R_COMM_EST
;
724 if (qp
->ibqp
.event_handler
) {
727 ev
.device
= qp
->ibqp
.device
;
728 ev
.element
.qp
= &qp
->ibqp
;
729 ev
.event
= IB_EVENT_COMM_EST
;
730 qp
->ibqp
.event_handler(&ev
, qp
->ibqp
.qp_context
);
734 void *qp_priv_alloc(struct rvt_dev_info
*rdi
, struct rvt_qp
*qp
,
737 struct hfi1_qp_priv
*priv
;
739 priv
= kzalloc_node(sizeof(*priv
), gfp
, rdi
->dparms
.node
);
741 return ERR_PTR(-ENOMEM
);
745 priv
->s_hdr
= kzalloc_node(sizeof(*priv
->s_hdr
), gfp
, rdi
->dparms
.node
);
748 return ERR_PTR(-ENOMEM
);
750 setup_timer(&priv
->s_rnr_timer
, hfi1_rc_rnr_retry
, (unsigned long)qp
);
751 qp
->s_timer
.function
= hfi1_rc_timeout
;
755 void qp_priv_free(struct rvt_dev_info
*rdi
, struct rvt_qp
*qp
)
757 struct hfi1_qp_priv
*priv
= qp
->priv
;
763 unsigned free_all_qps(struct rvt_dev_info
*rdi
)
765 struct hfi1_ibdev
*verbs_dev
= container_of(rdi
,
768 struct hfi1_devdata
*dd
= container_of(verbs_dev
,
772 unsigned qp_inuse
= 0;
774 for (n
= 0; n
< dd
->num_pports
; n
++) {
775 struct hfi1_ibport
*ibp
= &dd
->pport
[n
].ibport_data
;
778 if (rcu_dereference(ibp
->rvp
.qp
[0]))
780 if (rcu_dereference(ibp
->rvp
.qp
[1]))
788 void flush_qp_waiters(struct rvt_qp
*qp
)
791 hfi1_stop_rc_timers(qp
);
794 void stop_send_queue(struct rvt_qp
*qp
)
796 struct hfi1_qp_priv
*priv
= qp
->priv
;
798 cancel_work_sync(&priv
->s_iowait
.iowork
);
799 hfi1_del_timers_sync(qp
);
802 void quiesce_qp(struct rvt_qp
*qp
)
804 struct hfi1_qp_priv
*priv
= qp
->priv
;
806 iowait_sdma_drain(&priv
->s_iowait
);
811 void notify_qp_reset(struct rvt_qp
*qp
)
813 struct hfi1_qp_priv
*priv
= qp
->priv
;
821 iowait_sdma_drained
);
822 priv
->r_adefered
= 0;
827 * Switch to alternate path.
828 * The QP s_lock should be held and interrupts disabled.
830 void hfi1_migrate_qp(struct rvt_qp
*qp
)
832 struct hfi1_qp_priv
*priv
= qp
->priv
;
835 qp
->s_mig_state
= IB_MIG_MIGRATED
;
836 qp
->remote_ah_attr
= qp
->alt_ah_attr
;
837 qp
->port_num
= qp
->alt_ah_attr
.port_num
;
838 qp
->s_pkey_index
= qp
->s_alt_pkey_index
;
839 qp
->s_flags
|= RVT_S_AHG_CLEAR
;
840 priv
->s_sc
= ah_to_sc(qp
->ibqp
.device
, &qp
->remote_ah_attr
);
841 priv
->s_sde
= qp_to_sdma_engine(qp
, priv
->s_sc
);
843 ev
.device
= qp
->ibqp
.device
;
844 ev
.element
.qp
= &qp
->ibqp
;
845 ev
.event
= IB_EVENT_PATH_MIG
;
846 qp
->ibqp
.event_handler(&ev
, qp
->ibqp
.qp_context
);
849 int mtu_to_path_mtu(u32 mtu
)
851 return mtu_to_enum(mtu
, OPA_MTU_8192
);
854 u32
mtu_from_qp(struct rvt_dev_info
*rdi
, struct rvt_qp
*qp
, u32 pmtu
)
857 struct hfi1_ibdev
*verbs_dev
= container_of(rdi
,
860 struct hfi1_devdata
*dd
= container_of(verbs_dev
,
863 struct hfi1_ibport
*ibp
;
866 ibp
= &dd
->pport
[qp
->port_num
- 1].ibport_data
;
867 sc
= ibp
->sl_to_sc
[qp
->remote_ah_attr
.sl
];
868 vl
= sc_to_vlt(dd
, sc
);
870 mtu
= verbs_mtu_enum_to_int(qp
->ibqp
.device
, pmtu
);
871 if (vl
< PER_VL_SEND_CONTEXTS
)
872 mtu
= min_t(u32
, mtu
, dd
->vld
[vl
].mtu
);
876 int get_pmtu_from_attr(struct rvt_dev_info
*rdi
, struct rvt_qp
*qp
,
877 struct ib_qp_attr
*attr
)
879 int mtu
, pidx
= qp
->port_num
- 1;
880 struct hfi1_ibdev
*verbs_dev
= container_of(rdi
,
883 struct hfi1_devdata
*dd
= container_of(verbs_dev
,
886 mtu
= verbs_mtu_enum_to_int(qp
->ibqp
.device
, attr
->path_mtu
);
888 return -1; /* values less than 0 are error */
890 if (mtu
> dd
->pport
[pidx
].ibmtu
)
891 return mtu_to_enum(dd
->pport
[pidx
].ibmtu
, IB_MTU_2048
);
893 return attr
->path_mtu
;
896 void notify_error_qp(struct rvt_qp
*qp
)
898 struct hfi1_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
899 struct hfi1_qp_priv
*priv
= qp
->priv
;
901 write_seqlock(&dev
->iowait_lock
);
902 if (!list_empty(&priv
->s_iowait
.list
) && !(qp
->s_flags
& RVT_S_BUSY
)) {
903 qp
->s_flags
&= ~RVT_S_ANY_WAIT_IO
;
904 list_del_init(&priv
->s_iowait
.list
);
905 if (atomic_dec_and_test(&qp
->refcount
))
908 write_sequnlock(&dev
->iowait_lock
);
910 if (!(qp
->s_flags
& RVT_S_BUSY
)) {
913 rvt_put_mr(qp
->s_rdma_mr
);
914 qp
->s_rdma_mr
= NULL
;
921 * hfi1_error_port_qps - put a port's RC/UC qps into error state
923 * @sl: the service level.
925 * This function places all RC/UC qps with a given service level into error
926 * state. It is generally called to force upper lay apps to abandon stale qps
927 * after an sl->sc mapping change.
929 void hfi1_error_port_qps(struct hfi1_ibport
*ibp
, u8 sl
)
931 struct rvt_qp
*qp
= NULL
;
932 struct hfi1_pportdata
*ppd
= ppd_from_ibp(ibp
);
933 struct hfi1_ibdev
*dev
= &ppd
->dd
->verbs_dev
;
940 /* Deal only with RC/UC qps that use the given SL. */
941 for (n
= 0; n
< dev
->rdi
.qp_dev
->qp_table_size
; n
++) {
942 for (qp
= rcu_dereference(dev
->rdi
.qp_dev
->qp_table
[n
]); qp
;
943 qp
= rcu_dereference(qp
->next
)) {
944 if (qp
->port_num
== ppd
->port
&&
945 (qp
->ibqp
.qp_type
== IB_QPT_UC
||
946 qp
->ibqp
.qp_type
== IB_QPT_RC
) &&
947 qp
->remote_ah_attr
.sl
== sl
&&
948 (ib_rvt_state_ops
[qp
->state
] &
950 spin_lock_irq(&qp
->r_lock
);
951 spin_lock(&qp
->s_hlock
);
952 spin_lock(&qp
->s_lock
);
953 lastwqe
= rvt_error_qp(qp
,
955 spin_unlock(&qp
->s_lock
);
956 spin_unlock(&qp
->s_hlock
);
957 spin_unlock_irq(&qp
->r_lock
);
959 ev
.device
= qp
->ibqp
.device
;
960 ev
.element
.qp
= &qp
->ibqp
;
962 IB_EVENT_QP_LAST_WQE_REACHED
;
963 qp
->ibqp
.event_handler(&ev
,
964 qp
->ibqp
.qp_context
);