2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4 * Copyright (c) 2004 Intel Corporation. All rights reserved.
5 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/export.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
45 #include <rdma/ib_verbs.h>
46 #include <rdma/ib_cache.h>
47 #include <rdma/ib_addr.h>
49 #include "core_priv.h"
51 static const char * const ib_events
[] = {
52 [IB_EVENT_CQ_ERR
] = "CQ error",
53 [IB_EVENT_QP_FATAL
] = "QP fatal error",
54 [IB_EVENT_QP_REQ_ERR
] = "QP request error",
55 [IB_EVENT_QP_ACCESS_ERR
] = "QP access error",
56 [IB_EVENT_COMM_EST
] = "communication established",
57 [IB_EVENT_SQ_DRAINED
] = "send queue drained",
58 [IB_EVENT_PATH_MIG
] = "path migration successful",
59 [IB_EVENT_PATH_MIG_ERR
] = "path migration error",
60 [IB_EVENT_DEVICE_FATAL
] = "device fatal error",
61 [IB_EVENT_PORT_ACTIVE
] = "port active",
62 [IB_EVENT_PORT_ERR
] = "port error",
63 [IB_EVENT_LID_CHANGE
] = "LID change",
64 [IB_EVENT_PKEY_CHANGE
] = "P_key change",
65 [IB_EVENT_SM_CHANGE
] = "SM change",
66 [IB_EVENT_SRQ_ERR
] = "SRQ error",
67 [IB_EVENT_SRQ_LIMIT_REACHED
] = "SRQ limit reached",
68 [IB_EVENT_QP_LAST_WQE_REACHED
] = "last WQE reached",
69 [IB_EVENT_CLIENT_REREGISTER
] = "client reregister",
70 [IB_EVENT_GID_CHANGE
] = "GID changed",
73 const char *ib_event_msg(enum ib_event_type event
)
77 return (index
< ARRAY_SIZE(ib_events
) && ib_events
[index
]) ?
78 ib_events
[index
] : "unrecognized event";
80 EXPORT_SYMBOL(ib_event_msg
);
82 static const char * const wc_statuses
[] = {
83 [IB_WC_SUCCESS
] = "success",
84 [IB_WC_LOC_LEN_ERR
] = "local length error",
85 [IB_WC_LOC_QP_OP_ERR
] = "local QP operation error",
86 [IB_WC_LOC_EEC_OP_ERR
] = "local EE context operation error",
87 [IB_WC_LOC_PROT_ERR
] = "local protection error",
88 [IB_WC_WR_FLUSH_ERR
] = "WR flushed",
89 [IB_WC_MW_BIND_ERR
] = "memory management operation error",
90 [IB_WC_BAD_RESP_ERR
] = "bad response error",
91 [IB_WC_LOC_ACCESS_ERR
] = "local access error",
92 [IB_WC_REM_INV_REQ_ERR
] = "invalid request error",
93 [IB_WC_REM_ACCESS_ERR
] = "remote access error",
94 [IB_WC_REM_OP_ERR
] = "remote operation error",
95 [IB_WC_RETRY_EXC_ERR
] = "transport retry counter exceeded",
96 [IB_WC_RNR_RETRY_EXC_ERR
] = "RNR retry counter exceeded",
97 [IB_WC_LOC_RDD_VIOL_ERR
] = "local RDD violation error",
98 [IB_WC_REM_INV_RD_REQ_ERR
] = "remote invalid RD request",
99 [IB_WC_REM_ABORT_ERR
] = "operation aborted",
100 [IB_WC_INV_EECN_ERR
] = "invalid EE context number",
101 [IB_WC_INV_EEC_STATE_ERR
] = "invalid EE context state",
102 [IB_WC_FATAL_ERR
] = "fatal error",
103 [IB_WC_RESP_TIMEOUT_ERR
] = "response timeout error",
104 [IB_WC_GENERAL_ERR
] = "general error",
107 const char *ib_wc_status_msg(enum ib_wc_status status
)
109 size_t index
= status
;
111 return (index
< ARRAY_SIZE(wc_statuses
) && wc_statuses
[index
]) ?
112 wc_statuses
[index
] : "unrecognized status";
114 EXPORT_SYMBOL(ib_wc_status_msg
);
116 __attribute_const__
int ib_rate_to_mult(enum ib_rate rate
)
119 case IB_RATE_2_5_GBPS
: return 1;
120 case IB_RATE_5_GBPS
: return 2;
121 case IB_RATE_10_GBPS
: return 4;
122 case IB_RATE_20_GBPS
: return 8;
123 case IB_RATE_30_GBPS
: return 12;
124 case IB_RATE_40_GBPS
: return 16;
125 case IB_RATE_60_GBPS
: return 24;
126 case IB_RATE_80_GBPS
: return 32;
127 case IB_RATE_120_GBPS
: return 48;
131 EXPORT_SYMBOL(ib_rate_to_mult
);
133 __attribute_const__
enum ib_rate
mult_to_ib_rate(int mult
)
136 case 1: return IB_RATE_2_5_GBPS
;
137 case 2: return IB_RATE_5_GBPS
;
138 case 4: return IB_RATE_10_GBPS
;
139 case 8: return IB_RATE_20_GBPS
;
140 case 12: return IB_RATE_30_GBPS
;
141 case 16: return IB_RATE_40_GBPS
;
142 case 24: return IB_RATE_60_GBPS
;
143 case 32: return IB_RATE_80_GBPS
;
144 case 48: return IB_RATE_120_GBPS
;
145 default: return IB_RATE_PORT_CURRENT
;
148 EXPORT_SYMBOL(mult_to_ib_rate
);
150 __attribute_const__
int ib_rate_to_mbps(enum ib_rate rate
)
153 case IB_RATE_2_5_GBPS
: return 2500;
154 case IB_RATE_5_GBPS
: return 5000;
155 case IB_RATE_10_GBPS
: return 10000;
156 case IB_RATE_20_GBPS
: return 20000;
157 case IB_RATE_30_GBPS
: return 30000;
158 case IB_RATE_40_GBPS
: return 40000;
159 case IB_RATE_60_GBPS
: return 60000;
160 case IB_RATE_80_GBPS
: return 80000;
161 case IB_RATE_120_GBPS
: return 120000;
162 case IB_RATE_14_GBPS
: return 14062;
163 case IB_RATE_56_GBPS
: return 56250;
164 case IB_RATE_112_GBPS
: return 112500;
165 case IB_RATE_168_GBPS
: return 168750;
166 case IB_RATE_25_GBPS
: return 25781;
167 case IB_RATE_100_GBPS
: return 103125;
168 case IB_RATE_200_GBPS
: return 206250;
169 case IB_RATE_300_GBPS
: return 309375;
173 EXPORT_SYMBOL(ib_rate_to_mbps
);
175 __attribute_const__
enum rdma_transport_type
176 rdma_node_get_transport(enum rdma_node_type node_type
)
179 case RDMA_NODE_IB_CA
:
180 case RDMA_NODE_IB_SWITCH
:
181 case RDMA_NODE_IB_ROUTER
:
182 return RDMA_TRANSPORT_IB
;
184 return RDMA_TRANSPORT_IWARP
;
185 case RDMA_NODE_USNIC
:
186 return RDMA_TRANSPORT_USNIC
;
187 case RDMA_NODE_USNIC_UDP
:
188 return RDMA_TRANSPORT_USNIC_UDP
;
194 EXPORT_SYMBOL(rdma_node_get_transport
);
196 enum rdma_link_layer
rdma_port_get_link_layer(struct ib_device
*device
, u8 port_num
)
198 if (device
->get_link_layer
)
199 return device
->get_link_layer(device
, port_num
);
201 switch (rdma_node_get_transport(device
->node_type
)) {
202 case RDMA_TRANSPORT_IB
:
203 return IB_LINK_LAYER_INFINIBAND
;
204 case RDMA_TRANSPORT_IWARP
:
205 case RDMA_TRANSPORT_USNIC
:
206 case RDMA_TRANSPORT_USNIC_UDP
:
207 return IB_LINK_LAYER_ETHERNET
;
209 return IB_LINK_LAYER_UNSPECIFIED
;
212 EXPORT_SYMBOL(rdma_port_get_link_layer
);
214 /* Protection domains */
216 struct ib_pd
*ib_alloc_pd(struct ib_device
*device
)
220 pd
= device
->alloc_pd(device
, NULL
, NULL
);
225 atomic_set(&pd
->usecnt
, 0);
230 EXPORT_SYMBOL(ib_alloc_pd
);
232 int ib_dealloc_pd(struct ib_pd
*pd
)
234 if (atomic_read(&pd
->usecnt
))
237 return pd
->device
->dealloc_pd(pd
);
239 EXPORT_SYMBOL(ib_dealloc_pd
);
241 /* Address handles */
243 struct ib_ah
*ib_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
)
247 ah
= pd
->device
->create_ah(pd
, ah_attr
);
250 ah
->device
= pd
->device
;
253 atomic_inc(&pd
->usecnt
);
258 EXPORT_SYMBOL(ib_create_ah
);
260 int ib_init_ah_from_wc(struct ib_device
*device
, u8 port_num
, struct ib_wc
*wc
,
261 struct ib_grh
*grh
, struct ib_ah_attr
*ah_attr
)
267 memset(ah_attr
, 0, sizeof *ah_attr
);
268 if (rdma_cap_eth_ah(device
, port_num
)) {
269 if (!(wc
->wc_flags
& IB_WC_GRH
))
272 if (wc
->wc_flags
& IB_WC_WITH_SMAC
&&
273 wc
->wc_flags
& IB_WC_WITH_VLAN
) {
274 memcpy(ah_attr
->dmac
, wc
->smac
, ETH_ALEN
);
275 ah_attr
->vlan_id
= wc
->vlan_id
;
277 ret
= rdma_addr_find_dmac_by_grh(&grh
->dgid
, &grh
->sgid
,
278 ah_attr
->dmac
, &ah_attr
->vlan_id
);
283 ah_attr
->vlan_id
= 0xffff;
286 ah_attr
->dlid
= wc
->slid
;
287 ah_attr
->sl
= wc
->sl
;
288 ah_attr
->src_path_bits
= wc
->dlid_path_bits
;
289 ah_attr
->port_num
= port_num
;
291 if (wc
->wc_flags
& IB_WC_GRH
) {
292 ah_attr
->ah_flags
= IB_AH_GRH
;
293 ah_attr
->grh
.dgid
= grh
->sgid
;
295 ret
= ib_find_cached_gid(device
, &grh
->dgid
, &port_num
,
300 ah_attr
->grh
.sgid_index
= (u8
) gid_index
;
301 flow_class
= be32_to_cpu(grh
->version_tclass_flow
);
302 ah_attr
->grh
.flow_label
= flow_class
& 0xFFFFF;
303 ah_attr
->grh
.hop_limit
= 0xFF;
304 ah_attr
->grh
.traffic_class
= (flow_class
>> 20) & 0xFF;
308 EXPORT_SYMBOL(ib_init_ah_from_wc
);
310 struct ib_ah
*ib_create_ah_from_wc(struct ib_pd
*pd
, struct ib_wc
*wc
,
311 struct ib_grh
*grh
, u8 port_num
)
313 struct ib_ah_attr ah_attr
;
316 ret
= ib_init_ah_from_wc(pd
->device
, port_num
, wc
, grh
, &ah_attr
);
320 return ib_create_ah(pd
, &ah_attr
);
322 EXPORT_SYMBOL(ib_create_ah_from_wc
);
324 int ib_modify_ah(struct ib_ah
*ah
, struct ib_ah_attr
*ah_attr
)
326 return ah
->device
->modify_ah
?
327 ah
->device
->modify_ah(ah
, ah_attr
) :
330 EXPORT_SYMBOL(ib_modify_ah
);
332 int ib_query_ah(struct ib_ah
*ah
, struct ib_ah_attr
*ah_attr
)
334 return ah
->device
->query_ah
?
335 ah
->device
->query_ah(ah
, ah_attr
) :
338 EXPORT_SYMBOL(ib_query_ah
);
340 int ib_destroy_ah(struct ib_ah
*ah
)
346 ret
= ah
->device
->destroy_ah(ah
);
348 atomic_dec(&pd
->usecnt
);
352 EXPORT_SYMBOL(ib_destroy_ah
);
354 /* Shared receive queues */
356 struct ib_srq
*ib_create_srq(struct ib_pd
*pd
,
357 struct ib_srq_init_attr
*srq_init_attr
)
361 if (!pd
->device
->create_srq
)
362 return ERR_PTR(-ENOSYS
);
364 srq
= pd
->device
->create_srq(pd
, srq_init_attr
, NULL
);
367 srq
->device
= pd
->device
;
370 srq
->event_handler
= srq_init_attr
->event_handler
;
371 srq
->srq_context
= srq_init_attr
->srq_context
;
372 srq
->srq_type
= srq_init_attr
->srq_type
;
373 if (srq
->srq_type
== IB_SRQT_XRC
) {
374 srq
->ext
.xrc
.xrcd
= srq_init_attr
->ext
.xrc
.xrcd
;
375 srq
->ext
.xrc
.cq
= srq_init_attr
->ext
.xrc
.cq
;
376 atomic_inc(&srq
->ext
.xrc
.xrcd
->usecnt
);
377 atomic_inc(&srq
->ext
.xrc
.cq
->usecnt
);
379 atomic_inc(&pd
->usecnt
);
380 atomic_set(&srq
->usecnt
, 0);
385 EXPORT_SYMBOL(ib_create_srq
);
387 int ib_modify_srq(struct ib_srq
*srq
,
388 struct ib_srq_attr
*srq_attr
,
389 enum ib_srq_attr_mask srq_attr_mask
)
391 return srq
->device
->modify_srq
?
392 srq
->device
->modify_srq(srq
, srq_attr
, srq_attr_mask
, NULL
) :
395 EXPORT_SYMBOL(ib_modify_srq
);
397 int ib_query_srq(struct ib_srq
*srq
,
398 struct ib_srq_attr
*srq_attr
)
400 return srq
->device
->query_srq
?
401 srq
->device
->query_srq(srq
, srq_attr
) : -ENOSYS
;
403 EXPORT_SYMBOL(ib_query_srq
);
405 int ib_destroy_srq(struct ib_srq
*srq
)
408 enum ib_srq_type srq_type
;
409 struct ib_xrcd
*uninitialized_var(xrcd
);
410 struct ib_cq
*uninitialized_var(cq
);
413 if (atomic_read(&srq
->usecnt
))
417 srq_type
= srq
->srq_type
;
418 if (srq_type
== IB_SRQT_XRC
) {
419 xrcd
= srq
->ext
.xrc
.xrcd
;
420 cq
= srq
->ext
.xrc
.cq
;
423 ret
= srq
->device
->destroy_srq(srq
);
425 atomic_dec(&pd
->usecnt
);
426 if (srq_type
== IB_SRQT_XRC
) {
427 atomic_dec(&xrcd
->usecnt
);
428 atomic_dec(&cq
->usecnt
);
434 EXPORT_SYMBOL(ib_destroy_srq
);
438 static void __ib_shared_qp_event_handler(struct ib_event
*event
, void *context
)
440 struct ib_qp
*qp
= context
;
443 spin_lock_irqsave(&qp
->device
->event_handler_lock
, flags
);
444 list_for_each_entry(event
->element
.qp
, &qp
->open_list
, open_list
)
445 if (event
->element
.qp
->event_handler
)
446 event
->element
.qp
->event_handler(event
, event
->element
.qp
->qp_context
);
447 spin_unlock_irqrestore(&qp
->device
->event_handler_lock
, flags
);
450 static void __ib_insert_xrcd_qp(struct ib_xrcd
*xrcd
, struct ib_qp
*qp
)
452 mutex_lock(&xrcd
->tgt_qp_mutex
);
453 list_add(&qp
->xrcd_list
, &xrcd
->tgt_qp_list
);
454 mutex_unlock(&xrcd
->tgt_qp_mutex
);
457 static struct ib_qp
*__ib_open_qp(struct ib_qp
*real_qp
,
458 void (*event_handler
)(struct ib_event
*, void *),
464 qp
= kzalloc(sizeof *qp
, GFP_KERNEL
);
466 return ERR_PTR(-ENOMEM
);
468 qp
->real_qp
= real_qp
;
469 atomic_inc(&real_qp
->usecnt
);
470 qp
->device
= real_qp
->device
;
471 qp
->event_handler
= event_handler
;
472 qp
->qp_context
= qp_context
;
473 qp
->qp_num
= real_qp
->qp_num
;
474 qp
->qp_type
= real_qp
->qp_type
;
476 spin_lock_irqsave(&real_qp
->device
->event_handler_lock
, flags
);
477 list_add(&qp
->open_list
, &real_qp
->open_list
);
478 spin_unlock_irqrestore(&real_qp
->device
->event_handler_lock
, flags
);
483 struct ib_qp
*ib_open_qp(struct ib_xrcd
*xrcd
,
484 struct ib_qp_open_attr
*qp_open_attr
)
486 struct ib_qp
*qp
, *real_qp
;
488 if (qp_open_attr
->qp_type
!= IB_QPT_XRC_TGT
)
489 return ERR_PTR(-EINVAL
);
491 qp
= ERR_PTR(-EINVAL
);
492 mutex_lock(&xrcd
->tgt_qp_mutex
);
493 list_for_each_entry(real_qp
, &xrcd
->tgt_qp_list
, xrcd_list
) {
494 if (real_qp
->qp_num
== qp_open_attr
->qp_num
) {
495 qp
= __ib_open_qp(real_qp
, qp_open_attr
->event_handler
,
496 qp_open_attr
->qp_context
);
500 mutex_unlock(&xrcd
->tgt_qp_mutex
);
503 EXPORT_SYMBOL(ib_open_qp
);
505 struct ib_qp
*ib_create_qp(struct ib_pd
*pd
,
506 struct ib_qp_init_attr
*qp_init_attr
)
508 struct ib_qp
*qp
, *real_qp
;
509 struct ib_device
*device
;
511 device
= pd
? pd
->device
: qp_init_attr
->xrcd
->device
;
512 qp
= device
->create_qp(pd
, qp_init_attr
, NULL
);
518 qp
->qp_type
= qp_init_attr
->qp_type
;
520 atomic_set(&qp
->usecnt
, 0);
521 if (qp_init_attr
->qp_type
== IB_QPT_XRC_TGT
) {
522 qp
->event_handler
= __ib_shared_qp_event_handler
;
525 qp
->send_cq
= qp
->recv_cq
= NULL
;
527 qp
->xrcd
= qp_init_attr
->xrcd
;
528 atomic_inc(&qp_init_attr
->xrcd
->usecnt
);
529 INIT_LIST_HEAD(&qp
->open_list
);
532 qp
= __ib_open_qp(real_qp
, qp_init_attr
->event_handler
,
533 qp_init_attr
->qp_context
);
535 __ib_insert_xrcd_qp(qp_init_attr
->xrcd
, real_qp
);
537 real_qp
->device
->destroy_qp(real_qp
);
539 qp
->event_handler
= qp_init_attr
->event_handler
;
540 qp
->qp_context
= qp_init_attr
->qp_context
;
541 if (qp_init_attr
->qp_type
== IB_QPT_XRC_INI
) {
545 qp
->recv_cq
= qp_init_attr
->recv_cq
;
546 atomic_inc(&qp_init_attr
->recv_cq
->usecnt
);
547 qp
->srq
= qp_init_attr
->srq
;
549 atomic_inc(&qp_init_attr
->srq
->usecnt
);
553 qp
->send_cq
= qp_init_attr
->send_cq
;
556 atomic_inc(&pd
->usecnt
);
557 atomic_inc(&qp_init_attr
->send_cq
->usecnt
);
563 EXPORT_SYMBOL(ib_create_qp
);
565 static const struct {
567 enum ib_qp_attr_mask req_param
[IB_QPT_MAX
];
568 enum ib_qp_attr_mask req_param_add_eth
[IB_QPT_MAX
];
569 enum ib_qp_attr_mask opt_param
[IB_QPT_MAX
];
570 enum ib_qp_attr_mask opt_param_add_eth
[IB_QPT_MAX
];
571 } qp_state_table
[IB_QPS_ERR
+ 1][IB_QPS_ERR
+ 1] = {
573 [IB_QPS_RESET
] = { .valid
= 1 },
577 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
580 [IB_QPT_RAW_PACKET
] = IB_QP_PORT
,
581 [IB_QPT_UC
] = (IB_QP_PKEY_INDEX
|
584 [IB_QPT_RC
] = (IB_QP_PKEY_INDEX
|
587 [IB_QPT_XRC_INI
] = (IB_QP_PKEY_INDEX
|
590 [IB_QPT_XRC_TGT
] = (IB_QP_PKEY_INDEX
|
593 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
595 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
601 [IB_QPS_RESET
] = { .valid
= 1 },
602 [IB_QPS_ERR
] = { .valid
= 1 },
606 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
609 [IB_QPT_UC
] = (IB_QP_PKEY_INDEX
|
612 [IB_QPT_RC
] = (IB_QP_PKEY_INDEX
|
615 [IB_QPT_XRC_INI
] = (IB_QP_PKEY_INDEX
|
618 [IB_QPT_XRC_TGT
] = (IB_QP_PKEY_INDEX
|
621 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
623 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
630 [IB_QPT_UC
] = (IB_QP_AV
|
634 [IB_QPT_RC
] = (IB_QP_AV
|
638 IB_QP_MAX_DEST_RD_ATOMIC
|
639 IB_QP_MIN_RNR_TIMER
),
640 [IB_QPT_XRC_INI
] = (IB_QP_AV
|
644 [IB_QPT_XRC_TGT
] = (IB_QP_AV
|
648 IB_QP_MAX_DEST_RD_ATOMIC
|
649 IB_QP_MIN_RNR_TIMER
),
651 .req_param_add_eth
= {
652 [IB_QPT_RC
] = (IB_QP_SMAC
),
653 [IB_QPT_UC
] = (IB_QP_SMAC
),
654 [IB_QPT_XRC_INI
] = (IB_QP_SMAC
),
655 [IB_QPT_XRC_TGT
] = (IB_QP_SMAC
)
658 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
660 [IB_QPT_UC
] = (IB_QP_ALT_PATH
|
663 [IB_QPT_RC
] = (IB_QP_ALT_PATH
|
666 [IB_QPT_XRC_INI
] = (IB_QP_ALT_PATH
|
669 [IB_QPT_XRC_TGT
] = (IB_QP_ALT_PATH
|
672 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
674 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
677 .opt_param_add_eth
= {
678 [IB_QPT_RC
] = (IB_QP_ALT_SMAC
|
681 [IB_QPT_UC
] = (IB_QP_ALT_SMAC
|
684 [IB_QPT_XRC_INI
] = (IB_QP_ALT_SMAC
|
687 [IB_QPT_XRC_TGT
] = (IB_QP_ALT_SMAC
|
694 [IB_QPS_RESET
] = { .valid
= 1 },
695 [IB_QPS_ERR
] = { .valid
= 1 },
699 [IB_QPT_UD
] = IB_QP_SQ_PSN
,
700 [IB_QPT_UC
] = IB_QP_SQ_PSN
,
701 [IB_QPT_RC
] = (IB_QP_TIMEOUT
|
705 IB_QP_MAX_QP_RD_ATOMIC
),
706 [IB_QPT_XRC_INI
] = (IB_QP_TIMEOUT
|
710 IB_QP_MAX_QP_RD_ATOMIC
),
711 [IB_QPT_XRC_TGT
] = (IB_QP_TIMEOUT
|
713 [IB_QPT_SMI
] = IB_QP_SQ_PSN
,
714 [IB_QPT_GSI
] = IB_QP_SQ_PSN
,
717 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
719 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
722 IB_QP_PATH_MIG_STATE
),
723 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
726 IB_QP_MIN_RNR_TIMER
|
727 IB_QP_PATH_MIG_STATE
),
728 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
731 IB_QP_PATH_MIG_STATE
),
732 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
735 IB_QP_MIN_RNR_TIMER
|
736 IB_QP_PATH_MIG_STATE
),
737 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
739 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
745 [IB_QPS_RESET
] = { .valid
= 1 },
746 [IB_QPS_ERR
] = { .valid
= 1 },
750 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
752 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
755 IB_QP_PATH_MIG_STATE
),
756 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
759 IB_QP_PATH_MIG_STATE
|
760 IB_QP_MIN_RNR_TIMER
),
761 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
764 IB_QP_PATH_MIG_STATE
),
765 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
768 IB_QP_PATH_MIG_STATE
|
769 IB_QP_MIN_RNR_TIMER
),
770 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
772 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
779 [IB_QPT_UD
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
780 [IB_QPT_UC
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
781 [IB_QPT_RC
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
782 [IB_QPT_XRC_INI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
783 [IB_QPT_XRC_TGT
] = IB_QP_EN_SQD_ASYNC_NOTIFY
, /* ??? */
784 [IB_QPT_SMI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
785 [IB_QPT_GSI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
790 [IB_QPS_RESET
] = { .valid
= 1 },
791 [IB_QPS_ERR
] = { .valid
= 1 },
795 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
797 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
800 IB_QP_PATH_MIG_STATE
),
801 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
804 IB_QP_MIN_RNR_TIMER
|
805 IB_QP_PATH_MIG_STATE
),
806 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
809 IB_QP_PATH_MIG_STATE
),
810 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
813 IB_QP_MIN_RNR_TIMER
|
814 IB_QP_PATH_MIG_STATE
),
815 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
817 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
824 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
826 [IB_QPT_UC
] = (IB_QP_AV
|
830 IB_QP_PATH_MIG_STATE
),
831 [IB_QPT_RC
] = (IB_QP_PORT
|
836 IB_QP_MAX_QP_RD_ATOMIC
|
837 IB_QP_MAX_DEST_RD_ATOMIC
|
841 IB_QP_MIN_RNR_TIMER
|
842 IB_QP_PATH_MIG_STATE
),
843 [IB_QPT_XRC_INI
] = (IB_QP_PORT
|
848 IB_QP_MAX_QP_RD_ATOMIC
|
852 IB_QP_PATH_MIG_STATE
),
853 [IB_QPT_XRC_TGT
] = (IB_QP_PORT
|
856 IB_QP_MAX_DEST_RD_ATOMIC
|
860 IB_QP_MIN_RNR_TIMER
|
861 IB_QP_PATH_MIG_STATE
),
862 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
864 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
870 [IB_QPS_RESET
] = { .valid
= 1 },
871 [IB_QPS_ERR
] = { .valid
= 1 },
875 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
877 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
879 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
881 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
887 [IB_QPS_RESET
] = { .valid
= 1 },
888 [IB_QPS_ERR
] = { .valid
= 1 }
892 int ib_modify_qp_is_ok(enum ib_qp_state cur_state
, enum ib_qp_state next_state
,
893 enum ib_qp_type type
, enum ib_qp_attr_mask mask
,
894 enum rdma_link_layer ll
)
896 enum ib_qp_attr_mask req_param
, opt_param
;
898 if (cur_state
< 0 || cur_state
> IB_QPS_ERR
||
899 next_state
< 0 || next_state
> IB_QPS_ERR
)
902 if (mask
& IB_QP_CUR_STATE
&&
903 cur_state
!= IB_QPS_RTR
&& cur_state
!= IB_QPS_RTS
&&
904 cur_state
!= IB_QPS_SQD
&& cur_state
!= IB_QPS_SQE
)
907 if (!qp_state_table
[cur_state
][next_state
].valid
)
910 req_param
= qp_state_table
[cur_state
][next_state
].req_param
[type
];
911 opt_param
= qp_state_table
[cur_state
][next_state
].opt_param
[type
];
913 if (ll
== IB_LINK_LAYER_ETHERNET
) {
914 req_param
|= qp_state_table
[cur_state
][next_state
].
915 req_param_add_eth
[type
];
916 opt_param
|= qp_state_table
[cur_state
][next_state
].
917 opt_param_add_eth
[type
];
920 if ((mask
& req_param
) != req_param
)
923 if (mask
& ~(req_param
| opt_param
| IB_QP_STATE
))
928 EXPORT_SYMBOL(ib_modify_qp_is_ok
);
930 int ib_resolve_eth_l2_attrs(struct ib_qp
*qp
,
931 struct ib_qp_attr
*qp_attr
, int *qp_attr_mask
)
936 if ((*qp_attr_mask
& IB_QP_AV
) &&
937 (rdma_cap_eth_ah(qp
->device
, qp_attr
->ah_attr
.port_num
))) {
938 ret
= ib_query_gid(qp
->device
, qp_attr
->ah_attr
.port_num
,
939 qp_attr
->ah_attr
.grh
.sgid_index
, &sgid
);
942 if (rdma_link_local_addr((struct in6_addr
*)qp_attr
->ah_attr
.grh
.dgid
.raw
)) {
943 rdma_get_ll_mac((struct in6_addr
*)qp_attr
->ah_attr
.grh
.dgid
.raw
, qp_attr
->ah_attr
.dmac
);
944 rdma_get_ll_mac((struct in6_addr
*)sgid
.raw
, qp_attr
->smac
);
945 if (!(*qp_attr_mask
& IB_QP_VID
))
946 qp_attr
->vlan_id
= rdma_get_vlan_id(&sgid
);
948 ret
= rdma_addr_find_dmac_by_grh(&sgid
, &qp_attr
->ah_attr
.grh
.dgid
,
949 qp_attr
->ah_attr
.dmac
, &qp_attr
->vlan_id
);
952 ret
= rdma_addr_find_smac_by_sgid(&sgid
, qp_attr
->smac
, NULL
);
956 *qp_attr_mask
|= IB_QP_SMAC
;
957 if (qp_attr
->vlan_id
< 0xFFFF)
958 *qp_attr_mask
|= IB_QP_VID
;
963 EXPORT_SYMBOL(ib_resolve_eth_l2_attrs
);
966 int ib_modify_qp(struct ib_qp
*qp
,
967 struct ib_qp_attr
*qp_attr
,
972 ret
= ib_resolve_eth_l2_attrs(qp
, qp_attr
, &qp_attr_mask
);
976 return qp
->device
->modify_qp(qp
->real_qp
, qp_attr
, qp_attr_mask
, NULL
);
978 EXPORT_SYMBOL(ib_modify_qp
);
980 int ib_query_qp(struct ib_qp
*qp
,
981 struct ib_qp_attr
*qp_attr
,
983 struct ib_qp_init_attr
*qp_init_attr
)
985 return qp
->device
->query_qp
?
986 qp
->device
->query_qp(qp
->real_qp
, qp_attr
, qp_attr_mask
, qp_init_attr
) :
989 EXPORT_SYMBOL(ib_query_qp
);
991 int ib_close_qp(struct ib_qp
*qp
)
993 struct ib_qp
*real_qp
;
996 real_qp
= qp
->real_qp
;
1000 spin_lock_irqsave(&real_qp
->device
->event_handler_lock
, flags
);
1001 list_del(&qp
->open_list
);
1002 spin_unlock_irqrestore(&real_qp
->device
->event_handler_lock
, flags
);
1004 atomic_dec(&real_qp
->usecnt
);
1009 EXPORT_SYMBOL(ib_close_qp
);
1011 static int __ib_destroy_shared_qp(struct ib_qp
*qp
)
1013 struct ib_xrcd
*xrcd
;
1014 struct ib_qp
*real_qp
;
1017 real_qp
= qp
->real_qp
;
1018 xrcd
= real_qp
->xrcd
;
1020 mutex_lock(&xrcd
->tgt_qp_mutex
);
1022 if (atomic_read(&real_qp
->usecnt
) == 0)
1023 list_del(&real_qp
->xrcd_list
);
1026 mutex_unlock(&xrcd
->tgt_qp_mutex
);
1029 ret
= ib_destroy_qp(real_qp
);
1031 atomic_dec(&xrcd
->usecnt
);
1033 __ib_insert_xrcd_qp(xrcd
, real_qp
);
1039 int ib_destroy_qp(struct ib_qp
*qp
)
1042 struct ib_cq
*scq
, *rcq
;
1046 if (atomic_read(&qp
->usecnt
))
1049 if (qp
->real_qp
!= qp
)
1050 return __ib_destroy_shared_qp(qp
);
1057 ret
= qp
->device
->destroy_qp(qp
);
1060 atomic_dec(&pd
->usecnt
);
1062 atomic_dec(&scq
->usecnt
);
1064 atomic_dec(&rcq
->usecnt
);
1066 atomic_dec(&srq
->usecnt
);
1071 EXPORT_SYMBOL(ib_destroy_qp
);
1073 /* Completion queues */
1075 struct ib_cq
*ib_create_cq(struct ib_device
*device
,
1076 ib_comp_handler comp_handler
,
1077 void (*event_handler
)(struct ib_event
*, void *),
1078 void *cq_context
, int cqe
, int comp_vector
)
1082 cq
= device
->create_cq(device
, cqe
, comp_vector
, NULL
, NULL
);
1085 cq
->device
= device
;
1087 cq
->comp_handler
= comp_handler
;
1088 cq
->event_handler
= event_handler
;
1089 cq
->cq_context
= cq_context
;
1090 atomic_set(&cq
->usecnt
, 0);
1095 EXPORT_SYMBOL(ib_create_cq
);
1097 int ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
1099 return cq
->device
->modify_cq
?
1100 cq
->device
->modify_cq(cq
, cq_count
, cq_period
) : -ENOSYS
;
1102 EXPORT_SYMBOL(ib_modify_cq
);
1104 int ib_destroy_cq(struct ib_cq
*cq
)
1106 if (atomic_read(&cq
->usecnt
))
1109 return cq
->device
->destroy_cq(cq
);
1111 EXPORT_SYMBOL(ib_destroy_cq
);
1113 int ib_resize_cq(struct ib_cq
*cq
, int cqe
)
1115 return cq
->device
->resize_cq
?
1116 cq
->device
->resize_cq(cq
, cqe
, NULL
) : -ENOSYS
;
1118 EXPORT_SYMBOL(ib_resize_cq
);
1120 /* Memory regions */
1122 struct ib_mr
*ib_get_dma_mr(struct ib_pd
*pd
, int mr_access_flags
)
1127 err
= ib_check_mr_access(mr_access_flags
);
1129 return ERR_PTR(err
);
1131 mr
= pd
->device
->get_dma_mr(pd
, mr_access_flags
);
1134 mr
->device
= pd
->device
;
1137 atomic_inc(&pd
->usecnt
);
1138 atomic_set(&mr
->usecnt
, 0);
1143 EXPORT_SYMBOL(ib_get_dma_mr
);
1145 struct ib_mr
*ib_reg_phys_mr(struct ib_pd
*pd
,
1146 struct ib_phys_buf
*phys_buf_array
,
1148 int mr_access_flags
,
1154 err
= ib_check_mr_access(mr_access_flags
);
1156 return ERR_PTR(err
);
1158 if (!pd
->device
->reg_phys_mr
)
1159 return ERR_PTR(-ENOSYS
);
1161 mr
= pd
->device
->reg_phys_mr(pd
, phys_buf_array
, num_phys_buf
,
1162 mr_access_flags
, iova_start
);
1165 mr
->device
= pd
->device
;
1168 atomic_inc(&pd
->usecnt
);
1169 atomic_set(&mr
->usecnt
, 0);
1174 EXPORT_SYMBOL(ib_reg_phys_mr
);
1176 int ib_rereg_phys_mr(struct ib_mr
*mr
,
1179 struct ib_phys_buf
*phys_buf_array
,
1181 int mr_access_flags
,
1184 struct ib_pd
*old_pd
;
1187 ret
= ib_check_mr_access(mr_access_flags
);
1191 if (!mr
->device
->rereg_phys_mr
)
1194 if (atomic_read(&mr
->usecnt
))
1199 ret
= mr
->device
->rereg_phys_mr(mr
, mr_rereg_mask
, pd
,
1200 phys_buf_array
, num_phys_buf
,
1201 mr_access_flags
, iova_start
);
1203 if (!ret
&& (mr_rereg_mask
& IB_MR_REREG_PD
)) {
1204 atomic_dec(&old_pd
->usecnt
);
1205 atomic_inc(&pd
->usecnt
);
1210 EXPORT_SYMBOL(ib_rereg_phys_mr
);
1212 int ib_query_mr(struct ib_mr
*mr
, struct ib_mr_attr
*mr_attr
)
1214 return mr
->device
->query_mr
?
1215 mr
->device
->query_mr(mr
, mr_attr
) : -ENOSYS
;
1217 EXPORT_SYMBOL(ib_query_mr
);
1219 int ib_dereg_mr(struct ib_mr
*mr
)
1224 if (atomic_read(&mr
->usecnt
))
1228 ret
= mr
->device
->dereg_mr(mr
);
1230 atomic_dec(&pd
->usecnt
);
1234 EXPORT_SYMBOL(ib_dereg_mr
);
1236 struct ib_mr
*ib_create_mr(struct ib_pd
*pd
,
1237 struct ib_mr_init_attr
*mr_init_attr
)
1241 if (!pd
->device
->create_mr
)
1242 return ERR_PTR(-ENOSYS
);
1244 mr
= pd
->device
->create_mr(pd
, mr_init_attr
);
1247 mr
->device
= pd
->device
;
1250 atomic_inc(&pd
->usecnt
);
1251 atomic_set(&mr
->usecnt
, 0);
1256 EXPORT_SYMBOL(ib_create_mr
);
1258 int ib_destroy_mr(struct ib_mr
*mr
)
1263 if (atomic_read(&mr
->usecnt
))
1267 ret
= mr
->device
->destroy_mr(mr
);
1269 atomic_dec(&pd
->usecnt
);
1273 EXPORT_SYMBOL(ib_destroy_mr
);
1275 struct ib_mr
*ib_alloc_fast_reg_mr(struct ib_pd
*pd
, int max_page_list_len
)
1279 if (!pd
->device
->alloc_fast_reg_mr
)
1280 return ERR_PTR(-ENOSYS
);
1282 mr
= pd
->device
->alloc_fast_reg_mr(pd
, max_page_list_len
);
1285 mr
->device
= pd
->device
;
1288 atomic_inc(&pd
->usecnt
);
1289 atomic_set(&mr
->usecnt
, 0);
1294 EXPORT_SYMBOL(ib_alloc_fast_reg_mr
);
1296 struct ib_fast_reg_page_list
*ib_alloc_fast_reg_page_list(struct ib_device
*device
,
1297 int max_page_list_len
)
1299 struct ib_fast_reg_page_list
*page_list
;
1301 if (!device
->alloc_fast_reg_page_list
)
1302 return ERR_PTR(-ENOSYS
);
1304 page_list
= device
->alloc_fast_reg_page_list(device
, max_page_list_len
);
1306 if (!IS_ERR(page_list
)) {
1307 page_list
->device
= device
;
1308 page_list
->max_page_list_len
= max_page_list_len
;
1313 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list
);
1315 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list
*page_list
)
1317 page_list
->device
->free_fast_reg_page_list(page_list
);
1319 EXPORT_SYMBOL(ib_free_fast_reg_page_list
);
1321 /* Memory windows */
1323 struct ib_mw
*ib_alloc_mw(struct ib_pd
*pd
, enum ib_mw_type type
)
1327 if (!pd
->device
->alloc_mw
)
1328 return ERR_PTR(-ENOSYS
);
1330 mw
= pd
->device
->alloc_mw(pd
, type
);
1332 mw
->device
= pd
->device
;
1336 atomic_inc(&pd
->usecnt
);
1341 EXPORT_SYMBOL(ib_alloc_mw
);
1343 int ib_dealloc_mw(struct ib_mw
*mw
)
1349 ret
= mw
->device
->dealloc_mw(mw
);
1351 atomic_dec(&pd
->usecnt
);
1355 EXPORT_SYMBOL(ib_dealloc_mw
);
1357 /* "Fast" memory regions */
1359 struct ib_fmr
*ib_alloc_fmr(struct ib_pd
*pd
,
1360 int mr_access_flags
,
1361 struct ib_fmr_attr
*fmr_attr
)
1365 if (!pd
->device
->alloc_fmr
)
1366 return ERR_PTR(-ENOSYS
);
1368 fmr
= pd
->device
->alloc_fmr(pd
, mr_access_flags
, fmr_attr
);
1370 fmr
->device
= pd
->device
;
1372 atomic_inc(&pd
->usecnt
);
1377 EXPORT_SYMBOL(ib_alloc_fmr
);
1379 int ib_unmap_fmr(struct list_head
*fmr_list
)
1383 if (list_empty(fmr_list
))
1386 fmr
= list_entry(fmr_list
->next
, struct ib_fmr
, list
);
1387 return fmr
->device
->unmap_fmr(fmr_list
);
1389 EXPORT_SYMBOL(ib_unmap_fmr
);
1391 int ib_dealloc_fmr(struct ib_fmr
*fmr
)
1397 ret
= fmr
->device
->dealloc_fmr(fmr
);
1399 atomic_dec(&pd
->usecnt
);
1403 EXPORT_SYMBOL(ib_dealloc_fmr
);
1405 /* Multicast groups */
1407 int ib_attach_mcast(struct ib_qp
*qp
, union ib_gid
*gid
, u16 lid
)
1411 if (!qp
->device
->attach_mcast
)
1413 if (gid
->raw
[0] != 0xff || qp
->qp_type
!= IB_QPT_UD
)
1416 ret
= qp
->device
->attach_mcast(qp
, gid
, lid
);
1418 atomic_inc(&qp
->usecnt
);
1421 EXPORT_SYMBOL(ib_attach_mcast
);
1423 int ib_detach_mcast(struct ib_qp
*qp
, union ib_gid
*gid
, u16 lid
)
1427 if (!qp
->device
->detach_mcast
)
1429 if (gid
->raw
[0] != 0xff || qp
->qp_type
!= IB_QPT_UD
)
1432 ret
= qp
->device
->detach_mcast(qp
, gid
, lid
);
1434 atomic_dec(&qp
->usecnt
);
1437 EXPORT_SYMBOL(ib_detach_mcast
);
1439 struct ib_xrcd
*ib_alloc_xrcd(struct ib_device
*device
)
1441 struct ib_xrcd
*xrcd
;
1443 if (!device
->alloc_xrcd
)
1444 return ERR_PTR(-ENOSYS
);
1446 xrcd
= device
->alloc_xrcd(device
, NULL
, NULL
);
1447 if (!IS_ERR(xrcd
)) {
1448 xrcd
->device
= device
;
1450 atomic_set(&xrcd
->usecnt
, 0);
1451 mutex_init(&xrcd
->tgt_qp_mutex
);
1452 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
1457 EXPORT_SYMBOL(ib_alloc_xrcd
);
1459 int ib_dealloc_xrcd(struct ib_xrcd
*xrcd
)
1464 if (atomic_read(&xrcd
->usecnt
))
1467 while (!list_empty(&xrcd
->tgt_qp_list
)) {
1468 qp
= list_entry(xrcd
->tgt_qp_list
.next
, struct ib_qp
, xrcd_list
);
1469 ret
= ib_destroy_qp(qp
);
1474 return xrcd
->device
->dealloc_xrcd(xrcd
);
1476 EXPORT_SYMBOL(ib_dealloc_xrcd
);
1478 struct ib_flow
*ib_create_flow(struct ib_qp
*qp
,
1479 struct ib_flow_attr
*flow_attr
,
1482 struct ib_flow
*flow_id
;
1483 if (!qp
->device
->create_flow
)
1484 return ERR_PTR(-ENOSYS
);
1486 flow_id
= qp
->device
->create_flow(qp
, flow_attr
, domain
);
1487 if (!IS_ERR(flow_id
))
1488 atomic_inc(&qp
->usecnt
);
1491 EXPORT_SYMBOL(ib_create_flow
);
1493 int ib_destroy_flow(struct ib_flow
*flow_id
)
1496 struct ib_qp
*qp
= flow_id
->qp
;
1498 err
= qp
->device
->destroy_flow(flow_id
);
1500 atomic_dec(&qp
->usecnt
);
1503 EXPORT_SYMBOL(ib_destroy_flow
);
1505 int ib_check_mr_status(struct ib_mr
*mr
, u32 check_mask
,
1506 struct ib_mr_status
*mr_status
)
1508 return mr
->device
->check_mr_status
?
1509 mr
->device
->check_mr_status(mr
, check_mask
, mr_status
) : -ENOSYS
;
1511 EXPORT_SYMBOL(ib_check_mr_status
);