2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4 * Copyright (c) 2004 Intel Corporation. All rights reserved.
5 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/export.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
45 #include <linux/in6.h>
46 #include <net/addrconf.h>
48 #include <rdma/ib_verbs.h>
49 #include <rdma/ib_cache.h>
50 #include <rdma/ib_addr.h>
53 #include "core_priv.h"
55 static const char * const ib_events
[] = {
56 [IB_EVENT_CQ_ERR
] = "CQ error",
57 [IB_EVENT_QP_FATAL
] = "QP fatal error",
58 [IB_EVENT_QP_REQ_ERR
] = "QP request error",
59 [IB_EVENT_QP_ACCESS_ERR
] = "QP access error",
60 [IB_EVENT_COMM_EST
] = "communication established",
61 [IB_EVENT_SQ_DRAINED
] = "send queue drained",
62 [IB_EVENT_PATH_MIG
] = "path migration successful",
63 [IB_EVENT_PATH_MIG_ERR
] = "path migration error",
64 [IB_EVENT_DEVICE_FATAL
] = "device fatal error",
65 [IB_EVENT_PORT_ACTIVE
] = "port active",
66 [IB_EVENT_PORT_ERR
] = "port error",
67 [IB_EVENT_LID_CHANGE
] = "LID change",
68 [IB_EVENT_PKEY_CHANGE
] = "P_key change",
69 [IB_EVENT_SM_CHANGE
] = "SM change",
70 [IB_EVENT_SRQ_ERR
] = "SRQ error",
71 [IB_EVENT_SRQ_LIMIT_REACHED
] = "SRQ limit reached",
72 [IB_EVENT_QP_LAST_WQE_REACHED
] = "last WQE reached",
73 [IB_EVENT_CLIENT_REREGISTER
] = "client reregister",
74 [IB_EVENT_GID_CHANGE
] = "GID changed",
77 const char *__attribute_const__
ib_event_msg(enum ib_event_type event
)
81 return (index
< ARRAY_SIZE(ib_events
) && ib_events
[index
]) ?
82 ib_events
[index
] : "unrecognized event";
84 EXPORT_SYMBOL(ib_event_msg
);
86 static const char * const wc_statuses
[] = {
87 [IB_WC_SUCCESS
] = "success",
88 [IB_WC_LOC_LEN_ERR
] = "local length error",
89 [IB_WC_LOC_QP_OP_ERR
] = "local QP operation error",
90 [IB_WC_LOC_EEC_OP_ERR
] = "local EE context operation error",
91 [IB_WC_LOC_PROT_ERR
] = "local protection error",
92 [IB_WC_WR_FLUSH_ERR
] = "WR flushed",
93 [IB_WC_MW_BIND_ERR
] = "memory management operation error",
94 [IB_WC_BAD_RESP_ERR
] = "bad response error",
95 [IB_WC_LOC_ACCESS_ERR
] = "local access error",
96 [IB_WC_REM_INV_REQ_ERR
] = "invalid request error",
97 [IB_WC_REM_ACCESS_ERR
] = "remote access error",
98 [IB_WC_REM_OP_ERR
] = "remote operation error",
99 [IB_WC_RETRY_EXC_ERR
] = "transport retry counter exceeded",
100 [IB_WC_RNR_RETRY_EXC_ERR
] = "RNR retry counter exceeded",
101 [IB_WC_LOC_RDD_VIOL_ERR
] = "local RDD violation error",
102 [IB_WC_REM_INV_RD_REQ_ERR
] = "remote invalid RD request",
103 [IB_WC_REM_ABORT_ERR
] = "operation aborted",
104 [IB_WC_INV_EECN_ERR
] = "invalid EE context number",
105 [IB_WC_INV_EEC_STATE_ERR
] = "invalid EE context state",
106 [IB_WC_FATAL_ERR
] = "fatal error",
107 [IB_WC_RESP_TIMEOUT_ERR
] = "response timeout error",
108 [IB_WC_GENERAL_ERR
] = "general error",
111 const char *__attribute_const__
ib_wc_status_msg(enum ib_wc_status status
)
113 size_t index
= status
;
115 return (index
< ARRAY_SIZE(wc_statuses
) && wc_statuses
[index
]) ?
116 wc_statuses
[index
] : "unrecognized status";
118 EXPORT_SYMBOL(ib_wc_status_msg
);
120 __attribute_const__
int ib_rate_to_mult(enum ib_rate rate
)
123 case IB_RATE_2_5_GBPS
: return 1;
124 case IB_RATE_5_GBPS
: return 2;
125 case IB_RATE_10_GBPS
: return 4;
126 case IB_RATE_20_GBPS
: return 8;
127 case IB_RATE_30_GBPS
: return 12;
128 case IB_RATE_40_GBPS
: return 16;
129 case IB_RATE_60_GBPS
: return 24;
130 case IB_RATE_80_GBPS
: return 32;
131 case IB_RATE_120_GBPS
: return 48;
135 EXPORT_SYMBOL(ib_rate_to_mult
);
137 __attribute_const__
enum ib_rate
mult_to_ib_rate(int mult
)
140 case 1: return IB_RATE_2_5_GBPS
;
141 case 2: return IB_RATE_5_GBPS
;
142 case 4: return IB_RATE_10_GBPS
;
143 case 8: return IB_RATE_20_GBPS
;
144 case 12: return IB_RATE_30_GBPS
;
145 case 16: return IB_RATE_40_GBPS
;
146 case 24: return IB_RATE_60_GBPS
;
147 case 32: return IB_RATE_80_GBPS
;
148 case 48: return IB_RATE_120_GBPS
;
149 default: return IB_RATE_PORT_CURRENT
;
152 EXPORT_SYMBOL(mult_to_ib_rate
);
154 __attribute_const__
int ib_rate_to_mbps(enum ib_rate rate
)
157 case IB_RATE_2_5_GBPS
: return 2500;
158 case IB_RATE_5_GBPS
: return 5000;
159 case IB_RATE_10_GBPS
: return 10000;
160 case IB_RATE_20_GBPS
: return 20000;
161 case IB_RATE_30_GBPS
: return 30000;
162 case IB_RATE_40_GBPS
: return 40000;
163 case IB_RATE_60_GBPS
: return 60000;
164 case IB_RATE_80_GBPS
: return 80000;
165 case IB_RATE_120_GBPS
: return 120000;
166 case IB_RATE_14_GBPS
: return 14062;
167 case IB_RATE_56_GBPS
: return 56250;
168 case IB_RATE_112_GBPS
: return 112500;
169 case IB_RATE_168_GBPS
: return 168750;
170 case IB_RATE_25_GBPS
: return 25781;
171 case IB_RATE_100_GBPS
: return 103125;
172 case IB_RATE_200_GBPS
: return 206250;
173 case IB_RATE_300_GBPS
: return 309375;
177 EXPORT_SYMBOL(ib_rate_to_mbps
);
179 __attribute_const__
enum rdma_transport_type
180 rdma_node_get_transport(enum rdma_node_type node_type
)
183 case RDMA_NODE_IB_CA
:
184 case RDMA_NODE_IB_SWITCH
:
185 case RDMA_NODE_IB_ROUTER
:
186 return RDMA_TRANSPORT_IB
;
188 return RDMA_TRANSPORT_IWARP
;
189 case RDMA_NODE_USNIC
:
190 return RDMA_TRANSPORT_USNIC
;
191 case RDMA_NODE_USNIC_UDP
:
192 return RDMA_TRANSPORT_USNIC_UDP
;
198 EXPORT_SYMBOL(rdma_node_get_transport
);
200 enum rdma_link_layer
rdma_port_get_link_layer(struct ib_device
*device
, u8 port_num
)
202 if (device
->get_link_layer
)
203 return device
->get_link_layer(device
, port_num
);
205 switch (rdma_node_get_transport(device
->node_type
)) {
206 case RDMA_TRANSPORT_IB
:
207 return IB_LINK_LAYER_INFINIBAND
;
208 case RDMA_TRANSPORT_IWARP
:
209 case RDMA_TRANSPORT_USNIC
:
210 case RDMA_TRANSPORT_USNIC_UDP
:
211 return IB_LINK_LAYER_ETHERNET
;
213 return IB_LINK_LAYER_UNSPECIFIED
;
216 EXPORT_SYMBOL(rdma_port_get_link_layer
);
218 /* Protection domains */
221 * ib_alloc_pd - Allocates an unused protection domain.
222 * @device: The device on which to allocate the protection domain.
224 * A protection domain object provides an association between QPs, shared
225 * receive queues, address handles, memory regions, and memory windows.
227 * Every PD has a local_dma_lkey which can be used as the lkey value for local
230 struct ib_pd
*ib_alloc_pd(struct ib_device
*device
)
234 pd
= device
->alloc_pd(device
, NULL
, NULL
);
241 atomic_set(&pd
->usecnt
, 0);
243 if (device
->attrs
.device_cap_flags
& IB_DEVICE_LOCAL_DMA_LKEY
)
244 pd
->local_dma_lkey
= device
->local_dma_lkey
;
248 mr
= ib_get_dma_mr(pd
, IB_ACCESS_LOCAL_WRITE
);
251 return (struct ib_pd
*)mr
;
255 pd
->local_dma_lkey
= pd
->local_mr
->lkey
;
259 EXPORT_SYMBOL(ib_alloc_pd
);
262 * ib_dealloc_pd - Deallocates a protection domain.
263 * @pd: The protection domain to deallocate.
265 * It is an error to call this function while any resources in the pd still
266 * exist. The caller is responsible to synchronously destroy them and
267 * guarantee no new allocations will happen.
269 void ib_dealloc_pd(struct ib_pd
*pd
)
274 ret
= ib_dereg_mr(pd
->local_mr
);
279 /* uverbs manipulates usecnt with proper locking, while the kabi
280 requires the caller to guarantee we can't race here. */
281 WARN_ON(atomic_read(&pd
->usecnt
));
283 /* Making delalloc_pd a void return is a WIP, no driver should return
285 ret
= pd
->device
->dealloc_pd(pd
);
286 WARN_ONCE(ret
, "Infiniband HW driver failed dealloc_pd");
288 EXPORT_SYMBOL(ib_dealloc_pd
);
290 /* Address handles */
292 struct ib_ah
*ib_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
)
296 ah
= pd
->device
->create_ah(pd
, ah_attr
);
299 ah
->device
= pd
->device
;
302 atomic_inc(&pd
->usecnt
);
307 EXPORT_SYMBOL(ib_create_ah
);
309 static int ib_get_header_version(const union rdma_network_hdr
*hdr
)
311 const struct iphdr
*ip4h
= (struct iphdr
*)&hdr
->roce4grh
;
312 struct iphdr ip4h_checked
;
313 const struct ipv6hdr
*ip6h
= (struct ipv6hdr
*)&hdr
->ibgrh
;
315 /* If it's IPv6, the version must be 6, otherwise, the first
316 * 20 bytes (before the IPv4 header) are garbled.
318 if (ip6h
->version
!= 6)
319 return (ip4h
->version
== 4) ? 4 : 0;
320 /* version may be 6 or 4 because the first 20 bytes could be garbled */
322 /* RoCE v2 requires no options, thus header length
329 * We can't write on scattered buffers so we need to copy to
332 memcpy(&ip4h_checked
, ip4h
, sizeof(ip4h_checked
));
333 ip4h_checked
.check
= 0;
334 ip4h_checked
.check
= ip_fast_csum((u8
*)&ip4h_checked
, 5);
335 /* if IPv4 header checksum is OK, believe it */
336 if (ip4h
->check
== ip4h_checked
.check
)
341 static enum rdma_network_type
ib_get_net_type_by_grh(struct ib_device
*device
,
343 const struct ib_grh
*grh
)
347 if (rdma_protocol_ib(device
, port_num
))
348 return RDMA_NETWORK_IB
;
350 grh_version
= ib_get_header_version((union rdma_network_hdr
*)grh
);
352 if (grh_version
== 4)
353 return RDMA_NETWORK_IPV4
;
355 if (grh
->next_hdr
== IPPROTO_UDP
)
356 return RDMA_NETWORK_IPV6
;
358 return RDMA_NETWORK_ROCE_V1
;
361 struct find_gid_index_context
{
363 enum ib_gid_type gid_type
;
366 static bool find_gid_index(const union ib_gid
*gid
,
367 const struct ib_gid_attr
*gid_attr
,
370 struct find_gid_index_context
*ctx
=
371 (struct find_gid_index_context
*)context
;
373 if (ctx
->gid_type
!= gid_attr
->gid_type
)
376 if ((!!(ctx
->vlan_id
!= 0xffff) == !is_vlan_dev(gid_attr
->ndev
)) ||
377 (is_vlan_dev(gid_attr
->ndev
) &&
378 vlan_dev_vlan_id(gid_attr
->ndev
) != ctx
->vlan_id
))
384 static int get_sgid_index_from_eth(struct ib_device
*device
, u8 port_num
,
385 u16 vlan_id
, const union ib_gid
*sgid
,
386 enum ib_gid_type gid_type
,
389 struct find_gid_index_context context
= {.vlan_id
= vlan_id
,
390 .gid_type
= gid_type
};
392 return ib_find_gid_by_filter(device
, sgid
, port_num
, find_gid_index
,
393 &context
, gid_index
);
396 static int get_gids_from_rdma_hdr(union rdma_network_hdr
*hdr
,
397 enum rdma_network_type net_type
,
398 union ib_gid
*sgid
, union ib_gid
*dgid
)
400 struct sockaddr_in src_in
;
401 struct sockaddr_in dst_in
;
402 __be32 src_saddr
, dst_saddr
;
407 if (net_type
== RDMA_NETWORK_IPV4
) {
408 memcpy(&src_in
.sin_addr
.s_addr
,
409 &hdr
->roce4grh
.saddr
, 4);
410 memcpy(&dst_in
.sin_addr
.s_addr
,
411 &hdr
->roce4grh
.daddr
, 4);
412 src_saddr
= src_in
.sin_addr
.s_addr
;
413 dst_saddr
= dst_in
.sin_addr
.s_addr
;
414 ipv6_addr_set_v4mapped(src_saddr
,
415 (struct in6_addr
*)sgid
);
416 ipv6_addr_set_v4mapped(dst_saddr
,
417 (struct in6_addr
*)dgid
);
419 } else if (net_type
== RDMA_NETWORK_IPV6
||
420 net_type
== RDMA_NETWORK_IB
) {
421 *dgid
= hdr
->ibgrh
.dgid
;
422 *sgid
= hdr
->ibgrh
.sgid
;
429 int ib_init_ah_from_wc(struct ib_device
*device
, u8 port_num
,
430 const struct ib_wc
*wc
, const struct ib_grh
*grh
,
431 struct ib_ah_attr
*ah_attr
)
436 enum rdma_network_type net_type
= RDMA_NETWORK_IB
;
437 enum ib_gid_type gid_type
= IB_GID_TYPE_IB
;
442 memset(ah_attr
, 0, sizeof *ah_attr
);
443 if (rdma_cap_eth_ah(device
, port_num
)) {
444 if (wc
->wc_flags
& IB_WC_WITH_NETWORK_HDR_TYPE
)
445 net_type
= wc
->network_hdr_type
;
447 net_type
= ib_get_net_type_by_grh(device
, port_num
, grh
);
448 gid_type
= ib_network_to_gid_type(net_type
);
450 ret
= get_gids_from_rdma_hdr((union rdma_network_hdr
*)grh
, net_type
,
455 if (rdma_protocol_roce(device
, port_num
)) {
457 u16 vlan_id
= wc
->wc_flags
& IB_WC_WITH_VLAN
?
458 wc
->vlan_id
: 0xffff;
459 struct net_device
*idev
;
460 struct net_device
*resolved_dev
;
462 if (!(wc
->wc_flags
& IB_WC_GRH
))
465 if (!device
->get_netdev
)
468 idev
= device
->get_netdev(device
, port_num
);
472 ret
= rdma_addr_find_l2_eth_by_grh(&dgid
, &sgid
,
474 wc
->wc_flags
& IB_WC_WITH_VLAN
?
476 &if_index
, &hoplimit
);
482 resolved_dev
= dev_get_by_index(&init_net
, if_index
);
483 if (resolved_dev
->flags
& IFF_LOOPBACK
) {
484 dev_put(resolved_dev
);
486 dev_hold(resolved_dev
);
489 if (resolved_dev
!= idev
&& !rdma_is_upper_dev_rcu(idev
,
494 dev_put(resolved_dev
);
498 ret
= get_sgid_index_from_eth(device
, port_num
, vlan_id
,
499 &dgid
, gid_type
, &gid_index
);
504 ah_attr
->dlid
= wc
->slid
;
505 ah_attr
->sl
= wc
->sl
;
506 ah_attr
->src_path_bits
= wc
->dlid_path_bits
;
507 ah_attr
->port_num
= port_num
;
509 if (wc
->wc_flags
& IB_WC_GRH
) {
510 ah_attr
->ah_flags
= IB_AH_GRH
;
511 ah_attr
->grh
.dgid
= sgid
;
513 if (!rdma_cap_eth_ah(device
, port_num
)) {
514 ret
= ib_find_cached_gid_by_port(device
, &dgid
,
522 ah_attr
->grh
.sgid_index
= (u8
) gid_index
;
523 flow_class
= be32_to_cpu(grh
->version_tclass_flow
);
524 ah_attr
->grh
.flow_label
= flow_class
& 0xFFFFF;
525 ah_attr
->grh
.hop_limit
= hoplimit
;
526 ah_attr
->grh
.traffic_class
= (flow_class
>> 20) & 0xFF;
530 EXPORT_SYMBOL(ib_init_ah_from_wc
);
532 struct ib_ah
*ib_create_ah_from_wc(struct ib_pd
*pd
, const struct ib_wc
*wc
,
533 const struct ib_grh
*grh
, u8 port_num
)
535 struct ib_ah_attr ah_attr
;
538 ret
= ib_init_ah_from_wc(pd
->device
, port_num
, wc
, grh
, &ah_attr
);
542 return ib_create_ah(pd
, &ah_attr
);
544 EXPORT_SYMBOL(ib_create_ah_from_wc
);
546 int ib_modify_ah(struct ib_ah
*ah
, struct ib_ah_attr
*ah_attr
)
548 return ah
->device
->modify_ah
?
549 ah
->device
->modify_ah(ah
, ah_attr
) :
552 EXPORT_SYMBOL(ib_modify_ah
);
554 int ib_query_ah(struct ib_ah
*ah
, struct ib_ah_attr
*ah_attr
)
556 return ah
->device
->query_ah
?
557 ah
->device
->query_ah(ah
, ah_attr
) :
560 EXPORT_SYMBOL(ib_query_ah
);
562 int ib_destroy_ah(struct ib_ah
*ah
)
568 ret
= ah
->device
->destroy_ah(ah
);
570 atomic_dec(&pd
->usecnt
);
574 EXPORT_SYMBOL(ib_destroy_ah
);
576 /* Shared receive queues */
578 struct ib_srq
*ib_create_srq(struct ib_pd
*pd
,
579 struct ib_srq_init_attr
*srq_init_attr
)
583 if (!pd
->device
->create_srq
)
584 return ERR_PTR(-ENOSYS
);
586 srq
= pd
->device
->create_srq(pd
, srq_init_attr
, NULL
);
589 srq
->device
= pd
->device
;
592 srq
->event_handler
= srq_init_attr
->event_handler
;
593 srq
->srq_context
= srq_init_attr
->srq_context
;
594 srq
->srq_type
= srq_init_attr
->srq_type
;
595 if (srq
->srq_type
== IB_SRQT_XRC
) {
596 srq
->ext
.xrc
.xrcd
= srq_init_attr
->ext
.xrc
.xrcd
;
597 srq
->ext
.xrc
.cq
= srq_init_attr
->ext
.xrc
.cq
;
598 atomic_inc(&srq
->ext
.xrc
.xrcd
->usecnt
);
599 atomic_inc(&srq
->ext
.xrc
.cq
->usecnt
);
601 atomic_inc(&pd
->usecnt
);
602 atomic_set(&srq
->usecnt
, 0);
607 EXPORT_SYMBOL(ib_create_srq
);
609 int ib_modify_srq(struct ib_srq
*srq
,
610 struct ib_srq_attr
*srq_attr
,
611 enum ib_srq_attr_mask srq_attr_mask
)
613 return srq
->device
->modify_srq
?
614 srq
->device
->modify_srq(srq
, srq_attr
, srq_attr_mask
, NULL
) :
617 EXPORT_SYMBOL(ib_modify_srq
);
619 int ib_query_srq(struct ib_srq
*srq
,
620 struct ib_srq_attr
*srq_attr
)
622 return srq
->device
->query_srq
?
623 srq
->device
->query_srq(srq
, srq_attr
) : -ENOSYS
;
625 EXPORT_SYMBOL(ib_query_srq
);
627 int ib_destroy_srq(struct ib_srq
*srq
)
630 enum ib_srq_type srq_type
;
631 struct ib_xrcd
*uninitialized_var(xrcd
);
632 struct ib_cq
*uninitialized_var(cq
);
635 if (atomic_read(&srq
->usecnt
))
639 srq_type
= srq
->srq_type
;
640 if (srq_type
== IB_SRQT_XRC
) {
641 xrcd
= srq
->ext
.xrc
.xrcd
;
642 cq
= srq
->ext
.xrc
.cq
;
645 ret
= srq
->device
->destroy_srq(srq
);
647 atomic_dec(&pd
->usecnt
);
648 if (srq_type
== IB_SRQT_XRC
) {
649 atomic_dec(&xrcd
->usecnt
);
650 atomic_dec(&cq
->usecnt
);
656 EXPORT_SYMBOL(ib_destroy_srq
);
660 static void __ib_shared_qp_event_handler(struct ib_event
*event
, void *context
)
662 struct ib_qp
*qp
= context
;
665 spin_lock_irqsave(&qp
->device
->event_handler_lock
, flags
);
666 list_for_each_entry(event
->element
.qp
, &qp
->open_list
, open_list
)
667 if (event
->element
.qp
->event_handler
)
668 event
->element
.qp
->event_handler(event
, event
->element
.qp
->qp_context
);
669 spin_unlock_irqrestore(&qp
->device
->event_handler_lock
, flags
);
672 static void __ib_insert_xrcd_qp(struct ib_xrcd
*xrcd
, struct ib_qp
*qp
)
674 mutex_lock(&xrcd
->tgt_qp_mutex
);
675 list_add(&qp
->xrcd_list
, &xrcd
->tgt_qp_list
);
676 mutex_unlock(&xrcd
->tgt_qp_mutex
);
679 static struct ib_qp
*__ib_open_qp(struct ib_qp
*real_qp
,
680 void (*event_handler
)(struct ib_event
*, void *),
686 qp
= kzalloc(sizeof *qp
, GFP_KERNEL
);
688 return ERR_PTR(-ENOMEM
);
690 qp
->real_qp
= real_qp
;
691 atomic_inc(&real_qp
->usecnt
);
692 qp
->device
= real_qp
->device
;
693 qp
->event_handler
= event_handler
;
694 qp
->qp_context
= qp_context
;
695 qp
->qp_num
= real_qp
->qp_num
;
696 qp
->qp_type
= real_qp
->qp_type
;
698 spin_lock_irqsave(&real_qp
->device
->event_handler_lock
, flags
);
699 list_add(&qp
->open_list
, &real_qp
->open_list
);
700 spin_unlock_irqrestore(&real_qp
->device
->event_handler_lock
, flags
);
705 struct ib_qp
*ib_open_qp(struct ib_xrcd
*xrcd
,
706 struct ib_qp_open_attr
*qp_open_attr
)
708 struct ib_qp
*qp
, *real_qp
;
710 if (qp_open_attr
->qp_type
!= IB_QPT_XRC_TGT
)
711 return ERR_PTR(-EINVAL
);
713 qp
= ERR_PTR(-EINVAL
);
714 mutex_lock(&xrcd
->tgt_qp_mutex
);
715 list_for_each_entry(real_qp
, &xrcd
->tgt_qp_list
, xrcd_list
) {
716 if (real_qp
->qp_num
== qp_open_attr
->qp_num
) {
717 qp
= __ib_open_qp(real_qp
, qp_open_attr
->event_handler
,
718 qp_open_attr
->qp_context
);
722 mutex_unlock(&xrcd
->tgt_qp_mutex
);
725 EXPORT_SYMBOL(ib_open_qp
);
727 static struct ib_qp
*ib_create_xrc_qp(struct ib_qp
*qp
,
728 struct ib_qp_init_attr
*qp_init_attr
)
730 struct ib_qp
*real_qp
= qp
;
732 qp
->event_handler
= __ib_shared_qp_event_handler
;
735 qp
->send_cq
= qp
->recv_cq
= NULL
;
737 qp
->xrcd
= qp_init_attr
->xrcd
;
738 atomic_inc(&qp_init_attr
->xrcd
->usecnt
);
739 INIT_LIST_HEAD(&qp
->open_list
);
741 qp
= __ib_open_qp(real_qp
, qp_init_attr
->event_handler
,
742 qp_init_attr
->qp_context
);
744 __ib_insert_xrcd_qp(qp_init_attr
->xrcd
, real_qp
);
746 real_qp
->device
->destroy_qp(real_qp
);
750 struct ib_qp
*ib_create_qp(struct ib_pd
*pd
,
751 struct ib_qp_init_attr
*qp_init_attr
)
753 struct ib_device
*device
= pd
? pd
->device
: qp_init_attr
->xrcd
->device
;
758 * If the callers is using the RDMA API calculate the resources
759 * needed for the RDMA READ/WRITE operations.
761 * Note that these callers need to pass in a port number.
763 if (qp_init_attr
->cap
.max_rdma_ctxs
)
764 rdma_rw_init_qp(device
, qp_init_attr
);
766 qp
= device
->create_qp(pd
, qp_init_attr
, NULL
);
773 qp
->qp_type
= qp_init_attr
->qp_type
;
775 atomic_set(&qp
->usecnt
, 0);
777 spin_lock_init(&qp
->mr_lock
);
778 INIT_LIST_HEAD(&qp
->rdma_mrs
);
780 if (qp_init_attr
->qp_type
== IB_QPT_XRC_TGT
)
781 return ib_create_xrc_qp(qp
, qp_init_attr
);
783 qp
->event_handler
= qp_init_attr
->event_handler
;
784 qp
->qp_context
= qp_init_attr
->qp_context
;
785 if (qp_init_attr
->qp_type
== IB_QPT_XRC_INI
) {
789 qp
->recv_cq
= qp_init_attr
->recv_cq
;
790 atomic_inc(&qp_init_attr
->recv_cq
->usecnt
);
791 qp
->srq
= qp_init_attr
->srq
;
793 atomic_inc(&qp_init_attr
->srq
->usecnt
);
797 qp
->send_cq
= qp_init_attr
->send_cq
;
800 atomic_inc(&pd
->usecnt
);
801 atomic_inc(&qp_init_attr
->send_cq
->usecnt
);
803 if (qp_init_attr
->cap
.max_rdma_ctxs
) {
804 ret
= rdma_rw_init_mrs(qp
, qp_init_attr
);
806 pr_err("failed to init MR pool ret= %d\n", ret
);
814 EXPORT_SYMBOL(ib_create_qp
);
816 static const struct {
818 enum ib_qp_attr_mask req_param
[IB_QPT_MAX
];
819 enum ib_qp_attr_mask opt_param
[IB_QPT_MAX
];
820 } qp_state_table
[IB_QPS_ERR
+ 1][IB_QPS_ERR
+ 1] = {
822 [IB_QPS_RESET
] = { .valid
= 1 },
826 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
829 [IB_QPT_RAW_PACKET
] = IB_QP_PORT
,
830 [IB_QPT_UC
] = (IB_QP_PKEY_INDEX
|
833 [IB_QPT_RC
] = (IB_QP_PKEY_INDEX
|
836 [IB_QPT_XRC_INI
] = (IB_QP_PKEY_INDEX
|
839 [IB_QPT_XRC_TGT
] = (IB_QP_PKEY_INDEX
|
842 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
844 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
850 [IB_QPS_RESET
] = { .valid
= 1 },
851 [IB_QPS_ERR
] = { .valid
= 1 },
855 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
858 [IB_QPT_UC
] = (IB_QP_PKEY_INDEX
|
861 [IB_QPT_RC
] = (IB_QP_PKEY_INDEX
|
864 [IB_QPT_XRC_INI
] = (IB_QP_PKEY_INDEX
|
867 [IB_QPT_XRC_TGT
] = (IB_QP_PKEY_INDEX
|
870 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
872 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
879 [IB_QPT_UC
] = (IB_QP_AV
|
883 [IB_QPT_RC
] = (IB_QP_AV
|
887 IB_QP_MAX_DEST_RD_ATOMIC
|
888 IB_QP_MIN_RNR_TIMER
),
889 [IB_QPT_XRC_INI
] = (IB_QP_AV
|
893 [IB_QPT_XRC_TGT
] = (IB_QP_AV
|
897 IB_QP_MAX_DEST_RD_ATOMIC
|
898 IB_QP_MIN_RNR_TIMER
),
901 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
903 [IB_QPT_UC
] = (IB_QP_ALT_PATH
|
906 [IB_QPT_RC
] = (IB_QP_ALT_PATH
|
909 [IB_QPT_XRC_INI
] = (IB_QP_ALT_PATH
|
912 [IB_QPT_XRC_TGT
] = (IB_QP_ALT_PATH
|
915 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
917 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
923 [IB_QPS_RESET
] = { .valid
= 1 },
924 [IB_QPS_ERR
] = { .valid
= 1 },
928 [IB_QPT_UD
] = IB_QP_SQ_PSN
,
929 [IB_QPT_UC
] = IB_QP_SQ_PSN
,
930 [IB_QPT_RC
] = (IB_QP_TIMEOUT
|
934 IB_QP_MAX_QP_RD_ATOMIC
),
935 [IB_QPT_XRC_INI
] = (IB_QP_TIMEOUT
|
939 IB_QP_MAX_QP_RD_ATOMIC
),
940 [IB_QPT_XRC_TGT
] = (IB_QP_TIMEOUT
|
942 [IB_QPT_SMI
] = IB_QP_SQ_PSN
,
943 [IB_QPT_GSI
] = IB_QP_SQ_PSN
,
946 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
948 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
951 IB_QP_PATH_MIG_STATE
),
952 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
955 IB_QP_MIN_RNR_TIMER
|
956 IB_QP_PATH_MIG_STATE
),
957 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
960 IB_QP_PATH_MIG_STATE
),
961 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
964 IB_QP_MIN_RNR_TIMER
|
965 IB_QP_PATH_MIG_STATE
),
966 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
968 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
974 [IB_QPS_RESET
] = { .valid
= 1 },
975 [IB_QPS_ERR
] = { .valid
= 1 },
979 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
981 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
984 IB_QP_PATH_MIG_STATE
),
985 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
988 IB_QP_PATH_MIG_STATE
|
989 IB_QP_MIN_RNR_TIMER
),
990 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
993 IB_QP_PATH_MIG_STATE
),
994 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
997 IB_QP_PATH_MIG_STATE
|
998 IB_QP_MIN_RNR_TIMER
),
999 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
1001 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
1008 [IB_QPT_UD
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
1009 [IB_QPT_UC
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
1010 [IB_QPT_RC
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
1011 [IB_QPT_XRC_INI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
1012 [IB_QPT_XRC_TGT
] = IB_QP_EN_SQD_ASYNC_NOTIFY
, /* ??? */
1013 [IB_QPT_SMI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
1014 [IB_QPT_GSI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
1019 [IB_QPS_RESET
] = { .valid
= 1 },
1020 [IB_QPS_ERR
] = { .valid
= 1 },
1024 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
1026 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
1028 IB_QP_ACCESS_FLAGS
|
1029 IB_QP_PATH_MIG_STATE
),
1030 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
1032 IB_QP_ACCESS_FLAGS
|
1033 IB_QP_MIN_RNR_TIMER
|
1034 IB_QP_PATH_MIG_STATE
),
1035 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
1037 IB_QP_ACCESS_FLAGS
|
1038 IB_QP_PATH_MIG_STATE
),
1039 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
1041 IB_QP_ACCESS_FLAGS
|
1042 IB_QP_MIN_RNR_TIMER
|
1043 IB_QP_PATH_MIG_STATE
),
1044 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
1046 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
1053 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
1055 [IB_QPT_UC
] = (IB_QP_AV
|
1057 IB_QP_ACCESS_FLAGS
|
1059 IB_QP_PATH_MIG_STATE
),
1060 [IB_QPT_RC
] = (IB_QP_PORT
|
1065 IB_QP_MAX_QP_RD_ATOMIC
|
1066 IB_QP_MAX_DEST_RD_ATOMIC
|
1068 IB_QP_ACCESS_FLAGS
|
1070 IB_QP_MIN_RNR_TIMER
|
1071 IB_QP_PATH_MIG_STATE
),
1072 [IB_QPT_XRC_INI
] = (IB_QP_PORT
|
1077 IB_QP_MAX_QP_RD_ATOMIC
|
1079 IB_QP_ACCESS_FLAGS
|
1081 IB_QP_PATH_MIG_STATE
),
1082 [IB_QPT_XRC_TGT
] = (IB_QP_PORT
|
1085 IB_QP_MAX_DEST_RD_ATOMIC
|
1087 IB_QP_ACCESS_FLAGS
|
1089 IB_QP_MIN_RNR_TIMER
|
1090 IB_QP_PATH_MIG_STATE
),
1091 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
1093 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
1099 [IB_QPS_RESET
] = { .valid
= 1 },
1100 [IB_QPS_ERR
] = { .valid
= 1 },
1104 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
1106 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
1107 IB_QP_ACCESS_FLAGS
),
1108 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
1110 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
1116 [IB_QPS_RESET
] = { .valid
= 1 },
1117 [IB_QPS_ERR
] = { .valid
= 1 }
1121 int ib_modify_qp_is_ok(enum ib_qp_state cur_state
, enum ib_qp_state next_state
,
1122 enum ib_qp_type type
, enum ib_qp_attr_mask mask
,
1123 enum rdma_link_layer ll
)
1125 enum ib_qp_attr_mask req_param
, opt_param
;
1127 if (cur_state
< 0 || cur_state
> IB_QPS_ERR
||
1128 next_state
< 0 || next_state
> IB_QPS_ERR
)
1131 if (mask
& IB_QP_CUR_STATE
&&
1132 cur_state
!= IB_QPS_RTR
&& cur_state
!= IB_QPS_RTS
&&
1133 cur_state
!= IB_QPS_SQD
&& cur_state
!= IB_QPS_SQE
)
1136 if (!qp_state_table
[cur_state
][next_state
].valid
)
1139 req_param
= qp_state_table
[cur_state
][next_state
].req_param
[type
];
1140 opt_param
= qp_state_table
[cur_state
][next_state
].opt_param
[type
];
1142 if ((mask
& req_param
) != req_param
)
1145 if (mask
& ~(req_param
| opt_param
| IB_QP_STATE
))
1150 EXPORT_SYMBOL(ib_modify_qp_is_ok
);
1152 int ib_resolve_eth_dmac(struct ib_qp
*qp
,
1153 struct ib_qp_attr
*qp_attr
, int *qp_attr_mask
)
1157 if (*qp_attr_mask
& IB_QP_AV
) {
1158 if (qp_attr
->ah_attr
.port_num
< rdma_start_port(qp
->device
) ||
1159 qp_attr
->ah_attr
.port_num
> rdma_end_port(qp
->device
))
1162 if (!rdma_cap_eth_ah(qp
->device
, qp_attr
->ah_attr
.port_num
))
1165 if (rdma_link_local_addr((struct in6_addr
*)qp_attr
->ah_attr
.grh
.dgid
.raw
)) {
1166 rdma_get_ll_mac((struct in6_addr
*)qp_attr
->ah_attr
.grh
.dgid
.raw
,
1167 qp_attr
->ah_attr
.dmac
);
1170 struct ib_gid_attr sgid_attr
;
1174 ret
= ib_query_gid(qp
->device
,
1175 qp_attr
->ah_attr
.port_num
,
1176 qp_attr
->ah_attr
.grh
.sgid_index
,
1179 if (ret
|| !sgid_attr
.ndev
) {
1185 ifindex
= sgid_attr
.ndev
->ifindex
;
1187 ret
= rdma_addr_find_l2_eth_by_grh(&sgid
,
1188 &qp_attr
->ah_attr
.grh
.dgid
,
1189 qp_attr
->ah_attr
.dmac
,
1190 NULL
, &ifindex
, &hop_limit
);
1192 dev_put(sgid_attr
.ndev
);
1194 qp_attr
->ah_attr
.grh
.hop_limit
= hop_limit
;
1200 EXPORT_SYMBOL(ib_resolve_eth_dmac
);
1203 int ib_modify_qp(struct ib_qp
*qp
,
1204 struct ib_qp_attr
*qp_attr
,
1209 ret
= ib_resolve_eth_dmac(qp
, qp_attr
, &qp_attr_mask
);
1213 return qp
->device
->modify_qp(qp
->real_qp
, qp_attr
, qp_attr_mask
, NULL
);
1215 EXPORT_SYMBOL(ib_modify_qp
);
1217 int ib_query_qp(struct ib_qp
*qp
,
1218 struct ib_qp_attr
*qp_attr
,
1220 struct ib_qp_init_attr
*qp_init_attr
)
1222 return qp
->device
->query_qp
?
1223 qp
->device
->query_qp(qp
->real_qp
, qp_attr
, qp_attr_mask
, qp_init_attr
) :
1226 EXPORT_SYMBOL(ib_query_qp
);
1228 int ib_close_qp(struct ib_qp
*qp
)
1230 struct ib_qp
*real_qp
;
1231 unsigned long flags
;
1233 real_qp
= qp
->real_qp
;
1237 spin_lock_irqsave(&real_qp
->device
->event_handler_lock
, flags
);
1238 list_del(&qp
->open_list
);
1239 spin_unlock_irqrestore(&real_qp
->device
->event_handler_lock
, flags
);
1241 atomic_dec(&real_qp
->usecnt
);
1246 EXPORT_SYMBOL(ib_close_qp
);
1248 static int __ib_destroy_shared_qp(struct ib_qp
*qp
)
1250 struct ib_xrcd
*xrcd
;
1251 struct ib_qp
*real_qp
;
1254 real_qp
= qp
->real_qp
;
1255 xrcd
= real_qp
->xrcd
;
1257 mutex_lock(&xrcd
->tgt_qp_mutex
);
1259 if (atomic_read(&real_qp
->usecnt
) == 0)
1260 list_del(&real_qp
->xrcd_list
);
1263 mutex_unlock(&xrcd
->tgt_qp_mutex
);
1266 ret
= ib_destroy_qp(real_qp
);
1268 atomic_dec(&xrcd
->usecnt
);
1270 __ib_insert_xrcd_qp(xrcd
, real_qp
);
1276 int ib_destroy_qp(struct ib_qp
*qp
)
1279 struct ib_cq
*scq
, *rcq
;
1283 WARN_ON_ONCE(qp
->mrs_used
> 0);
1285 if (atomic_read(&qp
->usecnt
))
1288 if (qp
->real_qp
!= qp
)
1289 return __ib_destroy_shared_qp(qp
);
1297 rdma_rw_cleanup_mrs(qp
);
1299 ret
= qp
->device
->destroy_qp(qp
);
1302 atomic_dec(&pd
->usecnt
);
1304 atomic_dec(&scq
->usecnt
);
1306 atomic_dec(&rcq
->usecnt
);
1308 atomic_dec(&srq
->usecnt
);
1313 EXPORT_SYMBOL(ib_destroy_qp
);
1315 /* Completion queues */
1317 struct ib_cq
*ib_create_cq(struct ib_device
*device
,
1318 ib_comp_handler comp_handler
,
1319 void (*event_handler
)(struct ib_event
*, void *),
1321 const struct ib_cq_init_attr
*cq_attr
)
1325 cq
= device
->create_cq(device
, cq_attr
, NULL
, NULL
);
1328 cq
->device
= device
;
1330 cq
->comp_handler
= comp_handler
;
1331 cq
->event_handler
= event_handler
;
1332 cq
->cq_context
= cq_context
;
1333 atomic_set(&cq
->usecnt
, 0);
1338 EXPORT_SYMBOL(ib_create_cq
);
1340 int ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
1342 return cq
->device
->modify_cq
?
1343 cq
->device
->modify_cq(cq
, cq_count
, cq_period
) : -ENOSYS
;
1345 EXPORT_SYMBOL(ib_modify_cq
);
1347 int ib_destroy_cq(struct ib_cq
*cq
)
1349 if (atomic_read(&cq
->usecnt
))
1352 return cq
->device
->destroy_cq(cq
);
1354 EXPORT_SYMBOL(ib_destroy_cq
);
1356 int ib_resize_cq(struct ib_cq
*cq
, int cqe
)
1358 return cq
->device
->resize_cq
?
1359 cq
->device
->resize_cq(cq
, cqe
, NULL
) : -ENOSYS
;
1361 EXPORT_SYMBOL(ib_resize_cq
);
1363 /* Memory regions */
1365 struct ib_mr
*ib_get_dma_mr(struct ib_pd
*pd
, int mr_access_flags
)
1370 err
= ib_check_mr_access(mr_access_flags
);
1372 return ERR_PTR(err
);
1374 mr
= pd
->device
->get_dma_mr(pd
, mr_access_flags
);
1377 mr
->device
= pd
->device
;
1380 atomic_inc(&pd
->usecnt
);
1381 mr
->need_inval
= false;
1386 EXPORT_SYMBOL(ib_get_dma_mr
);
1388 int ib_dereg_mr(struct ib_mr
*mr
)
1390 struct ib_pd
*pd
= mr
->pd
;
1393 ret
= mr
->device
->dereg_mr(mr
);
1395 atomic_dec(&pd
->usecnt
);
1399 EXPORT_SYMBOL(ib_dereg_mr
);
1402 * ib_alloc_mr() - Allocates a memory region
1403 * @pd: protection domain associated with the region
1404 * @mr_type: memory region type
1405 * @max_num_sg: maximum sg entries available for registration.
1408 * Memory registeration page/sg lists must not exceed max_num_sg.
1409 * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
1410 * max_num_sg * used_page_size.
1413 struct ib_mr
*ib_alloc_mr(struct ib_pd
*pd
,
1414 enum ib_mr_type mr_type
,
1419 if (!pd
->device
->alloc_mr
)
1420 return ERR_PTR(-ENOSYS
);
1422 mr
= pd
->device
->alloc_mr(pd
, mr_type
, max_num_sg
);
1424 mr
->device
= pd
->device
;
1427 atomic_inc(&pd
->usecnt
);
1428 mr
->need_inval
= false;
1433 EXPORT_SYMBOL(ib_alloc_mr
);
1435 /* "Fast" memory regions */
1437 struct ib_fmr
*ib_alloc_fmr(struct ib_pd
*pd
,
1438 int mr_access_flags
,
1439 struct ib_fmr_attr
*fmr_attr
)
1443 if (!pd
->device
->alloc_fmr
)
1444 return ERR_PTR(-ENOSYS
);
1446 fmr
= pd
->device
->alloc_fmr(pd
, mr_access_flags
, fmr_attr
);
1448 fmr
->device
= pd
->device
;
1450 atomic_inc(&pd
->usecnt
);
1455 EXPORT_SYMBOL(ib_alloc_fmr
);
1457 int ib_unmap_fmr(struct list_head
*fmr_list
)
1461 if (list_empty(fmr_list
))
1464 fmr
= list_entry(fmr_list
->next
, struct ib_fmr
, list
);
1465 return fmr
->device
->unmap_fmr(fmr_list
);
1467 EXPORT_SYMBOL(ib_unmap_fmr
);
1469 int ib_dealloc_fmr(struct ib_fmr
*fmr
)
1475 ret
= fmr
->device
->dealloc_fmr(fmr
);
1477 atomic_dec(&pd
->usecnt
);
1481 EXPORT_SYMBOL(ib_dealloc_fmr
);
1483 /* Multicast groups */
1485 int ib_attach_mcast(struct ib_qp
*qp
, union ib_gid
*gid
, u16 lid
)
1489 if (!qp
->device
->attach_mcast
)
1491 if (gid
->raw
[0] != 0xff || qp
->qp_type
!= IB_QPT_UD
)
1494 ret
= qp
->device
->attach_mcast(qp
, gid
, lid
);
1496 atomic_inc(&qp
->usecnt
);
1499 EXPORT_SYMBOL(ib_attach_mcast
);
1501 int ib_detach_mcast(struct ib_qp
*qp
, union ib_gid
*gid
, u16 lid
)
1505 if (!qp
->device
->detach_mcast
)
1507 if (gid
->raw
[0] != 0xff || qp
->qp_type
!= IB_QPT_UD
)
1510 ret
= qp
->device
->detach_mcast(qp
, gid
, lid
);
1512 atomic_dec(&qp
->usecnt
);
1515 EXPORT_SYMBOL(ib_detach_mcast
);
1517 struct ib_xrcd
*ib_alloc_xrcd(struct ib_device
*device
)
1519 struct ib_xrcd
*xrcd
;
1521 if (!device
->alloc_xrcd
)
1522 return ERR_PTR(-ENOSYS
);
1524 xrcd
= device
->alloc_xrcd(device
, NULL
, NULL
);
1525 if (!IS_ERR(xrcd
)) {
1526 xrcd
->device
= device
;
1528 atomic_set(&xrcd
->usecnt
, 0);
1529 mutex_init(&xrcd
->tgt_qp_mutex
);
1530 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
1535 EXPORT_SYMBOL(ib_alloc_xrcd
);
1537 int ib_dealloc_xrcd(struct ib_xrcd
*xrcd
)
1542 if (atomic_read(&xrcd
->usecnt
))
1545 while (!list_empty(&xrcd
->tgt_qp_list
)) {
1546 qp
= list_entry(xrcd
->tgt_qp_list
.next
, struct ib_qp
, xrcd_list
);
1547 ret
= ib_destroy_qp(qp
);
1552 return xrcd
->device
->dealloc_xrcd(xrcd
);
1554 EXPORT_SYMBOL(ib_dealloc_xrcd
);
1556 struct ib_flow
*ib_create_flow(struct ib_qp
*qp
,
1557 struct ib_flow_attr
*flow_attr
,
1560 struct ib_flow
*flow_id
;
1561 if (!qp
->device
->create_flow
)
1562 return ERR_PTR(-ENOSYS
);
1564 flow_id
= qp
->device
->create_flow(qp
, flow_attr
, domain
);
1565 if (!IS_ERR(flow_id
))
1566 atomic_inc(&qp
->usecnt
);
1569 EXPORT_SYMBOL(ib_create_flow
);
1571 int ib_destroy_flow(struct ib_flow
*flow_id
)
1574 struct ib_qp
*qp
= flow_id
->qp
;
1576 err
= qp
->device
->destroy_flow(flow_id
);
1578 atomic_dec(&qp
->usecnt
);
1581 EXPORT_SYMBOL(ib_destroy_flow
);
1583 int ib_check_mr_status(struct ib_mr
*mr
, u32 check_mask
,
1584 struct ib_mr_status
*mr_status
)
1586 return mr
->device
->check_mr_status
?
1587 mr
->device
->check_mr_status(mr
, check_mask
, mr_status
) : -ENOSYS
;
1589 EXPORT_SYMBOL(ib_check_mr_status
);
1591 int ib_set_vf_link_state(struct ib_device
*device
, int vf
, u8 port
,
1594 if (!device
->set_vf_link_state
)
1597 return device
->set_vf_link_state(device
, vf
, port
, state
);
1599 EXPORT_SYMBOL(ib_set_vf_link_state
);
1601 int ib_get_vf_config(struct ib_device
*device
, int vf
, u8 port
,
1602 struct ifla_vf_info
*info
)
1604 if (!device
->get_vf_config
)
1607 return device
->get_vf_config(device
, vf
, port
, info
);
1609 EXPORT_SYMBOL(ib_get_vf_config
);
1611 int ib_get_vf_stats(struct ib_device
*device
, int vf
, u8 port
,
1612 struct ifla_vf_stats
*stats
)
1614 if (!device
->get_vf_stats
)
1617 return device
->get_vf_stats(device
, vf
, port
, stats
);
1619 EXPORT_SYMBOL(ib_get_vf_stats
);
1621 int ib_set_vf_guid(struct ib_device
*device
, int vf
, u8 port
, u64 guid
,
1624 if (!device
->set_vf_guid
)
1627 return device
->set_vf_guid(device
, vf
, port
, guid
, type
);
1629 EXPORT_SYMBOL(ib_set_vf_guid
);
1632 * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
1633 * and set it the memory region.
1634 * @mr: memory region
1635 * @sg: dma mapped scatterlist
1636 * @sg_nents: number of entries in sg
1637 * @sg_offset: offset in bytes into sg
1638 * @page_size: page vector desired page size
1641 * - The first sg element is allowed to have an offset.
1642 * - Each sg element must be aligned to page_size (or physically
1643 * contiguous to the previous element). In case an sg element has a
1644 * non contiguous offset, the mapping prefix will not include it.
1645 * - The last sg element is allowed to have length less than page_size.
1646 * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
1647 * then only max_num_sg entries will be mapped.
1648 * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS_REG, non of these
1649 * constraints holds and the page_size argument is ignored.
1651 * Returns the number of sg elements that were mapped to the memory region.
1653 * After this completes successfully, the memory region
1654 * is ready for registration.
1656 int ib_map_mr_sg(struct ib_mr
*mr
, struct scatterlist
*sg
, int sg_nents
,
1657 unsigned int sg_offset
, unsigned int page_size
)
1659 if (unlikely(!mr
->device
->map_mr_sg
))
1662 mr
->page_size
= page_size
;
1664 return mr
->device
->map_mr_sg(mr
, sg
, sg_nents
, sg_offset
);
1666 EXPORT_SYMBOL(ib_map_mr_sg
);
1669 * ib_sg_to_pages() - Convert the largest prefix of a sg list
1671 * @mr: memory region
1672 * @sgl: dma mapped scatterlist
1673 * @sg_nents: number of entries in sg
1674 * @sg_offset: offset in bytes into sg
1675 * @set_page: driver page assignment function pointer
1677 * Core service helper for drivers to convert the largest
1678 * prefix of given sg list to a page vector. The sg list
1679 * prefix converted is the prefix that meet the requirements
1682 * Returns the number of sg elements that were assigned to
1685 int ib_sg_to_pages(struct ib_mr
*mr
, struct scatterlist
*sgl
, int sg_nents
,
1686 unsigned int sg_offset
, int (*set_page
)(struct ib_mr
*, u64
))
1688 struct scatterlist
*sg
;
1689 u64 last_end_dma_addr
= 0;
1690 unsigned int last_page_off
= 0;
1691 u64 page_mask
= ~((u64
)mr
->page_size
- 1);
1694 mr
->iova
= sg_dma_address(&sgl
[0]) + sg_offset
;
1697 for_each_sg(sgl
, sg
, sg_nents
, i
) {
1698 u64 dma_addr
= sg_dma_address(sg
) + sg_offset
;
1699 unsigned int dma_len
= sg_dma_len(sg
) - sg_offset
;
1700 u64 end_dma_addr
= dma_addr
+ dma_len
;
1701 u64 page_addr
= dma_addr
& page_mask
;
1704 * For the second and later elements, check whether either the
1705 * end of element i-1 or the start of element i is not aligned
1706 * on a page boundary.
1708 if (i
&& (last_page_off
!= 0 || page_addr
!= dma_addr
)) {
1709 /* Stop mapping if there is a gap. */
1710 if (last_end_dma_addr
!= dma_addr
)
1714 * Coalesce this element with the last. If it is small
1715 * enough just update mr->length. Otherwise start
1716 * mapping from the next page.
1722 ret
= set_page(mr
, page_addr
);
1723 if (unlikely(ret
< 0))
1726 page_addr
+= mr
->page_size
;
1727 } while (page_addr
< end_dma_addr
);
1729 mr
->length
+= dma_len
;
1730 last_end_dma_addr
= end_dma_addr
;
1731 last_page_off
= end_dma_addr
& ~page_mask
;
1738 EXPORT_SYMBOL(ib_sg_to_pages
);
1740 struct ib_drain_cqe
{
1742 struct completion done
;
1745 static void ib_drain_qp_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1747 struct ib_drain_cqe
*cqe
= container_of(wc
->wr_cqe
, struct ib_drain_cqe
,
1750 complete(&cqe
->done
);
1754 * Post a WR and block until its completion is reaped for the SQ.
1756 static void __ib_drain_sq(struct ib_qp
*qp
)
1758 struct ib_qp_attr attr
= { .qp_state
= IB_QPS_ERR
};
1759 struct ib_drain_cqe sdrain
;
1760 struct ib_send_wr swr
= {}, *bad_swr
;
1763 if (qp
->send_cq
->poll_ctx
== IB_POLL_DIRECT
) {
1764 WARN_ONCE(qp
->send_cq
->poll_ctx
== IB_POLL_DIRECT
,
1765 "IB_POLL_DIRECT poll_ctx not supported for drain\n");
1769 swr
.wr_cqe
= &sdrain
.cqe
;
1770 sdrain
.cqe
.done
= ib_drain_qp_done
;
1771 init_completion(&sdrain
.done
);
1773 ret
= ib_modify_qp(qp
, &attr
, IB_QP_STATE
);
1775 WARN_ONCE(ret
, "failed to drain send queue: %d\n", ret
);
1779 ret
= ib_post_send(qp
, &swr
, &bad_swr
);
1781 WARN_ONCE(ret
, "failed to drain send queue: %d\n", ret
);
1785 wait_for_completion(&sdrain
.done
);
1789 * Post a WR and block until its completion is reaped for the RQ.
1791 static void __ib_drain_rq(struct ib_qp
*qp
)
1793 struct ib_qp_attr attr
= { .qp_state
= IB_QPS_ERR
};
1794 struct ib_drain_cqe rdrain
;
1795 struct ib_recv_wr rwr
= {}, *bad_rwr
;
1798 if (qp
->recv_cq
->poll_ctx
== IB_POLL_DIRECT
) {
1799 WARN_ONCE(qp
->recv_cq
->poll_ctx
== IB_POLL_DIRECT
,
1800 "IB_POLL_DIRECT poll_ctx not supported for drain\n");
1804 rwr
.wr_cqe
= &rdrain
.cqe
;
1805 rdrain
.cqe
.done
= ib_drain_qp_done
;
1806 init_completion(&rdrain
.done
);
1808 ret
= ib_modify_qp(qp
, &attr
, IB_QP_STATE
);
1810 WARN_ONCE(ret
, "failed to drain recv queue: %d\n", ret
);
1814 ret
= ib_post_recv(qp
, &rwr
, &bad_rwr
);
1816 WARN_ONCE(ret
, "failed to drain recv queue: %d\n", ret
);
1820 wait_for_completion(&rdrain
.done
);
1824 * ib_drain_sq() - Block until all SQ CQEs have been consumed by the
1826 * @qp: queue pair to drain
1828 * If the device has a provider-specific drain function, then
1829 * call that. Otherwise call the generic drain function
1834 * ensure there is room in the CQ and SQ for the drain work request and
1837 * allocate the CQ using ib_alloc_cq() and the CQ poll context cannot be
1840 * ensure that there are no other contexts that are posting WRs concurrently.
1841 * Otherwise the drain is not guaranteed.
1843 void ib_drain_sq(struct ib_qp
*qp
)
1845 if (qp
->device
->drain_sq
)
1846 qp
->device
->drain_sq(qp
);
1850 EXPORT_SYMBOL(ib_drain_sq
);
1853 * ib_drain_rq() - Block until all RQ CQEs have been consumed by the
1855 * @qp: queue pair to drain
1857 * If the device has a provider-specific drain function, then
1858 * call that. Otherwise call the generic drain function
1863 * ensure there is room in the CQ and RQ for the drain work request and
1866 * allocate the CQ using ib_alloc_cq() and the CQ poll context cannot be
1869 * ensure that there are no other contexts that are posting WRs concurrently.
1870 * Otherwise the drain is not guaranteed.
1872 void ib_drain_rq(struct ib_qp
*qp
)
1874 if (qp
->device
->drain_rq
)
1875 qp
->device
->drain_rq(qp
);
1879 EXPORT_SYMBOL(ib_drain_rq
);
1882 * ib_drain_qp() - Block until all CQEs have been consumed by the
1883 * application on both the RQ and SQ.
1884 * @qp: queue pair to drain
1888 * ensure there is room in the CQ(s), SQ, and RQ for drain work requests
1891 * allocate the CQs using ib_alloc_cq() and the CQ poll context cannot be
1894 * ensure that there are no other contexts that are posting WRs concurrently.
1895 * Otherwise the drain is not guaranteed.
1897 void ib_drain_qp(struct ib_qp
*qp
)
1903 EXPORT_SYMBOL(ib_drain_qp
);