2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * $Id: ipoib_verbs.c 1349 2004-12-16 21:09:43Z roland $
36 #include <rdma/ib_cache.h>
40 int ipoib_mcast_attach(struct net_device
*dev
, u16 mlid
, union ib_gid
*mgid
)
42 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
43 struct ib_qp_attr
*qp_attr
;
49 qp_attr
= kmalloc(sizeof *qp_attr
, GFP_KERNEL
);
53 if (ib_find_cached_pkey(priv
->ca
, priv
->port
, priv
->pkey
, &pkey_index
)) {
54 clear_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
58 set_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
60 /* set correct QKey for QP */
61 qp_attr
->qkey
= priv
->qkey
;
62 attr_mask
= IB_QP_QKEY
;
63 ret
= ib_modify_qp(priv
->qp
, qp_attr
, attr_mask
);
65 ipoib_warn(priv
, "failed to modify QP, ret = %d\n", ret
);
69 /* attach QP to multicast group */
70 down(&priv
->mcast_mutex
);
71 ret
= ib_attach_mcast(priv
->qp
, mgid
, mlid
);
72 up(&priv
->mcast_mutex
);
74 ipoib_warn(priv
, "failed to attach to multicast group, ret = %d\n", ret
);
81 int ipoib_mcast_detach(struct net_device
*dev
, u16 mlid
, union ib_gid
*mgid
)
83 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
86 down(&priv
->mcast_mutex
);
87 ret
= ib_detach_mcast(priv
->qp
, mgid
, mlid
);
88 up(&priv
->mcast_mutex
);
90 ipoib_warn(priv
, "ib_detach_mcast failed (result = %d)\n", ret
);
95 int ipoib_init_qp(struct net_device
*dev
)
97 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
100 struct ib_qp_attr qp_attr
;
104 * Search through the port P_Key table for the requested pkey value.
105 * The port has to be assigned to the respective IB partition in
108 ret
= ib_find_cached_pkey(priv
->ca
, priv
->port
, priv
->pkey
, &pkey_index
);
110 clear_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
113 set_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
115 qp_attr
.qp_state
= IB_QPS_INIT
;
117 qp_attr
.port_num
= priv
->port
;
118 qp_attr
.pkey_index
= pkey_index
;
124 ret
= ib_modify_qp(priv
->qp
, &qp_attr
, attr_mask
);
126 ipoib_warn(priv
, "failed to modify QP to init, ret = %d\n", ret
);
130 qp_attr
.qp_state
= IB_QPS_RTR
;
131 /* Can't set this in a INIT->RTR transition */
132 attr_mask
&= ~IB_QP_PORT
;
133 ret
= ib_modify_qp(priv
->qp
, &qp_attr
, attr_mask
);
135 ipoib_warn(priv
, "failed to modify QP to RTR, ret = %d\n", ret
);
139 qp_attr
.qp_state
= IB_QPS_RTS
;
141 attr_mask
|= IB_QP_SQ_PSN
;
142 attr_mask
&= ~IB_QP_PKEY_INDEX
;
143 ret
= ib_modify_qp(priv
->qp
, &qp_attr
, attr_mask
);
145 ipoib_warn(priv
, "failed to modify QP to RTS, ret = %d\n", ret
);
152 qp_attr
.qp_state
= IB_QPS_RESET
;
153 if (ib_modify_qp(priv
->qp
, &qp_attr
, IB_QP_STATE
))
154 ipoib_warn(priv
, "Failed to modify QP to RESET state\n");
159 int ipoib_transport_dev_init(struct net_device
*dev
, struct ib_device
*ca
)
161 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
162 struct ib_qp_init_attr init_attr
= {
164 .max_send_wr
= IPOIB_TX_RING_SIZE
,
165 .max_recv_wr
= IPOIB_RX_RING_SIZE
,
169 .sq_sig_type
= IB_SIGNAL_ALL_WR
,
173 priv
->pd
= ib_alloc_pd(priv
->ca
);
174 if (IS_ERR(priv
->pd
)) {
175 printk(KERN_WARNING
"%s: failed to allocate PD\n", ca
->name
);
179 priv
->cq
= ib_create_cq(priv
->ca
, ipoib_ib_completion
, NULL
, dev
,
180 IPOIB_TX_RING_SIZE
+ IPOIB_RX_RING_SIZE
+ 1);
181 if (IS_ERR(priv
->cq
)) {
182 printk(KERN_WARNING
"%s: failed to create CQ\n", ca
->name
);
186 if (ib_req_notify_cq(priv
->cq
, IB_CQ_NEXT_COMP
))
189 priv
->mr
= ib_get_dma_mr(priv
->pd
, IB_ACCESS_LOCAL_WRITE
);
190 if (IS_ERR(priv
->mr
)) {
191 printk(KERN_WARNING
"%s: ib_get_dma_mr failed\n", ca
->name
);
195 init_attr
.send_cq
= priv
->cq
;
196 init_attr
.recv_cq
= priv
->cq
,
198 priv
->qp
= ib_create_qp(priv
->pd
, &init_attr
);
199 if (IS_ERR(priv
->qp
)) {
200 printk(KERN_WARNING
"%s: failed to create QP\n", ca
->name
);
204 priv
->dev
->dev_addr
[1] = (priv
->qp
->qp_num
>> 16) & 0xff;
205 priv
->dev
->dev_addr
[2] = (priv
->qp
->qp_num
>> 8) & 0xff;
206 priv
->dev
->dev_addr
[3] = (priv
->qp
->qp_num
) & 0xff;
208 priv
->tx_sge
.lkey
= priv
->mr
->lkey
;
210 priv
->tx_wr
.opcode
= IB_WR_SEND
;
211 priv
->tx_wr
.sg_list
= &priv
->tx_sge
;
212 priv
->tx_wr
.num_sge
= 1;
213 priv
->tx_wr
.send_flags
= IB_SEND_SIGNALED
;
218 ib_dereg_mr(priv
->mr
);
221 ib_destroy_cq(priv
->cq
);
224 ib_dealloc_pd(priv
->pd
);
228 void ipoib_transport_dev_cleanup(struct net_device
*dev
)
230 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
233 if (ib_destroy_qp(priv
->qp
))
234 ipoib_warn(priv
, "ib_qp_destroy failed\n");
237 clear_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
240 if (ib_dereg_mr(priv
->mr
))
241 ipoib_warn(priv
, "ib_dereg_mr failed\n");
243 if (ib_destroy_cq(priv
->cq
))
244 ipoib_warn(priv
, "ib_cq_destroy failed\n");
246 if (ib_dealloc_pd(priv
->pd
))
247 ipoib_warn(priv
, "ib_dealloc_pd failed\n");
250 void ipoib_event(struct ib_event_handler
*handler
,
251 struct ib_event
*record
)
253 struct ipoib_dev_priv
*priv
=
254 container_of(handler
, struct ipoib_dev_priv
, event_handler
);
256 if (record
->event
== IB_EVENT_PORT_ACTIVE
||
257 record
->event
== IB_EVENT_LID_CHANGE
||
258 record
->event
== IB_EVENT_SM_CHANGE
) {
259 ipoib_dbg(priv
, "Port active event\n");
260 schedule_work(&priv
->flush_task
);