Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. | |
2a1d9b7f | 3 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. |
1da177e4 LT |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | * | |
33 | * $Id: ipoib_verbs.c 1349 2004-12-16 21:09:43Z roland $ | |
34 | */ | |
35 | ||
a4d61e84 | 36 | #include <rdma/ib_cache.h> |
1da177e4 LT |
37 | |
38 | #include "ipoib.h" | |
39 | ||
40 | int ipoib_mcast_attach(struct net_device *dev, u16 mlid, union ib_gid *mgid) | |
41 | { | |
42 | struct ipoib_dev_priv *priv = netdev_priv(dev); | |
43 | struct ib_qp_attr *qp_attr; | |
1da177e4 LT |
44 | int ret; |
45 | u16 pkey_index; | |
46 | ||
47 | ret = -ENOMEM; | |
48 | qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL); | |
49 | if (!qp_attr) | |
50 | goto out; | |
51 | ||
52 | if (ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index)) { | |
53 | clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); | |
54 | ret = -ENXIO; | |
55 | goto out; | |
56 | } | |
57 | set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); | |
58 | ||
59 | /* set correct QKey for QP */ | |
60 | qp_attr->qkey = priv->qkey; | |
3bc12e75 | 61 | ret = ib_modify_qp(priv->qp, qp_attr, IB_QP_QKEY); |
1da177e4 LT |
62 | if (ret) { |
63 | ipoib_warn(priv, "failed to modify QP, ret = %d\n", ret); | |
64 | goto out; | |
65 | } | |
66 | ||
67 | /* attach QP to multicast group */ | |
95ed644f | 68 | mutex_lock(&priv->mcast_mutex); |
1da177e4 | 69 | ret = ib_attach_mcast(priv->qp, mgid, mlid); |
95ed644f | 70 | mutex_unlock(&priv->mcast_mutex); |
1da177e4 LT |
71 | if (ret) |
72 | ipoib_warn(priv, "failed to attach to multicast group, ret = %d\n", ret); | |
73 | ||
74 | out: | |
75 | kfree(qp_attr); | |
76 | return ret; | |
77 | } | |
78 | ||
79 | int ipoib_mcast_detach(struct net_device *dev, u16 mlid, union ib_gid *mgid) | |
80 | { | |
81 | struct ipoib_dev_priv *priv = netdev_priv(dev); | |
82 | int ret; | |
83 | ||
95ed644f | 84 | mutex_lock(&priv->mcast_mutex); |
1da177e4 | 85 | ret = ib_detach_mcast(priv->qp, mgid, mlid); |
95ed644f | 86 | mutex_unlock(&priv->mcast_mutex); |
1da177e4 LT |
87 | if (ret) |
88 | ipoib_warn(priv, "ib_detach_mcast failed (result = %d)\n", ret); | |
89 | ||
90 | return ret; | |
91 | } | |
92 | ||
5b6810e0 | 93 | int ipoib_init_qp(struct net_device *dev) |
1da177e4 LT |
94 | { |
95 | struct ipoib_dev_priv *priv = netdev_priv(dev); | |
96 | int ret; | |
97 | u16 pkey_index; | |
98 | struct ib_qp_attr qp_attr; | |
99 | int attr_mask; | |
100 | ||
101 | /* | |
102 | * Search through the port P_Key table for the requested pkey value. | |
103 | * The port has to be assigned to the respective IB partition in | |
104 | * advance. | |
105 | */ | |
106 | ret = ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index); | |
107 | if (ret) { | |
108 | clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); | |
109 | return ret; | |
110 | } | |
111 | set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); | |
112 | ||
113 | qp_attr.qp_state = IB_QPS_INIT; | |
114 | qp_attr.qkey = 0; | |
115 | qp_attr.port_num = priv->port; | |
116 | qp_attr.pkey_index = pkey_index; | |
117 | attr_mask = | |
118 | IB_QP_QKEY | | |
119 | IB_QP_PORT | | |
120 | IB_QP_PKEY_INDEX | | |
121 | IB_QP_STATE; | |
122 | ret = ib_modify_qp(priv->qp, &qp_attr, attr_mask); | |
123 | if (ret) { | |
124 | ipoib_warn(priv, "failed to modify QP to init, ret = %d\n", ret); | |
125 | goto out_fail; | |
126 | } | |
127 | ||
128 | qp_attr.qp_state = IB_QPS_RTR; | |
129 | /* Can't set this in a INIT->RTR transition */ | |
130 | attr_mask &= ~IB_QP_PORT; | |
131 | ret = ib_modify_qp(priv->qp, &qp_attr, attr_mask); | |
132 | if (ret) { | |
133 | ipoib_warn(priv, "failed to modify QP to RTR, ret = %d\n", ret); | |
134 | goto out_fail; | |
135 | } | |
136 | ||
137 | qp_attr.qp_state = IB_QPS_RTS; | |
138 | qp_attr.sq_psn = 0; | |
139 | attr_mask |= IB_QP_SQ_PSN; | |
140 | attr_mask &= ~IB_QP_PKEY_INDEX; | |
141 | ret = ib_modify_qp(priv->qp, &qp_attr, attr_mask); | |
142 | if (ret) { | |
143 | ipoib_warn(priv, "failed to modify QP to RTS, ret = %d\n", ret); | |
144 | goto out_fail; | |
145 | } | |
146 | ||
147 | return 0; | |
148 | ||
149 | out_fail: | |
5b6810e0 RD |
150 | qp_attr.qp_state = IB_QPS_RESET; |
151 | if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE)) | |
152 | ipoib_warn(priv, "Failed to modify QP to RESET state\n"); | |
1da177e4 | 153 | |
5b6810e0 | 154 | return ret; |
1da177e4 LT |
155 | } |
156 | ||
157 | int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) | |
158 | { | |
159 | struct ipoib_dev_priv *priv = netdev_priv(dev); | |
160 | struct ib_qp_init_attr init_attr = { | |
161 | .cap = { | |
0f485251 SM |
162 | .max_send_wr = ipoib_sendq_size, |
163 | .max_recv_wr = ipoib_recvq_size, | |
1da177e4 LT |
164 | .max_send_sge = 1, |
165 | .max_recv_sge = 1 | |
166 | }, | |
167 | .sq_sig_type = IB_SIGNAL_ALL_WR, | |
168 | .qp_type = IB_QPT_UD | |
169 | }; | |
170 | ||
839fcaba MT |
171 | int ret, size; |
172 | ||
1da177e4 LT |
173 | priv->pd = ib_alloc_pd(priv->ca); |
174 | if (IS_ERR(priv->pd)) { | |
175 | printk(KERN_WARNING "%s: failed to allocate PD\n", ca->name); | |
176 | return -ENODEV; | |
177 | } | |
178 | ||
839fcaba MT |
179 | priv->mr = ib_get_dma_mr(priv->pd, IB_ACCESS_LOCAL_WRITE); |
180 | if (IS_ERR(priv->mr)) { | |
181 | printk(KERN_WARNING "%s: ib_get_dma_mr failed\n", ca->name); | |
182 | goto out_free_pd; | |
183 | } | |
184 | ||
185 | size = ipoib_sendq_size + ipoib_recvq_size + 1; | |
186 | ret = ipoib_cm_dev_init(dev); | |
187 | if (!ret) | |
188 | size += ipoib_recvq_size; | |
189 | ||
190 | priv->cq = ib_create_cq(priv->ca, ipoib_ib_completion, NULL, dev, size); | |
1da177e4 LT |
191 | if (IS_ERR(priv->cq)) { |
192 | printk(KERN_WARNING "%s: failed to create CQ\n", ca->name); | |
839fcaba | 193 | goto out_free_mr; |
1da177e4 LT |
194 | } |
195 | ||
196 | if (ib_req_notify_cq(priv->cq, IB_CQ_NEXT_COMP)) | |
197 | goto out_free_cq; | |
198 | ||
1da177e4 LT |
199 | init_attr.send_cq = priv->cq; |
200 | init_attr.recv_cq = priv->cq, | |
201 | ||
202 | priv->qp = ib_create_qp(priv->pd, &init_attr); | |
203 | if (IS_ERR(priv->qp)) { | |
204 | printk(KERN_WARNING "%s: failed to create QP\n", ca->name); | |
839fcaba | 205 | goto out_free_cq; |
1da177e4 LT |
206 | } |
207 | ||
208 | priv->dev->dev_addr[1] = (priv->qp->qp_num >> 16) & 0xff; | |
209 | priv->dev->dev_addr[2] = (priv->qp->qp_num >> 8) & 0xff; | |
210 | priv->dev->dev_addr[3] = (priv->qp->qp_num ) & 0xff; | |
211 | ||
212 | priv->tx_sge.lkey = priv->mr->lkey; | |
213 | ||
214 | priv->tx_wr.opcode = IB_WR_SEND; | |
215 | priv->tx_wr.sg_list = &priv->tx_sge; | |
216 | priv->tx_wr.num_sge = 1; | |
217 | priv->tx_wr.send_flags = IB_SEND_SIGNALED; | |
218 | ||
219 | return 0; | |
220 | ||
1da177e4 LT |
221 | out_free_cq: |
222 | ib_destroy_cq(priv->cq); | |
223 | ||
839fcaba MT |
224 | out_free_mr: |
225 | ib_dereg_mr(priv->mr); | |
226 | ||
1da177e4 LT |
227 | out_free_pd: |
228 | ib_dealloc_pd(priv->pd); | |
229 | return -ENODEV; | |
230 | } | |
231 | ||
232 | void ipoib_transport_dev_cleanup(struct net_device *dev) | |
233 | { | |
234 | struct ipoib_dev_priv *priv = netdev_priv(dev); | |
235 | ||
236 | if (priv->qp) { | |
237 | if (ib_destroy_qp(priv->qp)) | |
238 | ipoib_warn(priv, "ib_qp_destroy failed\n"); | |
239 | ||
240 | priv->qp = NULL; | |
241 | clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); | |
242 | } | |
243 | ||
1da177e4 LT |
244 | if (ib_destroy_cq(priv->cq)) |
245 | ipoib_warn(priv, "ib_cq_destroy failed\n"); | |
246 | ||
839fcaba MT |
247 | ipoib_cm_dev_cleanup(dev); |
248 | ||
249 | if (ib_dereg_mr(priv->mr)) | |
250 | ipoib_warn(priv, "ib_dereg_mr failed\n"); | |
251 | ||
1da177e4 LT |
252 | if (ib_dealloc_pd(priv->pd)) |
253 | ipoib_warn(priv, "ib_dealloc_pd failed\n"); | |
254 | } | |
255 | ||
256 | void ipoib_event(struct ib_event_handler *handler, | |
257 | struct ib_event *record) | |
258 | { | |
259 | struct ipoib_dev_priv *priv = | |
260 | container_of(handler, struct ipoib_dev_priv, event_handler); | |
261 | ||
4e37b956 | 262 | if (record->event == IB_EVENT_PORT_ERR || |
7a343d4c | 263 | record->event == IB_EVENT_PKEY_CHANGE || |
4e37b956 | 264 | record->event == IB_EVENT_PORT_ACTIVE || |
1da177e4 | 265 | record->event == IB_EVENT_LID_CHANGE || |
508e4341 LA |
266 | record->event == IB_EVENT_SM_CHANGE || |
267 | record->event == IB_EVENT_CLIENT_REREGISTER) { | |
4e37b956 | 268 | ipoib_dbg(priv, "Port state change event\n"); |
0b3ea082 | 269 | queue_work(ipoib_workqueue, &priv->flush_task); |
1da177e4 LT |
270 | } |
271 | } |