2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <asm-generic/kmap_types.h>
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/errno.h>
37 #include <linux/pci.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/slab.h>
40 #include <linux/io-mapping.h>
41 #include <linux/sched.h>
42 #include <rdma/ib_user_verbs.h>
43 #include <rdma/ib_smi.h>
44 #include <rdma/ib_umem.h>
48 #define DRIVER_NAME "mlx5_ib"
49 #define DRIVER_VERSION "2.2-1"
50 #define DRIVER_RELDATE "Feb 2014"
52 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
53 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
54 MODULE_LICENSE("Dual BSD/GPL");
55 MODULE_VERSION(DRIVER_VERSION
);
57 static int deprecated_prof_sel
= 2;
58 module_param_named(prof_sel
, deprecated_prof_sel
, int, 0444);
59 MODULE_PARM_DESC(prof_sel
, "profile selector. Deprecated here. Moved to module mlx5_core");
61 static char mlx5_version
[] =
62 DRIVER_NAME
": Mellanox Connect-IB Infiniband driver v"
63 DRIVER_VERSION
" (" DRIVER_RELDATE
")\n";
65 int mlx5_vector2eqn(struct mlx5_ib_dev
*dev
, int vector
, int *eqn
, int *irqn
)
67 struct mlx5_eq_table
*table
= &dev
->mdev
->priv
.eq_table
;
68 struct mlx5_eq
*eq
, *n
;
71 spin_lock(&table
->lock
);
72 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
73 if (eq
->index
== vector
) {
80 spin_unlock(&table
->lock
);
85 static int alloc_comp_eqs(struct mlx5_ib_dev
*dev
)
87 struct mlx5_eq_table
*table
= &dev
->mdev
->priv
.eq_table
;
88 char name
[MLX5_MAX_EQ_NAME
];
89 struct mlx5_eq
*eq
, *n
;
95 INIT_LIST_HEAD(&dev
->eqs_list
);
96 ncomp_vec
= table
->num_comp_vectors
;
97 nent
= MLX5_COMP_EQ_SIZE
;
98 for (i
= 0; i
< ncomp_vec
; i
++) {
99 eq
= kzalloc(sizeof(*eq
), GFP_KERNEL
);
105 snprintf(name
, MLX5_MAX_EQ_NAME
, "mlx5_comp%d", i
);
106 err
= mlx5_create_map_eq(dev
->mdev
, eq
,
107 i
+ MLX5_EQ_VEC_COMP_BASE
, nent
, 0,
108 name
, &dev
->mdev
->priv
.uuari
.uars
[0]);
113 mlx5_ib_dbg(dev
, "allocated completion EQN %d\n", eq
->eqn
);
115 spin_lock(&table
->lock
);
116 list_add_tail(&eq
->list
, &dev
->eqs_list
);
117 spin_unlock(&table
->lock
);
120 dev
->num_comp_vectors
= ncomp_vec
;
124 spin_lock(&table
->lock
);
125 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
127 spin_unlock(&table
->lock
);
128 if (mlx5_destroy_unmap_eq(dev
->mdev
, eq
))
129 mlx5_ib_warn(dev
, "failed to destroy EQ 0x%x\n", eq
->eqn
);
131 spin_lock(&table
->lock
);
133 spin_unlock(&table
->lock
);
137 static void free_comp_eqs(struct mlx5_ib_dev
*dev
)
139 struct mlx5_eq_table
*table
= &dev
->mdev
->priv
.eq_table
;
140 struct mlx5_eq
*eq
, *n
;
142 spin_lock(&table
->lock
);
143 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
145 spin_unlock(&table
->lock
);
146 if (mlx5_destroy_unmap_eq(dev
->mdev
, eq
))
147 mlx5_ib_warn(dev
, "failed to destroy EQ 0x%x\n", eq
->eqn
);
149 spin_lock(&table
->lock
);
151 spin_unlock(&table
->lock
);
154 static int mlx5_ib_query_device(struct ib_device
*ibdev
,
155 struct ib_device_attr
*props
)
157 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
158 struct ib_smp
*in_mad
= NULL
;
159 struct ib_smp
*out_mad
= NULL
;
160 struct mlx5_general_caps
*gen
;
166 gen
= &dev
->mdev
->caps
.gen
;
167 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
168 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
169 if (!in_mad
|| !out_mad
)
172 init_query_mad(in_mad
);
173 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
175 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
179 memset(props
, 0, sizeof(*props
));
181 props
->fw_ver
= ((u64
)fw_rev_maj(dev
->mdev
) << 32) |
182 (fw_rev_min(dev
->mdev
) << 16) |
183 fw_rev_sub(dev
->mdev
);
184 props
->device_cap_flags
= IB_DEVICE_CHANGE_PHY_PORT
|
185 IB_DEVICE_PORT_ACTIVE_EVENT
|
186 IB_DEVICE_SYS_IMAGE_GUID
|
187 IB_DEVICE_RC_RNR_NAK_GEN
;
189 if (flags
& MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR
)
190 props
->device_cap_flags
|= IB_DEVICE_BAD_PKEY_CNTR
;
191 if (flags
& MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR
)
192 props
->device_cap_flags
|= IB_DEVICE_BAD_QKEY_CNTR
;
193 if (flags
& MLX5_DEV_CAP_FLAG_APM
)
194 props
->device_cap_flags
|= IB_DEVICE_AUTO_PATH_MIG
;
195 props
->device_cap_flags
|= IB_DEVICE_LOCAL_DMA_LKEY
;
196 if (flags
& MLX5_DEV_CAP_FLAG_XRC
)
197 props
->device_cap_flags
|= IB_DEVICE_XRC
;
198 props
->device_cap_flags
|= IB_DEVICE_MEM_MGT_EXTENSIONS
;
199 if (flags
& MLX5_DEV_CAP_FLAG_SIG_HAND_OVER
) {
200 props
->device_cap_flags
|= IB_DEVICE_SIGNATURE_HANDOVER
;
201 /* At this stage no support for signature handover */
202 props
->sig_prot_cap
= IB_PROT_T10DIF_TYPE_1
|
203 IB_PROT_T10DIF_TYPE_2
|
204 IB_PROT_T10DIF_TYPE_3
;
205 props
->sig_guard_cap
= IB_GUARD_T10DIF_CRC
|
206 IB_GUARD_T10DIF_CSUM
;
208 if (flags
& MLX5_DEV_CAP_FLAG_BLOCK_MCAST
)
209 props
->device_cap_flags
|= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK
;
211 props
->vendor_id
= be32_to_cpup((__be32
*)(out_mad
->data
+ 36)) &
213 props
->vendor_part_id
= be16_to_cpup((__be16
*)(out_mad
->data
+ 30));
214 props
->hw_ver
= be32_to_cpup((__be32
*)(out_mad
->data
+ 32));
215 memcpy(&props
->sys_image_guid
, out_mad
->data
+ 4, 8);
217 props
->max_mr_size
= ~0ull;
218 props
->page_size_cap
= gen
->min_page_sz
;
219 props
->max_qp
= 1 << gen
->log_max_qp
;
220 props
->max_qp_wr
= gen
->max_wqes
;
221 max_rq_sg
= gen
->max_rq_desc_sz
/ sizeof(struct mlx5_wqe_data_seg
);
222 max_sq_sg
= (gen
->max_sq_desc_sz
- sizeof(struct mlx5_wqe_ctrl_seg
)) /
223 sizeof(struct mlx5_wqe_data_seg
);
224 props
->max_sge
= min(max_rq_sg
, max_sq_sg
);
225 props
->max_cq
= 1 << gen
->log_max_cq
;
226 props
->max_cqe
= gen
->max_cqes
- 1;
227 props
->max_mr
= 1 << gen
->log_max_mkey
;
228 props
->max_pd
= 1 << gen
->log_max_pd
;
229 props
->max_qp_rd_atom
= 1 << gen
->log_max_ra_req_qp
;
230 props
->max_qp_init_rd_atom
= 1 << gen
->log_max_ra_res_qp
;
231 props
->max_srq
= 1 << gen
->log_max_srq
;
232 props
->max_srq_wr
= gen
->max_srq_wqes
- 1;
233 props
->local_ca_ack_delay
= gen
->local_ca_ack_delay
;
234 props
->max_res_rd_atom
= props
->max_qp_rd_atom
* props
->max_qp
;
235 props
->max_srq_sge
= max_rq_sg
- 1;
236 props
->max_fast_reg_page_list_len
= (unsigned int)-1;
237 props
->local_ca_ack_delay
= gen
->local_ca_ack_delay
;
238 props
->atomic_cap
= IB_ATOMIC_NONE
;
239 props
->masked_atomic_cap
= IB_ATOMIC_NONE
;
240 props
->max_pkeys
= be16_to_cpup((__be16
*)(out_mad
->data
+ 28));
241 props
->max_mcast_grp
= 1 << gen
->log_max_mcg
;
242 props
->max_mcast_qp_attach
= gen
->max_qp_mcg
;
243 props
->max_total_mcast_qp_attach
= props
->max_mcast_qp_attach
*
244 props
->max_mcast_grp
;
245 props
->max_map_per_fmr
= INT_MAX
; /* no limit in ConnectIB */
247 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
248 if (dev
->mdev
->caps
.gen
.flags
& MLX5_DEV_CAP_FLAG_ON_DMND_PG
)
249 props
->device_cap_flags
|= IB_DEVICE_ON_DEMAND_PAGING
;
250 props
->odp_caps
= dev
->odp_caps
;
260 int mlx5_ib_query_port(struct ib_device
*ibdev
, u8 port
,
261 struct ib_port_attr
*props
)
263 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
264 struct ib_smp
*in_mad
= NULL
;
265 struct ib_smp
*out_mad
= NULL
;
266 struct mlx5_general_caps
*gen
;
267 int ext_active_speed
;
270 gen
= &dev
->mdev
->caps
.gen
;
271 if (port
< 1 || port
> gen
->num_ports
) {
272 mlx5_ib_warn(dev
, "invalid port number %d\n", port
);
276 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
277 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
278 if (!in_mad
|| !out_mad
)
281 memset(props
, 0, sizeof(*props
));
283 init_query_mad(in_mad
);
284 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
285 in_mad
->attr_mod
= cpu_to_be32(port
);
287 err
= mlx5_MAD_IFC(dev
, 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
289 mlx5_ib_warn(dev
, "err %d\n", err
);
294 props
->lid
= be16_to_cpup((__be16
*)(out_mad
->data
+ 16));
295 props
->lmc
= out_mad
->data
[34] & 0x7;
296 props
->sm_lid
= be16_to_cpup((__be16
*)(out_mad
->data
+ 18));
297 props
->sm_sl
= out_mad
->data
[36] & 0xf;
298 props
->state
= out_mad
->data
[32] & 0xf;
299 props
->phys_state
= out_mad
->data
[33] >> 4;
300 props
->port_cap_flags
= be32_to_cpup((__be32
*)(out_mad
->data
+ 20));
301 props
->gid_tbl_len
= out_mad
->data
[50];
302 props
->max_msg_sz
= 1 << gen
->log_max_msg
;
303 props
->pkey_tbl_len
= gen
->port
[port
- 1].pkey_table_len
;
304 props
->bad_pkey_cntr
= be16_to_cpup((__be16
*)(out_mad
->data
+ 46));
305 props
->qkey_viol_cntr
= be16_to_cpup((__be16
*)(out_mad
->data
+ 48));
306 props
->active_width
= out_mad
->data
[31] & 0xf;
307 props
->active_speed
= out_mad
->data
[35] >> 4;
308 props
->max_mtu
= out_mad
->data
[41] & 0xf;
309 props
->active_mtu
= out_mad
->data
[36] >> 4;
310 props
->subnet_timeout
= out_mad
->data
[51] & 0x1f;
311 props
->max_vl_num
= out_mad
->data
[37] >> 4;
312 props
->init_type_reply
= out_mad
->data
[41] >> 4;
314 /* Check if extended speeds (EDR/FDR/...) are supported */
315 if (props
->port_cap_flags
& IB_PORT_EXTENDED_SPEEDS_SUP
) {
316 ext_active_speed
= out_mad
->data
[62] >> 4;
318 switch (ext_active_speed
) {
320 props
->active_speed
= 16; /* FDR */
323 props
->active_speed
= 32; /* EDR */
328 /* If reported active speed is QDR, check if is FDR-10 */
329 if (props
->active_speed
== 4) {
330 if (gen
->ext_port_cap
[port
- 1] &
331 MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO
) {
332 init_query_mad(in_mad
);
333 in_mad
->attr_id
= MLX5_ATTR_EXTENDED_PORT_INFO
;
334 in_mad
->attr_mod
= cpu_to_be32(port
);
336 err
= mlx5_MAD_IFC(dev
, 1, 1, port
,
337 NULL
, NULL
, in_mad
, out_mad
);
341 /* Checking LinkSpeedActive for FDR-10 */
342 if (out_mad
->data
[15] & 0x1)
343 props
->active_speed
= 8;
354 static int mlx5_ib_query_gid(struct ib_device
*ibdev
, u8 port
, int index
,
357 struct ib_smp
*in_mad
= NULL
;
358 struct ib_smp
*out_mad
= NULL
;
361 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
362 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
363 if (!in_mad
|| !out_mad
)
366 init_query_mad(in_mad
);
367 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
368 in_mad
->attr_mod
= cpu_to_be32(port
);
370 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
374 memcpy(gid
->raw
, out_mad
->data
+ 8, 8);
376 init_query_mad(in_mad
);
377 in_mad
->attr_id
= IB_SMP_ATTR_GUID_INFO
;
378 in_mad
->attr_mod
= cpu_to_be32(index
/ 8);
380 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
384 memcpy(gid
->raw
+ 8, out_mad
->data
+ (index
% 8) * 8, 8);
392 static int mlx5_ib_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
395 struct ib_smp
*in_mad
= NULL
;
396 struct ib_smp
*out_mad
= NULL
;
399 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
400 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
401 if (!in_mad
|| !out_mad
)
404 init_query_mad(in_mad
);
405 in_mad
->attr_id
= IB_SMP_ATTR_PKEY_TABLE
;
406 in_mad
->attr_mod
= cpu_to_be32(index
/ 32);
408 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
412 *pkey
= be16_to_cpu(((__be16
*)out_mad
->data
)[index
% 32]);
420 struct mlx5_reg_node_desc
{
424 static int mlx5_ib_modify_device(struct ib_device
*ibdev
, int mask
,
425 struct ib_device_modify
*props
)
427 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
428 struct mlx5_reg_node_desc in
;
429 struct mlx5_reg_node_desc out
;
432 if (mask
& ~IB_DEVICE_MODIFY_NODE_DESC
)
435 if (!(mask
& IB_DEVICE_MODIFY_NODE_DESC
))
439 * If possible, pass node desc to FW, so it can generate
440 * a 144 trap. If cmd fails, just ignore.
442 memcpy(&in
, props
->node_desc
, 64);
443 err
= mlx5_core_access_reg(dev
->mdev
, &in
, sizeof(in
), &out
,
444 sizeof(out
), MLX5_REG_NODE_DESC
, 0, 1);
448 memcpy(ibdev
->node_desc
, props
->node_desc
, 64);
453 static int mlx5_ib_modify_port(struct ib_device
*ibdev
, u8 port
, int mask
,
454 struct ib_port_modify
*props
)
456 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
457 struct ib_port_attr attr
;
461 mutex_lock(&dev
->cap_mask_mutex
);
463 err
= mlx5_ib_query_port(ibdev
, port
, &attr
);
467 tmp
= (attr
.port_cap_flags
| props
->set_port_cap_mask
) &
468 ~props
->clr_port_cap_mask
;
470 err
= mlx5_set_port_caps(dev
->mdev
, port
, tmp
);
473 mutex_unlock(&dev
->cap_mask_mutex
);
477 static struct ib_ucontext
*mlx5_ib_alloc_ucontext(struct ib_device
*ibdev
,
478 struct ib_udata
*udata
)
480 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
481 struct mlx5_ib_alloc_ucontext_req_v2 req
;
482 struct mlx5_ib_alloc_ucontext_resp resp
;
483 struct mlx5_ib_ucontext
*context
;
484 struct mlx5_general_caps
*gen
;
485 struct mlx5_uuar_info
*uuari
;
486 struct mlx5_uar
*uars
;
495 gen
= &dev
->mdev
->caps
.gen
;
497 return ERR_PTR(-EAGAIN
);
499 memset(&req
, 0, sizeof(req
));
500 reqlen
= udata
->inlen
- sizeof(struct ib_uverbs_cmd_hdr
);
501 if (reqlen
== sizeof(struct mlx5_ib_alloc_ucontext_req
))
503 else if (reqlen
== sizeof(struct mlx5_ib_alloc_ucontext_req_v2
))
506 return ERR_PTR(-EINVAL
);
508 err
= ib_copy_from_udata(&req
, udata
, reqlen
);
512 if (req
.flags
|| req
.reserved
)
513 return ERR_PTR(-EINVAL
);
515 if (req
.total_num_uuars
> MLX5_MAX_UUARS
)
516 return ERR_PTR(-ENOMEM
);
518 if (req
.total_num_uuars
== 0)
519 return ERR_PTR(-EINVAL
);
521 req
.total_num_uuars
= ALIGN(req
.total_num_uuars
,
522 MLX5_NON_FP_BF_REGS_PER_PAGE
);
523 if (req
.num_low_latency_uuars
> req
.total_num_uuars
- 1)
524 return ERR_PTR(-EINVAL
);
526 num_uars
= req
.total_num_uuars
/ MLX5_NON_FP_BF_REGS_PER_PAGE
;
527 gross_uuars
= num_uars
* MLX5_BF_REGS_PER_PAGE
;
528 resp
.qp_tab_size
= 1 << gen
->log_max_qp
;
529 resp
.bf_reg_size
= gen
->bf_reg_size
;
530 resp
.cache_line_size
= L1_CACHE_BYTES
;
531 resp
.max_sq_desc_sz
= gen
->max_sq_desc_sz
;
532 resp
.max_rq_desc_sz
= gen
->max_rq_desc_sz
;
533 resp
.max_send_wqebb
= gen
->max_wqes
;
534 resp
.max_recv_wr
= gen
->max_wqes
;
535 resp
.max_srq_recv_wr
= gen
->max_srq_wqes
;
537 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
539 return ERR_PTR(-ENOMEM
);
541 uuari
= &context
->uuari
;
542 mutex_init(&uuari
->lock
);
543 uars
= kcalloc(num_uars
, sizeof(*uars
), GFP_KERNEL
);
549 uuari
->bitmap
= kcalloc(BITS_TO_LONGS(gross_uuars
),
550 sizeof(*uuari
->bitmap
),
552 if (!uuari
->bitmap
) {
557 * clear all fast path uuars
559 for (i
= 0; i
< gross_uuars
; i
++) {
561 if (uuarn
== 2 || uuarn
== 3)
562 set_bit(i
, uuari
->bitmap
);
565 uuari
->count
= kcalloc(gross_uuars
, sizeof(*uuari
->count
), GFP_KERNEL
);
571 for (i
= 0; i
< num_uars
; i
++) {
572 err
= mlx5_cmd_alloc_uar(dev
->mdev
, &uars
[i
].index
);
577 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
578 context
->ibucontext
.invalidate_range
= &mlx5_ib_invalidate_range
;
581 INIT_LIST_HEAD(&context
->db_page_list
);
582 mutex_init(&context
->db_page_mutex
);
584 resp
.tot_uuars
= req
.total_num_uuars
;
585 resp
.num_ports
= gen
->num_ports
;
586 err
= ib_copy_to_udata(udata
, &resp
,
587 sizeof(resp
) - sizeof(resp
.reserved
));
592 uuari
->num_low_latency_uuars
= req
.num_low_latency_uuars
;
594 uuari
->num_uars
= num_uars
;
595 return &context
->ibucontext
;
598 for (i
--; i
>= 0; i
--)
599 mlx5_cmd_free_uar(dev
->mdev
, uars
[i
].index
);
604 kfree(uuari
->bitmap
);
614 static int mlx5_ib_dealloc_ucontext(struct ib_ucontext
*ibcontext
)
616 struct mlx5_ib_ucontext
*context
= to_mucontext(ibcontext
);
617 struct mlx5_ib_dev
*dev
= to_mdev(ibcontext
->device
);
618 struct mlx5_uuar_info
*uuari
= &context
->uuari
;
621 for (i
= 0; i
< uuari
->num_uars
; i
++) {
622 if (mlx5_cmd_free_uar(dev
->mdev
, uuari
->uars
[i
].index
))
623 mlx5_ib_warn(dev
, "failed to free UAR 0x%x\n", uuari
->uars
[i
].index
);
627 kfree(uuari
->bitmap
);
634 static phys_addr_t
uar_index2pfn(struct mlx5_ib_dev
*dev
, int index
)
636 return (pci_resource_start(dev
->mdev
->pdev
, 0) >> PAGE_SHIFT
) + index
;
639 static int get_command(unsigned long offset
)
641 return (offset
>> MLX5_IB_MMAP_CMD_SHIFT
) & MLX5_IB_MMAP_CMD_MASK
;
644 static int get_arg(unsigned long offset
)
646 return offset
& ((1 << MLX5_IB_MMAP_CMD_SHIFT
) - 1);
649 static int get_index(unsigned long offset
)
651 return get_arg(offset
);
654 static int mlx5_ib_mmap(struct ib_ucontext
*ibcontext
, struct vm_area_struct
*vma
)
656 struct mlx5_ib_ucontext
*context
= to_mucontext(ibcontext
);
657 struct mlx5_ib_dev
*dev
= to_mdev(ibcontext
->device
);
658 struct mlx5_uuar_info
*uuari
= &context
->uuari
;
659 unsigned long command
;
663 command
= get_command(vma
->vm_pgoff
);
665 case MLX5_IB_MMAP_REGULAR_PAGE
:
666 if (vma
->vm_end
- vma
->vm_start
!= PAGE_SIZE
)
669 idx
= get_index(vma
->vm_pgoff
);
670 if (idx
>= uuari
->num_uars
)
673 pfn
= uar_index2pfn(dev
, uuari
->uars
[idx
].index
);
674 mlx5_ib_dbg(dev
, "uar idx 0x%lx, pfn 0x%llx\n", idx
,
675 (unsigned long long)pfn
);
677 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
678 if (io_remap_pfn_range(vma
, vma
->vm_start
, pfn
,
679 PAGE_SIZE
, vma
->vm_page_prot
))
682 mlx5_ib_dbg(dev
, "mapped WC at 0x%lx, PA 0x%llx\n",
684 (unsigned long long)pfn
<< PAGE_SHIFT
);
687 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES
:
697 static int alloc_pa_mkey(struct mlx5_ib_dev
*dev
, u32
*key
, u32 pdn
)
699 struct mlx5_create_mkey_mbox_in
*in
;
700 struct mlx5_mkey_seg
*seg
;
701 struct mlx5_core_mr mr
;
704 in
= kzalloc(sizeof(*in
), GFP_KERNEL
);
709 seg
->flags
= MLX5_PERM_LOCAL_READ
| MLX5_ACCESS_MODE_PA
;
710 seg
->flags_pd
= cpu_to_be32(pdn
| MLX5_MKEY_LEN64
);
711 seg
->qpn_mkey7_0
= cpu_to_be32(0xffffff << 8);
714 err
= mlx5_core_create_mkey(dev
->mdev
, &mr
, in
, sizeof(*in
),
717 mlx5_ib_warn(dev
, "failed to create mkey, %d\n", err
);
732 static void free_pa_mkey(struct mlx5_ib_dev
*dev
, u32 key
)
734 struct mlx5_core_mr mr
;
737 memset(&mr
, 0, sizeof(mr
));
739 err
= mlx5_core_destroy_mkey(dev
->mdev
, &mr
);
741 mlx5_ib_warn(dev
, "failed to destroy mkey 0x%x\n", key
);
744 static struct ib_pd
*mlx5_ib_alloc_pd(struct ib_device
*ibdev
,
745 struct ib_ucontext
*context
,
746 struct ib_udata
*udata
)
748 struct mlx5_ib_alloc_pd_resp resp
;
749 struct mlx5_ib_pd
*pd
;
752 pd
= kmalloc(sizeof(*pd
), GFP_KERNEL
);
754 return ERR_PTR(-ENOMEM
);
756 err
= mlx5_core_alloc_pd(to_mdev(ibdev
)->mdev
, &pd
->pdn
);
764 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
765 mlx5_core_dealloc_pd(to_mdev(ibdev
)->mdev
, pd
->pdn
);
767 return ERR_PTR(-EFAULT
);
770 err
= alloc_pa_mkey(to_mdev(ibdev
), &pd
->pa_lkey
, pd
->pdn
);
772 mlx5_core_dealloc_pd(to_mdev(ibdev
)->mdev
, pd
->pdn
);
781 static int mlx5_ib_dealloc_pd(struct ib_pd
*pd
)
783 struct mlx5_ib_dev
*mdev
= to_mdev(pd
->device
);
784 struct mlx5_ib_pd
*mpd
= to_mpd(pd
);
787 free_pa_mkey(mdev
, mpd
->pa_lkey
);
789 mlx5_core_dealloc_pd(mdev
->mdev
, mpd
->pdn
);
795 static int mlx5_ib_mcg_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
797 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
800 err
= mlx5_core_attach_mcg(dev
->mdev
, gid
, ibqp
->qp_num
);
802 mlx5_ib_warn(dev
, "failed attaching QPN 0x%x, MGID %pI6\n",
803 ibqp
->qp_num
, gid
->raw
);
808 static int mlx5_ib_mcg_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
810 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
813 err
= mlx5_core_detach_mcg(dev
->mdev
, gid
, ibqp
->qp_num
);
815 mlx5_ib_warn(dev
, "failed detaching QPN 0x%x, MGID %pI6\n",
816 ibqp
->qp_num
, gid
->raw
);
821 static int init_node_data(struct mlx5_ib_dev
*dev
)
823 struct ib_smp
*in_mad
= NULL
;
824 struct ib_smp
*out_mad
= NULL
;
827 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
828 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
829 if (!in_mad
|| !out_mad
)
832 init_query_mad(in_mad
);
833 in_mad
->attr_id
= IB_SMP_ATTR_NODE_DESC
;
835 err
= mlx5_MAD_IFC(dev
, 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
839 memcpy(dev
->ib_dev
.node_desc
, out_mad
->data
, 64);
841 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
843 err
= mlx5_MAD_IFC(dev
, 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
847 dev
->mdev
->rev_id
= be32_to_cpup((__be32
*)(out_mad
->data
+ 32));
848 memcpy(&dev
->ib_dev
.node_guid
, out_mad
->data
+ 12, 8);
856 static ssize_t
show_fw_pages(struct device
*device
, struct device_attribute
*attr
,
859 struct mlx5_ib_dev
*dev
=
860 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
862 return sprintf(buf
, "%d\n", dev
->mdev
->priv
.fw_pages
);
865 static ssize_t
show_reg_pages(struct device
*device
,
866 struct device_attribute
*attr
, char *buf
)
868 struct mlx5_ib_dev
*dev
=
869 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
871 return sprintf(buf
, "%d\n", atomic_read(&dev
->mdev
->priv
.reg_pages
));
874 static ssize_t
show_hca(struct device
*device
, struct device_attribute
*attr
,
877 struct mlx5_ib_dev
*dev
=
878 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
879 return sprintf(buf
, "MT%d\n", dev
->mdev
->pdev
->device
);
882 static ssize_t
show_fw_ver(struct device
*device
, struct device_attribute
*attr
,
885 struct mlx5_ib_dev
*dev
=
886 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
887 return sprintf(buf
, "%d.%d.%d\n", fw_rev_maj(dev
->mdev
),
888 fw_rev_min(dev
->mdev
), fw_rev_sub(dev
->mdev
));
891 static ssize_t
show_rev(struct device
*device
, struct device_attribute
*attr
,
894 struct mlx5_ib_dev
*dev
=
895 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
896 return sprintf(buf
, "%x\n", dev
->mdev
->rev_id
);
899 static ssize_t
show_board(struct device
*device
, struct device_attribute
*attr
,
902 struct mlx5_ib_dev
*dev
=
903 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
904 return sprintf(buf
, "%.*s\n", MLX5_BOARD_ID_LEN
,
905 dev
->mdev
->board_id
);
908 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
909 static DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
910 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
911 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
912 static DEVICE_ATTR(fw_pages
, S_IRUGO
, show_fw_pages
, NULL
);
913 static DEVICE_ATTR(reg_pages
, S_IRUGO
, show_reg_pages
, NULL
);
915 static struct device_attribute
*mlx5_class_attributes
[] = {
924 static void mlx5_ib_event(struct mlx5_core_dev
*dev
, void *context
,
925 enum mlx5_dev_event event
, unsigned long param
)
927 struct mlx5_ib_dev
*ibdev
= (struct mlx5_ib_dev
*)context
;
928 struct ib_event ibev
;
933 case MLX5_DEV_EVENT_SYS_ERROR
:
934 ibdev
->ib_active
= false;
935 ibev
.event
= IB_EVENT_DEVICE_FATAL
;
938 case MLX5_DEV_EVENT_PORT_UP
:
939 ibev
.event
= IB_EVENT_PORT_ACTIVE
;
943 case MLX5_DEV_EVENT_PORT_DOWN
:
944 ibev
.event
= IB_EVENT_PORT_ERR
;
948 case MLX5_DEV_EVENT_PORT_INITIALIZED
:
949 /* not used by ULPs */
952 case MLX5_DEV_EVENT_LID_CHANGE
:
953 ibev
.event
= IB_EVENT_LID_CHANGE
;
957 case MLX5_DEV_EVENT_PKEY_CHANGE
:
958 ibev
.event
= IB_EVENT_PKEY_CHANGE
;
962 case MLX5_DEV_EVENT_GUID_CHANGE
:
963 ibev
.event
= IB_EVENT_GID_CHANGE
;
967 case MLX5_DEV_EVENT_CLIENT_REREG
:
968 ibev
.event
= IB_EVENT_CLIENT_REREGISTER
;
973 ibev
.device
= &ibdev
->ib_dev
;
974 ibev
.element
.port_num
= port
;
976 if (port
< 1 || port
> ibdev
->num_ports
) {
977 mlx5_ib_warn(ibdev
, "warning: event on port %d\n", port
);
981 if (ibdev
->ib_active
)
982 ib_dispatch_event(&ibev
);
985 static void get_ext_port_caps(struct mlx5_ib_dev
*dev
)
987 struct mlx5_general_caps
*gen
;
990 gen
= &dev
->mdev
->caps
.gen
;
991 for (port
= 1; port
<= gen
->num_ports
; port
++)
992 mlx5_query_ext_port_caps(dev
, port
);
995 static int get_port_caps(struct mlx5_ib_dev
*dev
)
997 struct ib_device_attr
*dprops
= NULL
;
998 struct ib_port_attr
*pprops
= NULL
;
999 struct mlx5_general_caps
*gen
;
1003 gen
= &dev
->mdev
->caps
.gen
;
1004 pprops
= kmalloc(sizeof(*pprops
), GFP_KERNEL
);
1008 dprops
= kmalloc(sizeof(*dprops
), GFP_KERNEL
);
1012 err
= mlx5_ib_query_device(&dev
->ib_dev
, dprops
);
1014 mlx5_ib_warn(dev
, "query_device failed %d\n", err
);
1018 for (port
= 1; port
<= gen
->num_ports
; port
++) {
1019 err
= mlx5_ib_query_port(&dev
->ib_dev
, port
, pprops
);
1021 mlx5_ib_warn(dev
, "query_port %d failed %d\n", port
, err
);
1024 gen
->port
[port
- 1].pkey_table_len
= dprops
->max_pkeys
;
1025 gen
->port
[port
- 1].gid_table_len
= pprops
->gid_tbl_len
;
1026 mlx5_ib_dbg(dev
, "pkey_table_len %d, gid_table_len %d\n",
1027 dprops
->max_pkeys
, pprops
->gid_tbl_len
);
1037 static void destroy_umrc_res(struct mlx5_ib_dev
*dev
)
1041 err
= mlx5_mr_cache_cleanup(dev
);
1043 mlx5_ib_warn(dev
, "mr cache cleanup failed\n");
1045 mlx5_ib_destroy_qp(dev
->umrc
.qp
);
1046 ib_destroy_cq(dev
->umrc
.cq
);
1047 ib_dereg_mr(dev
->umrc
.mr
);
1048 ib_dealloc_pd(dev
->umrc
.pd
);
1055 static int create_umr_res(struct mlx5_ib_dev
*dev
)
1057 struct ib_qp_init_attr
*init_attr
= NULL
;
1058 struct ib_qp_attr
*attr
= NULL
;
1065 attr
= kzalloc(sizeof(*attr
), GFP_KERNEL
);
1066 init_attr
= kzalloc(sizeof(*init_attr
), GFP_KERNEL
);
1067 if (!attr
|| !init_attr
) {
1072 pd
= ib_alloc_pd(&dev
->ib_dev
);
1074 mlx5_ib_dbg(dev
, "Couldn't create PD for sync UMR QP\n");
1079 mr
= ib_get_dma_mr(pd
, IB_ACCESS_LOCAL_WRITE
);
1081 mlx5_ib_dbg(dev
, "Couldn't create DMA MR for sync UMR QP\n");
1086 cq
= ib_create_cq(&dev
->ib_dev
, mlx5_umr_cq_handler
, NULL
, NULL
, 128,
1089 mlx5_ib_dbg(dev
, "Couldn't create CQ for sync UMR QP\n");
1093 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
1095 init_attr
->send_cq
= cq
;
1096 init_attr
->recv_cq
= cq
;
1097 init_attr
->sq_sig_type
= IB_SIGNAL_ALL_WR
;
1098 init_attr
->cap
.max_send_wr
= MAX_UMR_WR
;
1099 init_attr
->cap
.max_send_sge
= 1;
1100 init_attr
->qp_type
= MLX5_IB_QPT_REG_UMR
;
1101 init_attr
->port_num
= 1;
1102 qp
= mlx5_ib_create_qp(pd
, init_attr
, NULL
);
1104 mlx5_ib_dbg(dev
, "Couldn't create sync UMR QP\n");
1108 qp
->device
= &dev
->ib_dev
;
1111 qp
->qp_type
= MLX5_IB_QPT_REG_UMR
;
1113 attr
->qp_state
= IB_QPS_INIT
;
1115 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
| IB_QP_PKEY_INDEX
|
1118 mlx5_ib_dbg(dev
, "Couldn't modify UMR QP\n");
1122 memset(attr
, 0, sizeof(*attr
));
1123 attr
->qp_state
= IB_QPS_RTR
;
1124 attr
->path_mtu
= IB_MTU_256
;
1126 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
, NULL
);
1128 mlx5_ib_dbg(dev
, "Couldn't modify umr QP to rtr\n");
1132 memset(attr
, 0, sizeof(*attr
));
1133 attr
->qp_state
= IB_QPS_RTS
;
1134 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
, NULL
);
1136 mlx5_ib_dbg(dev
, "Couldn't modify umr QP to rts\n");
1145 sema_init(&dev
->umrc
.sem
, MAX_UMR_WR
);
1146 ret
= mlx5_mr_cache_init(dev
);
1148 mlx5_ib_warn(dev
, "mr cache init failed %d\n", ret
);
1158 mlx5_ib_destroy_qp(qp
);
1175 static int create_dev_resources(struct mlx5_ib_resources
*devr
)
1177 struct ib_srq_init_attr attr
;
1178 struct mlx5_ib_dev
*dev
;
1181 dev
= container_of(devr
, struct mlx5_ib_dev
, devr
);
1183 devr
->p0
= mlx5_ib_alloc_pd(&dev
->ib_dev
, NULL
, NULL
);
1184 if (IS_ERR(devr
->p0
)) {
1185 ret
= PTR_ERR(devr
->p0
);
1188 devr
->p0
->device
= &dev
->ib_dev
;
1189 devr
->p0
->uobject
= NULL
;
1190 atomic_set(&devr
->p0
->usecnt
, 0);
1192 devr
->c0
= mlx5_ib_create_cq(&dev
->ib_dev
, 1, 0, NULL
, NULL
);
1193 if (IS_ERR(devr
->c0
)) {
1194 ret
= PTR_ERR(devr
->c0
);
1197 devr
->c0
->device
= &dev
->ib_dev
;
1198 devr
->c0
->uobject
= NULL
;
1199 devr
->c0
->comp_handler
= NULL
;
1200 devr
->c0
->event_handler
= NULL
;
1201 devr
->c0
->cq_context
= NULL
;
1202 atomic_set(&devr
->c0
->usecnt
, 0);
1204 devr
->x0
= mlx5_ib_alloc_xrcd(&dev
->ib_dev
, NULL
, NULL
);
1205 if (IS_ERR(devr
->x0
)) {
1206 ret
= PTR_ERR(devr
->x0
);
1209 devr
->x0
->device
= &dev
->ib_dev
;
1210 devr
->x0
->inode
= NULL
;
1211 atomic_set(&devr
->x0
->usecnt
, 0);
1212 mutex_init(&devr
->x0
->tgt_qp_mutex
);
1213 INIT_LIST_HEAD(&devr
->x0
->tgt_qp_list
);
1215 devr
->x1
= mlx5_ib_alloc_xrcd(&dev
->ib_dev
, NULL
, NULL
);
1216 if (IS_ERR(devr
->x1
)) {
1217 ret
= PTR_ERR(devr
->x1
);
1220 devr
->x1
->device
= &dev
->ib_dev
;
1221 devr
->x1
->inode
= NULL
;
1222 atomic_set(&devr
->x1
->usecnt
, 0);
1223 mutex_init(&devr
->x1
->tgt_qp_mutex
);
1224 INIT_LIST_HEAD(&devr
->x1
->tgt_qp_list
);
1226 memset(&attr
, 0, sizeof(attr
));
1227 attr
.attr
.max_sge
= 1;
1228 attr
.attr
.max_wr
= 1;
1229 attr
.srq_type
= IB_SRQT_XRC
;
1230 attr
.ext
.xrc
.cq
= devr
->c0
;
1231 attr
.ext
.xrc
.xrcd
= devr
->x0
;
1233 devr
->s0
= mlx5_ib_create_srq(devr
->p0
, &attr
, NULL
);
1234 if (IS_ERR(devr
->s0
)) {
1235 ret
= PTR_ERR(devr
->s0
);
1238 devr
->s0
->device
= &dev
->ib_dev
;
1239 devr
->s0
->pd
= devr
->p0
;
1240 devr
->s0
->uobject
= NULL
;
1241 devr
->s0
->event_handler
= NULL
;
1242 devr
->s0
->srq_context
= NULL
;
1243 devr
->s0
->srq_type
= IB_SRQT_XRC
;
1244 devr
->s0
->ext
.xrc
.xrcd
= devr
->x0
;
1245 devr
->s0
->ext
.xrc
.cq
= devr
->c0
;
1246 atomic_inc(&devr
->s0
->ext
.xrc
.xrcd
->usecnt
);
1247 atomic_inc(&devr
->s0
->ext
.xrc
.cq
->usecnt
);
1248 atomic_inc(&devr
->p0
->usecnt
);
1249 atomic_set(&devr
->s0
->usecnt
, 0);
1254 mlx5_ib_dealloc_xrcd(devr
->x1
);
1256 mlx5_ib_dealloc_xrcd(devr
->x0
);
1258 mlx5_ib_destroy_cq(devr
->c0
);
1260 mlx5_ib_dealloc_pd(devr
->p0
);
1265 static void destroy_dev_resources(struct mlx5_ib_resources
*devr
)
1267 mlx5_ib_destroy_srq(devr
->s0
);
1268 mlx5_ib_dealloc_xrcd(devr
->x0
);
1269 mlx5_ib_dealloc_xrcd(devr
->x1
);
1270 mlx5_ib_destroy_cq(devr
->c0
);
1271 mlx5_ib_dealloc_pd(devr
->p0
);
1274 static void *mlx5_ib_add(struct mlx5_core_dev
*mdev
)
1276 struct mlx5_ib_dev
*dev
;
1280 printk_once(KERN_INFO
"%s", mlx5_version
);
1282 dev
= (struct mlx5_ib_dev
*)ib_alloc_device(sizeof(*dev
));
1288 err
= get_port_caps(dev
);
1292 get_ext_port_caps(dev
);
1294 err
= alloc_comp_eqs(dev
);
1298 MLX5_INIT_DOORBELL_LOCK(&dev
->uar_lock
);
1300 strlcpy(dev
->ib_dev
.name
, "mlx5_%d", IB_DEVICE_NAME_MAX
);
1301 dev
->ib_dev
.owner
= THIS_MODULE
;
1302 dev
->ib_dev
.node_type
= RDMA_NODE_IB_CA
;
1303 dev
->ib_dev
.local_dma_lkey
= mdev
->caps
.gen
.reserved_lkey
;
1304 dev
->num_ports
= mdev
->caps
.gen
.num_ports
;
1305 dev
->ib_dev
.phys_port_cnt
= dev
->num_ports
;
1306 dev
->ib_dev
.num_comp_vectors
= dev
->num_comp_vectors
;
1307 dev
->ib_dev
.dma_device
= &mdev
->pdev
->dev
;
1309 dev
->ib_dev
.uverbs_abi_ver
= MLX5_IB_UVERBS_ABI_VERSION
;
1310 dev
->ib_dev
.uverbs_cmd_mask
=
1311 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
1312 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
1313 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
1314 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
1315 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
1316 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
1317 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
1318 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
1319 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
1320 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
1321 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
1322 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
1323 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
1324 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
1325 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
1326 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST
) |
1327 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST
) |
1328 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
1329 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
1330 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
1331 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
1332 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ
) |
1333 (1ull << IB_USER_VERBS_CMD_OPEN_QP
);
1334 dev
->ib_dev
.uverbs_ex_cmd_mask
=
1335 (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE
);
1337 dev
->ib_dev
.query_device
= mlx5_ib_query_device
;
1338 dev
->ib_dev
.query_port
= mlx5_ib_query_port
;
1339 dev
->ib_dev
.query_gid
= mlx5_ib_query_gid
;
1340 dev
->ib_dev
.query_pkey
= mlx5_ib_query_pkey
;
1341 dev
->ib_dev
.modify_device
= mlx5_ib_modify_device
;
1342 dev
->ib_dev
.modify_port
= mlx5_ib_modify_port
;
1343 dev
->ib_dev
.alloc_ucontext
= mlx5_ib_alloc_ucontext
;
1344 dev
->ib_dev
.dealloc_ucontext
= mlx5_ib_dealloc_ucontext
;
1345 dev
->ib_dev
.mmap
= mlx5_ib_mmap
;
1346 dev
->ib_dev
.alloc_pd
= mlx5_ib_alloc_pd
;
1347 dev
->ib_dev
.dealloc_pd
= mlx5_ib_dealloc_pd
;
1348 dev
->ib_dev
.create_ah
= mlx5_ib_create_ah
;
1349 dev
->ib_dev
.query_ah
= mlx5_ib_query_ah
;
1350 dev
->ib_dev
.destroy_ah
= mlx5_ib_destroy_ah
;
1351 dev
->ib_dev
.create_srq
= mlx5_ib_create_srq
;
1352 dev
->ib_dev
.modify_srq
= mlx5_ib_modify_srq
;
1353 dev
->ib_dev
.query_srq
= mlx5_ib_query_srq
;
1354 dev
->ib_dev
.destroy_srq
= mlx5_ib_destroy_srq
;
1355 dev
->ib_dev
.post_srq_recv
= mlx5_ib_post_srq_recv
;
1356 dev
->ib_dev
.create_qp
= mlx5_ib_create_qp
;
1357 dev
->ib_dev
.modify_qp
= mlx5_ib_modify_qp
;
1358 dev
->ib_dev
.query_qp
= mlx5_ib_query_qp
;
1359 dev
->ib_dev
.destroy_qp
= mlx5_ib_destroy_qp
;
1360 dev
->ib_dev
.post_send
= mlx5_ib_post_send
;
1361 dev
->ib_dev
.post_recv
= mlx5_ib_post_recv
;
1362 dev
->ib_dev
.create_cq
= mlx5_ib_create_cq
;
1363 dev
->ib_dev
.modify_cq
= mlx5_ib_modify_cq
;
1364 dev
->ib_dev
.resize_cq
= mlx5_ib_resize_cq
;
1365 dev
->ib_dev
.destroy_cq
= mlx5_ib_destroy_cq
;
1366 dev
->ib_dev
.poll_cq
= mlx5_ib_poll_cq
;
1367 dev
->ib_dev
.req_notify_cq
= mlx5_ib_arm_cq
;
1368 dev
->ib_dev
.get_dma_mr
= mlx5_ib_get_dma_mr
;
1369 dev
->ib_dev
.reg_user_mr
= mlx5_ib_reg_user_mr
;
1370 dev
->ib_dev
.dereg_mr
= mlx5_ib_dereg_mr
;
1371 dev
->ib_dev
.destroy_mr
= mlx5_ib_destroy_mr
;
1372 dev
->ib_dev
.attach_mcast
= mlx5_ib_mcg_attach
;
1373 dev
->ib_dev
.detach_mcast
= mlx5_ib_mcg_detach
;
1374 dev
->ib_dev
.process_mad
= mlx5_ib_process_mad
;
1375 dev
->ib_dev
.create_mr
= mlx5_ib_create_mr
;
1376 dev
->ib_dev
.alloc_fast_reg_mr
= mlx5_ib_alloc_fast_reg_mr
;
1377 dev
->ib_dev
.alloc_fast_reg_page_list
= mlx5_ib_alloc_fast_reg_page_list
;
1378 dev
->ib_dev
.free_fast_reg_page_list
= mlx5_ib_free_fast_reg_page_list
;
1379 dev
->ib_dev
.check_mr_status
= mlx5_ib_check_mr_status
;
1381 mlx5_ib_internal_query_odp_caps(dev
);
1383 if (mdev
->caps
.gen
.flags
& MLX5_DEV_CAP_FLAG_XRC
) {
1384 dev
->ib_dev
.alloc_xrcd
= mlx5_ib_alloc_xrcd
;
1385 dev
->ib_dev
.dealloc_xrcd
= mlx5_ib_dealloc_xrcd
;
1386 dev
->ib_dev
.uverbs_cmd_mask
|=
1387 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD
) |
1388 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD
);
1391 err
= init_node_data(dev
);
1395 mutex_init(&dev
->cap_mask_mutex
);
1397 err
= create_dev_resources(&dev
->devr
);
1401 err
= mlx5_ib_odp_init_one(dev
);
1405 err
= ib_register_device(&dev
->ib_dev
, NULL
);
1409 err
= create_umr_res(dev
);
1413 for (i
= 0; i
< ARRAY_SIZE(mlx5_class_attributes
); i
++) {
1414 err
= device_create_file(&dev
->ib_dev
.dev
,
1415 mlx5_class_attributes
[i
]);
1420 dev
->ib_active
= true;
1425 destroy_umrc_res(dev
);
1428 ib_unregister_device(&dev
->ib_dev
);
1431 mlx5_ib_odp_remove_one(dev
);
1434 destroy_dev_resources(&dev
->devr
);
1440 ib_dealloc_device((struct ib_device
*)dev
);
1445 static void mlx5_ib_remove(struct mlx5_core_dev
*mdev
, void *context
)
1447 struct mlx5_ib_dev
*dev
= context
;
1449 ib_unregister_device(&dev
->ib_dev
);
1450 destroy_umrc_res(dev
);
1451 mlx5_ib_odp_remove_one(dev
);
1452 destroy_dev_resources(&dev
->devr
);
1454 ib_dealloc_device(&dev
->ib_dev
);
1457 static struct mlx5_interface mlx5_ib_interface
= {
1459 .remove
= mlx5_ib_remove
,
1460 .event
= mlx5_ib_event
,
1463 static int __init
mlx5_ib_init(void)
1467 if (deprecated_prof_sel
!= 2)
1468 pr_warn("prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");
1470 err
= mlx5_ib_odp_init();
1474 err
= mlx5_register_interface(&mlx5_ib_interface
);
1481 mlx5_ib_odp_cleanup();
1485 static void __exit
mlx5_ib_cleanup(void)
1487 mlx5_unregister_interface(&mlx5_ib_interface
);
1488 mlx5_ib_odp_cleanup();
1491 module_init(mlx5_ib_init
);
1492 module_exit(mlx5_ib_cleanup
);