2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <asm-generic/kmap_types.h>
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/errno.h>
37 #include <linux/pci.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/slab.h>
40 #include <linux/io-mapping.h>
41 #include <linux/sched.h>
42 #include <rdma/ib_user_verbs.h>
43 #include <rdma/ib_smi.h>
44 #include <rdma/ib_umem.h>
48 #define DRIVER_NAME "mlx5_ib"
49 #define DRIVER_VERSION "2.2-1"
50 #define DRIVER_RELDATE "Feb 2014"
52 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
53 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
54 MODULE_LICENSE("Dual BSD/GPL");
55 MODULE_VERSION(DRIVER_VERSION
);
57 static int prof_sel
= 2;
58 module_param_named(prof_sel
, prof_sel
, int, 0444);
59 MODULE_PARM_DESC(prof_sel
, "profile selector. Valid range 0 - 2");
61 static char mlx5_version
[] =
62 DRIVER_NAME
": Mellanox Connect-IB Infiniband driver v"
63 DRIVER_VERSION
" (" DRIVER_RELDATE
")\n";
65 static struct mlx5_profile profile
[] = {
70 .mask
= MLX5_PROF_MASK_QP_SIZE
,
74 .mask
= MLX5_PROF_MASK_QP_SIZE
|
75 MLX5_PROF_MASK_MR_CACHE
,
144 int mlx5_vector2eqn(struct mlx5_ib_dev
*dev
, int vector
, int *eqn
, int *irqn
)
146 struct mlx5_eq_table
*table
= &dev
->mdev
.priv
.eq_table
;
147 struct mlx5_eq
*eq
, *n
;
150 spin_lock(&table
->lock
);
151 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
152 if (eq
->index
== vector
) {
159 spin_unlock(&table
->lock
);
164 static int alloc_comp_eqs(struct mlx5_ib_dev
*dev
)
166 struct mlx5_eq_table
*table
= &dev
->mdev
.priv
.eq_table
;
167 char name
[MLX5_MAX_EQ_NAME
];
168 struct mlx5_eq
*eq
, *n
;
174 INIT_LIST_HEAD(&dev
->eqs_list
);
175 ncomp_vec
= table
->num_comp_vectors
;
176 nent
= MLX5_COMP_EQ_SIZE
;
177 for (i
= 0; i
< ncomp_vec
; i
++) {
178 eq
= kzalloc(sizeof(*eq
), GFP_KERNEL
);
184 snprintf(name
, MLX5_MAX_EQ_NAME
, "mlx5_comp%d", i
);
185 err
= mlx5_create_map_eq(&dev
->mdev
, eq
,
186 i
+ MLX5_EQ_VEC_COMP_BASE
, nent
, 0,
187 name
, &dev
->mdev
.priv
.uuari
.uars
[0]);
192 mlx5_ib_dbg(dev
, "allocated completion EQN %d\n", eq
->eqn
);
194 spin_lock(&table
->lock
);
195 list_add_tail(&eq
->list
, &dev
->eqs_list
);
196 spin_unlock(&table
->lock
);
199 dev
->num_comp_vectors
= ncomp_vec
;
203 spin_lock(&table
->lock
);
204 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
206 spin_unlock(&table
->lock
);
207 if (mlx5_destroy_unmap_eq(&dev
->mdev
, eq
))
208 mlx5_ib_warn(dev
, "failed to destroy EQ 0x%x\n", eq
->eqn
);
210 spin_lock(&table
->lock
);
212 spin_unlock(&table
->lock
);
216 static void free_comp_eqs(struct mlx5_ib_dev
*dev
)
218 struct mlx5_eq_table
*table
= &dev
->mdev
.priv
.eq_table
;
219 struct mlx5_eq
*eq
, *n
;
221 spin_lock(&table
->lock
);
222 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
224 spin_unlock(&table
->lock
);
225 if (mlx5_destroy_unmap_eq(&dev
->mdev
, eq
))
226 mlx5_ib_warn(dev
, "failed to destroy EQ 0x%x\n", eq
->eqn
);
228 spin_lock(&table
->lock
);
230 spin_unlock(&table
->lock
);
233 static int mlx5_ib_query_device(struct ib_device
*ibdev
,
234 struct ib_device_attr
*props
)
236 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
237 struct ib_smp
*in_mad
= NULL
;
238 struct ib_smp
*out_mad
= NULL
;
244 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
245 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
246 if (!in_mad
|| !out_mad
)
249 init_query_mad(in_mad
);
250 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
252 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
256 memset(props
, 0, sizeof(*props
));
258 props
->fw_ver
= ((u64
)fw_rev_maj(&dev
->mdev
) << 32) |
259 (fw_rev_min(&dev
->mdev
) << 16) |
260 fw_rev_sub(&dev
->mdev
);
261 props
->device_cap_flags
= IB_DEVICE_CHANGE_PHY_PORT
|
262 IB_DEVICE_PORT_ACTIVE_EVENT
|
263 IB_DEVICE_SYS_IMAGE_GUID
|
264 IB_DEVICE_RC_RNR_NAK_GEN
;
265 flags
= dev
->mdev
.caps
.flags
;
266 if (flags
& MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR
)
267 props
->device_cap_flags
|= IB_DEVICE_BAD_PKEY_CNTR
;
268 if (flags
& MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR
)
269 props
->device_cap_flags
|= IB_DEVICE_BAD_QKEY_CNTR
;
270 if (flags
& MLX5_DEV_CAP_FLAG_APM
)
271 props
->device_cap_flags
|= IB_DEVICE_AUTO_PATH_MIG
;
272 props
->device_cap_flags
|= IB_DEVICE_LOCAL_DMA_LKEY
;
273 if (flags
& MLX5_DEV_CAP_FLAG_XRC
)
274 props
->device_cap_flags
|= IB_DEVICE_XRC
;
275 props
->device_cap_flags
|= IB_DEVICE_MEM_MGT_EXTENSIONS
;
276 if (flags
& MLX5_DEV_CAP_FLAG_SIG_HAND_OVER
) {
277 props
->device_cap_flags
|= IB_DEVICE_SIGNATURE_HANDOVER
;
278 /* At this stage no support for signature handover */
279 props
->sig_prot_cap
= IB_PROT_T10DIF_TYPE_1
|
280 IB_PROT_T10DIF_TYPE_2
|
281 IB_PROT_T10DIF_TYPE_3
;
282 props
->sig_guard_cap
= IB_GUARD_T10DIF_CRC
|
283 IB_GUARD_T10DIF_CSUM
;
286 props
->vendor_id
= be32_to_cpup((__be32
*)(out_mad
->data
+ 36)) &
288 props
->vendor_part_id
= be16_to_cpup((__be16
*)(out_mad
->data
+ 30));
289 props
->hw_ver
= be32_to_cpup((__be32
*)(out_mad
->data
+ 32));
290 memcpy(&props
->sys_image_guid
, out_mad
->data
+ 4, 8);
292 props
->max_mr_size
= ~0ull;
293 props
->page_size_cap
= dev
->mdev
.caps
.min_page_sz
;
294 props
->max_qp
= 1 << dev
->mdev
.caps
.log_max_qp
;
295 props
->max_qp_wr
= dev
->mdev
.caps
.max_wqes
;
296 max_rq_sg
= dev
->mdev
.caps
.max_rq_desc_sz
/ sizeof(struct mlx5_wqe_data_seg
);
297 max_sq_sg
= (dev
->mdev
.caps
.max_sq_desc_sz
- sizeof(struct mlx5_wqe_ctrl_seg
)) /
298 sizeof(struct mlx5_wqe_data_seg
);
299 props
->max_sge
= min(max_rq_sg
, max_sq_sg
);
300 props
->max_cq
= 1 << dev
->mdev
.caps
.log_max_cq
;
301 props
->max_cqe
= dev
->mdev
.caps
.max_cqes
- 1;
302 props
->max_mr
= 1 << dev
->mdev
.caps
.log_max_mkey
;
303 props
->max_pd
= 1 << dev
->mdev
.caps
.log_max_pd
;
304 props
->max_qp_rd_atom
= dev
->mdev
.caps
.max_ra_req_qp
;
305 props
->max_qp_init_rd_atom
= dev
->mdev
.caps
.max_ra_res_qp
;
306 props
->max_res_rd_atom
= props
->max_qp_rd_atom
* props
->max_qp
;
307 props
->max_srq
= 1 << dev
->mdev
.caps
.log_max_srq
;
308 props
->max_srq_wr
= dev
->mdev
.caps
.max_srq_wqes
- 1;
309 props
->max_srq_sge
= max_rq_sg
- 1;
310 props
->max_fast_reg_page_list_len
= (unsigned int)-1;
311 props
->local_ca_ack_delay
= dev
->mdev
.caps
.local_ca_ack_delay
;
312 props
->atomic_cap
= IB_ATOMIC_NONE
;
313 props
->masked_atomic_cap
= IB_ATOMIC_NONE
;
314 props
->max_pkeys
= be16_to_cpup((__be16
*)(out_mad
->data
+ 28));
315 props
->max_mcast_grp
= 1 << dev
->mdev
.caps
.log_max_mcg
;
316 props
->max_mcast_qp_attach
= dev
->mdev
.caps
.max_qp_mcg
;
317 props
->max_total_mcast_qp_attach
= props
->max_mcast_qp_attach
*
318 props
->max_mcast_grp
;
319 props
->max_map_per_fmr
= INT_MAX
; /* no limit in ConnectIB */
328 int mlx5_ib_query_port(struct ib_device
*ibdev
, u8 port
,
329 struct ib_port_attr
*props
)
331 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
332 struct ib_smp
*in_mad
= NULL
;
333 struct ib_smp
*out_mad
= NULL
;
334 int ext_active_speed
;
337 if (port
< 1 || port
> dev
->mdev
.caps
.num_ports
) {
338 mlx5_ib_warn(dev
, "invalid port number %d\n", port
);
342 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
343 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
344 if (!in_mad
|| !out_mad
)
347 memset(props
, 0, sizeof(*props
));
349 init_query_mad(in_mad
);
350 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
351 in_mad
->attr_mod
= cpu_to_be32(port
);
353 err
= mlx5_MAD_IFC(dev
, 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
355 mlx5_ib_warn(dev
, "err %d\n", err
);
360 props
->lid
= be16_to_cpup((__be16
*)(out_mad
->data
+ 16));
361 props
->lmc
= out_mad
->data
[34] & 0x7;
362 props
->sm_lid
= be16_to_cpup((__be16
*)(out_mad
->data
+ 18));
363 props
->sm_sl
= out_mad
->data
[36] & 0xf;
364 props
->state
= out_mad
->data
[32] & 0xf;
365 props
->phys_state
= out_mad
->data
[33] >> 4;
366 props
->port_cap_flags
= be32_to_cpup((__be32
*)(out_mad
->data
+ 20));
367 props
->gid_tbl_len
= out_mad
->data
[50];
368 props
->max_msg_sz
= 1 << to_mdev(ibdev
)->mdev
.caps
.log_max_msg
;
369 props
->pkey_tbl_len
= to_mdev(ibdev
)->mdev
.caps
.port
[port
- 1].pkey_table_len
;
370 props
->bad_pkey_cntr
= be16_to_cpup((__be16
*)(out_mad
->data
+ 46));
371 props
->qkey_viol_cntr
= be16_to_cpup((__be16
*)(out_mad
->data
+ 48));
372 props
->active_width
= out_mad
->data
[31] & 0xf;
373 props
->active_speed
= out_mad
->data
[35] >> 4;
374 props
->max_mtu
= out_mad
->data
[41] & 0xf;
375 props
->active_mtu
= out_mad
->data
[36] >> 4;
376 props
->subnet_timeout
= out_mad
->data
[51] & 0x1f;
377 props
->max_vl_num
= out_mad
->data
[37] >> 4;
378 props
->init_type_reply
= out_mad
->data
[41] >> 4;
380 /* Check if extended speeds (EDR/FDR/...) are supported */
381 if (props
->port_cap_flags
& IB_PORT_EXTENDED_SPEEDS_SUP
) {
382 ext_active_speed
= out_mad
->data
[62] >> 4;
384 switch (ext_active_speed
) {
386 props
->active_speed
= 16; /* FDR */
389 props
->active_speed
= 32; /* EDR */
394 /* If reported active speed is QDR, check if is FDR-10 */
395 if (props
->active_speed
== 4) {
396 if (dev
->mdev
.caps
.ext_port_cap
[port
- 1] &
397 MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO
) {
398 init_query_mad(in_mad
);
399 in_mad
->attr_id
= MLX5_ATTR_EXTENDED_PORT_INFO
;
400 in_mad
->attr_mod
= cpu_to_be32(port
);
402 err
= mlx5_MAD_IFC(dev
, 1, 1, port
,
403 NULL
, NULL
, in_mad
, out_mad
);
407 /* Checking LinkSpeedActive for FDR-10 */
408 if (out_mad
->data
[15] & 0x1)
409 props
->active_speed
= 8;
420 static int mlx5_ib_query_gid(struct ib_device
*ibdev
, u8 port
, int index
,
423 struct ib_smp
*in_mad
= NULL
;
424 struct ib_smp
*out_mad
= NULL
;
427 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
428 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
429 if (!in_mad
|| !out_mad
)
432 init_query_mad(in_mad
);
433 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
434 in_mad
->attr_mod
= cpu_to_be32(port
);
436 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
440 memcpy(gid
->raw
, out_mad
->data
+ 8, 8);
442 init_query_mad(in_mad
);
443 in_mad
->attr_id
= IB_SMP_ATTR_GUID_INFO
;
444 in_mad
->attr_mod
= cpu_to_be32(index
/ 8);
446 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
450 memcpy(gid
->raw
+ 8, out_mad
->data
+ (index
% 8) * 8, 8);
458 static int mlx5_ib_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
461 struct ib_smp
*in_mad
= NULL
;
462 struct ib_smp
*out_mad
= NULL
;
465 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
466 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
467 if (!in_mad
|| !out_mad
)
470 init_query_mad(in_mad
);
471 in_mad
->attr_id
= IB_SMP_ATTR_PKEY_TABLE
;
472 in_mad
->attr_mod
= cpu_to_be32(index
/ 32);
474 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
478 *pkey
= be16_to_cpu(((__be16
*)out_mad
->data
)[index
% 32]);
486 struct mlx5_reg_node_desc
{
490 static int mlx5_ib_modify_device(struct ib_device
*ibdev
, int mask
,
491 struct ib_device_modify
*props
)
493 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
494 struct mlx5_reg_node_desc in
;
495 struct mlx5_reg_node_desc out
;
498 if (mask
& ~IB_DEVICE_MODIFY_NODE_DESC
)
501 if (!(mask
& IB_DEVICE_MODIFY_NODE_DESC
))
505 * If possible, pass node desc to FW, so it can generate
506 * a 144 trap. If cmd fails, just ignore.
508 memcpy(&in
, props
->node_desc
, 64);
509 err
= mlx5_core_access_reg(&dev
->mdev
, &in
, sizeof(in
), &out
,
510 sizeof(out
), MLX5_REG_NODE_DESC
, 0, 1);
514 memcpy(ibdev
->node_desc
, props
->node_desc
, 64);
519 static int mlx5_ib_modify_port(struct ib_device
*ibdev
, u8 port
, int mask
,
520 struct ib_port_modify
*props
)
522 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
523 struct ib_port_attr attr
;
527 mutex_lock(&dev
->cap_mask_mutex
);
529 err
= mlx5_ib_query_port(ibdev
, port
, &attr
);
533 tmp
= (attr
.port_cap_flags
| props
->set_port_cap_mask
) &
534 ~props
->clr_port_cap_mask
;
536 err
= mlx5_set_port_caps(&dev
->mdev
, port
, tmp
);
539 mutex_unlock(&dev
->cap_mask_mutex
);
543 static struct ib_ucontext
*mlx5_ib_alloc_ucontext(struct ib_device
*ibdev
,
544 struct ib_udata
*udata
)
546 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
547 struct mlx5_ib_alloc_ucontext_req_v2 req
;
548 struct mlx5_ib_alloc_ucontext_resp resp
;
549 struct mlx5_ib_ucontext
*context
;
550 struct mlx5_uuar_info
*uuari
;
551 struct mlx5_uar
*uars
;
561 return ERR_PTR(-EAGAIN
);
563 memset(&req
, 0, sizeof(req
));
564 reqlen
= udata
->inlen
- sizeof(struct ib_uverbs_cmd_hdr
);
565 if (reqlen
== sizeof(struct mlx5_ib_alloc_ucontext_req
))
567 else if (reqlen
== sizeof(struct mlx5_ib_alloc_ucontext_req_v2
))
570 return ERR_PTR(-EINVAL
);
572 err
= ib_copy_from_udata(&req
, udata
, reqlen
);
576 if (req
.flags
|| req
.reserved
)
577 return ERR_PTR(-EINVAL
);
579 if (req
.total_num_uuars
> MLX5_MAX_UUARS
)
580 return ERR_PTR(-ENOMEM
);
582 if (req
.total_num_uuars
== 0)
583 return ERR_PTR(-EINVAL
);
585 req
.total_num_uuars
= ALIGN(req
.total_num_uuars
,
586 MLX5_NON_FP_BF_REGS_PER_PAGE
);
587 if (req
.num_low_latency_uuars
> req
.total_num_uuars
- 1)
588 return ERR_PTR(-EINVAL
);
590 num_uars
= req
.total_num_uuars
/ MLX5_NON_FP_BF_REGS_PER_PAGE
;
591 gross_uuars
= num_uars
* MLX5_BF_REGS_PER_PAGE
;
592 resp
.qp_tab_size
= 1 << dev
->mdev
.caps
.log_max_qp
;
593 resp
.bf_reg_size
= dev
->mdev
.caps
.bf_reg_size
;
594 resp
.cache_line_size
= L1_CACHE_BYTES
;
595 resp
.max_sq_desc_sz
= dev
->mdev
.caps
.max_sq_desc_sz
;
596 resp
.max_rq_desc_sz
= dev
->mdev
.caps
.max_rq_desc_sz
;
597 resp
.max_send_wqebb
= dev
->mdev
.caps
.max_wqes
;
598 resp
.max_recv_wr
= dev
->mdev
.caps
.max_wqes
;
599 resp
.max_srq_recv_wr
= dev
->mdev
.caps
.max_srq_wqes
;
601 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
603 return ERR_PTR(-ENOMEM
);
605 uuari
= &context
->uuari
;
606 mutex_init(&uuari
->lock
);
607 uars
= kcalloc(num_uars
, sizeof(*uars
), GFP_KERNEL
);
613 uuari
->bitmap
= kcalloc(BITS_TO_LONGS(gross_uuars
),
614 sizeof(*uuari
->bitmap
),
616 if (!uuari
->bitmap
) {
621 * clear all fast path uuars
623 for (i
= 0; i
< gross_uuars
; i
++) {
625 if (uuarn
== 2 || uuarn
== 3)
626 set_bit(i
, uuari
->bitmap
);
629 uuari
->count
= kcalloc(gross_uuars
, sizeof(*uuari
->count
), GFP_KERNEL
);
635 for (i
= 0; i
< num_uars
; i
++) {
636 err
= mlx5_cmd_alloc_uar(&dev
->mdev
, &uars
[i
].index
);
641 INIT_LIST_HEAD(&context
->db_page_list
);
642 mutex_init(&context
->db_page_mutex
);
644 resp
.tot_uuars
= req
.total_num_uuars
;
645 resp
.num_ports
= dev
->mdev
.caps
.num_ports
;
646 err
= ib_copy_to_udata(udata
, &resp
,
647 sizeof(resp
) - sizeof(resp
.reserved
));
652 uuari
->num_low_latency_uuars
= req
.num_low_latency_uuars
;
654 uuari
->num_uars
= num_uars
;
655 return &context
->ibucontext
;
658 for (i
--; i
>= 0; i
--)
659 mlx5_cmd_free_uar(&dev
->mdev
, uars
[i
].index
);
664 kfree(uuari
->bitmap
);
674 static int mlx5_ib_dealloc_ucontext(struct ib_ucontext
*ibcontext
)
676 struct mlx5_ib_ucontext
*context
= to_mucontext(ibcontext
);
677 struct mlx5_ib_dev
*dev
= to_mdev(ibcontext
->device
);
678 struct mlx5_uuar_info
*uuari
= &context
->uuari
;
681 for (i
= 0; i
< uuari
->num_uars
; i
++) {
682 if (mlx5_cmd_free_uar(&dev
->mdev
, uuari
->uars
[i
].index
))
683 mlx5_ib_warn(dev
, "failed to free UAR 0x%x\n", uuari
->uars
[i
].index
);
687 kfree(uuari
->bitmap
);
694 static phys_addr_t
uar_index2pfn(struct mlx5_ib_dev
*dev
, int index
)
696 return (pci_resource_start(dev
->mdev
.pdev
, 0) >> PAGE_SHIFT
) + index
;
699 static int get_command(unsigned long offset
)
701 return (offset
>> MLX5_IB_MMAP_CMD_SHIFT
) & MLX5_IB_MMAP_CMD_MASK
;
704 static int get_arg(unsigned long offset
)
706 return offset
& ((1 << MLX5_IB_MMAP_CMD_SHIFT
) - 1);
709 static int get_index(unsigned long offset
)
711 return get_arg(offset
);
714 static int mlx5_ib_mmap(struct ib_ucontext
*ibcontext
, struct vm_area_struct
*vma
)
716 struct mlx5_ib_ucontext
*context
= to_mucontext(ibcontext
);
717 struct mlx5_ib_dev
*dev
= to_mdev(ibcontext
->device
);
718 struct mlx5_uuar_info
*uuari
= &context
->uuari
;
719 unsigned long command
;
723 command
= get_command(vma
->vm_pgoff
);
725 case MLX5_IB_MMAP_REGULAR_PAGE
:
726 if (vma
->vm_end
- vma
->vm_start
!= PAGE_SIZE
)
729 idx
= get_index(vma
->vm_pgoff
);
730 pfn
= uar_index2pfn(dev
, uuari
->uars
[idx
].index
);
731 mlx5_ib_dbg(dev
, "uar idx 0x%lx, pfn 0x%llx\n", idx
,
732 (unsigned long long)pfn
);
734 if (idx
>= uuari
->num_uars
)
737 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
738 if (io_remap_pfn_range(vma
, vma
->vm_start
, pfn
,
739 PAGE_SIZE
, vma
->vm_page_prot
))
742 mlx5_ib_dbg(dev
, "mapped WC at 0x%lx, PA 0x%llx\n",
744 (unsigned long long)pfn
<< PAGE_SHIFT
);
747 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES
:
757 static int alloc_pa_mkey(struct mlx5_ib_dev
*dev
, u32
*key
, u32 pdn
)
759 struct mlx5_create_mkey_mbox_in
*in
;
760 struct mlx5_mkey_seg
*seg
;
761 struct mlx5_core_mr mr
;
764 in
= kzalloc(sizeof(*in
), GFP_KERNEL
);
769 seg
->flags
= MLX5_PERM_LOCAL_READ
| MLX5_ACCESS_MODE_PA
;
770 seg
->flags_pd
= cpu_to_be32(pdn
| MLX5_MKEY_LEN64
);
771 seg
->qpn_mkey7_0
= cpu_to_be32(0xffffff << 8);
774 err
= mlx5_core_create_mkey(&dev
->mdev
, &mr
, in
, sizeof(*in
),
777 mlx5_ib_warn(dev
, "failed to create mkey, %d\n", err
);
792 static void free_pa_mkey(struct mlx5_ib_dev
*dev
, u32 key
)
794 struct mlx5_core_mr mr
;
797 memset(&mr
, 0, sizeof(mr
));
799 err
= mlx5_core_destroy_mkey(&dev
->mdev
, &mr
);
801 mlx5_ib_warn(dev
, "failed to destroy mkey 0x%x\n", key
);
804 static struct ib_pd
*mlx5_ib_alloc_pd(struct ib_device
*ibdev
,
805 struct ib_ucontext
*context
,
806 struct ib_udata
*udata
)
808 struct mlx5_ib_alloc_pd_resp resp
;
809 struct mlx5_ib_pd
*pd
;
812 pd
= kmalloc(sizeof(*pd
), GFP_KERNEL
);
814 return ERR_PTR(-ENOMEM
);
816 err
= mlx5_core_alloc_pd(&to_mdev(ibdev
)->mdev
, &pd
->pdn
);
824 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
825 mlx5_core_dealloc_pd(&to_mdev(ibdev
)->mdev
, pd
->pdn
);
827 return ERR_PTR(-EFAULT
);
830 err
= alloc_pa_mkey(to_mdev(ibdev
), &pd
->pa_lkey
, pd
->pdn
);
832 mlx5_core_dealloc_pd(&to_mdev(ibdev
)->mdev
, pd
->pdn
);
841 static int mlx5_ib_dealloc_pd(struct ib_pd
*pd
)
843 struct mlx5_ib_dev
*mdev
= to_mdev(pd
->device
);
844 struct mlx5_ib_pd
*mpd
= to_mpd(pd
);
847 free_pa_mkey(mdev
, mpd
->pa_lkey
);
849 mlx5_core_dealloc_pd(&mdev
->mdev
, mpd
->pdn
);
855 static int mlx5_ib_mcg_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
857 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
860 err
= mlx5_core_attach_mcg(&dev
->mdev
, gid
, ibqp
->qp_num
);
862 mlx5_ib_warn(dev
, "failed attaching QPN 0x%x, MGID %pI6\n",
863 ibqp
->qp_num
, gid
->raw
);
868 static int mlx5_ib_mcg_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
870 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
873 err
= mlx5_core_detach_mcg(&dev
->mdev
, gid
, ibqp
->qp_num
);
875 mlx5_ib_warn(dev
, "failed detaching QPN 0x%x, MGID %pI6\n",
876 ibqp
->qp_num
, gid
->raw
);
881 static int init_node_data(struct mlx5_ib_dev
*dev
)
883 struct ib_smp
*in_mad
= NULL
;
884 struct ib_smp
*out_mad
= NULL
;
887 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
888 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
889 if (!in_mad
|| !out_mad
)
892 init_query_mad(in_mad
);
893 in_mad
->attr_id
= IB_SMP_ATTR_NODE_DESC
;
895 err
= mlx5_MAD_IFC(dev
, 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
899 memcpy(dev
->ib_dev
.node_desc
, out_mad
->data
, 64);
901 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
903 err
= mlx5_MAD_IFC(dev
, 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
907 dev
->mdev
.rev_id
= be32_to_cpup((__be32
*)(out_mad
->data
+ 32));
908 memcpy(&dev
->ib_dev
.node_guid
, out_mad
->data
+ 12, 8);
916 static ssize_t
show_fw_pages(struct device
*device
, struct device_attribute
*attr
,
919 struct mlx5_ib_dev
*dev
=
920 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
922 return sprintf(buf
, "%d\n", dev
->mdev
.priv
.fw_pages
);
925 static ssize_t
show_reg_pages(struct device
*device
,
926 struct device_attribute
*attr
, char *buf
)
928 struct mlx5_ib_dev
*dev
=
929 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
931 return sprintf(buf
, "%d\n", dev
->mdev
.priv
.reg_pages
);
934 static ssize_t
show_hca(struct device
*device
, struct device_attribute
*attr
,
937 struct mlx5_ib_dev
*dev
=
938 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
939 return sprintf(buf
, "MT%d\n", dev
->mdev
.pdev
->device
);
942 static ssize_t
show_fw_ver(struct device
*device
, struct device_attribute
*attr
,
945 struct mlx5_ib_dev
*dev
=
946 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
947 return sprintf(buf
, "%d.%d.%d\n", fw_rev_maj(&dev
->mdev
),
948 fw_rev_min(&dev
->mdev
), fw_rev_sub(&dev
->mdev
));
951 static ssize_t
show_rev(struct device
*device
, struct device_attribute
*attr
,
954 struct mlx5_ib_dev
*dev
=
955 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
956 return sprintf(buf
, "%x\n", dev
->mdev
.rev_id
);
959 static ssize_t
show_board(struct device
*device
, struct device_attribute
*attr
,
962 struct mlx5_ib_dev
*dev
=
963 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
964 return sprintf(buf
, "%.*s\n", MLX5_BOARD_ID_LEN
,
968 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
969 static DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
970 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
971 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
972 static DEVICE_ATTR(fw_pages
, S_IRUGO
, show_fw_pages
, NULL
);
973 static DEVICE_ATTR(reg_pages
, S_IRUGO
, show_reg_pages
, NULL
);
975 static struct device_attribute
*mlx5_class_attributes
[] = {
984 static void mlx5_ib_event(struct mlx5_core_dev
*dev
, enum mlx5_dev_event event
,
987 struct mlx5_ib_dev
*ibdev
= container_of(dev
, struct mlx5_ib_dev
, mdev
);
988 struct ib_event ibev
;
992 case MLX5_DEV_EVENT_SYS_ERROR
:
993 ibdev
->ib_active
= false;
994 ibev
.event
= IB_EVENT_DEVICE_FATAL
;
997 case MLX5_DEV_EVENT_PORT_UP
:
998 ibev
.event
= IB_EVENT_PORT_ACTIVE
;
1002 case MLX5_DEV_EVENT_PORT_DOWN
:
1003 ibev
.event
= IB_EVENT_PORT_ERR
;
1007 case MLX5_DEV_EVENT_PORT_INITIALIZED
:
1008 /* not used by ULPs */
1011 case MLX5_DEV_EVENT_LID_CHANGE
:
1012 ibev
.event
= IB_EVENT_LID_CHANGE
;
1016 case MLX5_DEV_EVENT_PKEY_CHANGE
:
1017 ibev
.event
= IB_EVENT_PKEY_CHANGE
;
1021 case MLX5_DEV_EVENT_GUID_CHANGE
:
1022 ibev
.event
= IB_EVENT_GID_CHANGE
;
1026 case MLX5_DEV_EVENT_CLIENT_REREG
:
1027 ibev
.event
= IB_EVENT_CLIENT_REREGISTER
;
1032 ibev
.device
= &ibdev
->ib_dev
;
1033 ibev
.element
.port_num
= port
;
1035 if (port
< 1 || port
> ibdev
->num_ports
) {
1036 mlx5_ib_warn(ibdev
, "warning: event on port %d\n", port
);
1040 if (ibdev
->ib_active
)
1041 ib_dispatch_event(&ibev
);
1044 static void get_ext_port_caps(struct mlx5_ib_dev
*dev
)
1048 for (port
= 1; port
<= dev
->mdev
.caps
.num_ports
; port
++)
1049 mlx5_query_ext_port_caps(dev
, port
);
1052 static int get_port_caps(struct mlx5_ib_dev
*dev
)
1054 struct ib_device_attr
*dprops
= NULL
;
1055 struct ib_port_attr
*pprops
= NULL
;
1059 pprops
= kmalloc(sizeof(*pprops
), GFP_KERNEL
);
1063 dprops
= kmalloc(sizeof(*dprops
), GFP_KERNEL
);
1067 err
= mlx5_ib_query_device(&dev
->ib_dev
, dprops
);
1069 mlx5_ib_warn(dev
, "query_device failed %d\n", err
);
1073 for (port
= 1; port
<= dev
->mdev
.caps
.num_ports
; port
++) {
1074 err
= mlx5_ib_query_port(&dev
->ib_dev
, port
, pprops
);
1076 mlx5_ib_warn(dev
, "query_port %d failed %d\n", port
, err
);
1079 dev
->mdev
.caps
.port
[port
- 1].pkey_table_len
= dprops
->max_pkeys
;
1080 dev
->mdev
.caps
.port
[port
- 1].gid_table_len
= pprops
->gid_tbl_len
;
1081 mlx5_ib_dbg(dev
, "pkey_table_len %d, gid_table_len %d\n",
1082 dprops
->max_pkeys
, pprops
->gid_tbl_len
);
1092 static void destroy_umrc_res(struct mlx5_ib_dev
*dev
)
1096 err
= mlx5_mr_cache_cleanup(dev
);
1098 mlx5_ib_warn(dev
, "mr cache cleanup failed\n");
1100 mlx5_ib_destroy_qp(dev
->umrc
.qp
);
1101 ib_destroy_cq(dev
->umrc
.cq
);
1102 ib_dereg_mr(dev
->umrc
.mr
);
1103 ib_dealloc_pd(dev
->umrc
.pd
);
1110 static int create_umr_res(struct mlx5_ib_dev
*dev
)
1112 struct ib_qp_init_attr
*init_attr
= NULL
;
1113 struct ib_qp_attr
*attr
= NULL
;
1120 attr
= kzalloc(sizeof(*attr
), GFP_KERNEL
);
1121 init_attr
= kzalloc(sizeof(*init_attr
), GFP_KERNEL
);
1122 if (!attr
|| !init_attr
) {
1127 pd
= ib_alloc_pd(&dev
->ib_dev
);
1129 mlx5_ib_dbg(dev
, "Couldn't create PD for sync UMR QP\n");
1134 mr
= ib_get_dma_mr(pd
, IB_ACCESS_LOCAL_WRITE
);
1136 mlx5_ib_dbg(dev
, "Couldn't create DMA MR for sync UMR QP\n");
1141 cq
= ib_create_cq(&dev
->ib_dev
, mlx5_umr_cq_handler
, NULL
, NULL
, 128,
1144 mlx5_ib_dbg(dev
, "Couldn't create CQ for sync UMR QP\n");
1148 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
1150 init_attr
->send_cq
= cq
;
1151 init_attr
->recv_cq
= cq
;
1152 init_attr
->sq_sig_type
= IB_SIGNAL_ALL_WR
;
1153 init_attr
->cap
.max_send_wr
= MAX_UMR_WR
;
1154 init_attr
->cap
.max_send_sge
= 1;
1155 init_attr
->qp_type
= MLX5_IB_QPT_REG_UMR
;
1156 init_attr
->port_num
= 1;
1157 qp
= mlx5_ib_create_qp(pd
, init_attr
, NULL
);
1159 mlx5_ib_dbg(dev
, "Couldn't create sync UMR QP\n");
1163 qp
->device
= &dev
->ib_dev
;
1166 qp
->qp_type
= MLX5_IB_QPT_REG_UMR
;
1168 attr
->qp_state
= IB_QPS_INIT
;
1170 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
| IB_QP_PKEY_INDEX
|
1173 mlx5_ib_dbg(dev
, "Couldn't modify UMR QP\n");
1177 memset(attr
, 0, sizeof(*attr
));
1178 attr
->qp_state
= IB_QPS_RTR
;
1179 attr
->path_mtu
= IB_MTU_256
;
1181 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
, NULL
);
1183 mlx5_ib_dbg(dev
, "Couldn't modify umr QP to rtr\n");
1187 memset(attr
, 0, sizeof(*attr
));
1188 attr
->qp_state
= IB_QPS_RTS
;
1189 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
, NULL
);
1191 mlx5_ib_dbg(dev
, "Couldn't modify umr QP to rts\n");
1200 sema_init(&dev
->umrc
.sem
, MAX_UMR_WR
);
1201 ret
= mlx5_mr_cache_init(dev
);
1203 mlx5_ib_warn(dev
, "mr cache init failed %d\n", ret
);
1213 mlx5_ib_destroy_qp(qp
);
1230 static int create_dev_resources(struct mlx5_ib_resources
*devr
)
1232 struct ib_srq_init_attr attr
;
1233 struct mlx5_ib_dev
*dev
;
1236 dev
= container_of(devr
, struct mlx5_ib_dev
, devr
);
1238 devr
->p0
= mlx5_ib_alloc_pd(&dev
->ib_dev
, NULL
, NULL
);
1239 if (IS_ERR(devr
->p0
)) {
1240 ret
= PTR_ERR(devr
->p0
);
1243 devr
->p0
->device
= &dev
->ib_dev
;
1244 devr
->p0
->uobject
= NULL
;
1245 atomic_set(&devr
->p0
->usecnt
, 0);
1247 devr
->c0
= mlx5_ib_create_cq(&dev
->ib_dev
, 1, 0, NULL
, NULL
);
1248 if (IS_ERR(devr
->c0
)) {
1249 ret
= PTR_ERR(devr
->c0
);
1252 devr
->c0
->device
= &dev
->ib_dev
;
1253 devr
->c0
->uobject
= NULL
;
1254 devr
->c0
->comp_handler
= NULL
;
1255 devr
->c0
->event_handler
= NULL
;
1256 devr
->c0
->cq_context
= NULL
;
1257 atomic_set(&devr
->c0
->usecnt
, 0);
1259 devr
->x0
= mlx5_ib_alloc_xrcd(&dev
->ib_dev
, NULL
, NULL
);
1260 if (IS_ERR(devr
->x0
)) {
1261 ret
= PTR_ERR(devr
->x0
);
1264 devr
->x0
->device
= &dev
->ib_dev
;
1265 devr
->x0
->inode
= NULL
;
1266 atomic_set(&devr
->x0
->usecnt
, 0);
1267 mutex_init(&devr
->x0
->tgt_qp_mutex
);
1268 INIT_LIST_HEAD(&devr
->x0
->tgt_qp_list
);
1270 devr
->x1
= mlx5_ib_alloc_xrcd(&dev
->ib_dev
, NULL
, NULL
);
1271 if (IS_ERR(devr
->x1
)) {
1272 ret
= PTR_ERR(devr
->x1
);
1275 devr
->x1
->device
= &dev
->ib_dev
;
1276 devr
->x1
->inode
= NULL
;
1277 atomic_set(&devr
->x1
->usecnt
, 0);
1278 mutex_init(&devr
->x1
->tgt_qp_mutex
);
1279 INIT_LIST_HEAD(&devr
->x1
->tgt_qp_list
);
1281 memset(&attr
, 0, sizeof(attr
));
1282 attr
.attr
.max_sge
= 1;
1283 attr
.attr
.max_wr
= 1;
1284 attr
.srq_type
= IB_SRQT_XRC
;
1285 attr
.ext
.xrc
.cq
= devr
->c0
;
1286 attr
.ext
.xrc
.xrcd
= devr
->x0
;
1288 devr
->s0
= mlx5_ib_create_srq(devr
->p0
, &attr
, NULL
);
1289 if (IS_ERR(devr
->s0
)) {
1290 ret
= PTR_ERR(devr
->s0
);
1293 devr
->s0
->device
= &dev
->ib_dev
;
1294 devr
->s0
->pd
= devr
->p0
;
1295 devr
->s0
->uobject
= NULL
;
1296 devr
->s0
->event_handler
= NULL
;
1297 devr
->s0
->srq_context
= NULL
;
1298 devr
->s0
->srq_type
= IB_SRQT_XRC
;
1299 devr
->s0
->ext
.xrc
.xrcd
= devr
->x0
;
1300 devr
->s0
->ext
.xrc
.cq
= devr
->c0
;
1301 atomic_inc(&devr
->s0
->ext
.xrc
.xrcd
->usecnt
);
1302 atomic_inc(&devr
->s0
->ext
.xrc
.cq
->usecnt
);
1303 atomic_inc(&devr
->p0
->usecnt
);
1304 atomic_set(&devr
->s0
->usecnt
, 0);
1309 mlx5_ib_dealloc_xrcd(devr
->x1
);
1311 mlx5_ib_dealloc_xrcd(devr
->x0
);
1313 mlx5_ib_destroy_cq(devr
->c0
);
1315 mlx5_ib_dealloc_pd(devr
->p0
);
1320 static void destroy_dev_resources(struct mlx5_ib_resources
*devr
)
1322 mlx5_ib_destroy_srq(devr
->s0
);
1323 mlx5_ib_dealloc_xrcd(devr
->x0
);
1324 mlx5_ib_dealloc_xrcd(devr
->x1
);
1325 mlx5_ib_destroy_cq(devr
->c0
);
1326 mlx5_ib_dealloc_pd(devr
->p0
);
1329 static int init_one(struct pci_dev
*pdev
,
1330 const struct pci_device_id
*id
)
1332 struct mlx5_core_dev
*mdev
;
1333 struct mlx5_ib_dev
*dev
;
1337 printk_once(KERN_INFO
"%s", mlx5_version
);
1339 dev
= (struct mlx5_ib_dev
*)ib_alloc_device(sizeof(*dev
));
1344 mdev
->event
= mlx5_ib_event
;
1345 if (prof_sel
>= ARRAY_SIZE(profile
)) {
1346 pr_warn("selected pofile out of range, selceting default\n");
1349 mdev
->profile
= &profile
[prof_sel
];
1350 err
= mlx5_dev_init(mdev
, pdev
);
1354 err
= get_port_caps(dev
);
1358 get_ext_port_caps(dev
);
1360 err
= alloc_comp_eqs(dev
);
1364 MLX5_INIT_DOORBELL_LOCK(&dev
->uar_lock
);
1366 strlcpy(dev
->ib_dev
.name
, "mlx5_%d", IB_DEVICE_NAME_MAX
);
1367 dev
->ib_dev
.owner
= THIS_MODULE
;
1368 dev
->ib_dev
.node_type
= RDMA_NODE_IB_CA
;
1369 dev
->ib_dev
.local_dma_lkey
= mdev
->caps
.reserved_lkey
;
1370 dev
->num_ports
= mdev
->caps
.num_ports
;
1371 dev
->ib_dev
.phys_port_cnt
= dev
->num_ports
;
1372 dev
->ib_dev
.num_comp_vectors
= dev
->num_comp_vectors
;
1373 dev
->ib_dev
.dma_device
= &mdev
->pdev
->dev
;
1375 dev
->ib_dev
.uverbs_abi_ver
= MLX5_IB_UVERBS_ABI_VERSION
;
1376 dev
->ib_dev
.uverbs_cmd_mask
=
1377 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
1378 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
1379 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
1380 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
1381 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
1382 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
1383 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
1384 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
1385 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
1386 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
1387 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
1388 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
1389 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
1390 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
1391 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
1392 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST
) |
1393 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST
) |
1394 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
1395 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
1396 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
1397 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
1398 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ
) |
1399 (1ull << IB_USER_VERBS_CMD_OPEN_QP
);
1401 dev
->ib_dev
.query_device
= mlx5_ib_query_device
;
1402 dev
->ib_dev
.query_port
= mlx5_ib_query_port
;
1403 dev
->ib_dev
.query_gid
= mlx5_ib_query_gid
;
1404 dev
->ib_dev
.query_pkey
= mlx5_ib_query_pkey
;
1405 dev
->ib_dev
.modify_device
= mlx5_ib_modify_device
;
1406 dev
->ib_dev
.modify_port
= mlx5_ib_modify_port
;
1407 dev
->ib_dev
.alloc_ucontext
= mlx5_ib_alloc_ucontext
;
1408 dev
->ib_dev
.dealloc_ucontext
= mlx5_ib_dealloc_ucontext
;
1409 dev
->ib_dev
.mmap
= mlx5_ib_mmap
;
1410 dev
->ib_dev
.alloc_pd
= mlx5_ib_alloc_pd
;
1411 dev
->ib_dev
.dealloc_pd
= mlx5_ib_dealloc_pd
;
1412 dev
->ib_dev
.create_ah
= mlx5_ib_create_ah
;
1413 dev
->ib_dev
.query_ah
= mlx5_ib_query_ah
;
1414 dev
->ib_dev
.destroy_ah
= mlx5_ib_destroy_ah
;
1415 dev
->ib_dev
.create_srq
= mlx5_ib_create_srq
;
1416 dev
->ib_dev
.modify_srq
= mlx5_ib_modify_srq
;
1417 dev
->ib_dev
.query_srq
= mlx5_ib_query_srq
;
1418 dev
->ib_dev
.destroy_srq
= mlx5_ib_destroy_srq
;
1419 dev
->ib_dev
.post_srq_recv
= mlx5_ib_post_srq_recv
;
1420 dev
->ib_dev
.create_qp
= mlx5_ib_create_qp
;
1421 dev
->ib_dev
.modify_qp
= mlx5_ib_modify_qp
;
1422 dev
->ib_dev
.query_qp
= mlx5_ib_query_qp
;
1423 dev
->ib_dev
.destroy_qp
= mlx5_ib_destroy_qp
;
1424 dev
->ib_dev
.post_send
= mlx5_ib_post_send
;
1425 dev
->ib_dev
.post_recv
= mlx5_ib_post_recv
;
1426 dev
->ib_dev
.create_cq
= mlx5_ib_create_cq
;
1427 dev
->ib_dev
.modify_cq
= mlx5_ib_modify_cq
;
1428 dev
->ib_dev
.resize_cq
= mlx5_ib_resize_cq
;
1429 dev
->ib_dev
.destroy_cq
= mlx5_ib_destroy_cq
;
1430 dev
->ib_dev
.poll_cq
= mlx5_ib_poll_cq
;
1431 dev
->ib_dev
.req_notify_cq
= mlx5_ib_arm_cq
;
1432 dev
->ib_dev
.get_dma_mr
= mlx5_ib_get_dma_mr
;
1433 dev
->ib_dev
.reg_user_mr
= mlx5_ib_reg_user_mr
;
1434 dev
->ib_dev
.dereg_mr
= mlx5_ib_dereg_mr
;
1435 dev
->ib_dev
.destroy_mr
= mlx5_ib_destroy_mr
;
1436 dev
->ib_dev
.attach_mcast
= mlx5_ib_mcg_attach
;
1437 dev
->ib_dev
.detach_mcast
= mlx5_ib_mcg_detach
;
1438 dev
->ib_dev
.process_mad
= mlx5_ib_process_mad
;
1439 dev
->ib_dev
.create_mr
= mlx5_ib_create_mr
;
1440 dev
->ib_dev
.alloc_fast_reg_mr
= mlx5_ib_alloc_fast_reg_mr
;
1441 dev
->ib_dev
.alloc_fast_reg_page_list
= mlx5_ib_alloc_fast_reg_page_list
;
1442 dev
->ib_dev
.free_fast_reg_page_list
= mlx5_ib_free_fast_reg_page_list
;
1443 dev
->ib_dev
.check_mr_status
= mlx5_ib_check_mr_status
;
1445 if (mdev
->caps
.flags
& MLX5_DEV_CAP_FLAG_XRC
) {
1446 dev
->ib_dev
.alloc_xrcd
= mlx5_ib_alloc_xrcd
;
1447 dev
->ib_dev
.dealloc_xrcd
= mlx5_ib_dealloc_xrcd
;
1448 dev
->ib_dev
.uverbs_cmd_mask
|=
1449 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD
) |
1450 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD
);
1453 err
= init_node_data(dev
);
1457 mutex_init(&dev
->cap_mask_mutex
);
1458 spin_lock_init(&dev
->mr_lock
);
1460 err
= create_dev_resources(&dev
->devr
);
1464 err
= ib_register_device(&dev
->ib_dev
, NULL
);
1468 err
= create_umr_res(dev
);
1472 for (i
= 0; i
< ARRAY_SIZE(mlx5_class_attributes
); i
++) {
1473 err
= device_create_file(&dev
->ib_dev
.dev
,
1474 mlx5_class_attributes
[i
]);
1479 dev
->ib_active
= true;
1484 destroy_umrc_res(dev
);
1487 ib_unregister_device(&dev
->ib_dev
);
1490 destroy_dev_resources(&dev
->devr
);
1496 mlx5_dev_cleanup(mdev
);
1499 ib_dealloc_device((struct ib_device
*)dev
);
1504 static void remove_one(struct pci_dev
*pdev
)
1506 struct mlx5_ib_dev
*dev
= mlx5_pci2ibdev(pdev
);
1508 destroy_umrc_res(dev
);
1509 ib_unregister_device(&dev
->ib_dev
);
1510 destroy_dev_resources(&dev
->devr
);
1512 mlx5_dev_cleanup(&dev
->mdev
);
1513 ib_dealloc_device(&dev
->ib_dev
);
1516 static DEFINE_PCI_DEVICE_TABLE(mlx5_ib_pci_table
) = {
1517 { PCI_VDEVICE(MELLANOX
, 4113) }, /* MT4113 Connect-IB */
1521 MODULE_DEVICE_TABLE(pci
, mlx5_ib_pci_table
);
1523 static struct pci_driver mlx5_ib_driver
= {
1524 .name
= DRIVER_NAME
,
1525 .id_table
= mlx5_ib_pci_table
,
1527 .remove
= remove_one
1530 static int __init
mlx5_ib_init(void)
1532 return pci_register_driver(&mlx5_ib_driver
);
1535 static void __exit
mlx5_ib_cleanup(void)
1537 pci_unregister_driver(&mlx5_ib_driver
);
1540 module_init(mlx5_ib_init
);
1541 module_exit(mlx5_ib_cleanup
);