2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <asm-generic/kmap_types.h>
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/errno.h>
37 #include <linux/pci.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/slab.h>
40 #include <linux/io-mapping.h>
41 #include <linux/sched.h>
42 #include <rdma/ib_user_verbs.h>
43 #include <rdma/ib_smi.h>
44 #include <rdma/ib_umem.h>
48 #define DRIVER_NAME "mlx5_ib"
49 #define DRIVER_VERSION "1.0"
50 #define DRIVER_RELDATE "June 2013"
52 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
53 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
54 MODULE_LICENSE("Dual BSD/GPL");
55 MODULE_VERSION(DRIVER_VERSION
);
57 static int prof_sel
= 2;
58 module_param_named(prof_sel
, prof_sel
, int, 0444);
59 MODULE_PARM_DESC(prof_sel
, "profile selector. Valid range 0 - 2");
61 static char mlx5_version
[] =
62 DRIVER_NAME
": Mellanox Connect-IB Infiniband driver v"
63 DRIVER_VERSION
" (" DRIVER_RELDATE
")\n";
65 static struct mlx5_profile profile
[] = {
70 .mask
= MLX5_PROF_MASK_QP_SIZE
,
74 .mask
= MLX5_PROF_MASK_QP_SIZE
|
75 MLX5_PROF_MASK_MR_CACHE
,
144 int mlx5_vector2eqn(struct mlx5_ib_dev
*dev
, int vector
, int *eqn
, int *irqn
)
146 struct mlx5_eq_table
*table
= &dev
->mdev
.priv
.eq_table
;
147 struct mlx5_eq
*eq
, *n
;
150 spin_lock(&table
->lock
);
151 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
152 if (eq
->index
== vector
) {
159 spin_unlock(&table
->lock
);
164 static int alloc_comp_eqs(struct mlx5_ib_dev
*dev
)
166 struct mlx5_eq_table
*table
= &dev
->mdev
.priv
.eq_table
;
167 char name
[MLX5_MAX_EQ_NAME
];
168 struct mlx5_eq
*eq
, *n
;
174 INIT_LIST_HEAD(&dev
->eqs_list
);
175 ncomp_vec
= table
->num_comp_vectors
;
176 nent
= MLX5_COMP_EQ_SIZE
;
177 for (i
= 0; i
< ncomp_vec
; i
++) {
178 eq
= kzalloc(sizeof(*eq
), GFP_KERNEL
);
184 snprintf(name
, MLX5_MAX_EQ_NAME
, "mlx5_comp%d", i
);
185 err
= mlx5_create_map_eq(&dev
->mdev
, eq
,
186 i
+ MLX5_EQ_VEC_COMP_BASE
, nent
, 0,
187 name
, &dev
->mdev
.priv
.uuari
.uars
[0]);
192 mlx5_ib_dbg(dev
, "allocated completion EQN %d\n", eq
->eqn
);
194 spin_lock(&table
->lock
);
195 list_add_tail(&eq
->list
, &dev
->eqs_list
);
196 spin_unlock(&table
->lock
);
199 dev
->num_comp_vectors
= ncomp_vec
;
203 spin_lock(&table
->lock
);
204 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
206 spin_unlock(&table
->lock
);
207 if (mlx5_destroy_unmap_eq(&dev
->mdev
, eq
))
208 mlx5_ib_warn(dev
, "failed to destroy EQ 0x%x\n", eq
->eqn
);
210 spin_lock(&table
->lock
);
212 spin_unlock(&table
->lock
);
216 static void free_comp_eqs(struct mlx5_ib_dev
*dev
)
218 struct mlx5_eq_table
*table
= &dev
->mdev
.priv
.eq_table
;
219 struct mlx5_eq
*eq
, *n
;
221 spin_lock(&table
->lock
);
222 list_for_each_entry_safe(eq
, n
, &dev
->eqs_list
, list
) {
224 spin_unlock(&table
->lock
);
225 if (mlx5_destroy_unmap_eq(&dev
->mdev
, eq
))
226 mlx5_ib_warn(dev
, "failed to destroy EQ 0x%x\n", eq
->eqn
);
228 spin_lock(&table
->lock
);
230 spin_unlock(&table
->lock
);
233 static int mlx5_ib_query_device(struct ib_device
*ibdev
,
234 struct ib_device_attr
*props
)
236 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
237 struct ib_smp
*in_mad
= NULL
;
238 struct ib_smp
*out_mad
= NULL
;
244 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
245 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
246 if (!in_mad
|| !out_mad
)
249 init_query_mad(in_mad
);
250 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
252 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
256 memset(props
, 0, sizeof(*props
));
258 props
->fw_ver
= ((u64
)fw_rev_maj(&dev
->mdev
) << 32) |
259 (fw_rev_min(&dev
->mdev
) << 16) |
260 fw_rev_sub(&dev
->mdev
);
261 props
->device_cap_flags
= IB_DEVICE_CHANGE_PHY_PORT
|
262 IB_DEVICE_PORT_ACTIVE_EVENT
|
263 IB_DEVICE_SYS_IMAGE_GUID
|
264 IB_DEVICE_RC_RNR_NAK_GEN
|
265 IB_DEVICE_BLOCK_MULTICAST_LOOPBACK
;
266 flags
= dev
->mdev
.caps
.flags
;
267 if (flags
& MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR
)
268 props
->device_cap_flags
|= IB_DEVICE_BAD_PKEY_CNTR
;
269 if (flags
& MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR
)
270 props
->device_cap_flags
|= IB_DEVICE_BAD_QKEY_CNTR
;
271 if (flags
& MLX5_DEV_CAP_FLAG_APM
)
272 props
->device_cap_flags
|= IB_DEVICE_AUTO_PATH_MIG
;
273 props
->device_cap_flags
|= IB_DEVICE_LOCAL_DMA_LKEY
;
274 if (flags
& MLX5_DEV_CAP_FLAG_XRC
)
275 props
->device_cap_flags
|= IB_DEVICE_XRC
;
276 props
->device_cap_flags
|= IB_DEVICE_MEM_MGT_EXTENSIONS
;
278 props
->vendor_id
= be32_to_cpup((__be32
*)(out_mad
->data
+ 36)) &
280 props
->vendor_part_id
= be16_to_cpup((__be16
*)(out_mad
->data
+ 30));
281 props
->hw_ver
= be32_to_cpup((__be32
*)(out_mad
->data
+ 32));
282 memcpy(&props
->sys_image_guid
, out_mad
->data
+ 4, 8);
284 props
->max_mr_size
= ~0ull;
285 props
->page_size_cap
= dev
->mdev
.caps
.min_page_sz
;
286 props
->max_qp
= 1 << dev
->mdev
.caps
.log_max_qp
;
287 props
->max_qp_wr
= dev
->mdev
.caps
.max_wqes
;
288 max_rq_sg
= dev
->mdev
.caps
.max_rq_desc_sz
/ sizeof(struct mlx5_wqe_data_seg
);
289 max_sq_sg
= (dev
->mdev
.caps
.max_sq_desc_sz
- sizeof(struct mlx5_wqe_ctrl_seg
)) /
290 sizeof(struct mlx5_wqe_data_seg
);
291 props
->max_sge
= min(max_rq_sg
, max_sq_sg
);
292 props
->max_cq
= 1 << dev
->mdev
.caps
.log_max_cq
;
293 props
->max_cqe
= dev
->mdev
.caps
.max_cqes
- 1;
294 props
->max_mr
= 1 << dev
->mdev
.caps
.log_max_mkey
;
295 props
->max_pd
= 1 << dev
->mdev
.caps
.log_max_pd
;
296 props
->max_qp_rd_atom
= dev
->mdev
.caps
.max_ra_req_qp
;
297 props
->max_qp_init_rd_atom
= dev
->mdev
.caps
.max_ra_res_qp
;
298 props
->max_res_rd_atom
= props
->max_qp_rd_atom
* props
->max_qp
;
299 props
->max_srq
= 1 << dev
->mdev
.caps
.log_max_srq
;
300 props
->max_srq_wr
= dev
->mdev
.caps
.max_srq_wqes
- 1;
301 props
->max_srq_sge
= max_rq_sg
- 1;
302 props
->max_fast_reg_page_list_len
= (unsigned int)-1;
303 props
->local_ca_ack_delay
= dev
->mdev
.caps
.local_ca_ack_delay
;
304 props
->atomic_cap
= IB_ATOMIC_NONE
;
305 props
->masked_atomic_cap
= IB_ATOMIC_NONE
;
306 props
->max_pkeys
= be16_to_cpup((__be16
*)(out_mad
->data
+ 28));
307 props
->max_mcast_grp
= 1 << dev
->mdev
.caps
.log_max_mcg
;
308 props
->max_mcast_qp_attach
= dev
->mdev
.caps
.max_qp_mcg
;
309 props
->max_total_mcast_qp_attach
= props
->max_mcast_qp_attach
*
310 props
->max_mcast_grp
;
311 props
->max_map_per_fmr
= INT_MAX
; /* no limit in ConnectIB */
320 int mlx5_ib_query_port(struct ib_device
*ibdev
, u8 port
,
321 struct ib_port_attr
*props
)
323 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
324 struct ib_smp
*in_mad
= NULL
;
325 struct ib_smp
*out_mad
= NULL
;
326 int ext_active_speed
;
329 if (port
< 1 || port
> dev
->mdev
.caps
.num_ports
) {
330 mlx5_ib_warn(dev
, "invalid port number %d\n", port
);
334 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
335 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
336 if (!in_mad
|| !out_mad
)
339 memset(props
, 0, sizeof(*props
));
341 init_query_mad(in_mad
);
342 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
343 in_mad
->attr_mod
= cpu_to_be32(port
);
345 err
= mlx5_MAD_IFC(dev
, 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
347 mlx5_ib_warn(dev
, "err %d\n", err
);
352 props
->lid
= be16_to_cpup((__be16
*)(out_mad
->data
+ 16));
353 props
->lmc
= out_mad
->data
[34] & 0x7;
354 props
->sm_lid
= be16_to_cpup((__be16
*)(out_mad
->data
+ 18));
355 props
->sm_sl
= out_mad
->data
[36] & 0xf;
356 props
->state
= out_mad
->data
[32] & 0xf;
357 props
->phys_state
= out_mad
->data
[33] >> 4;
358 props
->port_cap_flags
= be32_to_cpup((__be32
*)(out_mad
->data
+ 20));
359 props
->gid_tbl_len
= out_mad
->data
[50];
360 props
->max_msg_sz
= 1 << to_mdev(ibdev
)->mdev
.caps
.log_max_msg
;
361 props
->pkey_tbl_len
= to_mdev(ibdev
)->mdev
.caps
.port
[port
- 1].pkey_table_len
;
362 props
->bad_pkey_cntr
= be16_to_cpup((__be16
*)(out_mad
->data
+ 46));
363 props
->qkey_viol_cntr
= be16_to_cpup((__be16
*)(out_mad
->data
+ 48));
364 props
->active_width
= out_mad
->data
[31] & 0xf;
365 props
->active_speed
= out_mad
->data
[35] >> 4;
366 props
->max_mtu
= out_mad
->data
[41] & 0xf;
367 props
->active_mtu
= out_mad
->data
[36] >> 4;
368 props
->subnet_timeout
= out_mad
->data
[51] & 0x1f;
369 props
->max_vl_num
= out_mad
->data
[37] >> 4;
370 props
->init_type_reply
= out_mad
->data
[41] >> 4;
372 /* Check if extended speeds (EDR/FDR/...) are supported */
373 if (props
->port_cap_flags
& IB_PORT_EXTENDED_SPEEDS_SUP
) {
374 ext_active_speed
= out_mad
->data
[62] >> 4;
376 switch (ext_active_speed
) {
378 props
->active_speed
= 16; /* FDR */
381 props
->active_speed
= 32; /* EDR */
386 /* If reported active speed is QDR, check if is FDR-10 */
387 if (props
->active_speed
== 4) {
388 if (dev
->mdev
.caps
.ext_port_cap
[port
- 1] &
389 MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO
) {
390 init_query_mad(in_mad
);
391 in_mad
->attr_id
= MLX5_ATTR_EXTENDED_PORT_INFO
;
392 in_mad
->attr_mod
= cpu_to_be32(port
);
394 err
= mlx5_MAD_IFC(dev
, 1, 1, port
,
395 NULL
, NULL
, in_mad
, out_mad
);
399 /* Checking LinkSpeedActive for FDR-10 */
400 if (out_mad
->data
[15] & 0x1)
401 props
->active_speed
= 8;
412 static int mlx5_ib_query_gid(struct ib_device
*ibdev
, u8 port
, int index
,
415 struct ib_smp
*in_mad
= NULL
;
416 struct ib_smp
*out_mad
= NULL
;
419 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
420 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
421 if (!in_mad
|| !out_mad
)
424 init_query_mad(in_mad
);
425 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
426 in_mad
->attr_mod
= cpu_to_be32(port
);
428 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
432 memcpy(gid
->raw
, out_mad
->data
+ 8, 8);
434 init_query_mad(in_mad
);
435 in_mad
->attr_id
= IB_SMP_ATTR_GUID_INFO
;
436 in_mad
->attr_mod
= cpu_to_be32(index
/ 8);
438 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
442 memcpy(gid
->raw
+ 8, out_mad
->data
+ (index
% 8) * 8, 8);
450 static int mlx5_ib_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
453 struct ib_smp
*in_mad
= NULL
;
454 struct ib_smp
*out_mad
= NULL
;
457 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
458 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
459 if (!in_mad
|| !out_mad
)
462 init_query_mad(in_mad
);
463 in_mad
->attr_id
= IB_SMP_ATTR_PKEY_TABLE
;
464 in_mad
->attr_mod
= cpu_to_be32(index
/ 32);
466 err
= mlx5_MAD_IFC(to_mdev(ibdev
), 1, 1, port
, NULL
, NULL
, in_mad
, out_mad
);
470 *pkey
= be16_to_cpu(((__be16
*)out_mad
->data
)[index
% 32]);
478 struct mlx5_reg_node_desc
{
482 static int mlx5_ib_modify_device(struct ib_device
*ibdev
, int mask
,
483 struct ib_device_modify
*props
)
485 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
486 struct mlx5_reg_node_desc in
;
487 struct mlx5_reg_node_desc out
;
490 if (mask
& ~IB_DEVICE_MODIFY_NODE_DESC
)
493 if (!(mask
& IB_DEVICE_MODIFY_NODE_DESC
))
497 * If possible, pass node desc to FW, so it can generate
498 * a 144 trap. If cmd fails, just ignore.
500 memcpy(&in
, props
->node_desc
, 64);
501 err
= mlx5_core_access_reg(&dev
->mdev
, &in
, sizeof(in
), &out
,
502 sizeof(out
), MLX5_REG_NODE_DESC
, 0, 1);
506 memcpy(ibdev
->node_desc
, props
->node_desc
, 64);
511 static int mlx5_ib_modify_port(struct ib_device
*ibdev
, u8 port
, int mask
,
512 struct ib_port_modify
*props
)
514 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
515 struct ib_port_attr attr
;
519 mutex_lock(&dev
->cap_mask_mutex
);
521 err
= mlx5_ib_query_port(ibdev
, port
, &attr
);
525 tmp
= (attr
.port_cap_flags
| props
->set_port_cap_mask
) &
526 ~props
->clr_port_cap_mask
;
528 err
= mlx5_set_port_caps(&dev
->mdev
, port
, tmp
);
531 mutex_unlock(&dev
->cap_mask_mutex
);
535 static struct ib_ucontext
*mlx5_ib_alloc_ucontext(struct ib_device
*ibdev
,
536 struct ib_udata
*udata
)
538 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
539 struct mlx5_ib_alloc_ucontext_req req
;
540 struct mlx5_ib_alloc_ucontext_resp resp
;
541 struct mlx5_ib_ucontext
*context
;
542 struct mlx5_uuar_info
*uuari
;
543 struct mlx5_uar
*uars
;
551 return ERR_PTR(-EAGAIN
);
553 err
= ib_copy_from_udata(&req
, udata
, sizeof(req
));
557 if (req
.total_num_uuars
> MLX5_MAX_UUARS
)
558 return ERR_PTR(-ENOMEM
);
560 if (req
.total_num_uuars
== 0)
561 return ERR_PTR(-EINVAL
);
563 req
.total_num_uuars
= ALIGN(req
.total_num_uuars
,
564 MLX5_NON_FP_BF_REGS_PER_PAGE
);
565 if (req
.num_low_latency_uuars
> req
.total_num_uuars
- 1)
566 return ERR_PTR(-EINVAL
);
568 num_uars
= req
.total_num_uuars
/ MLX5_NON_FP_BF_REGS_PER_PAGE
;
569 gross_uuars
= num_uars
* MLX5_BF_REGS_PER_PAGE
;
570 resp
.qp_tab_size
= 1 << dev
->mdev
.caps
.log_max_qp
;
571 resp
.bf_reg_size
= dev
->mdev
.caps
.bf_reg_size
;
572 resp
.cache_line_size
= L1_CACHE_BYTES
;
573 resp
.max_sq_desc_sz
= dev
->mdev
.caps
.max_sq_desc_sz
;
574 resp
.max_rq_desc_sz
= dev
->mdev
.caps
.max_rq_desc_sz
;
575 resp
.max_send_wqebb
= dev
->mdev
.caps
.max_wqes
;
576 resp
.max_recv_wr
= dev
->mdev
.caps
.max_wqes
;
577 resp
.max_srq_recv_wr
= dev
->mdev
.caps
.max_srq_wqes
;
579 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
581 return ERR_PTR(-ENOMEM
);
583 uuari
= &context
->uuari
;
584 mutex_init(&uuari
->lock
);
585 uars
= kcalloc(num_uars
, sizeof(*uars
), GFP_KERNEL
);
591 uuari
->bitmap
= kcalloc(BITS_TO_LONGS(gross_uuars
),
592 sizeof(*uuari
->bitmap
),
594 if (!uuari
->bitmap
) {
599 * clear all fast path uuars
601 for (i
= 0; i
< gross_uuars
; i
++) {
603 if (uuarn
== 2 || uuarn
== 3)
604 set_bit(i
, uuari
->bitmap
);
607 uuari
->count
= kcalloc(gross_uuars
, sizeof(*uuari
->count
), GFP_KERNEL
);
613 for (i
= 0; i
< num_uars
; i
++) {
614 err
= mlx5_cmd_alloc_uar(&dev
->mdev
, &uars
[i
].index
);
619 INIT_LIST_HEAD(&context
->db_page_list
);
620 mutex_init(&context
->db_page_mutex
);
622 resp
.tot_uuars
= req
.total_num_uuars
;
623 resp
.num_ports
= dev
->mdev
.caps
.num_ports
;
624 err
= ib_copy_to_udata(udata
, &resp
,
625 sizeof(resp
) - sizeof(resp
.reserved
));
629 uuari
->num_low_latency_uuars
= req
.num_low_latency_uuars
;
631 uuari
->num_uars
= num_uars
;
632 return &context
->ibucontext
;
635 for (i
--; i
>= 0; i
--)
636 mlx5_cmd_free_uar(&dev
->mdev
, uars
[i
].index
);
641 kfree(uuari
->bitmap
);
651 static int mlx5_ib_dealloc_ucontext(struct ib_ucontext
*ibcontext
)
653 struct mlx5_ib_ucontext
*context
= to_mucontext(ibcontext
);
654 struct mlx5_ib_dev
*dev
= to_mdev(ibcontext
->device
);
655 struct mlx5_uuar_info
*uuari
= &context
->uuari
;
658 for (i
= 0; i
< uuari
->num_uars
; i
++) {
659 if (mlx5_cmd_free_uar(&dev
->mdev
, uuari
->uars
[i
].index
))
660 mlx5_ib_warn(dev
, "failed to free UAR 0x%x\n", uuari
->uars
[i
].index
);
664 kfree(uuari
->bitmap
);
671 static phys_addr_t
uar_index2pfn(struct mlx5_ib_dev
*dev
, int index
)
673 return (pci_resource_start(dev
->mdev
.pdev
, 0) >> PAGE_SHIFT
) + index
;
676 static int get_command(unsigned long offset
)
678 return (offset
>> MLX5_IB_MMAP_CMD_SHIFT
) & MLX5_IB_MMAP_CMD_MASK
;
681 static int get_arg(unsigned long offset
)
683 return offset
& ((1 << MLX5_IB_MMAP_CMD_SHIFT
) - 1);
686 static int get_index(unsigned long offset
)
688 return get_arg(offset
);
691 static int mlx5_ib_mmap(struct ib_ucontext
*ibcontext
, struct vm_area_struct
*vma
)
693 struct mlx5_ib_ucontext
*context
= to_mucontext(ibcontext
);
694 struct mlx5_ib_dev
*dev
= to_mdev(ibcontext
->device
);
695 struct mlx5_uuar_info
*uuari
= &context
->uuari
;
696 unsigned long command
;
700 command
= get_command(vma
->vm_pgoff
);
702 case MLX5_IB_MMAP_REGULAR_PAGE
:
703 if (vma
->vm_end
- vma
->vm_start
!= PAGE_SIZE
)
706 idx
= get_index(vma
->vm_pgoff
);
707 pfn
= uar_index2pfn(dev
, uuari
->uars
[idx
].index
);
708 mlx5_ib_dbg(dev
, "uar idx 0x%lx, pfn 0x%llx\n", idx
,
709 (unsigned long long)pfn
);
711 if (idx
>= uuari
->num_uars
)
714 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
715 if (io_remap_pfn_range(vma
, vma
->vm_start
, pfn
,
716 PAGE_SIZE
, vma
->vm_page_prot
))
719 mlx5_ib_dbg(dev
, "mapped WC at 0x%lx, PA 0x%llx\n",
721 (unsigned long long)pfn
<< PAGE_SHIFT
);
724 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES
:
734 static int alloc_pa_mkey(struct mlx5_ib_dev
*dev
, u32
*key
, u32 pdn
)
736 struct mlx5_create_mkey_mbox_in
*in
;
737 struct mlx5_mkey_seg
*seg
;
738 struct mlx5_core_mr mr
;
741 in
= kzalloc(sizeof(*in
), GFP_KERNEL
);
746 seg
->flags
= MLX5_PERM_LOCAL_READ
| MLX5_ACCESS_MODE_PA
;
747 seg
->flags_pd
= cpu_to_be32(pdn
| MLX5_MKEY_LEN64
);
748 seg
->qpn_mkey7_0
= cpu_to_be32(0xffffff << 8);
751 err
= mlx5_core_create_mkey(&dev
->mdev
, &mr
, in
, sizeof(*in
),
754 mlx5_ib_warn(dev
, "failed to create mkey, %d\n", err
);
769 static void free_pa_mkey(struct mlx5_ib_dev
*dev
, u32 key
)
771 struct mlx5_core_mr mr
;
774 memset(&mr
, 0, sizeof(mr
));
776 err
= mlx5_core_destroy_mkey(&dev
->mdev
, &mr
);
778 mlx5_ib_warn(dev
, "failed to destroy mkey 0x%x\n", key
);
781 static struct ib_pd
*mlx5_ib_alloc_pd(struct ib_device
*ibdev
,
782 struct ib_ucontext
*context
,
783 struct ib_udata
*udata
)
785 struct mlx5_ib_alloc_pd_resp resp
;
786 struct mlx5_ib_pd
*pd
;
789 pd
= kmalloc(sizeof(*pd
), GFP_KERNEL
);
791 return ERR_PTR(-ENOMEM
);
793 err
= mlx5_core_alloc_pd(&to_mdev(ibdev
)->mdev
, &pd
->pdn
);
801 if (ib_copy_to_udata(udata
, &resp
, sizeof(resp
))) {
802 mlx5_core_dealloc_pd(&to_mdev(ibdev
)->mdev
, pd
->pdn
);
804 return ERR_PTR(-EFAULT
);
807 err
= alloc_pa_mkey(to_mdev(ibdev
), &pd
->pa_lkey
, pd
->pdn
);
809 mlx5_core_dealloc_pd(&to_mdev(ibdev
)->mdev
, pd
->pdn
);
818 static int mlx5_ib_dealloc_pd(struct ib_pd
*pd
)
820 struct mlx5_ib_dev
*mdev
= to_mdev(pd
->device
);
821 struct mlx5_ib_pd
*mpd
= to_mpd(pd
);
824 free_pa_mkey(mdev
, mpd
->pa_lkey
);
826 mlx5_core_dealloc_pd(&mdev
->mdev
, mpd
->pdn
);
832 static int mlx5_ib_mcg_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
834 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
837 err
= mlx5_core_attach_mcg(&dev
->mdev
, gid
, ibqp
->qp_num
);
839 mlx5_ib_warn(dev
, "failed attaching QPN 0x%x, MGID %pI6\n",
840 ibqp
->qp_num
, gid
->raw
);
845 static int mlx5_ib_mcg_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
847 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
850 err
= mlx5_core_detach_mcg(&dev
->mdev
, gid
, ibqp
->qp_num
);
852 mlx5_ib_warn(dev
, "failed detaching QPN 0x%x, MGID %pI6\n",
853 ibqp
->qp_num
, gid
->raw
);
858 static int init_node_data(struct mlx5_ib_dev
*dev
)
860 struct ib_smp
*in_mad
= NULL
;
861 struct ib_smp
*out_mad
= NULL
;
864 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
865 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
866 if (!in_mad
|| !out_mad
)
869 init_query_mad(in_mad
);
870 in_mad
->attr_id
= IB_SMP_ATTR_NODE_DESC
;
872 err
= mlx5_MAD_IFC(dev
, 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
876 memcpy(dev
->ib_dev
.node_desc
, out_mad
->data
, 64);
878 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
880 err
= mlx5_MAD_IFC(dev
, 1, 1, 1, NULL
, NULL
, in_mad
, out_mad
);
884 dev
->mdev
.rev_id
= be32_to_cpup((__be32
*)(out_mad
->data
+ 32));
885 memcpy(&dev
->ib_dev
.node_guid
, out_mad
->data
+ 12, 8);
893 static ssize_t
show_fw_pages(struct device
*device
, struct device_attribute
*attr
,
896 struct mlx5_ib_dev
*dev
=
897 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
899 return sprintf(buf
, "%d\n", dev
->mdev
.priv
.fw_pages
);
902 static ssize_t
show_reg_pages(struct device
*device
,
903 struct device_attribute
*attr
, char *buf
)
905 struct mlx5_ib_dev
*dev
=
906 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
908 return sprintf(buf
, "%d\n", dev
->mdev
.priv
.reg_pages
);
911 static ssize_t
show_hca(struct device
*device
, struct device_attribute
*attr
,
914 struct mlx5_ib_dev
*dev
=
915 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
916 return sprintf(buf
, "MT%d\n", dev
->mdev
.pdev
->device
);
919 static ssize_t
show_fw_ver(struct device
*device
, struct device_attribute
*attr
,
922 struct mlx5_ib_dev
*dev
=
923 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
924 return sprintf(buf
, "%d.%d.%d\n", fw_rev_maj(&dev
->mdev
),
925 fw_rev_min(&dev
->mdev
), fw_rev_sub(&dev
->mdev
));
928 static ssize_t
show_rev(struct device
*device
, struct device_attribute
*attr
,
931 struct mlx5_ib_dev
*dev
=
932 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
933 return sprintf(buf
, "%x\n", dev
->mdev
.rev_id
);
936 static ssize_t
show_board(struct device
*device
, struct device_attribute
*attr
,
939 struct mlx5_ib_dev
*dev
=
940 container_of(device
, struct mlx5_ib_dev
, ib_dev
.dev
);
941 return sprintf(buf
, "%.*s\n", MLX5_BOARD_ID_LEN
,
945 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
946 static DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
947 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
948 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
949 static DEVICE_ATTR(fw_pages
, S_IRUGO
, show_fw_pages
, NULL
);
950 static DEVICE_ATTR(reg_pages
, S_IRUGO
, show_reg_pages
, NULL
);
952 static struct device_attribute
*mlx5_class_attributes
[] = {
961 static void mlx5_ib_event(struct mlx5_core_dev
*dev
, enum mlx5_dev_event event
,
964 struct mlx5_ib_dev
*ibdev
= container_of(dev
, struct mlx5_ib_dev
, mdev
);
965 struct ib_event ibev
;
969 case MLX5_DEV_EVENT_SYS_ERROR
:
970 ibdev
->ib_active
= false;
971 ibev
.event
= IB_EVENT_DEVICE_FATAL
;
974 case MLX5_DEV_EVENT_PORT_UP
:
975 ibev
.event
= IB_EVENT_PORT_ACTIVE
;
979 case MLX5_DEV_EVENT_PORT_DOWN
:
980 ibev
.event
= IB_EVENT_PORT_ERR
;
984 case MLX5_DEV_EVENT_PORT_INITIALIZED
:
985 /* not used by ULPs */
988 case MLX5_DEV_EVENT_LID_CHANGE
:
989 ibev
.event
= IB_EVENT_LID_CHANGE
;
993 case MLX5_DEV_EVENT_PKEY_CHANGE
:
994 ibev
.event
= IB_EVENT_PKEY_CHANGE
;
998 case MLX5_DEV_EVENT_GUID_CHANGE
:
999 ibev
.event
= IB_EVENT_GID_CHANGE
;
1003 case MLX5_DEV_EVENT_CLIENT_REREG
:
1004 ibev
.event
= IB_EVENT_CLIENT_REREGISTER
;
1009 ibev
.device
= &ibdev
->ib_dev
;
1010 ibev
.element
.port_num
= port
;
1012 if (port
< 1 || port
> ibdev
->num_ports
) {
1013 mlx5_ib_warn(ibdev
, "warning: event on port %d\n", port
);
1017 if (ibdev
->ib_active
)
1018 ib_dispatch_event(&ibev
);
1021 static void get_ext_port_caps(struct mlx5_ib_dev
*dev
)
1025 for (port
= 1; port
<= dev
->mdev
.caps
.num_ports
; port
++)
1026 mlx5_query_ext_port_caps(dev
, port
);
1029 static int get_port_caps(struct mlx5_ib_dev
*dev
)
1031 struct ib_device_attr
*dprops
= NULL
;
1032 struct ib_port_attr
*pprops
= NULL
;
1036 pprops
= kmalloc(sizeof(*pprops
), GFP_KERNEL
);
1040 dprops
= kmalloc(sizeof(*dprops
), GFP_KERNEL
);
1044 err
= mlx5_ib_query_device(&dev
->ib_dev
, dprops
);
1046 mlx5_ib_warn(dev
, "query_device failed %d\n", err
);
1050 for (port
= 1; port
<= dev
->mdev
.caps
.num_ports
; port
++) {
1051 err
= mlx5_ib_query_port(&dev
->ib_dev
, port
, pprops
);
1053 mlx5_ib_warn(dev
, "query_port %d failed %d\n", port
, err
);
1056 dev
->mdev
.caps
.port
[port
- 1].pkey_table_len
= dprops
->max_pkeys
;
1057 dev
->mdev
.caps
.port
[port
- 1].gid_table_len
= pprops
->gid_tbl_len
;
1058 mlx5_ib_dbg(dev
, "pkey_table_len %d, gid_table_len %d\n",
1059 dprops
->max_pkeys
, pprops
->gid_tbl_len
);
1069 static void destroy_umrc_res(struct mlx5_ib_dev
*dev
)
1073 err
= mlx5_mr_cache_cleanup(dev
);
1075 mlx5_ib_warn(dev
, "mr cache cleanup failed\n");
1077 mlx5_ib_destroy_qp(dev
->umrc
.qp
);
1078 ib_destroy_cq(dev
->umrc
.cq
);
1079 ib_dereg_mr(dev
->umrc
.mr
);
1080 ib_dealloc_pd(dev
->umrc
.pd
);
1087 static int create_umr_res(struct mlx5_ib_dev
*dev
)
1089 struct ib_qp_init_attr
*init_attr
= NULL
;
1090 struct ib_qp_attr
*attr
= NULL
;
1097 attr
= kzalloc(sizeof(*attr
), GFP_KERNEL
);
1098 init_attr
= kzalloc(sizeof(*init_attr
), GFP_KERNEL
);
1099 if (!attr
|| !init_attr
) {
1104 pd
= ib_alloc_pd(&dev
->ib_dev
);
1106 mlx5_ib_dbg(dev
, "Couldn't create PD for sync UMR QP\n");
1111 mr
= ib_get_dma_mr(pd
, IB_ACCESS_LOCAL_WRITE
);
1113 mlx5_ib_dbg(dev
, "Couldn't create DMA MR for sync UMR QP\n");
1118 cq
= ib_create_cq(&dev
->ib_dev
, mlx5_umr_cq_handler
, NULL
, NULL
, 128,
1121 mlx5_ib_dbg(dev
, "Couldn't create CQ for sync UMR QP\n");
1125 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
1127 init_attr
->send_cq
= cq
;
1128 init_attr
->recv_cq
= cq
;
1129 init_attr
->sq_sig_type
= IB_SIGNAL_ALL_WR
;
1130 init_attr
->cap
.max_send_wr
= MAX_UMR_WR
;
1131 init_attr
->cap
.max_send_sge
= 1;
1132 init_attr
->qp_type
= MLX5_IB_QPT_REG_UMR
;
1133 init_attr
->port_num
= 1;
1134 qp
= mlx5_ib_create_qp(pd
, init_attr
, NULL
);
1136 mlx5_ib_dbg(dev
, "Couldn't create sync UMR QP\n");
1140 qp
->device
= &dev
->ib_dev
;
1143 qp
->qp_type
= MLX5_IB_QPT_REG_UMR
;
1145 attr
->qp_state
= IB_QPS_INIT
;
1147 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
| IB_QP_PKEY_INDEX
|
1150 mlx5_ib_dbg(dev
, "Couldn't modify UMR QP\n");
1154 memset(attr
, 0, sizeof(*attr
));
1155 attr
->qp_state
= IB_QPS_RTR
;
1156 attr
->path_mtu
= IB_MTU_256
;
1158 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
, NULL
);
1160 mlx5_ib_dbg(dev
, "Couldn't modify umr QP to rtr\n");
1164 memset(attr
, 0, sizeof(*attr
));
1165 attr
->qp_state
= IB_QPS_RTS
;
1166 ret
= mlx5_ib_modify_qp(qp
, attr
, IB_QP_STATE
, NULL
);
1168 mlx5_ib_dbg(dev
, "Couldn't modify umr QP to rts\n");
1177 sema_init(&dev
->umrc
.sem
, MAX_UMR_WR
);
1178 ret
= mlx5_mr_cache_init(dev
);
1180 mlx5_ib_warn(dev
, "mr cache init failed %d\n", ret
);
1190 mlx5_ib_destroy_qp(qp
);
1207 static int create_dev_resources(struct mlx5_ib_resources
*devr
)
1209 struct ib_srq_init_attr attr
;
1210 struct mlx5_ib_dev
*dev
;
1213 dev
= container_of(devr
, struct mlx5_ib_dev
, devr
);
1215 devr
->p0
= mlx5_ib_alloc_pd(&dev
->ib_dev
, NULL
, NULL
);
1216 if (IS_ERR(devr
->p0
)) {
1217 ret
= PTR_ERR(devr
->p0
);
1220 devr
->p0
->device
= &dev
->ib_dev
;
1221 devr
->p0
->uobject
= NULL
;
1222 atomic_set(&devr
->p0
->usecnt
, 0);
1224 devr
->c0
= mlx5_ib_create_cq(&dev
->ib_dev
, 1, 0, NULL
, NULL
);
1225 if (IS_ERR(devr
->c0
)) {
1226 ret
= PTR_ERR(devr
->c0
);
1229 devr
->c0
->device
= &dev
->ib_dev
;
1230 devr
->c0
->uobject
= NULL
;
1231 devr
->c0
->comp_handler
= NULL
;
1232 devr
->c0
->event_handler
= NULL
;
1233 devr
->c0
->cq_context
= NULL
;
1234 atomic_set(&devr
->c0
->usecnt
, 0);
1236 devr
->x0
= mlx5_ib_alloc_xrcd(&dev
->ib_dev
, NULL
, NULL
);
1237 if (IS_ERR(devr
->x0
)) {
1238 ret
= PTR_ERR(devr
->x0
);
1241 devr
->x0
->device
= &dev
->ib_dev
;
1242 devr
->x0
->inode
= NULL
;
1243 atomic_set(&devr
->x0
->usecnt
, 0);
1244 mutex_init(&devr
->x0
->tgt_qp_mutex
);
1245 INIT_LIST_HEAD(&devr
->x0
->tgt_qp_list
);
1247 devr
->x1
= mlx5_ib_alloc_xrcd(&dev
->ib_dev
, NULL
, NULL
);
1248 if (IS_ERR(devr
->x1
)) {
1249 ret
= PTR_ERR(devr
->x1
);
1252 devr
->x1
->device
= &dev
->ib_dev
;
1253 devr
->x1
->inode
= NULL
;
1254 atomic_set(&devr
->x1
->usecnt
, 0);
1255 mutex_init(&devr
->x1
->tgt_qp_mutex
);
1256 INIT_LIST_HEAD(&devr
->x1
->tgt_qp_list
);
1258 memset(&attr
, 0, sizeof(attr
));
1259 attr
.attr
.max_sge
= 1;
1260 attr
.attr
.max_wr
= 1;
1261 attr
.srq_type
= IB_SRQT_XRC
;
1262 attr
.ext
.xrc
.cq
= devr
->c0
;
1263 attr
.ext
.xrc
.xrcd
= devr
->x0
;
1265 devr
->s0
= mlx5_ib_create_srq(devr
->p0
, &attr
, NULL
);
1266 if (IS_ERR(devr
->s0
)) {
1267 ret
= PTR_ERR(devr
->s0
);
1270 devr
->s0
->device
= &dev
->ib_dev
;
1271 devr
->s0
->pd
= devr
->p0
;
1272 devr
->s0
->uobject
= NULL
;
1273 devr
->s0
->event_handler
= NULL
;
1274 devr
->s0
->srq_context
= NULL
;
1275 devr
->s0
->srq_type
= IB_SRQT_XRC
;
1276 devr
->s0
->ext
.xrc
.xrcd
= devr
->x0
;
1277 devr
->s0
->ext
.xrc
.cq
= devr
->c0
;
1278 atomic_inc(&devr
->s0
->ext
.xrc
.xrcd
->usecnt
);
1279 atomic_inc(&devr
->s0
->ext
.xrc
.cq
->usecnt
);
1280 atomic_inc(&devr
->p0
->usecnt
);
1281 atomic_set(&devr
->s0
->usecnt
, 0);
1286 mlx5_ib_dealloc_xrcd(devr
->x1
);
1288 mlx5_ib_dealloc_xrcd(devr
->x0
);
1290 mlx5_ib_destroy_cq(devr
->c0
);
1292 mlx5_ib_dealloc_pd(devr
->p0
);
1297 static void destroy_dev_resources(struct mlx5_ib_resources
*devr
)
1299 mlx5_ib_destroy_srq(devr
->s0
);
1300 mlx5_ib_dealloc_xrcd(devr
->x0
);
1301 mlx5_ib_dealloc_xrcd(devr
->x1
);
1302 mlx5_ib_destroy_cq(devr
->c0
);
1303 mlx5_ib_dealloc_pd(devr
->p0
);
1306 static int init_one(struct pci_dev
*pdev
,
1307 const struct pci_device_id
*id
)
1309 struct mlx5_core_dev
*mdev
;
1310 struct mlx5_ib_dev
*dev
;
1314 printk_once(KERN_INFO
"%s", mlx5_version
);
1316 dev
= (struct mlx5_ib_dev
*)ib_alloc_device(sizeof(*dev
));
1321 mdev
->event
= mlx5_ib_event
;
1322 if (prof_sel
>= ARRAY_SIZE(profile
)) {
1323 pr_warn("selected pofile out of range, selceting default\n");
1326 mdev
->profile
= &profile
[prof_sel
];
1327 err
= mlx5_dev_init(mdev
, pdev
);
1331 err
= get_port_caps(dev
);
1335 get_ext_port_caps(dev
);
1337 err
= alloc_comp_eqs(dev
);
1341 MLX5_INIT_DOORBELL_LOCK(&dev
->uar_lock
);
1343 strlcpy(dev
->ib_dev
.name
, "mlx5_%d", IB_DEVICE_NAME_MAX
);
1344 dev
->ib_dev
.owner
= THIS_MODULE
;
1345 dev
->ib_dev
.node_type
= RDMA_NODE_IB_CA
;
1346 dev
->ib_dev
.local_dma_lkey
= mdev
->caps
.reserved_lkey
;
1347 dev
->num_ports
= mdev
->caps
.num_ports
;
1348 dev
->ib_dev
.phys_port_cnt
= dev
->num_ports
;
1349 dev
->ib_dev
.num_comp_vectors
= dev
->num_comp_vectors
;
1350 dev
->ib_dev
.dma_device
= &mdev
->pdev
->dev
;
1352 dev
->ib_dev
.uverbs_abi_ver
= MLX5_IB_UVERBS_ABI_VERSION
;
1353 dev
->ib_dev
.uverbs_cmd_mask
=
1354 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
1355 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
1356 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
1357 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
1358 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
1359 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
1360 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
1361 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
1362 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
1363 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
1364 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
1365 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
1366 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
1367 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
1368 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
1369 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST
) |
1370 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST
) |
1371 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
1372 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
1373 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
1374 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
1375 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ
) |
1376 (1ull << IB_USER_VERBS_CMD_OPEN_QP
);
1378 dev
->ib_dev
.query_device
= mlx5_ib_query_device
;
1379 dev
->ib_dev
.query_port
= mlx5_ib_query_port
;
1380 dev
->ib_dev
.query_gid
= mlx5_ib_query_gid
;
1381 dev
->ib_dev
.query_pkey
= mlx5_ib_query_pkey
;
1382 dev
->ib_dev
.modify_device
= mlx5_ib_modify_device
;
1383 dev
->ib_dev
.modify_port
= mlx5_ib_modify_port
;
1384 dev
->ib_dev
.alloc_ucontext
= mlx5_ib_alloc_ucontext
;
1385 dev
->ib_dev
.dealloc_ucontext
= mlx5_ib_dealloc_ucontext
;
1386 dev
->ib_dev
.mmap
= mlx5_ib_mmap
;
1387 dev
->ib_dev
.alloc_pd
= mlx5_ib_alloc_pd
;
1388 dev
->ib_dev
.dealloc_pd
= mlx5_ib_dealloc_pd
;
1389 dev
->ib_dev
.create_ah
= mlx5_ib_create_ah
;
1390 dev
->ib_dev
.query_ah
= mlx5_ib_query_ah
;
1391 dev
->ib_dev
.destroy_ah
= mlx5_ib_destroy_ah
;
1392 dev
->ib_dev
.create_srq
= mlx5_ib_create_srq
;
1393 dev
->ib_dev
.modify_srq
= mlx5_ib_modify_srq
;
1394 dev
->ib_dev
.query_srq
= mlx5_ib_query_srq
;
1395 dev
->ib_dev
.destroy_srq
= mlx5_ib_destroy_srq
;
1396 dev
->ib_dev
.post_srq_recv
= mlx5_ib_post_srq_recv
;
1397 dev
->ib_dev
.create_qp
= mlx5_ib_create_qp
;
1398 dev
->ib_dev
.modify_qp
= mlx5_ib_modify_qp
;
1399 dev
->ib_dev
.query_qp
= mlx5_ib_query_qp
;
1400 dev
->ib_dev
.destroy_qp
= mlx5_ib_destroy_qp
;
1401 dev
->ib_dev
.post_send
= mlx5_ib_post_send
;
1402 dev
->ib_dev
.post_recv
= mlx5_ib_post_recv
;
1403 dev
->ib_dev
.create_cq
= mlx5_ib_create_cq
;
1404 dev
->ib_dev
.modify_cq
= mlx5_ib_modify_cq
;
1405 dev
->ib_dev
.resize_cq
= mlx5_ib_resize_cq
;
1406 dev
->ib_dev
.destroy_cq
= mlx5_ib_destroy_cq
;
1407 dev
->ib_dev
.poll_cq
= mlx5_ib_poll_cq
;
1408 dev
->ib_dev
.req_notify_cq
= mlx5_ib_arm_cq
;
1409 dev
->ib_dev
.get_dma_mr
= mlx5_ib_get_dma_mr
;
1410 dev
->ib_dev
.reg_user_mr
= mlx5_ib_reg_user_mr
;
1411 dev
->ib_dev
.dereg_mr
= mlx5_ib_dereg_mr
;
1412 dev
->ib_dev
.attach_mcast
= mlx5_ib_mcg_attach
;
1413 dev
->ib_dev
.detach_mcast
= mlx5_ib_mcg_detach
;
1414 dev
->ib_dev
.process_mad
= mlx5_ib_process_mad
;
1415 dev
->ib_dev
.alloc_fast_reg_mr
= mlx5_ib_alloc_fast_reg_mr
;
1416 dev
->ib_dev
.alloc_fast_reg_page_list
= mlx5_ib_alloc_fast_reg_page_list
;
1417 dev
->ib_dev
.free_fast_reg_page_list
= mlx5_ib_free_fast_reg_page_list
;
1419 if (mdev
->caps
.flags
& MLX5_DEV_CAP_FLAG_XRC
) {
1420 dev
->ib_dev
.alloc_xrcd
= mlx5_ib_alloc_xrcd
;
1421 dev
->ib_dev
.dealloc_xrcd
= mlx5_ib_dealloc_xrcd
;
1422 dev
->ib_dev
.uverbs_cmd_mask
|=
1423 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD
) |
1424 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD
);
1427 err
= init_node_data(dev
);
1431 mutex_init(&dev
->cap_mask_mutex
);
1432 spin_lock_init(&dev
->mr_lock
);
1434 err
= create_dev_resources(&dev
->devr
);
1438 err
= ib_register_device(&dev
->ib_dev
, NULL
);
1442 err
= create_umr_res(dev
);
1446 for (i
= 0; i
< ARRAY_SIZE(mlx5_class_attributes
); i
++) {
1447 err
= device_create_file(&dev
->ib_dev
.dev
,
1448 mlx5_class_attributes
[i
]);
1453 dev
->ib_active
= true;
1458 destroy_umrc_res(dev
);
1461 ib_unregister_device(&dev
->ib_dev
);
1464 destroy_dev_resources(&dev
->devr
);
1470 mlx5_dev_cleanup(mdev
);
1473 ib_dealloc_device((struct ib_device
*)dev
);
1478 static void remove_one(struct pci_dev
*pdev
)
1480 struct mlx5_ib_dev
*dev
= mlx5_pci2ibdev(pdev
);
1482 destroy_umrc_res(dev
);
1483 ib_unregister_device(&dev
->ib_dev
);
1484 destroy_dev_resources(&dev
->devr
);
1486 mlx5_dev_cleanup(&dev
->mdev
);
1487 ib_dealloc_device(&dev
->ib_dev
);
1490 static DEFINE_PCI_DEVICE_TABLE(mlx5_ib_pci_table
) = {
1491 { PCI_VDEVICE(MELLANOX
, 4113) }, /* MT4113 Connect-IB */
1495 MODULE_DEVICE_TABLE(pci
, mlx5_ib_pci_table
);
1497 static struct pci_driver mlx5_ib_driver
= {
1498 .name
= DRIVER_NAME
,
1499 .id_table
= mlx5_ib_pci_table
,
1501 .remove
= remove_one
1504 static int __init
mlx5_ib_init(void)
1506 return pci_register_driver(&mlx5_ib_driver
);
1509 static void __exit
mlx5_ib_cleanup(void)
1511 pci_unregister_driver(&mlx5_ib_driver
);
1514 module_init(mlx5_ib_init
);
1515 module_exit(mlx5_ib_cleanup
);