2 * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37 #include <linux/compiler.h>
38 #include <linux/list.h>
39 #include <linux/mutex.h>
40 #include <linux/idr.h>
42 #include <rdma/ib_verbs.h>
43 #include <rdma/ib_umem.h>
44 #include <rdma/ib_mad.h>
46 #include <linux/mlx4/device.h>
47 #include <linux/mlx4/doorbell.h>
49 #define MLX4_IB_DRV_NAME "mlx4_ib"
54 #define pr_fmt(fmt) "<" MLX4_IB_DRV_NAME "> %s: " fmt, __func__
56 #define mlx4_ib_warn(ibdev, format, arg...) \
57 dev_warn((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg)
60 MLX4_IB_SQ_MIN_WQE_SHIFT
= 6,
61 MLX4_IB_MAX_HEADROOM
= 2048
64 #define MLX4_IB_SQ_HEADROOM(shift) ((MLX4_IB_MAX_HEADROOM >> (shift)) + 1)
65 #define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT))
67 struct mlx4_ib_ucontext
{
68 struct ib_ucontext ibucontext
;
70 struct list_head db_page_list
;
71 struct mutex db_page_mutex
;
80 struct ib_xrcd ibxrcd
;
86 struct mlx4_ib_cq_buf
{
91 struct mlx4_ib_cq_resize
{
92 struct mlx4_ib_cq_buf buf
;
99 struct mlx4_ib_cq_buf buf
;
100 struct mlx4_ib_cq_resize
*resize_buf
;
103 struct mutex resize_mutex
;
104 struct ib_umem
*umem
;
105 struct ib_umem
*resize_umem
;
111 struct ib_umem
*umem
;
114 struct mlx4_ib_fast_reg_page_list
{
115 struct ib_fast_reg_page_list ibfrpl
;
116 __be64
*mapped_page_list
;
122 struct mlx4_fmr mfmr
;
137 enum mlx4_ib_qp_flags
{
138 MLX4_IB_QP_LSO
= IB_QP_CREATE_IPOIB_UD_LSO
,
139 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK
= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
,
140 MLX4_IB_SRIOV_TUNNEL_QP
= 1 << 30,
141 MLX4_IB_SRIOV_SQP
= 1 << 31,
144 struct mlx4_ib_gid_entry
{
145 struct list_head list
;
151 enum mlx4_ib_qp_type
{
153 * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries
154 * here (and in that order) since the MAD layer uses them as
155 * indices into a 2-entry table.
157 MLX4_IB_QPT_SMI
= IB_QPT_SMI
,
158 MLX4_IB_QPT_GSI
= IB_QPT_GSI
,
160 MLX4_IB_QPT_RC
= IB_QPT_RC
,
161 MLX4_IB_QPT_UC
= IB_QPT_UC
,
162 MLX4_IB_QPT_UD
= IB_QPT_UD
,
163 MLX4_IB_QPT_RAW_IPV6
= IB_QPT_RAW_IPV6
,
164 MLX4_IB_QPT_RAW_ETHERTYPE
= IB_QPT_RAW_ETHERTYPE
,
165 MLX4_IB_QPT_RAW_PACKET
= IB_QPT_RAW_PACKET
,
166 MLX4_IB_QPT_XRC_INI
= IB_QPT_XRC_INI
,
167 MLX4_IB_QPT_XRC_TGT
= IB_QPT_XRC_TGT
,
169 MLX4_IB_QPT_PROXY_SMI_OWNER
= 1 << 16,
170 MLX4_IB_QPT_PROXY_SMI
= 1 << 17,
171 MLX4_IB_QPT_PROXY_GSI
= 1 << 18,
172 MLX4_IB_QPT_TUN_SMI_OWNER
= 1 << 19,
173 MLX4_IB_QPT_TUN_SMI
= 1 << 20,
174 MLX4_IB_QPT_TUN_GSI
= 1 << 21,
177 #define MLX4_IB_QPT_ANY_SRIOV (MLX4_IB_QPT_PROXY_SMI_OWNER | \
178 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER | \
179 MLX4_IB_QPT_TUN_SMI | MLX4_IB_QPT_TUN_GSI)
181 enum mlx4_ib_mad_ifc_flags
{
182 MLX4_MAD_IFC_IGNORE_MKEY
= 1,
183 MLX4_MAD_IFC_IGNORE_BKEY
= 2,
184 MLX4_MAD_IFC_IGNORE_KEYS
= (MLX4_MAD_IFC_IGNORE_MKEY
|
185 MLX4_MAD_IFC_IGNORE_BKEY
),
186 MLX4_MAD_IFC_NET_VIEW
= 4,
190 MLX4_NUM_TUNNEL_BUFS
= 256,
193 struct mlx4_ib_tunnel_header
{
208 struct mlx4_rcv_tunnel_hdr
{
209 __be32 flags_src_qp
; /* flags[6:5] is defined for VLANs:
210 * 0x0 - no vlan was in the packet
211 * 0x01 - C-VLAN was in the packet */
212 u8 g_ml_path
; /* gid bit stands for ipv6/4 header in RoCE */
216 __be16 slid_mac_47_32
;
220 struct mlx4_ib_proxy_sqp_hdr
{
222 struct mlx4_rcv_tunnel_hdr tun
;
231 struct mlx4_ib_wq rq
;
234 __be32 sq_signal_bits
;
235 unsigned sq_next_wqe
;
236 int sq_max_wqes_per_wr
;
238 struct mlx4_ib_wq sq
;
240 enum mlx4_ib_qp_type mlx4_ib_qp_type
;
241 struct ib_umem
*umem
;
254 struct list_head gid_list
;
255 struct list_head steering_rules
;
256 struct mlx4_ib_buf
*sqp_proxy_rcv
;
262 struct mlx4_srq msrq
;
270 struct ib_umem
*umem
;
277 union mlx4_ext_av av
;
280 struct mlx4_ib_demux_work
{
281 struct work_struct work
;
282 struct mlx4_ib_dev
*dev
;
289 struct mlx4_ib_tun_tx_buf
{
290 struct mlx4_ib_buf buf
;
294 struct mlx4_ib_demux_pv_qp
{
296 enum ib_qp_type proxy_qpt
;
297 struct mlx4_ib_buf
*ring
;
298 struct mlx4_ib_tun_tx_buf
*tx_ring
;
304 enum mlx4_ib_demux_pv_state
{
306 DEMUX_PV_STATE_STARTING
,
307 DEMUX_PV_STATE_ACTIVE
,
308 DEMUX_PV_STATE_DOWNING
,
311 struct mlx4_ib_demux_pv_ctx
{
314 enum mlx4_ib_demux_pv_state state
;
316 struct ib_device
*ib_dev
;
320 struct work_struct work
;
321 struct workqueue_struct
*wq
;
322 struct mlx4_ib_demux_pv_qp qp
[2];
325 struct mlx4_ib_demux_ctx
{
326 struct ib_device
*ib_dev
;
328 struct workqueue_struct
*wq
;
329 struct workqueue_struct
*ud_wq
;
331 __be64 subnet_prefix
;
332 __be64 guid_cache
[128];
333 struct mlx4_ib_dev
*dev
;
334 /* the following lock protects both mcg_table and mcg_mgid0_list */
335 struct mutex mcg_table_lock
;
336 struct rb_root mcg_table
;
337 struct list_head mcg_mgid0_list
;
338 struct workqueue_struct
*mcg_wq
;
339 struct mlx4_ib_demux_pv_ctx
**tun
;
341 int flushing
; /* flushing the work queue */
344 struct mlx4_ib_sriov
{
345 struct mlx4_ib_demux_ctx demux
[MLX4_MAX_PORTS
];
346 struct mlx4_ib_demux_pv_ctx
*sqps
[MLX4_MAX_PORTS
];
347 /* when using this spinlock you should use "irq" because
348 * it may be called from interrupt context.*/
349 spinlock_t going_down_lock
;
353 struct mlx4_ib_iboe
{
355 struct net_device
*netdevs
[MLX4_MAX_PORTS
];
356 struct notifier_block nb
;
357 union ib_gid gid_table
[MLX4_MAX_PORTS
][128];
361 u8 virt2phys_pkey
[MLX4_MFUNC_MAX
][MLX4_MAX_PORTS
][MLX4_MAX_PORT_PKEYS
];
362 u16 phys_pkey_cache
[MLX4_MAX_PORTS
][MLX4_MAX_PORT_PKEYS
];
363 struct list_head pkey_port_list
[MLX4_MFUNC_MAX
];
364 struct kobject
*device_parent
[MLX4_MFUNC_MAX
];
368 struct ib_device ib_dev
;
369 struct mlx4_dev
*dev
;
371 void __iomem
*uar_map
;
373 struct mlx4_uar priv_uar
;
375 MLX4_DECLARE_DOORBELL_LOCK(uar_lock
);
377 struct ib_mad_agent
*send_agent
[MLX4_MAX_PORTS
][2];
378 struct ib_ah
*sm_ah
[MLX4_MAX_PORTS
];
380 struct mlx4_ib_sriov sriov
;
382 struct mutex cap_mask_mutex
;
384 struct mlx4_ib_iboe iboe
;
385 int counters
[MLX4_MAX_PORTS
];
388 struct pkey_mgt pkeys
;
391 struct ib_event_work
{
392 struct work_struct work
;
393 struct mlx4_ib_dev
*ib_dev
;
394 struct mlx4_eqe ib_eqe
;
397 struct mlx4_ib_qp_tunnel_init_attr
{
398 struct ib_qp_init_attr init_attr
;
400 enum ib_qp_type proxy_qp_type
;
404 static inline struct mlx4_ib_dev
*to_mdev(struct ib_device
*ibdev
)
406 return container_of(ibdev
, struct mlx4_ib_dev
, ib_dev
);
409 static inline struct mlx4_ib_ucontext
*to_mucontext(struct ib_ucontext
*ibucontext
)
411 return container_of(ibucontext
, struct mlx4_ib_ucontext
, ibucontext
);
414 static inline struct mlx4_ib_pd
*to_mpd(struct ib_pd
*ibpd
)
416 return container_of(ibpd
, struct mlx4_ib_pd
, ibpd
);
419 static inline struct mlx4_ib_xrcd
*to_mxrcd(struct ib_xrcd
*ibxrcd
)
421 return container_of(ibxrcd
, struct mlx4_ib_xrcd
, ibxrcd
);
424 static inline struct mlx4_ib_cq
*to_mcq(struct ib_cq
*ibcq
)
426 return container_of(ibcq
, struct mlx4_ib_cq
, ibcq
);
429 static inline struct mlx4_ib_cq
*to_mibcq(struct mlx4_cq
*mcq
)
431 return container_of(mcq
, struct mlx4_ib_cq
, mcq
);
434 static inline struct mlx4_ib_mr
*to_mmr(struct ib_mr
*ibmr
)
436 return container_of(ibmr
, struct mlx4_ib_mr
, ibmr
);
439 static inline struct mlx4_ib_fast_reg_page_list
*to_mfrpl(struct ib_fast_reg_page_list
*ibfrpl
)
441 return container_of(ibfrpl
, struct mlx4_ib_fast_reg_page_list
, ibfrpl
);
444 static inline struct mlx4_ib_fmr
*to_mfmr(struct ib_fmr
*ibfmr
)
446 return container_of(ibfmr
, struct mlx4_ib_fmr
, ibfmr
);
448 static inline struct mlx4_ib_qp
*to_mqp(struct ib_qp
*ibqp
)
450 return container_of(ibqp
, struct mlx4_ib_qp
, ibqp
);
453 static inline struct mlx4_ib_qp
*to_mibqp(struct mlx4_qp
*mqp
)
455 return container_of(mqp
, struct mlx4_ib_qp
, mqp
);
458 static inline struct mlx4_ib_srq
*to_msrq(struct ib_srq
*ibsrq
)
460 return container_of(ibsrq
, struct mlx4_ib_srq
, ibsrq
);
463 static inline struct mlx4_ib_srq
*to_mibsrq(struct mlx4_srq
*msrq
)
465 return container_of(msrq
, struct mlx4_ib_srq
, msrq
);
468 static inline struct mlx4_ib_ah
*to_mah(struct ib_ah
*ibah
)
470 return container_of(ibah
, struct mlx4_ib_ah
, ibah
);
473 int mlx4_ib_init_sriov(struct mlx4_ib_dev
*dev
);
474 void mlx4_ib_close_sriov(struct mlx4_ib_dev
*dev
);
476 int mlx4_ib_db_map_user(struct mlx4_ib_ucontext
*context
, unsigned long virt
,
478 void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext
*context
, struct mlx4_db
*db
);
480 struct ib_mr
*mlx4_ib_get_dma_mr(struct ib_pd
*pd
, int acc
);
481 int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev
*dev
, struct mlx4_mtt
*mtt
,
482 struct ib_umem
*umem
);
483 struct ib_mr
*mlx4_ib_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
484 u64 virt_addr
, int access_flags
,
485 struct ib_udata
*udata
);
486 int mlx4_ib_dereg_mr(struct ib_mr
*mr
);
487 struct ib_mr
*mlx4_ib_alloc_fast_reg_mr(struct ib_pd
*pd
,
488 int max_page_list_len
);
489 struct ib_fast_reg_page_list
*mlx4_ib_alloc_fast_reg_page_list(struct ib_device
*ibdev
,
491 void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list
*page_list
);
493 int mlx4_ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
);
494 int mlx4_ib_resize_cq(struct ib_cq
*ibcq
, int entries
, struct ib_udata
*udata
);
495 struct ib_cq
*mlx4_ib_create_cq(struct ib_device
*ibdev
, int entries
, int vector
,
496 struct ib_ucontext
*context
,
497 struct ib_udata
*udata
);
498 int mlx4_ib_destroy_cq(struct ib_cq
*cq
);
499 int mlx4_ib_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
);
500 int mlx4_ib_arm_cq(struct ib_cq
*cq
, enum ib_cq_notify_flags flags
);
501 void __mlx4_ib_cq_clean(struct mlx4_ib_cq
*cq
, u32 qpn
, struct mlx4_ib_srq
*srq
);
502 void mlx4_ib_cq_clean(struct mlx4_ib_cq
*cq
, u32 qpn
, struct mlx4_ib_srq
*srq
);
504 struct ib_ah
*mlx4_ib_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
);
505 int mlx4_ib_query_ah(struct ib_ah
*ibah
, struct ib_ah_attr
*ah_attr
);
506 int mlx4_ib_destroy_ah(struct ib_ah
*ah
);
508 struct ib_srq
*mlx4_ib_create_srq(struct ib_pd
*pd
,
509 struct ib_srq_init_attr
*init_attr
,
510 struct ib_udata
*udata
);
511 int mlx4_ib_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
512 enum ib_srq_attr_mask attr_mask
, struct ib_udata
*udata
);
513 int mlx4_ib_query_srq(struct ib_srq
*srq
, struct ib_srq_attr
*srq_attr
);
514 int mlx4_ib_destroy_srq(struct ib_srq
*srq
);
515 void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq
*srq
, int wqe_index
);
516 int mlx4_ib_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
517 struct ib_recv_wr
**bad_wr
);
519 struct ib_qp
*mlx4_ib_create_qp(struct ib_pd
*pd
,
520 struct ib_qp_init_attr
*init_attr
,
521 struct ib_udata
*udata
);
522 int mlx4_ib_destroy_qp(struct ib_qp
*qp
);
523 int mlx4_ib_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
524 int attr_mask
, struct ib_udata
*udata
);
525 int mlx4_ib_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
, int qp_attr_mask
,
526 struct ib_qp_init_attr
*qp_init_attr
);
527 int mlx4_ib_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
528 struct ib_send_wr
**bad_wr
);
529 int mlx4_ib_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
530 struct ib_recv_wr
**bad_wr
);
532 int mlx4_MAD_IFC(struct mlx4_ib_dev
*dev
, int mad_ifc_flags
,
533 int port
, struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
534 void *in_mad
, void *response_mad
);
535 int mlx4_ib_process_mad(struct ib_device
*ibdev
, int mad_flags
, u8 port_num
,
536 struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
537 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
);
538 int mlx4_ib_mad_init(struct mlx4_ib_dev
*dev
);
539 void mlx4_ib_mad_cleanup(struct mlx4_ib_dev
*dev
);
541 struct ib_fmr
*mlx4_ib_fmr_alloc(struct ib_pd
*pd
, int mr_access_flags
,
542 struct ib_fmr_attr
*fmr_attr
);
543 int mlx4_ib_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
, int npages
,
545 int mlx4_ib_unmap_fmr(struct list_head
*fmr_list
);
546 int mlx4_ib_fmr_dealloc(struct ib_fmr
*fmr
);
547 int __mlx4_ib_query_port(struct ib_device
*ibdev
, u8 port
,
548 struct ib_port_attr
*props
, int netw_view
);
549 int __mlx4_ib_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
550 u16
*pkey
, int netw_view
);
552 int mlx4_ib_resolve_grh(struct mlx4_ib_dev
*dev
, const struct ib_ah_attr
*ah_attr
,
553 u8
*mac
, int *is_mcast
, u8 port
);
555 static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah
*ah
)
557 u8 port
= be32_to_cpu(ah
->av
.ib
.port_pd
) >> 24 & 3;
559 if (rdma_port_get_link_layer(ah
->ibah
.device
, port
) == IB_LINK_LAYER_ETHERNET
)
562 return !!(ah
->av
.ib
.g_slid
& 0x80);
565 int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx
*ctx
);
566 void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx
*ctx
, int destroy_wq
);
567 void clean_vf_mcast(struct mlx4_ib_demux_ctx
*ctx
, int slave
);
568 int mlx4_ib_mcg_init(void);
569 void mlx4_ib_mcg_destroy(void);
571 int mlx4_ib_find_real_gid(struct ib_device
*ibdev
, u8 port
, __be64 guid
);
573 int mlx4_ib_mcg_multiplex_handler(struct ib_device
*ibdev
, int port
, int slave
,
574 struct ib_sa_mad
*sa_mad
);
575 int mlx4_ib_mcg_demux_handler(struct ib_device
*ibdev
, int port
, int slave
,
576 struct ib_sa_mad
*mad
);
578 int mlx4_ib_add_mc(struct mlx4_ib_dev
*mdev
, struct mlx4_ib_qp
*mqp
,
581 void mlx4_ib_dispatch_event(struct mlx4_ib_dev
*dev
, u8 port_num
,
582 enum ib_event_type type
);
584 void mlx4_ib_tunnels_update_work(struct work_struct
*work
);
586 int mlx4_ib_send_to_slave(struct mlx4_ib_dev
*dev
, int slave
, u8 port
,
587 enum ib_qp_type qpt
, struct ib_wc
*wc
,
588 struct ib_grh
*grh
, struct ib_mad
*mad
);
589 int mlx4_ib_send_to_wire(struct mlx4_ib_dev
*dev
, int slave
, u8 port
,
590 enum ib_qp_type dest_qpt
, u16 pkey_index
, u32 remote_qpn
,
591 u32 qkey
, struct ib_ah_attr
*attr
, struct ib_mad
*mad
);
592 __be64
mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx
*ctx
);
594 #endif /* MLX4_IB_H */