1 /*******************************************************************
2 * This file is part of the Emulex RoCE Device Driver for *
3 * RoCE (RDMA over Converged Ethernet) adapters. *
4 * Copyright (C) 2008-2012 Emulex. All rights reserved. *
5 * EMULEX and SLI are trademarks of Emulex. *
8 * This program is free software; you can redistribute it and/or *
9 * modify it under the terms of version 2 of the GNU General *
10 * Public License as published by the Free Software Foundation. *
11 * This program is distributed in the hope that it will be useful. *
12 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
13 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
14 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
15 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
16 * TO BE LEGALLY INVALID. See the GNU General Public License for *
17 * more details, a copy of which can be found in the file COPYING *
18 * included with this package. *
20 * Contact Information:
21 * linux-drivers@emulex.com
25 * Costa Mesa, CA 92626
26 *******************************************************************/
31 #include <linux/mutex.h>
32 #include <linux/list.h>
33 #include <linux/spinlock.h>
34 #include <linux/pci.h>
36 #include <rdma/ib_verbs.h>
37 #include <rdma/ib_user_verbs.h>
38 #include <rdma/ib_addr.h>
41 #include "ocrdma_sli.h"
43 #define OCRDMA_ROCE_DRV_VERSION "10.6.0.0"
45 #define OCRDMA_ROCE_DRV_DESC "Emulex OneConnect RoCE Driver"
46 #define OCRDMA_NODE_DESC "Emulex OneConnect RoCE HCA"
48 #define OC_NAME_SH OCRDMA_NODE_DESC "(Skyhawk)"
49 #define OC_NAME_UNKNOWN OCRDMA_NODE_DESC "(Unknown)"
51 #define OC_SKH_DEVICE_PF 0x720
52 #define OC_SKH_DEVICE_VF 0x728
53 #define OCRDMA_MAX_AH 512
55 #define OCRDMA_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
57 #define convert_to_64bit(lo, hi) ((u64)hi << 32 | (u64)lo)
58 #define EQ_INTR_PER_SEC_THRSH_HI 150000
59 #define EQ_INTR_PER_SEC_THRSH_LOW 100000
60 #define EQ_AIC_MAX_EQD 20
61 #define EQ_AIC_MIN_EQD 0
63 void ocrdma_eqd_set_task(struct work_struct
*work
);
65 struct ocrdma_dev_attr
{
88 int max_pages_per_frmr
;
93 u8 cq_overflow_detect
;
99 u8 local_ca_ack_delay
;
104 struct ocrdma_dma_mem
{
115 struct ocrdma_queue_info
{
120 u16 entry_size
; /* Size of an element in the queue */
121 u16 id
; /* qid, where to ring the doorbell. */
126 struct ocrdma_aic_obj
{ /* Adaptive interrupt coalescing (AIC) info */
129 u64 prev_eq_intr_cnt
;
133 struct ocrdma_queue_info q
;
136 struct ocrdma_dev
*dev
;
138 struct ocrdma_aic_obj aic_obj
;
142 struct ocrdma_queue_info sq
;
143 struct ocrdma_queue_info cq
;
148 struct mutex lock
; /* for serializing mailbox commands on MQ */
149 wait_queue_head_t cmd_wait
;
157 struct ocrdma_hw_mr
{
168 struct ocrdma_pbl
*pbl_table
;
179 struct ib_umem
*umem
;
180 struct ocrdma_hw_mr hwmr
;
183 struct ocrdma_stats
{
185 struct ocrdma_dev
*dev
;
188 struct ocrdma_pd_resource_mgr
{
198 unsigned long *pd_norm_bitmap
;
199 unsigned long *pd_dpp_bitmap
;
200 bool pd_prealloc_valid
;
204 struct ocrdma_mqe mqe
;
212 u16 auto_speeds_supported
;
213 u16 fixed_speeds_supported
;
219 struct ib_device ibdev
;
220 struct ocrdma_dev_attr attr
;
222 struct mutex dev_lock
; /* provides syncronise access to device data */
223 spinlock_t flush_q_lock ____cacheline_aligned
;
225 struct ocrdma_cq
**cq_tbl
;
226 struct ocrdma_qp
**qp_tbl
;
228 struct ocrdma_eq
*eq_tbl
;
230 struct delayed_work eqd_work
;
234 union ib_gid
*sgid_tbl
;
235 /* provided synchronization to sgid table for
236 * updating gid entries triggered by notifier.
238 spinlock_t sgid_lock
;
241 struct ocrdma_cq
*gsi_sqcq
;
242 struct ocrdma_cq
*gsi_rqcq
;
245 struct ocrdma_av
*va
;
249 /* provide synchronization for av
254 struct ocrdma_pbl pbl
;
259 struct mqe_ctx mqe_ctx
;
261 struct be_dev_info nic_info
;
263 char model_number
[32];
266 struct list_head entry
;
270 u8 sl
; /* service level */
276 ulong last_stats_time
;
277 struct mutex stats_lock
; /* provide synch for debugfs operations */
278 struct stats_mem stats_mem
;
279 struct ocrdma_stats rsrc_stats
;
280 struct ocrdma_stats rx_stats
;
281 struct ocrdma_stats wqe_stats
;
282 struct ocrdma_stats tx_stats
;
283 struct ocrdma_stats db_err_stats
;
284 struct ocrdma_stats tx_qp_err_stats
;
285 struct ocrdma_stats rx_qp_err_stats
;
286 struct ocrdma_stats tx_dbg_stats
;
287 struct ocrdma_stats rx_dbg_stats
;
288 struct ocrdma_stats driver_stats
;
289 struct ocrdma_stats reset_stats
;
291 atomic_t async_err_stats
[OCRDMA_MAX_ASYNC_ERRORS
];
292 atomic_t cqe_err_stats
[OCRDMA_MAX_CQE_ERR
];
293 struct ocrdma_pd_resource_mgr
*pd_mgr
;
298 struct ocrdma_cqe
*va
;
300 u32 getp
; /* pointer to pending wrs to
301 * return to stack, wrap arounds
306 bool deferred_arm
, deferred_sol
;
309 spinlock_t cq_lock ____cacheline_aligned
; /* provide synchronization
312 /* syncronizes cq completion handler invoked from multiple context */
313 spinlock_t comp_handler_lock ____cacheline_aligned
;
317 struct ocrdma_ucontext
*ucontext
;
322 /* head of all qp's sq and rq for which cqes need to be flushed
325 struct list_head sq_head
, rq_head
;
330 struct ocrdma_ucontext
*uctx
;
339 struct ocrdma_av
*av
;
344 struct ocrdma_qp_hwq_info
{
345 u8
*va
; /* virtual address */
351 u16 dbid
; /* qid, where to ring the doorbell. */
359 struct ocrdma_qp_hwq_info rq
;
364 /* provide synchronization to multiple context(s) posting rqe */
365 spinlock_t q_lock ____cacheline_aligned
;
367 struct ocrdma_pd
*pd
;
375 struct ocrdma_qp_hwq_info sq
;
378 uint16_t dpp_wqe_idx
;
385 /* provide synchronization to multiple context(s) posting wqe, rqe */
386 spinlock_t q_lock ____cacheline_aligned
;
387 struct ocrdma_cq
*sq_cq
;
388 /* list maintained per CQ to flush SQ errors */
389 struct list_head sq_entry
;
392 struct ocrdma_qp_hwq_info rq
;
394 struct ocrdma_cq
*rq_cq
;
395 struct ocrdma_srq
*srq
;
396 /* list maintained per CQ to flush RQ errors */
397 struct list_head rq_entry
;
399 enum ocrdma_qp_state state
; /* QP state */
401 u32 max_ord
, max_ird
;
404 struct ocrdma_pd
*pd
;
406 enum ib_qp_type qp_type
;
415 struct ocrdma_ucontext
{
416 struct ib_ucontext ibucontext
;
418 struct list_head mm_head
;
419 struct mutex mm_list_lock
; /* protects list entries of mm type */
420 struct ocrdma_pd
*cntxt_pd
;
435 struct list_head entry
;
438 static inline struct ocrdma_dev
*get_ocrdma_dev(struct ib_device
*ibdev
)
440 return container_of(ibdev
, struct ocrdma_dev
, ibdev
);
443 static inline struct ocrdma_ucontext
*get_ocrdma_ucontext(struct ib_ucontext
446 return container_of(ibucontext
, struct ocrdma_ucontext
, ibucontext
);
449 static inline struct ocrdma_pd
*get_ocrdma_pd(struct ib_pd
*ibpd
)
451 return container_of(ibpd
, struct ocrdma_pd
, ibpd
);
454 static inline struct ocrdma_cq
*get_ocrdma_cq(struct ib_cq
*ibcq
)
456 return container_of(ibcq
, struct ocrdma_cq
, ibcq
);
459 static inline struct ocrdma_qp
*get_ocrdma_qp(struct ib_qp
*ibqp
)
461 return container_of(ibqp
, struct ocrdma_qp
, ibqp
);
464 static inline struct ocrdma_mr
*get_ocrdma_mr(struct ib_mr
*ibmr
)
466 return container_of(ibmr
, struct ocrdma_mr
, ibmr
);
469 static inline struct ocrdma_ah
*get_ocrdma_ah(struct ib_ah
*ibah
)
471 return container_of(ibah
, struct ocrdma_ah
, ibah
);
474 static inline struct ocrdma_srq
*get_ocrdma_srq(struct ib_srq
*ibsrq
)
476 return container_of(ibsrq
, struct ocrdma_srq
, ibsrq
);
479 static inline int is_cqe_valid(struct ocrdma_cq
*cq
, struct ocrdma_cqe
*cqe
)
482 cqe_valid
= le32_to_cpu(cqe
->flags_status_srcqpn
) & OCRDMA_CQE_VALID
;
483 return (cqe_valid
== cq
->phase
);
486 static inline int is_cqe_for_sq(struct ocrdma_cqe
*cqe
)
488 return (le32_to_cpu(cqe
->flags_status_srcqpn
) &
489 OCRDMA_CQE_QTYPE
) ? 0 : 1;
492 static inline int is_cqe_invalidated(struct ocrdma_cqe
*cqe
)
494 return (le32_to_cpu(cqe
->flags_status_srcqpn
) &
495 OCRDMA_CQE_INVALIDATE
) ? 1 : 0;
498 static inline int is_cqe_imm(struct ocrdma_cqe
*cqe
)
500 return (le32_to_cpu(cqe
->flags_status_srcqpn
) &
501 OCRDMA_CQE_IMM
) ? 1 : 0;
504 static inline int is_cqe_wr_imm(struct ocrdma_cqe
*cqe
)
506 return (le32_to_cpu(cqe
->flags_status_srcqpn
) &
507 OCRDMA_CQE_WRITE_IMM
) ? 1 : 0;
510 static inline int ocrdma_resolve_dmac(struct ocrdma_dev
*dev
,
511 struct ib_ah_attr
*ah_attr
, u8
*mac_addr
)
515 memcpy(&in6
, ah_attr
->grh
.dgid
.raw
, sizeof(in6
));
516 if (rdma_is_multicast_addr(&in6
))
517 rdma_get_mcast_mac(&in6
, mac_addr
);
518 else if (rdma_link_local_addr(&in6
))
519 rdma_get_ll_mac(&in6
, mac_addr
);
521 memcpy(mac_addr
, ah_attr
->dmac
, ETH_ALEN
);
525 static inline char *hca_name(struct ocrdma_dev
*dev
)
527 switch (dev
->nic_info
.pdev
->device
) {
528 case OC_SKH_DEVICE_PF
:
529 case OC_SKH_DEVICE_VF
:
532 return OC_NAME_UNKNOWN
;
536 static inline int ocrdma_get_eq_table_index(struct ocrdma_dev
*dev
,
541 for (indx
= 0; indx
< dev
->eq_cnt
; indx
++) {
542 if (dev
->eq_tbl
[indx
].q
.id
== eqid
)
549 static inline u8
ocrdma_get_asic_type(struct ocrdma_dev
*dev
)
551 if (dev
->nic_info
.dev_family
== 0xF && !dev
->asic_id
) {
552 pci_read_config_dword(
554 OCRDMA_SLI_ASIC_ID_OFFSET
, &dev
->asic_id
);
557 return (dev
->asic_id
& OCRDMA_SLI_ASIC_GEN_NUM_MASK
) >>
558 OCRDMA_SLI_ASIC_GEN_NUM_SHIFT
;
561 static inline u8
ocrdma_get_pfc_prio(u8
*pfc
, u8 prio
)
563 return *(pfc
+ prio
);
566 static inline u8
ocrdma_get_app_prio(u8
*app_prio
, u8 prio
)
568 return *(app_prio
+ prio
);
571 static inline u8
ocrdma_is_enabled_and_synced(u32 state
)
572 { /* May also be used to interpret TC-state, QCN-state
573 * Appl-state and Logical-link-state in future.
575 return (state
& OCRDMA_STATE_FLAG_ENABLED
) &&
576 (state
& OCRDMA_STATE_FLAG_SYNC
);