2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies.
5 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/sched.h>
37 #include <linux/pci.h>
38 #include <linux/errno.h>
39 #include <linux/kernel.h>
41 #include <linux/slab.h>
42 #include <linux/mlx4/cmd.h>
43 #include <linux/mlx4/qp.h>
44 #include <linux/if_ether.h>
45 #include <linux/etherdevice.h>
50 #define MLX4_MAC_VALID (1ull << 63)
53 struct list_head list
;
59 struct list_head list
;
74 struct list_head list
;
76 enum mlx4_protocol prot
;
77 enum mlx4_steer_type steer
;
82 RES_QP_BUSY
= RES_ANY_BUSY
,
84 /* QP number was allocated */
87 /* ICM memory for QP context was mapped */
90 /* QP is in hw ownership */
95 struct res_common com
;
100 struct list_head mcg_list
;
106 enum res_mtt_states
{
107 RES_MTT_BUSY
= RES_ANY_BUSY
,
111 static inline const char *mtt_states_str(enum res_mtt_states state
)
114 case RES_MTT_BUSY
: return "RES_MTT_BUSY";
115 case RES_MTT_ALLOCATED
: return "RES_MTT_ALLOCATED";
116 default: return "Unknown";
121 struct res_common com
;
126 enum res_mpt_states
{
127 RES_MPT_BUSY
= RES_ANY_BUSY
,
134 struct res_common com
;
140 RES_EQ_BUSY
= RES_ANY_BUSY
,
146 struct res_common com
;
151 RES_CQ_BUSY
= RES_ANY_BUSY
,
157 struct res_common com
;
162 enum res_srq_states
{
163 RES_SRQ_BUSY
= RES_ANY_BUSY
,
169 struct res_common com
;
175 enum res_counter_states
{
176 RES_COUNTER_BUSY
= RES_ANY_BUSY
,
177 RES_COUNTER_ALLOCATED
,
181 struct res_common com
;
185 enum res_xrcdn_states
{
186 RES_XRCD_BUSY
= RES_ANY_BUSY
,
191 struct res_common com
;
195 enum res_fs_rule_states
{
196 RES_FS_RULE_BUSY
= RES_ANY_BUSY
,
197 RES_FS_RULE_ALLOCATED
,
201 struct res_common com
;
205 static void *res_tracker_lookup(struct rb_root
*root
, u64 res_id
)
207 struct rb_node
*node
= root
->rb_node
;
210 struct res_common
*res
= container_of(node
, struct res_common
,
213 if (res_id
< res
->res_id
)
214 node
= node
->rb_left
;
215 else if (res_id
> res
->res_id
)
216 node
= node
->rb_right
;
223 static int res_tracker_insert(struct rb_root
*root
, struct res_common
*res
)
225 struct rb_node
**new = &(root
->rb_node
), *parent
= NULL
;
227 /* Figure out where to put new node */
229 struct res_common
*this = container_of(*new, struct res_common
,
233 if (res
->res_id
< this->res_id
)
234 new = &((*new)->rb_left
);
235 else if (res
->res_id
> this->res_id
)
236 new = &((*new)->rb_right
);
241 /* Add new node and rebalance tree. */
242 rb_link_node(&res
->node
, parent
, new);
243 rb_insert_color(&res
->node
, root
);
258 static const char *ResourceType(enum mlx4_resource rt
)
261 case RES_QP
: return "RES_QP";
262 case RES_CQ
: return "RES_CQ";
263 case RES_SRQ
: return "RES_SRQ";
264 case RES_MPT
: return "RES_MPT";
265 case RES_MTT
: return "RES_MTT";
266 case RES_MAC
: return "RES_MAC";
267 case RES_EQ
: return "RES_EQ";
268 case RES_COUNTER
: return "RES_COUNTER";
269 case RES_FS_RULE
: return "RES_FS_RULE";
270 case RES_XRCD
: return "RES_XRCD";
271 default: return "Unknown resource type !!!";
275 int mlx4_init_resource_tracker(struct mlx4_dev
*dev
)
277 struct mlx4_priv
*priv
= mlx4_priv(dev
);
281 priv
->mfunc
.master
.res_tracker
.slave_list
=
282 kzalloc(dev
->num_slaves
* sizeof(struct slave_list
),
284 if (!priv
->mfunc
.master
.res_tracker
.slave_list
)
287 for (i
= 0 ; i
< dev
->num_slaves
; i
++) {
288 for (t
= 0; t
< MLX4_NUM_OF_RESOURCE_TYPE
; ++t
)
289 INIT_LIST_HEAD(&priv
->mfunc
.master
.res_tracker
.
290 slave_list
[i
].res_list
[t
]);
291 mutex_init(&priv
->mfunc
.master
.res_tracker
.slave_list
[i
].mutex
);
294 mlx4_dbg(dev
, "Started init_resource_tracker: %ld slaves\n",
296 for (i
= 0 ; i
< MLX4_NUM_OF_RESOURCE_TYPE
; i
++)
297 priv
->mfunc
.master
.res_tracker
.res_tree
[i
] = RB_ROOT
;
299 spin_lock_init(&priv
->mfunc
.master
.res_tracker
.lock
);
303 void mlx4_free_resource_tracker(struct mlx4_dev
*dev
,
304 enum mlx4_res_tracker_free_type type
)
306 struct mlx4_priv
*priv
= mlx4_priv(dev
);
309 if (priv
->mfunc
.master
.res_tracker
.slave_list
) {
310 if (type
!= RES_TR_FREE_STRUCTS_ONLY
)
311 for (i
= 0 ; i
< dev
->num_slaves
; i
++)
312 if (type
== RES_TR_FREE_ALL
||
313 dev
->caps
.function
!= i
)
314 mlx4_delete_all_resources_for_slave(dev
, i
);
316 if (type
!= RES_TR_FREE_SLAVES_ONLY
) {
317 kfree(priv
->mfunc
.master
.res_tracker
.slave_list
);
318 priv
->mfunc
.master
.res_tracker
.slave_list
= NULL
;
323 static void update_pkey_index(struct mlx4_dev
*dev
, int slave
,
324 struct mlx4_cmd_mailbox
*inbox
)
326 u8 sched
= *(u8
*)(inbox
->buf
+ 64);
327 u8 orig_index
= *(u8
*)(inbox
->buf
+ 35);
329 struct mlx4_priv
*priv
= mlx4_priv(dev
);
332 port
= (sched
>> 6 & 1) + 1;
334 new_index
= priv
->virt2phys_pkey
[slave
][port
- 1][orig_index
];
335 *(u8
*)(inbox
->buf
+ 35) = new_index
;
338 static void update_gid(struct mlx4_dev
*dev
, struct mlx4_cmd_mailbox
*inbox
,
341 struct mlx4_qp_context
*qp_ctx
= inbox
->buf
+ 8;
342 enum mlx4_qp_optpar optpar
= be32_to_cpu(*(__be32
*) inbox
->buf
);
343 u32 ts
= (be32_to_cpu(qp_ctx
->flags
) >> 16) & 0xff;
345 if (MLX4_QP_ST_UD
== ts
)
346 qp_ctx
->pri_path
.mgid_index
= 0x80 | slave
;
348 if (MLX4_QP_ST_RC
== ts
|| MLX4_QP_ST_UC
== ts
) {
349 if (optpar
& MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH
)
350 qp_ctx
->pri_path
.mgid_index
= slave
& 0x7F;
351 if (optpar
& MLX4_QP_OPTPAR_ALT_ADDR_PATH
)
352 qp_ctx
->alt_path
.mgid_index
= slave
& 0x7F;
356 static int mpt_mask(struct mlx4_dev
*dev
)
358 return dev
->caps
.num_mpts
- 1;
361 static void *find_res(struct mlx4_dev
*dev
, u64 res_id
,
362 enum mlx4_resource type
)
364 struct mlx4_priv
*priv
= mlx4_priv(dev
);
366 return res_tracker_lookup(&priv
->mfunc
.master
.res_tracker
.res_tree
[type
],
370 static int get_res(struct mlx4_dev
*dev
, int slave
, u64 res_id
,
371 enum mlx4_resource type
,
374 struct res_common
*r
;
377 spin_lock_irq(mlx4_tlock(dev
));
378 r
= find_res(dev
, res_id
, type
);
384 if (r
->state
== RES_ANY_BUSY
) {
389 if (r
->owner
!= slave
) {
394 r
->from_state
= r
->state
;
395 r
->state
= RES_ANY_BUSY
;
398 *((struct res_common
**)res
) = r
;
401 spin_unlock_irq(mlx4_tlock(dev
));
405 int mlx4_get_slave_from_resource_id(struct mlx4_dev
*dev
,
406 enum mlx4_resource type
,
407 u64 res_id
, int *slave
)
410 struct res_common
*r
;
416 spin_lock(mlx4_tlock(dev
));
418 r
= find_res(dev
, id
, type
);
423 spin_unlock(mlx4_tlock(dev
));
428 static void put_res(struct mlx4_dev
*dev
, int slave
, u64 res_id
,
429 enum mlx4_resource type
)
431 struct res_common
*r
;
433 spin_lock_irq(mlx4_tlock(dev
));
434 r
= find_res(dev
, res_id
, type
);
436 r
->state
= r
->from_state
;
437 spin_unlock_irq(mlx4_tlock(dev
));
440 static struct res_common
*alloc_qp_tr(int id
)
444 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
448 ret
->com
.res_id
= id
;
449 ret
->com
.state
= RES_QP_RESERVED
;
451 INIT_LIST_HEAD(&ret
->mcg_list
);
452 spin_lock_init(&ret
->mcg_spl
);
453 atomic_set(&ret
->ref_count
, 0);
458 static struct res_common
*alloc_mtt_tr(int id
, int order
)
462 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
466 ret
->com
.res_id
= id
;
468 ret
->com
.state
= RES_MTT_ALLOCATED
;
469 atomic_set(&ret
->ref_count
, 0);
474 static struct res_common
*alloc_mpt_tr(int id
, int key
)
478 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
482 ret
->com
.res_id
= id
;
483 ret
->com
.state
= RES_MPT_RESERVED
;
489 static struct res_common
*alloc_eq_tr(int id
)
493 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
497 ret
->com
.res_id
= id
;
498 ret
->com
.state
= RES_EQ_RESERVED
;
503 static struct res_common
*alloc_cq_tr(int id
)
507 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
511 ret
->com
.res_id
= id
;
512 ret
->com
.state
= RES_CQ_ALLOCATED
;
513 atomic_set(&ret
->ref_count
, 0);
518 static struct res_common
*alloc_srq_tr(int id
)
522 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
526 ret
->com
.res_id
= id
;
527 ret
->com
.state
= RES_SRQ_ALLOCATED
;
528 atomic_set(&ret
->ref_count
, 0);
533 static struct res_common
*alloc_counter_tr(int id
)
535 struct res_counter
*ret
;
537 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
541 ret
->com
.res_id
= id
;
542 ret
->com
.state
= RES_COUNTER_ALLOCATED
;
547 static struct res_common
*alloc_xrcdn_tr(int id
)
549 struct res_xrcdn
*ret
;
551 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
555 ret
->com
.res_id
= id
;
556 ret
->com
.state
= RES_XRCD_ALLOCATED
;
561 static struct res_common
*alloc_fs_rule_tr(u64 id
, int qpn
)
563 struct res_fs_rule
*ret
;
565 ret
= kzalloc(sizeof *ret
, GFP_KERNEL
);
569 ret
->com
.res_id
= id
;
570 ret
->com
.state
= RES_FS_RULE_ALLOCATED
;
575 static struct res_common
*alloc_tr(u64 id
, enum mlx4_resource type
, int slave
,
578 struct res_common
*ret
;
582 ret
= alloc_qp_tr(id
);
585 ret
= alloc_mpt_tr(id
, extra
);
588 ret
= alloc_mtt_tr(id
, extra
);
591 ret
= alloc_eq_tr(id
);
594 ret
= alloc_cq_tr(id
);
597 ret
= alloc_srq_tr(id
);
600 printk(KERN_ERR
"implementation missing\n");
603 ret
= alloc_counter_tr(id
);
606 ret
= alloc_xrcdn_tr(id
);
609 ret
= alloc_fs_rule_tr(id
, extra
);
620 static int add_res_range(struct mlx4_dev
*dev
, int slave
, u64 base
, int count
,
621 enum mlx4_resource type
, int extra
)
625 struct mlx4_priv
*priv
= mlx4_priv(dev
);
626 struct res_common
**res_arr
;
627 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
628 struct rb_root
*root
= &tracker
->res_tree
[type
];
630 res_arr
= kzalloc(count
* sizeof *res_arr
, GFP_KERNEL
);
634 for (i
= 0; i
< count
; ++i
) {
635 res_arr
[i
] = alloc_tr(base
+ i
, type
, slave
, extra
);
637 for (--i
; i
>= 0; --i
)
645 spin_lock_irq(mlx4_tlock(dev
));
646 for (i
= 0; i
< count
; ++i
) {
647 if (find_res(dev
, base
+ i
, type
)) {
651 err
= res_tracker_insert(root
, res_arr
[i
]);
654 list_add_tail(&res_arr
[i
]->list
,
655 &tracker
->slave_list
[slave
].res_list
[type
]);
657 spin_unlock_irq(mlx4_tlock(dev
));
663 for (--i
; i
>= base
; --i
)
664 rb_erase(&res_arr
[i
]->node
, root
);
666 spin_unlock_irq(mlx4_tlock(dev
));
668 for (i
= 0; i
< count
; ++i
)
676 static int remove_qp_ok(struct res_qp
*res
)
678 if (res
->com
.state
== RES_QP_BUSY
|| atomic_read(&res
->ref_count
) ||
679 !list_empty(&res
->mcg_list
)) {
680 pr_err("resource tracker: fail to remove qp, state %d, ref_count %d\n",
681 res
->com
.state
, atomic_read(&res
->ref_count
));
683 } else if (res
->com
.state
!= RES_QP_RESERVED
) {
690 static int remove_mtt_ok(struct res_mtt
*res
, int order
)
692 if (res
->com
.state
== RES_MTT_BUSY
||
693 atomic_read(&res
->ref_count
)) {
694 printk(KERN_DEBUG
"%s-%d: state %s, ref_count %d\n",
696 mtt_states_str(res
->com
.state
),
697 atomic_read(&res
->ref_count
));
699 } else if (res
->com
.state
!= RES_MTT_ALLOCATED
)
701 else if (res
->order
!= order
)
707 static int remove_mpt_ok(struct res_mpt
*res
)
709 if (res
->com
.state
== RES_MPT_BUSY
)
711 else if (res
->com
.state
!= RES_MPT_RESERVED
)
717 static int remove_eq_ok(struct res_eq
*res
)
719 if (res
->com
.state
== RES_MPT_BUSY
)
721 else if (res
->com
.state
!= RES_MPT_RESERVED
)
727 static int remove_counter_ok(struct res_counter
*res
)
729 if (res
->com
.state
== RES_COUNTER_BUSY
)
731 else if (res
->com
.state
!= RES_COUNTER_ALLOCATED
)
737 static int remove_xrcdn_ok(struct res_xrcdn
*res
)
739 if (res
->com
.state
== RES_XRCD_BUSY
)
741 else if (res
->com
.state
!= RES_XRCD_ALLOCATED
)
747 static int remove_fs_rule_ok(struct res_fs_rule
*res
)
749 if (res
->com
.state
== RES_FS_RULE_BUSY
)
751 else if (res
->com
.state
!= RES_FS_RULE_ALLOCATED
)
757 static int remove_cq_ok(struct res_cq
*res
)
759 if (res
->com
.state
== RES_CQ_BUSY
)
761 else if (res
->com
.state
!= RES_CQ_ALLOCATED
)
767 static int remove_srq_ok(struct res_srq
*res
)
769 if (res
->com
.state
== RES_SRQ_BUSY
)
771 else if (res
->com
.state
!= RES_SRQ_ALLOCATED
)
777 static int remove_ok(struct res_common
*res
, enum mlx4_resource type
, int extra
)
781 return remove_qp_ok((struct res_qp
*)res
);
783 return remove_cq_ok((struct res_cq
*)res
);
785 return remove_srq_ok((struct res_srq
*)res
);
787 return remove_mpt_ok((struct res_mpt
*)res
);
789 return remove_mtt_ok((struct res_mtt
*)res
, extra
);
793 return remove_eq_ok((struct res_eq
*)res
);
795 return remove_counter_ok((struct res_counter
*)res
);
797 return remove_xrcdn_ok((struct res_xrcdn
*)res
);
799 return remove_fs_rule_ok((struct res_fs_rule
*)res
);
805 static int rem_res_range(struct mlx4_dev
*dev
, int slave
, u64 base
, int count
,
806 enum mlx4_resource type
, int extra
)
810 struct mlx4_priv
*priv
= mlx4_priv(dev
);
811 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
812 struct res_common
*r
;
814 spin_lock_irq(mlx4_tlock(dev
));
815 for (i
= base
; i
< base
+ count
; ++i
) {
816 r
= res_tracker_lookup(&tracker
->res_tree
[type
], i
);
821 if (r
->owner
!= slave
) {
825 err
= remove_ok(r
, type
, extra
);
830 for (i
= base
; i
< base
+ count
; ++i
) {
831 r
= res_tracker_lookup(&tracker
->res_tree
[type
], i
);
832 rb_erase(&r
->node
, &tracker
->res_tree
[type
]);
839 spin_unlock_irq(mlx4_tlock(dev
));
844 static int qp_res_start_move_to(struct mlx4_dev
*dev
, int slave
, int qpn
,
845 enum res_qp_states state
, struct res_qp
**qp
,
848 struct mlx4_priv
*priv
= mlx4_priv(dev
);
849 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
853 spin_lock_irq(mlx4_tlock(dev
));
854 r
= res_tracker_lookup(&tracker
->res_tree
[RES_QP
], qpn
);
857 else if (r
->com
.owner
!= slave
)
862 mlx4_dbg(dev
, "%s: failed RES_QP, 0x%llx\n",
863 __func__
, r
->com
.res_id
);
867 case RES_QP_RESERVED
:
868 if (r
->com
.state
== RES_QP_MAPPED
&& !alloc
)
871 mlx4_dbg(dev
, "failed RES_QP, 0x%llx\n", r
->com
.res_id
);
876 if ((r
->com
.state
== RES_QP_RESERVED
&& alloc
) ||
877 r
->com
.state
== RES_QP_HW
)
880 mlx4_dbg(dev
, "failed RES_QP, 0x%llx\n",
888 if (r
->com
.state
!= RES_QP_MAPPED
)
896 r
->com
.from_state
= r
->com
.state
;
897 r
->com
.to_state
= state
;
898 r
->com
.state
= RES_QP_BUSY
;
904 spin_unlock_irq(mlx4_tlock(dev
));
909 static int mr_res_start_move_to(struct mlx4_dev
*dev
, int slave
, int index
,
910 enum res_mpt_states state
, struct res_mpt
**mpt
)
912 struct mlx4_priv
*priv
= mlx4_priv(dev
);
913 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
917 spin_lock_irq(mlx4_tlock(dev
));
918 r
= res_tracker_lookup(&tracker
->res_tree
[RES_MPT
], index
);
921 else if (r
->com
.owner
!= slave
)
929 case RES_MPT_RESERVED
:
930 if (r
->com
.state
!= RES_MPT_MAPPED
)
935 if (r
->com
.state
!= RES_MPT_RESERVED
&&
936 r
->com
.state
!= RES_MPT_HW
)
941 if (r
->com
.state
!= RES_MPT_MAPPED
)
949 r
->com
.from_state
= r
->com
.state
;
950 r
->com
.to_state
= state
;
951 r
->com
.state
= RES_MPT_BUSY
;
957 spin_unlock_irq(mlx4_tlock(dev
));
962 static int eq_res_start_move_to(struct mlx4_dev
*dev
, int slave
, int index
,
963 enum res_eq_states state
, struct res_eq
**eq
)
965 struct mlx4_priv
*priv
= mlx4_priv(dev
);
966 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
970 spin_lock_irq(mlx4_tlock(dev
));
971 r
= res_tracker_lookup(&tracker
->res_tree
[RES_EQ
], index
);
974 else if (r
->com
.owner
!= slave
)
982 case RES_EQ_RESERVED
:
983 if (r
->com
.state
!= RES_EQ_HW
)
988 if (r
->com
.state
!= RES_EQ_RESERVED
)
997 r
->com
.from_state
= r
->com
.state
;
998 r
->com
.to_state
= state
;
999 r
->com
.state
= RES_EQ_BUSY
;
1005 spin_unlock_irq(mlx4_tlock(dev
));
1010 static int cq_res_start_move_to(struct mlx4_dev
*dev
, int slave
, int cqn
,
1011 enum res_cq_states state
, struct res_cq
**cq
)
1013 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1014 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1018 spin_lock_irq(mlx4_tlock(dev
));
1019 r
= res_tracker_lookup(&tracker
->res_tree
[RES_CQ
], cqn
);
1022 else if (r
->com
.owner
!= slave
)
1030 case RES_CQ_ALLOCATED
:
1031 if (r
->com
.state
!= RES_CQ_HW
)
1033 else if (atomic_read(&r
->ref_count
))
1040 if (r
->com
.state
!= RES_CQ_ALLOCATED
)
1051 r
->com
.from_state
= r
->com
.state
;
1052 r
->com
.to_state
= state
;
1053 r
->com
.state
= RES_CQ_BUSY
;
1059 spin_unlock_irq(mlx4_tlock(dev
));
1064 static int srq_res_start_move_to(struct mlx4_dev
*dev
, int slave
, int index
,
1065 enum res_cq_states state
, struct res_srq
**srq
)
1067 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1068 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1072 spin_lock_irq(mlx4_tlock(dev
));
1073 r
= res_tracker_lookup(&tracker
->res_tree
[RES_SRQ
], index
);
1076 else if (r
->com
.owner
!= slave
)
1084 case RES_SRQ_ALLOCATED
:
1085 if (r
->com
.state
!= RES_SRQ_HW
)
1087 else if (atomic_read(&r
->ref_count
))
1092 if (r
->com
.state
!= RES_SRQ_ALLOCATED
)
1101 r
->com
.from_state
= r
->com
.state
;
1102 r
->com
.to_state
= state
;
1103 r
->com
.state
= RES_SRQ_BUSY
;
1109 spin_unlock_irq(mlx4_tlock(dev
));
1114 static void res_abort_move(struct mlx4_dev
*dev
, int slave
,
1115 enum mlx4_resource type
, int id
)
1117 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1118 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1119 struct res_common
*r
;
1121 spin_lock_irq(mlx4_tlock(dev
));
1122 r
= res_tracker_lookup(&tracker
->res_tree
[type
], id
);
1123 if (r
&& (r
->owner
== slave
))
1124 r
->state
= r
->from_state
;
1125 spin_unlock_irq(mlx4_tlock(dev
));
1128 static void res_end_move(struct mlx4_dev
*dev
, int slave
,
1129 enum mlx4_resource type
, int id
)
1131 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1132 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1133 struct res_common
*r
;
1135 spin_lock_irq(mlx4_tlock(dev
));
1136 r
= res_tracker_lookup(&tracker
->res_tree
[type
], id
);
1137 if (r
&& (r
->owner
== slave
))
1138 r
->state
= r
->to_state
;
1139 spin_unlock_irq(mlx4_tlock(dev
));
1142 static int valid_reserved(struct mlx4_dev
*dev
, int slave
, int qpn
)
1144 return mlx4_is_qp_reserved(dev
, qpn
) &&
1145 (mlx4_is_master(dev
) || mlx4_is_guest_proxy(dev
, slave
, qpn
));
1148 static int fw_reserved(struct mlx4_dev
*dev
, int qpn
)
1150 return qpn
< dev
->caps
.reserved_qps_cnt
[MLX4_QP_REGION_FW
];
1153 static int qp_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1154 u64 in_param
, u64
*out_param
)
1163 case RES_OP_RESERVE
:
1164 count
= get_param_l(&in_param
);
1165 align
= get_param_h(&in_param
);
1166 err
= __mlx4_qp_reserve_range(dev
, count
, align
, &base
);
1170 err
= add_res_range(dev
, slave
, base
, count
, RES_QP
, 0);
1172 __mlx4_qp_release_range(dev
, base
, count
);
1175 set_param_l(out_param
, base
);
1177 case RES_OP_MAP_ICM
:
1178 qpn
= get_param_l(&in_param
) & 0x7fffff;
1179 if (valid_reserved(dev
, slave
, qpn
)) {
1180 err
= add_res_range(dev
, slave
, qpn
, 1, RES_QP
, 0);
1185 err
= qp_res_start_move_to(dev
, slave
, qpn
, RES_QP_MAPPED
,
1190 if (!fw_reserved(dev
, qpn
)) {
1191 err
= __mlx4_qp_alloc_icm(dev
, qpn
);
1193 res_abort_move(dev
, slave
, RES_QP
, qpn
);
1198 res_end_move(dev
, slave
, RES_QP
, qpn
);
1208 static int mtt_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1209 u64 in_param
, u64
*out_param
)
1215 if (op
!= RES_OP_RESERVE_AND_MAP
)
1218 order
= get_param_l(&in_param
);
1219 base
= __mlx4_alloc_mtt_range(dev
, order
);
1223 err
= add_res_range(dev
, slave
, base
, 1, RES_MTT
, order
);
1225 __mlx4_free_mtt_range(dev
, base
, order
);
1227 set_param_l(out_param
, base
);
1232 static int mpt_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1233 u64 in_param
, u64
*out_param
)
1238 struct res_mpt
*mpt
;
1241 case RES_OP_RESERVE
:
1242 index
= __mlx4_mpt_reserve(dev
);
1245 id
= index
& mpt_mask(dev
);
1247 err
= add_res_range(dev
, slave
, id
, 1, RES_MPT
, index
);
1249 __mlx4_mpt_release(dev
, index
);
1252 set_param_l(out_param
, index
);
1254 case RES_OP_MAP_ICM
:
1255 index
= get_param_l(&in_param
);
1256 id
= index
& mpt_mask(dev
);
1257 err
= mr_res_start_move_to(dev
, slave
, id
,
1258 RES_MPT_MAPPED
, &mpt
);
1262 err
= __mlx4_mpt_alloc_icm(dev
, mpt
->key
);
1264 res_abort_move(dev
, slave
, RES_MPT
, id
);
1268 res_end_move(dev
, slave
, RES_MPT
, id
);
1274 static int cq_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1275 u64 in_param
, u64
*out_param
)
1281 case RES_OP_RESERVE_AND_MAP
:
1282 err
= __mlx4_cq_alloc_icm(dev
, &cqn
);
1286 err
= add_res_range(dev
, slave
, cqn
, 1, RES_CQ
, 0);
1288 __mlx4_cq_free_icm(dev
, cqn
);
1292 set_param_l(out_param
, cqn
);
1302 static int srq_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1303 u64 in_param
, u64
*out_param
)
1309 case RES_OP_RESERVE_AND_MAP
:
1310 err
= __mlx4_srq_alloc_icm(dev
, &srqn
);
1314 err
= add_res_range(dev
, slave
, srqn
, 1, RES_SRQ
, 0);
1316 __mlx4_srq_free_icm(dev
, srqn
);
1320 set_param_l(out_param
, srqn
);
1330 static int mac_add_to_slave(struct mlx4_dev
*dev
, int slave
, u64 mac
, int port
)
1332 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1333 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1334 struct mac_res
*res
;
1336 res
= kzalloc(sizeof *res
, GFP_KERNEL
);
1340 res
->port
= (u8
) port
;
1341 list_add_tail(&res
->list
,
1342 &tracker
->slave_list
[slave
].res_list
[RES_MAC
]);
1346 static void mac_del_from_slave(struct mlx4_dev
*dev
, int slave
, u64 mac
,
1349 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1350 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1351 struct list_head
*mac_list
=
1352 &tracker
->slave_list
[slave
].res_list
[RES_MAC
];
1353 struct mac_res
*res
, *tmp
;
1355 list_for_each_entry_safe(res
, tmp
, mac_list
, list
) {
1356 if (res
->mac
== mac
&& res
->port
== (u8
) port
) {
1357 list_del(&res
->list
);
1364 static void rem_slave_macs(struct mlx4_dev
*dev
, int slave
)
1366 struct mlx4_priv
*priv
= mlx4_priv(dev
);
1367 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
1368 struct list_head
*mac_list
=
1369 &tracker
->slave_list
[slave
].res_list
[RES_MAC
];
1370 struct mac_res
*res
, *tmp
;
1372 list_for_each_entry_safe(res
, tmp
, mac_list
, list
) {
1373 list_del(&res
->list
);
1374 __mlx4_unregister_mac(dev
, res
->port
, res
->mac
);
1379 static int mac_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1380 u64 in_param
, u64
*out_param
)
1386 if (op
!= RES_OP_RESERVE_AND_MAP
)
1389 port
= get_param_l(out_param
);
1392 err
= __mlx4_register_mac(dev
, port
, mac
);
1394 set_param_l(out_param
, err
);
1399 err
= mac_add_to_slave(dev
, slave
, mac
, port
);
1401 __mlx4_unregister_mac(dev
, port
, mac
);
1406 static int vlan_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1407 u64 in_param
, u64
*out_param
)
1412 static int counter_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1413 u64 in_param
, u64
*out_param
)
1418 if (op
!= RES_OP_RESERVE
)
1421 err
= __mlx4_counter_alloc(dev
, &index
);
1425 err
= add_res_range(dev
, slave
, index
, 1, RES_COUNTER
, 0);
1427 __mlx4_counter_free(dev
, index
);
1429 set_param_l(out_param
, index
);
1434 static int xrcdn_alloc_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1435 u64 in_param
, u64
*out_param
)
1440 if (op
!= RES_OP_RESERVE
)
1443 err
= __mlx4_xrcd_alloc(dev
, &xrcdn
);
1447 err
= add_res_range(dev
, slave
, xrcdn
, 1, RES_XRCD
, 0);
1449 __mlx4_xrcd_free(dev
, xrcdn
);
1451 set_param_l(out_param
, xrcdn
);
1456 int mlx4_ALLOC_RES_wrapper(struct mlx4_dev
*dev
, int slave
,
1457 struct mlx4_vhcr
*vhcr
,
1458 struct mlx4_cmd_mailbox
*inbox
,
1459 struct mlx4_cmd_mailbox
*outbox
,
1460 struct mlx4_cmd_info
*cmd
)
1463 int alop
= vhcr
->op_modifier
;
1465 switch (vhcr
->in_modifier
) {
1467 err
= qp_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1468 vhcr
->in_param
, &vhcr
->out_param
);
1472 err
= mtt_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1473 vhcr
->in_param
, &vhcr
->out_param
);
1477 err
= mpt_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1478 vhcr
->in_param
, &vhcr
->out_param
);
1482 err
= cq_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1483 vhcr
->in_param
, &vhcr
->out_param
);
1487 err
= srq_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1488 vhcr
->in_param
, &vhcr
->out_param
);
1492 err
= mac_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1493 vhcr
->in_param
, &vhcr
->out_param
);
1497 err
= vlan_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1498 vhcr
->in_param
, &vhcr
->out_param
);
1502 err
= counter_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1503 vhcr
->in_param
, &vhcr
->out_param
);
1507 err
= xrcdn_alloc_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1508 vhcr
->in_param
, &vhcr
->out_param
);
1519 static int qp_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1528 case RES_OP_RESERVE
:
1529 base
= get_param_l(&in_param
) & 0x7fffff;
1530 count
= get_param_h(&in_param
);
1531 err
= rem_res_range(dev
, slave
, base
, count
, RES_QP
, 0);
1534 __mlx4_qp_release_range(dev
, base
, count
);
1536 case RES_OP_MAP_ICM
:
1537 qpn
= get_param_l(&in_param
) & 0x7fffff;
1538 err
= qp_res_start_move_to(dev
, slave
, qpn
, RES_QP_RESERVED
,
1543 if (!fw_reserved(dev
, qpn
))
1544 __mlx4_qp_free_icm(dev
, qpn
);
1546 res_end_move(dev
, slave
, RES_QP
, qpn
);
1548 if (valid_reserved(dev
, slave
, qpn
))
1549 err
= rem_res_range(dev
, slave
, qpn
, 1, RES_QP
, 0);
1558 static int mtt_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1559 u64 in_param
, u64
*out_param
)
1565 if (op
!= RES_OP_RESERVE_AND_MAP
)
1568 base
= get_param_l(&in_param
);
1569 order
= get_param_h(&in_param
);
1570 err
= rem_res_range(dev
, slave
, base
, 1, RES_MTT
, order
);
1572 __mlx4_free_mtt_range(dev
, base
, order
);
1576 static int mpt_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1582 struct res_mpt
*mpt
;
1585 case RES_OP_RESERVE
:
1586 index
= get_param_l(&in_param
);
1587 id
= index
& mpt_mask(dev
);
1588 err
= get_res(dev
, slave
, id
, RES_MPT
, &mpt
);
1592 put_res(dev
, slave
, id
, RES_MPT
);
1594 err
= rem_res_range(dev
, slave
, id
, 1, RES_MPT
, 0);
1597 __mlx4_mpt_release(dev
, index
);
1599 case RES_OP_MAP_ICM
:
1600 index
= get_param_l(&in_param
);
1601 id
= index
& mpt_mask(dev
);
1602 err
= mr_res_start_move_to(dev
, slave
, id
,
1603 RES_MPT_RESERVED
, &mpt
);
1607 __mlx4_mpt_free_icm(dev
, mpt
->key
);
1608 res_end_move(dev
, slave
, RES_MPT
, id
);
1618 static int cq_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1619 u64 in_param
, u64
*out_param
)
1625 case RES_OP_RESERVE_AND_MAP
:
1626 cqn
= get_param_l(&in_param
);
1627 err
= rem_res_range(dev
, slave
, cqn
, 1, RES_CQ
, 0);
1631 __mlx4_cq_free_icm(dev
, cqn
);
1642 static int srq_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1643 u64 in_param
, u64
*out_param
)
1649 case RES_OP_RESERVE_AND_MAP
:
1650 srqn
= get_param_l(&in_param
);
1651 err
= rem_res_range(dev
, slave
, srqn
, 1, RES_SRQ
, 0);
1655 __mlx4_srq_free_icm(dev
, srqn
);
1666 static int mac_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1667 u64 in_param
, u64
*out_param
)
1673 case RES_OP_RESERVE_AND_MAP
:
1674 port
= get_param_l(out_param
);
1675 mac_del_from_slave(dev
, slave
, in_param
, port
);
1676 __mlx4_unregister_mac(dev
, port
, in_param
);
1687 static int vlan_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1688 u64 in_param
, u64
*out_param
)
1693 static int counter_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1694 u64 in_param
, u64
*out_param
)
1699 if (op
!= RES_OP_RESERVE
)
1702 index
= get_param_l(&in_param
);
1703 err
= rem_res_range(dev
, slave
, index
, 1, RES_COUNTER
, 0);
1707 __mlx4_counter_free(dev
, index
);
1712 static int xrcdn_free_res(struct mlx4_dev
*dev
, int slave
, int op
, int cmd
,
1713 u64 in_param
, u64
*out_param
)
1718 if (op
!= RES_OP_RESERVE
)
1721 xrcdn
= get_param_l(&in_param
);
1722 err
= rem_res_range(dev
, slave
, xrcdn
, 1, RES_XRCD
, 0);
1726 __mlx4_xrcd_free(dev
, xrcdn
);
1731 int mlx4_FREE_RES_wrapper(struct mlx4_dev
*dev
, int slave
,
1732 struct mlx4_vhcr
*vhcr
,
1733 struct mlx4_cmd_mailbox
*inbox
,
1734 struct mlx4_cmd_mailbox
*outbox
,
1735 struct mlx4_cmd_info
*cmd
)
1738 int alop
= vhcr
->op_modifier
;
1740 switch (vhcr
->in_modifier
) {
1742 err
= qp_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1747 err
= mtt_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1748 vhcr
->in_param
, &vhcr
->out_param
);
1752 err
= mpt_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1757 err
= cq_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1758 vhcr
->in_param
, &vhcr
->out_param
);
1762 err
= srq_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1763 vhcr
->in_param
, &vhcr
->out_param
);
1767 err
= mac_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1768 vhcr
->in_param
, &vhcr
->out_param
);
1772 err
= vlan_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1773 vhcr
->in_param
, &vhcr
->out_param
);
1777 err
= counter_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1778 vhcr
->in_param
, &vhcr
->out_param
);
1782 err
= xrcdn_free_res(dev
, slave
, vhcr
->op_modifier
, alop
,
1783 vhcr
->in_param
, &vhcr
->out_param
);
1791 /* ugly but other choices are uglier */
1792 static int mr_phys_mpt(struct mlx4_mpt_entry
*mpt
)
1794 return (be32_to_cpu(mpt
->flags
) >> 9) & 1;
1797 static int mr_get_mtt_addr(struct mlx4_mpt_entry
*mpt
)
1799 return (int)be64_to_cpu(mpt
->mtt_addr
) & 0xfffffff8;
1802 static int mr_get_mtt_size(struct mlx4_mpt_entry
*mpt
)
1804 return be32_to_cpu(mpt
->mtt_sz
);
1807 static u32
mr_get_pd(struct mlx4_mpt_entry
*mpt
)
1809 return be32_to_cpu(mpt
->pd_flags
) & 0x00ffffff;
1812 static int mr_is_fmr(struct mlx4_mpt_entry
*mpt
)
1814 return be32_to_cpu(mpt
->pd_flags
) & MLX4_MPT_PD_FLAG_FAST_REG
;
1817 static int mr_is_bind_enabled(struct mlx4_mpt_entry
*mpt
)
1819 return be32_to_cpu(mpt
->flags
) & MLX4_MPT_FLAG_BIND_ENABLE
;
1822 static int mr_is_region(struct mlx4_mpt_entry
*mpt
)
1824 return be32_to_cpu(mpt
->flags
) & MLX4_MPT_FLAG_REGION
;
1827 static int qp_get_mtt_addr(struct mlx4_qp_context
*qpc
)
1829 return be32_to_cpu(qpc
->mtt_base_addr_l
) & 0xfffffff8;
1832 static int srq_get_mtt_addr(struct mlx4_srq_context
*srqc
)
1834 return be32_to_cpu(srqc
->mtt_base_addr_l
) & 0xfffffff8;
1837 static int qp_get_mtt_size(struct mlx4_qp_context
*qpc
)
1839 int page_shift
= (qpc
->log_page_size
& 0x3f) + 12;
1840 int log_sq_size
= (qpc
->sq_size_stride
>> 3) & 0xf;
1841 int log_sq_sride
= qpc
->sq_size_stride
& 7;
1842 int log_rq_size
= (qpc
->rq_size_stride
>> 3) & 0xf;
1843 int log_rq_stride
= qpc
->rq_size_stride
& 7;
1844 int srq
= (be32_to_cpu(qpc
->srqn
) >> 24) & 1;
1845 int rss
= (be32_to_cpu(qpc
->flags
) >> 13) & 1;
1846 int xrc
= (be32_to_cpu(qpc
->local_qpn
) >> 23) & 1;
1851 int page_offset
= (be32_to_cpu(qpc
->params2
) >> 6) & 0x3f;
1853 sq_size
= 1 << (log_sq_size
+ log_sq_sride
+ 4);
1854 rq_size
= (srq
|rss
|xrc
) ? 0 : (1 << (log_rq_size
+ log_rq_stride
+ 4));
1855 total_mem
= sq_size
+ rq_size
;
1857 roundup_pow_of_two((total_mem
+ (page_offset
<< 6)) >>
1863 static int check_mtt_range(struct mlx4_dev
*dev
, int slave
, int start
,
1864 int size
, struct res_mtt
*mtt
)
1866 int res_start
= mtt
->com
.res_id
;
1867 int res_size
= (1 << mtt
->order
);
1869 if (start
< res_start
|| start
+ size
> res_start
+ res_size
)
1874 int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev
*dev
, int slave
,
1875 struct mlx4_vhcr
*vhcr
,
1876 struct mlx4_cmd_mailbox
*inbox
,
1877 struct mlx4_cmd_mailbox
*outbox
,
1878 struct mlx4_cmd_info
*cmd
)
1881 int index
= vhcr
->in_modifier
;
1882 struct res_mtt
*mtt
;
1883 struct res_mpt
*mpt
;
1884 int mtt_base
= mr_get_mtt_addr(inbox
->buf
) / dev
->caps
.mtt_entry_sz
;
1890 id
= index
& mpt_mask(dev
);
1891 err
= mr_res_start_move_to(dev
, slave
, id
, RES_MPT_HW
, &mpt
);
1895 /* Disable memory windows for VFs. */
1896 if (!mr_is_region(inbox
->buf
)) {
1901 /* Make sure that the PD bits related to the slave id are zeros. */
1902 pd
= mr_get_pd(inbox
->buf
);
1903 pd_slave
= (pd
>> 17) & 0x7f;
1904 if (pd_slave
!= 0 && pd_slave
!= slave
) {
1909 if (mr_is_fmr(inbox
->buf
)) {
1910 /* FMR and Bind Enable are forbidden in slave devices. */
1911 if (mr_is_bind_enabled(inbox
->buf
)) {
1915 /* FMR and Memory Windows are also forbidden. */
1916 if (!mr_is_region(inbox
->buf
)) {
1922 phys
= mr_phys_mpt(inbox
->buf
);
1924 err
= get_res(dev
, slave
, mtt_base
, RES_MTT
, &mtt
);
1928 err
= check_mtt_range(dev
, slave
, mtt_base
,
1929 mr_get_mtt_size(inbox
->buf
), mtt
);
1936 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
1941 atomic_inc(&mtt
->ref_count
);
1942 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
1945 res_end_move(dev
, slave
, RES_MPT
, id
);
1950 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
1952 res_abort_move(dev
, slave
, RES_MPT
, id
);
1957 int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev
*dev
, int slave
,
1958 struct mlx4_vhcr
*vhcr
,
1959 struct mlx4_cmd_mailbox
*inbox
,
1960 struct mlx4_cmd_mailbox
*outbox
,
1961 struct mlx4_cmd_info
*cmd
)
1964 int index
= vhcr
->in_modifier
;
1965 struct res_mpt
*mpt
;
1968 id
= index
& mpt_mask(dev
);
1969 err
= mr_res_start_move_to(dev
, slave
, id
, RES_MPT_MAPPED
, &mpt
);
1973 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
1978 atomic_dec(&mpt
->mtt
->ref_count
);
1980 res_end_move(dev
, slave
, RES_MPT
, id
);
1984 res_abort_move(dev
, slave
, RES_MPT
, id
);
1989 int mlx4_QUERY_MPT_wrapper(struct mlx4_dev
*dev
, int slave
,
1990 struct mlx4_vhcr
*vhcr
,
1991 struct mlx4_cmd_mailbox
*inbox
,
1992 struct mlx4_cmd_mailbox
*outbox
,
1993 struct mlx4_cmd_info
*cmd
)
1996 int index
= vhcr
->in_modifier
;
1997 struct res_mpt
*mpt
;
2000 id
= index
& mpt_mask(dev
);
2001 err
= get_res(dev
, slave
, id
, RES_MPT
, &mpt
);
2005 if (mpt
->com
.from_state
!= RES_MPT_HW
) {
2010 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2013 put_res(dev
, slave
, id
, RES_MPT
);
2017 static int qp_get_rcqn(struct mlx4_qp_context
*qpc
)
2019 return be32_to_cpu(qpc
->cqn_recv
) & 0xffffff;
2022 static int qp_get_scqn(struct mlx4_qp_context
*qpc
)
2024 return be32_to_cpu(qpc
->cqn_send
) & 0xffffff;
2027 static u32
qp_get_srqn(struct mlx4_qp_context
*qpc
)
2029 return be32_to_cpu(qpc
->srqn
) & 0x1ffffff;
2032 static void adjust_proxy_tun_qkey(struct mlx4_dev
*dev
, struct mlx4_vhcr
*vhcr
,
2033 struct mlx4_qp_context
*context
)
2035 u32 qpn
= vhcr
->in_modifier
& 0xffffff;
2038 if (mlx4_get_parav_qkey(dev
, qpn
, &qkey
))
2041 /* adjust qkey in qp context */
2042 context
->qkey
= cpu_to_be32(qkey
);
2045 int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2046 struct mlx4_vhcr
*vhcr
,
2047 struct mlx4_cmd_mailbox
*inbox
,
2048 struct mlx4_cmd_mailbox
*outbox
,
2049 struct mlx4_cmd_info
*cmd
)
2052 int qpn
= vhcr
->in_modifier
& 0x7fffff;
2053 struct res_mtt
*mtt
;
2055 struct mlx4_qp_context
*qpc
= inbox
->buf
+ 8;
2056 int mtt_base
= qp_get_mtt_addr(qpc
) / dev
->caps
.mtt_entry_sz
;
2057 int mtt_size
= qp_get_mtt_size(qpc
);
2060 int rcqn
= qp_get_rcqn(qpc
);
2061 int scqn
= qp_get_scqn(qpc
);
2062 u32 srqn
= qp_get_srqn(qpc
) & 0xffffff;
2063 int use_srq
= (qp_get_srqn(qpc
) >> 24) & 1;
2064 struct res_srq
*srq
;
2065 int local_qpn
= be32_to_cpu(qpc
->local_qpn
) & 0xffffff;
2067 err
= qp_res_start_move_to(dev
, slave
, qpn
, RES_QP_HW
, &qp
, 0);
2070 qp
->local_qpn
= local_qpn
;
2072 err
= get_res(dev
, slave
, mtt_base
, RES_MTT
, &mtt
);
2076 err
= check_mtt_range(dev
, slave
, mtt_base
, mtt_size
, mtt
);
2080 err
= get_res(dev
, slave
, rcqn
, RES_CQ
, &rcq
);
2085 err
= get_res(dev
, slave
, scqn
, RES_CQ
, &scq
);
2092 err
= get_res(dev
, slave
, srqn
, RES_SRQ
, &srq
);
2097 adjust_proxy_tun_qkey(dev
, vhcr
, qpc
);
2098 update_pkey_index(dev
, slave
, inbox
);
2099 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2102 atomic_inc(&mtt
->ref_count
);
2104 atomic_inc(&rcq
->ref_count
);
2106 atomic_inc(&scq
->ref_count
);
2110 put_res(dev
, slave
, scqn
, RES_CQ
);
2113 atomic_inc(&srq
->ref_count
);
2114 put_res(dev
, slave
, srqn
, RES_SRQ
);
2117 put_res(dev
, slave
, rcqn
, RES_CQ
);
2118 put_res(dev
, slave
, mtt_base
, RES_MTT
);
2119 res_end_move(dev
, slave
, RES_QP
, qpn
);
2125 put_res(dev
, slave
, srqn
, RES_SRQ
);
2128 put_res(dev
, slave
, scqn
, RES_CQ
);
2130 put_res(dev
, slave
, rcqn
, RES_CQ
);
2132 put_res(dev
, slave
, mtt_base
, RES_MTT
);
2134 res_abort_move(dev
, slave
, RES_QP
, qpn
);
2139 static int eq_get_mtt_addr(struct mlx4_eq_context
*eqc
)
2141 return be32_to_cpu(eqc
->mtt_base_addr_l
) & 0xfffffff8;
2144 static int eq_get_mtt_size(struct mlx4_eq_context
*eqc
)
2146 int log_eq_size
= eqc
->log_eq_size
& 0x1f;
2147 int page_shift
= (eqc
->log_page_size
& 0x3f) + 12;
2149 if (log_eq_size
+ 5 < page_shift
)
2152 return 1 << (log_eq_size
+ 5 - page_shift
);
2155 static int cq_get_mtt_addr(struct mlx4_cq_context
*cqc
)
2157 return be32_to_cpu(cqc
->mtt_base_addr_l
) & 0xfffffff8;
2160 static int cq_get_mtt_size(struct mlx4_cq_context
*cqc
)
2162 int log_cq_size
= (be32_to_cpu(cqc
->logsize_usrpage
) >> 24) & 0x1f;
2163 int page_shift
= (cqc
->log_page_size
& 0x3f) + 12;
2165 if (log_cq_size
+ 5 < page_shift
)
2168 return 1 << (log_cq_size
+ 5 - page_shift
);
2171 int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2172 struct mlx4_vhcr
*vhcr
,
2173 struct mlx4_cmd_mailbox
*inbox
,
2174 struct mlx4_cmd_mailbox
*outbox
,
2175 struct mlx4_cmd_info
*cmd
)
2178 int eqn
= vhcr
->in_modifier
;
2179 int res_id
= (slave
<< 8) | eqn
;
2180 struct mlx4_eq_context
*eqc
= inbox
->buf
;
2181 int mtt_base
= eq_get_mtt_addr(eqc
) / dev
->caps
.mtt_entry_sz
;
2182 int mtt_size
= eq_get_mtt_size(eqc
);
2184 struct res_mtt
*mtt
;
2186 err
= add_res_range(dev
, slave
, res_id
, 1, RES_EQ
, 0);
2189 err
= eq_res_start_move_to(dev
, slave
, res_id
, RES_EQ_HW
, &eq
);
2193 err
= get_res(dev
, slave
, mtt_base
, RES_MTT
, &mtt
);
2197 err
= check_mtt_range(dev
, slave
, mtt_base
, mtt_size
, mtt
);
2201 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2205 atomic_inc(&mtt
->ref_count
);
2207 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2208 res_end_move(dev
, slave
, RES_EQ
, res_id
);
2212 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2214 res_abort_move(dev
, slave
, RES_EQ
, res_id
);
2216 rem_res_range(dev
, slave
, res_id
, 1, RES_EQ
, 0);
2220 static int get_containing_mtt(struct mlx4_dev
*dev
, int slave
, int start
,
2221 int len
, struct res_mtt
**res
)
2223 struct mlx4_priv
*priv
= mlx4_priv(dev
);
2224 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
2225 struct res_mtt
*mtt
;
2228 spin_lock_irq(mlx4_tlock(dev
));
2229 list_for_each_entry(mtt
, &tracker
->slave_list
[slave
].res_list
[RES_MTT
],
2231 if (!check_mtt_range(dev
, slave
, start
, len
, mtt
)) {
2233 mtt
->com
.from_state
= mtt
->com
.state
;
2234 mtt
->com
.state
= RES_MTT_BUSY
;
2239 spin_unlock_irq(mlx4_tlock(dev
));
2244 static int verify_qp_parameters(struct mlx4_dev
*dev
,
2245 struct mlx4_cmd_mailbox
*inbox
,
2246 enum qp_transition transition
, u8 slave
)
2249 struct mlx4_qp_context
*qp_ctx
;
2250 enum mlx4_qp_optpar optpar
;
2252 qp_ctx
= inbox
->buf
+ 8;
2253 qp_type
= (be32_to_cpu(qp_ctx
->flags
) >> 16) & 0xff;
2254 optpar
= be32_to_cpu(*(__be32
*) inbox
->buf
);
2259 switch (transition
) {
2260 case QP_TRANS_INIT2RTR
:
2261 case QP_TRANS_RTR2RTS
:
2262 case QP_TRANS_RTS2RTS
:
2263 case QP_TRANS_SQD2SQD
:
2264 case QP_TRANS_SQD2RTS
:
2265 if (slave
!= mlx4_master_func_num(dev
))
2266 /* slaves have only gid index 0 */
2267 if (optpar
& MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH
)
2268 if (qp_ctx
->pri_path
.mgid_index
)
2270 if (optpar
& MLX4_QP_OPTPAR_ALT_ADDR_PATH
)
2271 if (qp_ctx
->alt_path
.mgid_index
)
2286 int mlx4_WRITE_MTT_wrapper(struct mlx4_dev
*dev
, int slave
,
2287 struct mlx4_vhcr
*vhcr
,
2288 struct mlx4_cmd_mailbox
*inbox
,
2289 struct mlx4_cmd_mailbox
*outbox
,
2290 struct mlx4_cmd_info
*cmd
)
2292 struct mlx4_mtt mtt
;
2293 __be64
*page_list
= inbox
->buf
;
2294 u64
*pg_list
= (u64
*)page_list
;
2296 struct res_mtt
*rmtt
= NULL
;
2297 int start
= be64_to_cpu(page_list
[0]);
2298 int npages
= vhcr
->in_modifier
;
2301 err
= get_containing_mtt(dev
, slave
, start
, npages
, &rmtt
);
2305 /* Call the SW implementation of write_mtt:
2306 * - Prepare a dummy mtt struct
2307 * - Translate inbox contents to simple addresses in host endianess */
2308 mtt
.offset
= 0; /* TBD this is broken but I don't handle it since
2309 we don't really use it */
2312 for (i
= 0; i
< npages
; ++i
)
2313 pg_list
[i
+ 2] = (be64_to_cpu(page_list
[i
+ 2]) & ~1ULL);
2315 err
= __mlx4_write_mtt(dev
, &mtt
, be64_to_cpu(page_list
[0]), npages
,
2316 ((u64
*)page_list
+ 2));
2319 put_res(dev
, slave
, rmtt
->com
.res_id
, RES_MTT
);
2324 int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2325 struct mlx4_vhcr
*vhcr
,
2326 struct mlx4_cmd_mailbox
*inbox
,
2327 struct mlx4_cmd_mailbox
*outbox
,
2328 struct mlx4_cmd_info
*cmd
)
2330 int eqn
= vhcr
->in_modifier
;
2331 int res_id
= eqn
| (slave
<< 8);
2335 err
= eq_res_start_move_to(dev
, slave
, res_id
, RES_EQ_RESERVED
, &eq
);
2339 err
= get_res(dev
, slave
, eq
->mtt
->com
.res_id
, RES_MTT
, NULL
);
2343 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2347 atomic_dec(&eq
->mtt
->ref_count
);
2348 put_res(dev
, slave
, eq
->mtt
->com
.res_id
, RES_MTT
);
2349 res_end_move(dev
, slave
, RES_EQ
, res_id
);
2350 rem_res_range(dev
, slave
, res_id
, 1, RES_EQ
, 0);
2355 put_res(dev
, slave
, eq
->mtt
->com
.res_id
, RES_MTT
);
2357 res_abort_move(dev
, slave
, RES_EQ
, res_id
);
2362 int mlx4_GEN_EQE(struct mlx4_dev
*dev
, int slave
, struct mlx4_eqe
*eqe
)
2364 struct mlx4_priv
*priv
= mlx4_priv(dev
);
2365 struct mlx4_slave_event_eq_info
*event_eq
;
2366 struct mlx4_cmd_mailbox
*mailbox
;
2367 u32 in_modifier
= 0;
2372 if (!priv
->mfunc
.master
.slave_state
)
2375 event_eq
= &priv
->mfunc
.master
.slave_state
[slave
].event_eq
[eqe
->type
];
2377 /* Create the event only if the slave is registered */
2378 if (event_eq
->eqn
< 0)
2381 mutex_lock(&priv
->mfunc
.master
.gen_eqe_mutex
[slave
]);
2382 res_id
= (slave
<< 8) | event_eq
->eqn
;
2383 err
= get_res(dev
, slave
, res_id
, RES_EQ
, &req
);
2387 if (req
->com
.from_state
!= RES_EQ_HW
) {
2392 mailbox
= mlx4_alloc_cmd_mailbox(dev
);
2393 if (IS_ERR(mailbox
)) {
2394 err
= PTR_ERR(mailbox
);
2398 if (eqe
->type
== MLX4_EVENT_TYPE_CMD
) {
2400 eqe
->event
.cmd
.token
= cpu_to_be16(event_eq
->token
);
2403 memcpy(mailbox
->buf
, (u8
*) eqe
, 28);
2405 in_modifier
= (slave
& 0xff) | ((event_eq
->eqn
& 0xff) << 16);
2407 err
= mlx4_cmd(dev
, mailbox
->dma
, in_modifier
, 0,
2408 MLX4_CMD_GEN_EQE
, MLX4_CMD_TIME_CLASS_B
,
2411 put_res(dev
, slave
, res_id
, RES_EQ
);
2412 mutex_unlock(&priv
->mfunc
.master
.gen_eqe_mutex
[slave
]);
2413 mlx4_free_cmd_mailbox(dev
, mailbox
);
2417 put_res(dev
, slave
, res_id
, RES_EQ
);
2420 mutex_unlock(&priv
->mfunc
.master
.gen_eqe_mutex
[slave
]);
2424 int mlx4_QUERY_EQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2425 struct mlx4_vhcr
*vhcr
,
2426 struct mlx4_cmd_mailbox
*inbox
,
2427 struct mlx4_cmd_mailbox
*outbox
,
2428 struct mlx4_cmd_info
*cmd
)
2430 int eqn
= vhcr
->in_modifier
;
2431 int res_id
= eqn
| (slave
<< 8);
2435 err
= get_res(dev
, slave
, res_id
, RES_EQ
, &eq
);
2439 if (eq
->com
.from_state
!= RES_EQ_HW
) {
2444 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2447 put_res(dev
, slave
, res_id
, RES_EQ
);
2451 int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2452 struct mlx4_vhcr
*vhcr
,
2453 struct mlx4_cmd_mailbox
*inbox
,
2454 struct mlx4_cmd_mailbox
*outbox
,
2455 struct mlx4_cmd_info
*cmd
)
2458 int cqn
= vhcr
->in_modifier
;
2459 struct mlx4_cq_context
*cqc
= inbox
->buf
;
2460 int mtt_base
= cq_get_mtt_addr(cqc
) / dev
->caps
.mtt_entry_sz
;
2462 struct res_mtt
*mtt
;
2464 err
= cq_res_start_move_to(dev
, slave
, cqn
, RES_CQ_HW
, &cq
);
2467 err
= get_res(dev
, slave
, mtt_base
, RES_MTT
, &mtt
);
2470 err
= check_mtt_range(dev
, slave
, mtt_base
, cq_get_mtt_size(cqc
), mtt
);
2473 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2476 atomic_inc(&mtt
->ref_count
);
2478 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2479 res_end_move(dev
, slave
, RES_CQ
, cqn
);
2483 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2485 res_abort_move(dev
, slave
, RES_CQ
, cqn
);
2489 int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2490 struct mlx4_vhcr
*vhcr
,
2491 struct mlx4_cmd_mailbox
*inbox
,
2492 struct mlx4_cmd_mailbox
*outbox
,
2493 struct mlx4_cmd_info
*cmd
)
2496 int cqn
= vhcr
->in_modifier
;
2499 err
= cq_res_start_move_to(dev
, slave
, cqn
, RES_CQ_ALLOCATED
, &cq
);
2502 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2505 atomic_dec(&cq
->mtt
->ref_count
);
2506 res_end_move(dev
, slave
, RES_CQ
, cqn
);
2510 res_abort_move(dev
, slave
, RES_CQ
, cqn
);
2514 int mlx4_QUERY_CQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2515 struct mlx4_vhcr
*vhcr
,
2516 struct mlx4_cmd_mailbox
*inbox
,
2517 struct mlx4_cmd_mailbox
*outbox
,
2518 struct mlx4_cmd_info
*cmd
)
2520 int cqn
= vhcr
->in_modifier
;
2524 err
= get_res(dev
, slave
, cqn
, RES_CQ
, &cq
);
2528 if (cq
->com
.from_state
!= RES_CQ_HW
)
2531 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2533 put_res(dev
, slave
, cqn
, RES_CQ
);
2538 static int handle_resize(struct mlx4_dev
*dev
, int slave
,
2539 struct mlx4_vhcr
*vhcr
,
2540 struct mlx4_cmd_mailbox
*inbox
,
2541 struct mlx4_cmd_mailbox
*outbox
,
2542 struct mlx4_cmd_info
*cmd
,
2546 struct res_mtt
*orig_mtt
;
2547 struct res_mtt
*mtt
;
2548 struct mlx4_cq_context
*cqc
= inbox
->buf
;
2549 int mtt_base
= cq_get_mtt_addr(cqc
) / dev
->caps
.mtt_entry_sz
;
2551 err
= get_res(dev
, slave
, cq
->mtt
->com
.res_id
, RES_MTT
, &orig_mtt
);
2555 if (orig_mtt
!= cq
->mtt
) {
2560 err
= get_res(dev
, slave
, mtt_base
, RES_MTT
, &mtt
);
2564 err
= check_mtt_range(dev
, slave
, mtt_base
, cq_get_mtt_size(cqc
), mtt
);
2567 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2570 atomic_dec(&orig_mtt
->ref_count
);
2571 put_res(dev
, slave
, orig_mtt
->com
.res_id
, RES_MTT
);
2572 atomic_inc(&mtt
->ref_count
);
2574 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2578 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2580 put_res(dev
, slave
, orig_mtt
->com
.res_id
, RES_MTT
);
2586 int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2587 struct mlx4_vhcr
*vhcr
,
2588 struct mlx4_cmd_mailbox
*inbox
,
2589 struct mlx4_cmd_mailbox
*outbox
,
2590 struct mlx4_cmd_info
*cmd
)
2592 int cqn
= vhcr
->in_modifier
;
2596 err
= get_res(dev
, slave
, cqn
, RES_CQ
, &cq
);
2600 if (cq
->com
.from_state
!= RES_CQ_HW
)
2603 if (vhcr
->op_modifier
== 0) {
2604 err
= handle_resize(dev
, slave
, vhcr
, inbox
, outbox
, cmd
, cq
);
2608 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2610 put_res(dev
, slave
, cqn
, RES_CQ
);
2615 static int srq_get_mtt_size(struct mlx4_srq_context
*srqc
)
2617 int log_srq_size
= (be32_to_cpu(srqc
->state_logsize_srqn
) >> 24) & 0xf;
2618 int log_rq_stride
= srqc
->logstride
& 7;
2619 int page_shift
= (srqc
->log_page_size
& 0x3f) + 12;
2621 if (log_srq_size
+ log_rq_stride
+ 4 < page_shift
)
2624 return 1 << (log_srq_size
+ log_rq_stride
+ 4 - page_shift
);
2627 int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2628 struct mlx4_vhcr
*vhcr
,
2629 struct mlx4_cmd_mailbox
*inbox
,
2630 struct mlx4_cmd_mailbox
*outbox
,
2631 struct mlx4_cmd_info
*cmd
)
2634 int srqn
= vhcr
->in_modifier
;
2635 struct res_mtt
*mtt
;
2636 struct res_srq
*srq
;
2637 struct mlx4_srq_context
*srqc
= inbox
->buf
;
2638 int mtt_base
= srq_get_mtt_addr(srqc
) / dev
->caps
.mtt_entry_sz
;
2640 if (srqn
!= (be32_to_cpu(srqc
->state_logsize_srqn
) & 0xffffff))
2643 err
= srq_res_start_move_to(dev
, slave
, srqn
, RES_SRQ_HW
, &srq
);
2646 err
= get_res(dev
, slave
, mtt_base
, RES_MTT
, &mtt
);
2649 err
= check_mtt_range(dev
, slave
, mtt_base
, srq_get_mtt_size(srqc
),
2654 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2658 atomic_inc(&mtt
->ref_count
);
2660 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2661 res_end_move(dev
, slave
, RES_SRQ
, srqn
);
2665 put_res(dev
, slave
, mtt
->com
.res_id
, RES_MTT
);
2667 res_abort_move(dev
, slave
, RES_SRQ
, srqn
);
2672 int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2673 struct mlx4_vhcr
*vhcr
,
2674 struct mlx4_cmd_mailbox
*inbox
,
2675 struct mlx4_cmd_mailbox
*outbox
,
2676 struct mlx4_cmd_info
*cmd
)
2679 int srqn
= vhcr
->in_modifier
;
2680 struct res_srq
*srq
;
2682 err
= srq_res_start_move_to(dev
, slave
, srqn
, RES_SRQ_ALLOCATED
, &srq
);
2685 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2688 atomic_dec(&srq
->mtt
->ref_count
);
2690 atomic_dec(&srq
->cq
->ref_count
);
2691 res_end_move(dev
, slave
, RES_SRQ
, srqn
);
2696 res_abort_move(dev
, slave
, RES_SRQ
, srqn
);
2701 int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2702 struct mlx4_vhcr
*vhcr
,
2703 struct mlx4_cmd_mailbox
*inbox
,
2704 struct mlx4_cmd_mailbox
*outbox
,
2705 struct mlx4_cmd_info
*cmd
)
2708 int srqn
= vhcr
->in_modifier
;
2709 struct res_srq
*srq
;
2711 err
= get_res(dev
, slave
, srqn
, RES_SRQ
, &srq
);
2714 if (srq
->com
.from_state
!= RES_SRQ_HW
) {
2718 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2720 put_res(dev
, slave
, srqn
, RES_SRQ
);
2724 int mlx4_ARM_SRQ_wrapper(struct mlx4_dev
*dev
, int slave
,
2725 struct mlx4_vhcr
*vhcr
,
2726 struct mlx4_cmd_mailbox
*inbox
,
2727 struct mlx4_cmd_mailbox
*outbox
,
2728 struct mlx4_cmd_info
*cmd
)
2731 int srqn
= vhcr
->in_modifier
;
2732 struct res_srq
*srq
;
2734 err
= get_res(dev
, slave
, srqn
, RES_SRQ
, &srq
);
2738 if (srq
->com
.from_state
!= RES_SRQ_HW
) {
2743 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2745 put_res(dev
, slave
, srqn
, RES_SRQ
);
2749 int mlx4_GEN_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2750 struct mlx4_vhcr
*vhcr
,
2751 struct mlx4_cmd_mailbox
*inbox
,
2752 struct mlx4_cmd_mailbox
*outbox
,
2753 struct mlx4_cmd_info
*cmd
)
2756 int qpn
= vhcr
->in_modifier
& 0x7fffff;
2759 err
= get_res(dev
, slave
, qpn
, RES_QP
, &qp
);
2762 if (qp
->com
.from_state
!= RES_QP_HW
) {
2767 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2769 put_res(dev
, slave
, qpn
, RES_QP
);
2773 int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2774 struct mlx4_vhcr
*vhcr
,
2775 struct mlx4_cmd_mailbox
*inbox
,
2776 struct mlx4_cmd_mailbox
*outbox
,
2777 struct mlx4_cmd_info
*cmd
)
2779 struct mlx4_qp_context
*context
= inbox
->buf
+ 8;
2780 adjust_proxy_tun_qkey(dev
, vhcr
, context
);
2781 update_pkey_index(dev
, slave
, inbox
);
2782 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2785 int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2786 struct mlx4_vhcr
*vhcr
,
2787 struct mlx4_cmd_mailbox
*inbox
,
2788 struct mlx4_cmd_mailbox
*outbox
,
2789 struct mlx4_cmd_info
*cmd
)
2792 struct mlx4_qp_context
*qpc
= inbox
->buf
+ 8;
2794 err
= verify_qp_parameters(dev
, inbox
, QP_TRANS_INIT2RTR
, slave
);
2798 update_pkey_index(dev
, slave
, inbox
);
2799 update_gid(dev
, inbox
, (u8
)slave
);
2800 adjust_proxy_tun_qkey(dev
, vhcr
, qpc
);
2802 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2805 int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2806 struct mlx4_vhcr
*vhcr
,
2807 struct mlx4_cmd_mailbox
*inbox
,
2808 struct mlx4_cmd_mailbox
*outbox
,
2809 struct mlx4_cmd_info
*cmd
)
2812 struct mlx4_qp_context
*context
= inbox
->buf
+ 8;
2814 err
= verify_qp_parameters(dev
, inbox
, QP_TRANS_RTR2RTS
, slave
);
2818 update_pkey_index(dev
, slave
, inbox
);
2819 update_gid(dev
, inbox
, (u8
)slave
);
2820 adjust_proxy_tun_qkey(dev
, vhcr
, context
);
2821 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2824 int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2825 struct mlx4_vhcr
*vhcr
,
2826 struct mlx4_cmd_mailbox
*inbox
,
2827 struct mlx4_cmd_mailbox
*outbox
,
2828 struct mlx4_cmd_info
*cmd
)
2831 struct mlx4_qp_context
*context
= inbox
->buf
+ 8;
2833 err
= verify_qp_parameters(dev
, inbox
, QP_TRANS_RTS2RTS
, slave
);
2837 update_pkey_index(dev
, slave
, inbox
);
2838 update_gid(dev
, inbox
, (u8
)slave
);
2839 adjust_proxy_tun_qkey(dev
, vhcr
, context
);
2840 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2844 int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2845 struct mlx4_vhcr
*vhcr
,
2846 struct mlx4_cmd_mailbox
*inbox
,
2847 struct mlx4_cmd_mailbox
*outbox
,
2848 struct mlx4_cmd_info
*cmd
)
2850 struct mlx4_qp_context
*context
= inbox
->buf
+ 8;
2851 adjust_proxy_tun_qkey(dev
, vhcr
, context
);
2852 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2855 int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2856 struct mlx4_vhcr
*vhcr
,
2857 struct mlx4_cmd_mailbox
*inbox
,
2858 struct mlx4_cmd_mailbox
*outbox
,
2859 struct mlx4_cmd_info
*cmd
)
2862 struct mlx4_qp_context
*context
= inbox
->buf
+ 8;
2864 err
= verify_qp_parameters(dev
, inbox
, QP_TRANS_SQD2SQD
, slave
);
2868 adjust_proxy_tun_qkey(dev
, vhcr
, context
);
2869 update_gid(dev
, inbox
, (u8
)slave
);
2870 update_pkey_index(dev
, slave
, inbox
);
2871 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2874 int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2875 struct mlx4_vhcr
*vhcr
,
2876 struct mlx4_cmd_mailbox
*inbox
,
2877 struct mlx4_cmd_mailbox
*outbox
,
2878 struct mlx4_cmd_info
*cmd
)
2881 struct mlx4_qp_context
*context
= inbox
->buf
+ 8;
2883 err
= verify_qp_parameters(dev
, inbox
, QP_TRANS_SQD2RTS
, slave
);
2887 adjust_proxy_tun_qkey(dev
, vhcr
, context
);
2888 update_gid(dev
, inbox
, (u8
)slave
);
2889 update_pkey_index(dev
, slave
, inbox
);
2890 return mlx4_GEN_QP_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2893 int mlx4_2RST_QP_wrapper(struct mlx4_dev
*dev
, int slave
,
2894 struct mlx4_vhcr
*vhcr
,
2895 struct mlx4_cmd_mailbox
*inbox
,
2896 struct mlx4_cmd_mailbox
*outbox
,
2897 struct mlx4_cmd_info
*cmd
)
2900 int qpn
= vhcr
->in_modifier
& 0x7fffff;
2903 err
= qp_res_start_move_to(dev
, slave
, qpn
, RES_QP_MAPPED
, &qp
, 0);
2906 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
2910 atomic_dec(&qp
->mtt
->ref_count
);
2911 atomic_dec(&qp
->rcq
->ref_count
);
2912 atomic_dec(&qp
->scq
->ref_count
);
2914 atomic_dec(&qp
->srq
->ref_count
);
2915 res_end_move(dev
, slave
, RES_QP
, qpn
);
2919 res_abort_move(dev
, slave
, RES_QP
, qpn
);
2924 static struct res_gid
*find_gid(struct mlx4_dev
*dev
, int slave
,
2925 struct res_qp
*rqp
, u8
*gid
)
2927 struct res_gid
*res
;
2929 list_for_each_entry(res
, &rqp
->mcg_list
, list
) {
2930 if (!memcmp(res
->gid
, gid
, 16))
2936 static int add_mcg_res(struct mlx4_dev
*dev
, int slave
, struct res_qp
*rqp
,
2937 u8
*gid
, enum mlx4_protocol prot
,
2938 enum mlx4_steer_type steer
, u64 reg_id
)
2940 struct res_gid
*res
;
2943 res
= kzalloc(sizeof *res
, GFP_KERNEL
);
2947 spin_lock_irq(&rqp
->mcg_spl
);
2948 if (find_gid(dev
, slave
, rqp
, gid
)) {
2952 memcpy(res
->gid
, gid
, 16);
2955 res
->reg_id
= reg_id
;
2956 list_add_tail(&res
->list
, &rqp
->mcg_list
);
2959 spin_unlock_irq(&rqp
->mcg_spl
);
2964 static int rem_mcg_res(struct mlx4_dev
*dev
, int slave
, struct res_qp
*rqp
,
2965 u8
*gid
, enum mlx4_protocol prot
,
2966 enum mlx4_steer_type steer
, u64
*reg_id
)
2968 struct res_gid
*res
;
2971 spin_lock_irq(&rqp
->mcg_spl
);
2972 res
= find_gid(dev
, slave
, rqp
, gid
);
2973 if (!res
|| res
->prot
!= prot
|| res
->steer
!= steer
)
2976 *reg_id
= res
->reg_id
;
2977 list_del(&res
->list
);
2981 spin_unlock_irq(&rqp
->mcg_spl
);
2986 static int qp_attach(struct mlx4_dev
*dev
, struct mlx4_qp
*qp
, u8 gid
[16],
2987 int block_loopback
, enum mlx4_protocol prot
,
2988 enum mlx4_steer_type type
, u64
*reg_id
)
2990 switch (dev
->caps
.steering_mode
) {
2991 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
2992 return mlx4_trans_to_dmfs_attach(dev
, qp
, gid
, gid
[5],
2993 block_loopback
, prot
,
2995 case MLX4_STEERING_MODE_B0
:
2996 return mlx4_qp_attach_common(dev
, qp
, gid
,
2997 block_loopback
, prot
, type
);
3003 static int qp_detach(struct mlx4_dev
*dev
, struct mlx4_qp
*qp
, u8 gid
[16],
3004 enum mlx4_protocol prot
, enum mlx4_steer_type type
,
3007 switch (dev
->caps
.steering_mode
) {
3008 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
3009 return mlx4_flow_detach(dev
, reg_id
);
3010 case MLX4_STEERING_MODE_B0
:
3011 return mlx4_qp_detach_common(dev
, qp
, gid
, prot
, type
);
3017 int mlx4_QP_ATTACH_wrapper(struct mlx4_dev
*dev
, int slave
,
3018 struct mlx4_vhcr
*vhcr
,
3019 struct mlx4_cmd_mailbox
*inbox
,
3020 struct mlx4_cmd_mailbox
*outbox
,
3021 struct mlx4_cmd_info
*cmd
)
3023 struct mlx4_qp qp
; /* dummy for calling attach/detach */
3024 u8
*gid
= inbox
->buf
;
3025 enum mlx4_protocol prot
= (vhcr
->in_modifier
>> 28) & 0x7;
3030 int attach
= vhcr
->op_modifier
;
3031 int block_loopback
= vhcr
->in_modifier
>> 31;
3032 u8 steer_type_mask
= 2;
3033 enum mlx4_steer_type type
= (gid
[7] & steer_type_mask
) >> 1;
3035 qpn
= vhcr
->in_modifier
& 0xffffff;
3036 err
= get_res(dev
, slave
, qpn
, RES_QP
, &rqp
);
3042 err
= qp_attach(dev
, &qp
, gid
, block_loopback
, prot
,
3045 pr_err("Fail to attach rule to qp 0x%x\n", qpn
);
3048 err
= add_mcg_res(dev
, slave
, rqp
, gid
, prot
, type
, reg_id
);
3052 err
= rem_mcg_res(dev
, slave
, rqp
, gid
, prot
, type
, ®_id
);
3056 err
= qp_detach(dev
, &qp
, gid
, prot
, type
, reg_id
);
3058 pr_err("Fail to detach rule from qp 0x%x reg_id = 0x%llx\n",
3061 put_res(dev
, slave
, qpn
, RES_QP
);
3065 qp_detach(dev
, &qp
, gid
, prot
, type
, reg_id
);
3067 put_res(dev
, slave
, qpn
, RES_QP
);
3072 * MAC validation for Flow Steering rules.
3073 * VF can attach rules only with a mac address which is assigned to it.
3075 static int validate_eth_header_mac(int slave
, struct _rule_hw
*eth_header
,
3076 struct list_head
*rlist
)
3078 struct mac_res
*res
, *tmp
;
3081 /* make sure it isn't multicast or broadcast mac*/
3082 if (!is_multicast_ether_addr(eth_header
->eth
.dst_mac
) &&
3083 !is_broadcast_ether_addr(eth_header
->eth
.dst_mac
)) {
3084 list_for_each_entry_safe(res
, tmp
, rlist
, list
) {
3085 be_mac
= cpu_to_be64(res
->mac
<< 16);
3086 if (!memcmp(&be_mac
, eth_header
->eth
.dst_mac
, ETH_ALEN
))
3089 pr_err("MAC %pM doesn't belong to VF %d, Steering rule rejected\n",
3090 eth_header
->eth
.dst_mac
, slave
);
3097 * In case of missing eth header, append eth header with a MAC address
3098 * assigned to the VF.
3100 static int add_eth_header(struct mlx4_dev
*dev
, int slave
,
3101 struct mlx4_cmd_mailbox
*inbox
,
3102 struct list_head
*rlist
, int header_id
)
3104 struct mac_res
*res
, *tmp
;
3106 struct mlx4_net_trans_rule_hw_ctrl
*ctrl
;
3107 struct mlx4_net_trans_rule_hw_eth
*eth_header
;
3108 struct mlx4_net_trans_rule_hw_ipv4
*ip_header
;
3109 struct mlx4_net_trans_rule_hw_tcp_udp
*l4_header
;
3111 __be64 mac_msk
= cpu_to_be64(MLX4_MAC_MASK
<< 16);
3113 ctrl
= (struct mlx4_net_trans_rule_hw_ctrl
*)inbox
->buf
;
3115 eth_header
= (struct mlx4_net_trans_rule_hw_eth
*)(ctrl
+ 1);
3117 /* Clear a space in the inbox for eth header */
3118 switch (header_id
) {
3119 case MLX4_NET_TRANS_RULE_ID_IPV4
:
3121 (struct mlx4_net_trans_rule_hw_ipv4
*)(eth_header
+ 1);
3122 memmove(ip_header
, eth_header
,
3123 sizeof(*ip_header
) + sizeof(*l4_header
));
3125 case MLX4_NET_TRANS_RULE_ID_TCP
:
3126 case MLX4_NET_TRANS_RULE_ID_UDP
:
3127 l4_header
= (struct mlx4_net_trans_rule_hw_tcp_udp
*)
3129 memmove(l4_header
, eth_header
, sizeof(*l4_header
));
3134 list_for_each_entry_safe(res
, tmp
, rlist
, list
) {
3135 if (port
== res
->port
) {
3136 be_mac
= cpu_to_be64(res
->mac
<< 16);
3141 pr_err("Failed adding eth header to FS rule, Can't find matching MAC for port %d .\n",
3146 memset(eth_header
, 0, sizeof(*eth_header
));
3147 eth_header
->size
= sizeof(*eth_header
) >> 2;
3148 eth_header
->id
= cpu_to_be16(__sw_id_hw
[MLX4_NET_TRANS_RULE_ID_ETH
]);
3149 memcpy(eth_header
->dst_mac
, &be_mac
, ETH_ALEN
);
3150 memcpy(eth_header
->dst_mac_msk
, &mac_msk
, ETH_ALEN
);
3156 int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev
*dev
, int slave
,
3157 struct mlx4_vhcr
*vhcr
,
3158 struct mlx4_cmd_mailbox
*inbox
,
3159 struct mlx4_cmd_mailbox
*outbox
,
3160 struct mlx4_cmd_info
*cmd
)
3163 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3164 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3165 struct list_head
*rlist
= &tracker
->slave_list
[slave
].res_list
[RES_MAC
];
3169 struct mlx4_net_trans_rule_hw_ctrl
*ctrl
;
3170 struct _rule_hw
*rule_header
;
3173 if (dev
->caps
.steering_mode
!=
3174 MLX4_STEERING_MODE_DEVICE_MANAGED
)
3177 ctrl
= (struct mlx4_net_trans_rule_hw_ctrl
*)inbox
->buf
;
3178 qpn
= be32_to_cpu(ctrl
->qpn
) & 0xffffff;
3179 err
= get_res(dev
, slave
, qpn
, RES_QP
, &rqp
);
3181 pr_err("Steering rule with qpn 0x%x rejected.\n", qpn
);
3184 rule_header
= (struct _rule_hw
*)(ctrl
+ 1);
3185 header_id
= map_hw_to_sw_id(be16_to_cpu(rule_header
->id
));
3187 switch (header_id
) {
3188 case MLX4_NET_TRANS_RULE_ID_ETH
:
3189 if (validate_eth_header_mac(slave
, rule_header
, rlist
)) {
3194 case MLX4_NET_TRANS_RULE_ID_IB
:
3196 case MLX4_NET_TRANS_RULE_ID_IPV4
:
3197 case MLX4_NET_TRANS_RULE_ID_TCP
:
3198 case MLX4_NET_TRANS_RULE_ID_UDP
:
3199 pr_warn("Can't attach FS rule without L2 headers, adding L2 header.\n");
3200 if (add_eth_header(dev
, slave
, inbox
, rlist
, header_id
)) {
3204 vhcr
->in_modifier
+=
3205 sizeof(struct mlx4_net_trans_rule_hw_eth
) >> 2;
3208 pr_err("Corrupted mailbox.\n");
3213 err
= mlx4_cmd_imm(dev
, inbox
->dma
, &vhcr
->out_param
,
3214 vhcr
->in_modifier
, 0,
3215 MLX4_QP_FLOW_STEERING_ATTACH
, MLX4_CMD_TIME_CLASS_A
,
3220 err
= add_res_range(dev
, slave
, vhcr
->out_param
, 1, RES_FS_RULE
, qpn
);
3222 mlx4_err(dev
, "Fail to add flow steering resources.\n ");
3224 mlx4_cmd(dev
, vhcr
->out_param
, 0, 0,
3225 MLX4_QP_FLOW_STEERING_DETACH
, MLX4_CMD_TIME_CLASS_A
,
3229 atomic_inc(&rqp
->ref_count
);
3231 put_res(dev
, slave
, qpn
, RES_QP
);
3235 int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev
*dev
, int slave
,
3236 struct mlx4_vhcr
*vhcr
,
3237 struct mlx4_cmd_mailbox
*inbox
,
3238 struct mlx4_cmd_mailbox
*outbox
,
3239 struct mlx4_cmd_info
*cmd
)
3243 struct res_fs_rule
*rrule
;
3245 if (dev
->caps
.steering_mode
!=
3246 MLX4_STEERING_MODE_DEVICE_MANAGED
)
3249 err
= get_res(dev
, slave
, vhcr
->in_param
, RES_FS_RULE
, &rrule
);
3252 /* Release the rule form busy state before removal */
3253 put_res(dev
, slave
, vhcr
->in_param
, RES_FS_RULE
);
3254 err
= get_res(dev
, slave
, rrule
->qpn
, RES_QP
, &rqp
);
3258 err
= rem_res_range(dev
, slave
, vhcr
->in_param
, 1, RES_FS_RULE
, 0);
3260 mlx4_err(dev
, "Fail to remove flow steering resources.\n ");
3264 err
= mlx4_cmd(dev
, vhcr
->in_param
, 0, 0,
3265 MLX4_QP_FLOW_STEERING_DETACH
, MLX4_CMD_TIME_CLASS_A
,
3268 atomic_dec(&rqp
->ref_count
);
3270 put_res(dev
, slave
, rrule
->qpn
, RES_QP
);
3275 BUSY_MAX_RETRIES
= 10
3278 int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev
*dev
, int slave
,
3279 struct mlx4_vhcr
*vhcr
,
3280 struct mlx4_cmd_mailbox
*inbox
,
3281 struct mlx4_cmd_mailbox
*outbox
,
3282 struct mlx4_cmd_info
*cmd
)
3285 int index
= vhcr
->in_modifier
& 0xffff;
3287 err
= get_res(dev
, slave
, index
, RES_COUNTER
, NULL
);
3291 err
= mlx4_DMA_wrapper(dev
, slave
, vhcr
, inbox
, outbox
, cmd
);
3292 put_res(dev
, slave
, index
, RES_COUNTER
);
3296 static void detach_qp(struct mlx4_dev
*dev
, int slave
, struct res_qp
*rqp
)
3298 struct res_gid
*rgid
;
3299 struct res_gid
*tmp
;
3300 struct mlx4_qp qp
; /* dummy for calling attach/detach */
3302 list_for_each_entry_safe(rgid
, tmp
, &rqp
->mcg_list
, list
) {
3303 switch (dev
->caps
.steering_mode
) {
3304 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
3305 mlx4_flow_detach(dev
, rgid
->reg_id
);
3307 case MLX4_STEERING_MODE_B0
:
3308 qp
.qpn
= rqp
->local_qpn
;
3309 (void) mlx4_qp_detach_common(dev
, &qp
, rgid
->gid
,
3310 rgid
->prot
, rgid
->steer
);
3313 list_del(&rgid
->list
);
3318 static int _move_all_busy(struct mlx4_dev
*dev
, int slave
,
3319 enum mlx4_resource type
, int print
)
3321 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3322 struct mlx4_resource_tracker
*tracker
=
3323 &priv
->mfunc
.master
.res_tracker
;
3324 struct list_head
*rlist
= &tracker
->slave_list
[slave
].res_list
[type
];
3325 struct res_common
*r
;
3326 struct res_common
*tmp
;
3330 spin_lock_irq(mlx4_tlock(dev
));
3331 list_for_each_entry_safe(r
, tmp
, rlist
, list
) {
3332 if (r
->owner
== slave
) {
3334 if (r
->state
== RES_ANY_BUSY
) {
3337 "%s id 0x%llx is busy\n",
3342 r
->from_state
= r
->state
;
3343 r
->state
= RES_ANY_BUSY
;
3349 spin_unlock_irq(mlx4_tlock(dev
));
3354 static int move_all_busy(struct mlx4_dev
*dev
, int slave
,
3355 enum mlx4_resource type
)
3357 unsigned long begin
;
3362 busy
= _move_all_busy(dev
, slave
, type
, 0);
3363 if (time_after(jiffies
, begin
+ 5 * HZ
))
3370 busy
= _move_all_busy(dev
, slave
, type
, 1);
3374 static void rem_slave_qps(struct mlx4_dev
*dev
, int slave
)
3376 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3377 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3378 struct list_head
*qp_list
=
3379 &tracker
->slave_list
[slave
].res_list
[RES_QP
];
3387 err
= move_all_busy(dev
, slave
, RES_QP
);
3389 mlx4_warn(dev
, "rem_slave_qps: Could not move all qps to busy"
3390 "for slave %d\n", slave
);
3392 spin_lock_irq(mlx4_tlock(dev
));
3393 list_for_each_entry_safe(qp
, tmp
, qp_list
, com
.list
) {
3394 spin_unlock_irq(mlx4_tlock(dev
));
3395 if (qp
->com
.owner
== slave
) {
3396 qpn
= qp
->com
.res_id
;
3397 detach_qp(dev
, slave
, qp
);
3398 state
= qp
->com
.from_state
;
3399 while (state
!= 0) {
3401 case RES_QP_RESERVED
:
3402 spin_lock_irq(mlx4_tlock(dev
));
3403 rb_erase(&qp
->com
.node
,
3404 &tracker
->res_tree
[RES_QP
]);
3405 list_del(&qp
->com
.list
);
3406 spin_unlock_irq(mlx4_tlock(dev
));
3411 if (!valid_reserved(dev
, slave
, qpn
))
3412 __mlx4_qp_free_icm(dev
, qpn
);
3413 state
= RES_QP_RESERVED
;
3417 err
= mlx4_cmd(dev
, in_param
,
3420 MLX4_CMD_TIME_CLASS_A
,
3423 mlx4_dbg(dev
, "rem_slave_qps: failed"
3424 " to move slave %d qpn %d to"
3427 atomic_dec(&qp
->rcq
->ref_count
);
3428 atomic_dec(&qp
->scq
->ref_count
);
3429 atomic_dec(&qp
->mtt
->ref_count
);
3431 atomic_dec(&qp
->srq
->ref_count
);
3432 state
= RES_QP_MAPPED
;
3439 spin_lock_irq(mlx4_tlock(dev
));
3441 spin_unlock_irq(mlx4_tlock(dev
));
3444 static void rem_slave_srqs(struct mlx4_dev
*dev
, int slave
)
3446 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3447 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3448 struct list_head
*srq_list
=
3449 &tracker
->slave_list
[slave
].res_list
[RES_SRQ
];
3450 struct res_srq
*srq
;
3451 struct res_srq
*tmp
;
3458 err
= move_all_busy(dev
, slave
, RES_SRQ
);
3460 mlx4_warn(dev
, "rem_slave_srqs: Could not move all srqs to "
3461 "busy for slave %d\n", slave
);
3463 spin_lock_irq(mlx4_tlock(dev
));
3464 list_for_each_entry_safe(srq
, tmp
, srq_list
, com
.list
) {
3465 spin_unlock_irq(mlx4_tlock(dev
));
3466 if (srq
->com
.owner
== slave
) {
3467 srqn
= srq
->com
.res_id
;
3468 state
= srq
->com
.from_state
;
3469 while (state
!= 0) {
3471 case RES_SRQ_ALLOCATED
:
3472 __mlx4_srq_free_icm(dev
, srqn
);
3473 spin_lock_irq(mlx4_tlock(dev
));
3474 rb_erase(&srq
->com
.node
,
3475 &tracker
->res_tree
[RES_SRQ
]);
3476 list_del(&srq
->com
.list
);
3477 spin_unlock_irq(mlx4_tlock(dev
));
3484 err
= mlx4_cmd(dev
, in_param
, srqn
, 1,
3486 MLX4_CMD_TIME_CLASS_A
,
3489 mlx4_dbg(dev
, "rem_slave_srqs: failed"
3490 " to move slave %d srq %d to"
3494 atomic_dec(&srq
->mtt
->ref_count
);
3496 atomic_dec(&srq
->cq
->ref_count
);
3497 state
= RES_SRQ_ALLOCATED
;
3505 spin_lock_irq(mlx4_tlock(dev
));
3507 spin_unlock_irq(mlx4_tlock(dev
));
3510 static void rem_slave_cqs(struct mlx4_dev
*dev
, int slave
)
3512 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3513 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3514 struct list_head
*cq_list
=
3515 &tracker
->slave_list
[slave
].res_list
[RES_CQ
];
3524 err
= move_all_busy(dev
, slave
, RES_CQ
);
3526 mlx4_warn(dev
, "rem_slave_cqs: Could not move all cqs to "
3527 "busy for slave %d\n", slave
);
3529 spin_lock_irq(mlx4_tlock(dev
));
3530 list_for_each_entry_safe(cq
, tmp
, cq_list
, com
.list
) {
3531 spin_unlock_irq(mlx4_tlock(dev
));
3532 if (cq
->com
.owner
== slave
&& !atomic_read(&cq
->ref_count
)) {
3533 cqn
= cq
->com
.res_id
;
3534 state
= cq
->com
.from_state
;
3535 while (state
!= 0) {
3537 case RES_CQ_ALLOCATED
:
3538 __mlx4_cq_free_icm(dev
, cqn
);
3539 spin_lock_irq(mlx4_tlock(dev
));
3540 rb_erase(&cq
->com
.node
,
3541 &tracker
->res_tree
[RES_CQ
]);
3542 list_del(&cq
->com
.list
);
3543 spin_unlock_irq(mlx4_tlock(dev
));
3550 err
= mlx4_cmd(dev
, in_param
, cqn
, 1,
3552 MLX4_CMD_TIME_CLASS_A
,
3555 mlx4_dbg(dev
, "rem_slave_cqs: failed"
3556 " to move slave %d cq %d to"
3559 atomic_dec(&cq
->mtt
->ref_count
);
3560 state
= RES_CQ_ALLOCATED
;
3568 spin_lock_irq(mlx4_tlock(dev
));
3570 spin_unlock_irq(mlx4_tlock(dev
));
3573 static void rem_slave_mrs(struct mlx4_dev
*dev
, int slave
)
3575 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3576 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3577 struct list_head
*mpt_list
=
3578 &tracker
->slave_list
[slave
].res_list
[RES_MPT
];
3579 struct res_mpt
*mpt
;
3580 struct res_mpt
*tmp
;
3587 err
= move_all_busy(dev
, slave
, RES_MPT
);
3589 mlx4_warn(dev
, "rem_slave_mrs: Could not move all mpts to "
3590 "busy for slave %d\n", slave
);
3592 spin_lock_irq(mlx4_tlock(dev
));
3593 list_for_each_entry_safe(mpt
, tmp
, mpt_list
, com
.list
) {
3594 spin_unlock_irq(mlx4_tlock(dev
));
3595 if (mpt
->com
.owner
== slave
) {
3596 mptn
= mpt
->com
.res_id
;
3597 state
= mpt
->com
.from_state
;
3598 while (state
!= 0) {
3600 case RES_MPT_RESERVED
:
3601 __mlx4_mpt_release(dev
, mpt
->key
);
3602 spin_lock_irq(mlx4_tlock(dev
));
3603 rb_erase(&mpt
->com
.node
,
3604 &tracker
->res_tree
[RES_MPT
]);
3605 list_del(&mpt
->com
.list
);
3606 spin_unlock_irq(mlx4_tlock(dev
));
3611 case RES_MPT_MAPPED
:
3612 __mlx4_mpt_free_icm(dev
, mpt
->key
);
3613 state
= RES_MPT_RESERVED
;
3618 err
= mlx4_cmd(dev
, in_param
, mptn
, 0,
3620 MLX4_CMD_TIME_CLASS_A
,
3623 mlx4_dbg(dev
, "rem_slave_mrs: failed"
3624 " to move slave %d mpt %d to"
3628 atomic_dec(&mpt
->mtt
->ref_count
);
3629 state
= RES_MPT_MAPPED
;
3636 spin_lock_irq(mlx4_tlock(dev
));
3638 spin_unlock_irq(mlx4_tlock(dev
));
3641 static void rem_slave_mtts(struct mlx4_dev
*dev
, int slave
)
3643 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3644 struct mlx4_resource_tracker
*tracker
=
3645 &priv
->mfunc
.master
.res_tracker
;
3646 struct list_head
*mtt_list
=
3647 &tracker
->slave_list
[slave
].res_list
[RES_MTT
];
3648 struct res_mtt
*mtt
;
3649 struct res_mtt
*tmp
;
3655 err
= move_all_busy(dev
, slave
, RES_MTT
);
3657 mlx4_warn(dev
, "rem_slave_mtts: Could not move all mtts to "
3658 "busy for slave %d\n", slave
);
3660 spin_lock_irq(mlx4_tlock(dev
));
3661 list_for_each_entry_safe(mtt
, tmp
, mtt_list
, com
.list
) {
3662 spin_unlock_irq(mlx4_tlock(dev
));
3663 if (mtt
->com
.owner
== slave
) {
3664 base
= mtt
->com
.res_id
;
3665 state
= mtt
->com
.from_state
;
3666 while (state
!= 0) {
3668 case RES_MTT_ALLOCATED
:
3669 __mlx4_free_mtt_range(dev
, base
,
3671 spin_lock_irq(mlx4_tlock(dev
));
3672 rb_erase(&mtt
->com
.node
,
3673 &tracker
->res_tree
[RES_MTT
]);
3674 list_del(&mtt
->com
.list
);
3675 spin_unlock_irq(mlx4_tlock(dev
));
3685 spin_lock_irq(mlx4_tlock(dev
));
3687 spin_unlock_irq(mlx4_tlock(dev
));
3690 static void rem_slave_fs_rule(struct mlx4_dev
*dev
, int slave
)
3692 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3693 struct mlx4_resource_tracker
*tracker
=
3694 &priv
->mfunc
.master
.res_tracker
;
3695 struct list_head
*fs_rule_list
=
3696 &tracker
->slave_list
[slave
].res_list
[RES_FS_RULE
];
3697 struct res_fs_rule
*fs_rule
;
3698 struct res_fs_rule
*tmp
;
3703 err
= move_all_busy(dev
, slave
, RES_FS_RULE
);
3705 mlx4_warn(dev
, "rem_slave_fs_rule: Could not move all mtts to busy for slave %d\n",
3708 spin_lock_irq(mlx4_tlock(dev
));
3709 list_for_each_entry_safe(fs_rule
, tmp
, fs_rule_list
, com
.list
) {
3710 spin_unlock_irq(mlx4_tlock(dev
));
3711 if (fs_rule
->com
.owner
== slave
) {
3712 base
= fs_rule
->com
.res_id
;
3713 state
= fs_rule
->com
.from_state
;
3714 while (state
!= 0) {
3716 case RES_FS_RULE_ALLOCATED
:
3718 err
= mlx4_cmd(dev
, base
, 0, 0,
3719 MLX4_QP_FLOW_STEERING_DETACH
,
3720 MLX4_CMD_TIME_CLASS_A
,
3723 spin_lock_irq(mlx4_tlock(dev
));
3724 rb_erase(&fs_rule
->com
.node
,
3725 &tracker
->res_tree
[RES_FS_RULE
]);
3726 list_del(&fs_rule
->com
.list
);
3727 spin_unlock_irq(mlx4_tlock(dev
));
3737 spin_lock_irq(mlx4_tlock(dev
));
3739 spin_unlock_irq(mlx4_tlock(dev
));
3742 static void rem_slave_eqs(struct mlx4_dev
*dev
, int slave
)
3744 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3745 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3746 struct list_head
*eq_list
=
3747 &tracker
->slave_list
[slave
].res_list
[RES_EQ
];
3754 struct mlx4_cmd_mailbox
*mailbox
;
3756 err
= move_all_busy(dev
, slave
, RES_EQ
);
3758 mlx4_warn(dev
, "rem_slave_eqs: Could not move all eqs to "
3759 "busy for slave %d\n", slave
);
3761 spin_lock_irq(mlx4_tlock(dev
));
3762 list_for_each_entry_safe(eq
, tmp
, eq_list
, com
.list
) {
3763 spin_unlock_irq(mlx4_tlock(dev
));
3764 if (eq
->com
.owner
== slave
) {
3765 eqn
= eq
->com
.res_id
;
3766 state
= eq
->com
.from_state
;
3767 while (state
!= 0) {
3769 case RES_EQ_RESERVED
:
3770 spin_lock_irq(mlx4_tlock(dev
));
3771 rb_erase(&eq
->com
.node
,
3772 &tracker
->res_tree
[RES_EQ
]);
3773 list_del(&eq
->com
.list
);
3774 spin_unlock_irq(mlx4_tlock(dev
));
3780 mailbox
= mlx4_alloc_cmd_mailbox(dev
);
3781 if (IS_ERR(mailbox
)) {
3785 err
= mlx4_cmd_box(dev
, slave
, 0,
3788 MLX4_CMD_TIME_CLASS_A
,
3791 mlx4_dbg(dev
, "rem_slave_eqs: failed"
3792 " to move slave %d eqs %d to"
3793 " SW ownership\n", slave
, eqn
);
3794 mlx4_free_cmd_mailbox(dev
, mailbox
);
3795 atomic_dec(&eq
->mtt
->ref_count
);
3796 state
= RES_EQ_RESERVED
;
3804 spin_lock_irq(mlx4_tlock(dev
));
3806 spin_unlock_irq(mlx4_tlock(dev
));
3809 static void rem_slave_counters(struct mlx4_dev
*dev
, int slave
)
3811 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3812 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3813 struct list_head
*counter_list
=
3814 &tracker
->slave_list
[slave
].res_list
[RES_COUNTER
];
3815 struct res_counter
*counter
;
3816 struct res_counter
*tmp
;
3820 err
= move_all_busy(dev
, slave
, RES_COUNTER
);
3822 mlx4_warn(dev
, "rem_slave_counters: Could not move all counters to "
3823 "busy for slave %d\n", slave
);
3825 spin_lock_irq(mlx4_tlock(dev
));
3826 list_for_each_entry_safe(counter
, tmp
, counter_list
, com
.list
) {
3827 if (counter
->com
.owner
== slave
) {
3828 index
= counter
->com
.res_id
;
3829 rb_erase(&counter
->com
.node
,
3830 &tracker
->res_tree
[RES_COUNTER
]);
3831 list_del(&counter
->com
.list
);
3833 __mlx4_counter_free(dev
, index
);
3836 spin_unlock_irq(mlx4_tlock(dev
));
3839 static void rem_slave_xrcdns(struct mlx4_dev
*dev
, int slave
)
3841 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3842 struct mlx4_resource_tracker
*tracker
= &priv
->mfunc
.master
.res_tracker
;
3843 struct list_head
*xrcdn_list
=
3844 &tracker
->slave_list
[slave
].res_list
[RES_XRCD
];
3845 struct res_xrcdn
*xrcd
;
3846 struct res_xrcdn
*tmp
;
3850 err
= move_all_busy(dev
, slave
, RES_XRCD
);
3852 mlx4_warn(dev
, "rem_slave_xrcdns: Could not move all xrcdns to "
3853 "busy for slave %d\n", slave
);
3855 spin_lock_irq(mlx4_tlock(dev
));
3856 list_for_each_entry_safe(xrcd
, tmp
, xrcdn_list
, com
.list
) {
3857 if (xrcd
->com
.owner
== slave
) {
3858 xrcdn
= xrcd
->com
.res_id
;
3859 rb_erase(&xrcd
->com
.node
, &tracker
->res_tree
[RES_XRCD
]);
3860 list_del(&xrcd
->com
.list
);
3862 __mlx4_xrcd_free(dev
, xrcdn
);
3865 spin_unlock_irq(mlx4_tlock(dev
));
3868 void mlx4_delete_all_resources_for_slave(struct mlx4_dev
*dev
, int slave
)
3870 struct mlx4_priv
*priv
= mlx4_priv(dev
);
3872 mutex_lock(&priv
->mfunc
.master
.res_tracker
.slave_list
[slave
].mutex
);
3874 rem_slave_macs(dev
, slave
);
3875 rem_slave_fs_rule(dev
, slave
);
3876 rem_slave_qps(dev
, slave
);
3877 rem_slave_srqs(dev
, slave
);
3878 rem_slave_cqs(dev
, slave
);
3879 rem_slave_mrs(dev
, slave
);
3880 rem_slave_eqs(dev
, slave
);
3881 rem_slave_mtts(dev
, slave
);
3882 rem_slave_counters(dev
, slave
);
3883 rem_slave_xrcdns(dev
, slave
);
3884 mutex_unlock(&priv
->mfunc
.master
.res_tracker
.slave_list
[slave
].mutex
);