2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/export.h>
34 #include <linux/mlx5/driver.h>
35 #include <linux/mlx5/flow_table.h>
36 #include "mlx5_core.h"
39 struct mlx5_flow_table_group g
;
44 struct mlx5_flow_table
{
45 struct mlx5_core_dev
*dev
;
49 struct mutex mutex
; /* sync bitmap alloc */
51 struct mlx5_ftg
*group
;
52 unsigned long *bitmap
;
56 static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table
*ft
, u32 group_ix
,
57 u32 flow_index
, void *flow_context
)
59 u32 out
[MLX5_ST_SZ_DW(set_fte_out
)];
61 void *in_flow_context
;
63 MLX5_GET(flow_context
, flow_context
, destination_list_size
) *
64 MLX5_ST_SZ_BYTES(dest_format_struct
);
65 int inlen
= MLX5_ST_SZ_BYTES(set_fte_in
) + fcdls
;
68 in
= mlx5_vzalloc(inlen
);
70 mlx5_core_warn(ft
->dev
, "failed to allocate inbox\n");
74 MLX5_SET(set_fte_in
, in
, table_type
, ft
->type
);
75 MLX5_SET(set_fte_in
, in
, table_id
, ft
->id
);
76 MLX5_SET(set_fte_in
, in
, flow_index
, flow_index
);
77 MLX5_SET(set_fte_in
, in
, opcode
, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY
);
79 in_flow_context
= MLX5_ADDR_OF(set_fte_in
, in
, flow_context
);
80 memcpy(in_flow_context
, flow_context
,
81 MLX5_ST_SZ_BYTES(flow_context
) + fcdls
);
83 MLX5_SET(flow_context
, in_flow_context
, group_id
,
84 ft
->group
[group_ix
].id
);
86 memset(out
, 0, sizeof(out
));
87 err
= mlx5_cmd_exec_check_status(ft
->dev
, in
, inlen
, out
,
94 static void mlx5_del_flow_entry_cmd(struct mlx5_flow_table
*ft
, u32 flow_index
)
96 u32 in
[MLX5_ST_SZ_DW(delete_fte_in
)];
97 u32 out
[MLX5_ST_SZ_DW(delete_fte_out
)];
99 memset(in
, 0, sizeof(in
));
100 memset(out
, 0, sizeof(out
));
102 #define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v)
103 MLX5_SET_DFTEI(in
, table_type
, ft
->type
);
104 MLX5_SET_DFTEI(in
, table_id
, ft
->id
);
105 MLX5_SET_DFTEI(in
, flow_index
, flow_index
);
106 MLX5_SET_DFTEI(in
, opcode
, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY
);
108 mlx5_cmd_exec_check_status(ft
->dev
, in
, sizeof(in
), out
, sizeof(out
));
111 static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table
*ft
, int i
)
113 u32 in
[MLX5_ST_SZ_DW(destroy_flow_group_in
)];
114 u32 out
[MLX5_ST_SZ_DW(destroy_flow_group_out
)];
116 memset(in
, 0, sizeof(in
));
117 memset(out
, 0, sizeof(out
));
119 #define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v)
120 MLX5_SET_DFGI(in
, table_type
, ft
->type
);
121 MLX5_SET_DFGI(in
, table_id
, ft
->id
);
122 MLX5_SET_DFGI(in
, opcode
, MLX5_CMD_OP_DESTROY_FLOW_GROUP
);
123 MLX5_SET_DFGI(in
, group_id
, ft
->group
[i
].id
);
124 mlx5_cmd_exec_check_status(ft
->dev
, in
, sizeof(in
), out
, sizeof(out
));
127 static int mlx5_create_flow_group_cmd(struct mlx5_flow_table
*ft
, int i
)
129 u32 out
[MLX5_ST_SZ_DW(create_flow_group_out
)];
131 void *in_match_criteria
;
132 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
133 struct mlx5_flow_table_group
*g
= &ft
->group
[i
].g
;
134 u32 start_ix
= ft
->group
[i
].start_ix
;
135 u32 end_ix
= start_ix
+ (1 << g
->log_sz
) - 1;
138 in
= mlx5_vzalloc(inlen
);
140 mlx5_core_warn(ft
->dev
, "failed to allocate inbox\n");
143 in_match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, in
,
146 memset(out
, 0, sizeof(out
));
148 #define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v)
149 MLX5_SET_CFGI(in
, table_type
, ft
->type
);
150 MLX5_SET_CFGI(in
, table_id
, ft
->id
);
151 MLX5_SET_CFGI(in
, opcode
, MLX5_CMD_OP_CREATE_FLOW_GROUP
);
152 MLX5_SET_CFGI(in
, start_flow_index
, start_ix
);
153 MLX5_SET_CFGI(in
, end_flow_index
, end_ix
);
154 MLX5_SET_CFGI(in
, match_criteria_enable
, g
->match_criteria_enable
);
156 memcpy(in_match_criteria
, g
->match_criteria
,
157 MLX5_ST_SZ_BYTES(fte_match_param
));
159 err
= mlx5_cmd_exec_check_status(ft
->dev
, in
, inlen
, out
,
162 ft
->group
[i
].id
= MLX5_GET(create_flow_group_out
, out
,
170 static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table
*ft
)
174 for (i
= 0; i
< ft
->num_groups
; i
++)
175 mlx5_destroy_flow_group_cmd(ft
, i
);
178 static int mlx5_create_flow_table_groups(struct mlx5_flow_table
*ft
)
183 for (i
= 0; i
< ft
->num_groups
; i
++) {
184 err
= mlx5_create_flow_group_cmd(ft
, i
);
186 goto err_destroy_flow_table_groups
;
191 err_destroy_flow_table_groups
:
192 for (i
--; i
>= 0; i
--)
193 mlx5_destroy_flow_group_cmd(ft
, i
);
198 static int mlx5_create_flow_table_cmd(struct mlx5_flow_table
*ft
)
200 u32 in
[MLX5_ST_SZ_DW(create_flow_table_in
)];
201 u32 out
[MLX5_ST_SZ_DW(create_flow_table_out
)];
204 memset(in
, 0, sizeof(in
));
206 MLX5_SET(create_flow_table_in
, in
, table_type
, ft
->type
);
207 MLX5_SET(create_flow_table_in
, in
, level
, ft
->level
);
208 MLX5_SET(create_flow_table_in
, in
, log_size
, order_base_2(ft
->size
));
210 MLX5_SET(create_flow_table_in
, in
, opcode
,
211 MLX5_CMD_OP_CREATE_FLOW_TABLE
);
213 memset(out
, 0, sizeof(out
));
214 err
= mlx5_cmd_exec_check_status(ft
->dev
, in
, sizeof(in
), out
,
219 ft
->id
= MLX5_GET(create_flow_table_out
, out
, table_id
);
224 static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table
*ft
)
226 u32 in
[MLX5_ST_SZ_DW(destroy_flow_table_in
)];
227 u32 out
[MLX5_ST_SZ_DW(destroy_flow_table_out
)];
229 memset(in
, 0, sizeof(in
));
230 memset(out
, 0, sizeof(out
));
232 #define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v)
233 MLX5_SET_DFTI(in
, table_type
, ft
->type
);
234 MLX5_SET_DFTI(in
, table_id
, ft
->id
);
235 MLX5_SET_DFTI(in
, opcode
, MLX5_CMD_OP_DESTROY_FLOW_TABLE
);
237 mlx5_cmd_exec_check_status(ft
->dev
, in
, sizeof(in
), out
, sizeof(out
));
240 static int mlx5_find_group(struct mlx5_flow_table
*ft
, u8 match_criteria_enable
,
241 u32
*match_criteria
, int *group_ix
)
243 void *mc_outer
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
245 void *mc_misc
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
247 void *mc_inner
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
249 int mc_outer_sz
= MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4
);
250 int mc_misc_sz
= MLX5_ST_SZ_BYTES(fte_match_set_misc
);
251 int mc_inner_sz
= MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4
);
254 for (i
= 0; i
< ft
->num_groups
; i
++) {
255 struct mlx5_flow_table_group
*g
= &ft
->group
[i
].g
;
256 void *gmc_outer
= MLX5_ADDR_OF(fte_match_param
,
259 void *gmc_misc
= MLX5_ADDR_OF(fte_match_param
,
262 void *gmc_inner
= MLX5_ADDR_OF(fte_match_param
,
266 if (g
->match_criteria_enable
!= match_criteria_enable
)
269 if (match_criteria_enable
& MLX5_MATCH_OUTER_HEADERS
)
270 if (memcmp(mc_outer
, gmc_outer
, mc_outer_sz
))
273 if (match_criteria_enable
& MLX5_MATCH_MISC_PARAMETERS
)
274 if (memcmp(mc_misc
, gmc_misc
, mc_misc_sz
))
277 if (match_criteria_enable
& MLX5_MATCH_INNER_HEADERS
)
278 if (memcmp(mc_inner
, gmc_inner
, mc_inner_sz
))
288 static int alloc_flow_index(struct mlx5_flow_table
*ft
, int group_ix
, u32
*ix
)
290 struct mlx5_ftg
*g
= &ft
->group
[group_ix
];
293 mutex_lock(&ft
->mutex
);
295 *ix
= find_next_zero_bit(ft
->bitmap
, ft
->size
, g
->start_ix
);
296 if (*ix
>= (g
->start_ix
+ (1 << g
->g
.log_sz
)))
299 __set_bit(*ix
, ft
->bitmap
);
301 mutex_unlock(&ft
->mutex
);
306 static void mlx5_free_flow_index(struct mlx5_flow_table
*ft
, u32 ix
)
308 __clear_bit(ix
, ft
->bitmap
);
311 int mlx5_add_flow_table_entry(void *flow_table
, u8 match_criteria_enable
,
312 void *match_criteria
, void *flow_context
,
315 struct mlx5_flow_table
*ft
= flow_table
;
319 err
= mlx5_find_group(ft
, match_criteria_enable
, match_criteria
,
322 mlx5_core_warn(ft
->dev
, "mlx5_find_group failed\n");
326 err
= alloc_flow_index(ft
, group_ix
, flow_index
);
328 mlx5_core_warn(ft
->dev
, "alloc_flow_index failed\n");
332 return mlx5_set_flow_entry_cmd(ft
, group_ix
, *flow_index
, flow_context
);
334 EXPORT_SYMBOL(mlx5_add_flow_table_entry
);
336 void mlx5_del_flow_table_entry(void *flow_table
, u32 flow_index
)
338 struct mlx5_flow_table
*ft
= flow_table
;
340 mlx5_del_flow_entry_cmd(ft
, flow_index
);
341 mlx5_free_flow_index(ft
, flow_index
);
343 EXPORT_SYMBOL(mlx5_del_flow_table_entry
);
345 void *mlx5_create_flow_table(struct mlx5_core_dev
*dev
, u8 level
, u8 table_type
,
347 struct mlx5_flow_table_group
*group
)
349 struct mlx5_flow_table
*ft
;
357 for (i
= 0; i
< num_groups
; i
++)
358 ft_size
+= (1 << group
[i
].log_sz
);
360 ft
= kzalloc(sizeof(*ft
), GFP_KERNEL
);
361 gr
= kcalloc(num_groups
, sizeof(struct mlx5_ftg
), GFP_KERNEL
);
362 bm
= kcalloc(BITS_TO_LONGS(ft_size
), sizeof(uintptr_t), GFP_KERNEL
);
363 if (!ft
|| !gr
|| !bm
)
368 ft
->num_groups
= num_groups
;
370 ft
->type
= table_type
;
373 mutex_init(&ft
->mutex
);
375 for (i
= 0; i
< ft
->num_groups
; i
++) {
376 memcpy(&ft
->group
[i
].g
, &group
[i
], sizeof(*group
));
377 ft
->group
[i
].start_ix
= start_ix
;
378 start_ix
+= 1 << group
[i
].log_sz
;
381 err
= mlx5_create_flow_table_cmd(ft
);
385 err
= mlx5_create_flow_table_groups(ft
);
387 goto err_destroy_flow_table_cmd
;
391 err_destroy_flow_table_cmd
:
392 mlx5_destroy_flow_table_cmd(ft
);
395 mlx5_core_warn(dev
, "failed to alloc flow table\n");
402 EXPORT_SYMBOL(mlx5_create_flow_table
);
404 void mlx5_destroy_flow_table(void *flow_table
)
406 struct mlx5_flow_table
*ft
= flow_table
;
408 mlx5_destroy_flow_table_groups(ft
);
409 mlx5_destroy_flow_table_cmd(ft
);
414 EXPORT_SYMBOL(mlx5_destroy_flow_table
);
416 u32
mlx5_get_flow_table_id(void *flow_table
)
418 struct mlx5_flow_table
*ft
= flow_table
;
422 EXPORT_SYMBOL(mlx5_get_flow_table_id
);