diff options
Diffstat (limited to 'sys/dev/mlx5/mlx5_core')
29 files changed, 5939 insertions, 3825 deletions
diff --git a/sys/dev/mlx5/mlx5_core/eswitch.h b/sys/dev/mlx5/mlx5_core/eswitch.h index ca03da287543..50d06951bf07 100644 --- a/sys/dev/mlx5/mlx5_core/eswitch.h +++ b/sys/dev/mlx5/mlx5_core/eswitch.h @@ -29,6 +29,8 @@ #include <linux/if_ether.h> #include <dev/mlx5/device.h> +#define MLX5_ESWITCH_MANAGER(mdev) MLX5_CAP_GEN(mdev, eswitch_flow_table) + #define MLX5_MAX_UC_PER_VPORT(dev) \ (1 << MLX5_CAP_GEN(dev, log_max_current_uc_list)) @@ -83,15 +85,15 @@ struct l2addr_node { struct vport_ingress { struct mlx5_flow_table *acl; struct mlx5_flow_group *drop_grp; - struct mlx5_flow_rule *drop_rule; + struct mlx5_flow_handle *drop_rule; }; struct vport_egress { struct mlx5_flow_table *acl; struct mlx5_flow_group *allowed_vlans_grp; struct mlx5_flow_group *drop_grp; - struct mlx5_flow_rule *allowed_vlan; - struct mlx5_flow_rule *drop_rule; + struct mlx5_flow_handle *allowed_vlan; + struct mlx5_flow_handle *drop_rule; }; struct mlx5_vport { diff --git a/sys/dev/mlx5/mlx5_core/fs_chains.h b/sys/dev/mlx5/mlx5_core/fs_chains.h new file mode 100644 index 000000000000..e703a98981b6 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/fs_chains.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies. */ + +#ifndef __ML5_ESW_CHAINS_H__ +#define __ML5_ESW_CHAINS_H__ + +#include <dev/mlx5/fs.h> + +struct mlx5_fs_chains; + +enum mlx5_chains_flags { + MLX5_CHAINS_AND_PRIOS_SUPPORTED = BIT(0), + MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED = BIT(1), + MLX5_CHAINS_FT_TUNNEL_SUPPORTED = BIT(2), +}; + +struct mlx5_chains_attr { + enum mlx5_flow_namespace_type ns; + int fs_base_prio; + int fs_base_level; + u32 flags; + u32 max_grp_num; + struct mlx5_flow_table *default_ft; +}; + +bool +mlx5_chains_prios_supported(struct mlx5_fs_chains *chains); +bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains); +bool +mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains); +u32 +mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains); +u32 +mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains); +u32 +mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains); + +struct mlx5_flow_table * +mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, + u32 level); +void +mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, + u32 level); + +struct mlx5_flow_table * +mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains); + +struct mlx5_flow_table * +mlx5_chains_create_global_table(struct mlx5_fs_chains *chains); +void +mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains, + struct mlx5_flow_table *ft); + +int +mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain, + u32 *chain_mapping); +int +mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, + u32 chain_mapping); + +struct mlx5_fs_chains * +mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr); +void mlx5_chains_destroy(struct mlx5_fs_chains *chains); + +void +mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains, + struct mlx5_flow_table *ft); +void +mlx5_chains_print_info(struct mlx5_fs_chains *chains); + +#endif /* __ML5_ESW_CHAINS_H__ */ diff --git a/sys/dev/mlx5/mlx5_core/fs_cmd.h b/sys/dev/mlx5/mlx5_core/fs_cmd.h new file mode 100644 index 000000000000..a2b2d537ac45 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/fs_cmd.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _MLX5_FS_CMD_ +#define _MLX5_FS_CMD_ + +#include "fs_core.h" + +struct mlx5_flow_cmds { + int (*create_flow_table)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_table_attr *ft_attr, + struct mlx5_flow_table *next_ft); + int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft); + + int (*modify_flow_table)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_table *next_ft); + + int (*create_flow_group)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + u32 *in, + struct mlx5_flow_group *fg); + + int (*destroy_flow_group)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg); + + int (*create_fte)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg, + struct fs_fte *fte); + + int (*update_fte)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg, + int modify_mask, + struct fs_fte *fte); + + int (*delete_fte)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct fs_fte *fte); + + int (*update_root_ft)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + u32 underlay_qpn, + bool disconnect); + + int (*packet_reformat_alloc)(struct mlx5_flow_root_namespace *ns, + struct mlx5_pkt_reformat_params *params, + enum mlx5_flow_namespace_type namespace, + struct mlx5_pkt_reformat *pkt_reformat); + + void (*packet_reformat_dealloc)(struct mlx5_flow_root_namespace *ns, + struct mlx5_pkt_reformat *pkt_reformat); + + int (*modify_header_alloc)(struct mlx5_flow_root_namespace *ns, + u8 namespace, u8 num_actions, + void *modify_actions, + struct mlx5_modify_hdr *modify_hdr); + + void (*modify_header_dealloc)(struct mlx5_flow_root_namespace *ns, + struct mlx5_modify_hdr *modify_hdr); + + int (*set_peer)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_root_namespace *peer_ns); + + int (*create_ns)(struct mlx5_flow_root_namespace *ns); + int (*destroy_ns)(struct mlx5_flow_root_namespace *ns); + + u32 (*get_capabilities)(struct mlx5_flow_root_namespace *ns, + enum fs_flow_table_type ft_type); +}; + +int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id); +int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, + enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask, + u32 *id); +int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id); +int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id, + u64 *packets, u64 *bytes); + +int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len); +int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len, + u32 *out); + +const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type); +const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void); + +#endif diff --git a/sys/dev/mlx5/mlx5_core/fs_core.h b/sys/dev/mlx5/mlx5_core/fs_core.h index 05757f493469..1d7339e6b7d5 100644 --- a/sys/dev/mlx5/mlx5_core/fs_core.h +++ b/sys/dev/mlx5/mlx5_core/fs_core.h @@ -1,14 +1,11 @@ -/*- - * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. +/* + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE @@ -26,306 +23,327 @@ #ifndef _MLX5_FS_CORE_ #define _MLX5_FS_CORE_ -#include <asm/atomic.h> -#include <linux/completion.h> -#include <linux/mutex.h> #include <dev/mlx5/fs.h> -enum fs_type { +#define FDB_TC_MAX_CHAIN 3 +#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) +#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) + +/* The index of the last real chain (FT) + 1 as chain zero is valid as well */ +#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) + +#define FDB_TC_MAX_PRIO 16 +#define FDB_TC_LEVELS_PER_PRIO 2 + +struct mlx5_flow_definer { + enum mlx5_flow_namespace_type ns_type; + u32 id; +}; + +struct mlx5_modify_hdr { + enum mlx5_flow_namespace_type ns_type; + union { + u32 id; + }; +}; + +struct mlx5_pkt_reformat { + enum mlx5_flow_namespace_type ns_type; + int reformat_type; /* from mlx5_ifc */ + union { + u32 id; + }; +}; + +/* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only, + * and those are in parallel to one another when going over them to connect + * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one + * parallel namespace will not automatically connect to the first flow table + * found in any prio in any next namespace, but skip the entire containing + * TYPE_PRIO_CHAINS prio. + * + * This is used to implement tc chains, each chain of prios is a different + * namespace inside a containing TYPE_PRIO_CHAINS prio. + */ + +enum fs_node_type { FS_TYPE_NAMESPACE, FS_TYPE_PRIO, + FS_TYPE_PRIO_CHAINS, FS_TYPE_FLOW_TABLE, FS_TYPE_FLOW_GROUP, FS_TYPE_FLOW_ENTRY, FS_TYPE_FLOW_DEST }; -enum fs_ft_type { +/**********************************************************************************************************/ + + +#define fs_ft_type fs_flow_table_type +/************************************************************************************************************/ +enum fs_flow_table_type { FS_FT_NIC_RX = 0x0, + FS_FT_NIC_TX = 0x1, FS_FT_ESW_EGRESS_ACL = 0x2, FS_FT_ESW_INGRESS_ACL = 0x3, FS_FT_FDB = 0X4, - FS_FT_SNIFFER_RX = 0x5, - FS_FT_SNIFFER_TX = 0x6 + FS_FT_SNIFFER_RX = 0X5, + FS_FT_SNIFFER_TX = 0X6, + FS_FT_RDMA_RX = 0X7, + FS_FT_RDMA_TX = 0X8, + FS_FT_PORT_SEL = 0X9, + FS_FT_MAX_TYPE = FS_FT_PORT_SEL, +}; + +enum fs_flow_table_op_mod { + FS_FT_OP_MOD_NORMAL, + FS_FT_OP_MOD_LAG_DEMUX, }; enum fs_fte_status { FS_FTE_STATUS_EXISTING = 1UL << 0, }; -/* Should always be the first variable in the struct */ -struct fs_base { - struct list_head list; - struct fs_base *parent; - enum fs_type type; - struct kref refcount; +enum mlx5_flow_steering_mode { + MLX5_FLOW_STEERING_MODE_DMFS, + MLX5_FLOW_STEERING_MODE_SMFS +}; + +enum mlx5_flow_steering_capabilty { + MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0, + MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1, + MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2, +}; + +struct mlx5_flow_steering { + struct mlx5_core_dev *dev; + enum mlx5_flow_steering_mode mode; + struct kmem_cache *fgs_cache; + struct kmem_cache *ftes_cache; + struct mlx5_flow_root_namespace *root_ns; + struct mlx5_flow_root_namespace *fdb_root_ns; + struct mlx5_flow_namespace **fdb_sub_ns; + struct mlx5_flow_root_namespace **esw_egress_root_ns; + struct mlx5_flow_root_namespace **esw_ingress_root_ns; + struct mlx5_flow_root_namespace *sniffer_tx_root_ns; + struct mlx5_flow_root_namespace *sniffer_rx_root_ns; + struct mlx5_flow_root_namespace *rdma_rx_root_ns; + struct mlx5_flow_root_namespace *rdma_tx_root_ns; + struct mlx5_flow_root_namespace *egress_root_ns; + struct mlx5_flow_root_namespace *port_sel_root_ns; + int esw_egress_acl_vports; + int esw_ingress_acl_vports; +}; + +struct fs_node { + struct list_head list; + struct list_head children; + enum fs_node_type type; + struct fs_node *parent; + struct fs_node *root; /* lock the node for writing and traversing */ - struct mutex lock; - struct completion complete; - atomic_t users_refcount; - const char *name; + struct rw_semaphore lock; + refcount_t refcount; + bool active; + void (*del_hw_func)(struct fs_node *); + void (*del_sw_func)(struct fs_node *); + atomic_t version; }; struct mlx5_flow_rule { - struct fs_base base; + struct fs_node node; + struct mlx5_flow_table *ft; struct mlx5_flow_destination dest_attr; - struct list_head clients_data; - /*protect clients lits*/ - struct mutex clients_lock; -}; - -struct fs_fte { - struct fs_base base; - u32 val[MLX5_ST_SZ_DW(fte_match_param)]; - uint32_t dests_size; - struct list_head dests; - uint32_t index; /* index in ft */ - struct mlx5_flow_act flow_act; - u32 sw_action; /* enum mlx5_rule_fwd_action */ - enum fs_fte_status status; + /* next_ft should be accessed under chain_lock and only of + * destination type is FWD_NEXT_fT. + */ + struct list_head next_ft; + u32 sw_action; }; -struct fs_star_rule { - struct mlx5_flow_group *fg; - struct fs_fte *fte; +struct mlx5_flow_handle { + int num_rules; + struct mlx5_flow_rule *rule[]; }; +/* Type of children is mlx5_flow_group */ struct mlx5_flow_table { - struct fs_base base; - /* sorted list by start_index */ - struct list_head fgs; + struct fs_node node; + u32 id; + u16 vport; + unsigned int max_fte; + unsigned int level; + enum fs_flow_table_type type; + enum fs_flow_table_op_mod op_mod; struct { bool active; - unsigned int max_types; + unsigned int required_groups; unsigned int group_size; - unsigned int num_types; + unsigned int num_groups; unsigned int max_fte; } autogroup; - unsigned int max_fte; - unsigned int level; - uint32_t id; - u16 vport; - enum fs_ft_type type; - struct fs_star_rule star_rule; - unsigned int shared_refcount; + /* Protect fwd_rules */ + struct mutex lock; + /* FWD rules that point on this flow table */ + struct list_head fwd_rules; + u32 flags; + struct xarray fgs_xa; + enum mlx5_flow_table_miss_action def_miss_action; + struct mlx5_flow_namespace *ns; }; -enum fs_prio_flags { - MLX5_CORE_FS_PRIO_SHARED = 1 +struct mlx5_ft_underlay_qp { + struct list_head list; + u32 qpn; +}; + +#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_e00 +/* Calculate the fte_match_param length and without the reserved length. + * Make sure the reserved field is the last. + */ +#define MLX5_ST_SZ_DW_MATCH_PARAM \ + ((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \ + BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) != \ + MLX5_FLD_SZ_BYTES(fte_match_param, \ + MLX5_FTE_MATCH_PARAM_RESERVED) +\ + MLX5_BYTE_OFF(fte_match_param, \ + MLX5_FTE_MATCH_PARAM_RESERVED))) + +/* Type of children is mlx5_flow_rule */ +struct fs_fte { + struct fs_node node; + u32 val[MLX5_ST_SZ_DW_MATCH_PARAM]; + u32 dests_size; + u32 fwd_dests; + u32 index; + struct mlx5_flow_context flow_context; + struct mlx5_flow_act action; + enum fs_fte_status status; + struct mlx5_fc *counter; + int modify_mask; }; +/* Type of children is mlx5_flow_table/namespace */ struct fs_prio { - struct fs_base base; - struct list_head objs; /* each object is a namespace or ft */ - unsigned int max_ft; - unsigned int num_ft; - unsigned int max_ns; + struct fs_node node; + unsigned int num_levels; + unsigned int start_level; unsigned int prio; - /*When create shared flow table, this lock should be taken*/ - struct mutex shared_lock; - u8 flags; + unsigned int num_ft; }; +/* Type of children is fs_prio */ struct mlx5_flow_namespace { /* parent == NULL => root ns */ - struct fs_base base; - /* sorted by priority number */ - struct list_head prios; /* list of fs_prios */ - struct list_head list_notifiers; - struct rw_semaphore notifiers_rw_sem; - struct rw_semaphore dests_rw_sem; + struct fs_node node; + enum mlx5_flow_table_miss_action def_miss_action; +}; + +struct mlx5_flow_group_mask { + u8 match_criteria_enable; + u32 match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM]; +}; + +/* Type of children is fs_fte */ +struct mlx5_flow_group { + struct fs_node node; + struct mlx5_flow_group_mask mask; + u32 start_index; + u32 max_ftes; + struct ida fte_allocator; + u32 id; + struct xarray ftes_xa; }; struct mlx5_flow_root_namespace { struct mlx5_flow_namespace ns; - struct mlx5_flow_table *ft_level_0; - enum fs_ft_type table_type; + enum mlx5_flow_steering_mode mode; + enum fs_flow_table_type table_type; struct mlx5_core_dev *dev; struct mlx5_flow_table *root_ft; - /* When chaining flow-tables, this lock should be taken */ - struct mutex fs_chain_lock; + /* Should be held when chaining flow tables */ + struct mutex chain_lock; + struct list_head underlay_qpns; + const struct mlx5_flow_cmds *cmds; }; -struct mlx5_flow_group { - struct fs_base base; - struct list_head ftes; - struct mlx5_core_fs_mask mask; - uint32_t start_index; - uint32_t max_ftes; - uint32_t num_ftes; - uint32_t id; -}; +int mlx5_init_fc_stats(struct mlx5_core_dev *dev); +void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev); +void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, + struct delayed_work *dwork, + unsigned long delay); +void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev, + unsigned long interval); -struct mlx5_flow_handler { - struct list_head list; - rule_event_fn add_dst_cb; - rule_event_fn del_dst_cb; - void *client_context; - struct mlx5_flow_namespace *ns; -}; +const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void); -struct fs_client_priv_data { - struct mlx5_flow_handler *fs_handler; - struct list_head list; - void *client_dst_data; -}; +int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_root_namespace *peer_ns); -struct mlx5_modify_hdr { - enum mlx5_flow_namespace_type ns_type; - u32 id; -}; +int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns, + enum mlx5_flow_steering_mode mode); -struct mlx5_pkt_reformat { - enum mlx5_flow_namespace_type ns_type; - int reformat_type; /* from mlx5_ifc */ - u32 id; -}; +int mlx5_fs_core_alloc(struct mlx5_core_dev *dev); +void mlx5_fs_core_free(struct mlx5_core_dev *dev); +int mlx5_fs_core_init(struct mlx5_core_dev *dev); +void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev); -void _fs_remove_node(struct kref *kref); -#define fs_get_obj(v, _base) {v = container_of((_base), typeof(*v), base); } -#define fs_get_parent(v, child) {v = (child)->base.parent ? \ - container_of((child)->base.parent, \ - typeof(*v), base) : NULL; } +int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports); +void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev); +int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports); +void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev); -#define fs_list_for_each_entry(pos, cond, root) \ - list_for_each_entry(pos, root, base.list) \ - if (!(cond)) {} else +u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type); -#define fs_list_for_each_entry_continue(pos, cond, root) \ - list_for_each_entry_continue(pos, root, base.list) \ - if (!(cond)) {} else +struct mlx5_flow_root_namespace *find_root(struct fs_node *node); -#define fs_list_for_each_entry_reverse(pos, cond, root) \ - list_for_each_entry_reverse(pos, root, base.list) \ - if (!(cond)) {} else +#define fs_get_obj(v, _node) {v = container_of((_node), typeof(*v), node); } -#define fs_list_for_each_entry_continue_reverse(pos, cond, root) \ - list_for_each_entry_continue_reverse(pos, root, base.list) \ - if (!(cond)) {} else +#define fs_list_for_each_entry(pos, root) \ + list_for_each_entry(pos, root, node.list) -#define fs_for_each_ft(pos, prio) \ - fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_TABLE, \ - &(prio)->objs) +#define fs_list_for_each_entry_safe(pos, tmp, root) \ + list_for_each_entry_safe(pos, tmp, root, node.list) -#define fs_for_each_ft_reverse(pos, prio) \ - fs_list_for_each_entry_reverse(pos, \ - (pos)->base.type == FS_TYPE_FLOW_TABLE, \ - &(prio)->objs) +#define fs_for_each_ns_or_ft_reverse(pos, prio) \ + list_for_each_entry_reverse(pos, &(prio)->node.children, list) -#define fs_for_each_ns(pos, prio) \ - fs_list_for_each_entry(pos, \ - (pos)->base.type == FS_TYPE_NAMESPACE, \ - &(prio)->objs) - -#define fs_for_each_ns_or_ft_reverse(pos, prio) \ - list_for_each_entry_reverse(pos, &(prio)->objs, list) \ - if (!((pos)->type == FS_TYPE_NAMESPACE || \ - (pos)->type == FS_TYPE_FLOW_TABLE)) {} else - -#define fs_for_each_ns_or_ft(pos, prio) \ - list_for_each_entry(pos, &(prio)->objs, list) \ - if (!((pos)->type == FS_TYPE_NAMESPACE || \ - (pos)->type == FS_TYPE_FLOW_TABLE)) {} else - -#define fs_for_each_ns_or_ft_continue_reverse(pos, prio) \ - list_for_each_entry_continue_reverse(pos, &(prio)->objs, list) \ - if (!((pos)->type == FS_TYPE_NAMESPACE || \ - (pos)->type == FS_TYPE_FLOW_TABLE)) {} else - -#define fs_for_each_ns_or_ft_continue(pos, prio) \ - list_for_each_entry_continue(pos, &(prio)->objs, list) \ - if (!((pos)->type == FS_TYPE_NAMESPACE || \ - (pos)->type == FS_TYPE_FLOW_TABLE)) {} else +#define fs_for_each_ns_or_ft(pos, prio) \ + list_for_each_entry(pos, (&(prio)->node.children), list) #define fs_for_each_prio(pos, ns) \ - fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_PRIO, \ - &(ns)->prios) + fs_list_for_each_entry(pos, &(ns)->node.children) -#define fs_for_each_prio_reverse(pos, ns) \ - fs_list_for_each_entry_reverse(pos, (pos)->base.type == FS_TYPE_PRIO, \ - &(ns)->prios) +#define fs_for_each_ns(pos, prio) \ + fs_list_for_each_entry(pos, &(prio)->node.children) -#define fs_for_each_prio_continue(pos, ns) \ - fs_list_for_each_entry_continue(pos, (pos)->base.type == FS_TYPE_PRIO, \ - &(ns)->prios) +#define fs_for_each_ft(pos, prio) \ + fs_list_for_each_entry(pos, &(prio)->node.children) -#define fs_for_each_prio_continue_reverse(pos, ns) \ - fs_list_for_each_entry_continue_reverse(pos, \ - (pos)->base.type == FS_TYPE_PRIO, \ - &(ns)->prios) +#define fs_for_each_ft_safe(pos, tmp, prio) \ + fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children) #define fs_for_each_fg(pos, ft) \ - fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_GROUP, \ - &(ft)->fgs) + fs_list_for_each_entry(pos, &(ft)->node.children) #define fs_for_each_fte(pos, fg) \ - fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_ENTRY, \ - &(fg)->ftes) + fs_list_for_each_entry(pos, &(fg)->node.children) + #define fs_for_each_dst(pos, fte) \ - fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_DEST, \ - &(fte)->dests) - -int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev, - u16 vport, enum fs_ft_type type, unsigned int level, - unsigned int log_size, const char *name, unsigned int *table_id); - -int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev, - u16 vport, - enum fs_ft_type type, unsigned int table_id); - -int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev, - u32 *in, - u16 vport, - enum fs_ft_type type, unsigned int table_id, - unsigned int *group_id); - -int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev, - u16 vport, - enum fs_ft_type type, unsigned int table_id, - unsigned int group_id); - - -int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev, - u16 vport, - enum fs_fte_status *fte_status, - u32 *match_val, - enum fs_ft_type type, unsigned int table_id, - unsigned int index, unsigned int group_id, - struct mlx5_flow_act *flow_act, - u32 sw_action, int dest_size, - struct list_head *dests); /* mlx5_flow_desination */ - -int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev, - u16 vport, - enum fs_fte_status *fte_status, - enum fs_ft_type type, unsigned int table_id, - unsigned int index); - -int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev, - enum fs_ft_type type, - unsigned int id); - -int mlx5_init_fs(struct mlx5_core_dev *dev); -void mlx5_cleanup_fs(struct mlx5_core_dev *dev); -void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev, - unsigned long interval); + fs_list_for_each_entry(pos, &(fte)->node.children) + +#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \ + (type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \ + (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \ + (type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \ + (type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \ + (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \ + (type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) : \ + (type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) : \ + (type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) : \ + (type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) : \ + (type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) : \ + (BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\ + ) -int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev, - enum mlx5_flow_namespace_type namespace, - u8 num_actions, - void *modify_actions, - struct mlx5_modify_hdr *modify_hdr); -void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev, - struct mlx5_modify_hdr *modify_hdr); -int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev, - struct mlx5_pkt_reformat_params *params, - enum mlx5_flow_namespace_type namespace, - struct mlx5_pkt_reformat *pkt_reformat); -void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev, - struct mlx5_pkt_reformat *pkt_reformat); -int mlx5_init_fc_stats(struct mlx5_core_dev *dev); -void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev); -void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, - struct delayed_work *dwork, - unsigned long delay); #endif diff --git a/sys/dev/mlx5/mlx5_core/fs_ft_pool.h b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h new file mode 100644 index 000000000000..a5e4df624e27 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#ifndef __MLX5_FS_FT_POOL_H__ +#define __MLX5_FS_FT_POOL_H__ + +#include <linux/module.h> +#include <dev/mlx5/driver.h> +#include <dev/mlx5/mlx5_core/fs_core.h> +#include <linux/compiler.h> + +#define POOL_NEXT_SIZE BIT(30) + +int mlx5_ft_pool_init(struct mlx5_core_dev *dev); +void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev); + +int +mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type, + int desired_size); +void +mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz); + +#endif /* __MLX5_FS_FT_POOL_H__ */ diff --git a/sys/dev/mlx5/mlx5_core/fs_tcp.h b/sys/dev/mlx5/mlx5_core/fs_tcp.h index fa11ad9c4cb5..e2433af53a42 100644 --- a/sys/dev/mlx5/mlx5_core/fs_tcp.h +++ b/sys/dev/mlx5/mlx5_core/fs_tcp.h @@ -27,15 +27,15 @@ #define __MLX5E_ACCEL_FS_TCP_H__ struct inpcb; -struct mlx5_flow_rule; +struct mlx5_flow_handle; struct mlx5e_priv; int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *); void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *); -struct mlx5_flow_rule * +struct mlx5_flow_handle * mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *, struct inpcb *, uint32_t tirn, uint32_t flow_tag, uint16_t vlan_id); #define MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN 0xFFFF -void mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *); +void mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *); #endif /* __MLX5E_ACCEL_FS_TCP_H__ */ diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c index d46feb4b9e5b..86c721a83cb7 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c @@ -247,7 +247,7 @@ static void poll_timeout(struct mlx5_cmd_work_ent *ent) { struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd); - int poll_end = jiffies + + long poll_end = jiffies + msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC + 1000); u8 own; @@ -417,6 +417,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, case MLX5_CMD_OP_QUERY_VPORT_COUNTER: case MLX5_CMD_OP_ALLOC_Q_COUNTER: case MLX5_CMD_OP_QUERY_Q_COUNTER: + case MLX5_CMD_OP_QUERY_FLOW_COUNTER: case MLX5_CMD_OP_ALLOC_PD: case MLX5_CMD_OP_ALLOC_UAR: case MLX5_CMD_OP_CONFIG_INT_MODERATION: @@ -614,6 +615,9 @@ const char *mlx5_command_str(int command) MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJ); MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJ); MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJ); + MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER); + MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER); + MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER); default: return "unknown command opcode"; } } @@ -947,7 +951,7 @@ static const char *deliv_status_to_str(u8 status) static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) { - int timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); + unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); int err; if (ent->polling) { diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h index f0b1dde60323..f63bb2070bcf 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_core.h +++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h @@ -163,4 +163,14 @@ enum { u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); +/************************************************ TESTTEST********************************************/ +static inline int mlx5_init_fs(struct mlx5_core_dev *dev) +{ + return 0; +} + +static inline int mlx5_cleanup_fs(struct mlx5_core_dev *dev) +{ + return 0; +} #endif /* __MLX5_CORE_H__ */ diff --git a/sys/dev/mlx5/mlx5_core/mlx5_crypto.c b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c new file mode 100644 index 000000000000..03804219e0b3 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c @@ -0,0 +1,94 @@ +/*- + * Copyright (c) 2019-2021, Mellanox Technologies, Ltd. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "opt_rss.h" +#include "opt_ratelimit.h" + +#include <linux/kernel.h> +#include <linux/module.h> + +#include <dev/mlx5/driver.h> +#include <dev/mlx5/crypto.h> + +int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn, u32 key_type, + const void *p_key, u32 key_len, u32 *p_obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {}; + u64 general_obj_types; + int err; + + general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); + if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY)) + return -EINVAL; + + switch (key_len) { + case 128 / 8: + memcpy(MLX5_ADDR_OF(create_encryption_key_in, in, + encryption_key_object.key[4]), p_key, 128 / 8); + MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn); + MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size, + MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128); + MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type, + key_type); + break; + case 256 / 8: + memcpy(MLX5_ADDR_OF(create_encryption_key_in, in, + encryption_key_object.key[0]), p_key, 256 / 8); + MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn); + MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size, + MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256); + MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type, + key_type); + break; + default: + return -EINVAL; + } + + MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ); + MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err == 0) + *p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id); + + /* avoid leaking key on the stack */ + explicit_bzero(in, sizeof(in)); + + return err; +} + +int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid) +{ + u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {}; + + MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ); + MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + MLX5_SET(destroy_encryption_key_in, in, obj_id, oid); + + return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} diff --git a/sys/dev/mlx5/mlx5_core/mlx5_diagnostics.c b/sys/dev/mlx5/mlx5_core/mlx5_diagnostics.c index 182be547272a..9730ab3c56c1 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_diagnostics.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_diagnostics.c @@ -26,6 +26,8 @@ #include "opt_rss.h" #include "opt_ratelimit.h" +#define _WANT_SFF_8024_ID + #include <dev/mlx5/driver.h> #include <dev/mlx5/port.h> #include <dev/mlx5/diagnostics.h> diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c index 2d5b53b6482a..1090f8638171 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_eq.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c @@ -33,6 +33,7 @@ #include <dev/mlx5/mlx5_fpga/core.h> #include <dev/mlx5/mlx5_core/mlx5_core.h> #include <dev/mlx5/mlx5_core/eswitch.h> +#include <dev/mlx5/mlx5_accel/ipsec.h> #ifdef RSS #include <net/rss_config.h> @@ -165,6 +166,8 @@ static const char *eqe_type_str(u8 type) return "MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT"; case MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT: return "MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT"; + case MLX5_EVENT_TYPE_OBJECT_CHANGE: + return "MLX5_EVENT_TYPE_OBJECT_CHANGE"; default: return "Unrecognized event"; } @@ -370,6 +373,10 @@ static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq) mlx5_temp_warning_event(dev, eqe); break; + case MLX5_EVENT_TYPE_OBJECT_CHANGE: + mlx5_object_change_event(dev, eqe); + break; + default: mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n", eqe->type, eq->eqn); @@ -571,6 +578,10 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev) MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT); } + if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + async_event_mask |= + (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD, MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD); if (err) { @@ -679,9 +690,9 @@ static const char *mlx5_port_module_event_error_type_to_string(u8 error_type) unsigned int mlx5_query_module_status(struct mlx5_core_dev *dev, int module_num) { - if (module_num < 0 || module_num >= MLX5_MAX_PORTS) - return 0; /* undefined */ - return dev->module_status[module_num]; + if (module_num != dev->module_num) + return 0; /* module num doesn't equal to what FW reported */ + return dev->module_status; } static void mlx5_port_module_event(struct mlx5_core_dev *dev, @@ -729,8 +740,8 @@ static void mlx5_port_module_event(struct mlx5_core_dev *dev, "Module %u, unknown status %d\n", module_num, module_status); } /* store module status */ - if (module_num < MLX5_MAX_PORTS) - dev->module_status[module_num] = module_status; + dev->module_status = module_status; + dev->module_num = module_num; } static void mlx5_port_general_notification_event(struct mlx5_core_dev *dev, diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c index 15f5f0ff0336..30f04144502b 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c @@ -64,7 +64,7 @@ struct esw_uc_addr { /* E-Switch MC FDB table hash node */ struct esw_mc_addr { /* SRIOV only */ struct l2addr_node node; - struct mlx5_flow_rule *uplink_rule; /* Forward to uplink rule */ + struct mlx5_flow_handle *uplink_rule; /* Forward to uplink rule */ u32 refcnt; }; @@ -73,7 +73,7 @@ struct vport_addr { struct l2addr_node node; u8 action; u32 vport; - struct mlx5_flow_rule *flow_rule; /* SRIOV only */ + struct mlx5_flow_handle *flow_rule; /* SRIOV only */ }; enum { @@ -215,59 +215,54 @@ static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport, } /* E-Switch FDB */ -static struct mlx5_flow_rule * +static struct mlx5_flow_handle * esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport) { - int match_header = MLX5_MATCH_OUTER_HEADERS; - struct mlx5_flow_destination dest; - struct mlx5_flow_rule *flow_rule = NULL; + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_handle *flow_rule = NULL; struct mlx5_flow_act flow_act = {}; - u32 *match_v; - u32 *match_c; + struct mlx5_flow_spec *spec; u8 *dmac_v; u8 *dmac_c; - match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - if (!match_v || !match_c) { - printf("mlx5_core: WARN: ""FDB: Failed to alloc match parameters\n"); + spec = kzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + printf("mlx5_core: WARN: ""FDB: Failed to alloc flow spec\n"); goto out; } - dmac_v = MLX5_ADDR_OF(fte_match_param, match_v, + dmac_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.dmac_47_16); - dmac_c = MLX5_ADDR_OF(fte_match_param, match_c, + dmac_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers.dmac_47_16); + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; ether_addr_copy(dmac_v, mac); /* Match criteria mask */ memset(dmac_c, 0xff, 6); - dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT; - dest.vport_num = vport; + dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; + dest.vport.num = vport; esw_debug(esw->dev, "\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n", dmac_v, dmac_c, vport); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; flow_rule = - mlx5_add_flow_rule(esw->fdb_table.fdb, - match_header, - match_c, - match_v, - MLX5_FLOW_RULE_FWD_ACTION_DEST, - &flow_act, &dest); + mlx5_add_flow_rules(esw->fdb_table.fdb, spec, + &flow_act, &dest, 1); if (IS_ERR_OR_NULL(flow_rule)) { printf("mlx5_core: WARN: ""FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n", dmac_v, dmac_c, vport, PTR_ERR(flow_rule)); flow_rule = NULL; } out: - kfree(match_v); - kfree(match_c); + kfree(spec); return flow_rule; } static int esw_create_fdb_table(struct mlx5_eswitch *esw) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_core_dev *dev = esw->dev; struct mlx5_flow_namespace *root_ns; struct mlx5_flow_table *fdb; @@ -295,7 +290,9 @@ static int esw_create_fdb_table(struct mlx5_eswitch *esw) /* (-2) Since MaorG said so .. */ table_size = BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size)) - 2; - fdb = mlx5_create_flow_table(root_ns, 0, "FDB", table_size); + ft_attr.prio = FDB_SLOW_PATH; + ft_attr.max_fte = table_size; + fdb = mlx5_create_flow_table(root_ns, &ft_attr); if (IS_ERR_OR_NULL(fdb)) { err = PTR_ERR(fdb); esw_warn(dev, "Failed to create FDB Table err %d\n", err); @@ -397,7 +394,7 @@ static int esw_del_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr) mlx5_mpfs_del_mac(esw->dev, esw_uc->table_index); - mlx5_del_flow_rule(&vaddr->flow_rule); + mlx5_del_flow_rules(&vaddr->flow_rule); l2addr_hash_del(esw_uc); return 0; @@ -456,12 +453,12 @@ static int esw_del_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr) vport, mac, vaddr->flow_rule, esw_mc->refcnt, esw_mc->uplink_rule); - mlx5_del_flow_rule(&vaddr->flow_rule); + mlx5_del_flow_rules(&vaddr->flow_rule); if (--esw_mc->refcnt) return 0; - mlx5_del_flow_rule(&esw_mc->uplink_rule); + mlx5_del_flow_rules(&esw_mc->uplink_rule); l2addr_hash_del(esw_mc); return 0; @@ -602,13 +599,13 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_group *vlan_grp = NULL; struct mlx5_flow_group *drop_grp = NULL; struct mlx5_core_dev *dev = esw->dev; struct mlx5_flow_namespace *root_ns; struct mlx5_flow_table *acl; void *match_criteria; - char table_name[32]; u32 *flow_group_in; int table_size = 2; int err = 0; @@ -619,7 +616,7 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, esw_debug(dev, "Create vport[%d] egress ACL log_max_size(%d)\n", vport->vport, MLX5_CAP_ESW_EGRESS_ACL(dev, log_max_ft_size)); - root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_EGRESS); + root_ns = mlx5_get_flow_vport_acl_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_EGRESS, vport->vport); if (!root_ns) { esw_warn(dev, "Failed to get E-Switch egress flow namespace\n"); return; @@ -629,8 +626,10 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, if (!flow_group_in) return; - snprintf(table_name, 32, "egress_%d", vport->vport); - acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size); + ft_attr.max_fte = table_size; + if (vport->vport) + ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; + acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport); if (IS_ERR_OR_NULL(acl)) { err = PTR_ERR(acl); esw_warn(dev, "Failed to create E-Switch vport[%d] egress flow Table, err(%d)\n", @@ -678,8 +677,8 @@ out: static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { - mlx5_del_flow_rule(&vport->egress.allowed_vlan); - mlx5_del_flow_rule(&vport->egress.drop_rule); + mlx5_del_flow_rules(&vport->egress.allowed_vlan); + mlx5_del_flow_rules(&vport->egress.drop_rule); } static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw, @@ -703,12 +702,12 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_core_dev *dev = esw->dev; struct mlx5_flow_namespace *root_ns; struct mlx5_flow_table *acl; struct mlx5_flow_group *g; void *match_criteria; - char table_name[32]; u32 *flow_group_in; int table_size = 1; int err = 0; @@ -719,7 +718,7 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, esw_debug(dev, "Create vport[%d] ingress ACL log_max_size(%d)\n", vport->vport, MLX5_CAP_ESW_INGRESS_ACL(dev, log_max_ft_size)); - root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_INGRESS); + root_ns = mlx5_get_flow_vport_acl_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_INGRESS, vport->vport); if (!root_ns) { esw_warn(dev, "Failed to get E-Switch ingress flow namespace\n"); return; @@ -729,8 +728,10 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, if (!flow_group_in) return; - snprintf(table_name, 32, "ingress_%d", vport->vport); - acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size); + ft_attr.max_fte = table_size; + if (vport->vport) + ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; + acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport); if (IS_ERR_OR_NULL(acl)) { err = PTR_ERR(acl); esw_warn(dev, "Failed to create E-Switch vport[%d] ingress flow Table, err(%d)\n", @@ -763,7 +764,7 @@ out: static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { - mlx5_del_flow_rule(&vport->ingress.drop_rule); + mlx5_del_flow_rules(&vport->ingress.drop_rule); } static void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw, @@ -785,9 +786,7 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { struct mlx5_flow_act flow_act = {}; - struct mlx5_flow_destination dest; - u32 *match_v; - u32 *match_c; + struct mlx5_flow_spec *spec; int err = 0; if (IS_ERR_OR_NULL(vport->ingress.acl)) { @@ -806,35 +805,28 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw, "vport[%d] configure ingress rules, vlan(%d) qos(%d)\n", vport->vport, vport->vlan, vport->qos); - match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - if (!match_v || !match_c) { + spec = kzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { err = -ENOMEM; esw_warn(esw->dev, "vport[%d] configure ingress rules failed, err(%d)\n", vport->vport, err); goto out; } - MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag); - - dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT; - dest.vport_num = vport->vport; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; vport->ingress.drop_rule = - mlx5_add_flow_rule(vport->ingress.acl, - MLX5_MATCH_OUTER_HEADERS, - match_c, - match_v, - MLX5_FLOW_RULE_FWD_ACTION_DROP, - &flow_act, &dest); + mlx5_add_flow_rules(vport->ingress.acl, spec, + &flow_act, NULL, 0); if (IS_ERR_OR_NULL(vport->ingress.drop_rule)) { err = PTR_ERR(vport->ingress.drop_rule); printf("mlx5_core: WARN: ""vport[%d] configure ingress rules, err(%d)\n", vport->vport, err); vport->ingress.drop_rule = NULL; } out: - kfree(match_v); - kfree(match_c); + kfree(spec); return err; } @@ -842,9 +834,7 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { struct mlx5_flow_act flow_act = {}; - struct mlx5_flow_destination dest; - u32 *match_v; - u32 *match_c; + struct mlx5_flow_spec *spec; int err = 0; if (IS_ERR_OR_NULL(vport->egress.acl)) { @@ -862,9 +852,8 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw, "vport[%d] configure egress rules, vlan(%d) qos(%d)\n", vport->vport, vport->vlan, vport->qos); - match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - if (!match_v || !match_c) { + spec = kzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { err = -ENOMEM; esw_warn(esw->dev, "vport[%d] configure egress rules failed, err(%d)\n", vport->vport, err); @@ -872,21 +861,17 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw, } /* Allowed vlan rule */ - MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.first_vid); - MLX5_SET(fte_match_param, match_v, outer_headers.first_vid, vport->vlan); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vport->vlan); - dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT; - dest.vport_num = vport->vport; + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; vport->egress.allowed_vlan = - mlx5_add_flow_rule(vport->egress.acl, - MLX5_MATCH_OUTER_HEADERS, - match_c, - match_v, - MLX5_FLOW_RULE_FWD_ACTION_ALLOW, - &flow_act, &dest); + mlx5_add_flow_rules(vport->egress.acl, spec, + &flow_act, NULL, 0); if (IS_ERR_OR_NULL(vport->egress.allowed_vlan)) { err = PTR_ERR(vport->egress.allowed_vlan); printf("mlx5_core: WARN: ""vport[%d] configure egress allowed vlan rule failed, err(%d)\n", vport->vport, err); @@ -894,24 +879,17 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw, goto out; } - /* Drop others rule (star rule) */ - memset(match_c, 0, MLX5_ST_SZ_BYTES(fte_match_param)); - memset(match_v, 0, MLX5_ST_SZ_BYTES(fte_match_param)); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; vport->egress.drop_rule = - mlx5_add_flow_rule(vport->egress.acl, - 0, - match_c, - match_v, - MLX5_FLOW_RULE_FWD_ACTION_DROP, - &flow_act, &dest); + mlx5_add_flow_rules(vport->egress.acl, NULL, + &flow_act, NULL, 0); if (IS_ERR_OR_NULL(vport->egress.drop_rule)) { err = PTR_ERR(vport->egress.drop_rule); printf("mlx5_core: WARN: ""vport[%d] configure egress drop rule failed, err(%d)\n", vport->vport, err); vport->egress.drop_rule = NULL; } out: - kfree(match_v); - kfree(match_c); + kfree(spec); return err; } @@ -1030,7 +1008,7 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs) esw_warn(esw->dev, "E-Switch ingress ACL is not supported by FW\n"); if (!MLX5_CAP_ESW_EGRESS_ACL(esw->dev, ft_support)) - esw_warn(esw->dev, "E-Switch engress ACL is not supported by FW\n"); + esw_warn(esw->dev, "E-Switch egress ACL is not supported by FW\n"); esw_info(esw->dev, "E-Switch enable SRIOV: nvfs(%d)\n", nvfs); diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c deleted file mode 100644 index f3410249e67f..000000000000 --- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c +++ /dev/null @@ -1,102 +0,0 @@ -/*- - * Copyright (c) 2022 NVIDIA corporation & affiliates. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#include <dev/mlx5/driver.h> -#include <dev/mlx5/device.h> -#include <dev/mlx5/mlx5_ifc.h> -#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h> -#include <dev/mlx5/mlx5_core/mlx5_core.h> - -int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, - enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask, - u32 *id) -{ - u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {}; - u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {}; - int err; - - MLX5_SET(alloc_flow_counter_in, in, opcode, - MLX5_CMD_OP_ALLOC_FLOW_COUNTER); - MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask); - - err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out); - if (!err) - *id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id); - return err; -} - -int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id) -{ - return mlx5_cmd_fc_bulk_alloc(dev, 0, id); -} - -int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id) -{ - u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {}; - - MLX5_SET(dealloc_flow_counter_in, in, opcode, - MLX5_CMD_OP_DEALLOC_FLOW_COUNTER); - MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id); - return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in); -} - -int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id, - u64 *packets, u64 *bytes) -{ - u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) + - MLX5_ST_SZ_BYTES(traffic_counter)] = {}; - u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {}; - void *stats; - int err = 0; - - MLX5_SET(query_flow_counter_in, in, opcode, - MLX5_CMD_OP_QUERY_FLOW_COUNTER); - MLX5_SET(query_flow_counter_in, in, op_mod, 0); - MLX5_SET(query_flow_counter_in, in, flow_counter_id, id); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (err) - return err; - - stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics); - *packets = MLX5_GET64(traffic_counter, stats, packets); - *bytes = MLX5_GET64(traffic_counter, stats, octets); - return 0; -} - -int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len, - u32 *out) -{ - int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len); - u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {}; - - MLX5_SET(query_flow_counter_in, in, opcode, - MLX5_CMD_OP_QUERY_FLOW_COUNTER); - MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id); - MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len); - return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); -} - diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h deleted file mode 100644 index 3adebb3ca94c..000000000000 --- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef _MLX5_FC_CMD_ -#define _MLX5_FC_CMD_ - -#include "fs_core.h" - -int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id); -int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, - enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask, - u32 *id); -int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id); -int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id, - u64 *packets, u64 *bytes); - -int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len, - u32 *out); -static inline int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len) -{ - return MLX5_ST_SZ_BYTES(query_flow_counter_out) + - MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len; -} - -#endif diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c new file mode 100644 index 000000000000..21c1914fd864 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2020 Mellanox Technologies. + +#include <dev/mlx5/driver.h> +#include <dev/mlx5/mlx5_ifc.h> +#include <dev/mlx5/fs.h> + +#include "mlx5_core.h" +#include "fs_chains.h" +#include "fs_ft_pool.h" +#include "fs_core.h" + +#define chains_lock(chains) ((chains)->lock) +#define chains_xa(chains) ((chains)->chains_xa) +#define prios_xa(chains) ((chains)->prios_xa) +#define chains_default_ft(chains) ((chains)->chains_default_ft) +#define chains_end_ft(chains) ((chains)->chains_end_ft) +#define FT_TBL_SZ (64 * 1024) + +struct mlx5_fs_chains { + struct mlx5_core_dev *dev; + + struct xarray chains_xa; + struct xarray prios_xa; + /* Protects above chains_ht and prios_ht */ + struct mutex lock; + + struct mlx5_flow_table *chains_default_ft; + struct mlx5_flow_table *chains_end_ft; + + enum mlx5_flow_namespace_type ns; + u32 group_num; + u32 flags; + int fs_base_prio; + int fs_base_level; +}; + +struct fs_chain { + u32 chain; + + int ref; + int id; + uint32_t xa_idx; + + struct mlx5_fs_chains *chains; + struct list_head prios_list; + struct mlx5_flow_handle *restore_rule; + struct mlx5_modify_hdr *miss_modify_hdr; +}; + +struct prio_key { + u32 chain; + u32 prio; + u32 level; +}; + +struct prio { + struct list_head list; + + struct prio_key key; + uint32_t xa_idx; + + int ref; + + struct fs_chain *chain; + struct mlx5_flow_table *ft; + struct mlx5_flow_table *next_ft; + struct mlx5_flow_group *miss_group; + struct mlx5_flow_handle *miss_rule; +}; + +/* +static const struct rhashtable_params chain_params = { + .head_offset = offsetof(struct fs_chain, node), + .key_offset = offsetof(struct fs_chain, chain), + .key_len = sizeof_field(struct fs_chain, chain), + .automatic_shrinking = true, +}; + +static const struct rhashtable_params prio_params = { + .head_offset = offsetof(struct prio, node), + .key_offset = offsetof(struct prio, key), + .key_len = sizeof_field(struct prio, key), + .automatic_shrinking = true, +}; +*/ + +bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains) +{ + return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED; +} + +bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains) +{ + return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; +} + +bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains) +{ + return mlx5_chains_prios_supported(chains) && + mlx5_chains_ignore_flow_level_supported(chains); +} + +u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains) +{ + if (!mlx5_chains_prios_supported(chains)) + return 1; + + if (mlx5_chains_ignore_flow_level_supported(chains)) + return UINT_MAX - 1; + + /* We should get here only for eswitch case */ + return FDB_TC_MAX_CHAIN; +} + +u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains) +{ + return mlx5_chains_get_chain_range(chains) + 1; +} + +u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains) +{ + if (mlx5_chains_ignore_flow_level_supported(chains)) + return UINT_MAX; + + if (!chains->dev->priv.eswitch) + return 1; + + /* We should get here only for eswitch case */ + return FDB_TC_MAX_PRIO; +} + +static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains) +{ + if (mlx5_chains_ignore_flow_level_supported(chains)) + return UINT_MAX; + + /* Same value for FDB and NIC RX tables */ + return FDB_TC_LEVELS_PER_PRIO; +} + +void +mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains, + struct mlx5_flow_table *ft) +{ + chains_end_ft(chains) = ft; +} + +static struct mlx5_flow_table * +mlx5_chains_create_table(struct mlx5_fs_chains *chains, + u32 chain, u32 prio, u32 level) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_namespace *ns; + struct mlx5_flow_table *ft; + int sz; + + if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED) + ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | + MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); + + sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE; + ft_attr.max_fte = sz; + + /* We use chains_default_ft(chains) as the table's next_ft till + * ignore_flow_level is allowed on FT creation and not just for FTEs. + * Instead caller should add an explicit miss rule if needed. + */ + ft_attr.next_ft = chains_default_ft(chains); + + /* The root table(chain 0, prio 1, level 0) is required to be + * connected to the previous fs_core managed prio. + * We always create it, as a managed table, in order to align with + * fs_core logic. + */ + if (!mlx5_chains_ignore_flow_level_supported(chains) || + (chain == 0 && prio == 1 && level == 0)) { + ft_attr.level = chains->fs_base_level; + ft_attr.prio = chains->fs_base_prio; + ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ? + mlx5_get_fdb_sub_ns(chains->dev, chain) : + mlx5_get_flow_namespace(chains->dev, chains->ns); + } else { + ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED; + ft_attr.prio = chains->fs_base_prio; + /* Firmware doesn't allow us to create another level 0 table, + * so we create all unmanaged tables as level 1 (base + 1). + * + * To connect them, we use explicit miss rules with + * ignore_flow_level. Caller is responsible to create + * these rules (if needed). + */ + ft_attr.level = chains->fs_base_level + 1; + ns = mlx5_get_flow_namespace(chains->dev, chains->ns); + } + + ft_attr.autogroup.num_reserved_entries = 2; + ft_attr.autogroup.max_num_groups = chains->group_num; + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + if (IS_ERR(ft)) { + mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n", + (int)PTR_ERR(ft), chain, prio, level, sz); + return ft; + } + + return ft; +} + +static struct fs_chain * +mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain) +{ + struct fs_chain *chain_s = NULL; + int err; + + chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL); + if (!chain_s) + return ERR_PTR(-ENOMEM); + + chain_s->chains = chains; + chain_s->chain = chain; + INIT_LIST_HEAD(&chain_s->prios_list); + + err = xa_alloc(&chains_xa(chains), &chain_s->xa_idx, chain_s, + xa_limit_32b, GFP_KERNEL); + if (err) + goto err_insert; + + return chain_s; + +err_insert: + kvfree(chain_s); + return ERR_PTR(err); +} + +static void +mlx5_chains_destroy_chain(struct fs_chain *chain) +{ + struct mlx5_fs_chains *chains = chain->chains; + + xa_erase(&chains_xa(chains), chain->xa_idx); + kvfree(chain); +} + +static struct fs_chain * +mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain) +{ + struct fs_chain *chain_s = NULL; + unsigned long idx; + + xa_for_each(&chains_xa(chains), idx, chain_s) { + if (chain_s->chain == chain) + break; + } + + if (!chain_s) { + chain_s = mlx5_chains_create_chain(chains, chain); + if (IS_ERR(chain_s)) + return chain_s; + } + + chain_s->ref++; + + return chain_s; +} + +static struct mlx5_flow_handle * +mlx5_chains_add_miss_rule(struct fs_chain *chain, + struct mlx5_flow_table *ft, + struct mlx5_flow_table *next_ft) +{ + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act act = {}; + + act.flags = FLOW_ACT_NO_APPEND; + if (mlx5_chains_ignore_flow_level_supported(chain->chains)) + act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; + + act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = next_ft; + + return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1); +} + +static int +mlx5_chains_update_prio_prevs(struct prio *prio, + struct mlx5_flow_table *next_ft) +{ + struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {}; + struct fs_chain *chain = prio->chain; + struct prio *pos; + int n = 0, err; + + if (prio->key.level) + return 0; + + /* Iterate in reverse order until reaching the level 0 rule of + * the previous priority, adding all the miss rules first, so we can + * revert them if any of them fails. + */ + pos = prio; + list_for_each_entry_continue_reverse(pos, + &chain->prios_list, + list) { + miss_rules[n] = mlx5_chains_add_miss_rule(chain, + pos->ft, + next_ft); + if (IS_ERR(miss_rules[n])) { + err = PTR_ERR(miss_rules[n]); + goto err_prev_rule; + } + + n++; + if (!pos->key.level) + break; + } + + /* Success, delete old miss rules, and update the pointers. */ + n = 0; + pos = prio; + list_for_each_entry_continue_reverse(pos, + &chain->prios_list, + list) { + mlx5_del_flow_rules(&pos->miss_rule); + + pos->miss_rule = miss_rules[n]; + pos->next_ft = next_ft; + + n++; + if (!pos->key.level) + break; + } + + return 0; + +err_prev_rule: + while (--n >= 0) + mlx5_del_flow_rules(&miss_rules[n]); + + return err; +} + +static void +mlx5_chains_put_chain(struct fs_chain *chain) +{ + if (--chain->ref == 0) + mlx5_chains_destroy_chain(chain); +} + +static struct prio * +mlx5_chains_create_prio(struct mlx5_fs_chains *chains, + u32 chain, u32 prio, u32 level) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_handle *miss_rule; + struct mlx5_flow_group *miss_group; + struct mlx5_flow_table *next_ft; + struct mlx5_flow_table *ft; + struct fs_chain *chain_s; + struct list_head *pos; + struct prio *prio_s; + u32 *flow_group_in; + int err; + + chain_s = mlx5_chains_get_chain(chains, chain); + if (IS_ERR(chain_s)) + return ERR_CAST(chain_s); + + prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL); + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!prio_s || !flow_group_in) { + err = -ENOMEM; + goto err_alloc; + } + + /* Chain's prio list is sorted by prio and level. + * And all levels of some prio point to the next prio's level 0. + * Example list (prio, level): + * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0) + * In hardware, we will we have the following pointers: + * (3,0) -> (5,0) -> (7,0) -> Slow path + * (3,1) -> (5,0) + * (5,1) -> (7,0) + * (6,1) -> (7,0) + */ + + /* Default miss for each chain: */ + next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? + chains_default_ft(chains) : + chains_end_ft(chains); + list_for_each(pos, &chain_s->prios_list) { + struct prio *p = list_entry(pos, struct prio, list); + + /* exit on first pos that is larger */ + if (prio < p->key.prio || (prio == p->key.prio && + level < p->key.level)) { + /* Get next level 0 table */ + next_ft = p->key.level == 0 ? p->ft : p->next_ft; + break; + } + } + + ft = mlx5_chains_create_table(chains, chain, prio, level); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + goto err_create; + } + + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, + ft->max_fte - 2); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, + ft->max_fte - 1); + miss_group = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(miss_group)) { + err = PTR_ERR(miss_group); + goto err_group; + } + + /* Add miss rule to next_ft */ + miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft); + if (IS_ERR(miss_rule)) { + err = PTR_ERR(miss_rule); + goto err_miss_rule; + } + + prio_s->miss_group = miss_group; + prio_s->miss_rule = miss_rule; + prio_s->next_ft = next_ft; + prio_s->chain = chain_s; + prio_s->key.chain = chain; + prio_s->key.prio = prio; + prio_s->key.level = level; + prio_s->ft = ft; + + err = xa_alloc(&prios_xa(chains), &prio_s->xa_idx, prio_s, + xa_limit_32b, GFP_KERNEL); + if (err) + goto err_insert; + + list_add(&prio_s->list, pos->prev); + + /* Table is ready, connect it */ + err = mlx5_chains_update_prio_prevs(prio_s, ft); + if (err) + goto err_update; + + kvfree(flow_group_in); + return prio_s; + +err_update: + list_del(&prio_s->list); + xa_erase(&prios_xa(chains), prio_s->xa_idx); +err_insert: + mlx5_del_flow_rules(&miss_rule); +err_miss_rule: + mlx5_destroy_flow_group(miss_group); +err_group: + mlx5_destroy_flow_table(ft); +err_create: +err_alloc: + kvfree(prio_s); + kvfree(flow_group_in); + mlx5_chains_put_chain(chain_s); + return ERR_PTR(err); +} + +static void +mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains, + struct prio *prio) +{ + struct fs_chain *chain = prio->chain; + + WARN_ON(mlx5_chains_update_prio_prevs(prio, + prio->next_ft)); + + list_del(&prio->list); + xa_erase(&prios_xa(chains), prio->xa_idx); + mlx5_del_flow_rules(&prio->miss_rule); + mlx5_destroy_flow_group(prio->miss_group); + mlx5_destroy_flow_table(prio->ft); + mlx5_chains_put_chain(chain); + kvfree(prio); +} + +struct mlx5_flow_table * +mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, + u32 level) +{ + struct mlx5_flow_table *prev_fts; + struct prio *prio_s; + unsigned long idx; + int l = 0; + + if ((chain > mlx5_chains_get_chain_range(chains) && + chain != mlx5_chains_get_nf_ft_chain(chains)) || + prio > mlx5_chains_get_prio_range(chains) || + level > mlx5_chains_get_level_range(chains)) + return ERR_PTR(-EOPNOTSUPP); + + /* create earlier levels for correct fs_core lookup when + * connecting tables. + */ + for (l = 0; l < level; l++) { + prev_fts = mlx5_chains_get_table(chains, chain, prio, l); + if (IS_ERR(prev_fts)) { + prio_s = ERR_CAST(prev_fts); + goto err_get_prevs; + } + } + + mutex_lock(&chains_lock(chains)); + xa_for_each(&prios_xa(chains), idx, prio_s) { + if (chain == prio_s->key.chain && + prio == prio_s->key.prio && + level == prio_s->key.level) + break; + } + if (!prio_s) { + prio_s = mlx5_chains_create_prio(chains, chain, + prio, level); + if (IS_ERR(prio_s)) + goto err_create_prio; + } + + ++prio_s->ref; + mutex_unlock(&chains_lock(chains)); + + return prio_s->ft; + +err_create_prio: + mutex_unlock(&chains_lock(chains)); +err_get_prevs: + while (--l >= 0) + mlx5_chains_put_table(chains, chain, prio, l); + return ERR_CAST(prio_s); +} + +void +mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, + u32 level) +{ + struct prio *prio_s; + unsigned long idx; + + mutex_lock(&chains_lock(chains)); + + xa_for_each(&prios_xa(chains), idx, prio_s) { + if (chain == prio_s->key.chain && + prio == prio_s->key.prio && + level == prio_s->key.level) + break; + } + if (!prio_s) + goto err_get_prio; + + if (--prio_s->ref == 0) + mlx5_chains_destroy_prio(chains, prio_s); + mutex_unlock(&chains_lock(chains)); + + while (level-- > 0) + mlx5_chains_put_table(chains, chain, prio, level); + + return; + +err_get_prio: + mutex_unlock(&chains_lock(chains)); + WARN_ONCE(1, + "Couldn't find table: (chain: %d prio: %d level: %d)", + chain, prio, level); +} + +struct mlx5_flow_table * +mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains) +{ + return chains_end_ft(chains); +} + +struct mlx5_flow_table * +mlx5_chains_create_global_table(struct mlx5_fs_chains *chains) +{ + u32 chain, prio, level; + int err; + + if (!mlx5_chains_ignore_flow_level_supported(chains)) { + err = -EOPNOTSUPP; + + mlx5_core_warn(chains->dev, + "Couldn't create global flow table, ignore_flow_level not supported."); + goto err_ignore; + } + + chain = mlx5_chains_get_chain_range(chains), + prio = mlx5_chains_get_prio_range(chains); + level = mlx5_chains_get_level_range(chains); + + return mlx5_chains_create_table(chains, chain, prio, level); + +err_ignore: + return ERR_PTR(err); +} + +void +mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains, + struct mlx5_flow_table *ft) +{ + mlx5_destroy_flow_table(ft); +} + +static struct mlx5_fs_chains * +mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr) +{ + struct mlx5_fs_chains *chains; + + chains = kzalloc(sizeof(*chains), GFP_KERNEL); + if (!chains) + return ERR_PTR(-ENOMEM); + + chains->dev = dev; + chains->flags = attr->flags; + chains->ns = attr->ns; + chains->group_num = attr->max_grp_num; + chains->fs_base_prio = attr->fs_base_prio; + chains->fs_base_level = attr->fs_base_level; + chains_default_ft(chains) = chains_end_ft(chains) = attr->default_ft; + + xa_init(&chains_xa(chains)); + xa_init(&prios_xa(chains)); + + mutex_init(&chains_lock(chains)); + + return chains; +} + +static void +mlx5_chains_cleanup(struct mlx5_fs_chains *chains) +{ + mutex_destroy(&chains_lock(chains)); + xa_destroy(&prios_xa(chains)); + xa_destroy(&chains_xa(chains)); + + kfree(chains); +} + +struct mlx5_fs_chains * +mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr) +{ + struct mlx5_fs_chains *chains; + + chains = mlx5_chains_init(dev, attr); + + return chains; +} + +void +mlx5_chains_destroy(struct mlx5_fs_chains *chains) +{ + mlx5_chains_cleanup(chains); +} + +void +mlx5_chains_print_info(struct mlx5_fs_chains *chains) +{ + mlx5_core_dbg(chains->dev, "Flow table chains groups(%d)\n", chains->group_num); +} diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c index 0f827f0e69d3..b3c118fedc9b 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c @@ -1,468 +1,993 @@ -/*- - * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. +/* + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ -#include "opt_rss.h" -#include "opt_ratelimit.h" - -#include <linux/types.h> -#include <linux/module.h> -#include <dev/mlx5/mlx5_ifc.h> +#include <dev/mlx5/driver.h> #include <dev/mlx5/device.h> -#include <dev/mlx5/fs.h> +#include <dev/mlx5/mlx5_ifc.h> + +#include "fs_core.h" +#include "fs_cmd.h" +#include "fs_ft_pool.h" +#include "mlx5_core.h" +#include "eswitch.h" -#include <dev/mlx5/mlx5_core/fs_core.h> -#include <dev/mlx5/mlx5_core/mlx5_core.h> +static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + u32 underlay_qpn, + bool disconnect) +{ + return 0; +} -int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev, - enum fs_ft_type type, - unsigned int id) +static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_table_attr *ft_attr, + struct mlx5_flow_table *next_ft) { - u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {0}; - u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0}; + int max_fte = ft_attr->max_fte; - if (!dev) - return -EINVAL; + ft->max_fte = max_fte ? roundup_pow_of_two(max_fte) : 1; + + return 0; +} + +static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft) +{ + return 0; +} + +static int mlx5_cmd_stub_modify_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_table *next_ft) +{ + return 0; +} + +static int mlx5_cmd_stub_create_flow_group(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + u32 *in, + struct mlx5_flow_group *fg) +{ + return 0; +} + +static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg) +{ + return 0; +} + +static int mlx5_cmd_stub_create_fte(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *group, + struct fs_fte *fte) +{ + return 0; +} + +static int mlx5_cmd_stub_update_fte(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *group, + int modify_mask, + struct fs_fte *fte) +{ + return -EOPNOTSUPP; +} + +static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct fs_fte *fte) +{ + return 0; +} + +static int mlx5_cmd_stub_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, + struct mlx5_pkt_reformat_params *params, + enum mlx5_flow_namespace_type namespace, + struct mlx5_pkt_reformat *pkt_reformat) +{ + return 0; +} + +static void mlx5_cmd_stub_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns, + struct mlx5_pkt_reformat *pkt_reformat) +{ +} + +static int mlx5_cmd_stub_modify_header_alloc(struct mlx5_flow_root_namespace *ns, + u8 namespace, u8 num_actions, + void *modify_actions, + struct mlx5_modify_hdr *modify_hdr) +{ + return 0; +} + +static void mlx5_cmd_stub_modify_header_dealloc(struct mlx5_flow_root_namespace *ns, + struct mlx5_modify_hdr *modify_hdr) +{ +} + +static int mlx5_cmd_stub_set_peer(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_root_namespace *peer_ns) +{ + return 0; +} + +static int mlx5_cmd_stub_create_ns(struct mlx5_flow_root_namespace *ns) +{ + return 0; +} + +static int mlx5_cmd_stub_destroy_ns(struct mlx5_flow_root_namespace *ns) +{ + return 0; +} + +static u32 mlx5_cmd_stub_get_capabilities(struct mlx5_flow_root_namespace *ns, + enum fs_flow_table_type ft_type) +{ + return 0; +} + +static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, u32 underlay_qpn, + bool disconnect) +{ + u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + + if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) && + underlay_qpn == 0) + return 0; MLX5_SET(set_flow_table_root_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); - MLX5_SET(set_flow_table_root_in, in, table_type, type); - MLX5_SET(set_flow_table_root_in, in, table_id, id); + MLX5_SET(set_flow_table_root_in, in, table_type, ft->type); + + if (disconnect) + MLX5_SET(set_flow_table_root_in, in, op_mod, 1); + else + MLX5_SET(set_flow_table_root_in, in, table_id, ft->id); - return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn); + MLX5_SET(set_flow_table_root_in, in, vport_number, ft->vport); + MLX5_SET(set_flow_table_root_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + + return mlx5_cmd_exec_in(dev, set_flow_table_root, in); } -int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev, - u16 vport, enum fs_ft_type type, unsigned int level, - unsigned int log_size, const char *name, unsigned int *table_id) +static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_table_attr *ft_attr, + struct mlx5_flow_table *next_ft) { - u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {0}; - u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0}; + int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT); + int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); + int term = !!(ft->flags & MLX5_FLOW_TABLE_TERMINATION); + u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {}; + u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + unsigned int size; int err; - if (!dev) - return -EINVAL; + size = mlx5_ft_pool_get_avail_sz(dev, ft->type, ft_attr->max_fte); + if (!size) + return -ENOSPC; MLX5_SET(create_flow_table_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_TABLE); - MLX5_SET(create_flow_table_in, in, table_type, type); - MLX5_SET(create_flow_table_in, in, flow_table_context.level, level); - MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, - log_size); - if (strstr(name, FS_REFORMAT_KEYWORD) != NULL) - MLX5_SET(create_flow_table_in, in, - flow_table_context.reformat_en, 1); - if (vport) { - MLX5_SET(create_flow_table_in, in, vport_number, vport); - MLX5_SET(create_flow_table_in, in, other_vport, 1); + MLX5_SET(create_flow_table_in, in, uid, ft_attr->uid); + MLX5_SET(create_flow_table_in, in, table_type, ft->type); + MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level); + MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0); + MLX5_SET(create_flow_table_in, in, vport_number, ft->vport); + MLX5_SET(create_flow_table_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + + MLX5_SET(create_flow_table_in, in, flow_table_context.decap_en, + en_decap); + MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en, + en_encap); + MLX5_SET(create_flow_table_in, in, flow_table_context.termination_table, + term); + + switch (ft->op_mod) { + case FS_FT_OP_MOD_NORMAL: + if (next_ft) { + MLX5_SET(create_flow_table_in, in, + flow_table_context.table_miss_action, + MLX5_FLOW_TABLE_MISS_ACTION_FWD); + MLX5_SET(create_flow_table_in, in, + flow_table_context.table_miss_id, next_ft->id); + } else { + MLX5_SET(create_flow_table_in, in, + flow_table_context.table_miss_action, + ft->def_miss_action); + } + break; + + case FS_FT_OP_MOD_LAG_DEMUX: + MLX5_SET(create_flow_table_in, in, op_mod, 0x1); + if (next_ft) + MLX5_SET(create_flow_table_in, in, + flow_table_context.lag_master_next_table_id, + next_ft->id); + break; } - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (!err) - *table_id = MLX5_GET(create_flow_table_out, out, table_id); + err = mlx5_cmd_exec_inout(dev, create_flow_table, in, out); + if (!err) { + ft->id = MLX5_GET(create_flow_table_out, out, + table_id); + ft->max_fte = size; + } else { + mlx5_ft_pool_put_sz(ns->dev, size); + } return err; } -int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev, - u16 vport, - enum fs_ft_type type, unsigned int table_id) +static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft) { - u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {0}; - u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0}; - - if (!dev) - return -EINVAL; + u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + int err; MLX5_SET(destroy_flow_table_in, in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE); - MLX5_SET(destroy_flow_table_in, in, table_type, type); - MLX5_SET(destroy_flow_table_in, in, table_id, table_id); - if (vport) { - MLX5_SET(destroy_flow_table_in, in, vport_number, vport); - MLX5_SET(destroy_flow_table_in, in, other_vport, 1); + MLX5_SET(destroy_flow_table_in, in, table_type, ft->type); + MLX5_SET(destroy_flow_table_in, in, table_id, ft->id); + MLX5_SET(destroy_flow_table_in, in, vport_number, ft->vport); + MLX5_SET(destroy_flow_table_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + + err = mlx5_cmd_exec_in(dev, destroy_flow_table, in); + if (!err) + mlx5_ft_pool_put_sz(ns->dev, ft->max_fte); + + return err; +} + +static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_table *next_ft) +{ + u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + + MLX5_SET(modify_flow_table_in, in, opcode, + MLX5_CMD_OP_MODIFY_FLOW_TABLE); + MLX5_SET(modify_flow_table_in, in, table_type, ft->type); + MLX5_SET(modify_flow_table_in, in, table_id, ft->id); + + if (ft->op_mod == FS_FT_OP_MOD_LAG_DEMUX) { + MLX5_SET(modify_flow_table_in, in, modify_field_select, + MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID); + if (next_ft) { + MLX5_SET(modify_flow_table_in, in, + flow_table_context.lag_master_next_table_id, next_ft->id); + } else { + MLX5_SET(modify_flow_table_in, in, + flow_table_context.lag_master_next_table_id, 0); + } + } else { + MLX5_SET(modify_flow_table_in, in, vport_number, ft->vport); + MLX5_SET(modify_flow_table_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + MLX5_SET(modify_flow_table_in, in, modify_field_select, + MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID); + if (next_ft) { + MLX5_SET(modify_flow_table_in, in, + flow_table_context.table_miss_action, + MLX5_FLOW_TABLE_MISS_ACTION_FWD); + MLX5_SET(modify_flow_table_in, in, + flow_table_context.table_miss_id, + next_ft->id); + } else { + MLX5_SET(modify_flow_table_in, in, + flow_table_context.table_miss_action, + ft->def_miss_action); + } } - return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + return mlx5_cmd_exec_in(dev, modify_flow_table, in); } -int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev, - u32 *in, - u16 vport, - enum fs_ft_type type, unsigned int table_id, - unsigned int *group_id) +static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + u32 *in, + struct mlx5_flow_group *fg) { - u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0}; + u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {}; + struct mlx5_core_dev *dev = ns->dev; int err; - int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - if (!dev) - return -EINVAL; MLX5_SET(create_flow_group_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_GROUP); - MLX5_SET(create_flow_group_in, in, table_type, type); - MLX5_SET(create_flow_group_in, in, table_id, table_id); - if (vport) { - MLX5_SET(create_flow_group_in, in, vport_number, vport); - MLX5_SET(create_flow_group_in, in, other_vport, 1); - } - - err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); + MLX5_SET(create_flow_group_in, in, table_type, ft->type); + MLX5_SET(create_flow_group_in, in, table_id, ft->id); + MLX5_SET(create_flow_group_in, in, vport_number, ft->vport); + MLX5_SET(create_flow_group_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + err = mlx5_cmd_exec_inout(dev, create_flow_group, in, out); if (!err) - *group_id = MLX5_GET(create_flow_group_out, out, group_id); - + fg->id = MLX5_GET(create_flow_group_out, out, + group_id); return err; } -int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev, - u16 vport, - enum fs_ft_type type, unsigned int table_id, - unsigned int group_id) +static int mlx5_cmd_destroy_flow_group(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg) { - u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {0}; - u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0}; - - if (!dev) - return -EINVAL; + u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; MLX5_SET(destroy_flow_group_in, in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP); - MLX5_SET(destroy_flow_group_in, in, table_type, type); - MLX5_SET(destroy_flow_group_in, in, table_id, table_id); - MLX5_SET(destroy_flow_group_in, in, group_id, group_id); - if (vport) { - MLX5_SET(destroy_flow_group_in, in, vport_number, vport); - MLX5_SET(destroy_flow_group_in, in, other_vport, 1); - } - - return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + MLX5_SET(destroy_flow_group_in, in, table_type, ft->type); + MLX5_SET(destroy_flow_group_in, in, table_id, ft->id); + MLX5_SET(destroy_flow_group_in, in, group_id, fg->id); + MLX5_SET(destroy_flow_group_in, in, vport_number, ft->vport); + MLX5_SET(destroy_flow_group_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + return mlx5_cmd_exec_in(dev, destroy_flow_group, in); } -int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev, - u16 vport, - enum fs_fte_status *fte_status, - u32 *match_val, - enum fs_ft_type type, unsigned int table_id, - unsigned int index, unsigned int group_id, - struct mlx5_flow_act *flow_act, - u32 sw_action, int dest_size, - struct list_head *dests) /* mlx5_flow_desination */ +static int mlx5_set_extended_dest(struct mlx5_core_dev *dev, + struct fs_fte *fte, bool *extended_dest) { - u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0}; - u32 *in; - unsigned int inlen; + int fw_log_max_fdb_encap_uplink = + MLX5_CAP_ESW(dev, log_max_fdb_encap_uplink); + int num_fwd_destinations = 0; struct mlx5_flow_rule *dst; - void *in_flow_context; - void *in_match_value; - void *in_dests; - int err; - int opmod = 0; - int modify_mask = 0; - int atomic_mod_cap; - u32 prm_action = 0; - int count_list = 0; + int num_encap = 0; - if (sw_action != MLX5_FLOW_RULE_FWD_ACTION_DEST) - dest_size = 0; + *extended_dest = false; + if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)) + return 0; - if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_ALLOW) - prm_action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; + list_for_each_entry(dst, &fte->node.children, node.list) { + if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER || + dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_NONE) + continue; + if ((dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_VPORT || + dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) && + dst->dest_attr.vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID) + num_encap++; + num_fwd_destinations++; + } + if (num_fwd_destinations > 1 && num_encap > 0) + *extended_dest = true; - if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DROP) - prm_action |= MLX5_FLOW_CONTEXT_ACTION_DROP; + if (*extended_dest && !fw_log_max_fdb_encap_uplink) { + mlx5_core_warn(dev, "FW does not support extended destination"); + return -EOPNOTSUPP; + } + if (num_encap > (1 << fw_log_max_fdb_encap_uplink)) { + mlx5_core_warn(dev, "FW does not support more than %d encaps", + 1 << fw_log_max_fdb_encap_uplink); + return -EOPNOTSUPP; + } - if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DEST) - prm_action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + return 0; +} - if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) { - prm_action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; - count_list = 1; - } +static void +mlx5_cmd_set_fte_flow_meter(struct fs_fte *fte, void *in_flow_context) +{ + void *exe_aso_ctrl; + void *execute_aso; + + execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context, + execute_aso[0]); + MLX5_SET(execute_aso, execute_aso, valid, 1); + MLX5_SET(execute_aso, execute_aso, aso_object_id, + fte->action.exe_aso.object_id); + + exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id, + fte->action.exe_aso.return_reg_id); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type, + fte->action.exe_aso.type); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color, + fte->action.exe_aso.flow_meter.init_color); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id, + fte->action.exe_aso.flow_meter.meter_idx); +} - inlen = MLX5_ST_SZ_BYTES(set_fte_in) + - (dest_size + count_list) * MLX5_ST_SZ_BYTES(dest_format_struct); +static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, + int opmod, int modify_mask, + struct mlx5_flow_table *ft, + unsigned group_id, + struct fs_fte *fte) +{ + u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0}; + bool extended_dest = false; + struct mlx5_flow_rule *dst; + void *in_flow_context, *vlan; + void *in_match_value; + unsigned int inlen; + int dst_cnt_size; + void *in_dests; + u32 *in; + int err; - if (!dev) - return -EINVAL; + if (mlx5_set_extended_dest(dev, fte, &extended_dest)) + return -EOPNOTSUPP; - if (*fte_status & FS_FTE_STATUS_EXISTING) { - atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev, - flow_table_properties_nic_receive. - flow_modify_en); - if (!atomic_mod_cap) - return -ENOTSUPP; - opmod = 1; - modify_mask = 1 << - MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST; - } + if (!extended_dest) + dst_cnt_size = MLX5_ST_SZ_BYTES(dest_format_struct); + else + dst_cnt_size = MLX5_ST_SZ_BYTES(extended_dest_format); - in = mlx5_vzalloc(inlen); - if (!in) { - mlx5_core_warn(dev, "failed to allocate inbox\n"); + inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fte->dests_size * dst_cnt_size; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) return -ENOMEM; - } MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY); MLX5_SET(set_fte_in, in, op_mod, opmod); MLX5_SET(set_fte_in, in, modify_enable_mask, modify_mask); - MLX5_SET(set_fte_in, in, table_type, type); - MLX5_SET(set_fte_in, in, table_id, table_id); - MLX5_SET(set_fte_in, in, flow_index, index); - if (vport) { - MLX5_SET(set_fte_in, in, vport_number, vport); - MLX5_SET(set_fte_in, in, other_vport, 1); - } + MLX5_SET(set_fte_in, in, table_type, ft->type); + MLX5_SET(set_fte_in, in, table_id, ft->id); + MLX5_SET(set_fte_in, in, flow_index, fte->index); + MLX5_SET(set_fte_in, in, ignore_flow_level, + !!(fte->action.flags & FLOW_ACT_IGNORE_FLOW_LEVEL)); + + MLX5_SET(set_fte_in, in, vport_number, ft->vport); + MLX5_SET(set_fte_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context); MLX5_SET(flow_context, in_flow_context, group_id, group_id); - if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG) - MLX5_SET(flow_context, in_flow_context, flow_tag, flow_act->flow_tag); - if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR) { - MLX5_SET(flow_context, in_flow_context, modify_header_id, - flow_act->modify_hdr->id); - prm_action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; - } - if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT) { - MLX5_SET(flow_context, in_flow_context, packet_reformat_id, - flow_act->pkt_reformat->id); - prm_action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + + MLX5_SET(flow_context, in_flow_context, flow_tag, + fte->flow_context.flow_tag); + MLX5_SET(flow_context, in_flow_context, flow_source, + fte->flow_context.flow_source); + + MLX5_SET(flow_context, in_flow_context, extended_destination, + extended_dest); + if (extended_dest) { + u32 action; + + action = fte->action.action & + ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + MLX5_SET(flow_context, in_flow_context, action, action); + } else { + MLX5_SET(flow_context, in_flow_context, action, + fte->action.action); + if (fte->action.pkt_reformat) + MLX5_SET(flow_context, in_flow_context, packet_reformat_id, + fte->action.pkt_reformat->id); } - MLX5_SET(flow_context, in_flow_context, destination_list_size, - dest_size); + if (fte->action.modify_hdr) + MLX5_SET(flow_context, in_flow_context, modify_header_id, + fte->action.modify_hdr->id); + + MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_type, + fte->action.crypto.type); + MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_obj_id, + fte->action.crypto.obj_id); + + vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan); + + MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[0].ethtype); + MLX5_SET(vlan, vlan, vid, fte->action.vlan[0].vid); + MLX5_SET(vlan, vlan, prio, fte->action.vlan[0].prio); + + vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan_2); + + MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[1].ethtype); + MLX5_SET(vlan, vlan, vid, fte->action.vlan[1].vid); + MLX5_SET(vlan, vlan, prio, fte->action.vlan[1].prio); + in_match_value = MLX5_ADDR_OF(flow_context, in_flow_context, match_value); - memcpy(in_match_value, match_val, MLX5_ST_SZ_BYTES(fte_match_param)); + memcpy(in_match_value, &fte->val, sizeof(fte->val)); in_dests = MLX5_ADDR_OF(flow_context, in_flow_context, destination); + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { + int list_size = 0; - if (dest_size) { - list_for_each_entry(dst, dests, base.list) { + list_for_each_entry(dst, &fte->node.children, node.list) { + enum mlx5_flow_destination_type type = dst->dest_attr.type; + enum mlx5_ifc_flow_destination_type ifc_type; unsigned int id; - MLX5_SET(dest_format_struct, in_dests, destination_type, - dst->dest_attr.type); - if (dst->dest_attr.type == - MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE) + if (type == MLX5_FLOW_DESTINATION_TYPE_COUNTER) + continue; + + switch (type) { + case MLX5_FLOW_DESTINATION_TYPE_NONE: + continue; + case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM: + id = dst->dest_attr.ft_num; + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE; + break; + case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: id = dst->dest_attr.ft->id; - else + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE; + break; + case MLX5_FLOW_DESTINATION_TYPE_UPLINK: + case MLX5_FLOW_DESTINATION_TYPE_VPORT: + MLX5_SET(dest_format_struct, in_dests, + destination_eswitch_owner_vhca_id_valid, + !!(dst->dest_attr.vport.flags & + MLX5_FLOW_DEST_VPORT_VHCA_ID)); + MLX5_SET(dest_format_struct, in_dests, + destination_eswitch_owner_vhca_id, + dst->dest_attr.vport.vhca_id); + if (type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) { + /* destination_id is reserved */ + id = 0; + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK; + break; + } + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT; + id = dst->dest_attr.vport.num; + if (extended_dest && + dst->dest_attr.vport.pkt_reformat) { + MLX5_SET(dest_format_struct, in_dests, + packet_reformat, + !!(dst->dest_attr.vport.flags & + MLX5_FLOW_DEST_VPORT_REFORMAT_ID)); + MLX5_SET(extended_dest_format, in_dests, + packet_reformat_id, + dst->dest_attr.vport.pkt_reformat->id); + } + break; + case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER: + id = dst->dest_attr.sampler_id; + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; + break; + case MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE: + MLX5_SET(dest_format_struct, in_dests, + destination_table_type, dst->dest_attr.ft->type); + id = dst->dest_attr.ft->id; + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE; + break; + default: id = dst->dest_attr.tir_num; + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TIR; + } + + MLX5_SET(dest_format_struct, in_dests, destination_type, + ifc_type); MLX5_SET(dest_format_struct, in_dests, destination_id, id); - in_dests += MLX5_ST_SZ_BYTES(dest_format_struct); + in_dests += dst_cnt_size; + list_size++; } + + MLX5_SET(flow_context, in_flow_context, destination_list_size, + list_size); } - if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) { - MLX5_SET(dest_format_struct, in_dests, destination_id, - mlx5_fc_id(flow_act->counter)); - in_dests += MLX5_ST_SZ_BYTES(dest_format_struct); - MLX5_SET(flow_context, in_flow_context, flow_counter_list_size, 1); + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { + int max_list_size = BIT(MLX5_CAP_FLOWTABLE_TYPE(dev, + log_max_flow_counter, + ft->type)); + int list_size = 0; + + list_for_each_entry(dst, &fte->node.children, node.list) { + if (dst->dest_attr.type != + MLX5_FLOW_DESTINATION_TYPE_COUNTER) + continue; + + MLX5_SET(flow_counter_list, in_dests, flow_counter_id, + dst->dest_attr.counter_id); + in_dests += dst_cnt_size; + list_size++; + } + if (list_size > max_list_size) { + err = -EINVAL; + goto err_out; + } + + MLX5_SET(flow_context, in_flow_context, flow_counter_list_size, + list_size); } - MLX5_SET(flow_context, in_flow_context, action, prm_action); - err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); - if (!err) - *fte_status |= FS_FTE_STATUS_EXISTING; + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) { + if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) { + mlx5_cmd_set_fte_flow_meter(fte, in_flow_context); + } else { + err = -EOPNOTSUPP; + goto err_out; + } + } + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); +err_out: kvfree(in); + return err; +} + +static int mlx5_cmd_create_fte(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *group, + struct fs_fte *fte) +{ + struct mlx5_core_dev *dev = ns->dev; + unsigned int group_id = group->id; + + return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte); +} + +static int mlx5_cmd_update_fte(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg, + int modify_mask, + struct fs_fte *fte) +{ + int opmod; + struct mlx5_core_dev *dev = ns->dev; + int atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev, + flow_table_properties_nic_receive. + flow_modify_en); + if (!atomic_mod_cap) + return -EOPNOTSUPP; + opmod = 1; + + return mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, fg->id, fte); +} + +static int mlx5_cmd_delete_fte(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + struct fs_fte *fte) +{ + u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); + MLX5_SET(delete_fte_in, in, table_type, ft->type); + MLX5_SET(delete_fte_in, in, table_id, ft->id); + MLX5_SET(delete_fte_in, in, flow_index, fte->index); + MLX5_SET(delete_fte_in, in, vport_number, ft->vport); + MLX5_SET(delete_fte_in, in, other_vport, + !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); + + return mlx5_cmd_exec_in(dev, delete_fte, in); +} + +int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, + enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask, + u32 *id) +{ + u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {}; + u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {}; + int err; + + MLX5_SET(alloc_flow_counter_in, in, opcode, + MLX5_CMD_OP_ALLOC_FLOW_COUNTER); + MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask); + + err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out); + if (!err) + *id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id); return err; } -int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev, - u16 vport, - enum fs_fte_status *fte_status, - enum fs_ft_type type, unsigned int table_id, - unsigned int index) +int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id) +{ + return mlx5_cmd_fc_bulk_alloc(dev, 0, id); +} + +int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id) +{ + u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {}; + + MLX5_SET(dealloc_flow_counter_in, in, opcode, + MLX5_CMD_OP_DEALLOC_FLOW_COUNTER); + MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id); + return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in); +} + +int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id, + u64 *packets, u64 *bytes) +{ + u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) + + MLX5_ST_SZ_BYTES(traffic_counter)] = {}; + u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {}; + void *stats; + int err = 0; + + MLX5_SET(query_flow_counter_in, in, opcode, + MLX5_CMD_OP_QUERY_FLOW_COUNTER); + MLX5_SET(query_flow_counter_in, in, op_mod, 0); + MLX5_SET(query_flow_counter_in, in, flow_counter_id, id); + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics); + *packets = MLX5_GET64(traffic_counter, stats, packets); + *bytes = MLX5_GET64(traffic_counter, stats, octets); + return 0; +} + +int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len) { - u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {0}; - u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0}; + return MLX5_ST_SZ_BYTES(query_flow_counter_out) + + MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len; +} + +int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len, + u32 *out) +{ + int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len); + u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {}; + + MLX5_SET(query_flow_counter_in, in, opcode, + MLX5_CMD_OP_QUERY_FLOW_COUNTER); + MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id); + MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len); + return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); +} + +static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, + struct mlx5_pkt_reformat_params *params, + enum mlx5_flow_namespace_type namespace, + struct mlx5_pkt_reformat *pkt_reformat) +{ + u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {}; + struct mlx5_core_dev *dev = ns->dev; + void *packet_reformat_context_in; + int max_encap_size; + void *reformat; + int inlen; int err; + u32 *in; - if (!(*fte_status & FS_FTE_STATUS_EXISTING)) - return 0; + if (namespace == MLX5_FLOW_NAMESPACE_FDB || + namespace == MLX5_FLOW_NAMESPACE_FDB_BYPASS) + max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size); + else + max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size); - if (!dev) + if (params->size > max_encap_size) { + mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n", + params->size, max_encap_size); return -EINVAL; + } - MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); - MLX5_SET(delete_fte_in, in, table_type, type); - MLX5_SET(delete_fte_in, in, table_id, table_id); - MLX5_SET(delete_fte_in, in, flow_index, index); - if (vport) { - MLX5_SET(delete_fte_in, in, vport_number, vport); - MLX5_SET(delete_fte_in, in, other_vport, 1); + in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) + + params->size, GFP_KERNEL); + if (!in) + return -ENOMEM; + + packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in, + in, packet_reformat_context); + reformat = MLX5_ADDR_OF(packet_reformat_context_in, + packet_reformat_context_in, + reformat_data); + inlen = reformat - (void *)in + params->size; + + MLX5_SET(alloc_packet_reformat_context_in, in, opcode, + MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT); + MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, + reformat_data_size, params->size); + MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, + reformat_type, params->type); + MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, + reformat_param_0, params->param_0); + MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, + reformat_param_1, params->param_1); + if (params->data && params->size) + memcpy(reformat, params->data, params->size); + + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); + + pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out, + out, packet_reformat_id); + kfree(in); + return err; +} + +static void mlx5_cmd_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns, + struct mlx5_pkt_reformat *pkt_reformat) +{ + u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + + MLX5_SET(dealloc_packet_reformat_context_in, in, opcode, + MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT); + MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id, + pkt_reformat->id); + + mlx5_cmd_exec_in(dev, dealloc_packet_reformat_context, in); +} + +static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns, + u8 namespace, u8 num_actions, + void *modify_actions, + struct mlx5_modify_hdr *modify_hdr) +{ + u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {}; + int max_actions, actions_size, inlen, err; + struct mlx5_core_dev *dev = ns->dev; + void *actions_in; + u8 table_type; + u32 *in; + + switch (namespace) { + case MLX5_FLOW_NAMESPACE_FDB: + case MLX5_FLOW_NAMESPACE_FDB_BYPASS: + max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions); + table_type = FS_FT_FDB; + break; + case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC: + case MLX5_FLOW_NAMESPACE_KERNEL: + case MLX5_FLOW_NAMESPACE_BYPASS: + max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions); + table_type = FS_FT_NIC_RX; + break; + case MLX5_FLOW_NAMESPACE_EGRESS: + case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC: + case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC: + max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions); + table_type = FS_FT_NIC_TX; + break; + case MLX5_FLOW_NAMESPACE_ESW_INGRESS: + max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions); + table_type = FS_FT_ESW_INGRESS_ACL; + break; + case MLX5_FLOW_NAMESPACE_RDMA_TX: + max_actions = MLX5_CAP_FLOWTABLE_RDMA_TX(dev, max_modify_header_actions); + table_type = FS_FT_RDMA_TX; + break; + default: + return -EOPNOTSUPP; } - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (!err) - *fte_status = 0; + if (num_actions > max_actions) { + mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n", + num_actions, max_actions); + return -EOPNOTSUPP; + } + + actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions; + inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size; + + in = kzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + MLX5_SET(alloc_modify_header_context_in, in, opcode, + MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT); + MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type); + MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions); + + actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions); + memcpy(actions_in, modify_actions, actions_size); + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); + + modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id); + kfree(in); return err; } -int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev, - enum mlx5_flow_namespace_type namespace, - u8 num_actions, - void *modify_actions, - struct mlx5_modify_hdr *modify_hdr) -{ - u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {}; - int max_actions, actions_size, inlen, err; - void *actions_in; - u8 table_type; - u32 *in; - - switch (namespace) { - case MLX5_FLOW_NAMESPACE_FDB: - max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions); - table_type = FS_FT_FDB; - break; - case MLX5_FLOW_NAMESPACE_KERNEL: - case MLX5_FLOW_NAMESPACE_BYPASS: - max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions); - table_type = FS_FT_NIC_RX; - break; - case MLX5_FLOW_NAMESPACE_ESW_INGRESS: - max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions); - table_type = FS_FT_ESW_INGRESS_ACL; - break; - default: - return -EOPNOTSUPP; - } - - if (num_actions > max_actions) { - mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n", - num_actions, max_actions); - return -EOPNOTSUPP; - } - - actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions; - inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size; - - in = kzalloc(inlen, GFP_KERNEL); - if (!in) - return -ENOMEM; - - MLX5_SET(alloc_modify_header_context_in, in, opcode, - MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT); - MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type); - MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions); - - actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions); - memcpy(actions_in, modify_actions, actions_size); - - err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); - - modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id); - kfree(in); - - return err; -} - -void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev, - struct mlx5_modify_hdr *modify_hdr) -{ - u32 out[MLX5_ST_SZ_DW(dealloc_modify_header_context_out)] = {}; - u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {}; - - MLX5_SET(dealloc_modify_header_context_in, in, opcode, - MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT); - MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id, - modify_hdr->id); - - mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); -} - -int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev, - struct mlx5_pkt_reformat_params *params, - enum mlx5_flow_namespace_type namespace, - struct mlx5_pkt_reformat *pkt_reformat) -{ - u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {}; - void *packet_reformat_context_in; - int max_encap_size; - void *reformat; - int inlen; - int err; - u32 *in; - - if (namespace == MLX5_FLOW_NAMESPACE_FDB) - max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size); - else - max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size); - - if (params->size > max_encap_size) { - mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n", - params->size, max_encap_size); - return -EINVAL; - } - - in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) + - params->size, GFP_KERNEL); - if (!in) - return -ENOMEM; - - packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in, - in, packet_reformat_context); - reformat = MLX5_ADDR_OF(packet_reformat_context_in, - packet_reformat_context_in, - reformat_data); - inlen = reformat - (void *)in + params->size; - - MLX5_SET(alloc_packet_reformat_context_in, in, opcode, - MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT); - MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, - reformat_data_size, params->size); - MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, - reformat_type, params->type); - MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, - reformat_param_0, params->param_0); - MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, - reformat_param_1, params->param_1); - if (params->data && params->size) - memcpy(reformat, params->data, params->size); - - err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); - - pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out, - out, packet_reformat_id); - kfree(in); +static void mlx5_cmd_modify_header_dealloc(struct mlx5_flow_root_namespace *ns, + struct mlx5_modify_hdr *modify_hdr) +{ + u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {}; + struct mlx5_core_dev *dev = ns->dev; + + MLX5_SET(dealloc_modify_header_context_in, in, opcode, + MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT); + MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id, + modify_hdr->id); - return err; + mlx5_cmd_exec_in(dev, dealloc_modify_header_context, in); } -void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev, - struct mlx5_pkt_reformat *pkt_reformat) +static u32 mlx5_cmd_get_capabilities(struct mlx5_flow_root_namespace *ns, + enum fs_flow_table_type ft_type) { - u32 out[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_out)] = {}; - u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {}; + return 0; +} - MLX5_SET(dealloc_packet_reformat_context_in, in, opcode, - MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT); - MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id, - pkt_reformat->id); +static const struct mlx5_flow_cmds mlx5_flow_cmds = { + .create_flow_table = mlx5_cmd_create_flow_table, + .destroy_flow_table = mlx5_cmd_destroy_flow_table, + .modify_flow_table = mlx5_cmd_modify_flow_table, + .create_flow_group = mlx5_cmd_create_flow_group, + .destroy_flow_group = mlx5_cmd_destroy_flow_group, + .create_fte = mlx5_cmd_create_fte, + .update_fte = mlx5_cmd_update_fte, + .delete_fte = mlx5_cmd_delete_fte, + .update_root_ft = mlx5_cmd_update_root_ft, + .packet_reformat_alloc = mlx5_cmd_packet_reformat_alloc, + .packet_reformat_dealloc = mlx5_cmd_packet_reformat_dealloc, + .modify_header_alloc = mlx5_cmd_modify_header_alloc, + .modify_header_dealloc = mlx5_cmd_modify_header_dealloc, + .set_peer = mlx5_cmd_stub_set_peer, + .create_ns = mlx5_cmd_stub_create_ns, + .destroy_ns = mlx5_cmd_stub_destroy_ns, + .get_capabilities = mlx5_cmd_get_capabilities, +}; + +static const struct mlx5_flow_cmds mlx5_flow_cmd_stubs = { + .create_flow_table = mlx5_cmd_stub_create_flow_table, + .destroy_flow_table = mlx5_cmd_stub_destroy_flow_table, + .modify_flow_table = mlx5_cmd_stub_modify_flow_table, + .create_flow_group = mlx5_cmd_stub_create_flow_group, + .destroy_flow_group = mlx5_cmd_stub_destroy_flow_group, + .create_fte = mlx5_cmd_stub_create_fte, + .update_fte = mlx5_cmd_stub_update_fte, + .delete_fte = mlx5_cmd_stub_delete_fte, + .update_root_ft = mlx5_cmd_stub_update_root_ft, + .packet_reformat_alloc = mlx5_cmd_stub_packet_reformat_alloc, + .packet_reformat_dealloc = mlx5_cmd_stub_packet_reformat_dealloc, + .modify_header_alloc = mlx5_cmd_stub_modify_header_alloc, + .modify_header_dealloc = mlx5_cmd_stub_modify_header_dealloc, + .set_peer = mlx5_cmd_stub_set_peer, + .create_ns = mlx5_cmd_stub_create_ns, + .destroy_ns = mlx5_cmd_stub_destroy_ns, + .get_capabilities = mlx5_cmd_stub_get_capabilities, +}; + +const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void) +{ + return &mlx5_flow_cmds; +} + +static const struct mlx5_flow_cmds *mlx5_fs_cmd_get_stub_cmds(void) +{ + return &mlx5_flow_cmd_stubs; +} - mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type) +{ + switch (type) { + case FS_FT_NIC_RX: + case FS_FT_ESW_EGRESS_ACL: + case FS_FT_ESW_INGRESS_ACL: + case FS_FT_FDB: + case FS_FT_SNIFFER_RX: + case FS_FT_SNIFFER_TX: + case FS_FT_NIC_TX: + case FS_FT_RDMA_RX: + case FS_FT_RDMA_TX: + case FS_FT_PORT_SEL: + return mlx5_fs_cmd_get_fw_cmds(); + default: + return mlx5_fs_cmd_get_stub_cmds(); + } } diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c new file mode 100644 index 000000000000..8d93d4740462 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c @@ -0,0 +1,3522 @@ +/* + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/module.h> +#include <dev/mlx5/driver.h> +#include <dev/mlx5/mlx5_core/mlx5_core.h> +#include <dev/mlx5/mlx5_core/fs_core.h> +#include <linux/string.h> +#include <linux/compiler.h> + +#include "eswitch.h" +#include "fs_ft_pool.h" +#include "fs_cmd.h" + +#define down_write_nested(a, b) down_write(a) + +#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ + sizeof(struct init_tree_node)) + +#define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\ + ...) {.type = FS_TYPE_PRIO,\ + .min_ft_level = min_level_val,\ + .num_levels = num_levels_val,\ + .num_leaf_prios = num_prios_val,\ + .caps = caps_val,\ + .children = (struct init_tree_node[]) {__VA_ARGS__},\ + .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ +} + +#define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\ + ADD_PRIO(num_prios_val, 0, num_levels_val, {},\ + __VA_ARGS__)\ + +#define ADD_NS(def_miss_act, ...) {.type = FS_TYPE_NAMESPACE, \ + .def_miss_action = def_miss_act,\ + .children = (struct init_tree_node[]) {__VA_ARGS__},\ + .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ +} + +#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ + sizeof(long)) + +#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) + +#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ + .caps = (long[]) {__VA_ARGS__} } + +#define FS_CHAINING_CAPS FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \ + FS_CAP(flow_table_properties_nic_receive.modify_root), \ + FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \ + FS_CAP(flow_table_properties_nic_receive.flow_table_modify)) + +#define FS_CHAINING_CAPS_EGRESS \ + FS_REQUIRED_CAPS( \ + FS_CAP(flow_table_properties_nic_transmit.flow_modify_en), \ + FS_CAP(flow_table_properties_nic_transmit.modify_root), \ + FS_CAP(flow_table_properties_nic_transmit \ + .identified_miss_table_mode), \ + FS_CAP(flow_table_properties_nic_transmit.flow_table_modify)) + +#define FS_CHAINING_CAPS_RDMA_TX \ + FS_REQUIRED_CAPS( \ + FS_CAP(flow_table_properties_nic_transmit_rdma.flow_modify_en), \ + FS_CAP(flow_table_properties_nic_transmit_rdma.modify_root), \ + FS_CAP(flow_table_properties_nic_transmit_rdma \ + .identified_miss_table_mode), \ + FS_CAP(flow_table_properties_nic_transmit_rdma \ + .flow_table_modify)) + +#define LEFTOVERS_NUM_LEVELS 1 +#define LEFTOVERS_NUM_PRIOS 1 + +#define RDMA_RX_COUNTERS_PRIO_NUM_LEVELS 1 +#define RDMA_TX_COUNTERS_PRIO_NUM_LEVELS 1 + +#define BY_PASS_PRIO_NUM_LEVELS 1 +#define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\ + LEFTOVERS_NUM_PRIOS) + +#define KERNEL_RX_MACSEC_NUM_PRIOS 1 +#define KERNEL_RX_MACSEC_NUM_LEVELS 2 +#define KERNEL_RX_MACSEC_MIN_LEVEL (BY_PASS_MIN_LEVEL + KERNEL_RX_MACSEC_NUM_PRIOS) + +#define ETHTOOL_PRIO_NUM_LEVELS 1 +#define ETHTOOL_NUM_PRIOS 11 +#define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS) +/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}}, IPsec policy, + * IPsec RoCE policy + */ +#define KERNEL_NIC_PRIO_NUM_LEVELS 9 +#define KERNEL_NIC_NUM_PRIOS 1 +/* One more level for tc */ +#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1) + +#define KERNEL_NIC_TC_NUM_PRIOS 1 +#define KERNEL_NIC_TC_NUM_LEVELS 13 + +#define ANCHOR_NUM_LEVELS 1 +#define ANCHOR_NUM_PRIOS 1 +#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1) + +#define OFFLOADS_MAX_FT 2 +#define OFFLOADS_NUM_PRIOS 1 +#define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + OFFLOADS_NUM_PRIOS) + +#define LAG_PRIO_NUM_LEVELS 1 +#define LAG_NUM_PRIOS 1 +#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1) + +#define KERNEL_TX_IPSEC_NUM_PRIOS 1 +#define KERNEL_TX_IPSEC_NUM_LEVELS 3 +#define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS) + +#define KERNEL_TX_MACSEC_NUM_PRIOS 1 +#define KERNEL_TX_MACSEC_NUM_LEVELS 2 +#define KERNEL_TX_MACSEC_MIN_LEVEL (KERNEL_TX_IPSEC_MIN_LEVEL + KERNEL_TX_MACSEC_NUM_PRIOS) + +#define MAX_VPORTS 128 + +struct node_caps { + size_t arr_sz; + long *caps; +}; + +static struct init_tree_node { + enum fs_node_type type; + struct init_tree_node *children; + int ar_size; + struct node_caps caps; + int min_ft_level; + int num_leaf_prios; + int prio; + int num_levels; + enum mlx5_flow_table_miss_action def_miss_action; +} root_fs = { + .type = FS_TYPE_NAMESPACE, + .ar_size = 8, + .children = (struct init_tree_node[]){ + ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS, + BY_PASS_PRIO_NUM_LEVELS))), + ADD_PRIO(0, KERNEL_RX_MACSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(KERNEL_RX_MACSEC_NUM_PRIOS, + KERNEL_RX_MACSEC_NUM_LEVELS))), + ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS, + LAG_PRIO_NUM_LEVELS))), + ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS, + OFFLOADS_MAX_FT))), + ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS, + ETHTOOL_PRIO_NUM_LEVELS))), + ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {}, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS, + KERNEL_NIC_TC_NUM_LEVELS), + ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS, + KERNEL_NIC_PRIO_NUM_LEVELS))), + ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS, + LEFTOVERS_NUM_LEVELS))), + ADD_PRIO(0, ANCHOR_MIN_LEVEL, 0, {}, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS, + ANCHOR_NUM_LEVELS))), + } +}; + +static struct init_tree_node egress_root_fs = { + .type = FS_TYPE_NAMESPACE, + .ar_size = 3, + .children = (struct init_tree_node[]) { + ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0, + FS_CHAINING_CAPS_EGRESS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS, + BY_PASS_PRIO_NUM_LEVELS))), + ADD_PRIO(0, KERNEL_TX_IPSEC_MIN_LEVEL, 0, + FS_CHAINING_CAPS_EGRESS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(KERNEL_TX_IPSEC_NUM_PRIOS, + KERNEL_TX_IPSEC_NUM_LEVELS))), + ADD_PRIO(0, KERNEL_TX_MACSEC_MIN_LEVEL, 0, + FS_CHAINING_CAPS_EGRESS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(KERNEL_TX_MACSEC_NUM_PRIOS, + KERNEL_TX_MACSEC_NUM_LEVELS))), + } +}; + +enum { + RDMA_RX_IPSEC_PRIO, + RDMA_RX_COUNTERS_PRIO, + RDMA_RX_BYPASS_PRIO, + RDMA_RX_KERNEL_PRIO, +}; + +#define RDMA_RX_IPSEC_NUM_PRIOS 1 +#define RDMA_RX_IPSEC_NUM_LEVELS 2 +#define RDMA_RX_IPSEC_MIN_LEVEL (RDMA_RX_IPSEC_NUM_LEVELS) + +#define RDMA_RX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_REGULAR_PRIOS +#define RDMA_RX_KERNEL_MIN_LEVEL (RDMA_RX_BYPASS_MIN_LEVEL + 1) +#define RDMA_RX_COUNTERS_MIN_LEVEL (RDMA_RX_KERNEL_MIN_LEVEL + 2) + +static struct init_tree_node rdma_rx_root_fs = { + .type = FS_TYPE_NAMESPACE, + .ar_size = 4, + .children = (struct init_tree_node[]) { + [RDMA_RX_IPSEC_PRIO] = + ADD_PRIO(0, RDMA_RX_IPSEC_MIN_LEVEL, 0, + FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(RDMA_RX_IPSEC_NUM_PRIOS, + RDMA_RX_IPSEC_NUM_LEVELS))), + [RDMA_RX_COUNTERS_PRIO] = + ADD_PRIO(0, RDMA_RX_COUNTERS_MIN_LEVEL, 0, + FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(MLX5_RDMA_RX_NUM_COUNTERS_PRIOS, + RDMA_RX_COUNTERS_PRIO_NUM_LEVELS))), + [RDMA_RX_BYPASS_PRIO] = + ADD_PRIO(0, RDMA_RX_BYPASS_MIN_LEVEL, 0, + FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS, + BY_PASS_PRIO_NUM_LEVELS))), + [RDMA_RX_KERNEL_PRIO] = + ADD_PRIO(0, RDMA_RX_KERNEL_MIN_LEVEL, 0, + FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN, + ADD_MULTIPLE_PRIO(1, 1))), + } +}; + +enum { + RDMA_TX_COUNTERS_PRIO, + RDMA_TX_IPSEC_PRIO, + RDMA_TX_BYPASS_PRIO, +}; + +#define RDMA_TX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_PRIOS +#define RDMA_TX_COUNTERS_MIN_LEVEL (RDMA_TX_BYPASS_MIN_LEVEL + 1) + +#define RDMA_TX_IPSEC_NUM_PRIOS 1 +#define RDMA_TX_IPSEC_PRIO_NUM_LEVELS 1 +#define RDMA_TX_IPSEC_MIN_LEVEL (RDMA_TX_COUNTERS_MIN_LEVEL + RDMA_TX_IPSEC_NUM_PRIOS) + +static struct init_tree_node rdma_tx_root_fs = { + .type = FS_TYPE_NAMESPACE, + .ar_size = 3, + .children = (struct init_tree_node[]) { + [RDMA_TX_COUNTERS_PRIO] = + ADD_PRIO(0, RDMA_TX_COUNTERS_MIN_LEVEL, 0, + FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(MLX5_RDMA_TX_NUM_COUNTERS_PRIOS, + RDMA_TX_COUNTERS_PRIO_NUM_LEVELS))), + [RDMA_TX_IPSEC_PRIO] = + ADD_PRIO(0, RDMA_TX_IPSEC_MIN_LEVEL, 0, + FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(RDMA_TX_IPSEC_NUM_PRIOS, + RDMA_TX_IPSEC_PRIO_NUM_LEVELS))), + + [RDMA_TX_BYPASS_PRIO] = + ADD_PRIO(0, RDMA_TX_BYPASS_MIN_LEVEL, 0, + FS_CHAINING_CAPS_RDMA_TX, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(RDMA_TX_BYPASS_MIN_LEVEL, + BY_PASS_PRIO_NUM_LEVELS))), + } +}; + +enum fs_i_lock_class { + FS_LOCK_GRANDPARENT, + FS_LOCK_PARENT, + FS_LOCK_CHILD +}; + +static void del_hw_flow_table(struct fs_node *node); +static void del_hw_flow_group(struct fs_node *node); +static void del_hw_fte(struct fs_node *node); +static void del_sw_flow_table(struct fs_node *node); +static void del_sw_flow_group(struct fs_node *node); +static void del_sw_fte(struct fs_node *node); +static void del_sw_prio(struct fs_node *node); +static void del_sw_ns(struct fs_node *node); +/* Delete rule (destination) is special case that + * requires to lock the FTE for all the deletion process. + */ +static void del_sw_hw_rule(struct fs_node *node); +static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1, + struct mlx5_flow_destination *d2); +static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns); +static struct mlx5_flow_rule * +find_flow_rule(struct fs_fte *fte, + struct mlx5_flow_destination *dest); + +static void tree_init_node(struct fs_node *node, + void (*del_hw_func)(struct fs_node *), + void (*del_sw_func)(struct fs_node *)) +{ + refcount_set(&node->refcount, 1); + INIT_LIST_HEAD(&node->list); + INIT_LIST_HEAD(&node->children); + init_rwsem(&node->lock); + node->del_hw_func = del_hw_func; + node->del_sw_func = del_sw_func; + node->active = false; +} + +static void tree_add_node(struct fs_node *node, struct fs_node *parent) +{ + if (parent) + refcount_inc(&parent->refcount); + node->parent = parent; + + /* Parent is the root */ + if (!parent) + node->root = node; + else + node->root = parent->root; +} + +static int tree_get_node(struct fs_node *node) +{ + return refcount_inc_not_zero(&node->refcount); +} + +static void nested_down_read_ref_node(struct fs_node *node, + enum fs_i_lock_class class) +{ + if (node) { + down_read_nested(&node->lock, class); + refcount_inc(&node->refcount); + } +} + +static void nested_down_write_ref_node(struct fs_node *node, + enum fs_i_lock_class class) +{ + if (node) { + down_write_nested(&node->lock, class); + refcount_inc(&node->refcount); + } +} + +static void down_write_ref_node(struct fs_node *node, bool locked) +{ + if (node) { + if (!locked) + down_write(&node->lock); + refcount_inc(&node->refcount); + } +} + +static void up_read_ref_node(struct fs_node *node) +{ + refcount_dec(&node->refcount); + up_read(&node->lock); +} + +static void up_write_ref_node(struct fs_node *node, bool locked) +{ + refcount_dec(&node->refcount); + if (!locked) + up_write(&node->lock); +} + +static void tree_put_node(struct fs_node *node, bool locked) +{ + struct fs_node *parent_node = node->parent; + + if (refcount_dec_and_test(&node->refcount)) { + if (node->del_hw_func) + node->del_hw_func(node); + if (parent_node) { + down_write_ref_node(parent_node, locked); + list_del_init(&node->list); + } + node->del_sw_func(node); + if (parent_node) + up_write_ref_node(parent_node, locked); + node = NULL; + } + if (!node && parent_node) + tree_put_node(parent_node, locked); +} + +static int tree_remove_node(struct fs_node *node, bool locked) +{ + if (refcount_read(&node->refcount) > 1) { + refcount_dec(&node->refcount); + return -EEXIST; + } + tree_put_node(node, locked); + return 0; +} + +static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, + unsigned int prio) +{ + struct fs_prio *iter_prio; + + fs_for_each_prio(iter_prio, ns) { + if (iter_prio->prio == prio) + return iter_prio; + } + + return NULL; +} + +static bool is_fwd_next_action(u32 action) +{ + return action & (MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO | + MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS); +} + +static bool is_fwd_dest_type(enum mlx5_flow_destination_type type) +{ + return type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM || + type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE || + type == MLX5_FLOW_DESTINATION_TYPE_UPLINK || + type == MLX5_FLOW_DESTINATION_TYPE_VPORT || + type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER || + type == MLX5_FLOW_DESTINATION_TYPE_TIR || + type == MLX5_FLOW_DESTINATION_TYPE_RANGE || + type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; +} + +static bool check_valid_spec(const struct mlx5_flow_spec *spec) +{ + int i; + + for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++) + if (spec->match_value[i] & ~spec->match_criteria[i]) { + pr_warn("mlx5_core: match_value differs from match_criteria\n"); + return false; + } + + return true; +} + +struct mlx5_flow_root_namespace *find_root(struct fs_node *node) +{ + struct fs_node *root; + struct mlx5_flow_namespace *ns; + + root = node->root; + + if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) { + pr_warn("mlx5: flow steering node is not in tree or garbaged\n"); + return NULL; + } + + ns = container_of(root, struct mlx5_flow_namespace, node); + return container_of(ns, struct mlx5_flow_root_namespace, ns); +} + +static inline struct mlx5_flow_steering *get_steering(struct fs_node *node) +{ + struct mlx5_flow_root_namespace *root = find_root(node); + + if (root) + return root->dev->priv.steering; + return NULL; +} + +static inline struct mlx5_core_dev *get_dev(struct fs_node *node) +{ + struct mlx5_flow_root_namespace *root = find_root(node); + + if (root) + return root->dev; + return NULL; +} + +static void del_sw_ns(struct fs_node *node) +{ + kfree(node); +} + +static void del_sw_prio(struct fs_node *node) +{ + kfree(node); +} + +static void del_hw_flow_table(struct fs_node *node) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_table *ft; + struct mlx5_core_dev *dev; + int err; + + fs_get_obj(ft, node); + dev = get_dev(&ft->node); + root = find_root(&ft->node); + + if (node->active) { + err = root->cmds->destroy_flow_table(root, ft); + if (err) + mlx5_core_warn(dev, "flow steering can't destroy ft\n"); + } +} + +static void del_sw_flow_table(struct fs_node *node) +{ + struct mlx5_flow_table *ft; + struct fs_prio *prio; + + fs_get_obj(ft, node); + + xa_destroy(&ft->fgs_xa); + if (ft->node.parent) { + fs_get_obj(prio, ft->node.parent); + prio->num_ft--; + } + kfree(ft); +} + +static void modify_fte(struct fs_fte *fte) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct mlx5_core_dev *dev; + int err; + + fs_get_obj(fg, fte->node.parent); + fs_get_obj(ft, fg->node.parent); + dev = get_dev(&fte->node); + + root = find_root(&ft->node); + err = root->cmds->update_fte(root, ft, fg, fte->modify_mask, fte); + if (err) + mlx5_core_warn(dev, + "%s can't del rule fg id=%d fte_index=%d\n", + __func__, fg->id, fte->index); + fte->modify_mask = 0; +} + +static void del_sw_hw_rule(struct fs_node *node) +{ + struct mlx5_flow_rule *rule; + struct fs_fte *fte; + + fs_get_obj(rule, node); + fs_get_obj(fte, rule->node.parent); + if (is_fwd_next_action(rule->sw_action)) { + mutex_lock(&rule->dest_attr.ft->lock); + list_del(&rule->next_ft); + mutex_unlock(&rule->dest_attr.ft->lock); + } + + if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER) { + --fte->dests_size; + fte->modify_mask |= + BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) | + BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS); + fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT; + goto out; + } + + if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_PORT) { + --fte->dests_size; + fte->modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION); + fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_ALLOW; + goto out; + } + + if (is_fwd_dest_type(rule->dest_attr.type)) { + --fte->dests_size; + --fte->fwd_dests; + + if (!fte->fwd_dests) + fte->action.action &= + ~MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + fte->modify_mask |= + BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST); + goto out; + } +out: + kfree(rule); +} + +static void del_hw_fte(struct fs_node *node) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct mlx5_core_dev *dev; + struct fs_fte *fte; + int err; + + fs_get_obj(fte, node); + fs_get_obj(fg, fte->node.parent); + fs_get_obj(ft, fg->node.parent); + + WARN_ON(fte->dests_size); + dev = get_dev(&ft->node); + root = find_root(&ft->node); + if (node->active) { + err = root->cmds->delete_fte(root, ft, fte); + if (err) + mlx5_core_warn(dev, + "flow steering can't delete fte in index %d of flow group id %d\n", + fte->index, fg->id); + node->active = false; + } +} + +static void del_sw_fte(struct fs_node *node) +{ + struct mlx5_flow_steering *steering = get_steering(node); + struct mlx5_flow_group *fg; + struct fs_fte *deleted_fte; + struct fs_fte *fte; + + fs_get_obj(fte, node); + fs_get_obj(fg, fte->node.parent); + + deleted_fte = xa_erase(&fg->ftes_xa, fte->index); + WARN_ON(deleted_fte != fte); + ida_free(&fg->fte_allocator, fte->index - fg->start_index); + kmem_cache_free(steering->ftes_cache, fte); +} + +static void del_hw_flow_group(struct fs_node *node) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_group *fg; + struct mlx5_flow_table *ft; + struct mlx5_core_dev *dev; + + fs_get_obj(fg, node); + fs_get_obj(ft, fg->node.parent); + dev = get_dev(&ft->node); + + root = find_root(&ft->node); + if (fg->node.active && root->cmds->destroy_flow_group(root, ft, fg)) + mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n", + fg->id, ft->id); +} + +static void del_sw_flow_group(struct fs_node *node) +{ + struct mlx5_flow_steering *steering = get_steering(node); + struct mlx5_flow_group *deleted_fg; + struct mlx5_flow_group *fg; + struct mlx5_flow_table *ft; + + fs_get_obj(fg, node); + fs_get_obj(ft, fg->node.parent); + + xa_destroy(&fg->ftes_xa); + ida_destroy(&fg->fte_allocator); + if (ft->autogroup.active && + fg->max_ftes == ft->autogroup.group_size && + fg->start_index < ft->autogroup.max_fte) + ft->autogroup.num_groups--; + deleted_fg = xa_erase(&ft->fgs_xa, fg->start_index); + WARN_ON(deleted_fg != fg); + kmem_cache_free(steering->fgs_cache, fg); +} + +static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte) +{ + int index; + int ret; + + index = ida_alloc_max(&fg->fte_allocator, fg->max_ftes, GFP_KERNEL); + if (index < 0) + return index; + + fte->index = index + fg->start_index; + ret = xa_insert(&fg->ftes_xa, fte->index, fte, GFP_KERNEL); + if (ret) + goto err_ida_remove; + + tree_add_node(&fte->node, &fg->node); + list_add_tail(&fte->node.list, &fg->node.children); + return 0; + +err_ida_remove: + ida_free(&fg->fte_allocator, index); + return ret; +} + +static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act) +{ + struct mlx5_flow_steering *steering = get_steering(&ft->node); + struct fs_fte *fte; + + fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL); + if (!fte) + return ERR_PTR(-ENOMEM); + + memcpy(fte->val, &spec->match_value, sizeof(fte->val)); + fte->node.type = FS_TYPE_FLOW_ENTRY; + fte->action = *flow_act; + fte->flow_context = spec->flow_context; + + tree_init_node(&fte->node, del_hw_fte, del_sw_fte); + + return fte; +} + +static void dealloc_flow_group(struct mlx5_flow_steering *steering, + struct mlx5_flow_group *fg) +{ + xa_destroy(&fg->ftes_xa); + kmem_cache_free(steering->fgs_cache, fg); +} + +static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering, + u8 match_criteria_enable, + const void *match_criteria, + int start_index, + int end_index) +{ + struct mlx5_flow_group *fg; + + fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL); + if (!fg) + return ERR_PTR(-ENOMEM); + + xa_init(&fg->ftes_xa); + + ida_init(&fg->fte_allocator); + fg->mask.match_criteria_enable = match_criteria_enable; + memcpy(&fg->mask.match_criteria, match_criteria, + sizeof(fg->mask.match_criteria)); + fg->node.type = FS_TYPE_FLOW_GROUP; + fg->start_index = start_index; + fg->max_ftes = end_index - start_index + 1; + + return fg; +} + +static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *ft, + u8 match_criteria_enable, + const void *match_criteria, + int start_index, + int end_index, + struct list_head *prev) +{ + struct mlx5_flow_steering *steering = get_steering(&ft->node); + struct mlx5_flow_group *fg; + int ret; + + fg = alloc_flow_group(steering, match_criteria_enable, match_criteria, + start_index, end_index); + if (IS_ERR(fg)) + return fg; + + /* initialize refcnt, add to parent list */ + ret = xa_insert(&ft->fgs_xa, fg->start_index, fg, GFP_KERNEL); + if (ret) { + dealloc_flow_group(steering, fg); + return ERR_PTR(ret); + } + + tree_init_node(&fg->node, del_hw_flow_group, del_sw_flow_group); + tree_add_node(&fg->node, &ft->node); + /* Add node to group list */ + list_add(&fg->node.list, prev); + atomic_inc(&ft->node.version); + + return fg; +} + +static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, + enum fs_flow_table_type table_type, + enum fs_flow_table_op_mod op_mod, + u32 flags) +{ + struct mlx5_flow_table *ft; + + ft = kzalloc(sizeof(*ft), GFP_KERNEL); + if (!ft) + return ERR_PTR(-ENOMEM); + + xa_init(&ft->fgs_xa); + + ft->level = level; + ft->node.type = FS_TYPE_FLOW_TABLE; + ft->op_mod = op_mod; + ft->type = table_type; + ft->vport = vport; + ft->flags = flags; + INIT_LIST_HEAD(&ft->fwd_rules); + mutex_init(&ft->lock); + + return ft; +} + +/* If reverse is false, then we search for the first flow table in the + * root sub-tree from start(closest from right), else we search for the + * last flow table in the root sub-tree till start(closest from left). + */ +static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node *root, + struct list_head *start, + bool reverse) +{ +#define list_advance_entry(pos, reverse) \ + ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list)) + +#define list_for_each_advance_continue(pos, head, reverse) \ + for (pos = list_advance_entry(pos, reverse); \ + &pos->list != (head); \ + pos = list_advance_entry(pos, reverse)) + + struct fs_node *iter = list_entry(start, struct fs_node, list); + struct mlx5_flow_table *ft = NULL; + + if (!root || root->type == FS_TYPE_PRIO_CHAINS) + return NULL; + + list_for_each_advance_continue(iter, &root->children, reverse) { + if (iter->type == FS_TYPE_FLOW_TABLE) { + fs_get_obj(ft, iter); + return ft; + } + ft = find_closest_ft_recursive(iter, &iter->children, reverse); + if (ft) + return ft; + } + + return ft; +} + +/* If reverse is false then return the first flow table in next priority of + * prio in the tree, else return the last flow table in the previous priority + * of prio in the tree. + */ +static struct mlx5_flow_table *find_closest_ft(struct fs_prio *prio, bool reverse) +{ + struct mlx5_flow_table *ft = NULL; + struct fs_node *curr_node; + struct fs_node *parent; + + parent = prio->node.parent; + curr_node = &prio->node; + while (!ft && parent) { + ft = find_closest_ft_recursive(parent, &curr_node->list, reverse); + curr_node = parent; + parent = curr_node->parent; + } + return ft; +} + +/* Assuming all the tree is locked by mutex chain lock */ +static struct mlx5_flow_table *find_next_chained_ft(struct fs_prio *prio) +{ + return find_closest_ft(prio, false); +} + +/* Assuming all the tree is locked by mutex chain lock */ +static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio) +{ + return find_closest_ft(prio, true); +} + +static struct mlx5_flow_table *find_next_fwd_ft(struct mlx5_flow_table *ft, + struct mlx5_flow_act *flow_act) +{ + struct fs_prio *prio; + bool next_ns; + + next_ns = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; + fs_get_obj(prio, next_ns ? ft->ns->node.parent : ft->node.parent); + + return find_next_chained_ft(prio); +} + +static int connect_fts_in_prio(struct mlx5_core_dev *dev, + struct fs_prio *prio, + struct mlx5_flow_table *ft) +{ + struct mlx5_flow_root_namespace *root = find_root(&prio->node); + struct mlx5_flow_table *iter; + int err; + + fs_for_each_ft(iter, prio) { + err = root->cmds->modify_flow_table(root, iter, ft); + if (err) { + mlx5_core_err(dev, + "Failed to modify flow table id %d, type %d, err %d\n", + iter->id, iter->type, err); + /* The driver is out of sync with the FW */ + return err; + } + } + return 0; +} + +/* Connect flow tables from previous priority of prio to ft */ +static int connect_prev_fts(struct mlx5_core_dev *dev, + struct mlx5_flow_table *ft, + struct fs_prio *prio) +{ + struct mlx5_flow_table *prev_ft; + + prev_ft = find_prev_chained_ft(prio); + if (prev_ft) { + struct fs_prio *prev_prio; + + fs_get_obj(prev_prio, prev_ft->node.parent); + return connect_fts_in_prio(dev, prev_prio, ft); + } + return 0; +} + +static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio + *prio) +{ + struct mlx5_flow_root_namespace *root = find_root(&prio->node); + struct mlx5_ft_underlay_qp *uqp; + int min_level = INT_MAX; + int err = 0; + u32 qpn; + + if (root->root_ft) + min_level = root->root_ft->level; + + if (ft->level >= min_level) + return 0; + + if (list_empty(&root->underlay_qpns)) { + /* Don't set any QPN (zero) in case QPN list is empty */ + qpn = 0; + err = root->cmds->update_root_ft(root, ft, qpn, false); + } else { + list_for_each_entry(uqp, &root->underlay_qpns, list) { + qpn = uqp->qpn; + err = root->cmds->update_root_ft(root, ft, + qpn, false); + if (err) + break; + } + } + + if (err) + mlx5_core_warn(root->dev, + "Update root flow table of id(%u) qpn(%d) failed\n", + ft->id, qpn); + else + root->root_ft = ft; + + return err; +} + +static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule, + struct mlx5_flow_destination *dest) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct fs_fte *fte; + int modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST); + int err = 0; + + fs_get_obj(fte, rule->node.parent); + if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)) + return -EINVAL; + down_write_ref_node(&fte->node, false); + fs_get_obj(fg, fte->node.parent); + fs_get_obj(ft, fg->node.parent); + + memcpy(&rule->dest_attr, dest, sizeof(*dest)); + root = find_root(&ft->node); + err = root->cmds->update_fte(root, ft, fg, + modify_mask, fte); + up_write_ref_node(&fte->node, false); + + return err; +} + +int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle, + struct mlx5_flow_destination *new_dest, + struct mlx5_flow_destination *old_dest) +{ + int i; + + if (!old_dest) { + if (handle->num_rules != 1) + return -EINVAL; + return _mlx5_modify_rule_destination(handle->rule[0], + new_dest); + } + + for (i = 0; i < handle->num_rules; i++) { + if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr)) + return _mlx5_modify_rule_destination(handle->rule[i], + new_dest); + } + + return -EINVAL; +} + +/* Modify/set FWD rules that point on old_next_ft to point on new_next_ft */ +static int connect_fwd_rules(struct mlx5_core_dev *dev, + struct mlx5_flow_table *new_next_ft, + struct mlx5_flow_table *old_next_ft) +{ + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_rule *iter; + int err = 0; + + /* new_next_ft and old_next_ft could be NULL only + * when we create/destroy the anchor flow table. + */ + if (!new_next_ft || !old_next_ft) + return 0; + + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = new_next_ft; + + mutex_lock(&old_next_ft->lock); + list_splice_init(&old_next_ft->fwd_rules, &new_next_ft->fwd_rules); + mutex_unlock(&old_next_ft->lock); + list_for_each_entry(iter, &new_next_ft->fwd_rules, next_ft) { + if ((iter->sw_action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS) && + iter->ft->ns == new_next_ft->ns) + continue; + + err = _mlx5_modify_rule_destination(iter, &dest); + if (err) + pr_err("mlx5_core: failed to modify rule to point on flow table %d\n", + new_next_ft->id); + } + return 0; +} + +static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, + struct fs_prio *prio) +{ + struct mlx5_flow_table *next_ft, *first_ft; + int err = 0; + + /* Connect_prev_fts and update_root_ft_create are mutually exclusive */ + + first_ft = list_first_entry_or_null(&prio->node.children, + struct mlx5_flow_table, node.list); + if (!first_ft || first_ft->level > ft->level) { + err = connect_prev_fts(dev, ft, prio); + if (err) + return err; + + next_ft = first_ft ? first_ft : find_next_chained_ft(prio); + err = connect_fwd_rules(dev, ft, next_ft); + if (err) + return err; + } + + if (MLX5_CAP_FLOWTABLE(dev, + flow_table_properties_nic_receive.modify_root)) + err = update_root_ft_create(ft, prio); + return err; +} + +static void list_add_flow_table(struct mlx5_flow_table *ft, + struct fs_prio *prio) +{ + struct list_head *prev = &prio->node.children; + struct mlx5_flow_table *iter; + + fs_for_each_ft(iter, prio) { + if (iter->level > ft->level) + break; + prev = &iter->node.list; + } + list_add(&ft->node.list, prev); +} + +static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespace *ns, + struct mlx5_flow_table_attr *ft_attr, + enum fs_flow_table_op_mod op_mod, + u16 vport) +{ + struct mlx5_flow_root_namespace *root = find_root(&ns->node); + bool unmanaged = ft_attr->flags & MLX5_FLOW_TABLE_UNMANAGED; + struct mlx5_flow_table *next_ft; + struct fs_prio *fs_prio = NULL; + struct mlx5_flow_table *ft; + int err; + + if (!root) { + pr_err("mlx5: flow steering failed to find root of namespace\n"); + return ERR_PTR(-ENODEV); + } + + mutex_lock(&root->chain_lock); + fs_prio = find_prio(ns, ft_attr->prio); + if (!fs_prio) { + err = -EINVAL; + goto unlock_root; + } + if (!unmanaged) { + /* The level is related to the + * priority level range. + */ + if (ft_attr->level >= fs_prio->num_levels) { + err = -ENOSPC; + goto unlock_root; + } + + ft_attr->level += fs_prio->start_level; + } + + /* The level is related to the + * priority level range. + */ + ft = alloc_flow_table(ft_attr->level, + vport, + root->table_type, + op_mod, ft_attr->flags); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + goto unlock_root; + } + + tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table); + next_ft = unmanaged ? ft_attr->next_ft : + find_next_chained_ft(fs_prio); + ft->def_miss_action = ns->def_miss_action; + ft->ns = ns; + err = root->cmds->create_flow_table(root, ft, ft_attr, next_ft); + if (err) + goto free_ft; + + if (!unmanaged) { + err = connect_flow_table(root->dev, ft, fs_prio); + if (err) + goto destroy_ft; + } + + ft->node.active = true; + down_write_ref_node(&fs_prio->node, false); + if (!unmanaged) { + tree_add_node(&ft->node, &fs_prio->node); + list_add_flow_table(ft, fs_prio); + } else { + ft->node.root = fs_prio->node.root; + } + fs_prio->num_ft++; + up_write_ref_node(&fs_prio->node, false); + mutex_unlock(&root->chain_lock); + return ft; +destroy_ft: + root->cmds->destroy_flow_table(root, ft); +free_ft: + xa_destroy(&ft->fgs_xa); + kfree(ft); +unlock_root: + mutex_unlock(&root->chain_lock); + return ERR_PTR(err); +} + +struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, + struct mlx5_flow_table_attr *ft_attr) +{ + return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, 0); +} +EXPORT_SYMBOL(mlx5_create_flow_table); + +u32 mlx5_flow_table_id(struct mlx5_flow_table *ft) +{ + return ft->id; +} +EXPORT_SYMBOL(mlx5_flow_table_id); + +struct mlx5_flow_table * +mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, + struct mlx5_flow_table_attr *ft_attr, u16 vport) +{ + return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, vport); +} + +struct mlx5_flow_table* +mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns, + int prio, u32 level) +{ + struct mlx5_flow_table_attr ft_attr = {}; + + ft_attr.level = level; + ft_attr.prio = prio; + ft_attr.max_fte = 1; + + return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0); +} +EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table); + +#define MAX_FLOW_GROUP_SIZE BIT(24) +struct mlx5_flow_table* +mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, + struct mlx5_flow_table_attr *ft_attr) +{ + int num_reserved_entries = ft_attr->autogroup.num_reserved_entries; + int max_num_groups = ft_attr->autogroup.max_num_groups; + struct mlx5_flow_table *ft; + int autogroups_max_fte; + + ft = mlx5_create_flow_table(ns, ft_attr); + if (IS_ERR(ft)) + return ft; + + autogroups_max_fte = ft->max_fte - num_reserved_entries; + if (max_num_groups > autogroups_max_fte) + goto err_validate; + if (num_reserved_entries > ft->max_fte) + goto err_validate; + + /* Align the number of groups according to the largest group size */ + if (autogroups_max_fte / (max_num_groups + 1) > MAX_FLOW_GROUP_SIZE) + max_num_groups = (autogroups_max_fte / MAX_FLOW_GROUP_SIZE) - 1; + + ft->autogroup.active = true; + ft->autogroup.required_groups = max_num_groups; + ft->autogroup.max_fte = autogroups_max_fte; + /* We save place for flow groups in addition to max types */ + ft->autogroup.group_size = autogroups_max_fte / (max_num_groups + 1); + + return ft; + +err_validate: + mlx5_destroy_flow_table(ft); + return ERR_PTR(-ENOSPC); +} +EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); + +struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, + u32 *fg_in) +{ + struct mlx5_flow_root_namespace *root = find_root(&ft->node); + void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, + fg_in, match_criteria); + u8 match_criteria_enable = MLX5_GET(create_flow_group_in, + fg_in, + match_criteria_enable); + int start_index = MLX5_GET(create_flow_group_in, fg_in, + start_flow_index); + int end_index = MLX5_GET(create_flow_group_in, fg_in, + end_flow_index); + struct mlx5_flow_group *fg; + int err; + + if (ft->autogroup.active && start_index < ft->autogroup.max_fte) + return ERR_PTR(-EPERM); + + down_write_ref_node(&ft->node, false); + fg = alloc_insert_flow_group(ft, match_criteria_enable, match_criteria, + start_index, end_index, + ft->node.children.prev); + up_write_ref_node(&ft->node, false); + if (IS_ERR(fg)) + return fg; + + err = root->cmds->create_flow_group(root, ft, fg_in, fg); + if (err) { + tree_put_node(&fg->node, false); + return ERR_PTR(err); + } + fg->node.active = true; + + return fg; +} +EXPORT_SYMBOL(mlx5_create_flow_group); + +static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest) +{ + struct mlx5_flow_rule *rule; + + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return NULL; + + INIT_LIST_HEAD(&rule->next_ft); + rule->node.type = FS_TYPE_FLOW_DEST; + if (dest) + memcpy(&rule->dest_attr, dest, sizeof(*dest)); + else + rule->dest_attr.type = MLX5_FLOW_DESTINATION_TYPE_NONE; + + return rule; +} + +static struct mlx5_flow_handle *alloc_handle(int num_rules) +{ + struct mlx5_flow_handle *handle; + + handle = kzalloc(struct_size(handle, rule, num_rules), GFP_KERNEL); + if (!handle) + return NULL; + + handle->num_rules = num_rules; + + return handle; +} + +static void destroy_flow_handle(struct fs_fte *fte, + struct mlx5_flow_handle *handle, + struct mlx5_flow_destination *dest, + int i) +{ + for (; --i >= 0;) { + if (refcount_dec_and_test(&handle->rule[i]->node.refcount)) { + fte->dests_size--; + list_del(&handle->rule[i]->node.list); + kfree(handle->rule[i]); + } + } + kfree(handle); +} + +static struct mlx5_flow_handle * +create_flow_handle(struct fs_fte *fte, + struct mlx5_flow_destination *dest, + int dest_num, + int *modify_mask, + bool *new_rule) +{ + struct mlx5_flow_handle *handle; + struct mlx5_flow_rule *rule = NULL; + static int count = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS); + static int dst = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST); + int type; + int i = 0; + + handle = alloc_handle((dest_num) ? dest_num : 1); + if (!handle) + return ERR_PTR(-ENOMEM); + + do { + if (dest) { + rule = find_flow_rule(fte, dest + i); + if (rule) { + refcount_inc(&rule->node.refcount); + goto rule_found; + } + } + + *new_rule = true; + rule = alloc_rule(dest + i); + if (!rule) + goto free_rules; + + /* Add dest to dests list- we need flow tables to be in the + * end of the list for forward to next prio rules. + */ + tree_init_node(&rule->node, NULL, del_sw_hw_rule); + if (dest && + dest[i].type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) + list_add(&rule->node.list, &fte->node.children); + else + list_add_tail(&rule->node.list, &fte->node.children); + if (dest) { + fte->dests_size++; + + if (is_fwd_dest_type(dest[i].type)) + fte->fwd_dests++; + + type = dest[i].type == + MLX5_FLOW_DESTINATION_TYPE_COUNTER; + *modify_mask |= type ? count : dst; + } +rule_found: + handle->rule[i] = rule; + } while (++i < dest_num); + + return handle; + +free_rules: + destroy_flow_handle(fte, handle, dest, i); + return ERR_PTR(-ENOMEM); +} + +/* fte should not be deleted while calling this function */ +static struct mlx5_flow_handle * +add_rule_fte(struct fs_fte *fte, + struct mlx5_flow_group *fg, + struct mlx5_flow_destination *dest, + int dest_num, + bool update_action) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_handle *handle; + struct mlx5_flow_table *ft; + int modify_mask = 0; + int err; + bool new_rule = false; + + handle = create_flow_handle(fte, dest, dest_num, &modify_mask, + &new_rule); + if (IS_ERR(handle) || !new_rule) + goto out; + + if (update_action) + modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION); + + fs_get_obj(ft, fg->node.parent); + root = find_root(&fg->node); + if (!(fte->status & FS_FTE_STATUS_EXISTING)) + err = root->cmds->create_fte(root, ft, fg, fte); + else + err = root->cmds->update_fte(root, ft, fg, modify_mask, fte); + if (err) + goto free_handle; + + fte->node.active = true; + fte->status |= FS_FTE_STATUS_EXISTING; + atomic_inc(&fg->node.version); + +out: + return handle; + +free_handle: + destroy_flow_handle(fte, handle, dest, handle->num_rules); + return ERR_PTR(err); +} + +static struct mlx5_flow_group *alloc_auto_flow_group(struct mlx5_flow_table *ft, + const struct mlx5_flow_spec *spec) +{ + struct list_head *prev = &ft->node.children; + u32 max_fte = ft->autogroup.max_fte; + unsigned int candidate_index = 0; + unsigned int group_size = 0; + struct mlx5_flow_group *fg; + + if (!ft->autogroup.active) + return ERR_PTR(-ENOENT); + + if (ft->autogroup.num_groups < ft->autogroup.required_groups) + group_size = ft->autogroup.group_size; + + /* max_fte == ft->autogroup.max_types */ + if (group_size == 0) + group_size = 1; + + /* sorted by start_index */ + fs_for_each_fg(fg, ft) { + if (candidate_index + group_size > fg->start_index) + candidate_index = fg->start_index + fg->max_ftes; + else + break; + prev = &fg->node.list; + } + + if (candidate_index + group_size > max_fte) + return ERR_PTR(-ENOSPC); + + fg = alloc_insert_flow_group(ft, + spec->match_criteria_enable, + spec->match_criteria, + candidate_index, + candidate_index + group_size - 1, + prev); + if (IS_ERR(fg)) + goto out; + + if (group_size == ft->autogroup.group_size) + ft->autogroup.num_groups++; + +out: + return fg; +} + +static int create_auto_flow_group(struct mlx5_flow_table *ft, + struct mlx5_flow_group *fg) +{ + struct mlx5_flow_root_namespace *root = find_root(&ft->node); + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + void *match_criteria_addr; + u8 src_esw_owner_mask_on; + void *misc; + int err; + u32 *in; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + MLX5_SET(create_flow_group_in, in, match_criteria_enable, + fg->mask.match_criteria_enable); + MLX5_SET(create_flow_group_in, in, start_flow_index, fg->start_index); + MLX5_SET(create_flow_group_in, in, end_flow_index, fg->start_index + + fg->max_ftes - 1); + + misc = MLX5_ADDR_OF(fte_match_param, fg->mask.match_criteria, + misc_parameters); + src_esw_owner_mask_on = !!MLX5_GET(fte_match_set_misc, misc, + source_eswitch_owner_vhca_id); + MLX5_SET(create_flow_group_in, in, + source_eswitch_owner_vhca_id_valid, src_esw_owner_mask_on); + + match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, + in, match_criteria); + memcpy(match_criteria_addr, fg->mask.match_criteria, + sizeof(fg->mask.match_criteria)); + + err = root->cmds->create_flow_group(root, ft, in, fg); + if (!err) { + fg->node.active = true; + } + + kvfree(in); + return err; +} + +static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1, + struct mlx5_flow_destination *d2) +{ + if (d1->type == d2->type) { + if (((d1->type == MLX5_FLOW_DESTINATION_TYPE_VPORT || + d1->type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) && + d1->vport.num == d2->vport.num && + d1->vport.flags == d2->vport.flags && + ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_VHCA_ID) ? + (d1->vport.vhca_id == d2->vport.vhca_id) : true) && + ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID) ? + (d1->vport.pkt_reformat->id == + d2->vport.pkt_reformat->id) : true)) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE && + d1->ft == d2->ft) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR && + d1->tir_num == d2->tir_num) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM && + d1->ft_num == d2->ft_num) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER && + d1->sampler_id == d2->sampler_id) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_RANGE && + d1->range.field == d2->range.field && + d1->range.hit_ft == d2->range.hit_ft && + d1->range.miss_ft == d2->range.miss_ft && + d1->range.min == d2->range.min && + d1->range.max == d2->range.max)) + return true; + } + + return false; +} + +static struct mlx5_flow_rule *find_flow_rule(struct fs_fte *fte, + struct mlx5_flow_destination *dest) +{ + struct mlx5_flow_rule *rule; + + list_for_each_entry(rule, &fte->node.children, node.list) { + if (mlx5_flow_dests_cmp(&rule->dest_attr, dest)) + return rule; + } + return NULL; +} + +static bool check_conflicting_actions_vlan(const struct mlx5_fs_vlan *vlan0, + const struct mlx5_fs_vlan *vlan1) +{ + return vlan0->ethtype != vlan1->ethtype || + vlan0->vid != vlan1->vid || + vlan0->prio != vlan1->prio; +} + +static bool check_conflicting_actions(const struct mlx5_flow_act *act1, + const struct mlx5_flow_act *act2) +{ + u32 action1 = act1->action; + u32 action2 = act2->action; + u32 xored_actions; + + xored_actions = action1 ^ action2; + + /* if one rule only wants to count, it's ok */ + if (action1 == MLX5_FLOW_CONTEXT_ACTION_COUNT || + action2 == MLX5_FLOW_CONTEXT_ACTION_COUNT) + return false; + + if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT | + MLX5_FLOW_CONTEXT_ACTION_DECAP | + MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | + MLX5_FLOW_CONTEXT_ACTION_VLAN_POP | + MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | + MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 | + MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2)) + return true; + + if (action1 & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT && + act1->pkt_reformat != act2->pkt_reformat) + return true; + + if (action1 & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR && + act1->modify_hdr != act2->modify_hdr) + return true; + + if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH && + check_conflicting_actions_vlan(&act1->vlan[0], &act2->vlan[0])) + return true; + + if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 && + check_conflicting_actions_vlan(&act1->vlan[1], &act2->vlan[1])) + return true; + + return false; +} + +static int check_conflicting_ftes(struct fs_fte *fte, + const struct mlx5_flow_context *flow_context, + const struct mlx5_flow_act *flow_act) +{ + if (check_conflicting_actions(flow_act, &fte->action)) { + mlx5_core_warn(get_dev(&fte->node), + "Found two FTEs with conflicting actions\n"); + return -EEXIST; + } + + if ((flow_context->flags & FLOW_CONTEXT_HAS_TAG) && + fte->flow_context.flow_tag != flow_context->flow_tag) { + mlx5_core_warn(get_dev(&fte->node), + "FTE flow tag %u already exists with different flow tag %u\n", + fte->flow_context.flow_tag, + flow_context->flow_tag); + return -EEXIST; + } + + return 0; +} + +static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_destination *dest, + int dest_num, + struct fs_fte *fte) +{ + struct mlx5_flow_handle *handle; + int old_action; + int i; + int ret; + + ret = check_conflicting_ftes(fte, &spec->flow_context, flow_act); + if (ret) + return ERR_PTR(ret); + + old_action = fte->action.action; + fte->action.action |= flow_act->action; + handle = add_rule_fte(fte, fg, dest, dest_num, + old_action != flow_act->action); + if (IS_ERR(handle)) { + fte->action.action = old_action; + return handle; + } + + for (i = 0; i < handle->num_rules; i++) { + if (refcount_read(&handle->rule[i]->node.refcount) == 1) { + tree_add_node(&handle->rule[i]->node, &fte->node); + } + } + return handle; +} + +static bool counter_is_valid(u32 action) +{ + return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_ALLOW | + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)); +} + +static bool dest_is_valid(struct mlx5_flow_destination *dest, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_table *ft) +{ + bool ignore_level = flow_act->flags & FLOW_ACT_IGNORE_FLOW_LEVEL; + u32 action = flow_act->action; + + if (dest && (dest->type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)) + return counter_is_valid(action); + + if (!(action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)) + return true; + + if (ignore_level) { + if (ft->type != FS_FT_FDB && + ft->type != FS_FT_NIC_RX && + ft->type != FS_FT_NIC_TX) + return false; + + if (dest->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE && + ft->type != dest->ft->type) + return false; + } + + if (!dest || ((dest->type == + MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) && + (dest->ft->level <= ft->level && !ignore_level))) + return false; + return true; +} + +struct match_list { + struct list_head list; + struct mlx5_flow_group *g; +}; + +static void free_match_list(struct match_list *head, bool ft_locked) +{ + struct match_list *iter, *match_tmp; + + list_for_each_entry_safe(iter, match_tmp, &head->list, + list) { + tree_put_node(&iter->g->node, ft_locked); + list_del(&iter->list); + kfree(iter); + } +} + +#define xa_for_each_rcu(xa, index, entry) \ + for ((entry) = NULL, (index) = 0; \ + ((entry) = xa_next(xa, &index, (entry) != NULL)) != NULL; ) + +static int build_match_list(struct match_list *match_head, + struct mlx5_flow_table *ft, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_group *fg, + bool ft_locked) +{ + struct mlx5_flow_group *g; + unsigned long id; + + rcu_read_lock(); + INIT_LIST_HEAD(&match_head->list); + xa_for_each_rcu(&ft->fgs_xa, id, g) { + struct match_list *curr_match; + + if (memcmp(&g->mask, spec, sizeof_field(struct mlx5_flow_group, + mask))) + continue; + + if (fg && fg != g) + continue; + + if (unlikely(!tree_get_node(&g->node))) + continue; + + curr_match = kmalloc(sizeof(*curr_match), GFP_ATOMIC); + if (!curr_match) { + rcu_read_unlock(); + free_match_list(match_head, ft_locked); + return -ENOMEM; + } + curr_match->g = g; + list_add_tail(&curr_match->list, &match_head->list); + } + rcu_read_unlock(); + return 0; +} + +static u64 matched_fgs_get_version(struct list_head *match_head) +{ + struct match_list *iter; + u64 version = 0; + + list_for_each_entry(iter, match_head, list) + version += (u64)atomic_read(&iter->g->node.version); + return version; +} + +static struct fs_fte * +lookup_fte_locked(struct mlx5_flow_group *g, + const u32 *match_value, + bool take_write) +{ + struct fs_fte *fte_tmp; + unsigned long index; + + if (take_write) + nested_down_write_ref_node(&g->node, FS_LOCK_PARENT); + else + nested_down_read_ref_node(&g->node, FS_LOCK_PARENT); + xa_for_each_rcu(&g->ftes_xa, index, fte_tmp) { + if (!memcmp(match_value, fte_tmp->val, sizeof_field(struct fs_fte, val))) + break; + } + if (!fte_tmp || !tree_get_node(&fte_tmp->node)) { + fte_tmp = NULL; + goto out; + } + if (!fte_tmp->node.active) { + tree_put_node(&fte_tmp->node, false); + fte_tmp = NULL; + goto out; + } + + nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD); +out: + if (take_write) + up_write_ref_node(&g->node, false); + else + up_read_ref_node(&g->node); + return fte_tmp; +} + +static struct mlx5_flow_handle * +try_add_to_existing_fg(struct mlx5_flow_table *ft, + struct list_head *match_head, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_destination *dest, + int dest_num, + int ft_version) +{ + struct mlx5_flow_steering *steering = get_steering(&ft->node); + struct mlx5_flow_group *g; + struct mlx5_flow_handle *rule; + struct match_list *iter; + bool take_write = false; + struct fs_fte *fte; + u64 version = 0; + int err; + + fte = alloc_fte(ft, spec, flow_act); + if (IS_ERR(fte)) + return ERR_PTR(-ENOMEM); + +search_again_locked: + if (flow_act->flags & FLOW_ACT_NO_APPEND) + goto skip_search; + version = matched_fgs_get_version(match_head); + /* Try to find an fte with identical match value and attempt update its + * action. + */ + list_for_each_entry(iter, match_head, list) { + struct fs_fte *fte_tmp; + + g = iter->g; + fte_tmp = lookup_fte_locked(g, spec->match_value, take_write); + if (!fte_tmp) + continue; + rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte_tmp); + /* No error check needed here, because insert_fte() is not called */ + up_write_ref_node(&fte_tmp->node, false); + tree_put_node(&fte_tmp->node, false); + kmem_cache_free(steering->ftes_cache, fte); + return rule; + } + +skip_search: + /* No group with matching fte found, or we skipped the search. + * Try to add a new fte to any matching fg. + */ + + /* Check the ft version, for case that new flow group + * was added while the fgs weren't locked + */ + if (atomic_read(&ft->node.version) != ft_version) { + rule = ERR_PTR(-EAGAIN); + goto out; + } + + /* Check the fgs version. If version have changed it could be that an + * FTE with the same match value was added while the fgs weren't + * locked. + */ + if (!(flow_act->flags & FLOW_ACT_NO_APPEND) && + version != matched_fgs_get_version(match_head)) { + take_write = true; + goto search_again_locked; + } + + list_for_each_entry(iter, match_head, list) { + g = iter->g; + + nested_down_write_ref_node(&g->node, FS_LOCK_PARENT); + + if (!g->node.active) { + up_write_ref_node(&g->node, false); + continue; + } + + err = insert_fte(g, fte); + if (err) { + up_write_ref_node(&g->node, false); + if (err == -ENOSPC) + continue; + kmem_cache_free(steering->ftes_cache, fte); + return ERR_PTR(err); + } + + nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD); + up_write_ref_node(&g->node, false); + rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte); + up_write_ref_node(&fte->node, false); + if (IS_ERR(rule)) + tree_put_node(&fte->node, false); + return rule; + } + rule = ERR_PTR(-ENOENT); +out: + kmem_cache_free(steering->ftes_cache, fte); + return rule; +} + +static struct mlx5_flow_handle * +_mlx5_add_flow_rules(struct mlx5_flow_table *ft, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_destination *dest, + int dest_num) + +{ + struct mlx5_flow_steering *steering = get_steering(&ft->node); + struct mlx5_flow_handle *rule; + struct match_list match_head; + struct mlx5_flow_group *g; + bool take_write = false; + struct fs_fte *fte; + int version; + int err; + int i; + + if (!check_valid_spec(spec)) + return ERR_PTR(-EINVAL); + + if (flow_act->fg && ft->autogroup.active) + return ERR_PTR(-EINVAL); + + if (dest && dest_num <= 0) + return ERR_PTR(-EINVAL); + + for (i = 0; i < dest_num; i++) { + if (!dest_is_valid(&dest[i], flow_act, ft)) + return ERR_PTR(-EINVAL); + } + nested_down_read_ref_node(&ft->node, FS_LOCK_GRANDPARENT); +search_again_locked: + version = atomic_read(&ft->node.version); + + /* Collect all fgs which has a matching match_criteria */ + err = build_match_list(&match_head, ft, spec, flow_act->fg, take_write); + if (err) { + if (take_write) + up_write_ref_node(&ft->node, false); + else + up_read_ref_node(&ft->node); + return ERR_PTR(err); + } + + if (!take_write) + up_read_ref_node(&ft->node); + + rule = try_add_to_existing_fg(ft, &match_head.list, spec, flow_act, dest, + dest_num, version); + free_match_list(&match_head, take_write); + if (!IS_ERR(rule) || + (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN)) { + if (take_write) + up_write_ref_node(&ft->node, false); + return rule; + } + + if (!take_write) { + nested_down_write_ref_node(&ft->node, FS_LOCK_GRANDPARENT); + take_write = true; + } + + if (PTR_ERR(rule) == -EAGAIN || + version != atomic_read(&ft->node.version)) + goto search_again_locked; + + g = alloc_auto_flow_group(ft, spec); + if (IS_ERR(g)) { + rule = ERR_CAST(g); + up_write_ref_node(&ft->node, false); + return rule; + } + + fte = alloc_fte(ft, spec, flow_act); + if (IS_ERR(fte)) { + up_write_ref_node(&ft->node, false); + err = PTR_ERR(fte); + goto err_alloc_fte; + } + + nested_down_write_ref_node(&g->node, FS_LOCK_PARENT); + up_write_ref_node(&ft->node, false); + + err = create_auto_flow_group(ft, g); + if (err) + goto err_release_fg; + + err = insert_fte(g, fte); + if (err) + goto err_release_fg; + + nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD); + up_write_ref_node(&g->node, false); + rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte); + up_write_ref_node(&fte->node, false); + if (IS_ERR(rule)) + tree_put_node(&fte->node, false); + tree_put_node(&g->node, false); + return rule; + +err_release_fg: + up_write_ref_node(&g->node, false); + kmem_cache_free(steering->ftes_cache, fte); +err_alloc_fte: + tree_put_node(&g->node, false); + return ERR_PTR(err); +} + +static bool fwd_next_prio_supported(struct mlx5_flow_table *ft) +{ + return ((ft->type == FS_FT_NIC_RX) && + (MLX5_CAP_FLOWTABLE(get_dev(&ft->node), nic_rx_multi_path_tirs))); +} + +struct mlx5_flow_handle * +mlx5_add_flow_rules(struct mlx5_flow_table *ft, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_destination *dest, + int num_dest) +{ + struct mlx5_flow_root_namespace *root = find_root(&ft->node); + static const struct mlx5_flow_spec zero_spec = {}; + struct mlx5_flow_destination *gen_dest = NULL; + struct mlx5_flow_table *next_ft = NULL; + struct mlx5_flow_handle *handle = NULL; + u32 sw_action = flow_act->action; + int i; + + if (!spec) + spec = &zero_spec; + + if (!is_fwd_next_action(sw_action)) + return _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest); + + if (!fwd_next_prio_supported(ft)) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&root->chain_lock); + next_ft = find_next_fwd_ft(ft, flow_act); + if (!next_ft) { + handle = ERR_PTR(-EOPNOTSUPP); + goto unlock; + } + + gen_dest = kcalloc(num_dest + 1, sizeof(*dest), + GFP_KERNEL); + if (!gen_dest) { + handle = ERR_PTR(-ENOMEM); + goto unlock; + } + for (i = 0; i < num_dest; i++) + gen_dest[i] = dest[i]; + gen_dest[i].type = + MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + gen_dest[i].ft = next_ft; + dest = gen_dest; + num_dest++; + flow_act->action &= ~(MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO | + MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS); + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + handle = _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest); + if (IS_ERR(handle)) + goto unlock; + + if (list_empty(&handle->rule[num_dest - 1]->next_ft)) { + mutex_lock(&next_ft->lock); + list_add(&handle->rule[num_dest - 1]->next_ft, + &next_ft->fwd_rules); + mutex_unlock(&next_ft->lock); + handle->rule[num_dest - 1]->sw_action = sw_action; + handle->rule[num_dest - 1]->ft = ft; + } +unlock: + mutex_unlock(&root->chain_lock); + kfree(gen_dest); + return handle; +} +EXPORT_SYMBOL(mlx5_add_flow_rules); + +void mlx5_del_flow_rules(struct mlx5_flow_handle **pp) +{ + struct mlx5_flow_handle *handle; + struct fs_fte *fte; + int i; + + handle = *pp; + *pp = NULL; + if (IS_ERR_OR_NULL(handle)) + return; + + /* In order to consolidate the HW changes we lock the FTE for other + * changes, and increase its refcount, in order not to perform the + * "del" functions of the FTE. Will handle them here. + * The removal of the rules is done under locked FTE. + * After removing all the handle's rules, if there are remaining + * rules, it means we just need to modify the FTE in FW, and + * unlock/decrease the refcount we increased before. + * Otherwise, it means the FTE should be deleted. First delete the + * FTE in FW. Then, unlock the FTE, and proceed the tree_put_node of + * the FTE, which will handle the last decrease of the refcount, as + * well as required handling of its parent. + */ + fs_get_obj(fte, handle->rule[0]->node.parent); + down_write_ref_node(&fte->node, false); + for (i = handle->num_rules - 1; i >= 0; i--) + tree_remove_node(&handle->rule[i]->node, true); + if (list_empty(&fte->node.children)) { + fte->node.del_hw_func(&fte->node); + /* Avoid double call to del_hw_fte */ + fte->node.del_hw_func = NULL; + up_write_ref_node(&fte->node, false); + tree_put_node(&fte->node, false); + } else if (fte->dests_size) { + if (fte->modify_mask) + modify_fte(fte); + up_write_ref_node(&fte->node, false); + } else { + up_write_ref_node(&fte->node, false); + } + kfree(handle); +} +EXPORT_SYMBOL(mlx5_del_flow_rules); + +/* Assuming prio->node.children(flow tables) is sorted by level */ +static struct mlx5_flow_table *find_next_ft(struct mlx5_flow_table *ft) +{ + struct fs_prio *prio; + + fs_get_obj(prio, ft->node.parent); + + if (!list_is_last(&ft->node.list, &prio->node.children)) + return list_next_entry(ft, node.list); + return find_next_chained_ft(prio); +} + +static int update_root_ft_destroy(struct mlx5_flow_table *ft) +{ + struct mlx5_flow_root_namespace *root = find_root(&ft->node); + struct mlx5_ft_underlay_qp *uqp; + struct mlx5_flow_table *new_root_ft = NULL; + int err = 0; + u32 qpn; + + if (root->root_ft != ft) + return 0; + + new_root_ft = find_next_ft(ft); + if (!new_root_ft) { + root->root_ft = NULL; + return 0; + } + + if (list_empty(&root->underlay_qpns)) { + /* Don't set any QPN (zero) in case QPN list is empty */ + qpn = 0; + err = root->cmds->update_root_ft(root, new_root_ft, + qpn, false); + } else { + list_for_each_entry(uqp, &root->underlay_qpns, list) { + qpn = uqp->qpn; + err = root->cmds->update_root_ft(root, + new_root_ft, qpn, + false); + if (err) + break; + } + } + + if (err) + mlx5_core_warn(root->dev, + "Update root flow table of id(%u) qpn(%d) failed\n", + ft->id, qpn); + else + root->root_ft = new_root_ft; + + return 0; +} + +/* Connect flow table from previous priority to + * the next flow table. + */ +static int disconnect_flow_table(struct mlx5_flow_table *ft) +{ + struct mlx5_core_dev *dev = get_dev(&ft->node); + struct mlx5_flow_table *next_ft; + struct fs_prio *prio; + int err = 0; + + err = update_root_ft_destroy(ft); + if (err) + return err; + + fs_get_obj(prio, ft->node.parent); + if (!(list_first_entry(&prio->node.children, + struct mlx5_flow_table, + node.list) == ft)) + return 0; + + next_ft = find_next_ft(ft); + err = connect_fwd_rules(dev, next_ft, ft); + if (err) + return err; + + err = connect_prev_fts(dev, next_ft, prio); + if (err) + mlx5_core_warn(dev, "Failed to disconnect flow table %d\n", + ft->id); + return err; +} + +int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) +{ + struct mlx5_flow_root_namespace *root = find_root(&ft->node); + int err = 0; + + mutex_lock(&root->chain_lock); + if (!(ft->flags & MLX5_FLOW_TABLE_UNMANAGED)) + err = disconnect_flow_table(ft); + if (err) { + mutex_unlock(&root->chain_lock); + return err; + } + if (tree_remove_node(&ft->node, false)) + mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n", + ft->id); + mutex_unlock(&root->chain_lock); + + return err; +} +EXPORT_SYMBOL(mlx5_destroy_flow_table); + +void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) +{ + if (tree_remove_node(&fg->node, false)) + mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n", + fg->id); +} +EXPORT_SYMBOL(mlx5_destroy_flow_group); + +struct mlx5_flow_namespace *mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev, + int n) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + + if (!steering || !steering->fdb_sub_ns) + return NULL; + + return steering->fdb_sub_ns[n]; +} +EXPORT_SYMBOL(mlx5_get_fdb_sub_ns); + +static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type) +{ + switch (type) { + case MLX5_FLOW_NAMESPACE_BYPASS: + case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC: + case MLX5_FLOW_NAMESPACE_LAG: + case MLX5_FLOW_NAMESPACE_OFFLOADS: + case MLX5_FLOW_NAMESPACE_ETHTOOL: + case MLX5_FLOW_NAMESPACE_KERNEL: + case MLX5_FLOW_NAMESPACE_LEFTOVERS: + case MLX5_FLOW_NAMESPACE_ANCHOR: + return true; + default: + return false; + } +} + +struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, + enum mlx5_flow_namespace_type type) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + struct mlx5_flow_root_namespace *root_ns; + int prio = 0; + struct fs_prio *fs_prio; + struct mlx5_flow_namespace *ns; + + if (!steering) + return NULL; + + switch (type) { + case MLX5_FLOW_NAMESPACE_FDB: + if (steering->fdb_root_ns) + return &steering->fdb_root_ns->ns; + return NULL; + case MLX5_FLOW_NAMESPACE_PORT_SEL: + if (steering->port_sel_root_ns) + return &steering->port_sel_root_ns->ns; + return NULL; + case MLX5_FLOW_NAMESPACE_SNIFFER_RX: + if (steering->sniffer_rx_root_ns) + return &steering->sniffer_rx_root_ns->ns; + return NULL; + case MLX5_FLOW_NAMESPACE_SNIFFER_TX: + if (steering->sniffer_tx_root_ns) + return &steering->sniffer_tx_root_ns->ns; + return NULL; + case MLX5_FLOW_NAMESPACE_FDB_BYPASS: + root_ns = steering->fdb_root_ns; + prio = FDB_BYPASS_PATH; + break; + case MLX5_FLOW_NAMESPACE_EGRESS: + case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC: + case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC: + root_ns = steering->egress_root_ns; + prio = type - MLX5_FLOW_NAMESPACE_EGRESS; + break; + case MLX5_FLOW_NAMESPACE_RDMA_RX: + root_ns = steering->rdma_rx_root_ns; + prio = RDMA_RX_BYPASS_PRIO; + break; + case MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL: + root_ns = steering->rdma_rx_root_ns; + prio = RDMA_RX_KERNEL_PRIO; + break; + case MLX5_FLOW_NAMESPACE_RDMA_TX: + root_ns = steering->rdma_tx_root_ns; + break; + case MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS: + root_ns = steering->rdma_rx_root_ns; + prio = RDMA_RX_COUNTERS_PRIO; + break; + case MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS: + root_ns = steering->rdma_tx_root_ns; + prio = RDMA_TX_COUNTERS_PRIO; + break; + case MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC: + root_ns = steering->rdma_rx_root_ns; + prio = RDMA_RX_IPSEC_PRIO; + break; + case MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC: + root_ns = steering->rdma_tx_root_ns; + prio = RDMA_TX_IPSEC_PRIO; + break; + default: /* Must be NIC RX */ + WARN_ON(!is_nic_rx_ns(type)); + root_ns = steering->root_ns; + prio = type; + break; + } + + if (!root_ns) + return NULL; + + fs_prio = find_prio(&root_ns->ns, prio); + if (!fs_prio) + return NULL; + + ns = list_first_entry(&fs_prio->node.children, + typeof(*ns), + node.list); + + return ns; +} +EXPORT_SYMBOL(mlx5_get_flow_namespace); + +struct mlx5_flow_namespace *mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev, + enum mlx5_flow_namespace_type type, + int vport) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + + if (!steering) + return NULL; + + switch (type) { + case MLX5_FLOW_NAMESPACE_ESW_EGRESS: + if (vport >= steering->esw_egress_acl_vports) + return NULL; + if (steering->esw_egress_root_ns && + steering->esw_egress_root_ns[vport]) + return &steering->esw_egress_root_ns[vport]->ns; + else + return NULL; + case MLX5_FLOW_NAMESPACE_ESW_INGRESS: + if (vport >= steering->esw_ingress_acl_vports) + return NULL; + if (steering->esw_ingress_root_ns && + steering->esw_ingress_root_ns[vport]) + return &steering->esw_ingress_root_ns[vport]->ns; + else + return NULL; + default: + return NULL; + } +} + +static struct fs_prio *_fs_create_prio(struct mlx5_flow_namespace *ns, + unsigned int prio, + int num_levels, + enum fs_node_type type) +{ + struct fs_prio *fs_prio; + + fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); + if (!fs_prio) + return ERR_PTR(-ENOMEM); + + fs_prio->node.type = type; + tree_init_node(&fs_prio->node, NULL, del_sw_prio); + tree_add_node(&fs_prio->node, &ns->node); + fs_prio->num_levels = num_levels; + fs_prio->prio = prio; + list_add_tail(&fs_prio->node.list, &ns->node.children); + + return fs_prio; +} + +static struct fs_prio *fs_create_prio_chained(struct mlx5_flow_namespace *ns, + unsigned int prio, + int num_levels) +{ + return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO_CHAINS); +} + +static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, + unsigned int prio, int num_levels) +{ + return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO); +} + +static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace + *ns) +{ + ns->node.type = FS_TYPE_NAMESPACE; + + return ns; +} + +static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio, + int def_miss_act) +{ + struct mlx5_flow_namespace *ns; + + ns = kzalloc(sizeof(*ns), GFP_KERNEL); + if (!ns) + return ERR_PTR(-ENOMEM); + + fs_init_namespace(ns); + ns->def_miss_action = def_miss_act; + tree_init_node(&ns->node, NULL, del_sw_ns); + tree_add_node(&ns->node, &prio->node); + list_add_tail(&ns->node.list, &prio->node.children); + + return ns; +} + +static int create_leaf_prios(struct mlx5_flow_namespace *ns, int prio, + struct init_tree_node *prio_metadata) +{ + struct fs_prio *fs_prio; + int i; + + for (i = 0; i < prio_metadata->num_leaf_prios; i++) { + fs_prio = fs_create_prio(ns, prio++, prio_metadata->num_levels); + if (IS_ERR(fs_prio)) + return PTR_ERR(fs_prio); + } + return 0; +} + +#define FLOW_TABLE_BIT_SZ 1 +#define GET_FLOW_TABLE_CAP(dev, offset) \ + ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ + offset / 32)) >> \ + (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) +static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) +{ + int i; + + for (i = 0; i < caps->arr_sz; i++) { + if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) + return false; + } + return true; +} + +static int init_root_tree_recursive(struct mlx5_flow_steering *steering, + struct init_tree_node *init_node, + struct fs_node *fs_parent_node, + struct init_tree_node *init_parent_node, + int prio) +{ + int max_ft_level = MLX5_CAP_FLOWTABLE(steering->dev, + flow_table_properties_nic_receive. + max_ft_level); + struct mlx5_flow_namespace *fs_ns; + struct fs_prio *fs_prio; + struct fs_node *base; + int i; + int err; + + if (init_node->type == FS_TYPE_PRIO) { + if ((init_node->min_ft_level > max_ft_level) || + !has_required_caps(steering->dev, &init_node->caps)) + return 0; + + fs_get_obj(fs_ns, fs_parent_node); + if (init_node->num_leaf_prios) + return create_leaf_prios(fs_ns, prio, init_node); + fs_prio = fs_create_prio(fs_ns, prio, init_node->num_levels); + if (IS_ERR(fs_prio)) + return PTR_ERR(fs_prio); + base = &fs_prio->node; + } else if (init_node->type == FS_TYPE_NAMESPACE) { + fs_get_obj(fs_prio, fs_parent_node); + fs_ns = fs_create_namespace(fs_prio, init_node->def_miss_action); + if (IS_ERR(fs_ns)) + return PTR_ERR(fs_ns); + base = &fs_ns->node; + } else { + return -EINVAL; + } + prio = 0; + for (i = 0; i < init_node->ar_size; i++) { + err = init_root_tree_recursive(steering, &init_node->children[i], + base, init_node, prio); + if (err) + return err; + if (init_node->children[i].type == FS_TYPE_PRIO && + init_node->children[i].num_leaf_prios) { + prio += init_node->children[i].num_leaf_prios; + } + } + + return 0; +} + +static int init_root_tree(struct mlx5_flow_steering *steering, + struct init_tree_node *init_node, + struct fs_node *fs_parent_node) +{ + int err; + int i; + + for (i = 0; i < init_node->ar_size; i++) { + err = init_root_tree_recursive(steering, &init_node->children[i], + fs_parent_node, + init_node, i); + if (err) + return err; + } + return 0; +} + +static void del_sw_root_ns(struct fs_node *node) +{ + struct mlx5_flow_root_namespace *root_ns; + struct mlx5_flow_namespace *ns; + + fs_get_obj(ns, node); + root_ns = container_of(ns, struct mlx5_flow_root_namespace, ns); + mutex_destroy(&root_ns->chain_lock); + kfree(node); +} + +static struct mlx5_flow_root_namespace +*create_root_ns(struct mlx5_flow_steering *steering, + enum fs_flow_table_type table_type) +{ + const struct mlx5_flow_cmds *cmds = mlx5_fs_cmd_get_default(table_type); + struct mlx5_flow_root_namespace *root_ns; + struct mlx5_flow_namespace *ns; + + /* Create the root namespace */ + root_ns = kzalloc(sizeof(*root_ns), GFP_KERNEL); + if (!root_ns) + return NULL; + + root_ns->dev = steering->dev; + root_ns->table_type = table_type; + root_ns->cmds = cmds; + + INIT_LIST_HEAD(&root_ns->underlay_qpns); + + ns = &root_ns->ns; + fs_init_namespace(ns); + mutex_init(&root_ns->chain_lock); + tree_init_node(&ns->node, NULL, del_sw_root_ns); + tree_add_node(&ns->node, NULL); + + return root_ns; +} + +static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level); + +static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level) +{ + struct fs_prio *prio; + + fs_for_each_prio(prio, ns) { + /* This updates prio start_level and num_levels */ + set_prio_attrs_in_prio(prio, acc_level); + acc_level += prio->num_levels; + } + return acc_level; +} + +static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level) +{ + struct mlx5_flow_namespace *ns; + int acc_level_ns = acc_level; + + prio->start_level = acc_level; + fs_for_each_ns(ns, prio) { + /* This updates start_level and num_levels of ns's priority descendants */ + acc_level_ns = set_prio_attrs_in_ns(ns, acc_level); + + /* If this a prio with chains, and we can jump from one chain + * (namespace) to another, so we accumulate the levels + */ + if (prio->node.type == FS_TYPE_PRIO_CHAINS) + acc_level = acc_level_ns; + } + + if (!prio->num_levels) + prio->num_levels = acc_level_ns - prio->start_level; + WARN_ON(prio->num_levels < acc_level_ns - prio->start_level); +} + +static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns) +{ + struct mlx5_flow_namespace *ns = &root_ns->ns; + struct fs_prio *prio; + int start_level = 0; + + fs_for_each_prio(prio, ns) { + set_prio_attrs_in_prio(prio, start_level); + start_level += prio->num_levels; + } +} + +#define ANCHOR_PRIO 0 +#define ANCHOR_SIZE 1 +#define ANCHOR_LEVEL 0 +static int create_anchor_flow_table(struct mlx5_flow_steering *steering) +{ + struct mlx5_flow_namespace *ns = NULL; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_table *ft; + + ns = mlx5_get_flow_namespace(steering->dev, MLX5_FLOW_NAMESPACE_ANCHOR); + if (WARN_ON(!ns)) + return -EINVAL; + + ft_attr.max_fte = ANCHOR_SIZE; + ft_attr.level = ANCHOR_LEVEL; + ft_attr.prio = ANCHOR_PRIO; + + ft = mlx5_create_flow_table(ns, &ft_attr); + if (IS_ERR(ft)) { + mlx5_core_err(steering->dev, "Failed to create last anchor flow table"); + return PTR_ERR(ft); + } + return 0; +} + +static int init_root_ns(struct mlx5_flow_steering *steering) +{ + int err; + + steering->root_ns = create_root_ns(steering, FS_FT_NIC_RX); + if (!steering->root_ns) + return -ENOMEM; + + err = init_root_tree(steering, &root_fs, &steering->root_ns->ns.node); + if (err) + goto out_err; + + set_prio_attrs(steering->root_ns); + err = create_anchor_flow_table(steering); + if (err) + goto out_err; + + return 0; + +out_err: + cleanup_root_ns(steering->root_ns); + steering->root_ns = NULL; + return err; +} + +static void clean_tree(struct fs_node *node) +{ + if (node) { + struct fs_node *iter; + struct fs_node *temp; + + tree_get_node(node); + list_for_each_entry_safe(iter, temp, &node->children, list) + clean_tree(iter); + tree_put_node(node, false); + tree_remove_node(node, false); + } +} + +static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns) +{ + if (!root_ns) + return; + + clean_tree(&root_ns->ns.node); +} + +static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering) +{ + struct fs_prio *prio; + + steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX); + if (!steering->sniffer_tx_root_ns) + return -ENOMEM; + + /* Create single prio */ + prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1); + return PTR_ERR_OR_ZERO(prio); +} + +static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering) +{ + struct fs_prio *prio; + + steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX); + if (!steering->sniffer_rx_root_ns) + return -ENOMEM; + + /* Create single prio */ + prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1); + return PTR_ERR_OR_ZERO(prio); +} + +#define PORT_SEL_NUM_LEVELS 3 +static int init_port_sel_root_ns(struct mlx5_flow_steering *steering) +{ + struct fs_prio *prio; + + steering->port_sel_root_ns = create_root_ns(steering, FS_FT_PORT_SEL); + if (!steering->port_sel_root_ns) + return -ENOMEM; + + /* Create single prio */ + prio = fs_create_prio(&steering->port_sel_root_ns->ns, 0, + PORT_SEL_NUM_LEVELS); + return PTR_ERR_OR_ZERO(prio); +} + +static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering) +{ + int err; + + steering->rdma_rx_root_ns = create_root_ns(steering, FS_FT_RDMA_RX); + if (!steering->rdma_rx_root_ns) + return -ENOMEM; + + err = init_root_tree(steering, &rdma_rx_root_fs, + &steering->rdma_rx_root_ns->ns.node); + if (err) + goto out_err; + + set_prio_attrs(steering->rdma_rx_root_ns); + + return 0; + +out_err: + cleanup_root_ns(steering->rdma_rx_root_ns); + steering->rdma_rx_root_ns = NULL; + return err; +} + +static int init_rdma_tx_root_ns(struct mlx5_flow_steering *steering) +{ + int err; + + steering->rdma_tx_root_ns = create_root_ns(steering, FS_FT_RDMA_TX); + if (!steering->rdma_tx_root_ns) + return -ENOMEM; + + err = init_root_tree(steering, &rdma_tx_root_fs, + &steering->rdma_tx_root_ns->ns.node); + if (err) + goto out_err; + + set_prio_attrs(steering->rdma_tx_root_ns); + + return 0; + +out_err: + cleanup_root_ns(steering->rdma_tx_root_ns); + steering->rdma_tx_root_ns = NULL; + return err; +} + +/* FT and tc chains are stored in the same array so we can re-use the + * mlx5_get_fdb_sub_ns() and tc api for FT chains. + * When creating a new ns for each chain store it in the first available slot. + * Assume tc chains are created and stored first and only then the FT chain. + */ +static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering, + struct mlx5_flow_namespace *ns) +{ + int chain = 0; + + while (steering->fdb_sub_ns[chain]) + ++chain; + + steering->fdb_sub_ns[chain] = ns; +} + +static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering, + struct fs_prio *maj_prio) +{ + struct mlx5_flow_namespace *ns; + struct fs_prio *min_prio; + int prio; + + ns = fs_create_namespace(maj_prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF); + if (IS_ERR(ns)) + return PTR_ERR(ns); + + for (prio = 0; prio < FDB_TC_MAX_PRIO; prio++) { + min_prio = fs_create_prio(ns, prio, FDB_TC_LEVELS_PER_PRIO); + if (IS_ERR(min_prio)) + return PTR_ERR(min_prio); + } + + store_fdb_sub_ns_prio_chain(steering, ns); + + return 0; +} + +static int create_fdb_chains(struct mlx5_flow_steering *steering, + int fs_prio, + int chains) +{ + struct fs_prio *maj_prio; + int levels; + int chain; + int err; + + levels = FDB_TC_LEVELS_PER_PRIO * FDB_TC_MAX_PRIO * chains; + maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns, + fs_prio, + levels); + if (IS_ERR(maj_prio)) + return PTR_ERR(maj_prio); + + for (chain = 0; chain < chains; chain++) { + err = create_fdb_sub_ns_prio_chain(steering, maj_prio); + if (err) + return err; + } + + return 0; +} + +static int create_fdb_fast_path(struct mlx5_flow_steering *steering) +{ + int err; + + steering->fdb_sub_ns = kcalloc(FDB_NUM_CHAINS, + sizeof(*steering->fdb_sub_ns), + GFP_KERNEL); + if (!steering->fdb_sub_ns) + return -ENOMEM; + + err = create_fdb_chains(steering, FDB_TC_OFFLOAD, FDB_TC_MAX_CHAIN + 1); + if (err) + return err; + + err = create_fdb_chains(steering, FDB_FT_OFFLOAD, 1); + if (err) + return err; + + return 0; +} + +static int create_fdb_bypass(struct mlx5_flow_steering *steering) +{ + struct mlx5_flow_namespace *ns; + struct fs_prio *prio; + int i; + + prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH, 0); + if (IS_ERR(prio)) + return PTR_ERR(prio); + + ns = fs_create_namespace(prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF); + if (IS_ERR(ns)) + return PTR_ERR(ns); + + for (i = 0; i < MLX5_BY_PASS_NUM_REGULAR_PRIOS; i++) { + prio = fs_create_prio(ns, i, 1); + if (IS_ERR(prio)) + return PTR_ERR(prio); + } + return 0; +} + +static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering) +{ + cleanup_root_ns(steering->fdb_root_ns); + steering->fdb_root_ns = NULL; + kfree(steering->fdb_sub_ns); + steering->fdb_sub_ns = NULL; +} + +static int init_fdb_root_ns(struct mlx5_flow_steering *steering) +{ + struct fs_prio *maj_prio; + int err; + + steering->fdb_root_ns = create_root_ns(steering, FS_FT_FDB); + if (!steering->fdb_root_ns) + return -ENOMEM; + + err = create_fdb_bypass(steering); + if (err) + goto out_err; + + err = create_fdb_fast_path(steering); + if (err) + goto out_err; + + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_TC_MISS, 1); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 4); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + + /* We put this priority last, knowing that nothing will get here + * unless explicitly forwarded to. This is possible because the + * slow path tables have catch all rules and nothing gets passed + * those tables. + */ + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_PER_VPORT, 1); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + + set_prio_attrs(steering->fdb_root_ns); + return 0; + +out_err: + cleanup_fdb_root_ns(steering); + return err; +} + +static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport) +{ + struct fs_prio *prio; + + steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL); + if (!steering->esw_egress_root_ns[vport]) + return -ENOMEM; + + /* create 1 prio*/ + prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1); + return PTR_ERR_OR_ZERO(prio); +} + +static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport) +{ + struct fs_prio *prio; + + steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL); + if (!steering->esw_ingress_root_ns[vport]) + return -ENOMEM; + + /* create 1 prio*/ + prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1); + return PTR_ERR_OR_ZERO(prio); +} + +int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int err; + int i; + + steering->esw_egress_root_ns = + kcalloc(total_vports, + sizeof(*steering->esw_egress_root_ns), + GFP_KERNEL); + if (!steering->esw_egress_root_ns) + return -ENOMEM; + + for (i = 0; i < total_vports; i++) { + err = init_egress_acl_root_ns(steering, i); + if (err) + goto cleanup_root_ns; + } + steering->esw_egress_acl_vports = total_vports; + return 0; + +cleanup_root_ns: + for (i--; i >= 0; i--) + cleanup_root_ns(steering->esw_egress_root_ns[i]); + kfree(steering->esw_egress_root_ns); + steering->esw_egress_root_ns = NULL; + return err; +} + +void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int i; + + if (!steering->esw_egress_root_ns) + return; + + for (i = 0; i < steering->esw_egress_acl_vports; i++) + cleanup_root_ns(steering->esw_egress_root_ns[i]); + + kfree(steering->esw_egress_root_ns); + steering->esw_egress_root_ns = NULL; +} + +int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int err; + int i; + + steering->esw_ingress_root_ns = + kcalloc(total_vports, + sizeof(*steering->esw_ingress_root_ns), + GFP_KERNEL); + if (!steering->esw_ingress_root_ns) + return -ENOMEM; + + for (i = 0; i < total_vports; i++) { + err = init_ingress_acl_root_ns(steering, i); + if (err) + goto cleanup_root_ns; + } + steering->esw_ingress_acl_vports = total_vports; + return 0; + +cleanup_root_ns: + for (i--; i >= 0; i--) + cleanup_root_ns(steering->esw_ingress_root_ns[i]); + kfree(steering->esw_ingress_root_ns); + steering->esw_ingress_root_ns = NULL; + return err; +} + +void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int i; + + if (!steering->esw_ingress_root_ns) + return; + + for (i = 0; i < steering->esw_ingress_acl_vports; i++) + cleanup_root_ns(steering->esw_ingress_root_ns[i]); + + kfree(steering->esw_ingress_root_ns); + steering->esw_ingress_root_ns = NULL; +} + +u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_flow_namespace *ns; + + ns = mlx5_get_flow_namespace(dev, type); + if (!ns) + return 0; + + root = find_root(&ns->node); + if (!root) + return 0; + + return root->cmds->get_capabilities(root, root->table_type); +} + +static int init_egress_root_ns(struct mlx5_flow_steering *steering) +{ + int err; + + steering->egress_root_ns = create_root_ns(steering, + FS_FT_NIC_TX); + if (!steering->egress_root_ns) + return -ENOMEM; + + err = init_root_tree(steering, &egress_root_fs, + &steering->egress_root_ns->ns.node); + if (err) + goto cleanup; + set_prio_attrs(steering->egress_root_ns); + return 0; +cleanup: + cleanup_root_ns(steering->egress_root_ns); + steering->egress_root_ns = NULL; + return err; +} + +void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + + cleanup_root_ns(steering->root_ns); + cleanup_fdb_root_ns(steering); + cleanup_root_ns(steering->port_sel_root_ns); + cleanup_root_ns(steering->sniffer_rx_root_ns); + cleanup_root_ns(steering->sniffer_tx_root_ns); + cleanup_root_ns(steering->rdma_rx_root_ns); + cleanup_root_ns(steering->rdma_tx_root_ns); + cleanup_root_ns(steering->egress_root_ns); +} + +int mlx5_fs_core_init(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int err; + + if ((((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && + (MLX5_CAP_GEN(dev, nic_flow_table))) || + ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) && + MLX5_CAP_GEN(dev, ipoib_enhanced_offloads))) && + MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_support)) { + err = init_root_ns(steering); + if (err) + goto err; + } + + if (MLX5_ESWITCH_MANAGER(dev)) { + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ft_support)) { + err = init_fdb_root_ns(steering); + if (err) + goto err; + } + err = mlx5_fs_egress_acls_init(dev, MAX_VPORTS); + if (err) + goto err; + err = mlx5_fs_ingress_acls_init(dev, MAX_VPORTS); + if (err) + goto err; + } + + if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev, ft_support)) { + err = init_sniffer_rx_root_ns(steering); + if (err) + goto err; + } + + if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev, ft_support)) { + err = init_sniffer_tx_root_ns(steering); + if (err) + goto err; + } + + if (MLX5_CAP_FLOWTABLE_PORT_SELECTION(dev, ft_support)) { + err = init_port_sel_root_ns(steering); + if (err) + goto err; + } + + if (MLX5_CAP_FLOWTABLE_RDMA_RX(dev, ft_support) && + MLX5_CAP_FLOWTABLE_RDMA_RX(dev, table_miss_action_domain)) { + err = init_rdma_rx_root_ns(steering); + if (err) + goto err; + } + + if (MLX5_CAP_FLOWTABLE_RDMA_TX(dev, ft_support)) { + err = init_rdma_tx_root_ns(steering); + if (err) + goto err; + } + + if (MLX5_CAP_FLOWTABLE_NIC_TX(dev, ft_support)) { + err = init_egress_root_ns(steering); + if (err) + goto err; + } + + return 0; + +err: + mlx5_fs_core_cleanup(dev); + return err; +} + +void mlx5_fs_core_free(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + + kmem_cache_destroy(steering->ftes_cache); + kmem_cache_destroy(steering->fgs_cache); + kfree(steering); + mlx5_ft_pool_destroy(dev); + mlx5_cleanup_fc_stats(dev); +} + +int mlx5_fs_core_alloc(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering; + int err = 0; + + err = mlx5_init_fc_stats(dev); + if (err) + return err; + + err = mlx5_ft_pool_init(dev); + if (err) + goto err; + + steering = kzalloc(sizeof(*steering), GFP_KERNEL); + if (!steering) { + err = -ENOMEM; + goto err; + } + + steering->dev = dev; + dev->priv.steering = steering; + + steering->mode = MLX5_FLOW_STEERING_MODE_DMFS; + + steering->fgs_cache = kmem_cache_create("mlx5_fs_fgs", + sizeof(struct mlx5_flow_group), 0, + 0, NULL); + steering->ftes_cache = kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte), 0, + 0, NULL); + if (!steering->ftes_cache || !steering->fgs_cache) { + err = -ENOMEM; + goto err; + } + + return 0; + +err: + mlx5_fs_core_free(dev); + return err; +} + +int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn) +{ + struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns; + struct mlx5_ft_underlay_qp *new_uqp; + int err = 0; + + new_uqp = kzalloc(sizeof(*new_uqp), GFP_KERNEL); + if (!new_uqp) + return -ENOMEM; + + mutex_lock(&root->chain_lock); + + if (!root->root_ft) { + err = -EINVAL; + goto update_ft_fail; + } + + err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn, + false); + if (err) { + mlx5_core_warn(dev, "Failed adding underlay QPN (%u) to root FT err(%d)\n", + underlay_qpn, err); + goto update_ft_fail; + } + + new_uqp->qpn = underlay_qpn; + list_add_tail(&new_uqp->list, &root->underlay_qpns); + + mutex_unlock(&root->chain_lock); + + return 0; + +update_ft_fail: + mutex_unlock(&root->chain_lock); + kfree(new_uqp); + return err; +} +EXPORT_SYMBOL(mlx5_fs_add_rx_underlay_qpn); + +int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn) +{ + struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns; + struct mlx5_ft_underlay_qp *uqp; + bool found = false; + int err = 0; + + mutex_lock(&root->chain_lock); + list_for_each_entry(uqp, &root->underlay_qpns, list) { + if (uqp->qpn == underlay_qpn) { + found = true; + break; + } + } + + if (!found) { + mlx5_core_warn(dev, "Failed finding underlay qp (%u) in qpn list\n", + underlay_qpn); + err = -EINVAL; + goto out; + } + + err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn, + true); + if (err) + mlx5_core_warn(dev, "Failed removing underlay QPN (%u) from root FT err(%d)\n", + underlay_qpn, err); + + list_del(&uqp->list); + mutex_unlock(&root->chain_lock); + kfree(uqp); + + return 0; + +out: + mutex_unlock(&root->chain_lock); + return err; +} +EXPORT_SYMBOL(mlx5_fs_remove_rx_underlay_qpn); + +static struct mlx5_flow_root_namespace +*get_root_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type ns_type) +{ + struct mlx5_flow_namespace *ns; + + if (ns_type == MLX5_FLOW_NAMESPACE_ESW_EGRESS || + ns_type == MLX5_FLOW_NAMESPACE_ESW_INGRESS) + ns = mlx5_get_flow_vport_acl_namespace(dev, ns_type, 0); + else + ns = mlx5_get_flow_namespace(dev, ns_type); + if (!ns) + return NULL; + + return find_root(&ns->node); +} + +struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev, + enum mlx5_flow_namespace_type ns_type, + u8 num_actions, + void *modify_actions) +{ + struct mlx5_flow_root_namespace *root; + struct mlx5_modify_hdr *modify_hdr; + int err; + + root = get_root_namespace(dev, ns_type); + if (!root) + return ERR_PTR(-EOPNOTSUPP); + + modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL); + if (!modify_hdr) + return ERR_PTR(-ENOMEM); + + modify_hdr->ns_type = ns_type; + err = root->cmds->modify_header_alloc(root, ns_type, num_actions, + modify_actions, modify_hdr); + if (err) { + kfree(modify_hdr); + return ERR_PTR(err); + } + + return modify_hdr; +} +EXPORT_SYMBOL(mlx5_modify_header_alloc); + +void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev, + struct mlx5_modify_hdr *modify_hdr) +{ + struct mlx5_flow_root_namespace *root; + + root = get_root_namespace(dev, modify_hdr->ns_type); + if (WARN_ON(!root)) + return; + root->cmds->modify_header_dealloc(root, modify_hdr); + kfree(modify_hdr); +} +EXPORT_SYMBOL(mlx5_modify_header_dealloc); + +struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev, + struct mlx5_pkt_reformat_params *params, + enum mlx5_flow_namespace_type ns_type) +{ + struct mlx5_pkt_reformat *pkt_reformat; + struct mlx5_flow_root_namespace *root; + int err; + + root = get_root_namespace(dev, ns_type); + if (!root) + return ERR_PTR(-EOPNOTSUPP); + + pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL); + if (!pkt_reformat) + return ERR_PTR(-ENOMEM); + + pkt_reformat->ns_type = ns_type; + pkt_reformat->reformat_type = params->type; + err = root->cmds->packet_reformat_alloc(root, params, ns_type, + pkt_reformat); + if (err) { + kfree(pkt_reformat); + return ERR_PTR(err); + } + + return pkt_reformat; +} +EXPORT_SYMBOL(mlx5_packet_reformat_alloc); + +void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev, + struct mlx5_pkt_reformat *pkt_reformat) +{ + struct mlx5_flow_root_namespace *root; + + root = get_root_namespace(dev, pkt_reformat->ns_type); + if (WARN_ON(!root)) + return; + root->cmds->packet_reformat_dealloc(root, pkt_reformat); + kfree(pkt_reformat); +} +EXPORT_SYMBOL(mlx5_packet_reformat_dealloc); + +int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_root_namespace *peer_ns) +{ + if (peer_ns && ns->mode != peer_ns->mode) { + mlx5_core_err(ns->dev, + "Can't peer namespace of different steering mode\n"); + return -EINVAL; + } + + return ns->cmds->set_peer(ns, peer_ns); +} + +/* This function should be called only at init stage of the namespace. + * It is not safe to call this function while steering operations + * are executed in the namespace. + */ +int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns, + enum mlx5_flow_steering_mode mode) +{ + struct mlx5_flow_root_namespace *root; + const struct mlx5_flow_cmds *cmds; + int err; + + root = find_root(&ns->node); + if (&root->ns != ns) + /* Can't set cmds to non root namespace */ + return -EINVAL; + + if (root->table_type != FS_FT_FDB) + return -EOPNOTSUPP; + + if (root->mode == mode) + return 0; + + cmds = mlx5_fs_cmd_get_fw_cmds(); + if (!cmds) + return -EOPNOTSUPP; + + err = cmds->create_ns(root); + if (err) { + mlx5_core_err(root->dev, "Failed to create flow namespace (%d)\n", + err); + return err; + } + + root->cmds->destroy_ns(root); + root->cmds = cmds; + root->mode = mode; + + return 0; +} diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c index 7214c5256388..f8c7b3adc2c0 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c @@ -31,7 +31,7 @@ #include <linux/rbtree.h> #include <dev/mlx5/mlx5_core/mlx5_core.h> #include <dev/mlx5/mlx5_core/fs_core.h> -#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h> +#include <dev/mlx5/mlx5_core/fs_cmd.h> #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000) #define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000) diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c new file mode 100644 index 000000000000..70d9d235b629 --- /dev/null +++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021 Mellanox Technologies. */ + +#include "fs_ft_pool.h" + +/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS), + * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated + * for each flow table pool. We can allocate up to 16M of each pool, + * and we keep track of how much we used via mlx5_ft_pool_get_avail_sz. + * Firmware doesn't report any of this for now. + * ESW_POOL is expected to be sorted from large to small and match firmware + * pools. + */ +#define FT_SIZE (16 * 1024 * 1024) +static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024, + 1 * 1024 * 1024, + 64 * 1024, + 128, + 1 /* size for termination tables */ }; +struct mlx5_ft_pool { + int ft_left[ARRAY_SIZE(FT_POOLS)]; +}; + +int mlx5_ft_pool_init(struct mlx5_core_dev *dev) +{ + struct mlx5_ft_pool *ft_pool; + int i; + + ft_pool = kzalloc(sizeof(*ft_pool), GFP_KERNEL); + if (!ft_pool) + return -ENOMEM; + + for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) + ft_pool->ft_left[i] = FT_SIZE / FT_POOLS[i]; + + dev->priv.ft_pool = ft_pool; + return 0; +} + +void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev) +{ + kfree(dev->priv.ft_pool); +} + +int +mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type, + int desired_size) +{ + u32 max_ft_size = 1 << MLX5_CAP_FLOWTABLE_TYPE(dev, log_max_ft_size, table_type); + int i, found_i = -1; + + for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { + if (dev->priv.ft_pool->ft_left[i] && + (FT_POOLS[i] >= desired_size || desired_size == POOL_NEXT_SIZE) && + FT_POOLS[i] <= max_ft_size) { + found_i = i; + if (desired_size != POOL_NEXT_SIZE) + break; + } + } + + if (found_i != -1) { + --dev->priv.ft_pool->ft_left[found_i]; + return FT_POOLS[found_i]; + } + + return 0; +} + +void +mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz) +{ + int i; + + if (!sz) + return; + + for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { + if (sz == FT_POOLS[i]) { + ++dev->priv.ft_pool->ft_left[i]; + return; + } + } + + WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz); +} diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c index d7d63d7932a1..f69c36aa72de 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c @@ -81,12 +81,12 @@ accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) #endif void -mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule) +mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *rule) { - mlx5_del_flow_rule(&rule); + mlx5_del_flow_rules(&rule); } -struct mlx5_flow_rule * +struct mlx5_flow_handle * mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv, struct inpcb *inp, uint32_t tirn, uint32_t flow_tag, uint16_t vlan_id) @@ -96,18 +96,17 @@ mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv, #if defined(INET) || defined(INET6) struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp; #endif - struct mlx5_flow_rule *flow; + struct mlx5_flow_handle *flow; struct mlx5_flow_spec *spec; - struct mlx5_flow_act flow_act = { - .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, - .flow_tag = flow_tag, - }; + struct mlx5_flow_act flow_act = {}; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return (ERR_PTR(-ENOMEM)); spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG; + spec->flow_context.flow_tag = flow_tag; INP_RLOCK(inp); /* Set VLAN ID to match, if any. */ @@ -160,13 +159,9 @@ mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv, dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; dest.tir_num = tirn; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable, - spec->match_criteria, - spec->match_value, - MLX5_FLOW_RULE_FWD_ACTION_DEST, - &flow_act, - &dest); + flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); out: kvfree(spec); return (flow); @@ -175,18 +170,18 @@ out: static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type) { - static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)]; - static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)]; + static struct mlx5_flow_spec spec = {}; struct mlx5_flow_destination dest = {}; struct mlx5e_accel_fs_tcp *fs_tcp; - struct mlx5_flow_rule *rule; + struct mlx5_flow_handle *rule; struct mlx5_flow_act flow_act = { - .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, - .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG, + .action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, }; fs_tcp = &priv->fts.accel_tcp; + spec.flow_context.flags = FLOW_CONTEXT_HAS_TAG; + spec.flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; /* @@ -197,10 +192,11 @@ accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type) * of flow tables. */ dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ? - priv->fts.vlan.t : fs_tcp->tables[type + 1].t; + ((priv->fts.ipsec_ft) ? priv->fts.ipsec_ft : priv->fts.vlan.t) : + fs_tcp->tables[type + 1].t; - rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value, - MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); + rule = mlx5_add_flow_rules(fs_tcp->tables[type].t, &spec, &flow_act, + &dest, 1); if (IS_ERR(rule)) return (PTR_ERR(rule)); @@ -317,11 +313,13 @@ static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type) { struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type]; + struct mlx5_flow_table_attr ft_attr = {}; int err; ft->num_groups = 0; - ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp", - MLX5E_ACCEL_FS_TCP_TABLE_SIZE); + ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE; + ft_attr.level = type; + ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -365,7 +363,7 @@ mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) return; for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { - mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]); + mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]); accel_fs_tcp_destroy_table(priv, i); } } @@ -402,7 +400,7 @@ mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) err_destroy_rules: while (i--) - mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]); + mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]); i = MLX5E_ACCEL_FS_TCP_NUM_TYPES; err_destroy_tables: diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c deleted file mode 100644 index b76ea7b60582..000000000000 --- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c +++ /dev/null @@ -1,2874 +0,0 @@ -/*- - * Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "opt_rss.h" -#include "opt_ratelimit.h" - -#include <linux/module.h> -#include <dev/mlx5/driver.h> -#include <dev/mlx5/mlx5_core/mlx5_core.h> -#include <dev/mlx5/mlx5_core/fs_core.h> -#include <linux/string.h> -#include <linux/compiler.h> - -#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ - sizeof(struct init_tree_node)) - -#define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \ - ...) {.type = FS_TYPE_PRIO,\ - .name = name_val,\ - .min_ft_level = min_level_val,\ - .flags = flags_val,\ - .max_ft = max_ft_val,\ - .caps = caps_val,\ - .children = (struct init_tree_node[]) {__VA_ARGS__},\ - .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ -} - -#define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\ - ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\ - __VA_ARGS__)\ - -#define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\ - .name = name_val,\ - .children = (struct init_tree_node[]) {__VA_ARGS__},\ - .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ -} - -#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ - sizeof(long)) - -#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) - -#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ - .caps = (long[]) {__VA_ARGS__}} - -/* Flowtable sizes: */ -#define BYPASS_MAX_FT 5 -#define BYPASS_PRIO_MAX_FT 1 -#define OFFLOADS_MAX_FT 2 -#define KERNEL_MAX_FT 5 -#define LEFTOVER_MAX_FT 1 - -/* Flowtable levels: */ -#define OFFLOADS_MIN_LEVEL 3 -#define KERNEL_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1) -#define LEFTOVER_MIN_LEVEL (KERNEL_MIN_LEVEL + 1) -#define BYPASS_MIN_LEVEL (MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL) - -struct node_caps { - size_t arr_sz; - long *caps; -}; - -struct init_tree_node { - enum fs_type type; - const char *name; - struct init_tree_node *children; - int ar_size; - struct node_caps caps; - u8 flags; - int min_ft_level; - int prio; - int max_ft; -} root_fs = { - .type = FS_TYPE_NAMESPACE, - .name = "root", - .ar_size = 4, - .children = (struct init_tree_node[]) { - ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0, - FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), - FS_CAP(flow_table_properties_nic_receive.modify_root)), - ADD_NS("by_pass_ns", - ADD_FT_PRIO("prio0", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio1", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio2", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio3", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio4", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio5", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio6", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio7", 0, - BYPASS_PRIO_MAX_FT), - ADD_FT_PRIO("prio-mcast", 0, - BYPASS_PRIO_MAX_FT))), - ADD_PRIO("offloads_prio", 0, OFFLOADS_MIN_LEVEL, 0, {}, - ADD_NS("offloads_ns", - ADD_FT_PRIO("prio_offloads-0", 0, - OFFLOADS_MAX_FT))), - ADD_PRIO("kernel_prio", 0, KERNEL_MIN_LEVEL, 0, {}, - ADD_NS("kernel_ns", - ADD_FT_PRIO("prio_kernel-0", 0, - KERNEL_MAX_FT))), - ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED, - LEFTOVER_MIN_LEVEL, 0, - FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), - FS_CAP(flow_table_properties_nic_receive.modify_root)), - ADD_NS("leftover_ns", - ADD_FT_PRIO("leftovers_prio-0", - MLX5_CORE_FS_PRIO_SHARED, - LEFTOVER_MAX_FT))) - } -}; - -/* Tree creation functions */ - -static struct mlx5_flow_root_namespace *find_root(struct fs_base *node) -{ - struct fs_base *parent; - - /* Make sure we only read it once while we go up the tree */ - while ((parent = node->parent)) - node = parent; - - if (node->type != FS_TYPE_NAMESPACE) { - return NULL; - } - - return container_of(container_of(node, - struct mlx5_flow_namespace, - base), - struct mlx5_flow_root_namespace, - ns); -} - -static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node) -{ - struct mlx5_flow_root_namespace *root = find_root(node); - - if (root) - return root->dev; - return NULL; -} - -static void fs_init_node(struct fs_base *node, - unsigned int refcount) -{ - kref_init(&node->refcount); - atomic_set(&node->users_refcount, refcount); - init_completion(&node->complete); - INIT_LIST_HEAD(&node->list); - mutex_init(&node->lock); -} - -static void _fs_add_node(struct fs_base *node, - const char *name, - struct fs_base *parent) -{ - if (parent) - atomic_inc(&parent->users_refcount); - node->name = kstrdup_const(name, GFP_KERNEL); - node->parent = parent; -} - -static void fs_add_node(struct fs_base *node, - struct fs_base *parent, const char *name, - unsigned int refcount) -{ - fs_init_node(node, refcount); - _fs_add_node(node, name, parent); -} - -static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), - bool parent_locked); - -static void fs_del_dst(struct mlx5_flow_rule *dst); -static void _fs_del_ft(struct mlx5_flow_table *ft); -static void fs_del_fg(struct mlx5_flow_group *fg); -static void fs_del_fte(struct fs_fte *fte); - -static void cmd_remove_node(struct fs_base *base) -{ - switch (base->type) { - case FS_TYPE_FLOW_DEST: - fs_del_dst(container_of(base, struct mlx5_flow_rule, base)); - break; - case FS_TYPE_FLOW_TABLE: - _fs_del_ft(container_of(base, struct mlx5_flow_table, base)); - break; - case FS_TYPE_FLOW_GROUP: - fs_del_fg(container_of(base, struct mlx5_flow_group, base)); - break; - case FS_TYPE_FLOW_ENTRY: - fs_del_fte(container_of(base, struct fs_fte, base)); - break; - default: - break; - } -} - -static void __fs_remove_node(struct kref *kref) -{ - struct fs_base *node = container_of(kref, struct fs_base, refcount); - - if (node->parent) { - if (node->type == FS_TYPE_FLOW_DEST) - mutex_lock(&node->parent->parent->lock); - mutex_lock(&node->parent->lock); - } - mutex_lock(&node->lock); - cmd_remove_node(node); - mutex_unlock(&node->lock); - complete(&node->complete); - if (node->parent) { - mutex_unlock(&node->parent->lock); - if (node->type == FS_TYPE_FLOW_DEST) - mutex_unlock(&node->parent->parent->lock); - _fs_put(node->parent, _fs_remove_node, false); - } -} - -void _fs_remove_node(struct kref *kref) -{ - struct fs_base *node = container_of(kref, struct fs_base, refcount); - - __fs_remove_node(kref); - kfree_const(node->name); - kfree(node); -} - -static void fs_get(struct fs_base *node) -{ - atomic_inc(&node->users_refcount); -} - -static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), - bool parent_locked) -{ - struct fs_base *parent_node = node->parent; - - if (parent_node && !parent_locked) - mutex_lock(&parent_node->lock); - if (atomic_dec_and_test(&node->users_refcount)) { - if (parent_node) { - /*remove from parent's list*/ - list_del_init(&node->list); - mutex_unlock(&parent_node->lock); - } - kref_put(&node->refcount, kref_cb); - if (parent_node && parent_locked) - mutex_lock(&parent_node->lock); - } else if (parent_node && !parent_locked) { - mutex_unlock(&parent_node->lock); - } -} - -static void fs_put(struct fs_base *node) -{ - _fs_put(node, __fs_remove_node, false); -} - -static void fs_put_parent_locked(struct fs_base *node) -{ - _fs_put(node, __fs_remove_node, true); -} - -static void fs_remove_node(struct fs_base *node) -{ - fs_put(node); - wait_for_completion(&node->complete); - kfree_const(node->name); - kfree(node); -} - -static void fs_remove_node_parent_locked(struct fs_base *node) -{ - fs_put_parent_locked(node); - wait_for_completion(&node->complete); - kfree_const(node->name); - kfree(node); -} - -static struct fs_fte *fs_alloc_fte(u32 sw_action, - struct mlx5_flow_act *flow_act, - u32 *match_value, - unsigned int index) -{ - struct fs_fte *fte; - - - fte = kzalloc(sizeof(*fte), GFP_KERNEL); - if (!fte) - return ERR_PTR(-ENOMEM); - - memcpy(fte->val, match_value, sizeof(fte->val)); - fte->base.type = FS_TYPE_FLOW_ENTRY; - fte->dests_size = 0; - fte->index = index; - INIT_LIST_HEAD(&fte->dests); - fte->flow_act = *flow_act; - fte->sw_action = sw_action; - - return fte; -} - -static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft, - struct mlx5_flow_group *fg, - u32 *match_value, - unsigned int index) -{ - int err; - struct fs_fte *fte; - struct mlx5_flow_rule *dst; - struct mlx5_flow_act flow_act = { - .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, - .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG, - }; - - if (fg->num_ftes == fg->max_ftes) - return ERR_PTR(-ENOSPC); - - fte = fs_alloc_fte(MLX5_FLOW_RULE_FWD_ACTION_DEST, - &flow_act, match_value, index); - if (IS_ERR(fte)) - return fte; - - /*create dst*/ - dst = kzalloc(sizeof(*dst), GFP_KERNEL); - if (!dst) { - err = -ENOMEM; - goto free_fte; - } - - fte->base.parent = &fg->base; - fte->dests_size = 1; - dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE; - dst->base.parent = &fte->base; - list_add(&dst->base.list, &fte->dests); - /* assumed that the callee creates the star rules sorted by index */ - list_add_tail(&fte->base.list, &fg->ftes); - fg->num_ftes++; - - return fte; - -free_fte: - kfree(fte); - return ERR_PTR(err); -} - -/* assume that fte can't be changed */ -static void free_star_fte_entry(struct fs_fte *fte) -{ - struct mlx5_flow_group *fg; - struct mlx5_flow_rule *dst, *temp; - - fs_get_parent(fg, fte); - - list_for_each_entry_safe(dst, temp, &fte->dests, base.list) { - fte->dests_size--; - list_del(&dst->base.list); - kfree(dst); - } - - list_del(&fte->base.list); - fg->num_ftes--; - kfree(fte); -} - -static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in) -{ - struct mlx5_flow_group *fg; - void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, - create_fg_in, match_criteria); - u8 match_criteria_enable = MLX5_GET(create_flow_group_in, - create_fg_in, - match_criteria_enable); - fg = kzalloc(sizeof(*fg), GFP_KERNEL); - if (!fg) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&fg->ftes); - fg->mask.match_criteria_enable = match_criteria_enable; - memcpy(&fg->mask.match_criteria, match_criteria, - sizeof(fg->mask.match_criteria)); - fg->base.type = FS_TYPE_FLOW_GROUP; - fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in, - start_flow_index); - fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in, - end_flow_index) - fg->start_index + 1; - return fg; -} - -static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio); -static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, - struct fs_prio *prio); - -/* assumed src_ft and dst_ft can't be freed */ -static int fs_set_star_rule(struct mlx5_core_dev *dev, - struct mlx5_flow_table *src_ft, - struct mlx5_flow_table *dst_ft) -{ - struct mlx5_flow_rule *src_dst; - struct fs_fte *src_fte; - int err = 0; - u32 *match_value; - int match_len = MLX5_ST_SZ_BYTES(fte_match_param); - - src_dst = list_first_entry(&src_ft->star_rule.fte->dests, - struct mlx5_flow_rule, base.list); - match_value = mlx5_vzalloc(match_len); - if (!match_value) { - mlx5_core_warn(dev, "failed to allocate inbox\n"); - return -ENOMEM; - } - /*Create match context*/ - - fs_get_parent(src_fte, src_dst); - - src_dst->dest_attr.ft = dst_ft; - if (dst_ft) { - err = mlx5_cmd_fs_set_fte(dev, - src_ft->vport, - &src_fte->status, - match_value, src_ft->type, - src_ft->id, src_fte->index, - src_ft->star_rule.fg->id, - &src_fte->flow_act, - src_fte->sw_action, - src_fte->dests_size, - &src_fte->dests); - if (err) - goto free; - - fs_get(&dst_ft->base); - } else { - mlx5_cmd_fs_delete_fte(dev, - src_ft->vport, - &src_fte->status, - src_ft->type, src_ft->id, - src_fte->index); - } - -free: - kvfree(match_value); - return err; -} - -static int connect_prev_fts(struct fs_prio *locked_prio, - struct fs_prio *prev_prio, - struct mlx5_flow_table *next_ft) -{ - struct mlx5_flow_table *iter; - int err = 0; - struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base); - - if (!dev) - return -ENODEV; - - mutex_lock(&prev_prio->base.lock); - fs_for_each_ft(iter, prev_prio) { - struct mlx5_flow_rule *src_dst = - list_first_entry(&iter->star_rule.fte->dests, - struct mlx5_flow_rule, base.list); - struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft; - - if (prev_ft == next_ft) - continue; - - err = fs_set_star_rule(dev, iter, next_ft); - if (err) { - mlx5_core_warn(dev, - "mlx5: flow steering can't connect prev and next\n"); - goto unlock; - } else { - /* Assume ft's prio is locked */ - if (prev_ft) { - struct fs_prio *prio; - - fs_get_parent(prio, prev_ft); - if (prio == locked_prio) - fs_put_parent_locked(&prev_ft->base); - else - fs_put(&prev_ft->base); - } - } - } - -unlock: - mutex_unlock(&prev_prio->base.lock); - return 0; -} - -static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) -{ - struct mlx5_flow_group *fg; - int err; - u32 *fg_in; - u32 *match_value; - struct mlx5_flow_table *next_ft; - struct mlx5_flow_table *prev_ft; - struct mlx5_flow_root_namespace *root = find_root(&prio->base); - int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - int match_len = MLX5_ST_SZ_BYTES(fte_match_param); - - fg_in = mlx5_vzalloc(fg_inlen); - if (!fg_in) { - mlx5_core_warn(root->dev, "failed to allocate inbox\n"); - return -ENOMEM; - } - - match_value = mlx5_vzalloc(match_len); - if (!match_value) { - mlx5_core_warn(root->dev, "failed to allocate inbox\n"); - kvfree(fg_in); - return -ENOMEM; - } - - MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte); - MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte); - fg = fs_alloc_fg(fg_in); - if (IS_ERR(fg)) { - err = PTR_ERR(fg); - goto out; - } - ft->star_rule.fg = fg; - err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base), - fg_in, ft->vport, ft->type, - ft->id, - &fg->id); - if (err) - goto free_fg; - - ft->star_rule.fte = alloc_star_ft_entry(ft, fg, - match_value, - ft->max_fte); - if (IS_ERR(ft->star_rule.fte)) - goto free_star_rule; - - mutex_lock(&root->fs_chain_lock); - next_ft = find_next_ft(prio); - err = fs_set_star_rule(root->dev, ft, next_ft); - if (err) { - mutex_unlock(&root->fs_chain_lock); - goto free_star_rule; - } - if (next_ft) { - struct fs_prio *parent; - - fs_get_parent(parent, next_ft); - fs_put(&next_ft->base); - } - prev_ft = find_prev_ft(ft, prio); - if (prev_ft) { - struct fs_prio *prev_parent; - - fs_get_parent(prev_parent, prev_ft); - - err = connect_prev_fts(NULL, prev_parent, ft); - if (err) { - mutex_unlock(&root->fs_chain_lock); - goto destroy_chained_star_rule; - } - fs_put(&prev_ft->base); - } - mutex_unlock(&root->fs_chain_lock); - kvfree(fg_in); - kvfree(match_value); - - return 0; - -destroy_chained_star_rule: - fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL); - if (next_ft) - fs_put(&next_ft->base); -free_star_rule: - free_star_fte_entry(ft->star_rule.fte); - mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport, - ft->type, ft->id, - fg->id); -free_fg: - kfree(fg); -out: - kvfree(fg_in); - kvfree(match_value); - return err; -} - -static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) -{ - int err; - struct mlx5_flow_root_namespace *root; - struct mlx5_core_dev *dev = fs_get_dev(&prio->base); - struct mlx5_flow_table *prev_ft, *next_ft; - struct fs_prio *prev_prio; - - WARN_ON(!dev); - - root = find_root(&prio->base); - if (!root) - mlx5_core_err(dev, - "flow steering failed to find root of priority %s", - prio->base.name); - - /* In order to ensure atomic deletion, first update - * prev ft to point on the next ft. - */ - mutex_lock(&root->fs_chain_lock); - prev_ft = find_prev_ft(ft, prio); - next_ft = find_next_ft(prio); - if (prev_ft) { - fs_get_parent(prev_prio, prev_ft); - /*Prev is connected to ft, only if ft is the first(last) in the prio*/ - err = connect_prev_fts(prio, prev_prio, next_ft); - if (err) - mlx5_core_warn(root->dev, - "flow steering can't connect prev and next of flow table\n"); - fs_put(&prev_ft->base); - } - - err = fs_set_star_rule(root->dev, ft, NULL); - /*One put is for fs_get in find next ft*/ - if (next_ft) { - fs_put(&next_ft->base); - if (!err) - fs_put(&next_ft->base); - } - - mutex_unlock(&root->fs_chain_lock); - err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id, - ft->star_rule.fg->id); - if (err) - mlx5_core_warn(dev, - "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index, - ft->base.name); - free_star_fte_entry(ft->star_rule.fte); - - kfree(ft->star_rule.fg); - ft->star_rule.fg = NULL; -} - -static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, - unsigned int prio) -{ - struct fs_prio *iter_prio; - - fs_for_each_prio(iter_prio, ns) { - if (iter_prio->prio == prio) - return iter_prio; - } - - return NULL; -} - -static unsigned int _alloc_new_level(struct fs_prio *prio, - struct mlx5_flow_namespace *match); - -static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns, - struct fs_prio *prio) -{ - unsigned int level = 0; - struct fs_prio *p; - - if (!ns) - return 0; - - mutex_lock(&ns->base.lock); - fs_for_each_prio(p, ns) { - if (p != prio) - level += p->max_ft; - else - break; - } - mutex_unlock(&ns->base.lock); - - fs_get_parent(prio, ns); - if (prio) - WARN_ON(prio->base.type != FS_TYPE_PRIO); - - return level + _alloc_new_level(prio, ns); -} - -/* Called under lock of priority, hence locking all upper objects */ -static unsigned int _alloc_new_level(struct fs_prio *prio, - struct mlx5_flow_namespace *match) -{ - struct mlx5_flow_namespace *ns; - struct fs_base *it; - unsigned int level = 0; - - if (!prio) - return 0; - - mutex_lock(&prio->base.lock); - fs_for_each_ns_or_ft_reverse(it, prio) { - if (it->type == FS_TYPE_NAMESPACE) { - struct fs_prio *p; - - fs_get_obj(ns, it); - - if (match != ns) { - mutex_lock(&ns->base.lock); - fs_for_each_prio(p, ns) - level += p->max_ft; - mutex_unlock(&ns->base.lock); - } else { - break; - } - } else { - struct mlx5_flow_table *ft; - - fs_get_obj(ft, it); - mutex_unlock(&prio->base.lock); - return level + ft->level + 1; - } - } - - fs_get_parent(ns, prio); - mutex_unlock(&prio->base.lock); - return __alloc_new_level(ns, prio) + level; -} - -static unsigned int alloc_new_level(struct fs_prio *prio) -{ - return _alloc_new_level(prio, NULL); -} - -static int update_root_ft_create(struct mlx5_flow_root_namespace *root, - struct mlx5_flow_table *ft) -{ - int err = 0; - int min_level = INT_MAX; - - if (root->root_ft) - min_level = root->root_ft->level; - - if (ft->level < min_level) - err = mlx5_cmd_update_root_ft(root->dev, ft->type, - ft->id); - else - return err; - - if (err) - mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", - ft->id); - else - root->root_ft = ft; - - return err; -} - -static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns, - u16 vport, - struct fs_prio *fs_prio, - int max_fte, - const char *name) -{ - struct mlx5_flow_table *ft; - int err; - int log_table_sz; - int ft_size; - char gen_name[20]; - struct mlx5_flow_root_namespace *root = find_root(&ns->base); - struct mlx5_core_dev *dev = fs_get_dev(&ns->base); - - if (!root) { - mlx5_core_err(dev, - "flow steering failed to find root of namespace %s", - ns->base.name); - return ERR_PTR(-ENODEV); - } - - if (fs_prio->num_ft == fs_prio->max_ft) - return ERR_PTR(-ENOSPC); - - ft = kzalloc(sizeof(*ft), GFP_KERNEL); - if (!ft) - return ERR_PTR(-ENOMEM); - - fs_init_node(&ft->base, 1); - INIT_LIST_HEAD(&ft->fgs); - - /* Temporarily WA until we expose the level set in the API */ - if (root->table_type == FS_FT_ESW_EGRESS_ACL || - root->table_type == FS_FT_ESW_INGRESS_ACL) - ft->level = 0; - else - ft->level = alloc_new_level(fs_prio); - - ft->base.type = FS_TYPE_FLOW_TABLE; - ft->vport = vport; - ft->type = root->table_type; - /*Two entries are reserved for star rules*/ - ft_size = roundup_pow_of_two(max_fte + 2); - /*User isn't aware to those rules*/ - ft->max_fte = ft_size - 2; - log_table_sz = ilog2(ft_size); - - if (name == NULL || name[0] == '\0') { - snprintf(gen_name, sizeof(gen_name), "flow_table_%u", ft->id); - name = gen_name; - } - - err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type, - ft->level, log_table_sz, name, &ft->id); - if (err) - goto free_ft; - - err = create_star_rule(ft, fs_prio); - if (err) - goto del_ft; - - if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev, - flow_table_properties_nic_receive.modify_root)) { - err = update_root_ft_create(root, ft); - if (err) - goto destroy_star_rule; - } - - _fs_add_node(&ft->base, name, &fs_prio->base); - - list_add_tail(&ft->base.list, &fs_prio->objs); - fs_prio->num_ft++; - - return ft; - -destroy_star_rule: - destroy_star_rule(ft, fs_prio); -del_ft: - mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id); -free_ft: - kfree(ft); - return ERR_PTR(err); -} - -static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns, - u16 vport, - unsigned int prio, - int max_fte, - const char *name) -{ - struct fs_prio *fs_prio = NULL; - fs_prio = find_prio(ns, prio); - if (!fs_prio) - return ERR_PTR(-EINVAL); - - return _create_ft_common(ns, vport, fs_prio, max_fte, name); -} - - -static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, - struct list_head *start); - -static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, - struct list_head *start); - -static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio) -{ - struct mlx5_flow_table *ft; - - ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs); - if (ft) { - ft->shared_refcount++; - return ft; - } - - return NULL; -} - -struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, - int prio, - const char *name, - int num_flow_table_entries, - int max_num_groups, - int num_reserved_entries) -{ - struct mlx5_flow_table *ft = NULL; - struct fs_prio *fs_prio; - bool is_shared_prio; - - if (max_num_groups > (num_flow_table_entries - num_reserved_entries)) - return ERR_PTR(-EINVAL); - if (num_reserved_entries > num_flow_table_entries) - return ERR_PTR(-EINVAL); - - fs_prio = find_prio(ns, prio); - if (!fs_prio) - return ERR_PTR(-EINVAL); - - is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED; - if (is_shared_prio) { - mutex_lock(&fs_prio->shared_lock); - ft = mlx5_create_autogrouped_shared_flow_table(fs_prio); - } - - if (ft) - goto return_ft; - - ft = create_ft_common(ns, 0, prio, num_flow_table_entries, - name); - if (IS_ERR(ft)) - goto return_ft; - - ft->autogroup.active = true; - ft->autogroup.max_types = max_num_groups; - ft->autogroup.max_fte = num_flow_table_entries - num_reserved_entries; - /* We save place for flow groups in addition to max types */ - ft->autogroup.group_size = ft->autogroup.max_fte / (max_num_groups + 1); - - if (is_shared_prio) - ft->shared_refcount = 1; - -return_ft: - if (is_shared_prio) - mutex_unlock(&fs_prio->shared_lock); - return ft; -} -EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); - -struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, - u16 vport, - int prio, - const char *name, - int num_flow_table_entries) -{ - return create_ft_common(ns, vport, prio, num_flow_table_entries, name); -} -EXPORT_SYMBOL(mlx5_create_vport_flow_table); - -struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, - int prio, - const char *name, - int num_flow_table_entries) -{ - return create_ft_common(ns, 0, prio, num_flow_table_entries, name); -} -EXPORT_SYMBOL(mlx5_create_flow_table); - -static void _fs_del_ft(struct mlx5_flow_table *ft) -{ - int err; - struct mlx5_core_dev *dev = fs_get_dev(&ft->base); - struct fs_prio *prio; - - err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id); - if (err) - mlx5_core_warn(dev, "flow steering can't destroy ft %s\n", - ft->base.name); - - fs_get_parent(prio, ft); - prio->num_ft--; -} - -static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root, - struct mlx5_flow_table *ft) -{ - int err = 0; - struct fs_prio *prio; - struct mlx5_flow_table *next_ft = NULL; - struct mlx5_flow_table *put_ft = NULL; - - if (root->root_ft != ft) - return 0; - - fs_get_parent(prio, ft); - /*Assuming objs containis only flow tables and - * flow tables are sorted by level. - */ - if (!list_is_last(&ft->base.list, &prio->objs)) { - next_ft = list_next_entry(ft, base.list); - } else { - next_ft = find_next_ft(prio); - put_ft = next_ft; - } - - if (next_ft) { - err = mlx5_cmd_update_root_ft(root->dev, next_ft->type, - next_ft->id); - if (err) - mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", - ft->id); - } - if (!err) - root->root_ft = next_ft; - - if (put_ft) - fs_put(&put_ft->base); - - return err; -} - -/*Objects in the same prio are destroyed in the reverse order they were createrd*/ -int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) -{ - int err = 0; - struct fs_prio *prio; - struct mlx5_flow_root_namespace *root; - bool is_shared_prio; - struct mlx5_core_dev *dev; - - fs_get_parent(prio, ft); - root = find_root(&prio->base); - dev = fs_get_dev(&prio->base); - - if (!root) { - mlx5_core_err(dev, - "flow steering failed to find root of priority %s", - prio->base.name); - return -ENODEV; - } - - is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED; - if (is_shared_prio) { - mutex_lock(&prio->shared_lock); - if (ft->shared_refcount > 1) { - --ft->shared_refcount; - fs_put(&ft->base); - mutex_unlock(&prio->shared_lock); - return 0; - } - } - - mutex_lock(&prio->base.lock); - mutex_lock(&ft->base.lock); - - err = update_root_ft_destroy(root, ft); - if (err) - goto unlock_ft; - - /* delete two last entries */ - destroy_star_rule(ft, prio); - - mutex_unlock(&ft->base.lock); - fs_remove_node_parent_locked(&ft->base); - mutex_unlock(&prio->base.lock); - if (is_shared_prio) - mutex_unlock(&prio->shared_lock); - - return err; - -unlock_ft: - mutex_unlock(&ft->base.lock); - mutex_unlock(&prio->base.lock); - if (is_shared_prio) - mutex_unlock(&prio->shared_lock); - - return err; -} -EXPORT_SYMBOL(mlx5_destroy_flow_table); - -static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev, - struct mlx5_flow_table *ft, - struct list_head *prev, - u32 *fg_in, - int refcount) -{ - struct mlx5_flow_group *fg; - unsigned int group_size; - int err; - char name[20]; - - fg = fs_alloc_fg(fg_in); - if (IS_ERR(fg)) - return fg; - - group_size = MLX5_GET(create_flow_group_in, fg_in, end_flow_index) - - MLX5_GET(create_flow_group_in, fg_in, start_flow_index) + 1; - err = mlx5_cmd_fs_create_fg(dev, fg_in, - ft->vport, ft->type, ft->id, - &fg->id); - if (err) - goto free_fg; - - mutex_lock(&ft->base.lock); - - if (ft->autogroup.active && group_size == ft->autogroup.group_size) - ft->autogroup.num_types++; - - snprintf(name, sizeof(name), "group_%u", fg->id); - /*Add node to tree*/ - fs_add_node(&fg->base, &ft->base, name, refcount); - /*Add node to group list*/ - list_add(&fg->base.list, prev); - mutex_unlock(&ft->base.lock); - - return fg; - -free_fg: - kfree(fg); - return ERR_PTR(err); -} - -struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, - u32 *in) -{ - struct mlx5_flow_group *fg; - struct mlx5_core_dev *dev = fs_get_dev(&ft->base); - unsigned int start_index; - - start_index = MLX5_GET(create_flow_group_in, in, start_flow_index); - if (!dev) - return ERR_PTR(-ENODEV); - - if (ft->autogroup.active && start_index < ft->autogroup.max_fte) - return ERR_PTR(-EPERM); - - fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1); - - return fg; -} -EXPORT_SYMBOL(mlx5_create_flow_group); - -/*Group is destoyed when all the rules in the group were removed*/ -static void fs_del_fg(struct mlx5_flow_group *fg) -{ - struct mlx5_flow_table *parent_ft; - struct mlx5_core_dev *dev; - - fs_get_parent(parent_ft, fg); - dev = fs_get_dev(&parent_ft->base); - WARN_ON(!dev); - - if (parent_ft->autogroup.active && - fg->max_ftes == parent_ft->autogroup.group_size && - fg->start_index < parent_ft->autogroup.max_fte) - parent_ft->autogroup.num_types--; - - if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport, - parent_ft->type, - parent_ft->id, fg->id)) - mlx5_core_warn(dev, "flow steering can't destroy fg\n"); -} - -void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) -{ - fs_remove_node(&fg->base); -} -EXPORT_SYMBOL(mlx5_destroy_flow_group); - -static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size) -{ - unsigned int i; - - /* TODO: optimize by comparing 64bits when possible */ - for (i = 0; i < size; i++, mask++, val1++, val2++) - if ((*((u8 *)val1) & (*(u8 *)mask)) != - ((*(u8 *)val2) & (*(u8 *)mask))) - return false; - - return true; -} - -bool fs_match_exact_val(struct mlx5_core_fs_mask *mask, - void *val1, void *val2) -{ - if (mask->match_criteria_enable & - 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) { - void *fte_match1 = MLX5_ADDR_OF(fte_match_param, - val1, outer_headers); - void *fte_match2 = MLX5_ADDR_OF(fte_match_param, - val2, outer_headers); - void *fte_mask = MLX5_ADDR_OF(fte_match_param, - mask->match_criteria, outer_headers); - - if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, - MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) - return false; - } - - if (mask->match_criteria_enable & - 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) { - void *fte_match1 = MLX5_ADDR_OF(fte_match_param, - val1, misc_parameters); - void *fte_match2 = MLX5_ADDR_OF(fte_match_param, - val2, misc_parameters); - void *fte_mask = MLX5_ADDR_OF(fte_match_param, - mask->match_criteria, misc_parameters); - - if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, - MLX5_ST_SZ_BYTES(fte_match_set_misc))) - return false; - } - if (mask->match_criteria_enable & - 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) { - void *fte_match1 = MLX5_ADDR_OF(fte_match_param, - val1, inner_headers); - void *fte_match2 = MLX5_ADDR_OF(fte_match_param, - val2, inner_headers); - void *fte_mask = MLX5_ADDR_OF(fte_match_param, - mask->match_criteria, inner_headers); - - if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, - MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) - return false; - } - return true; -} - -bool fs_match_exact_mask(u8 match_criteria_enable1, - u8 match_criteria_enable2, - void *mask1, void *mask2) -{ - return match_criteria_enable1 == match_criteria_enable2 && - !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param)); -} - -static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, - struct list_head *start); - -static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio, - struct list_head *start) -{ - struct fs_base *it = container_of(start, struct fs_base, list); - - if (!prio) - return NULL; - - fs_for_each_ns_or_ft_continue_reverse(it, prio) { - struct mlx5_flow_namespace *ns; - struct mlx5_flow_table *ft; - - if (it->type == FS_TYPE_FLOW_TABLE) { - fs_get_obj(ft, it); - fs_get(&ft->base); - return ft; - } - - fs_get_obj(ns, it); - WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); - - ft = find_first_ft_in_ns_reverse(ns, &ns->prios); - if (ft) - return ft; - } - - return NULL; -} - -static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio, - struct list_head *start) -{ - struct mlx5_flow_table *ft; - - if (!prio) - return NULL; - - mutex_lock(&prio->base.lock); - ft = _find_first_ft_in_prio_reverse(prio, start); - mutex_unlock(&prio->base.lock); - - return ft; -} - -static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, - struct list_head *start) -{ - struct fs_prio *prio; - - if (!ns) - return NULL; - - fs_get_obj(prio, container_of(start, struct fs_base, list)); - mutex_lock(&ns->base.lock); - fs_for_each_prio_continue_reverse(prio, ns) { - struct mlx5_flow_table *ft; - - ft = find_first_ft_in_prio_reverse(prio, &prio->objs); - if (ft) { - mutex_unlock(&ns->base.lock); - return ft; - } - } - mutex_unlock(&ns->base.lock); - - return NULL; -} - -/* Returned a held ft, assumed curr is protected, assumed curr's parent is - * locked - */ -static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, - struct fs_prio *prio) -{ - struct mlx5_flow_table *ft = NULL; - struct fs_base *curr_base; - - if (!curr) - return NULL; - - /* prio has either namespace or flow-tables, but not both */ - if (!list_empty(&prio->objs) && - list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) != - curr) - return NULL; - - while (!ft && prio) { - struct mlx5_flow_namespace *ns; - - fs_get_parent(ns, prio); - ft = find_first_ft_in_ns_reverse(ns, &prio->base.list); - curr_base = &ns->base; - fs_get_parent(prio, ns); - - if (prio && !ft) - ft = find_first_ft_in_prio_reverse(prio, - &curr_base->list); - } - return ft; -} - -static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio, - struct list_head *start) -{ - struct fs_base *it = container_of(start, struct fs_base, list); - - if (!prio) - return NULL; - - fs_for_each_ns_or_ft_continue(it, prio) { - struct mlx5_flow_namespace *ns; - struct mlx5_flow_table *ft; - - if (it->type == FS_TYPE_FLOW_TABLE) { - fs_get_obj(ft, it); - fs_get(&ft->base); - return ft; - } - - fs_get_obj(ns, it); - WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); - - ft = find_first_ft_in_ns(ns, &ns->prios); - if (ft) - return ft; - } - - return NULL; -} - -static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, - struct list_head *start) -{ - struct mlx5_flow_table *ft; - - if (!prio) - return NULL; - - mutex_lock(&prio->base.lock); - ft = _find_first_ft_in_prio(prio, start); - mutex_unlock(&prio->base.lock); - - return ft; -} - -static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, - struct list_head *start) -{ - struct fs_prio *prio; - - if (!ns) - return NULL; - - fs_get_obj(prio, container_of(start, struct fs_base, list)); - mutex_lock(&ns->base.lock); - fs_for_each_prio_continue(prio, ns) { - struct mlx5_flow_table *ft; - - ft = find_first_ft_in_prio(prio, &prio->objs); - if (ft) { - mutex_unlock(&ns->base.lock); - return ft; - } - } - mutex_unlock(&ns->base.lock); - - return NULL; -} - -/* returned a held ft, assumed curr is protected, assumed curr's parent is - * locked - */ -static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio) -{ - struct mlx5_flow_table *ft = NULL; - struct fs_base *curr_base; - - while (!ft && prio) { - struct mlx5_flow_namespace *ns; - - fs_get_parent(ns, prio); - ft = find_first_ft_in_ns(ns, &prio->base.list); - curr_base = &ns->base; - fs_get_parent(prio, ns); - - if (!ft && prio) - ft = _find_first_ft_in_prio(prio, &curr_base->list); - } - return ft; -} - - -/* called under ft mutex lock */ -static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, - u8 match_criteria_enable, - u32 *match_criteria) -{ - unsigned int group_size; - unsigned int candidate_index = 0; - struct mlx5_flow_group *g; - struct mlx5_flow_group *ret; - struct list_head *prev = &ft->fgs; - struct mlx5_core_dev *dev; - u32 *in; - int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - void *match_criteria_addr; - u32 max_fte = ft->autogroup.max_fte; - - if (!ft->autogroup.active) - return ERR_PTR(-ENOENT); - - dev = fs_get_dev(&ft->base); - if (!dev) - return ERR_PTR(-ENODEV); - - in = mlx5_vzalloc(inlen); - if (!in) { - mlx5_core_warn(dev, "failed to allocate inbox\n"); - return ERR_PTR(-ENOMEM); - } - - - if (ft->autogroup.num_types < ft->autogroup.max_types) - group_size = ft->autogroup.group_size; - else - group_size = 1; - - if (group_size == 0) { - mlx5_core_warn(dev, - "flow steering can't create group size of 0\n"); - ret = ERR_PTR(-EINVAL); - goto out; - } - - /* sorted by start_index */ - fs_for_each_fg(g, ft) { - if (candidate_index + group_size > g->start_index) - candidate_index = g->start_index + g->max_ftes; - else - break; - prev = &g->base.list; - } - - if (candidate_index + group_size > max_fte) { - ret = ERR_PTR(-ENOSPC); - goto out; - } - - MLX5_SET(create_flow_group_in, in, match_criteria_enable, - match_criteria_enable); - MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index); - MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index + - group_size - 1); - match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, - in, match_criteria); - memcpy(match_criteria_addr, match_criteria, - MLX5_ST_SZ_BYTES(fte_match_param)); - - ret = fs_create_fg(dev, ft, prev, in, 0); -out: - kvfree(in); - return ret; -} - -static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node) -{ - struct mlx5_flow_namespace *ns = NULL; - - while (node && (node->type != FS_TYPE_NAMESPACE || - list_empty(&container_of(node, struct - mlx5_flow_namespace, - base)->list_notifiers))) - node = node->parent; - - if (node) - fs_get_obj(ns, node); - - return ns; -} - - -/*Assumption- fte is locked*/ -static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst, - struct fs_fte *fte) -{ - struct mlx5_flow_namespace *ns; - struct mlx5_flow_handler *iter_handler; - struct fs_client_priv_data *iter_client; - void *data; - bool is_new_rule = list_first_entry(&fte->dests, - struct mlx5_flow_rule, - base.list) == dst; - int err; - - ns = get_ns_with_notifiers(&fte->base); - if (!ns) - return; - - down_read(&ns->notifiers_rw_sem); - list_for_each_entry(iter_handler, &ns->list_notifiers, - list) { - if (iter_handler->add_dst_cb) { - data = NULL; - mutex_lock(&dst->clients_lock); - list_for_each_entry( - iter_client, &dst->clients_data, list) { - if (iter_client->fs_handler == iter_handler) { - data = iter_client->client_dst_data; - break; - } - } - mutex_unlock(&dst->clients_lock); - err = iter_handler->add_dst_cb(dst, - is_new_rule, - data, - iter_handler->client_context); - if (err) - break; - } - } - up_read(&ns->notifiers_rw_sem); -} - -static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst, - struct fs_fte *fte) -{ - struct mlx5_flow_namespace *ns; - struct mlx5_flow_handler *iter_handler; - struct fs_client_priv_data *iter_client; - void *data; - bool ctx_changed = (fte->dests_size == 0); - - ns = get_ns_with_notifiers(&fte->base); - if (!ns) - return; - down_read(&ns->notifiers_rw_sem); - list_for_each_entry(iter_handler, &ns->list_notifiers, - list) { - data = NULL; - mutex_lock(&dst->clients_lock); - list_for_each_entry(iter_client, &dst->clients_data, list) { - if (iter_client->fs_handler == iter_handler) { - data = iter_client->client_dst_data; - break; - } - } - mutex_unlock(&dst->clients_lock); - if (iter_handler->del_dst_cb) { - iter_handler->del_dst_cb(dst, ctx_changed, data, - iter_handler->client_context); - } - } - up_read(&ns->notifiers_rw_sem); -} - -/* fte should not be deleted while calling this function */ -static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte, - struct mlx5_flow_group *fg, - struct mlx5_flow_destination *dest) -{ - struct mlx5_flow_table *ft; - struct mlx5_flow_rule *dst; - int err; - - dst = kzalloc(sizeof(*dst), GFP_KERNEL); - if (!dst) - return ERR_PTR(-ENOMEM); - - memcpy(&dst->dest_attr, dest, sizeof(*dest)); - dst->base.type = FS_TYPE_FLOW_DEST; - INIT_LIST_HEAD(&dst->clients_data); - mutex_init(&dst->clients_lock); - fs_get_parent(ft, fg); - /*Add dest to dests list- added as first element after the head*/ - list_add_tail(&dst->base.list, &fte->dests); - fte->dests_size++; - err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base), - ft->vport, - &fte->status, - fte->val, ft->type, - ft->id, fte->index, fg->id, &fte->flow_act, - fte->sw_action, fte->dests_size, &fte->dests); - if (err) - goto free_dst; - - list_del(&dst->base.list); - - return dst; - -free_dst: - list_del(&dst->base.list); - kfree(dst); - fte->dests_size--; - return ERR_PTR(err); -} - -static char *get_dest_name(struct mlx5_flow_destination *dest) -{ - char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL); - - switch (dest->type) { - case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE: - snprintf(name, 20, "dest_%s_%u", "flow_table", - dest->ft->id); - return name; - case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT: - snprintf(name, 20, "dest_%s_%u", "vport", - dest->vport_num); - return name; - case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR: - snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num); - return name; - default: - kfree(name); - return NULL; - } -} - -/* assumed fg is locked */ -static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg, - struct list_head **prev) -{ - struct fs_fte *fte; - unsigned int start = fg->start_index; - - if (prev) - *prev = &fg->ftes; - - /* assumed list is sorted by index */ - fs_for_each_fte(fte, fg) { - if (fte->index != start) - return start; - start++; - if (prev) - *prev = &fte->base.list; - } - - return start; -} - - -static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg, - u32 *match_value, - u32 sw_action, - struct mlx5_flow_act *flow_act, - struct list_head **prev) -{ - struct fs_fte *fte; - int index = 0; - - index = fs_get_free_fg_index(fg, prev); - fte = fs_alloc_fte(sw_action, flow_act, match_value, index); - if (IS_ERR(fte)) - return fte; - - return fte; -} - -static void add_rule_to_tree(struct mlx5_flow_rule *rule, - struct fs_fte *fte) -{ - char *dest_name; - - dest_name = get_dest_name(&rule->dest_attr); - fs_add_node(&rule->base, &fte->base, dest_name, 1); - /* re-add to list, since fs_add_node reset our list */ - list_add_tail(&rule->base.list, &fte->dests); - kfree(dest_name); - call_to_add_rule_notifiers(rule, fte); -} - -static void fs_del_dst(struct mlx5_flow_rule *dst) -{ - struct mlx5_flow_table *ft; - struct mlx5_flow_group *fg; - struct fs_fte *fte; - u32 *match_value; - struct mlx5_core_dev *dev = fs_get_dev(&dst->base); - int match_len = MLX5_ST_SZ_BYTES(fte_match_param); - int err; - - WARN_ON(!dev); - - match_value = mlx5_vzalloc(match_len); - if (!match_value) { - mlx5_core_warn(dev, "failed to allocate inbox\n"); - return; - } - - fs_get_parent(fte, dst); - fs_get_parent(fg, fte); - sx_assert(&fg->base.lock.sx, SX_XLOCKED); - memcpy(match_value, fte->val, sizeof(fte->val)); - /* ft can't be changed as fg is locked */ - fs_get_parent(ft, fg); - list_del(&dst->base.list); - fte->dests_size--; - if (fte->dests_size) { - err = mlx5_cmd_fs_set_fte(dev, ft->vport, - &fte->status, match_value, ft->type, - ft->id, fte->index, fg->id, - &fte->flow_act, fte->sw_action, - fte->dests_size, &fte->dests); - if (err) { - mlx5_core_warn(dev, "%s can't delete dst %s\n", - __func__, dst->base.name); - goto err; - } - } - call_to_del_rule_notifiers(dst, fte); -err: - kvfree(match_value); -} - -static void fs_del_fte(struct fs_fte *fte) -{ - struct mlx5_flow_table *ft; - struct mlx5_flow_group *fg; - int err; - struct mlx5_core_dev *dev; - - fs_get_parent(fg, fte); - fs_get_parent(ft, fg); - - dev = fs_get_dev(&ft->base); - WARN_ON(!dev); - - err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status, - ft->type, ft->id, fte->index); - if (err) - mlx5_core_warn(dev, "flow steering can't delete fte %s\n", - fte->base.name); - - fg->num_ftes--; -} - -static bool check_conflicting_actions(const struct mlx5_flow_act *act1, - const struct mlx5_flow_act *act2) -{ - u32 action1 = act1->actions; - u32 action2 = act2->actions; - u32 xored_actions; - - xored_actions = action1 ^ action2; - - if (xored_actions & (MLX5_FLOW_ACT_ACTIONS_FLOW_TAG)) - return true; - - if (action1 & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG && - act1->flow_tag != act2->flow_tag) - return true; - - /* Can even have complex actions in merged rules */ - if (action1 & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR) - return true; - - if (action1 & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT) - return true; - - if (action1 & MLX5_FLOW_ACT_ACTIONS_COUNT) - return true; - - return false; -} - -/* assuming parent fg is locked */ -/* Add dst algorithm */ -static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg, - u32 *match_value, - u32 sw_action, - struct mlx5_flow_act *flow_act, - struct mlx5_flow_destination *dest) -{ - struct fs_fte *fte; - struct mlx5_flow_rule *dst; - struct mlx5_flow_table *ft; - struct list_head *prev; - char fte_name[20]; - - mutex_lock(&fg->base.lock); - if (flow_act->flags & MLX5_FLOW_ACT_NO_APPEND) - goto insert_fte; - - fs_for_each_fte(fte, fg) { - /* TODO: Check of size against PRM max size */ - mutex_lock(&fte->base.lock); - if (fs_match_exact_val(&fg->mask, match_value, &fte->val) && - sw_action == fte->sw_action && - !check_conflicting_actions(flow_act, &fte->flow_act)) { - dst = _fs_add_dst_fte(fte, fg, dest); - mutex_unlock(&fte->base.lock); - if (IS_ERR(dst)) - goto unlock_fg; - goto add_rule; - } - mutex_unlock(&fte->base.lock); - } - -insert_fte: - fs_get_parent(ft, fg); - if (fg->num_ftes == fg->max_ftes) { - dst = ERR_PTR(-ENOSPC); - goto unlock_fg; - } - - fte = fs_create_fte(fg, match_value, sw_action, flow_act, &prev); - if (IS_ERR(fte)) { - dst = (void *)fte; - goto unlock_fg; - } - dst = _fs_add_dst_fte(fte, fg, dest); - if (IS_ERR(dst)) { - kfree(fte); - goto unlock_fg; - } - - fg->num_ftes++; - - snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index); - /* Add node to tree */ - fs_add_node(&fte->base, &fg->base, fte_name, 0); - list_add(&fte->base.list, prev); -add_rule: - add_rule_to_tree(dst, fte); -unlock_fg: - mutex_unlock(&fg->base.lock); - return dst; -} - -static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft, - u8 match_criteria_enable, - u32 *match_criteria, - u32 *match_value, - u32 sw_action, - struct mlx5_flow_act *flow_act, - struct mlx5_flow_destination *dest) -{ - /*? where dst_entry is allocated*/ - struct mlx5_flow_group *g; - struct mlx5_flow_rule *dst; - - fs_get(&ft->base); - mutex_lock(&ft->base.lock); - fs_for_each_fg(g, ft) - if (fs_match_exact_mask(g->mask.match_criteria_enable, - match_criteria_enable, - g->mask.match_criteria, - match_criteria)) { - mutex_unlock(&ft->base.lock); - - dst = fs_add_dst_fg(g, match_value, sw_action, flow_act, dest); - if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC) - goto unlock; - } - mutex_unlock(&ft->base.lock); - - g = create_autogroup(ft, match_criteria_enable, match_criteria); - if (IS_ERR(g)) { - dst = (void *)g; - goto unlock; - } - - dst = fs_add_dst_fg(g, match_value, - sw_action, flow_act, dest); - if (IS_ERR(dst)) { - /* Remove assumes refcount > 0 and autogroup creates a group - * with a refcount = 0. - */ - fs_get(&g->base); - fs_remove_node(&g->base); - goto unlock; - } - -unlock: - fs_put(&ft->base); - return dst; -} - -struct mlx5_flow_rule * -mlx5_add_flow_rule(struct mlx5_flow_table *ft, - u8 match_criteria_enable, - u32 *match_criteria, - u32 *match_value, - u32 sw_action, - struct mlx5_flow_act *flow_act, - struct mlx5_flow_destination *dest) -{ - struct mlx5_flow_rule *dst; - struct mlx5_flow_namespace *ns; - - ns = get_ns_with_notifiers(&ft->base); - if (ns) - down_read(&ns->dests_rw_sem); - dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria, - match_value, sw_action, flow_act, dest); - if (ns) - up_read(&ns->dests_rw_sem); - - return dst; - - -} -EXPORT_SYMBOL(mlx5_add_flow_rule); - -void mlx5_del_flow_rule(struct mlx5_flow_rule **pp) -{ - struct mlx5_flow_namespace *ns; - struct mlx5_flow_rule *dst; - - dst = *pp; - *pp = NULL; - - if (IS_ERR_OR_NULL(dst)) - return; - ns = get_ns_with_notifiers(&dst->base); - if (ns) - down_read(&ns->dests_rw_sem); - fs_remove_node(&dst->base); - if (ns) - up_read(&ns->dests_rw_sem); -} -EXPORT_SYMBOL(mlx5_del_flow_rule); - -#define MLX5_CORE_FS_ROOT_NS_NAME "root" -#define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root" -#define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root" -#define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root" -#define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root" -#define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root" -#define MLX5_CORE_FS_PRIO_MAX_FT 4 -#define MLX5_CORE_FS_PRIO_MAX_NS 1 - -static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, - unsigned prio, int max_ft, - const char *name, u8 flags) -{ - struct fs_prio *fs_prio; - - fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); - if (!fs_prio) - return ERR_PTR(-ENOMEM); - - fs_prio->base.type = FS_TYPE_PRIO; - fs_add_node(&fs_prio->base, &ns->base, name, 1); - fs_prio->max_ft = max_ft; - fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS; - fs_prio->prio = prio; - fs_prio->flags = flags; - list_add_tail(&fs_prio->base.list, &ns->prios); - INIT_LIST_HEAD(&fs_prio->objs); - mutex_init(&fs_prio->shared_lock); - - return fs_prio; -} - -static void cleanup_root_ns(struct mlx5_core_dev *dev) -{ - struct mlx5_flow_root_namespace *root_ns = dev->root_ns; - struct fs_prio *iter_prio; - - if (!root_ns) - return; - - /* stage 1 */ - fs_for_each_prio(iter_prio, &root_ns->ns) { - struct mlx5_flow_namespace *iter_ns; - - fs_for_each_ns(iter_ns, iter_prio) { - while (!list_empty(&iter_ns->prios)) { - struct fs_base *iter_prio2 = - list_first_entry(&iter_ns->prios, - struct fs_base, - list); - - fs_remove_node(iter_prio2); - } - } - } - - /* stage 2 */ - fs_for_each_prio(iter_prio, &root_ns->ns) { - while (!list_empty(&iter_prio->objs)) { - struct fs_base *iter_ns = - list_first_entry(&iter_prio->objs, - struct fs_base, - list); - - fs_remove_node(iter_ns); - } - } - /* stage 3 */ - while (!list_empty(&root_ns->ns.prios)) { - struct fs_base *iter_prio = - list_first_entry(&root_ns->ns.prios, - struct fs_base, - list); - - fs_remove_node(iter_prio); - } - - fs_remove_node(&root_ns->ns.base); - dev->root_ns = NULL; -} - -static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev, - struct mlx5_flow_root_namespace *root_ns) -{ - struct fs_base *prio; - - if (!root_ns) - return; - - if (!list_empty(&root_ns->ns.prios)) { - prio = list_first_entry(&root_ns->ns.prios, - struct fs_base, - list); - fs_remove_node(prio); - } - fs_remove_node(&root_ns->ns.base); - root_ns = NULL; -} - -void mlx5_cleanup_fs(struct mlx5_core_dev *dev) -{ - mlx5_cleanup_fc_stats(dev); - cleanup_root_ns(dev); - cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns); - cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns); - cleanup_single_prio_root_ns(dev, dev->fdb_root_ns); - cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns); - cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns); -} - -static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace - *ns) -{ - ns->base.type = FS_TYPE_NAMESPACE; - init_rwsem(&ns->dests_rw_sem); - init_rwsem(&ns->notifiers_rw_sem); - INIT_LIST_HEAD(&ns->prios); - INIT_LIST_HEAD(&ns->list_notifiers); - - return ns; -} - -static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev, - enum fs_ft_type - table_type, - char *name) -{ - struct mlx5_flow_root_namespace *root_ns; - struct mlx5_flow_namespace *ns; - - /* create the root namespace */ - root_ns = mlx5_vzalloc(sizeof(*root_ns)); - if (!root_ns) - goto err; - - root_ns->dev = dev; - root_ns->table_type = table_type; - mutex_init(&root_ns->fs_chain_lock); - - ns = &root_ns->ns; - fs_init_namespace(ns); - fs_add_node(&ns->base, NULL, name, 1); - - return root_ns; -err: - return NULL; -} - -static int init_fdb_root_ns(struct mlx5_core_dev *dev) -{ - struct fs_prio *prio; - - dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB, - MLX5_CORE_FS_FDB_ROOT_NS_NAME); - if (!dev->fdb_root_ns) - return -ENOMEM; - - /* create 1 prio*/ - prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0); - if (IS_ERR(prio)) - return PTR_ERR(prio); - else - return 0; -} - -#define MAX_VPORTS 128 - -static int init_egress_acl_root_ns(struct mlx5_core_dev *dev) -{ - struct fs_prio *prio; - - dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL, - MLX5_CORE_FS_ESW_EGRESS_ACL); - if (!dev->esw_egress_root_ns) - return -ENOMEM; - - /* create 1 prio*/ - prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS, - "esw_egress_prio", 0); - if (IS_ERR(prio)) - return PTR_ERR(prio); - else - return 0; -} - -static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev) -{ - struct fs_prio *prio; - - dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL, - MLX5_CORE_FS_ESW_INGRESS_ACL); - if (!dev->esw_ingress_root_ns) - return -ENOMEM; - - /* create 1 prio*/ - prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS, - "esw_ingress_prio", 0); - if (IS_ERR(prio)) - return PTR_ERR(prio); - else - return 0; -} - -static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev) -{ - struct fs_prio *prio; - - dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX, - MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME); - if (!dev->sniffer_rx_root_ns) - return -ENOMEM; - - /* create 1 prio*/ - prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1, - "sniffer_prio", 0); - if (IS_ERR(prio)) - return PTR_ERR(prio); - else - return 0; -} - - -static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev) -{ - struct fs_prio *prio; - - dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX, - MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME); - if (!dev->sniffer_tx_root_ns) - return -ENOMEM; - - /* create 1 prio*/ - prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1, - "sniffer_prio", 0); - if (IS_ERR(prio)) - return PTR_ERR(prio); - else - return 0; -} - -static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio, - const char *name) -{ - struct mlx5_flow_namespace *ns; - - ns = kzalloc(sizeof(*ns), GFP_KERNEL); - if (!ns) - return ERR_PTR(-ENOMEM); - - fs_init_namespace(ns); - fs_add_node(&ns->base, &prio->base, name, 1); - list_add_tail(&ns->base.list, &prio->objs); - - return ns; -} - -#define FLOW_TABLE_BIT_SZ 1 -#define GET_FLOW_TABLE_CAP(dev, offset) \ - ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ - offset / 32)) >> \ - (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) - -static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) -{ - int i; - - for (i = 0; i < caps->arr_sz; i++) { - if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) - return false; - } - return true; -} - -static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, - struct init_tree_node *node, struct fs_base *base_parent, - struct init_tree_node *tree_parent) -{ - struct mlx5_flow_namespace *fs_ns; - struct fs_prio *fs_prio; - int priority; - struct fs_base *base; - int i; - int err = 0; - - if (node->type == FS_TYPE_PRIO) { - if ((node->min_ft_level > max_ft_level) || - !has_required_caps(dev, &node->caps)) - goto out; - - fs_get_obj(fs_ns, base_parent); - priority = node - tree_parent->children; - fs_prio = fs_create_prio(fs_ns, priority, - node->max_ft, - node->name, node->flags); - if (IS_ERR(fs_prio)) { - err = PTR_ERR(fs_prio); - goto out; - } - base = &fs_prio->base; - } else if (node->type == FS_TYPE_NAMESPACE) { - fs_get_obj(fs_prio, base_parent); - fs_ns = fs_create_namespace(fs_prio, node->name); - if (IS_ERR(fs_ns)) { - err = PTR_ERR(fs_ns); - goto out; - } - base = &fs_ns->base; - } else { - return -EINVAL; - } - for (i = 0; i < node->ar_size; i++) { - err = _init_root_tree(dev, max_ft_level, &node->children[i], base, - node); - if (err) - break; - } -out: - return err; -} - -static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, - struct init_tree_node *node, struct fs_base *parent) -{ - int i; - struct mlx5_flow_namespace *fs_ns; - int err = 0; - - fs_get_obj(fs_ns, parent); - for (i = 0; i < node->ar_size; i++) { - err = _init_root_tree(dev, max_ft_level, - &node->children[i], &fs_ns->base, node); - if (err) - break; - } - return err; -} - -static int sum_max_ft_in_prio(struct fs_prio *prio); -static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns) -{ - struct fs_prio *prio; - int sum = 0; - - fs_for_each_prio(prio, ns) { - sum += sum_max_ft_in_prio(prio); - } - return sum; -} - -static int sum_max_ft_in_prio(struct fs_prio *prio) -{ - int sum = 0; - struct fs_base *it; - struct mlx5_flow_namespace *ns; - - if (prio->max_ft) - return prio->max_ft; - - fs_for_each_ns_or_ft(it, prio) { - if (it->type == FS_TYPE_FLOW_TABLE) - continue; - - fs_get_obj(ns, it); - sum += sum_max_ft_in_ns(ns); - } - prio->max_ft = sum; - return sum; -} - -static void set_max_ft(struct mlx5_flow_namespace *ns) -{ - struct fs_prio *prio; - - if (!ns) - return; - - fs_for_each_prio(prio, ns) - sum_max_ft_in_prio(prio); -} - -static int init_root_ns(struct mlx5_core_dev *dev) -{ - int max_ft_level = MLX5_CAP_FLOWTABLE(dev, - flow_table_properties_nic_receive. - max_ft_level); - - dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX, - MLX5_CORE_FS_ROOT_NS_NAME); - if (IS_ERR_OR_NULL(dev->root_ns)) - goto err; - - - if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base)) - goto err; - - set_max_ft(&dev->root_ns->ns); - - return 0; -err: - return -ENOMEM; -} - -u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule) -{ - struct fs_base *pbase; - struct mlx5_flow_group *fg; - - pbase = rule->base.parent; - WARN_ON(!pbase); - pbase = pbase->parent; - WARN_ON(!pbase); - - fs_get_obj(fg, pbase); - return fg->mask.match_criteria_enable; -} - -void mlx5_get_match_value(u32 *match_value, - struct mlx5_flow_rule *rule) -{ - struct fs_base *pbase; - struct fs_fte *fte; - - pbase = rule->base.parent; - WARN_ON(!pbase); - fs_get_obj(fte, pbase); - - memcpy(match_value, fte->val, sizeof(fte->val)); -} - -void mlx5_get_match_criteria(u32 *match_criteria, - struct mlx5_flow_rule *rule) -{ - struct fs_base *pbase; - struct mlx5_flow_group *fg; - - pbase = rule->base.parent; - WARN_ON(!pbase); - pbase = pbase->parent; - WARN_ON(!pbase); - - fs_get_obj(fg, pbase); - memcpy(match_criteria, &fg->mask.match_criteria, - sizeof(fg->mask.match_criteria)); -} - -int mlx5_init_fs(struct mlx5_core_dev *dev) -{ - int err; - - if (MLX5_CAP_GEN(dev, nic_flow_table)) { - err = init_root_ns(dev); - if (err) - goto err; - } - - err = init_fdb_root_ns(dev); - if (err) - goto err; - - err = init_egress_acl_root_ns(dev); - if (err) - goto err; - - err = init_ingress_acl_root_ns(dev); - if (err) - goto err; - - err = init_sniffer_tx_root_ns(dev); - if (err) - goto err; - - err = init_sniffer_rx_root_ns(dev); - if (err) - goto err; - - err = mlx5_init_fc_stats(dev); - if (err) - goto err; - - return 0; -err: - mlx5_cleanup_fs(dev); - return err; -} - -struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, - enum mlx5_flow_namespace_type type) -{ - struct mlx5_flow_root_namespace *root_ns = dev->root_ns; - int prio; - static struct fs_prio *fs_prio; - struct mlx5_flow_namespace *ns; - - switch (type) { - case MLX5_FLOW_NAMESPACE_BYPASS: - prio = 0; - break; - case MLX5_FLOW_NAMESPACE_OFFLOADS: - prio = 1; - break; - case MLX5_FLOW_NAMESPACE_KERNEL: - prio = 2; - break; - case MLX5_FLOW_NAMESPACE_LEFTOVERS: - prio = 3; - break; - case MLX5_FLOW_NAMESPACE_FDB: - if (dev->fdb_root_ns) - return &dev->fdb_root_ns->ns; - else - return NULL; - case MLX5_FLOW_NAMESPACE_ESW_EGRESS: - if (dev->esw_egress_root_ns) - return &dev->esw_egress_root_ns->ns; - else - return NULL; - case MLX5_FLOW_NAMESPACE_ESW_INGRESS: - if (dev->esw_ingress_root_ns) - return &dev->esw_ingress_root_ns->ns; - else - return NULL; - case MLX5_FLOW_NAMESPACE_SNIFFER_RX: - if (dev->sniffer_rx_root_ns) - return &dev->sniffer_rx_root_ns->ns; - else - return NULL; - case MLX5_FLOW_NAMESPACE_SNIFFER_TX: - if (dev->sniffer_tx_root_ns) - return &dev->sniffer_tx_root_ns->ns; - else - return NULL; - default: - return NULL; - } - - if (!root_ns) - return NULL; - - fs_prio = find_prio(&root_ns->ns, prio); - if (!fs_prio) - return NULL; - - ns = list_first_entry(&fs_prio->objs, - typeof(*ns), - base.list); - - return ns; -} -EXPORT_SYMBOL(mlx5_get_flow_namespace); - - -int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule, - struct mlx5_flow_handler *fs_handler, - void *client_data) -{ - struct fs_client_priv_data *priv_data; - - mutex_lock(&rule->clients_lock); - /*Check that hanlder isn't exists in the list already*/ - list_for_each_entry(priv_data, &rule->clients_data, list) { - if (priv_data->fs_handler == fs_handler) { - priv_data->client_dst_data = client_data; - goto unlock; - } - } - priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); - if (!priv_data) { - mutex_unlock(&rule->clients_lock); - return -ENOMEM; - } - - priv_data->client_dst_data = client_data; - priv_data->fs_handler = fs_handler; - list_add(&priv_data->list, &rule->clients_data); - -unlock: - mutex_unlock(&rule->clients_lock); - - return 0; -} - -static int remove_from_clients(struct mlx5_flow_rule *rule, - bool ctx_changed, - void *client_data, - void *context) -{ - struct fs_client_priv_data *iter_client; - struct fs_client_priv_data *temp_client; - struct mlx5_flow_handler *handler = (struct - mlx5_flow_handler*)context; - - mutex_lock(&rule->clients_lock); - list_for_each_entry_safe(iter_client, temp_client, - &rule->clients_data, list) { - if (iter_client->fs_handler == handler) { - list_del(&iter_client->list); - kfree(iter_client); - break; - } - } - mutex_unlock(&rule->clients_lock); - - return 0; -} - -struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev, - enum mlx5_flow_namespace_type ns_type, - rule_event_fn add_cb, - rule_event_fn del_cb, - void *context) -{ - struct mlx5_flow_namespace *ns; - struct mlx5_flow_handler *handler; - - ns = mlx5_get_flow_namespace(dev, ns_type); - if (!ns) - return ERR_PTR(-EINVAL); - - handler = kzalloc(sizeof(*handler), GFP_KERNEL); - if (!handler) - return ERR_PTR(-ENOMEM); - - handler->add_dst_cb = add_cb; - handler->del_dst_cb = del_cb; - handler->client_context = context; - handler->ns = ns; - down_write(&ns->notifiers_rw_sem); - list_add_tail(&handler->list, &ns->list_notifiers); - up_write(&ns->notifiers_rw_sem); - - return handler; -} - -static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, - rule_event_fn add_rule_cb, - void *context); - -void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler) -{ - struct mlx5_flow_namespace *ns = handler->ns; - - /*Remove from dst's clients*/ - down_write(&ns->dests_rw_sem); - down_write(&ns->notifiers_rw_sem); - iterate_rules_in_ns(ns, remove_from_clients, handler); - list_del(&handler->list); - up_write(&ns->notifiers_rw_sem); - up_write(&ns->dests_rw_sem); - kfree(handler); -} - -static void iterate_rules_in_ft(struct mlx5_flow_table *ft, - rule_event_fn add_rule_cb, - void *context) -{ - struct mlx5_flow_group *iter_fg; - struct fs_fte *iter_fte; - struct mlx5_flow_rule *iter_rule; - int err = 0; - bool is_new_rule; - - mutex_lock(&ft->base.lock); - fs_for_each_fg(iter_fg, ft) { - mutex_lock(&iter_fg->base.lock); - fs_for_each_fte(iter_fte, iter_fg) { - mutex_lock(&iter_fte->base.lock); - is_new_rule = true; - fs_for_each_dst(iter_rule, iter_fte) { - fs_get(&iter_rule->base); - err = add_rule_cb(iter_rule, - is_new_rule, - NULL, - context); - fs_put_parent_locked(&iter_rule->base); - if (err) - break; - is_new_rule = false; - } - mutex_unlock(&iter_fte->base.lock); - if (err) - break; - } - mutex_unlock(&iter_fg->base.lock); - if (err) - break; - } - mutex_unlock(&ft->base.lock); -} - -static void iterate_rules_in_prio(struct fs_prio *prio, - rule_event_fn add_rule_cb, - void *context) -{ - struct fs_base *it; - - mutex_lock(&prio->base.lock); - fs_for_each_ns_or_ft(it, prio) { - if (it->type == FS_TYPE_FLOW_TABLE) { - struct mlx5_flow_table *ft; - - fs_get_obj(ft, it); - iterate_rules_in_ft(ft, add_rule_cb, context); - } else { - struct mlx5_flow_namespace *ns; - - fs_get_obj(ns, it); - iterate_rules_in_ns(ns, add_rule_cb, context); - } - } - mutex_unlock(&prio->base.lock); -} - -static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, - rule_event_fn add_rule_cb, - void *context) -{ - struct fs_prio *iter_prio; - - mutex_lock(&ns->base.lock); - fs_for_each_prio(iter_prio, ns) { - iterate_rules_in_prio(iter_prio, add_rule_cb, context); - } - mutex_unlock(&ns->base.lock); -} - -void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns, - rule_event_fn add_rule_cb, - void *context) -{ - down_write(&ns->dests_rw_sem); - down_read(&ns->notifiers_rw_sem); - iterate_rules_in_ns(ns, add_rule_cb, context); - up_read(&ns->notifiers_rw_sem); - up_write(&ns->dests_rw_sem); -} - - -void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list) -{ - struct mlx5_flow_rule_node *iter_node; - struct mlx5_flow_rule_node *temp_node; - - list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) { - list_del(&iter_node->list); - kfree(iter_node); - } - - kfree(rules_list); -} - -#define ROCEV1_ETHERTYPE 0x8915 -static int set_rocev1_rules(struct list_head *rules_list) -{ - struct mlx5_flow_rule_node *rocev1_rule; - - rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL); - if (!rocev1_rule) - return -ENOMEM; - - rocev1_rule->match_criteria_enable = - 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; - MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype, - 0xffff); - MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype, - ROCEV1_ETHERTYPE); - - list_add_tail(&rocev1_rule->list, rules_list); - - return 0; -} - -#define ROCEV2_UDP_PORT 4791 -static int set_rocev2_rules(struct list_head *rules_list) -{ - struct mlx5_flow_rule_node *ipv4_rule; - struct mlx5_flow_rule_node *ipv6_rule; - - ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL); - if (!ipv4_rule) - return -ENOMEM; - - ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL); - if (!ipv6_rule) { - kfree(ipv4_rule); - return -ENOMEM; - } - - ipv4_rule->match_criteria_enable = - 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; - MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype, - 0xffff); - MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype, - 0x0800); - MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol, - 0xff); - MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol, - IPPROTO_UDP); - MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport, - 0xffff); - MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport, - ROCEV2_UDP_PORT); - - ipv6_rule->match_criteria_enable = - 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; - MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype, - 0xffff); - MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype, - 0x86dd); - MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol, - 0xff); - MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol, - IPPROTO_UDP); - MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport, - 0xffff); - MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport, - ROCEV2_UDP_PORT); - - list_add_tail(&ipv4_rule->list, rules_list); - list_add_tail(&ipv6_rule->list, rules_list); - - return 0; -} - - -struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode) -{ - int err = 0; - struct mlx5_flow_rules_list *rules_list = - kzalloc(sizeof(*rules_list), GFP_KERNEL); - - if (!rules_list) - return NULL; - - INIT_LIST_HEAD(&rules_list->head); - - if (roce_mode & MLX5_ROCE_VERSION_1_CAP) { - err = set_rocev1_rules(&rules_list->head); - if (err) - goto free_list; - } - if (roce_mode & MLX5_ROCE_VERSION_2_CAP) - err = set_rocev2_rules(&rules_list->head); - if (err) - goto free_list; - - return rules_list; - -free_list: - mlx5_del_flow_rules_list(rules_list); - return NULL; -} - -struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev, - enum mlx5_flow_namespace_type ns_type, - u8 num_actions, - void *modify_actions) -{ - struct mlx5_modify_hdr *modify_hdr; - int err; - - modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL); - if (!modify_hdr) - return ERR_PTR(-ENOMEM); - - modify_hdr->ns_type = ns_type; - err = mlx5_cmd_modify_header_alloc(dev, ns_type, num_actions, - modify_actions, modify_hdr); - if (err) { - kfree(modify_hdr); - return ERR_PTR(err); - } - - return modify_hdr; -} -EXPORT_SYMBOL(mlx5_modify_header_alloc); - -void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev, - struct mlx5_modify_hdr *modify_hdr) -{ - mlx5_cmd_modify_header_dealloc(dev, modify_hdr); - kfree(modify_hdr); -} -EXPORT_SYMBOL(mlx5_modify_header_dealloc); - -struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev, - struct mlx5_pkt_reformat_params *params, - enum mlx5_flow_namespace_type ns_type) -{ - struct mlx5_pkt_reformat *pkt_reformat; - int err; - - pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL); - if (!pkt_reformat) - return ERR_PTR(-ENOMEM); - - pkt_reformat->ns_type = ns_type; - pkt_reformat->reformat_type = params->type; - err = mlx5_cmd_packet_reformat_alloc(dev, params, ns_type, - pkt_reformat); - if (err) { - kfree(pkt_reformat); - return ERR_PTR(err); - } - - return pkt_reformat; -} -EXPORT_SYMBOL(mlx5_packet_reformat_alloc); - -void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev, - struct mlx5_pkt_reformat *pkt_reformat) -{ - mlx5_cmd_packet_reformat_dealloc(dev, pkt_reformat); - kfree(pkt_reformat); -} -EXPORT_SYMBOL(mlx5_packet_reformat_dealloc); - diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fw.c b/sys/dev/mlx5/mlx5_core/mlx5_fw.c index 1a4956b09d32..233bd4a38c91 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_fw.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_fw.c @@ -240,6 +240,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN(dev, ipsec_offload)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_IPSEC); + if (err) + return err; + } + err = mlx5_core_query_special_contexts(dev); if (err) return err; @@ -300,7 +306,7 @@ int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev) #define MLX5_FAST_TEARDOWN_WAIT_MS 3000 int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev) { - int end, delay_ms = MLX5_FAST_TEARDOWN_WAIT_MS; + unsigned long end, delay_ms = MLX5_FAST_TEARDOWN_WAIT_MS; u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {}; u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {}; int state; @@ -337,7 +343,7 @@ int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev) } while (!time_after(jiffies, end)); if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { - mlx5_core_err(dev, "NIC IFC still %d after %ums.\n", + mlx5_core_err(dev, "NIC IFC still %d after %lums.\n", mlx5_get_nic_state(dev), delay_ms); return -EIO; } diff --git a/sys/dev/mlx5/mlx5_core/mlx5_health.c b/sys/dev/mlx5/mlx5_core/mlx5_health.c index f4049d23d75d..bedd51eb02e4 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_health.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_health.c @@ -265,7 +265,8 @@ mlx5_health_allow_reset(struct mlx5_core_dev *dev) #define MLX5_NIC_STATE_POLL_MS 5 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) { - int end, delay_ms = MLX5_CRDUMP_WAIT_MS; + unsigned long end; + int delay_ms = MLX5_CRDUMP_WAIT_MS; u32 fatal_error; int lock = -EBUSY; @@ -445,7 +446,7 @@ static void health_care(struct work_struct *work) spin_unlock_irqrestore(&health->wq_lock, flags); } -static int get_next_poll_jiffies(void) +static unsigned long get_next_poll_jiffies(void) { unsigned long next; diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c index 6b9b63a24714..221781327b51 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_main.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c @@ -26,6 +26,7 @@ #include "opt_rss.h" #include "opt_ratelimit.h" +#include "opt_ipsec.h" #include <linux/kmod.h> #include <linux/module.h> @@ -52,8 +53,12 @@ #include <dev/mlx5/mlx5_core/diag_cnt.h> #ifdef PCI_IOV #include <sys/nv.h> +#include <sys/socket.h> #include <dev/pci/pci_iov.h> #include <sys/iov_schema.h> +#include <sys/iov.h> +#include <net/if.h> +#include <net/if_vlan_var.h> #endif static const char mlx5_version[] = "Mellanox Core driver " @@ -63,6 +68,9 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1); MODULE_DEPEND(mlx5, mlxfw, 1, 1, 1); MODULE_DEPEND(mlx5, firmware, 1, 1, 1); +#ifdef IPSEC_OFFLOAD +MODULE_DEPEND(mlx5, ipsec, 1, 1, 1); +#endif MODULE_VERSION(mlx5, 1); SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, @@ -225,6 +233,7 @@ static void mlx5_set_driver_version(struct mlx5_core_dev *dev) #ifdef PCI_IOV static const char iov_mac_addr_name[] = "mac-addr"; +static const char iov_vlan_name[] = "vlan"; static const char iov_node_guid_name[] = "node-guid"; static const char iov_port_guid_name[] = "port-guid"; #endif @@ -752,8 +761,8 @@ static inline int fw_initializing(struct mlx5_core_dev *dev) static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili, u32 warn_time_mili) { - int warn = jiffies + msecs_to_jiffies(warn_time_mili); - int end = jiffies + msecs_to_jiffies(max_wait_mili); + unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili); + unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); int err = 0; MPASS(max_wait_mili > warn_time_mili); @@ -765,8 +774,8 @@ static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili, } if (warn_time_mili && time_after(jiffies, warn)) { mlx5_core_warn(dev, - "Waiting for FW initialization, timeout abort in %u s\n", - (unsigned)(jiffies_to_msecs(end - warn) / 1000)); + "Waiting for FW initialization, timeout abort in %lu s\n", + (unsigned long)(jiffies_to_msecs(end - warn) / 1000)); warn = jiffies + msecs_to_jiffies(warn_time_mili); } msleep(FW_INIT_WAIT_MS); @@ -1204,7 +1213,7 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, goto err_stop_eqs; } - err = mlx5_init_fs(dev); + err = mlx5_fs_core_init(dev); if (err) { mlx5_core_err(dev, "flow steering init %d\n", err); goto err_free_comp_eqs; @@ -1322,7 +1331,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, mlx5_diag_cnt_cleanup(dev); mlx5_fpga_device_stop(dev); mlx5_mpfs_destroy(dev); - mlx5_cleanup_fs(dev); + mlx5_fs_core_cleanup(dev); mlx5_wait_for_reclaim_vfs_pages(dev); free_comp_eqs(dev); mlx5_stop_eqs(dev); @@ -1689,10 +1698,16 @@ static int init_one(struct pci_dev *pdev, mlx5_pagealloc_init(dev); + err = mlx5_fs_core_alloc(dev); + if (err) { + mlx5_core_err(dev, "Failed to alloc flow steering\n"); + goto clean_health; + } + err = mlx5_load_one(dev, priv, true); if (err) { mlx5_core_err(dev, "mlx5_load_one failed %d\n", err); - goto clean_health; + goto clean_fs; } mlx5_fwdump_prep(dev); @@ -1714,6 +1729,8 @@ static int init_one(struct pci_dev *pdev, vf_schema = pci_iov_schema_alloc_node(); pci_iov_schema_add_unicast_mac(vf_schema, iov_mac_addr_name, 0, NULL); + pci_iov_schema_add_vlan(vf_schema, + iov_vlan_name, 0, 0); pci_iov_schema_add_uint64(vf_schema, iov_node_guid_name, 0, 0); pci_iov_schema_add_uint64(vf_schema, iov_port_guid_name, @@ -1736,6 +1753,8 @@ static int init_one(struct pci_dev *pdev, pci_save_state(pdev); return 0; +clean_fs: + mlx5_fs_core_free(dev); clean_health: mlx5_pagealloc_cleanup(dev); mlx5_health_cleanup(dev); @@ -1767,6 +1786,7 @@ static void remove_one(struct pci_dev *pdev) (long long)(dev->priv.fw_pages * MLX5_ADAPTER_PAGE_SIZE)); } + mlx5_fs_core_free(dev); mlx5_pagealloc_cleanup(dev); mlx5_health_cleanup(dev); mlx5_fwdump_clean(dev); @@ -1950,6 +1970,25 @@ mlx5_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) } } + if (nvlist_exists_number(vf_config, iov_vlan_name)) { + uint16_t vlan = nvlist_get_number(vf_config, iov_vlan_name); + + if (vlan == DOT1Q_VID_NULL) + error = ENOTSUP; + else { + if (vlan == VF_VLAN_TRUNK) + vlan = DOT1Q_VID_NULL; + + error = -mlx5_eswitch_set_vport_vlan(priv->eswitch, + vfnum + 1, vlan, 0); + } + if (error != 0) { + mlx5_core_err(core_dev, + "setting VLAN for VF %d failed, error %d\n", + vfnum + 1, error); + } + } + if (nvlist_exists_number(vf_config, iov_node_guid_name)) { node_guid = nvlist_get_number(vf_config, iov_node_guid_name); error = -mlx5_modify_nic_vport_node_guid(core_dev, vfnum + 1, @@ -2103,7 +2142,7 @@ static const struct pci_device_id mlx5_core_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0xa2d3) }, /* BlueField integrated ConnectX-5 network controller VF */ { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */ { PCI_VDEVICE(MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */ - { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 integrated ConnectX-8 network controller */ + { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 Family integrated network controller */ { } }; diff --git a/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c b/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c index 6207442e756f..b1798f909ee5 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c @@ -519,7 +519,7 @@ enum { s64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev) { - int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); + unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); s64 prevpages = 0; s64 npages = 0; @@ -557,7 +557,7 @@ static int optimal_reclaimed_pages(void) int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) { - int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); + unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); struct mlx5_fw_page *fwp; struct rb_node *p; int nclaimed = 0; diff --git a/sys/dev/mlx5/mlx5_core/mlx5_tls.c b/sys/dev/mlx5/mlx5_core/mlx5_tls.c index b3a49c603fed..3ed209e2028d 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_tls.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_tls.c @@ -33,66 +33,6 @@ #include <dev/mlx5/mlx5_core/mlx5_core.h> #include <dev/mlx5/mlx5_core/transobj.h> -int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn, - const void *p_key, u32 key_len, u32 *p_obj_id) -{ - u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; - u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {}; - u64 general_obj_types; - int err; - - general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); - if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY)) - return -EINVAL; - - switch (key_len) { - case 128 / 8: - memcpy(MLX5_ADDR_OF(create_encryption_key_in, in, - encryption_key_object.key[4]), p_key, 128 / 8); - MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn); - MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128); - MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK); - break; - case 256 / 8: - memcpy(MLX5_ADDR_OF(create_encryption_key_in, in, - encryption_key_object.key[0]), p_key, 256 / 8); - MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn); - MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256); - MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK); - break; - default: - return -EINVAL; - } - - MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ); - MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); - - err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); - if (err == 0) - *p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id); - - /* avoid leaking key on the stack */ - memset(in, 0, sizeof(in)); - - return err; -} - -int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid) -{ - u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {}; - u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {}; - - MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ); - MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); - MLX5_SET(destroy_encryption_key_in, in, obj_id, oid); - - return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); -} - int mlx5_tls_open_tis(struct mlx5_core_dev *mdev, int tc, int tdn, int pdn, u32 *p_tisn) { u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; diff --git a/sys/dev/mlx5/mlx5_core/mlx5_transobj.c b/sys/dev/mlx5/mlx5_core/mlx5_transobj.c index 6d375d371597..c62969d8d172 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_transobj.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_transobj.c @@ -166,6 +166,18 @@ int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen, return err; } + +int +mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 *in, int inlen) +{ + u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {0}; + int err; + + MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR); + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); + return (err); +} + void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u32 uid) { u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {0}; diff --git a/sys/dev/mlx5/mlx5_core/transobj.h b/sys/dev/mlx5/mlx5_core/transobj.h index 1cc40ca8b1b7..6a21d7db90c8 100644 --- a/sys/dev/mlx5/mlx5_core/transobj.h +++ b/sys/dev/mlx5/mlx5_core/transobj.h @@ -40,6 +40,7 @@ void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn); int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out); int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *tirn); +int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 *in, int inlen); void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u32 uid); int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *tisn); diff --git a/sys/dev/mlx5/mlx5_core/wq.h b/sys/dev/mlx5/mlx5_core/wq.h index c996eca41114..2e1d6a6fcff0 100644 --- a/sys/dev/mlx5/mlx5_core/wq.h +++ b/sys/dev/mlx5/mlx5_core/wq.h @@ -27,6 +27,7 @@ #define __MLX5_WQ_H__ #include <dev/mlx5/mlx5_ifc.h> +#include <dev/mlx5/cq.h> struct mlx5_wq_param { int linear; @@ -136,6 +137,22 @@ static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq) *wq->db = cpu_to_be32(wq->cc & 0xffffff); } +static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq) +{ + u32 ci = mlx5_cqwq_get_ci(wq); + struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci); + u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; + u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; + + if (cqe_ownership_bit != sw_ownership_val) + return NULL; + + /* ensure cqe content is read after cqe ownership bit */ + atomic_thread_fence_acq(); + + return cqe; +} + static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq) { return wq->cur_sz == wq->sz_m1; |