diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index 9d623e38d783d735ce6f99382e40bbfd42544e16..461a43f338e6e0ab1083c74ee7f500f0105f6411 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -104,6 +104,18 @@ config MLX5_TC_CT
If unsure, set to Y
+config MLX5_TC_SAMPLE
+ bool "MLX5 TC sample offload support"
+ depends on MLX5_CLS_ACT
+ default y
+ help
+ Say Y here if you want to support offloading sample rules via tc
+ sample action.
+ If set to N, will not be able to configure tc rules with sample
+ action.
+
+ If unsure, set to Y
+
config MLX5_CORE_EN_DCB
bool "Data Center Bridging (DCB) Support"
default y
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 9cf7de72df52dbd2533497f530e3a0006e52def8..8bde58379ac65d29fcee82faade76aedda0c1e3b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -37,9 +37,10 @@ mlx5_core-$(CONFIG_MLX5_EN_RXNFC) += en_fs_ethtool.o
mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) += en_dcbnl.o en/port_buffer.o
mlx5_core-$(CONFIG_PCI_HYPERV_INTERFACE) += en/hv_vhca_stats.o
mlx5_core-$(CONFIG_MLX5_ESWITCH) += lag_mp.o lib/geneve.o lib/port_tun.o \
- en_rep.o en/rep/bond.o en/mod_hdr.o
+ en_rep.o en/rep/bond.o en/mod_hdr.o \
+ en/mapping.o
mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \
- en/mapping.o lib/fs_chains.o en/tc_tun.o \
+ lib/fs_chains.o en/tc_tun.o \
esw/indir_table.o en/tc_tun_encap.o \
en/tc_tun_vxlan.o en/tc_tun_gre.o en/tc_tun_geneve.o \
en/tc_tun_mplsoudp.o diag/en_tc_tracepoint.o
@@ -53,7 +54,8 @@ mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offlo
mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \
esw/acl/egress_lgcy.o esw/acl/egress_ofld.o \
esw/acl/ingress_lgcy.o esw/acl/ingress_ofld.o \
- esw/devlink_port.o
+ esw/devlink_port.o esw/vporttbl.o
+mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += esw/sample.o
mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
index 373bd89484cb9af966ea8c368ed006761ed79a21..1d5ce07b83f456402b47ca7f3cfd02a4114ca9ff 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
@@ -29,6 +29,7 @@ struct mlx5e_tc_table {
struct netdev_net_notifier netdevice_nn;
struct mlx5_tc_ct_priv *ct;
+ struct mapping_ctx *mapping;
};
struct mlx5e_flow_table {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
index 11a44d30adc7cdce32e6f9665eff9b6c27a93612..6cdc52d50a488a9a372719f7fb5252fa76f85f97 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
@@ -17,6 +17,7 @@
#include "en/mapping.h"
#include "en/tc_tun.h"
#include "lib/port_tun.h"
+#include "esw/sample.h"
struct mlx5e_rep_indr_block_priv {
struct net_device *netdev;
@@ -611,19 +612,46 @@ static bool mlx5e_restore_tunnel(struct mlx5e_priv *priv, struct sk_buff *skb,
return true;
}
+
+static bool mlx5e_restore_skb(struct sk_buff *skb, u32 chain, u32 reg_c1,
+ struct mlx5e_tc_update_priv *tc_priv)
+{
+ struct mlx5e_priv *priv = netdev_priv(skb->dev);
+ u32 tunnel_id = reg_c1 >> ESW_TUN_OFFSET;
+
+ if (chain) {
+ struct mlx5_rep_uplink_priv *uplink_priv;
+ struct mlx5e_rep_priv *uplink_rpriv;
+ struct tc_skb_ext *tc_skb_ext;
+ struct mlx5_eswitch *esw;
+ u32 zone_restore_id;
+
+ tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT);
+ if (!tc_skb_ext) {
+ WARN_ON(1);
+ return false;
+ }
+ tc_skb_ext->chain = chain;
+ zone_restore_id = reg_c1 & ESW_ZONE_ID_MASK;
+ esw = priv->mdev->priv.eswitch;
+ uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
+ uplink_priv = &uplink_rpriv->uplink_priv;
+ if (!mlx5e_tc_ct_restore_flow(uplink_priv->ct_priv, skb,
+ zone_restore_id))
+ return false;
+ }
+ return mlx5e_restore_tunnel(priv, skb, tc_priv, tunnel_id);
+}
#endif /* CONFIG_NET_TC_SKB_EXT */
bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe,
struct sk_buff *skb,
struct mlx5e_tc_update_priv *tc_priv)
{
-#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
- u32 chain = 0, reg_c0, reg_c1, tunnel_id, zone_restore_id;
- struct mlx5_rep_uplink_priv *uplink_priv;
- struct mlx5e_rep_priv *uplink_rpriv;
- struct tc_skb_ext *tc_skb_ext;
+ struct mlx5_mapped_obj mapped_obj;
struct mlx5_eswitch *esw;
struct mlx5e_priv *priv;
+ u32 reg_c0, reg_c1;
int err;
reg_c0 = (be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK);
@@ -639,36 +667,29 @@ bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe,
priv = netdev_priv(skb->dev);
esw = priv->mdev->priv.eswitch;
-
- err = mlx5_get_chain_for_tag(esw_chains(esw), reg_c0, &chain);
+ err = mapping_find(esw->offloads.reg_c0_obj_pool, reg_c0, &mapped_obj);
if (err) {
netdev_dbg(priv->netdev,
- "Couldn't find chain for chain tag: %d, err: %d\n",
+ "Couldn't find mapped object for reg_c0: %d, err: %d\n",
reg_c0, err);
return false;
}
- if (chain) {
- tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT);
- if (!tc_skb_ext) {
- WARN_ON(1);
- return false;
- }
-
- tc_skb_ext->chain = chain;
-
- zone_restore_id = reg_c1 & ESW_ZONE_ID_MASK;
-
- uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
- uplink_priv = &uplink_rpriv->uplink_priv;
- if (!mlx5e_tc_ct_restore_flow(uplink_priv->ct_priv, skb,
- zone_restore_id))
- return false;
- }
-
- tunnel_id = reg_c1 >> ESW_TUN_OFFSET;
- return mlx5e_restore_tunnel(priv, skb, tc_priv, tunnel_id);
+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
+ if (mapped_obj.type == MLX5_MAPPED_OBJ_CHAIN)
+ return mlx5e_restore_skb(skb, mapped_obj.chain, reg_c1, tc_priv);
#endif /* CONFIG_NET_TC_SKB_EXT */
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ if (mapped_obj.type == MLX5_MAPPED_OBJ_SAMPLE) {
+ mlx5_esw_sample_skb(skb, &mapped_obj);
+ return false;
+ }
+#endif /* CONFIG_MLX5_TC_SAMPLE */
+ if (mapped_obj.type != MLX5_MAPPED_OBJ_SAMPLE &&
+ mapped_obj.type != MLX5_MAPPED_OBJ_CHAIN) {
+ netdev_dbg(priv->netdev, "Invalid mapped object type: %d\n", mapped_obj.type);
+ return false;
+ }
return true;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
index c223591ffc220902c6b8ff06f411d79ac7b3e231..d1599b7b944bf046c7cbce09f4339ce78b7a06d2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
@@ -27,6 +27,7 @@ enum {
MLX5E_TC_FLOW_FLAG_L3_TO_L2_DECAP = MLX5E_TC_FLOW_BASE + 8,
MLX5E_TC_FLOW_FLAG_TUN_RX = MLX5E_TC_FLOW_BASE + 9,
MLX5E_TC_FLOW_FLAG_FAILED = MLX5E_TC_FLOW_BASE + 10,
+ MLX5E_TC_FLOW_FLAG_SAMPLE = MLX5E_TC_FLOW_BASE + 11,
};
struct mlx5e_tc_flow_parse_attr {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index 931fa619cb010bec872604592dc0ef09ed2ecef9..22585015c7a7160e0abb394076df5066b6341b25 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -89,6 +89,7 @@ struct mlx5_rep_uplink_priv {
struct mapping_ctx *tunnel_enc_opts_mapping;
struct mlx5_tc_ct_priv *ct_priv;
+ struct mlx5_esw_psample *esw_psample;
/* support eswitch vports bonding */
struct mlx5e_rep_bond *bond;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index bb1e0d442b5c3d8c797510a51e7618d1bacd8ce6..d157d1b9cad645a082acf232646c20cf44670899 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -47,6 +47,7 @@
#include <net/tc_act/tc_pedit.h>
#include <net/tc_act/tc_csum.h>
#include <net/tc_act/tc_mpls.h>
+#include <net/psample.h>
#include <net/arp.h>
#include <net/ipv6_stubs.h>
#include <net/bareudp.h>
@@ -65,6 +66,7 @@
#include "en/mod_hdr.h"
#include "en/tc_priv.h"
#include "en/tc_tun_encap.h"
+#include "esw/sample.h"
#include "lib/devcom.h"
#include "lib/geneve.h"
#include "lib/fs_chains.h"
@@ -221,6 +223,25 @@ get_ct_priv(struct mlx5e_priv *priv)
return priv->fs.tc.ct;
}
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+static struct mlx5_esw_psample *
+get_sample_priv(struct mlx5e_priv *priv)
+{
+ struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
+ struct mlx5_rep_uplink_priv *uplink_priv;
+ struct mlx5e_rep_priv *uplink_rpriv;
+
+ if (is_mdev_switchdev_mode(priv->mdev)) {
+ uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
+ uplink_priv = &uplink_rpriv->uplink_priv;
+
+ return uplink_priv->esw_psample;
+ }
+
+ return NULL;
+}
+#endif
+
struct mlx5_flow_handle *
mlx5_tc_rule_insert(struct mlx5e_priv *priv,
struct mlx5_flow_spec *spec,
@@ -1090,6 +1111,10 @@ mlx5e_tc_offload_fdb_rules(struct mlx5_eswitch *esw,
rule = mlx5_tc_ct_flow_offload(get_ct_priv(flow->priv),
flow, spec, attr,
mod_hdr_acts);
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ } else if (flow_flag_test(flow, SAMPLE)) {
+ rule = mlx5_esw_sample_offload(get_sample_priv(flow->priv), spec, attr);
+#endif
} else {
rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
}
@@ -1125,6 +1150,13 @@ void mlx5e_tc_unoffload_fdb_rules(struct mlx5_eswitch *esw,
return;
}
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ if (flow_flag_test(flow, SAMPLE)) {
+ mlx5_esw_sample_unoffload(get_sample_priv(flow->priv), flow->rule[0], attr);
+ return;
+ }
+#endif
+
if (attr->esw_attr->split_count)
mlx5_eswitch_del_fwd_rule(esw, flow->rule[1], attr);
@@ -1481,6 +1513,7 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
if (flow_flag_test(flow, L3_TO_L2_DECAP))
mlx5e_detach_decap(priv, flow);
+ kfree(flow->attr->esw_attr->sample);
kfree(flow->attr);
}
@@ -3627,6 +3660,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv,
bool ft_flow = mlx5e_is_ft_flow(flow);
const struct flow_action_entry *act;
struct mlx5_esw_flow_attr *esw_attr;
+ struct mlx5_sample_attr sample = {};
bool encap = false, decap = false;
u32 action = attr->action;
int err, i, if_count = 0;
@@ -3881,6 +3915,10 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv,
attr->dest_chain = act->chain_index;
break;
case FLOW_ACTION_CT:
+ if (flow_flag_test(flow, SAMPLE)) {
+ NL_SET_ERR_MSG_MOD(extack, "Sample action with connection tracking is not supported");
+ return -EOPNOTSUPP;
+ }
err = mlx5_tc_ct_parse_action(get_ct_priv(priv), attr, act, extack);
if (err)
return err;
@@ -3888,6 +3926,17 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv,
flow_flag_set(flow, CT);
esw_attr->split_count = esw_attr->out_count;
break;
+ case FLOW_ACTION_SAMPLE:
+ if (flow_flag_test(flow, CT)) {
+ NL_SET_ERR_MSG_MOD(extack, "Sample action with connection tracking is not supported");
+ return -EOPNOTSUPP;
+ }
+ sample.rate = act->sample.rate;
+ sample.group_num = act->sample.psample_group->group_num;
+ if (act->sample.truncate)
+ sample.trunc_size = act->sample.trunc_size;
+ flow_flag_set(flow, SAMPLE);
+ break;
default:
NL_SET_ERR_MSG_MOD(extack, "The offload action is not supported");
return -EOPNOTSUPP;
@@ -3966,6 +4015,16 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv,
return -EOPNOTSUPP;
}
+ /* Allocate sample attribute only when there is a sample action and
+ * no errors after parsing.
+ */
+ if (flow_flag_test(flow, SAMPLE)) {
+ esw_attr->sample = kzalloc(sizeof(*esw_attr->sample), GFP_KERNEL);
+ if (!esw_attr->sample)
+ return -ENOMEM;
+ *esw_attr->sample = sample;
+ }
+
return 0;
}
@@ -4731,6 +4790,7 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
{
struct mlx5e_tc_table *tc = &priv->fs.tc;
struct mlx5_core_dev *dev = priv->mdev;
+ struct mapping_ctx *chains_mapping;
struct mlx5_chains_attr attr = {};
int err;
@@ -4745,15 +4805,22 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
lockdep_set_class(&tc->ht.mutex, &tc_ht_lock_key);
- if (MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ignore_flow_level)) {
+ chains_mapping = mapping_create(sizeof(struct mlx5_mapped_obj),
+ MLX5E_TC_TABLE_CHAIN_TAG_MASK, true);
+ if (IS_ERR(chains_mapping)) {
+ err = PTR_ERR(chains_mapping);
+ goto err_mapping;
+ }
+ tc->mapping = chains_mapping;
+
+ if (MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ignore_flow_level))
attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
- attr.max_restore_tag = MLX5E_TC_TABLE_CHAIN_TAG_MASK;
- }
attr.ns = MLX5_FLOW_NAMESPACE_KERNEL;
attr.max_ft_sz = mlx5e_tc_nic_get_ft_size(dev);
attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS;
attr.default_ft = mlx5e_vlan_get_flowtable(priv->fs.vlan);
+ attr.mapping = chains_mapping;
tc->chains = mlx5_chains_create(dev, &attr);
if (IS_ERR(tc->chains)) {
@@ -4780,6 +4847,8 @@ err_reg:
mlx5_tc_ct_clean(tc->ct);
mlx5_chains_destroy(tc->chains);
err_chains:
+ mapping_destroy(chains_mapping);
+err_mapping:
rhashtable_destroy(&tc->ht);
return err;
}
@@ -4814,6 +4883,7 @@ void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv)
mutex_destroy(&tc->t_lock);
mlx5_tc_ct_clean(tc->ct);
+ mapping_destroy(tc->mapping);
mlx5_chains_destroy(tc->chains);
}
@@ -4837,6 +4907,10 @@ int mlx5e_tc_esw_init(struct rhashtable *tc_ht)
&esw->offloads.mod_hdr,
MLX5_FLOW_NAMESPACE_FDB);
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ uplink_priv->esw_psample = mlx5_esw_sample_init(netdev_priv(priv->netdev));
+#endif
+
mapping = mapping_create(sizeof(struct tunnel_match_key),
TUNNEL_INFO_BITS_MASK, true);
if (IS_ERR(mapping)) {
@@ -4874,6 +4948,9 @@ err_ht_init:
err_enc_opts_mapping:
mapping_destroy(uplink_priv->tunnel_mapping);
err_tun_mapping:
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ mlx5_esw_sample_cleanup(uplink_priv->esw_psample);
+#endif
mlx5_tc_ct_clean(uplink_priv->ct_priv);
netdev_warn(priv->netdev,
"Failed to initialize tc (eswitch), err: %d", err);
@@ -4892,6 +4969,9 @@ void mlx5e_tc_esw_cleanup(struct rhashtable *tc_ht)
mapping_destroy(uplink_priv->tunnel_enc_opts_mapping);
mapping_destroy(uplink_priv->tunnel_mapping);
+#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ mlx5_esw_sample_cleanup(uplink_priv->esw_psample);
+#endif
mlx5_tc_ct_clean(uplink_priv->ct_priv);
}
@@ -4973,6 +5053,7 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe,
u32 chain = 0, chain_tag, reg_b, zone_restore_id;
struct mlx5e_priv *priv = netdev_priv(skb->dev);
struct mlx5e_tc_table *tc = &priv->fs.tc;
+ struct mlx5_mapped_obj mapped_obj;
struct tc_skb_ext *tc_skb_ext;
int err;
@@ -4980,7 +5061,7 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe,
chain_tag = reg_b & MLX5E_TC_TABLE_CHAIN_TAG_MASK;
- err = mlx5_get_chain_for_tag(nic_chains(priv), chain_tag, &chain);
+ err = mapping_find(tc->mapping, chain_tag, &mapped_obj);
if (err) {
netdev_dbg(priv->netdev,
"Couldn't find chain for chain tag: %d, err: %d\n",
@@ -4988,7 +5069,8 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe,
return false;
}
- if (chain) {
+ if (mapped_obj.type == MLX5_MAPPED_OBJ_CHAIN) {
+ chain = mapped_obj.chain;
tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT);
if (WARN_ON(!tc_skb_ext))
return false;
@@ -5001,6 +5083,9 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe,
if (!mlx5e_tc_ct_restore_flow(tc->ct, skb,
zone_restore_id))
return false;
+ } else {
+ netdev_dbg(priv->netdev, "Invalid mapped object type: %d\n", mapped_obj.type);
+ return false;
}
#endif /* CONFIG_NET_TC_SKB_EXT */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/sample.c
new file mode 100644
index 0000000000000000000000000000000000000000..794012c5c476593ba6ac35da6b51bf66734b78da
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/sample.c
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#include <linux/skbuff.h>
+#include <net/psample.h>
+#include "en/mapping.h"
+#include "esw/sample.h"
+#include "eswitch.h"
+#include "en_tc.h"
+#include "fs_core.h"
+
+#define MLX5_ESW_VPORT_TBL_SIZE_SAMPLE (64 * 1024)
+
+static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_sample_ns = {
+ .max_fte = MLX5_ESW_VPORT_TBL_SIZE_SAMPLE,
+ .max_num_groups = 0, /* default num of groups */
+ .flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | MLX5_FLOW_TABLE_TUNNEL_EN_DECAP,
+};
+
+struct mlx5_esw_psample {
+ struct mlx5e_priv *priv;
+ struct mlx5_flow_table *termtbl;
+ struct mlx5_flow_handle *termtbl_rule;
+ DECLARE_HASHTABLE(hashtbl, 8);
+ struct mutex ht_lock; /* protect hashtbl */
+ DECLARE_HASHTABLE(restore_hashtbl, 8);
+ struct mutex restore_lock; /* protect restore_hashtbl */
+};
+
+struct mlx5_sampler {
+ struct hlist_node hlist;
+ u32 sampler_id;
+ u32 sample_ratio;
+ u32 sample_table_id;
+ u32 default_table_id;
+ int count;
+};
+
+struct mlx5_sample_flow {
+ struct mlx5_sampler *sampler;
+ struct mlx5_sample_restore *restore;
+ struct mlx5_flow_attr *pre_attr;
+ struct mlx5_flow_handle *pre_rule;
+ struct mlx5_flow_handle *rule;
+};
+
+struct mlx5_sample_restore {
+ struct hlist_node hlist;
+ struct mlx5_modify_hdr *modify_hdr;
+ struct mlx5_flow_handle *rule;
+ u32 obj_id;
+ int count;
+};
+
+static int
+sampler_termtbl_create(struct mlx5_esw_psample *esw_psample)
+{
+ struct mlx5_core_dev *dev = esw_psample->priv->mdev;
+ struct mlx5_eswitch *esw = dev->priv.eswitch;
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_destination dest = {};
+ struct mlx5_flow_namespace *root_ns;
+ struct mlx5_flow_act act = {};
+ int err;
+
+ if (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, termination_table)) {
+ mlx5_core_warn(dev, "termination table is not supported\n");
+ return -EOPNOTSUPP;
+ }
+
+ root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
+ if (!root_ns) {
+ mlx5_core_warn(dev, "failed to get FDB flow namespace\n");
+ return -EOPNOTSUPP;
+ }
+
+ ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED;
+ ft_attr.autogroup.max_num_groups = 1;
+ ft_attr.prio = FDB_SLOW_PATH;
+ ft_attr.max_fte = 1;
+ ft_attr.level = 1;
+ esw_psample->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
+ if (IS_ERR(esw_psample->termtbl)) {
+ err = PTR_ERR(esw_psample->termtbl);
+ mlx5_core_warn(dev, "failed to create termtbl, err: %d\n", err);
+ return err;
+ }
+
+ act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ dest.vport.num = esw->manager_vport;
+ esw_psample->termtbl_rule = mlx5_add_flow_rules(esw_psample->termtbl, NULL, &act, &dest, 1);
+ if (IS_ERR(esw_psample->termtbl_rule)) {
+ err = PTR_ERR(esw_psample->termtbl_rule);
+ mlx5_core_warn(dev, "failed to create termtbl rule, err: %d\n", err);
+ mlx5_destroy_flow_table(esw_psample->termtbl);
+ return err;
+ }
+
+ return 0;
+}
+
+static void
+sampler_termtbl_destroy(struct mlx5_esw_psample *esw_psample)
+{
+ mlx5_del_flow_rules(esw_psample->termtbl_rule);
+ mlx5_destroy_flow_table(esw_psample->termtbl);
+}
+
+static int
+sampler_obj_create(struct mlx5_core_dev *mdev, struct mlx5_sampler *sampler)
+{
+ u32 in[MLX5_ST_SZ_DW(create_sampler_obj_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+ u64 general_obj_types;
+ void *obj;
+ int err;
+
+ general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
+ if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_SAMPLER))
+ return -EOPNOTSUPP;
+ if (!MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, ignore_flow_level))
+ return -EOPNOTSUPP;
+
+ obj = MLX5_ADDR_OF(create_sampler_obj_in, in, sampler_object);
+ MLX5_SET(sampler_obj, obj, table_type, FS_FT_FDB);
+ MLX5_SET(sampler_obj, obj, ignore_flow_level, 1);
+ MLX5_SET(sampler_obj, obj, level, 1);
+ MLX5_SET(sampler_obj, obj, sample_ratio, sampler->sample_ratio);
+ MLX5_SET(sampler_obj, obj, sample_table_id, sampler->sample_table_id);
+ MLX5_SET(sampler_obj, obj, default_table_id, sampler->default_table_id);
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_SAMPLER);
+
+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+ if (!err)
+ sampler->sampler_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+
+ return err;
+}
+
+static void
+sampler_obj_destroy(struct mlx5_core_dev *mdev, u32 sampler_id)
+{
+ u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_SAMPLER);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sampler_id);
+
+ mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static u32
+sampler_hash(u32 sample_ratio, u32 default_table_id)
+{
+ return jhash_2words(sample_ratio, default_table_id, 0);
+}
+
+static int
+sampler_cmp(u32 sample_ratio1, u32 default_table_id1, u32 sample_ratio2, u32 default_table_id2)
+{
+ return sample_ratio1 != sample_ratio2 || default_table_id1 != default_table_id2;
+}
+
+static struct mlx5_sampler *
+sampler_get(struct mlx5_esw_psample *esw_psample, u32 sample_ratio, u32 default_table_id)
+{
+ struct mlx5_sampler *sampler;
+ u32 hash_key;
+ int err;
+
+ mutex_lock(&esw_psample->ht_lock);
+ hash_key = sampler_hash(sample_ratio, default_table_id);
+ hash_for_each_possible(esw_psample->hashtbl, sampler, hlist, hash_key)
+ if (!sampler_cmp(sampler->sample_ratio, sampler->default_table_id,
+ sample_ratio, default_table_id))
+ goto add_ref;
+
+ sampler = kzalloc(sizeof(*sampler), GFP_KERNEL);
+ if (!sampler) {
+ err = -ENOMEM;
+ goto err_alloc;
+ }
+
+ sampler->sample_table_id = esw_psample->termtbl->id;
+ sampler->default_table_id = default_table_id;
+ sampler->sample_ratio = sample_ratio;
+
+ err = sampler_obj_create(esw_psample->priv->mdev, sampler);
+ if (err)
+ goto err_create;
+
+ hash_add(esw_psample->hashtbl, &sampler->hlist, hash_key);
+
+add_ref:
+ sampler->count++;
+ mutex_unlock(&esw_psample->ht_lock);
+ return sampler;
+
+err_create:
+ kfree(sampler);
+err_alloc:
+ mutex_unlock(&esw_psample->ht_lock);
+ return ERR_PTR(err);
+}
+
+static void
+sampler_put(struct mlx5_esw_psample *esw_psample, struct mlx5_sampler *sampler)
+{
+ mutex_lock(&esw_psample->ht_lock);
+ if (--sampler->count == 0) {
+ hash_del(&sampler->hlist);
+ sampler_obj_destroy(esw_psample->priv->mdev, sampler->sampler_id);
+ kfree(sampler);
+ }
+ mutex_unlock(&esw_psample->ht_lock);
+}
+
+static struct mlx5_modify_hdr *
+sample_metadata_rule_get(struct mlx5_core_dev *mdev, u32 obj_id)
+{
+ struct mlx5e_tc_mod_hdr_acts mod_acts = {};
+ struct mlx5_modify_hdr *modify_hdr;
+ int err;
+
+ err = mlx5e_tc_match_to_reg_set(mdev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB,
+ CHAIN_TO_REG, obj_id);
+ if (err)
+ goto err_set_regc0;
+
+ modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_FDB,
+ mod_acts.num_actions,
+ mod_acts.actions);
+ if (IS_ERR(modify_hdr)) {
+ err = PTR_ERR(modify_hdr);
+ goto err_modify_hdr;
+ }
+
+ dealloc_mod_hdr_actions(&mod_acts);
+ return modify_hdr;
+
+err_modify_hdr:
+ dealloc_mod_hdr_actions(&mod_acts);
+err_set_regc0:
+ return ERR_PTR(err);
+}
+
+static struct mlx5_sample_restore *
+sample_restore_get(struct mlx5_esw_psample *esw_psample, u32 obj_id)
+{
+ struct mlx5_core_dev *mdev = esw_psample->priv->mdev;
+ struct mlx5_eswitch *esw = mdev->priv.eswitch;
+ struct mlx5_sample_restore *restore;
+ struct mlx5_modify_hdr *modify_hdr;
+ int err;
+
+ mutex_lock(&esw_psample->restore_lock);
+ hash_for_each_possible(esw_psample->restore_hashtbl, restore, hlist, obj_id)
+ if (restore->obj_id == obj_id)
+ goto add_ref;
+
+ restore = kzalloc(sizeof(*restore), GFP_KERNEL);
+ if (!restore) {
+ err = -ENOMEM;
+ goto err_alloc;
+ }
+ restore->obj_id = obj_id;
+
+ modify_hdr = sample_metadata_rule_get(mdev, obj_id);
+ if (IS_ERR(modify_hdr)) {
+ err = PTR_ERR(modify_hdr);
+ goto err_modify_hdr;
+ }
+ restore->modify_hdr = modify_hdr;
+
+ restore->rule = esw_add_restore_rule(esw, obj_id);
+ if (IS_ERR(restore->rule)) {
+ err = PTR_ERR(restore->rule);
+ goto err_restore;
+ }
+
+ hash_add(esw_psample->restore_hashtbl, &restore->hlist, obj_id);
+add_ref:
+ restore->count++;
+ mutex_unlock(&esw_psample->restore_lock);
+ return restore;
+
+err_restore:
+ mlx5_modify_header_dealloc(mdev, restore->modify_hdr);
+err_modify_hdr:
+ kfree(restore);
+err_alloc:
+ mutex_unlock(&esw_psample->restore_lock);
+ return ERR_PTR(err);
+}
+
+static void
+sample_restore_put(struct mlx5_esw_psample *esw_psample, struct mlx5_sample_restore *restore)
+{
+ mutex_lock(&esw_psample->restore_lock);
+ if (--restore->count == 0)
+ hash_del(&restore->hlist);
+ mutex_unlock(&esw_psample->restore_lock);
+
+ if (!restore->count) {
+ mlx5_del_flow_rules(restore->rule);
+ mlx5_modify_header_dealloc(esw_psample->priv->mdev, restore->modify_hdr);
+ kfree(restore);
+ }
+}
+
+void mlx5_esw_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj)
+{
+ u32 trunc_size = mapped_obj->sample.trunc_size;
+ struct psample_group psample_group = {};
+ struct psample_metadata md = {};
+
+ md.trunc_size = trunc_size ? min(trunc_size, skb->len) : skb->len;
+ md.in_ifindex = skb->dev->ifindex;
+ psample_group.group_num = mapped_obj->sample.group_id;
+ psample_group.net = &init_net;
+ skb_push(skb, skb->mac_len);
+
+ psample_sample_packet(&psample_group, skb, mapped_obj->sample.rate, &md);
+}
+
+/* For the following typical flow table:
+ *
+ * +-------------------------------+
+ * + original flow table +
+ * +-------------------------------+
+ * + original match +
+ * +-------------------------------+
+ * + sample action + other actions +
+ * +-------------------------------+
+ *
+ * We translate the tc filter with sample action to the following HW model:
+ *
+ * +---------------------+
+ * + original flow table +
+ * +---------------------+
+ * + original match +
+ * +---------------------+
+ * |
+ * v
+ * +------------------------------------------------+
+ * + Flow Sampler Object +
+ * +------------------------------------------------+
+ * + sample ratio +
+ * +------------------------------------------------+
+ * + sample table id | default table id +
+ * +------------------------------------------------+
+ * | |
+ * v v
+ * +-----------------------------+ +----------------------------------------+
+ * + sample table + + default table per <vport, chain, prio> +
+ * +-----------------------------+ +----------------------------------------+
+ * + forward to management vport + + original match +
+ * +-----------------------------+ +----------------------------------------+
+ * + other actions +
+ * +----------------------------------------+
+ */
+struct mlx5_flow_handle *
+mlx5_esw_sample_offload(struct mlx5_esw_psample *esw_psample,
+ struct mlx5_flow_spec *spec,
+ struct mlx5_flow_attr *attr)
+{
+ struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
+ struct mlx5_vport_tbl_attr per_vport_tbl_attr;
+ struct mlx5_esw_flow_attr *pre_esw_attr;
+ struct mlx5_mapped_obj restore_obj = {};
+ struct mlx5_sample_flow *sample_flow;
+ struct mlx5_sample_attr *sample_attr;
+ struct mlx5_flow_table *default_tbl;
+ struct mlx5_flow_attr *pre_attr;
+ struct mlx5_eswitch *esw;
+ u32 obj_id;
+ int err;
+
+ if (IS_ERR_OR_NULL(esw_psample))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ /* If slow path flag is set, eg. when the neigh is invalid for encap,
+ * don't offload sample action.
+ */
+ esw = esw_psample->priv->mdev->priv.eswitch;
+ if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)
+ return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
+
+ sample_flow = kzalloc(sizeof(*sample_flow), GFP_KERNEL);
+ if (!sample_flow)
+ return ERR_PTR(-ENOMEM);
+ esw_attr->sample->sample_flow = sample_flow;
+
+ /* Allocate default table per vport, chain and prio. Otherwise, there is
+ * only one default table for the same sampler object. Rules with different
+ * prio and chain may overlap. For CT sample action, per vport default
+ * table is needed to resotre the metadata.
+ */
+ per_vport_tbl_attr.chain = attr->chain;
+ per_vport_tbl_attr.prio = attr->prio;
+ per_vport_tbl_attr.vport = esw_attr->in_rep->vport;
+ per_vport_tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
+ default_tbl = mlx5_esw_vporttbl_get(esw, &per_vport_tbl_attr);
+ if (IS_ERR(default_tbl)) {
+ err = PTR_ERR(default_tbl);
+ goto err_default_tbl;
+ }
+
+ /* Perform the original matches on the default table.
+ * Offload all actions except the sample action.
+ */
+ esw_attr->sample->sample_default_tbl = default_tbl;
+ /* When offloading sample and encap action, if there is no valid
+ * neigh data struct, a slow path rule is offloaded first. Source
+ * port metadata match is set at that time. A per vport table is
+ * already allocated. No need to match it again. So clear the source
+ * port metadata match.
+ */
+ mlx5_eswitch_clear_rule_source_port(esw, spec);
+ sample_flow->rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
+ if (IS_ERR(sample_flow->rule)) {
+ err = PTR_ERR(sample_flow->rule);
+ goto err_offload_rule;
+ }
+
+ /* Create sampler object. */
+ sample_flow->sampler = sampler_get(esw_psample, esw_attr->sample->rate, default_tbl->id);
+ if (IS_ERR(sample_flow->sampler)) {
+ err = PTR_ERR(sample_flow->sampler);
+ goto err_sampler;
+ }
+
+ /* Create an id mapping reg_c0 value to sample object. */
+ restore_obj.type = MLX5_MAPPED_OBJ_SAMPLE;
+ restore_obj.sample.group_id = esw_attr->sample->group_num;
+ restore_obj.sample.rate = esw_attr->sample->rate;
+ restore_obj.sample.trunc_size = esw_attr->sample->trunc_size;
+ err = mapping_add(esw->offloads.reg_c0_obj_pool, &restore_obj, &obj_id);
+ if (err)
+ goto err_obj_id;
+ esw_attr->sample->restore_obj_id = obj_id;
+
+ /* Create sample restore context. */
+ sample_flow->restore = sample_restore_get(esw_psample, obj_id);
+ if (IS_ERR(sample_flow->restore)) {
+ err = PTR_ERR(sample_flow->restore);
+ goto err_sample_restore;
+ }
+
+ /* Perform the original matches on the original table. Offload the
+ * sample action. The destination is the sampler object.
+ */
+ pre_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB);
+ if (!pre_attr) {
+ err = -ENOMEM;
+ goto err_alloc_flow_attr;
+ }
+ sample_attr = kzalloc(sizeof(*sample_attr), GFP_KERNEL);
+ if (!sample_attr) {
+ err = -ENOMEM;
+ goto err_alloc_sample_attr;
+ }
+ pre_esw_attr = pre_attr->esw_attr;
+ pre_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+ pre_attr->modify_hdr = sample_flow->restore->modify_hdr;
+ pre_attr->flags = MLX5_ESW_ATTR_FLAG_SAMPLE;
+ pre_attr->chain = attr->chain;
+ pre_attr->prio = attr->prio;
+ pre_esw_attr->sample = sample_attr;
+ pre_esw_attr->sample->sampler_id = sample_flow->sampler->sampler_id;
+ pre_esw_attr->in_mdev = esw_attr->in_mdev;
+ pre_esw_attr->in_rep = esw_attr->in_rep;
+ sample_flow->pre_rule = mlx5_eswitch_add_offloaded_rule(esw, spec, pre_attr);
+ if (IS_ERR(sample_flow->pre_rule)) {
+ err = PTR_ERR(sample_flow->pre_rule);
+ goto err_pre_offload_rule;
+ }
+ sample_flow->pre_attr = pre_attr;
+
+ return sample_flow->rule;
+
+err_pre_offload_rule:
+ kfree(sample_attr);
+err_alloc_sample_attr:
+ kfree(pre_attr);
+err_alloc_flow_attr:
+ sample_restore_put(esw_psample, sample_flow->restore);
+err_sample_restore:
+ mapping_remove(esw->offloads.reg_c0_obj_pool, obj_id);
+err_obj_id:
+ sampler_put(esw_psample, sample_flow->sampler);
+err_sampler:
+ /* For sample offload, rule is added in default_tbl. No need to call
+ * mlx5_esw_chains_put_table()
+ */
+ attr->prio = 0;
+ attr->chain = 0;
+ mlx5_eswitch_del_offloaded_rule(esw, sample_flow->rule, attr);
+err_offload_rule:
+ mlx5_esw_vporttbl_put(esw, &per_vport_tbl_attr);
+err_default_tbl:
+ return ERR_PTR(err);
+}
+
+void
+mlx5_esw_sample_unoffload(struct mlx5_esw_psample *esw_psample,
+ struct mlx5_flow_handle *rule,
+ struct mlx5_flow_attr *attr)
+{
+ struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
+ struct mlx5_sample_flow *sample_flow;
+ struct mlx5_vport_tbl_attr tbl_attr;
+ struct mlx5_flow_attr *pre_attr;
+ struct mlx5_eswitch *esw;
+
+ if (IS_ERR_OR_NULL(esw_psample))
+ return;
+
+ /* If slow path flag is set, sample action is not offloaded.
+ * No need to delete sample rule.
+ */
+ esw = esw_psample->priv->mdev->priv.eswitch;
+ if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
+ mlx5_eswitch_del_offloaded_rule(esw, rule, attr);
+ return;
+ }
+
+ sample_flow = esw_attr->sample->sample_flow;
+ pre_attr = sample_flow->pre_attr;
+ memset(pre_attr, 0, sizeof(*pre_attr));
+ esw = esw_psample->priv->mdev->priv.eswitch;
+ mlx5_eswitch_del_offloaded_rule(esw, sample_flow->pre_rule, pre_attr);
+ mlx5_eswitch_del_offloaded_rule(esw, sample_flow->rule, attr);
+
+ sample_restore_put(esw_psample, sample_flow->restore);
+ mapping_remove(esw->offloads.reg_c0_obj_pool, esw_attr->sample->restore_obj_id);
+ sampler_put(esw_psample, sample_flow->sampler);
+ tbl_attr.chain = attr->chain;
+ tbl_attr.prio = attr->prio;
+ tbl_attr.vport = esw_attr->in_rep->vport;
+ tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
+ mlx5_esw_vporttbl_put(esw, &tbl_attr);
+
+ kfree(pre_attr->esw_attr->sample);
+ kfree(pre_attr);
+ kfree(sample_flow);
+}
+
+struct mlx5_esw_psample *
+mlx5_esw_sample_init(struct mlx5e_priv *priv)
+{
+ struct mlx5_esw_psample *esw_psample;
+ int err;
+
+ esw_psample = kzalloc(sizeof(*esw_psample), GFP_KERNEL);
+ if (!esw_psample)
+ return ERR_PTR(-ENOMEM);
+ esw_psample->priv = priv;
+ err = sampler_termtbl_create(esw_psample);
+ if (err)
+ goto err_termtbl;
+
+ mutex_init(&esw_psample->ht_lock);
+ mutex_init(&esw_psample->restore_lock);
+
+ return esw_psample;
+
+err_termtbl:
+ kfree(esw_psample);
+ return ERR_PTR(err);
+}
+
+void
+mlx5_esw_sample_cleanup(struct mlx5_esw_psample *esw_psample)
+{
+ if (IS_ERR_OR_NULL(esw_psample))
+ return;
+
+ mutex_destroy(&esw_psample->restore_lock);
+ mutex_destroy(&esw_psample->ht_lock);
+ sampler_termtbl_destroy(esw_psample);
+ kfree(esw_psample);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/sample.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/sample.h
new file mode 100644
index 0000000000000000000000000000000000000000..2a3f4be10030bcb845c2a8df660fd5f644a8c860
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/sample.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#ifndef __MLX5_EN_TC_SAMPLE_H__
+#define __MLX5_EN_TC_SAMPLE_H__
+
+#include "en.h"
+#include "eswitch.h"
+
+struct mlx5e_priv;
+struct mlx5_flow_attr;
+struct mlx5_esw_psample;
+
+struct mlx5_sample_attr {
+ u32 group_num;
+ u32 rate;
+ u32 trunc_size;
+ u32 restore_obj_id;
+ u32 sampler_id;
+ struct mlx5_flow_table *sample_default_tbl;
+ struct mlx5_sample_flow *sample_flow;
+};
+
+void mlx5_esw_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj);
+
+struct mlx5_flow_handle *
+mlx5_esw_sample_offload(struct mlx5_esw_psample *sample_priv,
+ struct mlx5_flow_spec *spec,
+ struct mlx5_flow_attr *attr);
+
+void
+mlx5_esw_sample_unoffload(struct mlx5_esw_psample *sample_priv,
+ struct mlx5_flow_handle *rule,
+ struct mlx5_flow_attr *attr);
+
+struct mlx5_esw_psample *
+mlx5_esw_sample_init(struct mlx5e_priv *priv);
+
+void
+mlx5_esw_sample_cleanup(struct mlx5_esw_psample *esw_psample);
+
+#endif /* __MLX5_EN_TC_SAMPLE_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c
new file mode 100644
index 0000000000000000000000000000000000000000..9e72118f2e4c0cf4b4ea227b93aab40bbd407c19
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2021 Mellanox Technologies.
+
+#include "eswitch.h"
+
+/* This struct is used as a key to the hash table and we need it to be packed
+ * so hash result is consistent
+ */
+struct mlx5_vport_key {
+ u32 chain;
+ u16 prio;
+ u16 vport;
+ u16 vhca_id;
+ const struct esw_vport_tbl_namespace *vport_ns;
+} __packed;
+
+struct mlx5_vport_table {
+ struct hlist_node hlist;
+ struct mlx5_flow_table *fdb;
+ u32 num_rules;
+ struct mlx5_vport_key key;
+};
+
+static struct mlx5_flow_table *
+esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns,
+ const struct esw_vport_tbl_namespace *vport_ns)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_table *fdb;
+
+ if (vport_ns->max_num_groups)
+ ft_attr.autogroup.max_num_groups = vport_ns->max_num_groups;
+ else
+ ft_attr.autogroup.max_num_groups = esw->params.large_group_num;
+ ft_attr.max_fte = vport_ns->max_fte;
+ ft_attr.prio = FDB_PER_VPORT;
+ ft_attr.flags = vport_ns->flags;
+ fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+ if (IS_ERR(fdb)) {
+ esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n",
+ PTR_ERR(fdb));
+ }
+
+ return fdb;
+}
+
+static u32 flow_attr_to_vport_key(struct mlx5_eswitch *esw,
+ struct mlx5_vport_tbl_attr *attr,
+ struct mlx5_vport_key *key)
+{
+ key->vport = attr->vport;
+ key->chain = attr->chain;
+ key->prio = attr->prio;
+ key->vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
+ key->vport_ns = attr->vport_ns;
+ return jhash(key, sizeof(*key), 0);
+}
+
+/* caller must hold vports.lock */
+static struct mlx5_vport_table *
+esw_vport_tbl_lookup(struct mlx5_eswitch *esw, struct mlx5_vport_key *skey, u32 key)
+{
+ struct mlx5_vport_table *e;
+
+ hash_for_each_possible(esw->fdb_table.offloads.vports.table, e, hlist, key)
+ if (!memcmp(&e->key, skey, sizeof(*skey)))
+ return e;
+
+ return NULL;
+}
+
+struct mlx5_flow_table *
+mlx5_esw_vporttbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr)
+{
+ struct mlx5_core_dev *dev = esw->dev;
+ struct mlx5_flow_namespace *ns;
+ struct mlx5_flow_table *fdb;
+ struct mlx5_vport_table *e;
+ struct mlx5_vport_key skey;
+ u32 hkey;
+
+ mutex_lock(&esw->fdb_table.offloads.vports.lock);
+ hkey = flow_attr_to_vport_key(esw, attr, &skey);
+ e = esw_vport_tbl_lookup(esw, &skey, hkey);
+ if (e) {
+ e->num_rules++;
+ goto out;
+ }
+
+ e = kzalloc(sizeof(*e), GFP_KERNEL);
+ if (!e) {
+ fdb = ERR_PTR(-ENOMEM);
+ goto err_alloc;
+ }
+
+ ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
+ if (!ns) {
+ esw_warn(dev, "Failed to get FDB namespace\n");
+ fdb = ERR_PTR(-ENOENT);
+ goto err_ns;
+ }
+
+ fdb = esw_vport_tbl_create(esw, ns, attr->vport_ns);
+ if (IS_ERR(fdb))
+ goto err_ns;
+
+ e->fdb = fdb;
+ e->num_rules = 1;
+ e->key = skey;
+ hash_add(esw->fdb_table.offloads.vports.table, &e->hlist, hkey);
+out:
+ mutex_unlock(&esw->fdb_table.offloads.vports.lock);
+ return e->fdb;
+
+err_ns:
+ kfree(e);
+err_alloc:
+ mutex_unlock(&esw->fdb_table.offloads.vports.lock);
+ return fdb;
+}
+
+void
+mlx5_esw_vporttbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr)
+{
+ struct mlx5_vport_table *e;
+ struct mlx5_vport_key key;
+ u32 hkey;
+
+ mutex_lock(&esw->fdb_table.offloads.vports.lock);
+ hkey = flow_attr_to_vport_key(esw, attr, &key);
+ e = esw_vport_tbl_lookup(esw, &key, hkey);
+ if (!e || --e->num_rules)
+ goto out;
+
+ hash_del(&e->hlist);
+ mlx5_destroy_flow_table(e->fdb);
+ kfree(e);
+out:
+ mutex_unlock(&esw->fdb_table.offloads.vports.lock);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 64db903068c112c442f640b63c30e9c80c91a01c..deafb0e0378709261024f0add6d0c00fd6528658 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -46,6 +46,24 @@
#include "lib/fs_chains.h"
#include "sf/sf.h"
#include "en/tc_ct.h"
+#include "esw/sample.h"
+
+enum mlx5_mapped_obj_type {
+ MLX5_MAPPED_OBJ_CHAIN,
+ MLX5_MAPPED_OBJ_SAMPLE,
+};
+
+struct mlx5_mapped_obj {
+ enum mlx5_mapped_obj_type type;
+ union {
+ u32 chain;
+ struct {
+ u32 group_id;
+ u32 rate;
+ u32 trunc_size;
+ } sample;
+ };
+};
#ifdef CONFIG_MLX5_ESWITCH
@@ -206,6 +224,7 @@ struct mlx5_esw_offload {
struct mlx5_flow_table *ft_offloads_restore;
struct mlx5_flow_group *restore_group;
struct mlx5_modify_hdr *restore_copy_hdr_id;
+ struct mapping_ctx *reg_c0_obj_pool;
struct mlx5_flow_table *ft_offloads;
struct mlx5_flow_group *vport_rx_group;
@@ -357,6 +376,9 @@ void
mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw,
struct mlx5_termtbl_handle *tt);
+void
+mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec);
+
struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
struct mlx5_flow_spec *spec,
@@ -404,6 +426,7 @@ enum {
MLX5_ESW_ATTR_FLAG_SLOW_PATH = BIT(1),
MLX5_ESW_ATTR_FLAG_NO_IN_PORT = BIT(2),
MLX5_ESW_ATTR_FLAG_SRC_REWRITE = BIT(3),
+ MLX5_ESW_ATTR_FLAG_SAMPLE = BIT(4),
};
struct mlx5_esw_flow_attr {
@@ -428,6 +451,7 @@ struct mlx5_esw_flow_attr {
} dests[MLX5_MAX_FLOW_FWD_VPORTS];
struct mlx5_rx_tun_attr *rx_tun_attr;
struct mlx5_pkt_reformat *decap_pkt_reformat;
+ struct mlx5_sample_attr *sample;
};
int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
@@ -713,13 +737,26 @@ void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
struct mlx5_vport *vport);
-int mlx5_esw_vport_tbl_get(struct mlx5_eswitch *esw);
-void mlx5_esw_vport_tbl_put(struct mlx5_eswitch *esw);
+struct esw_vport_tbl_namespace {
+ int max_fte;
+ int max_num_groups;
+ u32 flags;
+};
+
+struct mlx5_vport_tbl_attr {
+ u16 chain;
+ u16 prio;
+ u16 vport;
+ const struct esw_vport_tbl_namespace *vport_ns;
+};
+
+struct mlx5_flow_table *
+mlx5_esw_vporttbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr);
+void
+mlx5_esw_vporttbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr);
struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag);
-u32
-esw_get_max_restore_tag(struct mlx5_eswitch *esw);
int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num);
void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index d5de6bf622ce47352eaf8ace4b5245d732840071..ac92ffc8a5d3d69fb8cc5feb09753194e9ba688b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -47,6 +47,7 @@
#include "lib/eq.h"
#include "lib/fs_chains.h"
#include "en_tc.h"
+#include "en/mapping.h"
/* There are two match-all miss flows, one for unicast dst mac and
* one for multicast.
@@ -54,184 +55,14 @@
#define MLX5_ESW_MISS_FLOWS (2)
#define UPLINK_REP_INDEX 0
-/* Per vport tables */
-
-#define MLX5_ESW_VPORT_TABLE_SIZE 128
-
-/* This struct is used as a key to the hash table and we need it to be packed
- * so hash result is consistent
- */
-struct mlx5_vport_key {
- u32 chain;
- u16 prio;
- u16 vport;
- u16 vhca_id;
-} __packed;
-
-struct mlx5_vport_tbl_attr {
- u16 chain;
- u16 prio;
- u16 vport;
-};
-
-struct mlx5_vport_table {
- struct hlist_node hlist;
- struct mlx5_flow_table *fdb;
- u32 num_rules;
- struct mlx5_vport_key key;
-};
-
+#define MLX5_ESW_VPORT_TBL_SIZE 128
#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
-static struct mlx5_flow_table *
-esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns)
-{
- struct mlx5_flow_table_attr ft_attr = {};
- struct mlx5_flow_table *fdb;
-
- ft_attr.autogroup.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS;
- ft_attr.max_fte = MLX5_ESW_VPORT_TABLE_SIZE;
- ft_attr.prio = FDB_PER_VPORT;
- fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
- if (IS_ERR(fdb)) {
- esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n",
- PTR_ERR(fdb));
- }
-
- return fdb;
-}
-
-static u32 flow_attr_to_vport_key(struct mlx5_eswitch *esw,
- struct mlx5_vport_tbl_attr *attr,
- struct mlx5_vport_key *key)
-{
- key->vport = attr->vport;
- key->chain = attr->chain;
- key->prio = attr->prio;
- key->vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
- return jhash(key, sizeof(*key), 0);
-}
-
-/* caller must hold vports.lock */
-static struct mlx5_vport_table *
-esw_vport_tbl_lookup(struct mlx5_eswitch *esw, struct mlx5_vport_key *skey, u32 key)
-{
- struct mlx5_vport_table *e;
-
- hash_for_each_possible(esw->fdb_table.offloads.vports.table, e, hlist, key)
- if (!memcmp(&e->key, skey, sizeof(*skey)))
- return e;
-
- return NULL;
-}
-
-static void
-esw_vport_tbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr)
-{
- struct mlx5_vport_table *e;
- struct mlx5_vport_key key;
- u32 hkey;
-
- mutex_lock(&esw->fdb_table.offloads.vports.lock);
- hkey = flow_attr_to_vport_key(esw, attr, &key);
- e = esw_vport_tbl_lookup(esw, &key, hkey);
- if (!e || --e->num_rules)
- goto out;
-
- hash_del(&e->hlist);
- mlx5_destroy_flow_table(e->fdb);
- kfree(e);
-out:
- mutex_unlock(&esw->fdb_table.offloads.vports.lock);
-}
-
-static struct mlx5_flow_table *
-esw_vport_tbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr)
-{
- struct mlx5_core_dev *dev = esw->dev;
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_table *fdb;
- struct mlx5_vport_table *e;
- struct mlx5_vport_key skey;
- u32 hkey;
-
- mutex_lock(&esw->fdb_table.offloads.vports.lock);
- hkey = flow_attr_to_vport_key(esw, attr, &skey);
- e = esw_vport_tbl_lookup(esw, &skey, hkey);
- if (e) {
- e->num_rules++;
- goto out;
- }
-
- e = kzalloc(sizeof(*e), GFP_KERNEL);
- if (!e) {
- fdb = ERR_PTR(-ENOMEM);
- goto err_alloc;
- }
-
- ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
- if (!ns) {
- esw_warn(dev, "Failed to get FDB namespace\n");
- fdb = ERR_PTR(-ENOENT);
- goto err_ns;
- }
-
- fdb = esw_vport_tbl_create(esw, ns);
- if (IS_ERR(fdb))
- goto err_ns;
-
- e->fdb = fdb;
- e->num_rules = 1;
- e->key = skey;
- hash_add(esw->fdb_table.offloads.vports.table, &e->hlist, hkey);
-out:
- mutex_unlock(&esw->fdb_table.offloads.vports.lock);
- return e->fdb;
-
-err_ns:
- kfree(e);
-err_alloc:
- mutex_unlock(&esw->fdb_table.offloads.vports.lock);
- return fdb;
-}
-
-int mlx5_esw_vport_tbl_get(struct mlx5_eswitch *esw)
-{
- struct mlx5_vport_tbl_attr attr;
- struct mlx5_flow_table *fdb;
- struct mlx5_vport *vport;
- int i;
-
- attr.chain = 0;
- attr.prio = 1;
- mlx5_esw_for_all_vports(esw, i, vport) {
- attr.vport = vport->vport;
- fdb = esw_vport_tbl_get(esw, &attr);
- if (IS_ERR(fdb))
- goto out;
- }
- return 0;
-
-out:
- mlx5_esw_vport_tbl_put(esw);
- return PTR_ERR(fdb);
-}
-
-void mlx5_esw_vport_tbl_put(struct mlx5_eswitch *esw)
-{
- struct mlx5_vport_tbl_attr attr;
- struct mlx5_vport *vport;
- int i;
-
- attr.chain = 0;
- attr.prio = 1;
- mlx5_esw_for_all_vports(esw, i, vport) {
- attr.vport = vport->vport;
- esw_vport_tbl_put(esw, &attr);
- }
-}
-
-/* End: Per vport tables */
+static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
+ .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
+ .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
+ .flags = 0,
+};
static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
u16 vport_num)
@@ -255,6 +86,26 @@ mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
}
+/* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
+ * are not needed as well in the following process. So clear them all for simplicity.
+ */
+void
+mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
+{
+ if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
+ void *misc2;
+
+ misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
+ MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
+
+ misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
+ MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
+
+ if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
+ spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
+ }
+}
+
static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
struct mlx5_flow_spec *spec,
@@ -325,6 +176,19 @@ esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
true);
}
+static int
+esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_esw_flow_attr *esw_attr,
+ int i)
+{
+ flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
+ dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
+ dest[i].sampler_id = esw_attr->sample->sampler_id;
+
+ return 0;
+}
+
static int
esw_setup_ft_dest(struct mlx5_flow_destination *dest,
struct mlx5_flow_act *flow_act,
@@ -554,7 +418,10 @@ esw_setup_dests(struct mlx5_flow_destination *dest,
MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
attr->flags |= MLX5_ESW_ATTR_FLAG_SRC_REWRITE;
- if (attr->dest_ft) {
+ if (attr->flags & MLX5_ESW_ATTR_FLAG_SAMPLE) {
+ esw_setup_sampler_dest(dest, flow_act, esw_attr, *i);
+ (*i)++;
+ } else if (attr->dest_ft) {
esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
(*i)++;
} else if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
@@ -657,12 +524,16 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
flow_act.modify_hdr = attr->modify_hdr;
- if (split) {
+ /* esw_attr->sample is allocated only when there is a sample action */
+ if (esw_attr->sample && esw_attr->sample->sample_default_tbl) {
+ fdb = esw_attr->sample->sample_default_tbl;
+ } else if (split) {
fwd_attr.chain = attr->chain;
fwd_attr.prio = attr->prio;
fwd_attr.vport = esw_attr->in_rep->vport;
+ fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
- fdb = esw_vport_tbl_get(esw, &fwd_attr);
+ fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
} else {
if (attr->chain || attr->prio)
fdb = mlx5_chains_get_table(chains, attr->chain,
@@ -694,7 +565,7 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
err_add_rule:
if (split)
- esw_vport_tbl_put(esw, &fwd_attr);
+ mlx5_esw_vporttbl_put(esw, &fwd_attr);
else if (attr->chain || attr->prio)
mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
err_esw_get:
@@ -727,7 +598,8 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
fwd_attr.chain = attr->chain;
fwd_attr.prio = attr->prio;
fwd_attr.vport = esw_attr->in_rep->vport;
- fwd_fdb = esw_vport_tbl_get(esw, &fwd_attr);
+ fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
+ fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
if (IS_ERR(fwd_fdb)) {
rule = ERR_CAST(fwd_fdb);
goto err_get_fwd;
@@ -772,7 +644,7 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
return rule;
err_chain_src_rewrite:
esw_put_dest_tables_loop(esw, attr, 0, i);
- esw_vport_tbl_put(esw, &fwd_attr);
+ mlx5_esw_vporttbl_put(esw, &fwd_attr);
err_get_fwd:
mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
err_get_fast:
@@ -807,15 +679,16 @@ __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
fwd_attr.chain = attr->chain;
fwd_attr.prio = attr->prio;
fwd_attr.vport = esw_attr->in_rep->vport;
+ fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
}
if (fwd_rule) {
- esw_vport_tbl_put(esw, &fwd_attr);
+ mlx5_esw_vporttbl_put(esw, &fwd_attr);
mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count);
} else {
if (split)
- esw_vport_tbl_put(esw, &fwd_attr);
+ mlx5_esw_vporttbl_put(esw, &fwd_attr);
else if (attr->chain || attr->prio)
mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
esw_cleanup_dests(esw, attr);
@@ -1453,7 +1326,7 @@ esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
misc_parameters_2);
MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
- ESW_CHAIN_TAG_METADATA_MASK);
+ ESW_REG_C0_USER_DATA_METADATA_MASK);
misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
misc_parameters_2);
MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
@@ -1479,12 +1352,6 @@ esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
return flow_rule;
}
-u32
-esw_get_max_restore_tag(struct mlx5_eswitch *esw)
-{
- return ESW_CHAIN_TAG_METADATA_MASK;
-}
-
#define MAX_PF_SQ 256
#define MAX_SQ_NVPORTS 32
@@ -1514,6 +1381,44 @@ static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
}
#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
+static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
+{
+ struct mlx5_vport_tbl_attr attr;
+ struct mlx5_vport *vport;
+ int i;
+
+ attr.chain = 0;
+ attr.prio = 1;
+ mlx5_esw_for_all_vports(esw, i, vport) {
+ attr.vport = vport->vport;
+ attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
+ mlx5_esw_vporttbl_put(esw, &attr);
+ }
+}
+
+static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
+{
+ struct mlx5_vport_tbl_attr attr;
+ struct mlx5_flow_table *fdb;
+ struct mlx5_vport *vport;
+ int i;
+
+ attr.chain = 0;
+ attr.prio = 1;
+ mlx5_esw_for_all_vports(esw, i, vport) {
+ attr.vport = vport->vport;
+ attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
+ fdb = mlx5_esw_vporttbl_get(esw, &attr);
+ if (IS_ERR(fdb))
+ goto out;
+ }
+ return 0;
+
+out:
+ esw_vport_tbl_put(esw);
+ return PTR_ERR(fdb);
+}
+
#define fdb_modify_header_fwd_to_table_supported(esw) \
(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
@@ -1563,7 +1468,7 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
attr.max_ft_sz = fdb_max;
attr.max_grp_num = esw->params.large_group_num;
attr.default_ft = miss_fdb;
- attr.max_restore_tag = esw_get_max_restore_tag(esw);
+ attr.mapping = esw->offloads.reg_c0_obj_pool;
chains = mlx5_chains_create(dev, &attr);
if (IS_ERR(chains)) {
@@ -1591,7 +1496,7 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
/* Open level 1 for split fdb rules now if prios isn't supported */
if (!mlx5_chains_prios_supported(chains)) {
- err = mlx5_esw_vport_tbl_get(esw);
+ err = esw_vport_tbl_get(esw);
if (err)
goto level_1_err;
}
@@ -1615,7 +1520,7 @@ static void
esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
{
if (!mlx5_chains_prios_supported(chains))
- mlx5_esw_vport_tbl_put(esw);
+ esw_vport_tbl_put(esw);
mlx5_chains_put_table(chains, 0, 1, 0);
mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
mlx5_chains_destroy(chains);
@@ -2057,7 +1962,7 @@ static int esw_create_restore_table(struct mlx5_eswitch *esw)
goto out_free;
}
- ft_attr.max_fte = 1 << ESW_CHAIN_TAG_METADATA_BITS;
+ ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
ft = mlx5_create_flow_table(ns, &ft_attr);
if (IS_ERR(ft)) {
err = PTR_ERR(ft);
@@ -2072,7 +1977,7 @@ static int esw_create_restore_table(struct mlx5_eswitch *esw)
misc_parameters_2);
MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
- ESW_CHAIN_TAG_METADATA_MASK);
+ ESW_REG_C0_USER_DATA_METADATA_MASK);
MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
ft_attr.max_fte - 1);
@@ -2730,6 +2635,7 @@ static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
int esw_offloads_enable(struct mlx5_eswitch *esw)
{
+ struct mapping_ctx *reg_c0_obj_pool;
struct mlx5_vport *vport;
int err, i;
@@ -2757,6 +2663,15 @@ int esw_offloads_enable(struct mlx5_eswitch *esw)
if (err)
goto err_vport_metadata;
+ reg_c0_obj_pool = mapping_create(sizeof(struct mlx5_mapped_obj),
+ ESW_REG_C0_USER_DATA_METADATA_MASK,
+ true);
+ if (IS_ERR(reg_c0_obj_pool)) {
+ err = PTR_ERR(reg_c0_obj_pool);
+ goto err_pool;
+ }
+ esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
+
err = esw_offloads_steering_init(esw);
if (err)
goto err_steering_init;
@@ -2783,6 +2698,8 @@ err_vports:
err_uplink:
esw_offloads_steering_cleanup(esw);
err_steering_init:
+ mapping_destroy(reg_c0_obj_pool);
+err_pool:
esw_set_passing_vport_metadata(esw, false);
err_vport_metadata:
esw_offloads_metadata_uninit(esw);
@@ -2821,6 +2738,7 @@ void esw_offloads_disable(struct mlx5_eswitch *esw)
esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
esw_set_passing_vport_metadata(esw, false);
esw_offloads_steering_cleanup(esw);
+ mapping_destroy(esw->offloads.reg_c0_obj_pool);
esw_offloads_metadata_uninit(esw);
esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
mlx5_rdma_disable_roce(esw->dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
index 381325b4a863ea7670161e2c10cabcfb3d87c576..00ef10a1a9f86dabe1078d4d2f0a6ef5124e6322 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
@@ -7,15 +7,11 @@
#include "lib/fs_chains.h"
#include "en/mapping.h"
-#include "mlx5_core.h"
#include "fs_core.h"
-#include "eswitch.h"
-#include "en.h"
#include "en_tc.h"
#define chains_lock(chains) ((chains)->lock)
#define chains_ht(chains) ((chains)->chains_ht)
-#define chains_mapping(chains) ((chains)->chains_mapping)
#define prios_ht(chains) ((chains)->prios_ht)
#define ft_pool_left(chains) ((chains)->ft_left)
#define tc_default_ft(chains) ((chains)->tc_default_ft)
@@ -300,7 +296,7 @@ create_chain_restore(struct fs_chain *chain)
!mlx5_chains_prios_supported(chains))
return 0;
- err = mapping_add(chains_mapping(chains), &chain->chain, &index);
+ err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
if (err)
return err;
if (index == MLX5_FS_DEFAULT_FLOW_TAG) {
@@ -310,10 +306,8 @@ create_chain_restore(struct fs_chain *chain)
*
* This case isn't possible with MLX5_FS_DEFAULT_FLOW_TAG = 0.
*/
- err = mapping_add(chains_mapping(chains),
- &chain->chain, &index);
- mapping_remove(chains_mapping(chains),
- MLX5_FS_DEFAULT_FLOW_TAG);
+ err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
+ mapping_remove(chains->chains_mapping, MLX5_FS_DEFAULT_FLOW_TAG);
if (err)
return err;
}
@@ -361,7 +355,7 @@ err_mod_hdr:
mlx5_del_flow_rules(chain->restore_rule);
err_rule:
/* Datapath can't find this mapping, so we can safely remove it */
- mapping_remove(chains_mapping(chains), chain->id);
+ mapping_remove(chains->chains_mapping, chain->id);
return err;
}
@@ -376,7 +370,7 @@ static void destroy_chain_restore(struct fs_chain *chain)
mlx5_del_flow_rules(chain->restore_rule);
mlx5_modify_header_dealloc(chains->dev, chain->miss_modify_hdr);
- mapping_remove(chains_mapping(chains), chain->id);
+ mapping_remove(chains->chains_mapping, chain->id);
}
static struct fs_chain *
@@ -797,7 +791,6 @@ static struct mlx5_fs_chains *
mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
{
struct mlx5_fs_chains *chains_priv;
- struct mapping_ctx *mapping;
u32 max_flow_counter;
int err;
@@ -816,6 +809,7 @@ mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
chains_priv->flags = attr->flags;
chains_priv->ns = attr->ns;
chains_priv->group_num = attr->max_grp_num;
+ chains_priv->chains_mapping = attr->mapping;
tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
@@ -832,20 +826,10 @@ mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
if (err)
goto init_prios_ht_err;
- mapping = mapping_create(sizeof(u32), attr->max_restore_tag,
- true);
- if (IS_ERR(mapping)) {
- err = PTR_ERR(mapping);
- goto mapping_err;
- }
- chains_mapping(chains_priv) = mapping;
-
mutex_init(&chains_lock(chains_priv));
return chains_priv;
-mapping_err:
- rhashtable_destroy(&prios_ht(chains_priv));
init_prios_ht_err:
rhashtable_destroy(&chains_ht(chains_priv));
init_chains_ht_err:
@@ -857,7 +841,6 @@ static void
mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
{
mutex_destroy(&chains_lock(chains));
- mapping_destroy(chains_mapping(chains));
rhashtable_destroy(&prios_ht(chains));
rhashtable_destroy(&chains_ht(chains));
@@ -884,25 +867,18 @@ int
mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
u32 *chain_mapping)
{
- return mapping_add(chains_mapping(chains), &chain, chain_mapping);
+ struct mapping_ctx *ctx = chains->chains_mapping;
+ struct mlx5_mapped_obj mapped_obj = {};
+
+ mapped_obj.type = MLX5_MAPPED_OBJ_CHAIN;
+ mapped_obj.chain = chain;
+ return mapping_add(ctx, &mapped_obj, chain_mapping);
}
int
mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
{
- return mapping_remove(chains_mapping(chains), chain_mapping);
-}
-
-int mlx5_get_chain_for_tag(struct mlx5_fs_chains *chains, u32 tag,
- u32 *chain)
-{
- int err;
+ struct mapping_ctx *ctx = chains->chains_mapping;
- err = mapping_find(chains_mapping(chains), tag, chain);
- if (err) {
- mlx5_core_warn(chains->dev, "Can't find chain for tag: %d\n", tag);
- return -ENOENT;
- }
-
- return 0;
+ return mapping_remove(ctx, chain_mapping);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
index 6d5be31b05dd758ef2fe4e099b57504259501649..e96f345e7dae7b7f2d4460094dfb8ef593ebcf89 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
@@ -7,6 +7,7 @@
#include <linux/mlx5/fs.h>
struct mlx5_fs_chains;
+struct mlx5_mapped_obj;
enum mlx5_chains_flags {
MLX5_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),
@@ -20,7 +21,7 @@ struct mlx5_chains_attr {
u32 max_ft_sz;
u32 max_grp_num;
struct mlx5_flow_table *default_ft;
- u32 max_restore_tag;
+ struct mapping_ctx *mapping;
};
#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
@@ -63,9 +64,6 @@ struct mlx5_fs_chains *
mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr);
void mlx5_chains_destroy(struct mlx5_fs_chains *chains);
-int
-mlx5_get_chain_for_tag(struct mlx5_fs_chains *chains, u32 tag, u32 *chain);
-
void
mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
struct mlx5_flow_table *ft);
diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h
index 994c2c8cb4fd74600a103fd7a0dd3efd3d0bc668..125ae482383b269be3a3e46e18dd4f0648ceb67a 100644
--- a/include/linux/mlx5/eswitch.h
+++ b/include/linux/mlx5/eswitch.h
@@ -74,20 +74,19 @@ bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw);
bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw);
/* Reg C0 usage:
- * Reg C0 = < ESW_PFNUM_BITS(4) | ESW_VPORT BITS(12) | ESW_CHAIN_TAG(16) >
+ * Reg C0 = < ESW_PFNUM_BITS(4) | ESW_VPORT BITS(12) | ESW_REG_C0_OBJ(16) >
*
* Highest 4 bits of the reg c0 is the PF_NUM (range 0-15), 12 bits of
* unique non-zero vport id (range 1-4095). The rest (lowest 16 bits) is left
- * for tc chain tag restoration.
+ * for user data objects managed by a common mapping context.
* PFNUM + VPORT comprise the SOURCE_PORT matching.
*/
#define ESW_VPORT_BITS 12
#define ESW_PFNUM_BITS 4
#define ESW_SOURCE_PORT_METADATA_BITS (ESW_PFNUM_BITS + ESW_VPORT_BITS)
#define ESW_SOURCE_PORT_METADATA_OFFSET (32 - ESW_SOURCE_PORT_METADATA_BITS)
-#define ESW_CHAIN_TAG_METADATA_BITS (32 - ESW_SOURCE_PORT_METADATA_BITS)
-#define ESW_CHAIN_TAG_METADATA_MASK GENMASK(ESW_CHAIN_TAG_METADATA_BITS - 1,\
- 0)
+#define ESW_REG_C0_USER_DATA_METADATA_BITS (32 - ESW_SOURCE_PORT_METADATA_BITS)
+#define ESW_REG_C0_USER_DATA_METADATA_MASK GENMASK(ESW_REG_C0_USER_DATA_METADATA_BITS - 1, 0)
static inline u32 mlx5_eswitch_get_vport_metadata_mask(void)
{