diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox')
102 files changed, 5154 insertions, 1891 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 6ec7d6e0181d..3c3e84100d5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -8,7 +8,6 @@ config MLX5_CORE depends on PCI select AUXILIARY_BUS select NET_DEVLINK - depends on VXLAN || !VXLAN depends on MLXFW || !MLXFW depends on PTP_1588_CLOCK_OPTIONAL depends on PCI_HYPERV_INTERFACE || !PCI_HYPERV_INTERFACE @@ -208,3 +207,14 @@ config MLX5_DPLL help DPLL support in Mellanox Technologies ConnectX NICs. +config MLX5_EN_PSP + bool "Mellanox Technologies support for PSP cryptography-offload acceleration" + depends on INET_PSP + depends on MLX5_CORE_EN + default y + help + mlx5 device offload support for Google PSP Security Protocol offload. + Adds support for PSP encryption offload and for SPI and key generation + interfaces to PSP Stack which supports PSP crypto offload. + + If unsure, say Y. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index a253c73db9e5..8ffa286a18f5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -17,7 +17,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o diag/reporter_vnic.o \ - fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o + fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o lib/nv_param.o # # Netdev basic @@ -69,7 +69,7 @@ mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += en/tc/sample.o # Core extra # mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \ - ecpf.o rdma.o esw/legacy.o \ + ecpf.o rdma.o esw/legacy.o esw/adj_vport.o \ esw/devlink_port.o esw/vporttbl.o esw/qos.o esw/ipsec.o mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \ @@ -85,7 +85,9 @@ mlx5_core-$(CONFIG_MLX5_BRIDGE) += esw/bridge.o esw/bridge_mcast.o esw/bridge mlx5_core-$(CONFIG_HWMON) += hwmon.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o -mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o +ifneq ($(CONFIG_VXLAN),) + mlx5_core-y += lib/vxlan.o +endif mlx5_core-$(CONFIG_PTP_1588_CLOCK) += lib/clock.o mlx5_core-$(CONFIG_PCI_HYPERV_INTERFACE) += lib/hv.o lib/hv_vhca.o @@ -110,6 +112,8 @@ mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/ktls_stats.o \ en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \ en_accel/ktls_tx.o en_accel/ktls_rx.o +mlx5_core-$(CONFIG_MLX5_EN_PSP) += en_accel/psp.o en_accel/psp_rxtx.o + # # SW Steering # diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index e395ef5f356e..722282cebce9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -294,6 +294,10 @@ static void poll_timeout(struct mlx5_cmd_work_ent *ent) return; } cond_resched(); + if (mlx5_cmd_is_down(dev)) { + ent->ret = -ENXIO; + return; + } } while (time_before(jiffies, poll_end)); ent->ret = -ETIMEDOUT; @@ -1070,7 +1074,7 @@ static void cmd_work_handler(struct work_struct *work) poll_timeout(ent); /* make sure we read the descriptor after ownership is SW */ rmb(); - mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT)); + mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, !!ent->ret); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c index 1fd403713baf..e9f319a9bdd6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c @@ -145,7 +145,6 @@ int mlx5_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, mlx5_core_dbg(dev, "failed adding CP 0x%x to debug file system\n", cq->cqn); - cq->uar = dev->priv.uar; cq->irqn = eq->core.irqn; return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 2c0e0c16ca90..fceea83abbd7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -10,6 +10,7 @@ #include "esw/qos.h" #include "sf/dev/dev.h" #include "sf/sf.h" +#include "lib/nv_param.h" static int mlx5_devlink_flash_update(struct devlink *devlink, struct devlink_flash_update_params *params, @@ -203,11 +204,6 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, return 0; } - if (mlx5_lag_is_active(dev)) { - NL_SET_ERR_MSG_MOD(extack, "reload is unsupported in Lag mode"); - return -EOPNOTSUPP; - } - if (mlx5_core_is_mp_slave(dev)) { NL_SET_ERR_MSG_MOD(extack, "reload is unsupported for multi port slave"); return -EOPNOTSUPP; @@ -534,6 +530,25 @@ mlx5_devlink_hairpin_queue_size_validate(struct devlink *devlink, u32 id, return 0; } +static int mlx5_devlink_num_doorbells_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *mdev = devlink_priv(devlink); + u32 val32 = val.vu32; + u32 max_num_channels; + + max_num_channels = mlx5e_get_max_num_channels(mdev); + if (val32 > max_num_channels) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Requested num_doorbells (%u) exceeds maximum number of channels (%u)", + val32, max_num_channels); + return -EINVAL; + } + + return 0; +} + static void mlx5_devlink_hairpin_params_init_values(struct devlink *devlink) { struct mlx5_core_dev *dev = devlink_priv(devlink); @@ -613,6 +628,9 @@ static const struct devlink_param mlx5_devlink_eth_params[] = { "hairpin_queue_size", DEVLINK_PARAM_TYPE_U32, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL, mlx5_devlink_hairpin_queue_size_validate), + DEVLINK_PARAM_GENERIC(NUM_DOORBELLS, + BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL, + mlx5_devlink_num_doorbells_validate), }; static int mlx5_devlink_eth_params_register(struct devlink *devlink) @@ -636,6 +654,10 @@ static int mlx5_devlink_eth_params_register(struct devlink *devlink) mlx5_devlink_hairpin_params_init_values(devlink); + value.vu32 = MLX5_DEFAULT_NUM_DOORBELLS; + devl_param_driverinit_value_set(devlink, + DEVLINK_PARAM_GENERIC_ID_NUM_DOORBELLS, + value); return 0; } @@ -650,6 +672,105 @@ static void mlx5_devlink_eth_params_unregister(struct devlink *devlink) ARRAY_SIZE(mlx5_devlink_eth_params)); } +#define MLX5_PCIE_CONG_THRESH_MAX 10000 +#define MLX5_PCIE_CONG_THRESH_DEF_LOW 7500 +#define MLX5_PCIE_CONG_THRESH_DEF_HIGH 9000 + +static int +mlx5_devlink_pcie_cong_thresh_validate(struct devlink *devl, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + if (val.vu16 > MLX5_PCIE_CONG_THRESH_MAX) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Value %u > max supported (%u)", + val.vu16, MLX5_PCIE_CONG_THRESH_MAX); + + return -EINVAL; + } + + switch (id) { + case MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_LOW: + case MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_HIGH: + case MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_LOW: + case MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_HIGH: + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static void mlx5_devlink_pcie_cong_init_values(struct devlink *devlink) +{ + union devlink_param_value value; + u32 id; + + value.vu16 = MLX5_PCIE_CONG_THRESH_DEF_LOW; + id = MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_LOW; + devl_param_driverinit_value_set(devlink, id, value); + + value.vu16 = MLX5_PCIE_CONG_THRESH_DEF_HIGH; + id = MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_HIGH; + devl_param_driverinit_value_set(devlink, id, value); + + value.vu16 = MLX5_PCIE_CONG_THRESH_DEF_LOW; + id = MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_LOW; + devl_param_driverinit_value_set(devlink, id, value); + + value.vu16 = MLX5_PCIE_CONG_THRESH_DEF_HIGH; + id = MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_HIGH; + devl_param_driverinit_value_set(devlink, id, value); +} + +static const struct devlink_param mlx5_devlink_pcie_cong_params[] = { + DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_LOW, + "pcie_cong_inbound_low", DEVLINK_PARAM_TYPE_U16, + BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL, + mlx5_devlink_pcie_cong_thresh_validate), + DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_HIGH, + "pcie_cong_inbound_high", DEVLINK_PARAM_TYPE_U16, + BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL, + mlx5_devlink_pcie_cong_thresh_validate), + DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_LOW, + "pcie_cong_outbound_low", DEVLINK_PARAM_TYPE_U16, + BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL, + mlx5_devlink_pcie_cong_thresh_validate), + DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_HIGH, + "pcie_cong_outbound_high", DEVLINK_PARAM_TYPE_U16, + BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL, + mlx5_devlink_pcie_cong_thresh_validate), +}; + +static int mlx5_devlink_pcie_cong_params_register(struct devlink *devlink) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + int err; + + if (!mlx5_pcie_cong_event_supported(dev)) + return 0; + + err = devl_params_register(devlink, mlx5_devlink_pcie_cong_params, + ARRAY_SIZE(mlx5_devlink_pcie_cong_params)); + if (err) + return err; + + mlx5_devlink_pcie_cong_init_values(devlink); + + return 0; +} + +static void mlx5_devlink_pcie_cong_params_unregister(struct devlink *devlink) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + + if (!mlx5_pcie_cong_event_supported(dev)) + return; + + devl_params_unregister(devlink, mlx5_devlink_pcie_cong_params, + ARRAY_SIZE(mlx5_devlink_pcie_cong_params)); +} + static int mlx5_devlink_enable_rdma_validate(struct devlink *devlink, u32 id, union devlink_param_value val, struct netlink_ext_ack *extack) @@ -895,8 +1016,20 @@ int mlx5_devlink_params_register(struct devlink *devlink) if (err) goto max_uc_list_err; + err = mlx5_devlink_pcie_cong_params_register(devlink); + if (err) + goto pcie_cong_err; + + err = mlx5_nv_param_register_dl_params(devlink); + if (err) + goto nv_param_err; + return 0; +nv_param_err: + mlx5_devlink_pcie_cong_params_unregister(devlink); +pcie_cong_err: + mlx5_devlink_max_uc_list_params_unregister(devlink); max_uc_list_err: mlx5_devlink_auxdev_params_unregister(devlink); auxdev_reg_err: @@ -907,6 +1040,8 @@ auxdev_reg_err: void mlx5_devlink_params_unregister(struct devlink *devlink) { + mlx5_nv_param_unregister_dl_params(devlink); + mlx5_devlink_pcie_cong_params_unregister(devlink); mlx5_devlink_max_uc_list_params_unregister(devlink); mlx5_devlink_auxdev_params_unregister(devlink); devl_params_unregister(devlink, mlx5_devlink_params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index 961f75da6227..c9555119a661 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -22,6 +22,11 @@ enum mlx5_devlink_param_id { MLX5_DEVLINK_PARAM_ID_ESW_MULTIPORT, MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES, MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_LOW, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_HIGH, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_LOW, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_HIGH, + MLX5_DEVLINK_PARAM_ID_CQE_COMPRESSION_TYPE }; struct mlx5_trap_ctx { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c index 86253a89c24c..7cae0c6e5e8a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. */ +#include <linux/mlx5/vport.h> + #include "reporter_vnic.h" #include "en_stats.h" #include "devlink.h" @@ -105,6 +107,15 @@ void mlx5_reporter_vnic_diagnose_counters(struct mlx5_core_dev *dev, } if (MLX5_CAP_GEN(dev, nic_cap_reg)) mlx5_reporter_vnic_diagnose_counter_icm(dev, fmsg, vport_num, other_vport); + if (MLX5_CAP_GEN(dev, vnic_env_cnt_bar_uar_access)) + devlink_fmsg_u32_pair_put(fmsg, "bar_uar_access", + VNIC_ENV_GET(&vnic, bar_uar_access)); + if (MLX5_CAP_GEN(dev, vnic_env_cnt_odp_page_fault)) { + devlink_fmsg_u32_pair_put(fmsg, "odp_local_triggered_page_fault", + VNIC_ENV_GET(&vnic, odp_local_triggered_page_fault)); + devlink_fmsg_u32_pair_put(fmsg, "odp_remote_triggered_page_fault", + VNIC_ENV_GET(&vnic, odp_remote_triggered_page_fault)); + } devlink_fmsg_obj_nest_end(fmsg); devlink_fmsg_pair_nest_end(fmsg); @@ -133,11 +144,11 @@ void mlx5_reporter_vnic_create(struct mlx5_core_dev *dev) health->vnic_reporter = devlink_health_reporter_create(devlink, &mlx5_reporter_vnic_ops, - 0, dev); + dev); if (IS_ERR(health->vnic_reporter)) mlx5_core_warn(dev, - "Failed to create vnic reporter, err = %ld\n", - PTR_ERR(health->vnic_reporter)); + "Failed to create vnic reporter, err = %pe\n", + health->vnic_reporter); } void mlx5_reporter_vnic_destroy(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 0dd3bc0f4caa..14e3207b14e7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -47,6 +47,7 @@ #include <linux/rhashtable.h> #include <net/udp_tunnel.h> #include <net/switchdev.h> +#include <net/psp/types.h> #include <net/xdp.h> #include <linux/dim.h> #include <linux/bits.h> @@ -68,7 +69,7 @@ struct page_pool; #define MLX5E_METADATA_ETHER_TYPE (0x8CE4) #define MLX5E_METADATA_ETHER_LEN 8 -#define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN) +#define MLX5E_ETH_HARD_MTU (ETH_HLEN + PSP_ENCAP_HLEN + PSP_TRL_SIZE + VLAN_HLEN + ETH_FCS_LEN) #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu)) #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu)) @@ -344,6 +345,7 @@ struct mlx5e_cq { /* data path - accessed per napi poll */ u16 event_ctr; struct napi_struct *napi; + struct mlx5_uars_page *uar; struct mlx5_core_cq mcq; struct mlx5e_ch_stats *ch_stats; @@ -788,6 +790,7 @@ struct mlx5e_channel { int vec_ix; int sd_ix; int cpu; + struct mlx5_sq_bfreg *bfreg; /* Sync between icosq recovery and XSK enable/disable. */ struct mutex icosq_recovery_lock; @@ -936,6 +939,9 @@ struct mlx5e_priv { #ifdef CONFIG_MLX5_EN_IPSEC struct mlx5e_ipsec *ipsec; #endif +#ifdef CONFIG_MLX5_EN_PSP + struct mlx5e_psp *psp; +#endif #ifdef CONFIG_MLX5_EN_TLS struct mlx5e_tls *tls; #endif @@ -950,6 +956,7 @@ struct mlx5e_priv { struct mlx5e_mqprio_rl *mqprio_rl; struct dentry *dfs_root; struct mlx5_devcom_comp_dev *devcom; + struct ethtool_fec_hist_range *fec_ranges; }; struct mlx5e_dev { @@ -1060,6 +1067,7 @@ struct mlx5e_create_cq_param { struct mlx5e_ch_stats *ch_stats; int node; int ix; + struct mlx5_uars_page *uar; }; struct mlx5e_cq_param; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index ac65e3191480..c3408b3f7010 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -57,7 +57,7 @@ struct mlx5e_l2_table { bool promisc_enabled; }; -#define MLX5E_NUM_INDIR_TIRS (MLX5_NUM_TT - 1) +#define MLX5E_NUM_INDIR_TIRS (MLX5_NUM_INDIR_TIRS) #define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\ MLX5_HASH_FIELD_SEL_DST_IP) @@ -88,7 +88,7 @@ enum { #ifdef CONFIG_MLX5_EN_ARFS MLX5E_ARFS_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, #endif -#ifdef CONFIG_MLX5_EN_IPSEC +#if defined(CONFIG_MLX5_EN_IPSEC) || defined(CONFIG_MLX5_EN_PSP) MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, MLX5E_ACCEL_FS_POL_FT_LEVEL, @@ -132,7 +132,8 @@ struct mlx5e_ptp_fs; void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, struct mlx5e_rx_res *rx_res, - struct ttc_params *ttc_params, bool tunnel); + struct ttc_params *ttc_params, bool tunnel, + bool ipsec_rss); void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs); int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c index b4f3bd7d346e..195863b2c013 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c @@ -138,8 +138,8 @@ void mlx5e_hv_vhca_stats_create(struct mlx5e_priv *priv) if (IS_ERR_OR_NULL(agent)) { if (IS_ERR(agent)) netdev_warn(priv->netdev, - "Failed to create hv vhca stats agent, err = %ld\n", - PTR_ERR(agent)); + "Failed to create hv vhca stats agent, err = %pe\n", + agent); kvfree(priv->stats_agent.buf); return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 3cca06a74cf9..3692298e10f2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -6,6 +6,7 @@ #include "en/port.h" #include "en_accel/en_accel.h" #include "en_accel/ipsec.h" +#include "en_accel/psp.h" #include <linux/dim.h> #include <net/page_pool/types.h> #include <net/xdp_sock_drv.h> @@ -611,6 +612,7 @@ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e .ch_stats = c->stats, .node = cpu_to_node(c->cpu), .ix = c->vec_ix, + .uar = c->bfreg->up, }; } @@ -810,7 +812,7 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev, { void *cqc = param->cqc; - MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index); if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD); } @@ -1003,7 +1005,8 @@ void mlx5e_build_sq_param(struct mlx5_core_dev *mdev, bool allow_swp; allow_swp = mlx5_geneve_tx_allowed(mdev) || - (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO); + (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO) || + mlx5_is_psp_device(mdev); mlx5e_build_sq_param_common(mdev, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); MLX5_SET(sqc, sqc, allow_swp, allow_swp); @@ -1229,7 +1232,6 @@ static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev, void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, struct mlx5e_sq_param *param) { void *sqc = param->sqc; @@ -1256,7 +1258,7 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(mdev); mlx5e_build_sq_param(mdev, params, &cparam->txq_sq); - mlx5e_build_xdpsq_param(mdev, params, NULL, &cparam->xdp_sq); + mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq); mlx5e_build_icosq_param(mdev, icosq_log_wq_sz, &cparam->icosq); mlx5e_build_async_icosq_param(mdev, async_icosq_log_wq_sz, &cparam->async_icosq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 488ccdbc1e2c..00617c65fe3c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -51,6 +51,7 @@ struct mlx5e_create_sq_param { u32 tisn; u8 tis_lst_sz; u8 min_inline_mode; + u32 uar_page; }; /* Striding RQ dynamic parameters */ @@ -132,7 +133,6 @@ void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev, struct mlx5e_cq_param *param); void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, struct mlx5e_sq_param *param); int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c b/drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c index 0ed017569a19..2eb666a46f39 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +#include "../devlink.h" #include "en.h" #include "pcie_cong_event.h" @@ -23,6 +24,7 @@ struct mlx5e_pcie_cong_stats { u32 pci_bw_inbound_low; u32 pci_bw_outbound_high; u32 pci_bw_outbound_low; + u32 pci_bw_stale_event; }; struct mlx5e_pcie_cong_event { @@ -41,13 +43,6 @@ struct mlx5e_pcie_cong_event { struct mlx5e_pcie_cong_stats stats; }; -/* In units of 0.01 % */ -static const struct mlx5e_pcie_cong_thresh default_thresh_config = { - .inbound_high = 9000, - .inbound_low = 7500, - .outbound_high = 9000, - .outbound_low = 7500, -}; static const struct counter_desc mlx5e_pcie_cong_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats, @@ -58,6 +53,8 @@ static const struct counter_desc mlx5e_pcie_cong_stats_desc[] = { pci_bw_outbound_high) }, { MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats, pci_bw_outbound_low) }, + { MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats, + pci_bw_stale_event) }, }; #define NUM_PCIE_CONG_COUNTERS ARRAY_SIZE(mlx5e_pcie_cong_stats_desc) @@ -218,8 +215,10 @@ static void mlx5e_pcie_cong_event_work(struct work_struct *work) } changes = cong_event->state ^ new_cong_state; - if (!changes) + if (!changes) { + cong_event->stats.pci_bw_stale_event++; return; + } cong_event->state = new_cong_state; @@ -249,8 +248,60 @@ static int mlx5e_pcie_cong_event_handler(struct notifier_block *nb, return NOTIFY_OK; } +static int +mlx5e_pcie_cong_get_thresh_config(struct mlx5_core_dev *dev, + struct mlx5e_pcie_cong_thresh *config) +{ + u32 ids[4] = { + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_LOW, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_HIGH, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_LOW, + MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_HIGH, + }; + struct devlink *devlink = priv_to_devlink(dev); + union devlink_param_value val[4]; + + for (int i = 0; i < 4; i++) { + u32 id = ids[i]; + int err; + + err = devl_param_driverinit_value_get(devlink, id, &val[i]); + if (err) + return err; + } + + config->inbound_low = val[0].vu16; + config->inbound_high = val[1].vu16; + config->outbound_low = val[2].vu16; + config->outbound_high = val[3].vu16; + + return 0; +} + +static int +mlx5e_thresh_config_validate(struct mlx5_core_dev *mdev, + const struct mlx5e_pcie_cong_thresh *config) +{ + int err = 0; + + if (config->inbound_low >= config->inbound_high) { + err = -EINVAL; + mlx5_core_err(mdev, "PCIe inbound congestion threshold configuration invalid: low (%u) >= high (%u).\n", + config->inbound_low, config->inbound_high); + } + + if (config->outbound_low >= config->outbound_high) { + err = -EINVAL; + mlx5_core_err(mdev, "PCIe outbound congestion threshold configuration invalid: low (%u) >= high (%u).\n", + config->outbound_low, config->outbound_high); + } + + return err; +} + int mlx5e_pcie_cong_event_init(struct mlx5e_priv *priv) { + struct mlx5e_pcie_cong_thresh thresh_config = {}; struct mlx5e_pcie_cong_event *cong_event; struct mlx5_core_dev *mdev = priv->mdev; int err; @@ -258,6 +309,16 @@ int mlx5e_pcie_cong_event_init(struct mlx5e_priv *priv) if (!mlx5_pcie_cong_event_supported(mdev)) return 0; + err = mlx5e_pcie_cong_get_thresh_config(mdev, &thresh_config); + if (WARN_ON(err)) + return err; + + err = mlx5e_thresh_config_validate(mdev, &thresh_config); + if (err) { + mlx5_core_err(mdev, "PCIe congestion event feature disabled\n"); + return err; + } + cong_event = kvzalloc_node(sizeof(*cong_event), GFP_KERNEL, mdev->priv.numa_node); if (!cong_event) @@ -269,7 +330,7 @@ int mlx5e_pcie_cong_event_init(struct mlx5e_priv *priv) cong_event->priv = priv; - err = mlx5_cmd_pcie_cong_event_set(mdev, &default_thresh_config, + err = mlx5_cmd_pcie_cong_event_set(mdev, &thresh_config, &cong_event->obj_id); if (err) { mlx5_core_warn(mdev, "Error creating a PCIe congestion event object\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h index 66d276a1be83..f4a19ffbb641 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h @@ -66,23 +66,11 @@ struct mlx5e_port_buffer { struct mlx5e_bufferx_reg buffer[MLX5E_MAX_NETWORK_BUFFER]; }; -#ifdef CONFIG_MLX5_CORE_EN_DCB int mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv, u32 change, unsigned int mtu, struct ieee_pfc *pfc, u32 *buffer_size, u8 *prio2buffer); -#else -static inline int -mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv, - u32 change, unsigned int mtu, - void *pfc, - u32 *buffer_size, - u8 *prio2buffer) -{ - return 0; -} -#endif int mlx5e_port_query_buffer(struct mlx5e_priv *priv, struct mlx5e_port_buffer *port_buffer); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 391b4e9c9dc4..c93ee969ea64 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -334,7 +334,7 @@ static int mlx5e_ptp_alloc_txqsq(struct mlx5e_ptp *c, int txq_ix, sq->mdev = mdev; sq->ch_ix = MLX5E_PTP_CHANNEL_IX; sq->txq_ix = txq_ix; - sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + sq->uar_map = c->bfreg->map; sq->min_inline_mode = params->tx_min_inline_mode; sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); sq->stats = &c->priv->ptp_stats.sq[tc]; @@ -486,6 +486,7 @@ static int mlx5e_ptp_open_txqsq(struct mlx5e_ptp *c, u32 tisn, csp.wq_ctrl = &txqsq->wq_ctrl; csp.min_inline_mode = txqsq->min_inline_mode; csp.ts_cqe_to_dest_cqn = ptpsq->ts_cq.mcq.cqn; + csp.uar_page = c->bfreg->index; err = mlx5e_create_sq_rdy(c->mdev, sqp, &csp, 0, &txqsq->sqn); if (err) @@ -577,6 +578,7 @@ static int mlx5e_ptp_open_tx_cqs(struct mlx5e_ptp *c, ccp.ch_stats = c->stats; ccp.napi = &c->napi; ccp.ix = MLX5E_PTP_CHANNEL_IX; + ccp.uar = c->bfreg->up; cq_param = &cparams->txq_sq_param.cqp; @@ -626,6 +628,7 @@ static int mlx5e_ptp_open_rx_cq(struct mlx5e_ptp *c, ccp.ch_stats = c->stats; ccp.napi = &c->napi; ccp.ix = MLX5E_PTP_CHANNEL_IX; + ccp.uar = c->bfreg->up; cq_param = &cparams->rq_param.cqp; @@ -900,6 +903,7 @@ int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params, c->num_tc = mlx5e_get_dcb_num_tc(params); c->stats = &priv->ptp_stats.ch; c->lag_port = lag_port; + c->bfreg = &mdev->priv.bfreg; err = mlx5e_ptp_set_state(c, params); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index 883c044852f1..1b3c9648220b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -66,6 +66,7 @@ struct mlx5e_ptp { struct mlx5_core_dev *mdev; struct hwtstamp_config *tstamp; DECLARE_BITMAP(state, MLX5E_PTP_STATE_NUM_STATES); + struct mlx5_sq_bfreg *bfreg; }; static inline bool mlx5e_use_ptpsq(struct sk_buff *skb) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index 0f5d7ea8956f..9d1c677814e0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -488,8 +488,8 @@ static int mlx5_esw_bridge_switchdev_event(struct notifier_block *nb, fdb_info, br_offloads); if (IS_ERR(work)) { - WARN_ONCE(1, "Failed to init switchdev work, err=%ld", - PTR_ERR(work)); + WARN_ONCE(1, "Failed to init switchdev work, err=%pe", + work); return notifier_from_errno(PTR_ERR(work)); } @@ -527,7 +527,8 @@ void mlx5e_rep_bridge_init(struct mlx5e_priv *priv) br_offloads = mlx5_esw_bridge_init(esw); rtnl_unlock(); if (IS_ERR(br_offloads)) { - esw_warn(mdev, "Failed to init esw bridge (err=%ld)\n", PTR_ERR(br_offloads)); + esw_warn(mdev, "Failed to init esw bridge (err=%pe)\n", + br_offloads); return; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index 16c44d628eda..b1415992ffa2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -651,25 +651,29 @@ void mlx5e_reporter_icosq_resume_recovery(struct mlx5e_channel *c) mutex_unlock(&c->icosq_recovery_lock); } +#define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500 +#define MLX5E_REPORTER_RX_BURST_PERIOD 500 + static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = { .name = "rx", .recover = mlx5e_rx_reporter_recover, .diagnose = mlx5e_rx_reporter_diagnose, .dump = mlx5e_rx_reporter_dump, + .default_graceful_period = MLX5E_REPORTER_RX_GRACEFUL_PERIOD, + .default_burst_period = MLX5E_REPORTER_RX_BURST_PERIOD, }; -#define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500 - void mlx5e_reporter_rx_create(struct mlx5e_priv *priv) { + struct devlink_port *port = priv->netdev->devlink_port; struct devlink_health_reporter *reporter; - reporter = devlink_port_health_reporter_create(priv->netdev->devlink_port, + reporter = devlink_port_health_reporter_create(port, &mlx5_rx_reporter_ops, - MLX5E_REPORTER_RX_GRACEFUL_PERIOD, priv); + priv); if (IS_ERR(reporter)) { - netdev_warn(priv->netdev, "Failed to create rx reporter, err = %ld\n", - PTR_ERR(reporter)); + netdev_warn(priv->netdev, "Failed to create rx reporter, err = %pe\n", + reporter); return; } priv->rx_reporter = reporter; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 85d5cb39b107..9e2cf191ed30 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -539,26 +539,30 @@ void mlx5e_reporter_tx_ptpsq_unhealthy(struct mlx5e_ptpsq *ptpsq) mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); } +#define MLX5E_REPORTER_TX_GRACEFUL_PERIOD 500 +#define MLX5E_REPORTER_TX_BURST_PERIOD 500 + static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = { .name = "tx", .recover = mlx5e_tx_reporter_recover, .diagnose = mlx5e_tx_reporter_diagnose, .dump = mlx5e_tx_reporter_dump, + .default_graceful_period = MLX5E_REPORTER_TX_GRACEFUL_PERIOD, + .default_burst_period = MLX5E_REPORTER_TX_BURST_PERIOD, }; -#define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500 - void mlx5e_reporter_tx_create(struct mlx5e_priv *priv) { + struct devlink_port *port = priv->netdev->devlink_port; struct devlink_health_reporter *reporter; - reporter = devlink_port_health_reporter_create(priv->netdev->devlink_port, + reporter = devlink_port_health_reporter_create(port, &mlx5_tx_reporter_ops, - MLX5_REPORTER_TX_GRACEFUL_PERIOD, priv); + priv); if (IS_ERR(reporter)) { netdev_warn(priv->netdev, - "Failed to create tx reporter, err = %ld\n", - PTR_ERR(reporter)); + "Failed to create tx reporter, err = %pe\n", + reporter); return; } priv->tx_reporter = reporter; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c index c68ba0e58fa6..c96cbc4b0dbf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c @@ -75,15 +75,14 @@ struct mlx5e_rss { struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS]; struct mlx5e_rqt rqt; struct mlx5_core_dev *mdev; /* primary */ - u32 drop_rqn; - bool inner_ft_support; + struct mlx5e_rss_params params; bool enabled; refcount_t refcnt; }; bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss) { - return rss->inner_ft_support; + return rss->params.inner_ft_support; } void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels) @@ -91,7 +90,7 @@ void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_ch rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels); } -int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev, +int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, u32 actual_table_size, u32 max_table_size) { indir->table = kvmalloc_array(max_table_size, sizeof(*indir->table), GFP_KERNEL); @@ -139,7 +138,8 @@ static struct mlx5e_rss *mlx5e_rss_init_copy(const struct mlx5e_rss *from) if (!rss) return ERR_PTR(-ENOMEM); - err = mlx5e_rss_params_indir_init(&rss->indir, from->mdev, from->indir.actual_table_size, + err = mlx5e_rss_params_indir_init(&rss->indir, + from->indir.actual_table_size, from->indir.max_table_size); if (err) goto err_free_rss; @@ -192,11 +192,12 @@ mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt) return rss_tt; } -static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, - enum mlx5_traffic_types tt, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner) +static int +mlx5e_rss_create_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner) { + bool rss_inner = rss->params.inner_ft_support; struct mlx5e_rss_params_traffic_type rss_tt; struct mlx5e_tir_builder *builder; struct mlx5e_tir **tir_p; @@ -204,7 +205,7 @@ static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, u32 rqtn; int err; - if (inner && !rss->inner_ft_support) { + if (inner && !rss_inner) { mlx5e_rss_warn(rss->mdev, "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n", tt); @@ -227,8 +228,8 @@ static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, rqtn = mlx5e_rqt_get_rqtn(&rss->rqt); mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn, - rqtn, rss->inner_ft_support); - mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param); + rqtn, rss_inner); + mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param); rss_tt = mlx5e_rss_get_tt_config(rss, tt); mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner); @@ -264,15 +265,16 @@ static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types *tir_p = NULL; } -static int mlx5e_rss_create_tirs(struct mlx5e_rss *rss, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner) +static int +mlx5e_rss_create_tirs(struct mlx5e_rss *rss, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner) { enum mlx5_traffic_types tt, max_tt; int err; for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { - err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner); + err = mlx5e_rss_create_tir(rss, tt, pkt_merge_param, inner); if (err) goto err_destroy_tirs; } @@ -335,7 +337,7 @@ static int mlx5e_rss_update_tirs(struct mlx5e_rss *rss) tt, err); } - if (!rss->inner_ft_support) + if (!rss->params.inner_ft_support) continue; err = mlx5e_rss_update_tir(rss, tt, true); @@ -355,14 +357,16 @@ static int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss) refcount_set(&rss->refcnt, 1); return mlx5e_rqt_init_direct(&rss->rqt, rss->mdev, true, - rss->drop_rqn, rss->indir.max_table_size); + rss->params.drop_rqn, + rss->indir.max_table_size); } -struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - enum mlx5e_rss_init_type type, unsigned int nch, - unsigned int max_nch) +struct mlx5e_rss * +mlx5e_rss_init(struct mlx5_core_dev *mdev, + const struct mlx5e_rss_params *params, + const struct mlx5e_rss_init_params *init_params) { + u32 rqt_max_size, rqt_size; struct mlx5e_rss *rss; int err; @@ -370,29 +374,31 @@ struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_suppo if (!rss) return ERR_PTR(-ENOMEM); - err = mlx5e_rss_params_indir_init(&rss->indir, mdev, - mlx5e_rqt_size(mdev, nch), - mlx5e_rqt_size(mdev, max_nch)); + rqt_size = mlx5e_rqt_size(mdev, init_params->nch); + rqt_max_size = mlx5e_rqt_size(mdev, init_params->max_nch); + err = mlx5e_rss_params_indir_init(&rss->indir, rqt_size, rqt_max_size); if (err) goto err_free_rss; rss->mdev = mdev; - rss->inner_ft_support = inner_ft_support; - rss->drop_rqn = drop_rqn; + rss->params = *params; err = mlx5e_rss_init_no_tirs(rss); if (err) goto err_free_indir; - if (type == MLX5E_RSS_INIT_NO_TIRS) + if (init_params->type == MLX5E_RSS_INIT_NO_TIRS) goto out; - err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, false); + err = mlx5e_rss_create_tirs(rss, init_params->pkt_merge_param, + false); if (err) goto err_destroy_rqt; - if (inner_ft_support) { - err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, true); + if (params->inner_ft_support) { + err = mlx5e_rss_create_tirs(rss, + init_params->pkt_merge_param, + true); if (err) goto err_destroy_tirs; } @@ -418,7 +424,7 @@ int mlx5e_rss_cleanup(struct mlx5e_rss *rss) mlx5e_rss_destroy_tirs(rss, false); - if (rss->inner_ft_support) + if (rss->params.inner_ft_support) mlx5e_rss_destroy_tirs(rss, true); mlx5e_rqt_destroy(&rss->rqt); @@ -448,7 +454,7 @@ u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, { struct mlx5e_tir *tir; - WARN_ON(inner && !rss->inner_ft_support); + WARN_ON(inner && !rss->params.inner_ft_support); tir = rss_get_tir(rss, tt, inner); WARN_ON(!tir); @@ -468,10 +474,10 @@ bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool /* Fill the "tirn" output parameter. * Create the requested TIR if it's its first usage. */ -int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, - enum mlx5_traffic_types tt, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner, u32 *tirn) +int +mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner, u32 *tirn) { struct mlx5e_tir *tir; @@ -479,7 +485,7 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, if (!tir) { /* TIR doesn't exist, create one */ int err; - err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner); + err = mlx5e_rss_create_tir(rss, tt, pkt_merge_param, inner); if (err) return err; tir = rss_get_tir(rss, tt, inner); @@ -512,10 +518,11 @@ void mlx5e_rss_disable(struct mlx5e_rss *rss) int err; rss->enabled = false; - err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn, NULL); + err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->params.drop_rqn, NULL); if (err) mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n", - mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err); + mlx5e_rqt_get_rqtn(&rss->rqt), + rss->params.drop_rqn, err); } int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, @@ -548,7 +555,7 @@ int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, } inner_tir: - if (!rss->inner_ft_support) + if (!rss->params.inner_ft_support) continue; tir = rss_get_tir(rss, tt, true); @@ -681,7 +688,7 @@ int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, return err; } - if (!(rss->inner_ft_support)) + if (!(rss->params.inner_ft_support)) return 0; err = mlx5e_rss_update_tir(rss, tt, true); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h index c6c1b2847cf5..5fb03cd0a411 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h @@ -13,19 +13,31 @@ enum mlx5e_rss_init_type { MLX5E_RSS_INIT_TIRS }; +struct mlx5e_rss_init_params { + enum mlx5e_rss_init_type type; + const struct mlx5e_packet_merge_param *pkt_merge_param; + unsigned int nch; + unsigned int max_nch; +}; + +struct mlx5e_rss_params { + bool inner_ft_support; + u32 drop_rqn; +}; + struct mlx5e_rss_params_traffic_type mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt); struct mlx5e_rss; -int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev, +int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, u32 actual_table_size, u32 max_table_size); void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir); void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels); -struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - enum mlx5e_rss_init_type type, unsigned int nch, - unsigned int max_nch); +struct mlx5e_rss * +mlx5e_rss_init(struct mlx5_core_dev *mdev, + const struct mlx5e_rss_params *params, + const struct mlx5e_rss_init_params *init_params); int mlx5e_rss_cleanup(struct mlx5e_rss *rss); void mlx5e_rss_refcnt_inc(struct mlx5e_rss *rss); @@ -37,10 +49,10 @@ u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner); bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner); u32 mlx5e_rss_get_rqtn(struct mlx5e_rss *rss); -int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, - enum mlx5_traffic_types tt, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner, u32 *tirn); +int +mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner, u32 *tirn); void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns); void mlx5e_rss_disable(struct mlx5e_rss *rss); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index a2acbfee2b77..ac26a32845d0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -54,17 +54,30 @@ static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res, unsigned int init_nch) { bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; + struct mlx5e_rss_init_params init_params; + struct mlx5e_rss_params rss_params; struct mlx5e_rss *rss; if (WARN_ON(res->rss[0])) return -EINVAL; - rss = mlx5e_rss_init(res->mdev, inner_ft_support, res->drop_rqn, - &res->pkt_merge_param, MLX5E_RSS_INIT_TIRS, init_nch, res->max_nch); + init_params = (struct mlx5e_rss_init_params) { + .type = MLX5E_RSS_INIT_TIRS, + .pkt_merge_param = &res->pkt_merge_param, + .nch = init_nch, + .max_nch = res->max_nch, + }; + + rss_params = (struct mlx5e_rss_params) { + .inner_ft_support = inner_ft_support, + .drop_rqn = res->drop_rqn, + }; + + rss = mlx5e_rss_init(res->mdev, &rss_params, &init_params); if (IS_ERR(rss)) return PTR_ERR(rss); - mlx5e_rss_set_indir_uniform(rss, init_nch); + mlx5e_rss_set_indir_uniform(rss, init_params.nch); res->rss[0] = rss; @@ -74,18 +87,30 @@ static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res, int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 rss_idx, unsigned int init_nch) { bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; + struct mlx5e_rss_init_params init_params; + struct mlx5e_rss_params rss_params; struct mlx5e_rss *rss; if (WARN_ON_ONCE(res->rss[rss_idx])) return -ENOSPC; - rss = mlx5e_rss_init(res->mdev, inner_ft_support, res->drop_rqn, - &res->pkt_merge_param, MLX5E_RSS_INIT_NO_TIRS, init_nch, - res->max_nch); + init_params = (struct mlx5e_rss_init_params) { + .type = MLX5E_RSS_INIT_NO_TIRS, + .pkt_merge_param = &res->pkt_merge_param, + .nch = init_nch, + .max_nch = res->max_nch, + }; + + rss_params = (struct mlx5e_rss_params) { + .inner_ft_support = inner_ft_support, + .drop_rqn = res->drop_rqn, + }; + + rss = mlx5e_rss_init(res->mdev, &rss_params, &init_params); if (IS_ERR(rss)) return PTR_ERR(rss); - mlx5e_rss_set_indir_uniform(rss, init_nch); + mlx5e_rss_set_indir_uniform(rss, init_params.nch); if (res->rss_active) { u32 *vhca_ids = get_vhca_ids(res, 0); @@ -438,7 +463,7 @@ static void mlx5e_rx_res_ptp_destroy(struct mlx5e_rx_res *res) struct mlx5e_rx_res * mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features features, unsigned int max_nch, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, + const struct mlx5e_packet_merge_param *pkt_merge_param, unsigned int init_nch) { bool multi_vhca = features & MLX5E_RX_RES_FEATURE_MULTI_VHCA; @@ -454,7 +479,7 @@ mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features featu res->max_nch = max_nch; res->drop_rqn = drop_rqn; - res->pkt_merge_param = *init_pkt_merge_param; + res->pkt_merge_param = *pkt_merge_param; init_rwsem(&res->pkt_merge_param_sem); err = mlx5e_rx_res_rss_init_def(res, init_nch); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index 1d049e2aa264..65a857c215e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -27,7 +27,7 @@ enum mlx5e_rx_res_features { struct mlx5e_rx_res * mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features features, unsigned int max_nch, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, + const struct mlx5e_packet_merge_param *pkt_merge_param, unsigned int init_nch); void mlx5e_rx_res_destroy(struct mlx5e_rx_res *res); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_hmfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_hmfs.c index 01d522b02947..d3db6146fcad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_hmfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_hmfs.c @@ -136,8 +136,8 @@ mlx5_ct_fs_hmfs_matcher_get(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, hws_bwc_matcher = mlx5_ct_fs_hmfs_matcher_create(fs, tbl, spec, ipv4, tcp, gre); if (IS_ERR(hws_bwc_matcher)) { netdev_warn(fs->netdev, - "ct_fs_hmfs: failed to create bwc matcher (nat %d, ipv4 %d, tcp %d, gre %d), err: %ld\n", - nat, ipv4, tcp, gre, PTR_ERR(hws_bwc_matcher)); + "ct_fs_hmfs: failed to create bwc matcher (nat %d, ipv4 %d, tcp %d, gre %d), err: %pe\n", + nat, ipv4, tcp, gre, hws_bwc_matcher); hmfs_matcher = ERR_CAST(hws_bwc_matcher); goto out_unlock; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c index 0c97c5899904..4d6924b644c9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c @@ -148,8 +148,8 @@ mlx5_ct_fs_smfs_matcher_get(struct mlx5_ct_fs *fs, bool nat, bool ipv4, bool tcp dr_matcher = mlx5_ct_fs_smfs_matcher_create(fs, tbl, ipv4, tcp, gre, prio); if (IS_ERR(dr_matcher)) { netdev_warn(fs->netdev, - "ct_fs_smfs: failed to create matcher (nat %d, ipv4 %d, tcp %d, gre %d), err: %ld\n", - nat, ipv4, tcp, gre, PTR_ERR(dr_matcher)); + "ct_fs_smfs: failed to create matcher (nat %d, ipv4 %d, tcp %d, gre %d), err: %pe\n", + nat, ipv4, tcp, gre, dr_matcher); smfs_matcher = ERR_CAST(dr_matcher); goto out_unlock; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c index 8afcec0c5d3c..896f718483c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c @@ -93,8 +93,8 @@ mlx5e_int_port_create_rx_rule(struct mlx5_eswitch *esw, flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, &flow_act, dest, 1); if (IS_ERR(flow_rule)) - mlx5_core_warn(esw->dev, "ft offloads: Failed to add internal vport rx rule err %ld\n", - PTR_ERR(flow_rule)); + mlx5_core_warn(esw->dev, "ft offloads: Failed to add internal vport rx rule err %pe\n", + flow_rule); kvfree(spec); @@ -322,8 +322,8 @@ mlx5e_tc_int_port_init(struct mlx5e_priv *priv) sizeof(u32) * 2, (1 << ESW_VPORT_BITS) - 1, true); if (IS_ERR(int_port_priv->metadata_mapping)) { - mlx5_core_warn(priv->mdev, "Can't allocate metadata mapping of int port offload, err=%ld\n", - PTR_ERR(int_port_priv->metadata_mapping)); + mlx5_core_warn(priv->mdev, "Can't allocate metadata mapping of int port offload, err=%pe\n", + int_port_priv->metadata_mapping); goto err_mapping; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c index 2162d776fe35..a14f216048cd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c @@ -1,7 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* Copyright (c) 2018 Mellanox Technologies. */ -#include <net/inet_ecn.h> +#include <net/flow.h> +#include <net/inet_dscp.h> #include <net/vxlan.h> #include <net/gre.h> #include <net/geneve.h> @@ -233,7 +234,7 @@ int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv, int err; /* add the IP fields */ - attr.fl.fl4.flowi4_tos = tun_key->tos & ~INET_ECN_MASK; + attr.fl.fl4.flowi4_dscp = inet_dsfield_to_dscp(tun_key->tos); attr.fl.fl4.daddr = tun_key->u.ipv4.dst; attr.fl.fl4.saddr = tun_key->u.ipv4.src; attr.ttl = tun_key->ttl; @@ -349,7 +350,7 @@ int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv, int err; /* add the IP fields */ - attr.fl.fl4.flowi4_tos = tun_key->tos & ~INET_ECN_MASK; + attr.fl.fl4.flowi4_dscp = inet_dsfield_to_dscp(tun_key->tos); attr.fl.fl4.daddr = tun_key->u.ipv4.dst; attr.fl.fl4.saddr = tun_key->u.ipv4.src; attr.ttl = tun_key->ttl; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c index a0fc76a1bc08..0735d10f2bac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c @@ -172,8 +172,8 @@ void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv, &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(e->pkt_reformat)) { - mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %lu\n", - PTR_ERR(e->pkt_reformat)); + mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %pe\n", + e->pkt_reformat); return; } e->flags |= MLX5_ENCAP_ENTRY_VALID; @@ -1845,8 +1845,8 @@ static int mlx5e_tc_tun_fib_event(struct notifier_block *nb, unsigned long event queue_work(priv->wq, &fib_work->work); } else if (IS_ERR(fib_work)) { NL_SET_ERR_MSG_MOD(info->extack, "Failed to init fib work"); - mlx5_core_warn(priv->mdev, "Failed to init fib work, %ld\n", - PTR_ERR(fib_work)); + mlx5_core_warn(priv->mdev, "Failed to init fib work, %pe\n", + fib_work); } break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c index b5c19396e096..996fcdb5a29d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c @@ -76,6 +76,7 @@ static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct mlx5e_trap *t) ccp.ch_stats = t->stats; ccp.napi = &t->napi; ccp.ix = 0; + ccp.uar = mdev->priv.bfreg.up; err = mlx5e_open_cq(priv->mdev, trap_moder, &rq_param->cqp, &ccp, &rq->cq); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index 5dc04bbfc71b..6760bb0336df 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -309,10 +309,7 @@ mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map, static inline void mlx5e_cq_arm(struct mlx5e_cq *cq) { - struct mlx5_core_cq *mcq; - - mcq = &cq->mcq; - mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc); + mlx5_cq_arm(&cq->mcq, MLX5_CQ_DB_REQ_NOT, cq->uar->map, cq->wq.cc); } static inline struct mlx5e_sq_dma * diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index d743e823362a..dbd88eb5c082 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -54,7 +54,7 @@ static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev, struct mlx5e_channel_param *cparam) { mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq); - mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq); + mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq); } static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 33e32584b07f..8bef99e8367e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -42,6 +42,8 @@ #include <en_accel/macsec.h> #include "en.h" #include "en/txrx.h" +#include "en_accel/psp.h" +#include "en_accel/psp_rxtx.h" #if IS_ENABLED(CONFIG_GENEVE) #include <net/geneve.h> @@ -119,6 +121,9 @@ struct mlx5e_accel_tx_state { #ifdef CONFIG_MLX5_EN_IPSEC struct mlx5e_accel_tx_ipsec_state ipsec; #endif +#ifdef CONFIG_MLX5_EN_PSP + struct mlx5e_accel_tx_psp_state psp_st; +#endif }; static inline bool mlx5e_accel_tx_begin(struct net_device *dev, @@ -137,6 +142,13 @@ static inline bool mlx5e_accel_tx_begin(struct net_device *dev, return false; #endif +#ifdef CONFIG_MLX5_EN_PSP + if (mlx5e_psp_is_offload(skb, dev)) { + if (unlikely(!mlx5e_psp_handle_tx_skb(dev, skb, &state->psp_st))) + return false; + } +#endif + #ifdef CONFIG_MLX5_EN_IPSEC if (test_bit(MLX5E_SQ_STATE_IPSEC, &sq->state) && xfrm_offload(skb)) { if (unlikely(!mlx5e_ipsec_handle_tx_skb(dev, skb, &state->ipsec))) @@ -157,8 +169,14 @@ static inline bool mlx5e_accel_tx_begin(struct net_device *dev, } static inline unsigned int mlx5e_accel_tx_ids_len(struct mlx5e_txqsq *sq, + struct sk_buff *skb, struct mlx5e_accel_tx_state *state) { +#ifdef CONFIG_MLX5_EN_PSP + if (mlx5e_psp_is_offload_state(&state->psp_st)) + return mlx5e_psp_tx_ids_len(&state->psp_st); +#endif + #ifdef CONFIG_MLX5_EN_IPSEC if (test_bit(MLX5E_SQ_STATE_IPSEC, &sq->state)) return mlx5e_ipsec_tx_ids_len(&state->ipsec); @@ -172,8 +190,14 @@ static inline unsigned int mlx5e_accel_tx_ids_len(struct mlx5e_txqsq *sq, static inline void mlx5e_accel_tx_eseg(struct mlx5e_priv *priv, struct sk_buff *skb, + struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { +#ifdef CONFIG_MLX5_EN_PSP + if (mlx5e_psp_is_offload_state(&accel->psp_st)) + mlx5e_psp_tx_build_eseg(priv, skb, &accel->psp_st, eseg); +#endif + #ifdef CONFIG_MLX5_EN_IPSEC if (xfrm_offload(skb)) mlx5e_ipsec_tx_build_eseg(priv, skb, eseg); @@ -199,6 +223,11 @@ static inline void mlx5e_accel_tx_finish(struct mlx5e_txqsq *sq, mlx5e_ktls_handle_tx_wqe(&wqe->ctrl, &state->tls); #endif +#ifdef CONFIG_MLX5_EN_PSP + if (mlx5e_psp_is_offload_state(&state->psp_st)) + mlx5e_psp_handle_tx_wqe(wqe, &state->psp_st, inlseg); +#endif + #ifdef CONFIG_MLX5_EN_IPSEC if (test_bit(MLX5E_SQ_STATE_IPSEC, &sq->state) && state->ipsec.xo && state->ipsec.tailen) @@ -208,21 +237,40 @@ static inline void mlx5e_accel_tx_finish(struct mlx5e_txqsq *sq, static inline int mlx5e_accel_init_rx(struct mlx5e_priv *priv) { - return mlx5e_ktls_init_rx(priv); + int err; + + err = mlx5_accel_psp_fs_init_rx_tables(priv); + if (err) + goto out; + + err = mlx5e_ktls_init_rx(priv); + if (err) + mlx5_accel_psp_fs_cleanup_rx_tables(priv); + +out: + return err; } static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) { mlx5e_ktls_cleanup_rx(priv); + mlx5_accel_psp_fs_cleanup_rx_tables(priv); } static inline int mlx5e_accel_init_tx(struct mlx5e_priv *priv) { + int err; + + err = mlx5_accel_psp_fs_init_tx_tables(priv); + if (err) + return err; + return mlx5e_ktls_init_tx(priv); } static inline void mlx5e_accel_cleanup_tx(struct mlx5e_priv *priv) { mlx5e_ktls_cleanup_tx(priv); + mlx5_accel_psp_fs_cleanup_tx_tables(priv); } #endif /* __MLX5E_EN_ACCEL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c index 4f83e3172767..1febdc5b81f9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c @@ -138,7 +138,7 @@ struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs, flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); if (IS_ERR(flow)) - fs_err(fs, "mlx5_add_flow_rules() failed, flow is %ld\n", PTR_ERR(flow)); + fs_err(fs, "mlx5_add_flow_rules() failed, flow is %pe\n", flow); out: kvfree(spec); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 65dc3529283b..6ccfc2af07b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -61,6 +61,7 @@ struct mlx5e_ipsec_rx { struct mlx5_flow_table *pol_miss_ft; struct mlx5_flow_handle *pol_miss_rule; u8 allow_tunnel_mode : 1; + u8 ttc_rules_added : 1; }; /* IPsec RX flow steering */ @@ -585,6 +586,20 @@ out: return err; } +static struct mlx5_flow_destination +ipsec_rx_decrypted_pkt_def_dest(struct mlx5_ttc_table *ttc, u32 family) +{ + struct mlx5_flow_destination dest; + + if (!mlx5_ttc_has_esp_flow_group(ttc)) + return mlx5_ttc_get_default_dest(ttc, family2tt(family)); + + dest.ft = mlx5_get_ttc_flow_table(ttc); + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + + return dest; +} + static void ipsec_rx_update_default_dest(struct mlx5e_ipsec_rx *rx, struct mlx5_flow_destination *old_dest, struct mlx5_flow_destination *new_dest) @@ -598,10 +613,10 @@ static void handle_ipsec_rx_bringup(struct mlx5e_ipsec *ipsec, u32 family) { struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination old_dest, new_dest; - old_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false), - family2tt(family)); + old_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family); mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, ns, &old_dest, family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, MLX5E_NIC_PRIO); @@ -614,12 +629,12 @@ static void handle_ipsec_rx_bringup(struct mlx5e_ipsec *ipsec, u32 family) static void handle_ipsec_rx_cleanup(struct mlx5e_ipsec *ipsec, u32 family) { struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination old_dest, new_dest; old_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); old_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - new_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false), - family2tt(family)); + new_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family); ipsec_rx_update_default_dest(rx, &old_dest, &new_dest); mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, ipsec->mdev); @@ -669,10 +684,13 @@ static void ipsec_mpv_work_handler(struct work_struct *_work) complete(&work->master_priv->ipsec->comp); } -static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, u32 family) +static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, u32 family) { struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + if (rx->ttc_rules_added) + mlx5_ttc_destroy_ipsec_rules(ttc); mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); } @@ -707,7 +725,7 @@ static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, { /* disconnect */ if (rx != ipsec->rx_esw) - ipsec_rx_ft_disconnect(ipsec, family); + ipsec_rx_ft_disconnect(ipsec, rx, family); mlx5_del_flow_rules(rx->sa.rule); mlx5_destroy_flow_group(rx->sa.group); @@ -764,7 +782,7 @@ static int ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, if (rx == ipsec->rx_esw) return mlx5_esw_ipsec_rx_status_pass_dest_get(ipsec, dest); - *dest = mlx5_ttc_get_default_dest(attr->ttc, family2tt(attr->family)); + *dest = ipsec_rx_decrypted_pkt_def_dest(attr->ttc, attr->family); err = mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, attr->ns, dest, attr->family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, attr->prio); @@ -807,10 +825,16 @@ static void ipsec_rx_ft_connect(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx_create_attr *attr) { struct mlx5_flow_destination dest = {}; + struct mlx5_ttc_table *ttc, *inner_ttc; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = rx->ft.sa; - mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest); + if (mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest)) + return; + + ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + inner_ttc = mlx5e_fs_get_ttc(ipsec->fs, true); + rx->ttc_rules_added = !mlx5_ttc_create_ipsec_rules(ttc, inner_ttc); } static int ipsec_rx_chains_create_miss(struct mlx5e_ipsec *ipsec, @@ -1705,8 +1729,8 @@ static int setup_modify_header(struct mlx5e_ipsec *ipsec, int type, u32 val, u8 modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, num_of_actions, action); if (IS_ERR(modify_hdr)) { - mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", - PTR_ERR(modify_hdr)); + mlx5_core_err(mdev, "Failed to allocate modify_header %pe\n", + modify_hdr); return PTR_ERR(modify_hdr); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index 3cc640669247..45b0d19e735c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -40,7 +40,7 @@ #include "en/txrx.h" /* Bit31: IPsec marker, Bit30: reserved, Bit29-24: IPsec syndrome, Bit23-0: IPsec obj id */ -#define MLX5_IPSEC_METADATA_MARKER(metadata) (((metadata) >> 31) & 0x1) +#define MLX5_IPSEC_METADATA_MARKER(metadata) ((((metadata) >> 30) & 0x3) == 0x2) #define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(5, 0)) #define MLX5_IPSEC_METADATA_HANDLE(metadata) ((metadata) & GENMASK(23, 0)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 65ccb33edafb..d7a11ff9bbdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -498,9 +498,9 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) depth += sizeof(struct iphdr); th = (void *)iph + sizeof(struct iphdr); - sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, - iph->saddr, th->source, iph->daddr, - th->dest, netdev->ifindex); + sk = inet_lookup_established(net, iph->saddr, th->source, + iph->daddr, th->dest, + netdev->ifindex); #if IS_ENABLED(CONFIG_IPV6) } else { struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph; @@ -508,8 +508,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) depth += sizeof(struct ipv6hdr); th = (void *)ipv6h + sizeof(struct ipv6hdr); - sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, - &ipv6h->saddr, th->source, + sk = __inet6_lookup_established(net, &ipv6h->saddr, th->source, &ipv6h->daddr, ntohs(th->dest), netdev->ifindex, 0); #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 6ab02f3fc291..528b04d4de41 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -1676,7 +1676,7 @@ void mlx5e_macsec_tx_build_eseg(struct mlx5e_macsec *macsec, if (!fs_id) return; - eseg->flow_table_metadata = cpu_to_be32(MLX5_ETH_WQE_FT_META_MACSEC | fs_id << 2); + eseg->flow_table_metadata = cpu_to_be32(MLX5_MACSEC_TX_METADATA(fs_id)); } void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c new file mode 100644 index 000000000000..b4cb131c5f81 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c @@ -0,0 +1,952 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +#include <linux/mlx5/device.h> +#include <net/psp.h> +#include <linux/psp.h> +#include "mlx5_core.h" +#include "psp.h" +#include "lib/crypto.h" +#include "en_accel/psp.h" +#include "fs_core.h" + +enum accel_fs_psp_type { + ACCEL_FS_PSP4, + ACCEL_FS_PSP6, + ACCEL_FS_PSP_NUM_TYPES, +}; + +enum accel_psp_syndrome { + PSP_OK = 0, + PSP_ICV_FAIL, + PSP_BAD_TRAILER, +}; + +struct mlx5e_psp_tx { + struct mlx5_flow_namespace *ns; + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct mlx5_flow_handle *rule; + struct mutex mutex; /* Protect PSP TX steering */ + u32 refcnt; +}; + +struct mlx5e_psp_rx_err { + struct mlx5_flow_table *ft; + struct mlx5_flow_handle *rule; + struct mlx5_flow_handle *drop_rule; + struct mlx5_modify_hdr *copy_modify_hdr; +}; + +struct mlx5e_accel_fs_psp_prot { + struct mlx5_flow_table *ft; + struct mlx5_flow_group *miss_group; + struct mlx5_flow_handle *miss_rule; + struct mlx5_flow_destination default_dest; + struct mlx5e_psp_rx_err rx_err; + u32 refcnt; + struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ + struct mlx5_flow_handle *def_rule; +}; + +struct mlx5e_accel_fs_psp { + struct mlx5e_accel_fs_psp_prot fs_prot[ACCEL_FS_PSP_NUM_TYPES]; +}; + +struct mlx5e_psp_fs { + struct mlx5_core_dev *mdev; + struct mlx5e_psp_tx *tx_fs; + /* Rx manage */ + struct mlx5e_flow_steering *fs; + struct mlx5e_accel_fs_psp *rx_fs; +}; + +/* PSP RX flow steering */ +static enum mlx5_traffic_types fs_psp2tt(enum accel_fs_psp_type i) +{ + if (i == ACCEL_FS_PSP4) + return MLX5_TT_IPV4_UDP; + + return MLX5_TT_IPV6_UDP; +} + +static void accel_psp_fs_rx_err_del_rules(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_rx_err *rx_err) +{ + if (rx_err->drop_rule) { + mlx5_del_flow_rules(rx_err->drop_rule); + rx_err->drop_rule = NULL; + } + + if (rx_err->rule) { + mlx5_del_flow_rules(rx_err->rule); + rx_err->rule = NULL; + } + + if (rx_err->copy_modify_hdr) { + mlx5_modify_header_dealloc(fs->mdev, rx_err->copy_modify_hdr); + rx_err->copy_modify_hdr = NULL; + } +} + +static void accel_psp_fs_rx_err_destroy_ft(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_rx_err *rx_err) +{ + accel_psp_fs_rx_err_del_rules(fs, rx_err); + + if (rx_err->ft) { + mlx5_destroy_flow_table(rx_err->ft); + rx_err->ft = NULL; + } +} + +static void accel_psp_setup_syndrome_match(struct mlx5_flow_spec *spec, + enum accel_psp_syndrome syndrome) +{ + void *misc_params_2; + + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + misc_params_2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); + MLX5_SET_TO_ONES(fte_match_set_misc2, misc_params_2, psp_syndrome); + misc_params_2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); + MLX5_SET(fte_match_set_misc2, misc_params_2, psp_syndrome, syndrome); +} + +static int accel_psp_fs_rx_err_add_rule(struct mlx5e_psp_fs *fs, + struct mlx5e_accel_fs_psp_prot *fs_prot, + struct mlx5e_psp_rx_err *rx_err) +{ + u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + struct mlx5_core_dev *mdev = fs->mdev; + struct mlx5_flow_act flow_act = {}; + struct mlx5_modify_hdr *modify_hdr; + struct mlx5_flow_handle *fte; + struct mlx5_flow_spec *spec; + int err = 0; + + spec = kzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + /* Action to copy 7 bit psp_syndrome to regB[23:29] */ + MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); + MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_PSP_SYNDROME); + MLX5_SET(copy_action_in, action, src_offset, 0); + MLX5_SET(copy_action_in, action, length, 7); + MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); + MLX5_SET(copy_action_in, action, dst_offset, 23); + + modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, + 1, action); + if (IS_ERR(modify_hdr)) { + err = PTR_ERR(modify_hdr); + mlx5_core_err(mdev, + "fail to alloc psp copy modify_header_id err=%d\n", err); + goto out_spec; + } + + accel_psp_setup_syndrome_match(spec, PSP_OK); + /* create fte */ + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + flow_act.modify_hdr = modify_hdr; + fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, + &fs_prot->default_dest, 1); + if (IS_ERR(fte)) { + err = PTR_ERR(fte); + mlx5_core_err(mdev, "fail to add psp rx err copy rule err=%d\n", err); + goto out; + } + rx_err->rule = fte; + + /* add default drop rule */ + memset(spec, 0, sizeof(*spec)); + memset(&flow_act, 0, sizeof(flow_act)); + /* create fte */ + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; + fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, NULL, 0); + if (IS_ERR(fte)) { + err = PTR_ERR(fte); + mlx5_core_err(mdev, "fail to add psp rx err drop rule err=%d\n", err); + goto out_drop_rule; + } + rx_err->drop_rule = fte; + rx_err->copy_modify_hdr = modify_hdr; + + goto out_spec; + +out_drop_rule: + mlx5_del_flow_rules(rx_err->rule); + rx_err->rule = NULL; +out: + mlx5_modify_header_dealloc(mdev, modify_hdr); +out_spec: + kfree(spec); + return err; +} + +static int accel_psp_fs_rx_err_create_ft(struct mlx5e_psp_fs *fs, + struct mlx5e_accel_fs_psp_prot *fs_prot, + struct mlx5e_psp_rx_err *rx_err) +{ + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs->fs, false); + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_table *ft; + int err; + + ft_attr.max_fte = 2; + ft_attr.autogroup.max_num_groups = 2; + ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; // MLX5E_ACCEL_FS_TCP_FT_LEVEL + ft_attr.prio = MLX5E_NIC_PRIO; + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + mlx5_core_err(fs->mdev, "fail to create psp rx inline ft err=%d\n", err); + return err; + } + + rx_err->ft = ft; + err = accel_psp_fs_rx_err_add_rule(fs, fs_prot, rx_err); + if (err) + goto out_err; + + return 0; + +out_err: + mlx5_destroy_flow_table(ft); + rx_err->ft = NULL; + return err; +} + +static void accel_psp_fs_rx_fs_destroy(struct mlx5e_accel_fs_psp_prot *fs_prot) +{ + if (fs_prot->def_rule) { + mlx5_del_flow_rules(fs_prot->def_rule); + fs_prot->def_rule = NULL; + } + + if (fs_prot->miss_rule) { + mlx5_del_flow_rules(fs_prot->miss_rule); + fs_prot->miss_rule = NULL; + } + + if (fs_prot->miss_group) { + mlx5_destroy_flow_group(fs_prot->miss_group); + fs_prot->miss_group = NULL; + } + + if (fs_prot->ft) { + mlx5_destroy_flow_table(fs_prot->ft); + fs_prot->ft = NULL; + } +} + +static void setup_fte_udp_psp(struct mlx5_flow_spec *spec, u16 udp_port) +{ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, udp_dport, 0xffff); + MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, udp_dport, udp_port); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); + MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, IPPROTO_UDP); +} + +static int accel_psp_fs_rx_create_ft(struct mlx5e_psp_fs *fs, + struct mlx5e_accel_fs_psp_prot *fs_prot) +{ + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs->fs, false); + u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_modify_hdr *modify_hdr = NULL; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_destination dest = {}; + struct mlx5_core_dev *mdev = fs->mdev; + struct mlx5_flow_group *miss_group; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5_flow_table *ft; + u32 *flow_group_in; + int err = 0; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!flow_group_in || !spec) { + err = -ENOMEM; + goto out; + } + + /* Create FT */ + ft_attr.max_fte = 2; + ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = 1; + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + mlx5_core_err(mdev, "fail to create psp rx ft err=%d\n", err); + goto out_err; + } + fs_prot->ft = ft; + + /* Create miss_group */ + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); + miss_group = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(miss_group)) { + err = PTR_ERR(miss_group); + mlx5_core_err(mdev, "fail to create psp rx miss_group err=%d\n", err); + goto out_err; + } + fs_prot->miss_group = miss_group; + + /* Create miss rule */ + rule = mlx5_add_flow_rules(ft, spec, &flow_act, &fs_prot->default_dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "fail to create psp rx miss_rule err=%d\n", err); + goto out_err; + } + fs_prot->miss_rule = rule; + + /* Add default Rx psp rule */ + setup_fte_udp_psp(spec, PSP_DEFAULT_UDP_PORT); + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_PSP; + /* Set bit[31, 30] PSP marker */ + /* Set bit[29-23] psp_syndrome is set in error FT */ +#define MLX5E_PSP_MARKER_BIT (BIT(30) | BIT(31)) + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); + MLX5_SET(set_action_in, action, data, MLX5E_PSP_MARKER_BIT); + MLX5_SET(set_action_in, action, offset, 0); + MLX5_SET(set_action_in, action, length, 32); + + modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 1, action); + if (IS_ERR(modify_hdr)) { + err = PTR_ERR(modify_hdr); + mlx5_core_err(mdev, "fail to alloc psp set modify_header_id err=%d\n", err); + modify_hdr = NULL; + goto out_err; + } + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | + MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + flow_act.modify_hdr = modify_hdr; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = fs_prot->rx_err.ft; + rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, + "fail to add psp rule Rx decryption, err=%d, flow_act.action = %#04X\n", + err, flow_act.action); + goto out_err; + } + + fs_prot->def_rule = rule; + goto out; + +out_err: + accel_psp_fs_rx_fs_destroy(fs_prot); +out: + kvfree(flow_group_in); + kvfree(spec); + return err; +} + +static int accel_psp_fs_rx_destroy(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) +{ + struct mlx5e_accel_fs_psp_prot *fs_prot; + struct mlx5e_accel_fs_psp *accel_psp; + + accel_psp = fs->rx_fs; + + /* The netdev unreg already happened, so all offloaded rule are already removed */ + fs_prot = &accel_psp->fs_prot[type]; + + accel_psp_fs_rx_fs_destroy(fs_prot); + + accel_psp_fs_rx_err_destroy_ft(fs, &fs_prot->rx_err); + + return 0; +} + +static int accel_psp_fs_rx_create(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) +{ + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false); + struct mlx5e_accel_fs_psp_prot *fs_prot; + struct mlx5e_accel_fs_psp *accel_psp; + int err; + + accel_psp = fs->rx_fs; + fs_prot = &accel_psp->fs_prot[type]; + + fs_prot->default_dest = mlx5_ttc_get_default_dest(ttc, fs_psp2tt(type)); + + err = accel_psp_fs_rx_err_create_ft(fs, fs_prot, &fs_prot->rx_err); + if (err) + return err; + + err = accel_psp_fs_rx_create_ft(fs, fs_prot); + if (err) + accel_psp_fs_rx_err_destroy_ft(fs, &fs_prot->rx_err); + + return err; +} + +static int accel_psp_fs_rx_ft_get(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) +{ + struct mlx5e_accel_fs_psp_prot *fs_prot; + struct mlx5_flow_destination dest = {}; + struct mlx5e_accel_fs_psp *accel_psp; + struct mlx5_ttc_table *ttc; + int err = 0; + + if (!fs || !fs->rx_fs) + return -EINVAL; + + ttc = mlx5e_fs_get_ttc(fs->fs, false); + accel_psp = fs->rx_fs; + fs_prot = &accel_psp->fs_prot[type]; + mutex_lock(&fs_prot->prot_mutex); + if (fs_prot->refcnt++) + goto out; + + /* create FT */ + err = accel_psp_fs_rx_create(fs, type); + if (err) { + fs_prot->refcnt--; + goto out; + } + + /* connect */ + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = fs_prot->ft; + mlx5_ttc_fwd_dest(ttc, fs_psp2tt(type), &dest); + +out: + mutex_unlock(&fs_prot->prot_mutex); + return err; +} + +static void accel_psp_fs_rx_ft_put(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) +{ + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false); + struct mlx5e_accel_fs_psp_prot *fs_prot; + struct mlx5e_accel_fs_psp *accel_psp; + + accel_psp = fs->rx_fs; + fs_prot = &accel_psp->fs_prot[type]; + mutex_lock(&fs_prot->prot_mutex); + if (--fs_prot->refcnt) + goto out; + + /* disconnect */ + mlx5_ttc_fwd_default_dest(ttc, fs_psp2tt(type)); + + /* remove FT */ + accel_psp_fs_rx_destroy(fs, type); + +out: + mutex_unlock(&fs_prot->prot_mutex); +} + +static void accel_psp_fs_cleanup_rx(struct mlx5e_psp_fs *fs) +{ + struct mlx5e_accel_fs_psp_prot *fs_prot; + struct mlx5e_accel_fs_psp *accel_psp; + enum accel_fs_psp_type i; + + if (!fs->rx_fs) + return; + + accel_psp = fs->rx_fs; + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { + fs_prot = &accel_psp->fs_prot[i]; + mutex_destroy(&fs_prot->prot_mutex); + WARN_ON(fs_prot->refcnt); + } + kfree(fs->rx_fs); + fs->rx_fs = NULL; +} + +static int accel_psp_fs_init_rx(struct mlx5e_psp_fs *fs) +{ + struct mlx5e_accel_fs_psp_prot *fs_prot; + struct mlx5e_accel_fs_psp *accel_psp; + enum accel_fs_psp_type i; + + accel_psp = kzalloc(sizeof(*accel_psp), GFP_KERNEL); + if (!accel_psp) + return -ENOMEM; + + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { + fs_prot = &accel_psp->fs_prot[i]; + mutex_init(&fs_prot->prot_mutex); + } + + fs->rx_fs = accel_psp; + + return 0; +} + +void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv) +{ + int i; + + if (!priv->psp) + return; + + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) + accel_psp_fs_rx_ft_put(priv->psp->fs, i); +} + +int mlx5_accel_psp_fs_init_rx_tables(struct mlx5e_priv *priv) +{ + struct mlx5e_psp_fs *fs; + int err, i; + + if (!priv->psp) + return 0; + + fs = priv->psp->fs; + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { + err = accel_psp_fs_rx_ft_get(fs, i); + if (err) + goto out_err; + } + + return 0; + +out_err: + i--; + while (i >= 0) { + accel_psp_fs_rx_ft_put(fs, i); + --i; + } + + return err; +} + +static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_core_dev *mdev = fs->mdev; + struct mlx5_flow_act flow_act = {}; + u32 *in, *mc, *outer_headers_c; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_psp_tx *tx_fs; + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + int err = 0; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + in = kvzalloc(inlen, GFP_KERNEL); + if (!spec || !in) { + err = -ENOMEM; + goto out; + } + + ft_attr.max_fte = 1; +#define MLX5E_PSP_PRIO 0 + ft_attr.prio = MLX5E_PSP_PRIO; +#define MLX5E_PSP_LEVEL 0 + ft_attr.level = MLX5E_PSP_LEVEL; + ft_attr.autogroup.max_num_groups = 1; + + tx_fs = fs->tx_fs; + ft = mlx5_create_flow_table(tx_fs->ns, &ft_attr); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + mlx5_core_err(mdev, "PSP: fail to add psp tx flow table, err = %d\n", err); + goto out; + } + + mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + fg = mlx5_create_flow_group(ft, in); + if (IS_ERR(fg)) { + err = PTR_ERR(fg); + mlx5_core_err(mdev, "PSP: fail to add psp tx flow group, err = %d\n", err); + goto err_create_fg; + } + + setup_fte_udp_psp(spec, PSP_DEFAULT_UDP_PORT); + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_PSP; + flow_act.flags |= FLOW_ACT_NO_APPEND; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; + rule = mlx5_add_flow_rules(ft, spec, &flow_act, NULL, 0); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "PSP: fail to add psp tx flow rule, err = %d\n", err); + goto err_add_flow_rule; + } + + tx_fs->ft = ft; + tx_fs->fg = fg; + tx_fs->rule = rule; + goto out; + +err_add_flow_rule: + mlx5_destroy_flow_group(fg); +err_create_fg: + mlx5_destroy_flow_table(ft); +out: + kvfree(in); + kvfree(spec); + return err; +} + +static void accel_psp_fs_tx_destroy(struct mlx5e_psp_tx *tx_fs) +{ + if (!tx_fs->ft) + return; + + mlx5_del_flow_rules(tx_fs->rule); + mlx5_destroy_flow_group(tx_fs->fg); + mlx5_destroy_flow_table(tx_fs->ft); +} + +static int accel_psp_fs_tx_ft_get(struct mlx5e_psp_fs *fs) +{ + struct mlx5e_psp_tx *tx_fs = fs->tx_fs; + int err = 0; + + mutex_lock(&tx_fs->mutex); + if (tx_fs->refcnt++) + goto out; + + err = accel_psp_fs_tx_create_ft_table(fs); + if (err) + tx_fs->refcnt--; +out: + mutex_unlock(&tx_fs->mutex); + return err; +} + +static void accel_psp_fs_tx_ft_put(struct mlx5e_psp_fs *fs) +{ + struct mlx5e_psp_tx *tx_fs = fs->tx_fs; + + mutex_lock(&tx_fs->mutex); + if (--tx_fs->refcnt) + goto out; + + accel_psp_fs_tx_destroy(tx_fs); +out: + mutex_unlock(&tx_fs->mutex); +} + +static void accel_psp_fs_cleanup_tx(struct mlx5e_psp_fs *fs) +{ + struct mlx5e_psp_tx *tx_fs = fs->tx_fs; + + if (!tx_fs) + return; + + mutex_destroy(&tx_fs->mutex); + WARN_ON(tx_fs->refcnt); + kfree(tx_fs); + fs->tx_fs = NULL; +} + +static int accel_psp_fs_init_tx(struct mlx5e_psp_fs *fs) +{ + struct mlx5_flow_namespace *ns; + struct mlx5e_psp_tx *tx_fs; + + ns = mlx5_get_flow_namespace(fs->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); + if (!ns) + return -EOPNOTSUPP; + + tx_fs = kzalloc(sizeof(*tx_fs), GFP_KERNEL); + if (!tx_fs) + return -ENOMEM; + + mutex_init(&tx_fs->mutex); + tx_fs->ns = ns; + fs->tx_fs = tx_fs; + return 0; +} + +void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv) +{ + if (!priv->psp) + return; + + accel_psp_fs_tx_ft_put(priv->psp->fs); +} + +int mlx5_accel_psp_fs_init_tx_tables(struct mlx5e_priv *priv) +{ + if (!priv->psp) + return 0; + + return accel_psp_fs_tx_ft_get(priv->psp->fs); +} + +static void mlx5e_accel_psp_fs_cleanup(struct mlx5e_psp_fs *fs) +{ + accel_psp_fs_cleanup_rx(fs); + accel_psp_fs_cleanup_tx(fs); + kfree(fs); +} + +static struct mlx5e_psp_fs *mlx5e_accel_psp_fs_init(struct mlx5e_priv *priv) +{ + struct mlx5e_psp_fs *fs; + int err = 0; + + fs = kzalloc(sizeof(*fs), GFP_KERNEL); + if (!fs) + return ERR_PTR(-ENOMEM); + + fs->mdev = priv->mdev; + err = accel_psp_fs_init_tx(fs); + if (err) + goto err_tx; + + fs->fs = priv->fs; + err = accel_psp_fs_init_rx(fs); + if (err) + goto err_rx; + + return fs; + +err_rx: + accel_psp_fs_cleanup_tx(fs); +err_tx: + kfree(fs); + return ERR_PTR(err); +} + +static int +mlx5e_psp_set_config(struct psp_dev *psd, struct psp_dev_config *conf, + struct netlink_ext_ack *extack) +{ + return 0; /* TODO: this should actually do things to the device */ +} + +static int +mlx5e_psp_generate_key_spi(struct mlx5_core_dev *mdev, + enum mlx5_psp_gen_spi_in_key_size keysz, + unsigned int keysz_bytes, + struct psp_key_parsed *key) +{ + u32 out[MLX5_ST_SZ_DW(psp_gen_spi_out) + MLX5_ST_SZ_DW(key_spi)] = {}; + u32 in[MLX5_ST_SZ_DW(psp_gen_spi_in)] = {}; + void *outkey; + int err; + + WARN_ON_ONCE(keysz_bytes > PSP_MAX_KEY); + + MLX5_SET(psp_gen_spi_in, in, opcode, MLX5_CMD_OP_PSP_GEN_SPI); + MLX5_SET(psp_gen_spi_in, in, key_size, keysz); + MLX5_SET(psp_gen_spi_in, in, num_of_spi, 1); + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + outkey = MLX5_ADDR_OF(psp_gen_spi_out, out, key_spi); + key->spi = cpu_to_be32(MLX5_GET(key_spi, outkey, spi)); + memcpy(key->key, MLX5_ADDR_OF(key_spi, outkey, key) + 32 - keysz_bytes, + keysz_bytes); + + return 0; +} + +static int +mlx5e_psp_rx_spi_alloc(struct psp_dev *psd, u32 version, + struct psp_key_parsed *assoc, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + enum mlx5_psp_gen_spi_in_key_size keysz; + u8 keysz_bytes; + + switch (version) { + case PSP_VERSION_HDR0_AES_GCM_128: + keysz = MLX5_PSP_GEN_SPI_IN_KEY_SIZE_128; + keysz_bytes = 16; + break; + case PSP_VERSION_HDR0_AES_GCM_256: + keysz = MLX5_PSP_GEN_SPI_IN_KEY_SIZE_256; + keysz_bytes = 32; + break; + default: + return -EINVAL; + } + + return mlx5e_psp_generate_key_spi(priv->mdev, keysz, keysz_bytes, assoc); +} + +struct psp_key { + u32 id; +}; + +static int mlx5e_psp_assoc_add(struct psp_dev *psd, struct psp_assoc *pas, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + struct mlx5_core_dev *mdev = priv->mdev; + struct psp_key_parsed *tx = &pas->tx; + struct mlx5e_psp *psp = priv->psp; + struct psp_key *nkey; + int err; + + mdev = priv->mdev; + nkey = (struct psp_key *)pas->drv_data; + + err = mlx5_create_encryption_key(mdev, tx->key, + psp_key_size(pas->version), + MLX5_ACCEL_OBJ_PSP_KEY, + &nkey->id); + if (err) { + mlx5_core_err(mdev, "Failed to create encryption key (err = %d)\n", err); + return err; + } + + atomic_inc(&psp->tx_key_cnt); + return 0; +} + +static void mlx5e_psp_assoc_del(struct psp_dev *psd, struct psp_assoc *pas) +{ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + struct mlx5e_psp *psp = priv->psp; + struct psp_key *nkey; + + nkey = (struct psp_key *)pas->drv_data; + mlx5_destroy_encryption_key(priv->mdev, nkey->id); + atomic_dec(&psp->tx_key_cnt); +} + +static int mlx5e_psp_rotate_key(struct mlx5_core_dev *mdev) +{ + u32 in[MLX5_ST_SZ_DW(psp_rotate_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(psp_rotate_key_out)]; + + MLX5_SET(psp_rotate_key_in, in, opcode, + MLX5_CMD_OP_PSP_ROTATE_KEY); + + return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static int +mlx5e_psp_key_rotate(struct psp_dev *psd, struct netlink_ext_ack *exack) +{ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + + /* no support for protecting against external rotations */ + psd->generation = 0; + + return mlx5e_psp_rotate_key(priv->mdev); +} + +static struct psp_dev_ops mlx5_psp_ops = { + .set_config = mlx5e_psp_set_config, + .rx_spi_alloc = mlx5e_psp_rx_spi_alloc, + .tx_key_add = mlx5e_psp_assoc_add, + .tx_key_del = mlx5e_psp_assoc_del, + .key_rotate = mlx5e_psp_key_rotate, +}; + +void mlx5e_psp_unregister(struct mlx5e_priv *priv) +{ + if (!priv->psp || !priv->psp->psp) + return; + + psp_dev_unregister(priv->psp->psp); +} + +void mlx5e_psp_register(struct mlx5e_priv *priv) +{ + /* FW Caps missing */ + if (!priv->psp) + return; + + priv->psp->caps.assoc_drv_spc = sizeof(u32); + priv->psp->caps.versions = 1 << PSP_VERSION_HDR0_AES_GCM_128; + if (MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_encrypt) && + MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_decrypt)) + priv->psp->caps.versions |= 1 << PSP_VERSION_HDR0_AES_GCM_256; + + priv->psp->psp = psp_dev_create(priv->netdev, &mlx5_psp_ops, + &priv->psp->caps, NULL); + if (IS_ERR(priv->psp->psp)) + mlx5_core_err(priv->mdev, "PSP failed to register due to %pe\n", + priv->psp->psp); +} + +int mlx5e_psp_init(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_psp_fs *fs; + struct mlx5e_psp *psp; + int err; + + if (!mlx5_is_psp_device(mdev)) { + mlx5_core_dbg(mdev, "PSP offload not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp)) { + mlx5_core_dbg(mdev, "SWP not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp_csum)) { + mlx5_core_dbg(mdev, "SWP checksum not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp_csum_l4_partial)) { + mlx5_core_dbg(mdev, "SWP L4 partial checksum not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp_lso)) { + mlx5_core_dbg(mdev, "PSP LSO not supported\n"); + return -EOPNOTSUPP; + } + + psp = kzalloc(sizeof(*psp), GFP_KERNEL); + if (!psp) + return -ENOMEM; + + priv->psp = psp; + fs = mlx5e_accel_psp_fs_init(priv); + if (IS_ERR(fs)) { + err = PTR_ERR(fs); + goto out_err; + } + + psp->fs = fs; + + mlx5_core_dbg(priv->mdev, "PSP attached to netdevice\n"); + return 0; + +out_err: + priv->psp = NULL; + kfree(psp); + return err; +} + +void mlx5e_psp_cleanup(struct mlx5e_priv *priv) +{ + struct mlx5e_psp *psp = priv->psp; + + if (!psp) + return; + + WARN_ON(atomic_read(&psp->tx_key_cnt)); + mlx5e_accel_psp_fs_cleanup(psp->fs); + priv->psp = NULL; + kfree(psp); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h new file mode 100644 index 000000000000..42bb671fb2cb --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5E_ACCEL_PSP_H__ +#define __MLX5E_ACCEL_PSP_H__ +#if IS_ENABLED(CONFIG_MLX5_EN_PSP) +#include <net/psp/types.h> +#include "en.h" + +struct mlx5e_psp { + struct psp_dev *psp; + struct psp_dev_caps caps; + struct mlx5e_psp_fs *fs; + atomic_t tx_key_cnt; +}; + +static inline bool mlx5_is_psp_device(struct mlx5_core_dev *mdev) +{ + if (!MLX5_CAP_GEN(mdev, psp)) + return false; + + if (!MLX5_CAP_PSP(mdev, psp_crypto_offload) || + !MLX5_CAP_PSP(mdev, psp_crypto_esp_aes_gcm_128_encrypt) || + !MLX5_CAP_PSP(mdev, psp_crypto_esp_aes_gcm_128_decrypt)) + return false; + + return true; +} + +int mlx5_accel_psp_fs_init_rx_tables(struct mlx5e_priv *priv); +void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv); +int mlx5_accel_psp_fs_init_tx_tables(struct mlx5e_priv *priv); +void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv); +void mlx5e_psp_register(struct mlx5e_priv *priv); +void mlx5e_psp_unregister(struct mlx5e_priv *priv); +int mlx5e_psp_init(struct mlx5e_priv *priv); +void mlx5e_psp_cleanup(struct mlx5e_priv *priv); +#else +static inline int mlx5_accel_psp_fs_init_rx_tables(struct mlx5e_priv *priv) +{ + return 0; +} + +static inline void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv) { } +static inline int mlx5_accel_psp_fs_init_tx_tables(struct mlx5e_priv *priv) +{ + return 0; +} + +static inline void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv) { } +static inline bool mlx5_is_psp_device(struct mlx5_core_dev *mdev) +{ + return false; +} + +static inline void mlx5e_psp_register(struct mlx5e_priv *priv) { } +static inline void mlx5e_psp_unregister(struct mlx5e_priv *priv) { } +static inline int mlx5e_psp_init(struct mlx5e_priv *priv) { return 0; } +static inline void mlx5e_psp_cleanup(struct mlx5e_priv *priv) { } +#endif /* CONFIG_MLX5_EN_PSP */ +#endif /* __MLX5E_ACCEL_PSP_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c new file mode 100644 index 000000000000..828bff1137af --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <linux/skbuff.h> +#include <linux/ip.h> +#include <linux/udp.h> +#include <net/protocol.h> +#include <net/udp.h> +#include <net/ip6_checksum.h> +#include <net/psp/types.h> + +#include "en.h" +#include "psp.h" +#include "en_accel/psp_rxtx.h" +#include "en_accel/psp.h" + +enum { + MLX5E_PSP_OFFLOAD_RX_SYNDROME_DECRYPTED, + MLX5E_PSP_OFFLOAD_RX_SYNDROME_AUTH_FAILED, + MLX5E_PSP_OFFLOAD_RX_SYNDROME_BAD_TRAILER, +}; + +static void mlx5e_psp_set_swp(struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_eth_seg *eseg) +{ + /* Tunnel Mode: + * SWP: OutL3 InL3 InL4 + * Pkt: MAC IP ESP IP L4 + * + * Transport Mode: + * SWP: OutL3 OutL4 + * Pkt: MAC IP ESP L4 + * + * Tunnel(VXLAN TCP/UDP) over Transport Mode + * SWP: OutL3 InL3 InL4 + * Pkt: MAC IP ESP UDP VXLAN IP L4 + */ + u8 inner_ipproto = 0; + struct ethhdr *eth; + + /* Shared settings */ + eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; + if (skb->protocol == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; + + if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) { + inner_ipproto = skb->inner_ipproto; + /* Set SWP additional flags for packet of type IP|UDP|PSP|[ TCP | UDP ] */ + switch (inner_ipproto) { + case IPPROTO_UDP: + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; + fallthrough; + case IPPROTO_TCP: + eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; + break; + default: + break; + } + } else { + /* IP in IP tunneling like vxlan*/ + if (skb->inner_protocol_type != ENCAP_TYPE_ETHER) + return; + + eth = (struct ethhdr *)skb_inner_mac_header(skb); + switch (ntohs(eth->h_proto)) { + case ETH_P_IP: + inner_ipproto = ((struct iphdr *)((char *)skb->data + + skb_inner_network_offset(skb)))->protocol; + break; + case ETH_P_IPV6: + inner_ipproto = ((struct ipv6hdr *)((char *)skb->data + + skb_inner_network_offset(skb)))->nexthdr; + break; + default: + break; + } + + /* Tunnel(VXLAN TCP/UDP) over Transport Mode PSP i.e. PSP payload is vxlan tunnel */ + switch (inner_ipproto) { + case IPPROTO_UDP: + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; + fallthrough; + case IPPROTO_TCP: + eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; + eseg->swp_inner_l4_offset = + (skb->csum_start + skb->head - skb->data) / 2; + if (skb->protocol == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; + break; + default: + break; + } + + psp_st->inner_ipproto = inner_ipproto; + } +} + +static bool mlx5e_psp_set_state(struct mlx5e_priv *priv, + struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st) +{ + struct psp_assoc *pas; + bool ret = false; + + rcu_read_lock(); + pas = psp_skb_get_assoc_rcu(skb); + if (!pas) + goto out; + + ret = true; + psp_st->tailen = PSP_TRL_SIZE; + psp_st->spi = pas->tx.spi; + psp_st->ver = pas->version; + psp_st->keyid = *(u32 *)pas->drv_data; + +out: + rcu_read_unlock(); + return ret; +} + +bool mlx5e_psp_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, + struct mlx5_cqe64 *cqe) +{ + u32 psp_meta_data = be32_to_cpu(cqe->ft_metadata); + struct mlx5e_priv *priv = netdev_priv(netdev); + u16 dev_id = priv->psp->psp->id; + bool strip_icv = true; + u8 generation = 0; + + /* TBD: report errors as SW counters to ethtool, any further handling ? */ + if (MLX5_PSP_METADATA_SYNDROME(psp_meta_data) != MLX5E_PSP_OFFLOAD_RX_SYNDROME_DECRYPTED) + goto drop; + + if (psp_dev_rcv(skb, dev_id, generation, strip_icv)) + goto drop; + + skb->decrypted = 1; + return false; + +drop: + kfree_skb(skb); + return true; +} + +void mlx5e_psp_tx_build_eseg(struct mlx5e_priv *priv, struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_eth_seg *eseg) +{ + if (!mlx5_is_psp_device(priv->mdev)) + return; + + if (unlikely(skb->protocol != htons(ETH_P_IP) && + skb->protocol != htons(ETH_P_IPV6))) + return; + + mlx5e_psp_set_swp(skb, psp_st, eseg); + /* Special WA for PSP LSO in ConnectX7 */ + eseg->swp_outer_l3_offset = 0; + eseg->swp_inner_l3_offset = 0; + + eseg->flow_table_metadata |= cpu_to_be32(psp_st->keyid); + eseg->trailer |= cpu_to_be32(MLX5_ETH_WQE_INSERT_TRAILER) | + cpu_to_be32(MLX5_ETH_WQE_TRAILER_HDR_OUTER_L4_ASSOC); +} + +void mlx5e_psp_handle_tx_wqe(struct mlx5e_tx_wqe *wqe, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_inline_seg *inlseg) +{ + inlseg->byte_count = cpu_to_be32(psp_st->tailen | MLX5_INLINE_SEG); +} + +bool mlx5e_psp_handle_tx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st) +{ + struct mlx5e_priv *priv = netdev_priv(netdev); + struct net *net = sock_net(skb->sk); + const struct ipv6hdr *ip6; + struct tcphdr *th; + + if (!mlx5e_psp_set_state(priv, skb, psp_st)) + return true; + + /* psp_encap of the packet */ + if (!psp_dev_encapsulate(net, skb, psp_st->spi, psp_st->ver, 0)) { + kfree_skb_reason(skb, SKB_DROP_REASON_PSP_OUTPUT); + return false; + } + if (skb_is_gso(skb)) { + ip6 = ipv6_hdr(skb); + th = inner_tcp_hdr(skb); + + th->check = ~tcp_v6_check(skb_shinfo(skb)->gso_size + inner_tcp_hdrlen(skb), &ip6->saddr, + &ip6->daddr, 0); + } + + return true; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h new file mode 100644 index 000000000000..70289c921bd6 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5E_PSP_RXTX_H__ +#define __MLX5E_PSP_RXTX_H__ + +#include <linux/skbuff.h> +#include <net/xfrm.h> +#include <net/psp.h> +#include "en.h" +#include "en/txrx.h" + +/* Bit30: PSP marker, Bit29-23: PSP syndrome, Bit22-0: PSP obj id */ +#define MLX5_PSP_METADATA_MARKER(metadata) ((((metadata) >> 30) & 0x3) == 0x3) +#define MLX5_PSP_METADATA_SYNDROME(metadata) (((metadata) >> 23) & GENMASK(6, 0)) +#define MLX5_PSP_METADATA_HANDLE(metadata) ((metadata) & GENMASK(22, 0)) + +struct mlx5e_accel_tx_psp_state { + u32 tailen; + u32 keyid; + __be32 spi; + u8 inner_ipproto; + u8 ver; +}; + +#ifdef CONFIG_MLX5_EN_PSP +static inline bool mlx5e_psp_is_offload_state(struct mlx5e_accel_tx_psp_state *psp_state) +{ + return (psp_state->tailen != 0); +} + +static inline bool mlx5e_psp_is_offload(struct sk_buff *skb, struct net_device *netdev) +{ + bool ret; + + rcu_read_lock(); + ret = !!psp_skb_get_assoc_rcu(skb); + rcu_read_unlock(); + return ret; +} + +bool mlx5e_psp_handle_tx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st); + +void mlx5e_psp_tx_build_eseg(struct mlx5e_priv *priv, struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_eth_seg *eseg); + +void mlx5e_psp_handle_tx_wqe(struct mlx5e_tx_wqe *wqe, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_inline_seg *inlseg); + +static inline bool mlx5e_psp_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_eth_seg *eseg) +{ + u8 inner_ipproto; + + if (!mlx5e_psp_is_offload_state(psp_st)) + return false; + + inner_ipproto = psp_st->inner_ipproto; + eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; + if (inner_ipproto) { + eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM; + if (inner_ipproto == IPPROTO_TCP || inner_ipproto == IPPROTO_UDP) + eseg->cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM; + if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) + sq->stats->csum_partial_inner++; + } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { + eseg->cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM; + sq->stats->csum_partial_inner++; + } + + return true; +} + +static inline unsigned int mlx5e_psp_tx_ids_len(struct mlx5e_accel_tx_psp_state *psp_st) +{ + return psp_st->tailen; +} + +static inline bool mlx5e_psp_is_rx_flow(struct mlx5_cqe64 *cqe) +{ + return MLX5_PSP_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata)); +} + +bool mlx5e_psp_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, + struct mlx5_cqe64 *cqe); +#else +static inline bool mlx5e_psp_is_offload_state(struct mlx5e_accel_tx_psp_state *psp_state) +{ + return false; +} + +static inline bool mlx5e_psp_is_offload(struct sk_buff *skb, struct net_device *netdev) +{ + return false; +} + +static inline bool mlx5e_psp_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, + struct mlx5e_accel_tx_psp_state *psp_st, + struct mlx5_wqe_eth_seg *eseg) +{ + return false; +} + +static inline bool mlx5e_psp_is_rx_flow(struct mlx5_cqe64 *cqe) +{ + return false; +} + +static inline bool mlx5e_psp_offload_handle_rx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5_cqe64 *cqe) +{ + return false; +} +#endif /* CONFIG_MLX5_EN_PSP */ +#endif /* __MLX5E_PSP_RXTX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 6ed3a32b7e22..30424ccad584 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include "devlink.h" #include "en.h" #include "lib/crypto.h" @@ -140,9 +141,22 @@ err_close_tises: return err; } +static unsigned int +mlx5e_get_devlink_param_num_doorbells(struct mlx5_core_dev *dev) +{ + const u32 param_id = DEVLINK_PARAM_GENERIC_ID_NUM_DOORBELLS; + struct devlink *devlink = priv_to_devlink(dev); + union devlink_param_value val; + int err; + + err = devl_param_driverinit_value_get(devlink, param_id, &val); + return err ? MLX5_DEFAULT_NUM_DOORBELLS : val.vu32; +} + int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; + unsigned int num_doorbells, i; int err; err = mlx5_core_alloc_pd(mdev, &res->pdn); @@ -163,17 +177,30 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises) goto err_dealloc_transport_domain; } - err = mlx5_alloc_bfreg(mdev, &res->bfreg, false, false); - if (err) { - mlx5_core_err(mdev, "alloc bfreg failed, %d\n", err); + num_doorbells = min(mlx5e_get_devlink_param_num_doorbells(mdev), + mlx5e_get_max_num_channels(mdev)); + res->bfregs = kcalloc(num_doorbells, sizeof(*res->bfregs), GFP_KERNEL); + if (!res->bfregs) { + err = -ENOMEM; goto err_destroy_mkey; } + for (i = 0; i < num_doorbells; i++) { + err = mlx5_alloc_bfreg(mdev, res->bfregs + i, false, false); + if (err) { + mlx5_core_warn(mdev, + "could only allocate %d/%d doorbells, err %d.\n", + i, num_doorbells, err); + break; + } + } + res->num_bfregs = i; + if (create_tises) { err = mlx5e_create_tises(mdev, res->tisn); if (err) { mlx5_core_err(mdev, "alloc tises failed, %d\n", err); - goto err_destroy_bfreg; + goto err_destroy_bfregs; } res->tisn_valid = true; } @@ -183,15 +210,17 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises) mdev->mlx5e_res.dek_priv = mlx5_crypto_dek_init(mdev); if (IS_ERR(mdev->mlx5e_res.dek_priv)) { - mlx5_core_err(mdev, "crypto dek init failed, %ld\n", - PTR_ERR(mdev->mlx5e_res.dek_priv)); + mlx5_core_err(mdev, "crypto dek init failed, %pe\n", + mdev->mlx5e_res.dek_priv); mdev->mlx5e_res.dek_priv = NULL; } return 0; -err_destroy_bfreg: - mlx5_free_bfreg(mdev, &res->bfreg); +err_destroy_bfregs: + for (i = 0; i < res->num_bfregs; i++) + mlx5_free_bfreg(mdev, res->bfregs + i); + kfree(res->bfregs); err_destroy_mkey: mlx5_core_destroy_mkey(mdev, res->mkey); err_dealloc_transport_domain: @@ -209,7 +238,9 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) mdev->mlx5e_res.dek_priv = NULL; if (res->tisn_valid) mlx5e_destroy_tises(mdev, res->tisn); - mlx5_free_bfreg(mdev, &res->bfreg); + for (unsigned int i = 0; i < res->num_bfregs; i++) + mlx5_free_bfreg(mdev, res->bfregs + i); + kfree(res->bfregs); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); mlx5_core_dealloc_pd(mdev, res->pdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index d507366d773e..53e5ae252eac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1494,7 +1494,8 @@ static int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param * } static int mlx5e_rxfh_hfunc_check(struct mlx5e_priv *priv, - const struct ethtool_rxfh_param *rxfh) + const struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { unsigned int count; @@ -1504,8 +1505,10 @@ static int mlx5e_rxfh_hfunc_check(struct mlx5e_priv *priv, unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8(); if (count > xor8_max_channels) { - netdev_err(priv->netdev, "%s: Cannot set RSS hash function to XOR, current number of channels (%d) exceeds the maximum allowed for XOR8 RSS hfunc (%d)\n", - __func__, count, xor8_max_channels); + NL_SET_ERR_MSG_FMT_MOD( + extack, + "Number of channels (%u) exceeds the max for XOR8 RSS (%u)", + count, xor8_max_channels); return -EINVAL; } } @@ -1524,7 +1527,7 @@ static int mlx5e_set_rxfh(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh); + err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); if (err) goto unlock; @@ -1550,7 +1553,7 @@ static int mlx5e_create_rxfh_context(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh); + err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); if (err) goto unlock; @@ -1590,7 +1593,7 @@ static int mlx5e_modify_rxfh_context(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh); + err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); if (err) goto unlock; @@ -1927,11 +1930,12 @@ static int mlx5e_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) } static void mlx5e_get_fec_stats(struct net_device *netdev, - struct ethtool_fec_stats *fec_stats) + struct ethtool_fec_stats *fec_stats, + struct ethtool_fec_hist *hist) { struct mlx5e_priv *priv = netdev_priv(netdev); - mlx5e_stats_fec_get(priv, fec_stats); + mlx5e_stats_fec_get(priv, fec_stats, hist); } static int mlx5e_get_fecparam(struct net_device *netdev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 265c4ca85f7d..8928d2dcd43f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -905,6 +905,9 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_flow_steering *fs, ft_attr->prio = MLX5E_NIC_PRIO; for (tt = 0; tt < MLX5_NUM_TT; tt++) { + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? @@ -914,9 +917,17 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_flow_steering *fs, } } +static bool mlx5e_ipsec_rss_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, ipsec_next_header) && + MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, outer_l4_type_ext) && + MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, inner_l4_type_ext); +} + void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, struct mlx5e_rx_res *rx_res, - struct ttc_params *ttc_params, bool tunnel) + struct ttc_params *ttc_params, bool tunnel, + bool ipsec_rss) { struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr; @@ -927,7 +938,13 @@ void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, ft_attr->level = MLX5E_TTC_FT_LEVEL; ft_attr->prio = MLX5E_NIC_PRIO; + ttc_params->ipsec_rss = ipsec_rss && + mlx5e_ipsec_rss_supported(fs->mdev); + for (tt = 0; tt < MLX5_NUM_TT; tt++) { + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? @@ -1293,7 +1310,7 @@ int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs, { struct ttc_params ttc_params = {}; - mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true); + mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true, true); fs->ttc = mlx5_create_ttc_table(fs->mdev, &ttc_params); return PTR_ERR_OR_ZERO(fs->ttc); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 15eded36b872..a56825921c23 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -49,10 +49,10 @@ #include "en.h" #include "en/dim.h" #include "en/txrx.h" -#include "en/port_buffer.h" #include "en_tc.h" #include "en_rep.h" #include "en_accel/ipsec.h" +#include "en_accel/psp.h" #include "en_accel/macsec.h" #include "en_accel/en_accel.h" #include "en_accel/ktls.h" @@ -233,9 +233,13 @@ static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data) static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data) { + struct mlx5_devcom_match_attr attr = { + .key.val = *data, + }; + priv->devcom = mlx5_devcom_register_component(priv->mdev->priv.devc, MLX5_DEVCOM_MPV, - *data, + &attr, mlx5e_devcom_event_mpv, priv); if (IS_ERR(priv->devcom)) @@ -778,13 +782,6 @@ static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) bitmap_free(rq->mpwqe.shampo->bitmap); } -static bool mlx5_rq_needs_separate_hd_pool(struct mlx5e_rq *rq) -{ - struct netdev_rx_queue *rxq = __netif_get_rx_queue(rq->netdev, rq->ix); - - return !!rxq->mp_params.mp_ops; -} - static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rqp, @@ -823,7 +820,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, hd_pool_size = (rq->mpwqe.shampo->hd_per_wqe * wq_size) / MLX5E_SHAMPO_WQ_HEADER_PER_PAGE; - if (mlx5_rq_needs_separate_hd_pool(rq)) { + if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) { /* Separate page pool for shampo headers */ struct page_pool_params pp_params = { }; @@ -1537,7 +1534,7 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, sq->pdev = c->pdev; sq->mkey_be = c->mkey_be; sq->channel = c; - sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + sq->uar_map = c->bfreg->map; sq->min_inline_mode = params->tx_min_inline_mode; sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN; sq->xsk_pool = xsk_pool; @@ -1622,7 +1619,7 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c, int err; sq->channel = c; - sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + sq->uar_map = c->bfreg->map; sq->reserved_room = param->stop_room; param->wq.db_numa_node = cpu_to_node(c->cpu); @@ -1707,7 +1704,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, sq->priv = c->priv; sq->ch_ix = c->ix; sq->txq_ix = txq_ix; - sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + sq->uar_map = c->bfreg->map; sq->min_inline_mode = params->tx_min_inline_mode; sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); @@ -1783,7 +1780,7 @@ static int mlx5e_create_sq(struct mlx5_core_dev *mdev, MLX5_SET(sqc, sqc, flush_in_error_en, 1); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); - MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); + MLX5_SET(wq, wq, uar_page, csp->uar_page); MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); @@ -1887,6 +1884,7 @@ int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = sq->min_inline_mode; + csp.uar_page = c->bfreg->index; err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); if (err) goto err_free_txqsq; @@ -2057,6 +2055,7 @@ static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = params->tx_min_inline_mode; + csp.uar_page = c->bfreg->index; err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); if (err) goto err_free_icosq; @@ -2117,6 +2116,7 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = sq->min_inline_mode; + csp.uar_page = c->bfreg->index; set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); @@ -2187,6 +2187,7 @@ static void mlx5e_close_xdpredirect_sq(struct mlx5e_xdpsq *xdpsq) static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, struct net_device *netdev, struct workqueue_struct *workqueue, + struct mlx5_uars_page *uar, struct mlx5e_cq_param *param, struct mlx5e_cq *cq) { @@ -2218,6 +2219,7 @@ static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, cq->mdev = mdev; cq->netdev = netdev; cq->workqueue = workqueue; + cq->uar = uar; return 0; } @@ -2233,7 +2235,8 @@ static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev, param->wq.db_numa_node = ccp->node; param->eq_ix = ccp->ix; - err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, param, cq); + err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, + ccp->uar, param, cq); cq->napi = ccp->napi; cq->ch_stats = ccp->ch_stats; @@ -2278,7 +2281,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode)); MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); - MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, uar_page, cq->uar->index); MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); @@ -2745,6 +2748,20 @@ void mlx5e_trigger_napi_sched(struct napi_struct *napi) local_bh_enable(); } +static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c) +{ + struct mlx5e_hw_objs *hw_objs = &c->mdev->mlx5e_res.hw_objs; + + /* No dedicated Ethernet doorbells, use the global one. */ + if (hw_objs->num_bfregs == 0) { + c->bfreg = &c->mdev->priv.bfreg; + return; + } + + /* Round-robin between doorbells. */ + c->bfreg = hw_objs->bfregs + c->vec_ix % hw_objs->num_bfregs; +} + static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct mlx5e_params *params, struct xsk_buff_pool *xsk_pool, @@ -2799,6 +2816,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, c->aff_mask = irq_get_effective_affinity_mask(irq); c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); + mlx5e_channel_pick_doorbell(c); + netif_napi_add_config_locked(netdev, &c->napi, mlx5e_napi_poll, ix); netif_napi_set_irq_locked(&c->napi, irq); @@ -3041,11 +3060,9 @@ int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) struct mlx5e_params *params = &priv->channels.params; struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; - u16 mtu, prev_mtu; + u16 mtu; int err; - mlx5e_query_mtu(mdev, params, &prev_mtu); - err = mlx5e_set_mtu(mdev, params, params->sw_mtu); if (err) return err; @@ -3055,18 +3072,6 @@ int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", __func__, mtu, params->sw_mtu); - if (mtu != prev_mtu && MLX5_BUFFER_SUPPORTED(mdev)) { - err = mlx5e_port_manual_buffer_config(priv, 0, mtu, - NULL, NULL, NULL); - if (err) { - netdev_warn(netdev, "%s: Failed to set Xon/Xoff values with MTU %d (err %d), setting back to previous MTU %d\n", - __func__, mtu, err, prev_mtu); - - mlx5e_set_mtu(mdev, params, prev_mtu); - return err; - } - } - params->sw_mtu = mtu; return 0; } @@ -3584,7 +3589,8 @@ static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); - return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, param, cq); + return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, + mdev->priv.bfreg.up, param, cq); } int mlx5e_open_drop_rq(struct mlx5e_priv *priv, @@ -5640,12 +5646,36 @@ static int mlx5e_queue_start(struct net_device *dev, void *newq, return 0; } +static struct device *mlx5e_queue_get_dma_dev(struct net_device *dev, + int queue_index) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5e_channels *channels; + struct device *pdev = NULL; + struct mlx5e_channel *ch; + + channels = &priv->channels; + + mutex_lock(&priv->state_lock); + + if (queue_index >= channels->num) + goto out; + + ch = channels->c[queue_index]; + pdev = ch->pdev; +out: + mutex_unlock(&priv->state_lock); + + return pdev; +} + static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = { .ndo_queue_mem_size = sizeof(struct mlx5_qmgmt_data), .ndo_queue_mem_alloc = mlx5e_queue_mem_alloc, .ndo_queue_mem_free = mlx5e_queue_mem_free, .ndo_queue_start = mlx5e_queue_start, .ndo_queue_stop = mlx5e_queue_stop, + .ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev, }; static void mlx5e_build_nic_netdev(struct net_device *netdev) @@ -5873,6 +5903,10 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, } priv->fs = fs; + err = mlx5e_psp_init(priv); + if (err) + mlx5_core_err(mdev, "PSP initialization failed, %d\n", err); + err = mlx5e_ktls_init(priv); if (err) mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); @@ -5885,6 +5919,7 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, if (take_rtnl) rtnl_lock(); + mlx5e_psp_register(priv); /* update XDP supported features */ mlx5e_set_xdp_feature(netdev); @@ -5897,7 +5932,9 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) { mlx5e_health_destroy_reporters(priv); + mlx5e_psp_unregister(priv); mlx5e_ktls_cleanup(priv); + mlx5e_psp_cleanup(priv); mlx5e_fs_cleanup(priv->fs); debugfs_remove_recursive(priv->dfs_root); priv->fs = NULL; @@ -6227,8 +6264,15 @@ int mlx5e_priv_init(struct mlx5e_priv *priv, if (!priv->channel_stats) goto err_free_tx_rates; + priv->fec_ranges = kcalloc(ETHTOOL_FEC_HIST_MAX, + sizeof(*priv->fec_ranges), GFP_KERNEL); + if (!priv->fec_ranges) + goto err_free_channel_stats; + return 0; +err_free_channel_stats: + kfree(priv->channel_stats); err_free_tx_rates: kfree(priv->tx_rates); err_free_txq2sq_stats: @@ -6252,6 +6296,7 @@ void mlx5e_priv_cleanup(struct mlx5e_priv *priv) if (!priv->mdev) return; + kfree(priv->fec_ranges); for (i = 0; i < priv->stats_nch; i++) kvfree(priv->channel_stats[i]); kfree(priv->channel_stats); @@ -6745,6 +6790,7 @@ static void _mlx5e_remove(struct auxiliary_device *adev) * is already unregistered before changing to NIC profile. */ if (priv->netdev->reg_state == NETREG_REGISTERED) { + mlx5e_psp_unregister(priv); unregister_netdev(priv->netdev); _mlx5e_suspend(adev, false); } else { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index cd0242eb008c..0335ca8277ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -974,7 +974,7 @@ static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) MLX5_FLOW_NAMESPACE_KERNEL), false); /* The inner_ttc in the ttc params is intentionally not set */ - mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false); + mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false, false); if (rep->vport != MLX5_VPORT_UPLINK) /* To give uplik rep TTC a lower level for chaining from root ft */ @@ -1447,11 +1447,11 @@ static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv, reporter = devl_port_health_reporter_create(dl_port, &mlx5_rep_vnic_reporter_ops, - 0, rpriv); + rpriv); if (IS_ERR(reporter)) { mlx5_core_err(priv->mdev, - "Failed to create representor vnic reporter, err = %ld\n", - PTR_ERR(reporter)); + "Failed to create representor vnic reporter, err = %pe\n", + reporter); return; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index b8c609d91d11..263d5628ee44 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -51,6 +51,7 @@ #include "ipoib/ipoib.h" #include "en_accel/ipsec.h" #include "en_accel/macsec.h" +#include "en_accel/psp_rxtx.h" #include "en_accel/ipsec_rxtx.h" #include "en_accel/ktls_txrx.h" #include "en/xdp.h" @@ -1289,8 +1290,12 @@ static void mlx5e_shampo_update_ipv4_tcp_hdr(struct mlx5e_rq *rq, struct iphdr * tcp->check = ~tcp_v4_check(skb->len - tcp_off, ipv4->saddr, ipv4->daddr, 0); skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4; - if (ntohs(ipv4->id) == rq->hw_gro_data->second_ip_id) - skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID; + if (ntohs(ipv4->id) == rq->hw_gro_data->second_ip_id) { + bool encap = rq->hw_gro_data->fk.control.flags & FLOW_DIS_ENCAPSULATION; + + skb_shinfo(skb)->gso_type |= encap ? SKB_GSO_TCP_FIXEDID_INNER : + SKB_GSO_TCP_FIXEDID; + } skb->csum_start = (unsigned char *)tcp - skb->head; skb->csum_offset = offsetof(struct tcphdr, check); @@ -1521,6 +1526,11 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, skb->ip_summed = CHECKSUM_COMPLETE; skb->csum = csum_unfold((__force __sum16)cqe->check_sum); + if (unlikely(mlx5e_psp_is_rx_flow(cqe))) { + /* TBD: PSP csum complete corrections for now chose csum_unnecessary path */ + goto csum_unnecessary; + } + if (test_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state)) return; /* CQE csum covers all received bytes */ @@ -1549,7 +1559,7 @@ csum_none: #define MLX5E_CE_BIT_MASK 0x80 -static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, +static inline bool mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, u32 cqe_bcnt, struct mlx5e_rq *rq, struct sk_buff *skb) @@ -1563,6 +1573,11 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, if (unlikely(get_cqe_tls_offload(cqe))) mlx5e_ktls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt); + if (unlikely(mlx5e_psp_is_rx_flow(cqe))) { + if (mlx5e_psp_offload_handle_rx_skb(netdev, skb, cqe)) + return true; + } + if (unlikely(mlx5_ipsec_is_rx_flow(cqe))) mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, be32_to_cpu(cqe->ft_metadata)); @@ -1608,9 +1623,11 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, if (unlikely(mlx5e_skb_is_multicast(skb))) stats->mcast_packets++; + + return false; } -static void mlx5e_shampo_complete_rx_cqe(struct mlx5e_rq *rq, +static bool mlx5e_shampo_complete_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, u32 cqe_bcnt, struct sk_buff *skb) @@ -1620,16 +1637,20 @@ static void mlx5e_shampo_complete_rx_cqe(struct mlx5e_rq *rq, stats->packets++; stats->bytes += cqe_bcnt; if (NAPI_GRO_CB(skb)->count != 1) - return; - mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb); + return false; + + if (mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb)) + return true; + skb_reset_network_header(skb); if (!skb_flow_dissect_flow_keys(skb, &rq->hw_gro_data->fk, 0)) { napi_gro_receive(rq->cq.napi, skb); rq->hw_gro_data->skb = NULL; } + return false; } -static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, +static inline bool mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, u32 cqe_bcnt, struct sk_buff *skb) @@ -1638,7 +1659,7 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, stats->packets++; stats->bytes += cqe_bcnt; - mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb); + return mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb); } static inline @@ -1796,10 +1817,9 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi if (xdp_buff_has_frags(&mxbuf->xdp)) { /* sinfo->nr_frags is reset by build_skb, calculate again. */ - xdp_update_skb_shared_info(skb, wi - head_wi - 1, - sinfo->xdp_frags_size, truesize, - xdp_buff_is_frag_pfmemalloc( - &mxbuf->xdp)); + xdp_update_skb_frags_info(skb, wi - head_wi - 1, + sinfo->xdp_frags_size, truesize, + xdp_buff_get_skb_flags(&mxbuf->xdp)); for (struct mlx5e_wqe_frag_info *pwi = head_wi + 1; pwi < wi; pwi++) pwi->frag_page->frags++; @@ -1855,7 +1875,8 @@ static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) goto wq_cyc_pop; } - mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); + if (mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb)) + goto wq_cyc_pop; if (mlx5e_cqe_regb_chain(cqe)) if (!mlx5e_tc_update_skb_nic(cqe, skb)) { @@ -1902,7 +1923,8 @@ static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) goto wq_cyc_pop; } - mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); + if (mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb)) + goto wq_cyc_pop; if (rep->vlan && skb_vlan_tag_present(skb)) skb_vlan_pop(skb); @@ -1951,7 +1973,8 @@ static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 if (!skb) goto mpwrq_cqe_out; - mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); + if (mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb)) + goto mpwrq_cqe_out; mlx5e_rep_tc_receive(cqe, rq, skb); @@ -2105,10 +2128,10 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w struct mlx5e_frag_page *pagep; /* sinfo->nr_frags is reset by build_skb, calculate again. */ - xdp_update_skb_shared_info(skb, frag_page - head_page, - sinfo->xdp_frags_size, truesize, - xdp_buff_is_frag_pfmemalloc( - &mxbuf->xdp)); + xdp_update_skb_frags_info(skb, frag_page - head_page, + sinfo->xdp_frags_size, + truesize, + xdp_buff_get_skb_flags(&mxbuf->xdp)); pagep = head_page; do @@ -2122,10 +2145,10 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w if (xdp_buff_has_frags(&mxbuf->xdp)) { struct mlx5e_frag_page *pagep; - xdp_update_skb_shared_info(skb, sinfo->nr_frags, - sinfo->xdp_frags_size, truesize, - xdp_buff_is_frag_pfmemalloc( - &mxbuf->xdp)); + xdp_update_skb_frags_info(skb, sinfo->nr_frags, + sinfo->xdp_frags_size, + truesize, + xdp_buff_get_skb_flags(&mxbuf->xdp)); pagep = frag_page - sinfo->nr_frags; do @@ -2388,7 +2411,10 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq stats->hds_nosplit_bytes += data_bcnt; } - mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb); + if (mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb)) { + *skb = NULL; + goto free_hd_entry; + } if (flush && rq->hw_gro_data->skb) mlx5e_shampo_flush_skb(rq, cqe, match); free_hd_entry: @@ -2446,7 +2472,8 @@ static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cq if (!skb) goto mpwrq_cqe_out; - mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); + if (mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb)) + goto mpwrq_cqe_out; if (mlx5e_cqe_regb_chain(cqe)) if (!mlx5e_tc_update_skb_nic(cqe, skb)) { @@ -2779,7 +2806,8 @@ static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe if (!skb) goto wq_cyc_pop; - mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); + if (mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb)) + goto wq_cyc_pop; skb_push(skb, ETH_HLEN); mlx5_devlink_trap_report(rq->mdev, trap_id, skb, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index c6185ddba04b..7c029a7d0fd7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1446,16 +1446,13 @@ static void fec_set_rs_stats(struct ethtool_fec_stats *fec_stats, u32 *ppcnt) } static void fec_set_block_stats(struct mlx5e_priv *priv, + int mode, struct ethtool_fec_stats *fec_stats) { struct mlx5_core_dev *mdev = priv->mdev; u32 out[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); - int mode = fec_active_mode(mdev); - - if (mode == MLX5E_FEC_NOFEC) - return; MLX5_SET(ppcnt_reg, in, local_port, 1); MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP); @@ -1494,14 +1491,130 @@ static void fec_set_corrected_bits_total(struct mlx5e_priv *priv, phy_corrected_bits); } +#define MLX5_RS_HISTOGRAM_ENTRIES \ + (MLX5_FLD_SZ_BYTES(rs_histogram_cntrs, hist) / \ + MLX5_FLD_SZ_BYTES(rs_histogram_cntrs, hist[0])) + +enum { + MLX5E_HISTOGRAM_FEC_RS_544_514 = 1, + MLX5E_HISTOGRAM_FEC_LLRS = 2, + MLX5E_HISTOGRAM_FEC_RS_528_514 = 3, +}; + +static bool fec_rs_validate_hist_type(int mode, int hist_type) +{ + switch (mode) { + case MLX5E_FEC_RS_528_514: + return hist_type == MLX5E_HISTOGRAM_FEC_RS_528_514; + case MLX5E_FEC_RS_544_514_INTERLEAVED_QUAD: + case MLX5E_FEC_RS_544_514: + return hist_type == MLX5E_HISTOGRAM_FEC_RS_544_514; + case MLX5E_FEC_LLRS_272_257_1: + return hist_type == MLX5E_HISTOGRAM_FEC_LLRS; + default: + break; + } + + return false; +} + +static u8 +fec_rs_histogram_fill_ranges(struct mlx5e_priv *priv, int mode, + const struct ethtool_fec_hist_range **ranges) +{ + struct mlx5_core_dev *mdev = priv->mdev; + u32 out[MLX5_ST_SZ_DW(pphcr_reg)] = {0}; + u32 in[MLX5_ST_SZ_DW(pphcr_reg)] = {0}; + int sz = MLX5_ST_SZ_BYTES(pphcr_reg); + u8 hist_type, num_of_bins; + + memset(priv->fec_ranges, 0, + ETHTOOL_FEC_HIST_MAX * sizeof(*priv->fec_ranges)); + MLX5_SET(pphcr_reg, in, local_port, 1); + if (mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPHCR, 0, 0)) + return 0; + + hist_type = MLX5_GET(pphcr_reg, out, active_hist_type); + if (!fec_rs_validate_hist_type(mode, hist_type)) + return 0; + + num_of_bins = MLX5_GET(pphcr_reg, out, num_of_bins); + if (WARN_ON_ONCE(num_of_bins > MLX5_RS_HISTOGRAM_ENTRIES)) + return 0; + + for (int i = 0; i < num_of_bins; i++) { + void *bin_range = MLX5_ADDR_OF(pphcr_reg, out, bin_range[i]); + + priv->fec_ranges[i].high = MLX5_GET(bin_range_layout, bin_range, + high_val); + priv->fec_ranges[i].low = MLX5_GET(bin_range_layout, bin_range, + low_val); + } + *ranges = priv->fec_ranges; + + return num_of_bins; +} + +static void fec_rs_histogram_fill_stats(struct mlx5e_priv *priv, + u8 num_of_bins, + struct ethtool_fec_hist *hist) +{ + struct mlx5_core_dev *mdev = priv->mdev; + u32 out[MLX5_ST_SZ_DW(ppcnt_reg)] = {0}; + u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0}; + int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); + void *rs_histogram_cntrs; + + MLX5_SET(ppcnt_reg, in, local_port, 1); + MLX5_SET(ppcnt_reg, in, grp, MLX5_RS_FEC_HISTOGRAM_GROUP); + if (mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0)) + return; + + rs_histogram_cntrs = MLX5_ADDR_OF(ppcnt_reg, out, + counter_set.rs_histogram_cntrs); + /* Guaranteed that num_of_bins is less than MLX5E_FEC_RS_HIST_MAX + * by fec_rs_histogram_fill_ranges(). + */ + for (int i = 0; i < num_of_bins; i++) + hist->values[i].sum = MLX5_GET64(rs_histogram_cntrs, + rs_histogram_cntrs, + hist[i]); +} + +static void fec_set_histograms_stats(struct mlx5e_priv *priv, int mode, + struct ethtool_fec_hist *hist) +{ + u8 num_of_bins; + + switch (mode) { + case MLX5E_FEC_RS_528_514: + case MLX5E_FEC_RS_544_514: + case MLX5E_FEC_LLRS_272_257_1: + case MLX5E_FEC_RS_544_514_INTERLEAVED_QUAD: + num_of_bins = + fec_rs_histogram_fill_ranges(priv, mode, &hist->ranges); + if (num_of_bins) + return fec_rs_histogram_fill_stats(priv, num_of_bins, + hist); + break; + default: + return; + } +} + void mlx5e_stats_fec_get(struct mlx5e_priv *priv, - struct ethtool_fec_stats *fec_stats) + struct ethtool_fec_stats *fec_stats, + struct ethtool_fec_hist *hist) { - if (!MLX5_CAP_PCAM_FEATURE(priv->mdev, ppcnt_statistical_group)) + int mode = fec_active_mode(priv->mdev); + + if (mode == MLX5E_FEC_NOFEC || + !MLX5_CAP_PCAM_FEATURE(priv->mdev, ppcnt_statistical_group)) return; fec_set_corrected_bits_total(priv, fec_stats); - fec_set_block_stats(priv, fec_stats); + fec_set_block_stats(priv, mode, fec_stats); + fec_set_histograms_stats(priv, mode, hist); } #define PPORT_ETH_EXT_OFF(c) \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 72dbcc1928ef..09f155acb461 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -117,7 +117,8 @@ void mlx5e_stats_update_ndo_stats(struct mlx5e_priv *priv); void mlx5e_stats_pause_get(struct mlx5e_priv *priv, struct ethtool_pause_stats *pause_stats); void mlx5e_stats_fec_get(struct mlx5e_priv *priv, - struct ethtool_fec_stats *fec_stats); + struct ethtool_fec_stats *fec_stats, + struct ethtool_fec_hist *hist); void mlx5e_stats_eth_phy_get(struct mlx5e_priv *priv, struct ethtool_eth_phy_stats *phy_stats); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 32c07a8b03d1..00c2763e57ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -66,6 +66,7 @@ #include "lib/devcom.h" #include "lib/geneve.h" #include "lib/fs_chains.h" +#include "lib/mlx5.h" #include "diag/en_tc_tracepoint.h" #include <asm/div64.h> #include "lag/lag.h" @@ -757,11 +758,11 @@ static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp) struct mlx5e_priv *priv = hp->func_priv; struct mlx5_core_dev *mdev = priv->mdev; struct mlx5e_rss_params_indir indir; + u32 rqt_size; int err; - err = mlx5e_rss_params_indir_init(&indir, mdev, - mlx5e_rqt_size(mdev, hp->num_channels), - mlx5e_rqt_size(mdev, hp->num_channels)); + rqt_size = mlx5e_rqt_size(mdev, hp->num_channels); + err = mlx5e_rss_params_indir_init(&indir, rqt_size, rqt_size); if (err) return err; @@ -837,6 +838,9 @@ static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin *hp, ttc_params->ns_type = MLX5_FLOW_NAMESPACE_KERNEL; for (tt = 0; tt < MLX5_NUM_TT; tt++) { + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? @@ -5387,12 +5391,13 @@ void mlx5e_tc_ht_cleanup(struct rhashtable *tc_ht) int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv) { const size_t sz_enc_opts = sizeof(struct tunnel_match_enc_opts); + struct mlx5_devcom_match_attr attr = {}; struct netdev_phys_item_id ppid; struct mlx5e_rep_priv *rpriv; struct mapping_ctx *mapping; struct mlx5_eswitch *esw; struct mlx5e_priv *priv; - u64 mapping_id, key; + u64 mapping_id; int err = 0; rpriv = container_of(uplink_priv, struct mlx5e_rep_priv, uplink_priv); @@ -5448,8 +5453,10 @@ int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv) err = netif_get_port_parent_id(priv->netdev, &ppid, false); if (!err) { - memcpy(&key, &ppid.id, sizeof(key)); - mlx5_esw_offloads_devcom_init(esw, key); + memcpy(&attr.key.val, &ppid.id, sizeof(attr.key.val)); + attr.flags = MLX5_DEVCOM_MATCH_FLAGS_NS; + attr.net = mlx5_core_net(esw->dev); + mlx5_esw_offloads_devcom_init(esw, &attr); } return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 319061d31602..b7227afcb51d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -39,6 +39,7 @@ #include "ipoib/ipoib.h" #include "en_accel/en_accel.h" #include "en_accel/ipsec_rxtx.h" +#include "en_accel/psp_rxtx.h" #include "en_accel/macsec.h" #include "en/ptp.h" #include <net/ipv6.h> @@ -120,6 +121,11 @@ mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg) { +#ifdef CONFIG_MLX5_EN_PSP + if (unlikely(mlx5e_psp_txwqe_build_eseg_csum(sq, skb, &accel->psp_st, eseg))) + return; +#endif + if (unlikely(mlx5e_ipsec_txwqe_build_eseg_csum(sq, skb, eseg))) return; @@ -297,7 +303,7 @@ static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq *sq, struct sk_buff *skb, stats->packets++; } - attr->insz = mlx5e_accel_tx_ids_len(sq, accel); + attr->insz = mlx5e_accel_tx_ids_len(sq, skb, accel); stats->bytes += attr->num_bytes; } @@ -653,7 +659,7 @@ static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg) { if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) - eseg->flow_table_metadata = + eseg->flow_table_metadata |= cpu_to_be32(mlx5e_ptp_metadata_fifo_peek(&ptpsq->metadata_freelist)); } @@ -661,7 +667,7 @@ static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq * struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { - mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); + mlx5e_accel_tx_eseg(priv, skb, accel, eseg, ihs); mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); if (unlikely(sq->ptpsq)) mlx5e_cqe_ts_id_eseg(sq->ptpsq, skb, eseg); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 1ab77159409d..25499da177bc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -32,9 +32,7 @@ enum { MLX5_EQ_STATE_ALWAYS_ARMED = 0xb, }; -enum { - MLX5_EQ_DOORBEL_OFFSET = 0x40, -}; +#define MLX5_EQ_DOORBELL_OFFSET 0x40 /* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is @@ -309,7 +307,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz); - MLX5_SET(eqc, eqc, uar_page, priv->uar->index); + MLX5_SET(eqc, eqc, uar_page, priv->bfreg.up->index); MLX5_SET(eqc, eqc, intr, vecidx); MLX5_SET(eqc, eqc, log_page_size, eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); @@ -322,7 +320,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, eq->eqn = MLX5_GET(create_eq_out, out, eq_number); eq->irqn = pci_irq_vector(dev->pdev, vecidx); eq->dev = dev; - eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET; + eq->doorbell = priv->bfreg.up->map + MLX5_EQ_DOORBELL_OFFSET; err = mlx5_debug_eq_add(dev, eq); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_lgcy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_lgcy.c index 7dd1dc3f77c7..c9a1654d83a2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_lgcy.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_lgcy.c @@ -87,8 +87,8 @@ int esw_acl_egress_lgcy_setup(struct mlx5_eswitch *esw, drop_counter = mlx5_fc_create(esw->dev, false); if (IS_ERR(drop_counter)) { esw_warn(esw->dev, - "vport[%d] configure egress drop rule counter err(%ld)\n", - vport->vport, PTR_ERR(drop_counter)); + "vport[%d] configure egress drop rule counter err(%pe)\n", + vport->vport, drop_counter); drop_counter = NULL; } vport->egress.legacy.drop_counter = drop_counter; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c new file mode 100644 index 000000000000..0091ba697bae --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "fs_core.h" +#include "eswitch.h" + +enum { + MLX5_ADJ_VPORT_DISCONNECT = 0x0, + MLX5_ADJ_VPORT_CONNECT = 0x1, +}; + +static int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, + u16 vport, bool connect) +{ + u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; + + MLX5_SET(modify_vport_state_in, in, opcode, + MLX5_CMD_OP_MODIFY_VPORT_STATE); + MLX5_SET(modify_vport_state_in, in, op_mod, + MLX5_VPORT_STATE_OP_MOD_ESW_VPORT); + MLX5_SET(modify_vport_state_in, in, other_vport, 1); + MLX5_SET(modify_vport_state_in, in, vport_number, vport); + MLX5_SET(modify_vport_state_in, in, ingress_connect_valid, 1); + MLX5_SET(modify_vport_state_in, in, egress_connect_valid, 1); + MLX5_SET(modify_vport_state_in, in, ingress_connect, connect); + MLX5_SET(modify_vport_state_in, in, egress_connect, connect); + + return mlx5_cmd_exec_in(dev, modify_vport_state, in); +} + +static void mlx5_esw_destroy_esw_vport(struct mlx5_core_dev *dev, u16 vport) +{ + u32 in[MLX5_ST_SZ_DW(destroy_esw_vport_in)] = {}; + + MLX5_SET(destroy_esw_vport_in, in, opcode, + MLX5_CMD_OPCODE_DESTROY_ESW_VPORT); + MLX5_SET(destroy_esw_vport_in, in, vport_num, vport); + + mlx5_cmd_exec_in(dev, destroy_esw_vport, in); +} + +static int mlx5_esw_create_esw_vport(struct mlx5_core_dev *dev, u16 vhca_id, + u16 *vport_num) +{ + u32 out[MLX5_ST_SZ_DW(create_esw_vport_out)] = {}; + u32 in[MLX5_ST_SZ_DW(create_esw_vport_in)] = {}; + int err; + + MLX5_SET(create_esw_vport_in, in, opcode, + MLX5_CMD_OPCODE_CREATE_ESW_VPORT); + MLX5_SET(create_esw_vport_in, in, managed_vhca_id, vhca_id); + + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (!err) + *vport_num = MLX5_GET(create_esw_vport_out, out, vport_num); + + return err; +} + +static int mlx5_esw_adj_vport_create(struct mlx5_eswitch *esw, u16 vhca_id, + const void *rid_info_reg) +{ + struct mlx5_vport *vport; + u16 vport_num; + int err; + + err = mlx5_esw_create_esw_vport(esw->dev, vhca_id, &vport_num); + if (err) { + esw_warn(esw->dev, + "Failed to create adjacent vport for vhca_id %d, err %d\n", + vhca_id, err); + return err; + } + + esw_debug(esw->dev, "Created adjacent vport[%d] %d for vhca_id 0x%x\n", + esw->last_vport_idx, vport_num, vhca_id); + + err = mlx5_esw_vport_alloc(esw, esw->last_vport_idx++, vport_num); + if (err) + goto destroy_esw_vport; + + xa_set_mark(&esw->vports, vport_num, MLX5_ESW_VPT_VF); + vport = mlx5_eswitch_get_vport(esw, vport_num); + vport->adjacent = true; + vport->vhca_id = vhca_id; + + vport->adj_info.parent_pci_devfn = + MLX5_GET(function_vhca_rid_info_reg, rid_info_reg, + parent_pci_device_function); + vport->adj_info.function_id = + MLX5_GET(function_vhca_rid_info_reg, rid_info_reg, function_id); + + mlx5_fs_vport_egress_acl_ns_add(esw->dev->priv.steering, vport->index); + mlx5_fs_vport_ingress_acl_ns_add(esw->dev->priv.steering, vport->index); + err = mlx5_esw_offloads_rep_add(esw, vport); + if (err) + goto acl_ns_remove; + + mlx5_esw_adj_vport_modify(esw->dev, vport_num, MLX5_ADJ_VPORT_CONNECT); + return 0; + +acl_ns_remove: + mlx5_fs_vport_ingress_acl_ns_remove(esw->dev->priv.steering, + vport->index); + mlx5_fs_vport_egress_acl_ns_remove(esw->dev->priv.steering, + vport->index); + mlx5_esw_vport_free(esw, vport); +destroy_esw_vport: + mlx5_esw_destroy_esw_vport(esw->dev, vport_num); + return err; +} + +static void mlx5_esw_adj_vport_destroy(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) +{ + u16 vport_num = vport->vport; + + esw_debug(esw->dev, "Destroying adjacent vport %d for vhca_id 0x%x\n", + vport_num, vport->vhca_id); + mlx5_esw_adj_vport_modify(esw->dev, vport_num, + MLX5_ADJ_VPORT_DISCONNECT); + mlx5_esw_offloads_rep_remove(esw, vport); + mlx5_fs_vport_egress_acl_ns_remove(esw->dev->priv.steering, + vport->index); + mlx5_fs_vport_ingress_acl_ns_remove(esw->dev->priv.steering, + vport->index); + mlx5_esw_vport_free(esw, vport); + /* Reset the vport index back so new adj vports can use this index. + * When vport count can incrementally change, this needs to be modified. + */ + esw->last_vport_idx--; + mlx5_esw_destroy_esw_vport(esw->dev, vport_num); +} + +void mlx5_esw_adjacent_vhcas_cleanup(struct mlx5_eswitch *esw) +{ + struct mlx5_vport *vport; + unsigned long i; + + if (!MLX5_CAP_GEN_2(esw->dev, delegated_vhca_max)) + return; + + mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { + if (!vport->adjacent) + continue; + mlx5_esw_adj_vport_destroy(esw, vport); + } +} + +void mlx5_esw_adjacent_vhcas_setup(struct mlx5_eswitch *esw) +{ + u32 delegated_vhca_max = MLX5_CAP_GEN_2(esw->dev, delegated_vhca_max); + u32 in[MLX5_ST_SZ_DW(query_delegated_vhca_in)] = {}; + int outlen, err, i = 0; + u8 *out; + u32 count; + + if (!delegated_vhca_max) + return; + + outlen = MLX5_ST_SZ_BYTES(query_delegated_vhca_out) + + delegated_vhca_max * + MLX5_ST_SZ_BYTES(delegated_function_vhca_rid_info); + + esw_debug(esw->dev, "delegated_vhca_max=%d\n", delegated_vhca_max); + + out = kvzalloc(outlen, GFP_KERNEL); + if (!out) + return; + + MLX5_SET(query_delegated_vhca_in, in, opcode, + MLX5_CMD_OPCODE_QUERY_DELEGATED_VHCA); + + err = mlx5_cmd_exec(esw->dev, in, sizeof(in), out, outlen); + if (err) { + kvfree(out); + esw_warn(esw->dev, "Failed to query delegated vhca, err %d\n", + err); + return; + } + + count = MLX5_GET(query_delegated_vhca_out, out, functions_count); + esw_debug(esw->dev, "Delegated vhca functions count %d\n", count); + + for (i = 0; i < count; i++) { + const void *rid_info, *rid_info_reg; + u16 vhca_id; + + rid_info = MLX5_ADDR_OF(query_delegated_vhca_out, out, + delegated_function_vhca_rid_info[i]); + + rid_info_reg = MLX5_ADDR_OF(delegated_function_vhca_rid_info, + rid_info, function_vhca_rid_info); + + vhca_id = MLX5_GET(function_vhca_rid_info_reg, rid_info_reg, + vhca_id); + esw_debug(esw->dev, "Delegating vhca_id 0x%x\n", vhca_id); + + err = mlx5_esw_adj_vport_create(esw, vhca_id, rid_info_reg); + if (err) { + esw_warn(esw->dev, + "Failed to init adjacent vhca 0x%x, err %d\n", + vhca_id, err); + break; + } + } + + kvfree(out); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index 76e35c827da0..60e10047770f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -81,7 +81,8 @@ mlx5_esw_bridge_table_create(int max_fte, u32 level, struct mlx5_eswitch *esw) ft_attr.prio = FDB_BR_OFFLOAD; fdb = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(fdb)) - esw_warn(dev, "Failed to create bridge FDB Table (err=%ld)\n", PTR_ERR(fdb)); + esw_warn(dev, "Failed to create bridge FDB Table (err=%pe)\n", + fdb); return fdb; } @@ -121,8 +122,8 @@ mlx5_esw_bridge_ingress_vlan_proto_fg_create(unsigned int from, unsigned int to, kvfree(in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create VLAN(proto=%x) flow group for bridge ingress table (err=%ld)\n", - vlan_proto, PTR_ERR(fg)); + "Failed to create VLAN(proto=%x) flow group for bridge ingress table (err=%pe)\n", + vlan_proto, fg); return fg; } @@ -180,8 +181,8 @@ mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(unsigned int from, unsigned fg = mlx5_create_flow_group(ingress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create bridge ingress table VLAN filter flow group (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create bridge ingress table VLAN filter flow group (err=%pe)\n", + fg); kvfree(in); return fg; } @@ -237,8 +238,8 @@ mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow fg = mlx5_create_flow_group(ingress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create MAC flow group for bridge ingress table (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create MAC flow group for bridge ingress table (err=%pe)\n", + fg); kvfree(in); return fg; @@ -274,8 +275,8 @@ mlx5_esw_bridge_egress_vlan_proto_fg_create(unsigned int from, unsigned int to, fg = mlx5_create_flow_group(egress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create VLAN flow group for bridge egress table (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create VLAN flow group for bridge egress table (err=%pe)\n", + fg); kvfree(in); return fg; } @@ -324,8 +325,8 @@ mlx5_esw_bridge_egress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_ fg = mlx5_create_flow_group(egress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create bridge egress table MAC flow group (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create bridge egress table MAC flow group (err=%pe)\n", + fg); kvfree(in); return fg; } @@ -354,8 +355,8 @@ mlx5_esw_bridge_egress_miss_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow fg = mlx5_create_flow_group(egress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create bridge egress table miss flow group (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create bridge egress table miss flow group (err=%pe)\n", + fg); kvfree(in); return fg; } @@ -501,8 +502,8 @@ mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, if (mlx5_esw_bridge_pkt_reformat_vlan_pop_supported(esw)) { miss_fg = mlx5_esw_bridge_egress_miss_fg_create(esw, egress_ft); if (IS_ERR(miss_fg)) { - esw_warn(esw->dev, "Failed to create miss flow group (err=%ld)\n", - PTR_ERR(miss_fg)); + esw_warn(esw->dev, "Failed to create miss flow group (err=%pe)\n", + miss_fg); miss_fg = NULL; goto skip_miss_flow; } @@ -510,8 +511,8 @@ mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, miss_pkt_reformat = mlx5_esw_bridge_pkt_reformat_vlan_pop_create(esw); if (IS_ERR(miss_pkt_reformat)) { esw_warn(esw->dev, - "Failed to alloc packet reformat REMOVE_HEADER (err=%ld)\n", - PTR_ERR(miss_pkt_reformat)); + "Failed to alloc packet reformat REMOVE_HEADER (err=%pe)\n", + miss_pkt_reformat); miss_pkt_reformat = NULL; mlx5_destroy_flow_group(miss_fg); miss_fg = NULL; @@ -522,8 +523,8 @@ mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, br_offloads->skip_ft, miss_pkt_reformat); if (IS_ERR(miss_handle)) { - esw_warn(esw->dev, "Failed to create miss flow (err=%ld)\n", - PTR_ERR(miss_handle)); + esw_warn(esw->dev, "Failed to create miss flow (err=%pe)\n", + miss_handle); miss_handle = NULL; mlx5_packet_reformat_dealloc(esw->dev, miss_pkt_reformat); miss_pkt_reformat = NULL; @@ -1048,8 +1049,8 @@ mlx5_esw_bridge_vlan_push_create(u16 vlan_proto, struct mlx5_esw_bridge_vlan *vl &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(pkt_reformat)) { - esw_warn(esw->dev, "Failed to alloc packet reformat INSERT_HEADER (err=%ld)\n", - PTR_ERR(pkt_reformat)); + esw_warn(esw->dev, "Failed to alloc packet reformat INSERT_HEADER (err=%pe)\n", + pkt_reformat); return PTR_ERR(pkt_reformat); } @@ -1076,8 +1077,8 @@ mlx5_esw_bridge_vlan_pop_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_e pkt_reformat = mlx5_esw_bridge_pkt_reformat_vlan_pop_create(esw); if (IS_ERR(pkt_reformat)) { - esw_warn(esw->dev, "Failed to alloc packet reformat REMOVE_HEADER (err=%ld)\n", - PTR_ERR(pkt_reformat)); + esw_warn(esw->dev, "Failed to alloc packet reformat REMOVE_HEADER (err=%pe)\n", + pkt_reformat); return PTR_ERR(pkt_reformat); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index c33accadae0f..cf88a106d80d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -27,6 +27,7 @@ static void mlx5_esw_offloads_pf_vf_devlink_port_attrs_set(struct mlx5_eswitch * { struct mlx5_core_dev *dev = esw->dev; struct netdev_phys_item_id ppid = {}; + struct mlx5_vport *vport; u32 controller_num = 0; bool external; u16 pfnum; @@ -42,10 +43,18 @@ static void mlx5_esw_offloads_pf_vf_devlink_port_attrs_set(struct mlx5_eswitch * dl_port->attrs.switch_id.id_len = ppid.id_len; devlink_port_attrs_pci_pf_set(dl_port, controller_num, pfnum, external); } else if (mlx5_eswitch_is_vf_vport(esw, vport_num)) { + u16 func_id = vport_num - 1; + + vport = mlx5_eswitch_get_vport(esw, vport_num); memcpy(dl_port->attrs.switch_id.id, ppid.id, ppid.id_len); dl_port->attrs.switch_id.id_len = ppid.id_len; + if (vport->adjacent) { + func_id = vport->adj_info.function_id; + pfnum = vport->adj_info.parent_pci_devfn; + } + devlink_port_attrs_pci_vf_set(dl_port, controller_num, pfnum, - vport_num - 1, external); + func_id, external); } else if (mlx5_core_is_ec_vf_vport(esw->dev, vport_num)) { u16 base_vport = mlx5_core_ec_vf_vport_base(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index 5f2d6c35f1ad..56e6f54b1e2e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -971,8 +971,9 @@ esw_qos_vport_tc_enable(struct mlx5_vport *vport, enum sched_node_type type, max_level = 1 << MLX5_CAP_QOS(vport_node->esw->dev, log_esw_max_sched_depth); if (new_level > max_level) { - NL_SET_ERR_MSG_MOD(extack, - "TC arbitration on leafs is not supported beyond max scheduling depth"); + NL_SET_ERR_MSG_FMT_MOD(extack, + "TC arbitration on leafs is not supported beyond max depth %d", + max_level); return -EOPNOTSUPP; } } @@ -1444,8 +1445,9 @@ static int esw_qos_node_enable_tc_arbitration(struct mlx5_esw_sched_node *node, new_level = node->level + 1; max_level = 1 << MLX5_CAP_QOS(node->esw->dev, log_esw_max_sched_depth); if (new_level > max_level) { - NL_SET_ERR_MSG_MOD(extack, - "TC arbitration on nodes is not supported beyond max scheduling depth"); + NL_SET_ERR_MSG_FMT_MOD(extack, + "TC arbitration on nodes is not supported beyond max depth %d", + max_level); return -EOPNOTSUPP; } @@ -1997,8 +1999,9 @@ mlx5_esw_qos_node_validate_set_parent(struct mlx5_esw_sched_node *node, max_level = 1 << MLX5_CAP_QOS(node->esw->dev, log_esw_max_sched_depth); if (new_level > max_level) { - NL_SET_ERR_MSG_MOD(extack, - "Node hierarchy depth exceeds the maximum supported level"); + NL_SET_ERR_MSG_FMT_MOD(extack, + "Node hierarchy depth %d exceeds the maximum supported level %d", + new_level, max_level); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c index 749c3957a128..407062096a82 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c @@ -45,8 +45,8 @@ esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns, ft_attr.flags = vport_ns->flags; fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); if (IS_ERR(fdb)) { - esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n", - PTR_ERR(fdb)); + esw_warn(esw->dev, "Failed to create per vport FDB Table err %pe\n", + fdb); } return fdb; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 4917d185d0c3..e2ffb87b94cb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -257,8 +257,8 @@ __esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u16 vport, bool rx_rule, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) { esw_warn(esw->dev, - "FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n", - dmac_v, dmac_c, vport, PTR_ERR(flow_rule)); + "FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%pe)\n", + dmac_v, dmac_c, vport, flow_rule); flow_rule = NULL; } @@ -820,6 +820,7 @@ static int mlx5_esw_vport_caps_get(struct mlx5_eswitch *esw, struct mlx5_vport * hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); vport->info.roce_enabled = MLX5_GET(cmd_hca_cap, hca_caps, roce); + vport->vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); if (!MLX5_CAP_GEN_MAX(esw->dev, hca_cap_2)) goto out_free; @@ -839,6 +840,18 @@ out_free: return err; } +bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id) +{ + struct mlx5_vport *vport; + + vport = mlx5_eswitch_get_vport(esw, vportn); + if (IS_ERR(vport) || MLX5_VPORT_INVAL_VHCA_ID(vport)) + return false; + + *vhca_id = vport->vhca_id; + return true; +} + static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { bool vst_mode_steering = esw_vst_mode_is_steering(esw); @@ -929,7 +942,7 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, if (!mlx5_esw_is_manager_vport(esw, vport_num) && MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { - ret = mlx5_esw_vport_vhca_id_set(esw, vport_num); + ret = mlx5_esw_vport_vhca_id_map(esw, vport); if (ret) goto err_vhca_mapping; } @@ -973,7 +986,7 @@ void mlx5_esw_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport) if (!mlx5_esw_is_manager_vport(esw, vport_num) && MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) - mlx5_esw_vport_vhca_id_clear(esw, vport_num); + mlx5_esw_vport_vhca_id_unmap(esw, vport); if (vport->vport != MLX5_VPORT_PF && (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled)) @@ -1038,6 +1051,25 @@ const u32 *mlx5_esw_query_functions(struct mlx5_core_dev *dev) return ERR_PTR(err); } +static int mlx5_esw_host_functions_enabled_query(struct mlx5_eswitch *esw) +{ + const u32 *query_host_out; + + if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) + return 0; + + query_host_out = mlx5_esw_query_functions(esw->dev); + if (IS_ERR(query_host_out)) + return PTR_ERR(query_host_out); + + esw->esw_funcs.host_funcs_disabled = + MLX5_GET(query_esw_functions_out, query_host_out, + host_params_context.host_pf_not_exist); + + kvfree(query_host_out); + return 0; +} + static void mlx5_eswitch_event_handler_register(struct mlx5_eswitch *esw) { if (esw->mode == MLX5_ESWITCH_OFFLOADS && mlx5_eswitch_is_funcs_handler(esw->dev)) { @@ -1185,7 +1217,8 @@ void mlx5_eswitch_unload_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs) unsigned long i; mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) { - if (!vport->enabled) + /* Adjacent VFs are unloaded separately */ + if (!vport->enabled || vport->adjacent) continue; mlx5_eswitch_unload_pf_vf_vport(esw, vport->vport); } @@ -1204,6 +1237,42 @@ static void mlx5_eswitch_unload_ec_vf_vports(struct mlx5_eswitch *esw, } } +static void mlx5_eswitch_unload_adj_vf_vports(struct mlx5_eswitch *esw) +{ + struct mlx5_vport *vport; + unsigned long i; + + mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { + if (!vport->enabled || !vport->adjacent) + continue; + mlx5_eswitch_unload_pf_vf_vport(esw, vport->vport); + } +} + +static int +mlx5_eswitch_load_adj_vf_vports(struct mlx5_eswitch *esw, + enum mlx5_eswitch_vport_event enabled_events) +{ + struct mlx5_vport *vport; + unsigned long i; + int err; + + mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { + if (!vport->adjacent) + continue; + err = mlx5_eswitch_load_pf_vf_vport(esw, vport->vport, + enabled_events); + if (err) + goto unload_adj_vf_vport; + } + + return 0; + +unload_adj_vf_vport: + mlx5_eswitch_unload_adj_vf_vports(esw); + return err; +} + int mlx5_eswitch_load_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs, enum mlx5_eswitch_vport_event enabled_events) { @@ -1278,17 +1347,19 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw, esw->mode == MLX5_ESWITCH_LEGACY; /* Enable PF vport */ - if (pf_needed) { + if (pf_needed && mlx5_esw_host_functions_enabled(esw->dev)) { ret = mlx5_eswitch_load_pf_vf_vport(esw, MLX5_VPORT_PF, enabled_events); if (ret) return ret; } - /* Enable external host PF HCA */ - ret = host_pf_enable_hca(esw->dev); - if (ret) - goto pf_hca_err; + if (mlx5_esw_host_functions_enabled(esw->dev)) { + /* Enable external host PF HCA */ + ret = host_pf_enable_hca(esw->dev); + if (ret) + goto pf_hca_err; + } /* Enable ECPF vport */ if (mlx5_ecpf_vport_exists(esw->dev)) { @@ -1311,8 +1382,16 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw, enabled_events); if (ret) goto vf_err; + + /* Enable adjacent VF vports */ + ret = mlx5_eswitch_load_adj_vf_vports(esw, enabled_events); + if (ret) + goto unload_vf_vports; + return 0; +unload_vf_vports: + mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); vf_err: if (mlx5_core_ec_sriov_enabled(esw->dev)) mlx5_eswitch_unload_ec_vf_vports(esw, esw->esw_funcs.num_ec_vfs); @@ -1320,9 +1399,10 @@ ec_vf_err: if (mlx5_ecpf_vport_exists(esw->dev)) mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_ECPF); ecpf_err: - host_pf_disable_hca(esw->dev); + if (mlx5_esw_host_functions_enabled(esw->dev)) + host_pf_disable_hca(esw->dev); pf_hca_err: - if (pf_needed) + if (pf_needed && mlx5_esw_host_functions_enabled(esw->dev)) mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_PF); return ret; } @@ -1332,6 +1412,8 @@ pf_hca_err: */ void mlx5_eswitch_disable_pf_vf_vports(struct mlx5_eswitch *esw) { + mlx5_eswitch_unload_adj_vf_vports(esw); + mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); if (mlx5_core_ec_sriov_enabled(esw->dev)) @@ -1342,10 +1424,12 @@ void mlx5_eswitch_disable_pf_vf_vports(struct mlx5_eswitch *esw) mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_ECPF); } - host_pf_disable_hca(esw->dev); + if (mlx5_esw_host_functions_enabled(esw->dev)) + host_pf_disable_hca(esw->dev); - if (mlx5_core_is_ecpf_esw_manager(esw->dev) || - esw->mode == MLX5_ESWITCH_LEGACY) + if ((mlx5_core_is_ecpf_esw_manager(esw->dev) || + esw->mode == MLX5_ESWITCH_LEGACY) && + mlx5_esw_host_functions_enabled(esw->dev)) mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_PF); } @@ -1402,19 +1486,76 @@ static void mlx5_esw_mode_change_notify(struct mlx5_eswitch *esw, u16 mode) blocking_notifier_call_chain(&esw->n_head, 0, &info); } +static int mlx5_esw_egress_acls_init(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int total_vports = mlx5_eswitch_get_total_vports(dev); + int err; + int i; + + for (i = 0; i < total_vports; i++) { + err = mlx5_fs_vport_egress_acl_ns_add(steering, i); + if (err) + goto acl_ns_remove; + } + return 0; + +acl_ns_remove: + while (i--) + mlx5_fs_vport_egress_acl_ns_remove(steering, i); + return err; +} + +static void mlx5_esw_egress_acls_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int total_vports = mlx5_eswitch_get_total_vports(dev); + int i; + + for (i = total_vports - 1; i >= 0; i--) + mlx5_fs_vport_egress_acl_ns_remove(steering, i); +} + +static int mlx5_esw_ingress_acls_init(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int total_vports = mlx5_eswitch_get_total_vports(dev); + int err; + int i; + + for (i = 0; i < total_vports; i++) { + err = mlx5_fs_vport_ingress_acl_ns_add(steering, i); + if (err) + goto acl_ns_remove; + } + return 0; + +acl_ns_remove: + while (i--) + mlx5_fs_vport_ingress_acl_ns_remove(steering, i); + return err; +} + +static void mlx5_esw_ingress_acls_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_steering *steering = dev->priv.steering; + int total_vports = mlx5_eswitch_get_total_vports(dev); + int i; + + for (i = total_vports - 1; i >= 0; i--) + mlx5_fs_vport_ingress_acl_ns_remove(steering, i); +} + static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) { struct mlx5_core_dev *dev = esw->dev; - int total_vports; int err; if (esw->flags & MLX5_ESWITCH_VPORT_ACL_NS_CREATED) return 0; - total_vports = mlx5_eswitch_get_total_vports(dev); - if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) { - err = mlx5_fs_egress_acls_init(dev, total_vports); + err = mlx5_esw_egress_acls_init(dev); if (err) return err; } else { @@ -1422,7 +1563,7 @@ static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) } if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) { - err = mlx5_fs_ingress_acls_init(dev, total_vports); + err = mlx5_esw_ingress_acls_init(dev); if (err) goto err; } else { @@ -1433,7 +1574,7 @@ static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) err: if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) - mlx5_fs_egress_acls_cleanup(dev); + mlx5_esw_egress_acls_cleanup(dev); return err; } @@ -1443,9 +1584,9 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) esw->flags &= ~MLX5_ESWITCH_VPORT_ACL_NS_CREATED; if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) - mlx5_fs_ingress_acls_cleanup(dev); + mlx5_esw_ingress_acls_cleanup(dev); if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) - mlx5_fs_egress_acls_cleanup(dev); + mlx5_esw_egress_acls_cleanup(dev); } /** @@ -1674,7 +1815,8 @@ int mlx5_esw_sf_max_hpf_functions(struct mlx5_core_dev *dev, u16 *max_sfs, u16 * void *hca_caps; int err; - if (!mlx5_core_is_ecpf(dev)) { + if (!mlx5_core_is_ecpf(dev) || + !mlx5_esw_host_functions_enabled(dev)) { *max_sfs = 0; return 0; } @@ -1696,8 +1838,7 @@ out_free: return err; } -static int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw, - int index, u16 vport_num) +int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw, int index, u16 vport_num) { struct mlx5_vport *vport; int err; @@ -1710,6 +1851,7 @@ static int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw, vport->vport = vport_num; vport->index = index; vport->info.link_state = MLX5_VPORT_ADMIN_STATE_AUTO; + vport->vhca_id = MLX5_VHCA_ID_INVALID; INIT_WORK(&vport->vport_change_handler, esw_vport_change_handler); err = xa_insert(&esw->vports, vport_num, vport, GFP_KERNEL); if (err) @@ -1723,8 +1865,9 @@ insert_err: return err; } -static void mlx5_esw_vport_free(struct mlx5_eswitch *esw, struct mlx5_vport *vport) +void mlx5_esw_vport_free(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { + esw->total_vports--; xa_erase(&esw->vports, vport->vport); kfree(vport); } @@ -1750,21 +1893,23 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw) xa_init(&esw->vports); - err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_PF); - if (err) - goto err; - if (esw->first_host_vport == MLX5_VPORT_PF) - xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN); - idx++; - - for (i = 0; i < mlx5_core_max_vfs(dev); i++) { - err = mlx5_esw_vport_alloc(esw, idx, idx); + if (mlx5_esw_host_functions_enabled(dev)) { + err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_PF); if (err) goto err; - xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_VF); - xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN); + if (esw->first_host_vport == MLX5_VPORT_PF) + xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN); idx++; + for (i = 0; i < mlx5_core_max_vfs(dev); i++) { + err = mlx5_esw_vport_alloc(esw, idx, idx); + if (err) + goto err; + xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_VF); + xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN); + idx++; + } } + base_sf_num = mlx5_sf_start_function_id(dev); for (i = 0; i < mlx5_sf_max_functions(dev); i++) { err = mlx5_esw_vport_alloc(esw, idx, base_sf_num + i); @@ -1806,6 +1951,9 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw) err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_UPLINK); if (err) goto err; + + /* Adjacent vports or other dynamically create vports will use this */ + esw->last_vport_idx = ++idx; return 0; err: @@ -1864,6 +2012,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) goto free_esw; esw->dev = dev; + dev->priv.eswitch = esw; esw->manager_vport = mlx5_eswitch_manager_vport(dev); esw->first_host_vport = mlx5_eswitch_first_host_vport_num(dev); @@ -1874,11 +2023,14 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) goto abort; } + err = mlx5_esw_host_functions_enabled_query(esw); + if (err) + goto abort; + err = mlx5_esw_vports_init(esw); if (err) goto abort; - dev->priv.eswitch = esw; err = esw_offloads_init(esw); if (err) goto reps_err; @@ -2410,3 +2562,11 @@ void mlx5_eswitch_unblock_ipsec(struct mlx5_core_dev *dev) dev->num_ipsec_offloads--; mutex_unlock(&esw->state_lock); } + +bool mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev) +{ + if (!dev->priv.eswitch) + return true; + + return !dev->priv.eswitch->esw_funcs.host_funcs_disabled; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 45506ad56847..df3756d7e52e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -197,6 +197,11 @@ static inline struct mlx5_vport *mlx5_devlink_port_vport_get(struct devlink_port return mlx5_devlink_port_get(dl_port)->vport; } +#define MLX5_VHCA_ID_INVALID (-1) + +#define MLX5_VPORT_INVAL_VHCA_ID(vport) \ + ((vport)->vhca_id == MLX5_VHCA_ID_INVALID) + struct mlx5_vport { struct mlx5_core_dev *dev; struct hlist_head uc_list[MLX5_L2_ADDR_HASH_SIZE]; @@ -209,6 +214,13 @@ struct mlx5_vport { struct vport_egress egress; u32 default_metadata; u32 metadata; + int vhca_id; + + bool adjacent; /* delegated vhca from adjacent function */ + struct { + u16 parent_pci_devfn; /* Adjacent parent PCI device function */ + u16 function_id; /* Function ID of the delegated VPort */ + } adj_info; struct mlx5_vport_info info; @@ -323,6 +335,7 @@ struct mlx5_host_work { struct mlx5_esw_functions { struct mlx5_nb nb; + bool host_funcs_disabled; u16 num_vfs; u16 num_ec_vfs; }; @@ -377,6 +390,7 @@ struct mlx5_eswitch { struct mlx5_esw_bridge_offloads *br_offloads; struct mlx5_esw_offload offloads; + u32 last_vport_idx; int mode; u16 manager_vport; u16 first_host_vport; @@ -410,6 +424,8 @@ int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32 /* E-Switch API */ int mlx5_eswitch_init(struct mlx5_core_dev *dev); void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw); +int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw, int index, u16 vport_num); +void mlx5_esw_vport_free(struct mlx5_eswitch *esw, struct mlx5_vport *vport); #define MLX5_ESWITCH_IGNORE_NUM_VFS (-1) int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs); @@ -417,7 +433,8 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs); void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf); void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw); void mlx5_eswitch_disable(struct mlx5_eswitch *esw); -void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key); +void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, + const struct mlx5_devcom_match_attr *attr); void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw); bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw); int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, @@ -615,6 +632,9 @@ bool mlx5_esw_multipath_prereq(struct mlx5_core_dev *dev0, const u32 *mlx5_esw_query_functions(struct mlx5_core_dev *dev); +void mlx5_esw_adjacent_vhcas_setup(struct mlx5_eswitch *esw); +void mlx5_esw_adjacent_vhcas_cleanup(struct mlx5_eswitch *esw); + #define MLX5_DEBUG_ESWITCH_MASK BIT(3) #define esw_info(__dev, format, ...) \ @@ -817,9 +837,17 @@ struct devlink_port *mlx5_esw_offloads_devlink_port(struct mlx5_eswitch *esw, u1 int mlx5_esw_sf_max_hpf_functions(struct mlx5_core_dev *dev, u16 *max_sfs, u16 *sf_base_id); -int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num); -void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num); +int mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num); +bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id); + +void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw, + const struct mlx5_vport *vport); +int mlx5_esw_offloads_rep_add(struct mlx5_eswitch *esw, + const struct mlx5_vport *vport); /** * struct mlx5_esw_event_info - Indicates eswitch mode changed/changing. @@ -893,6 +921,7 @@ int mlx5_esw_ipsec_vf_packet_offload_set(struct mlx5_eswitch *esw, struct mlx5_v bool enable); int mlx5_esw_ipsec_vf_packet_offload_supported(struct mlx5_core_dev *dev, u16 vport_num); +bool mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } @@ -900,7 +929,9 @@ static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {} static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { return 0; } static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) {} static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw) {} -static inline void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) {} +static inline void +mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, + const struct mlx5_devcom_match_attr *attr) {} static inline void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) {} static inline bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) { return false; } static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; } @@ -960,6 +991,19 @@ static inline bool mlx5_eswitch_block_ipsec(struct mlx5_core_dev *dev) } static inline void mlx5_eswitch_unblock_ipsec(struct mlx5_core_dev *dev) {} + +static inline bool +mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev) +{ + return true; +} + +static inline bool +mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id) +{ + return -EOPNOTSUPP; +} + #endif /* CONFIG_MLX5_ESWITCH */ #endif /* __MLX5_ESWITCH_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index bee906661282..52c3de24bea3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1016,8 +1016,8 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) - esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", - PTR_ERR(flow_rule)); + esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %pe\n", + flow_rule); out: kvfree(spec); return flow_rule; @@ -1065,8 +1065,8 @@ mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) - esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", - vport_num, PTR_ERR(flow_rule)); + esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %pe\n", + vport_num, flow_rule); kvfree(spec); return flow_rule; @@ -1213,7 +1213,8 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); - if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + if (mlx5_core_is_ecpf_esw_manager(peer_dev) && + mlx5_esw_host_functions_enabled(peer_dev)) { peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, MLX5_VPORT_PF); @@ -1239,19 +1240,21 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, flows[peer_vport->index] = flow; } - mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, - mlx5_core_max_vfs(peer_dev)) { - esw_set_peer_miss_rule_source_port(esw, - peer_esw, - spec, peer_vport->vport); + if (mlx5_esw_host_functions_enabled(esw->dev)) { + mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_vfs(peer_dev)) { + esw_set_peer_miss_rule_source_port(esw, peer_esw, + spec, + peer_vport->vport); - flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), - spec, &flow_act, &dest, 1); - if (IS_ERR(flow)) { - err = PTR_ERR(flow); - goto add_vf_flow_err; + flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), + spec, &flow_act, &dest, 1); + if (IS_ERR(flow)) { + err = PTR_ERR(flow); + goto add_vf_flow_err; + } + flows[peer_vport->index] = flow; } - flows[peer_vport->index] = flow; } if (mlx5_core_ec_sriov_enabled(peer_dev)) { @@ -1301,7 +1304,9 @@ add_vf_flow_err: mlx5_del_flow_rules(flows[peer_vport->index]); } add_ecpf_flow_err: - if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + + if (mlx5_core_is_ecpf_esw_manager(peer_dev) && + mlx5_esw_host_functions_enabled(peer_dev)) { peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); mlx5_del_flow_rules(flows[peer_vport->index]); } @@ -2154,7 +2159,9 @@ mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, &flow_act, dest, 1); if (IS_ERR(flow_rule)) { - esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); + esw_warn(esw->dev, + "fs offloads: Failed to add vport rx rule err %pe\n", + flow_rule); goto out; } @@ -2173,8 +2180,8 @@ static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) &flow_act, NULL, 0); if (IS_ERR(flow_rule)) { esw_warn(esw->dev, - "fs offloads: Failed to add vport rx drop rule err %ld\n", - PTR_ERR(flow_rule)); + "fs offloads: Failed to add vport rx drop rule err %pe\n", + flow_rule); return PTR_ERR(flow_rule); } @@ -2373,7 +2380,20 @@ static int esw_offloads_start(struct mlx5_eswitch *esw, return 0; } -static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) +void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw, + const struct mlx5_vport *vport) +{ + struct mlx5_eswitch_rep *rep = xa_load(&esw->offloads.vport_reps, + vport->vport); + + if (!rep) + return; + xa_erase(&esw->offloads.vport_reps, vport->vport); + kfree(rep); +} + +int mlx5_esw_offloads_rep_add(struct mlx5_eswitch *esw, + const struct mlx5_vport *vport) { struct mlx5_eswitch_rep *rep; int rep_type; @@ -2385,9 +2405,19 @@ static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx rep->vport = vport->vport; rep->vport_index = vport->index; - for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) - atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); - + for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { + if (!esw->offloads.rep_ops[rep_type]) { + atomic_set(&rep->rep_data[rep_type].state, + REP_UNREGISTERED); + continue; + } + /* Dynamic/delegated vports add their representors after + * mlx5_eswitch_register_vport_reps, so mark them as registered + * for them to be loaded later with the others. + */ + rep->esw = esw; + atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); + } err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); if (err) goto insert_err; @@ -2425,7 +2455,7 @@ static int esw_offloads_init_reps(struct mlx5_eswitch *esw) xa_init(&esw->offloads.vport_reps); mlx5_esw_for_each_vport(esw, i, vport) { - err = mlx5_esw_offloads_rep_init(esw, vport); + err = mlx5_esw_offloads_rep_add(esw, vport); if (err) goto err; } @@ -3076,7 +3106,8 @@ err_out: return err; } -void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) +void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, + const struct mlx5_devcom_match_attr *attr) { int i; @@ -3095,7 +3126,7 @@ void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) esw->num_peers = 0; esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, MLX5_DEVCOM_ESW_OFFLOADS, - key, + attr, mlx5_esw_offloads_devcom_event, esw); if (IS_ERR(esw->devcom)) @@ -3533,6 +3564,8 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) int err; mutex_init(&esw->offloads.termtbl_mutex); + mlx5_esw_adjacent_vhcas_setup(esw); + err = mlx5_rdma_enable_roce(esw->dev); if (err) goto err_roce; @@ -3597,6 +3630,7 @@ err_vport_metadata: err_metadata: mlx5_rdma_disable_roce(esw->dev); err_roce: + mlx5_esw_adjacent_vhcas_cleanup(esw); mutex_destroy(&esw->offloads.termtbl_mutex); return err; } @@ -3630,6 +3664,7 @@ void esw_offloads_disable(struct mlx5_eswitch *esw) mapping_destroy(esw->offloads.reg_c0_obj_pool); esw_offloads_metadata_uninit(esw); mlx5_rdma_disable_roce(esw->dev); + mlx5_esw_adjacent_vhcas_cleanup(esw); mutex_destroy(&esw->offloads.termtbl_mutex); } @@ -3739,6 +3774,29 @@ void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) up_write(&esw->mode_lock); } +/* Returns false only when uplink netdev exists and its netns is different from + * devlink's netns. True for all others so entering switchdev mode is allowed. + */ +static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink, + bool immutable) +{ + struct mlx5_core_dev *mdev = devlink_priv(devlink); + struct net_device *netdev; + bool ret; + + netdev = mlx5_uplink_netdev_get(mdev); + if (!netdev) + return true; + + rtnl_lock(); + netdev->netns_immutable = immutable; + ret = net_eq(dev_net(netdev), devlink_net(devlink)); + rtnl_unlock(); + + mlx5_uplink_netdev_put(mdev, netdev); + return ret; +} + int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { @@ -3781,6 +3839,14 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, esw->eswitch_operation_in_progress = true; up_write(&esw->mode_lock); + if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && + !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) { + NL_SET_ERR_MSG_MOD(extack, + "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); + err = -EINVAL; + goto skip; + } + if (mode == DEVLINK_ESWITCH_MODE_LEGACY) esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_eswitch_disable_locked(esw); @@ -3799,6 +3865,8 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, } skip: + if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && err) + mlx5_devlink_netdev_netns_immutable_set(devlink, false); down_write(&esw->mode_lock); esw->eswitch_operation_in_progress = false; unlock: @@ -4059,7 +4127,8 @@ mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) { /* Currently, only ECPF based device has representor for host PF. */ if (vport_num == MLX5_VPORT_PF && - !mlx5_core_is_ecpf_esw_manager(esw->dev)) + (!mlx5_core_is_ecpf_esw_manager(esw->dev) || + !mlx5_esw_host_functions_enabled(esw->dev))) return false; if (vport_num == MLX5_VPORT_ECPF && @@ -4161,23 +4230,28 @@ u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, } EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); -int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) +int mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { u16 *old_entry, *vhca_map_entry, vhca_id; - int err; - err = mlx5_vport_get_vhca_id(esw->dev, vport_num, &vhca_id); - if (err) { - esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", - vport_num, err); - return err; + if (WARN_ONCE(MLX5_VPORT_INVAL_VHCA_ID(vport), + "vport %d vhca_id is not set", vport->vport)) { + int err; + + err = mlx5_vport_get_vhca_id(vport->dev, vport->vport, + &vhca_id); + if (err) + return err; + vport->vhca_id = vhca_id; } + vhca_id = vport->vhca_id; vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); if (!vhca_map_entry) return -ENOMEM; - *vhca_map_entry = vport_num; + *vhca_map_entry = vport->vport; old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); if (xa_is_err(old_entry)) { kfree(vhca_map_entry); @@ -4187,17 +4261,12 @@ int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) return 0; } -void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) +void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { - u16 *vhca_map_entry, vhca_id; - int err; - - err = mlx5_vport_get_vhca_id(esw->dev, vport_num, &vhca_id); - if (err) - esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", - vport_num, err); + u16 *vhca_map_entry; - vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); + vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vport->vhca_id); kfree(vhca_map_entry); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c index c4de6bf8d1b6..cb1319974f83 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c @@ -475,7 +475,6 @@ static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size) *conn->cq.mcq.arm_db = 0; conn->cq.mcq.vector = 0; conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete; - conn->cq.mcq.uar = fdev->conn_res.uar; tasklet_setup(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet); mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 80245c38dbad..2db3ffb0a2b2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -2793,30 +2793,32 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, } EXPORT_SYMBOL(mlx5_get_flow_namespace); +struct mlx5_vport_acl_root_ns { + u16 vport_idx; + struct mlx5_flow_root_namespace *root_ns; +}; + struct mlx5_flow_namespace * mlx5_get_flow_vport_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type, int vport_idx) { struct mlx5_flow_steering *steering = dev->priv.steering; + struct mlx5_vport_acl_root_ns *vport_ns; if (!steering) return NULL; switch (type) { case MLX5_FLOW_NAMESPACE_ESW_EGRESS: - if (vport_idx >= steering->esw_egress_acl_vports) - return NULL; - if (steering->esw_egress_root_ns && - steering->esw_egress_root_ns[vport_idx]) - return &steering->esw_egress_root_ns[vport_idx]->ns; + vport_ns = xa_load(&steering->esw_egress_root_ns, vport_idx); + if (vport_ns) + return &vport_ns->root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_ESW_INGRESS: - if (vport_idx >= steering->esw_ingress_acl_vports) - return NULL; - if (steering->esw_ingress_root_ns && - steering->esw_ingress_root_ns[vport_idx]) - return &steering->esw_ingress_root_ns[vport_idx]->ns; + vport_ns = xa_load(&steering->esw_ingress_root_ns, vport_idx); + if (vport_ns) + return &vport_ns->root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: @@ -3575,118 +3577,102 @@ out_err: return err; } -static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport) +static void +mlx5_fs_remove_vport_acl_root_ns(struct xarray *esw_acl_root_ns, u16 vport_idx) { - struct fs_prio *prio; - - steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL); - if (!steering->esw_egress_root_ns[vport]) - return -ENOMEM; + struct mlx5_vport_acl_root_ns *vport_ns; - /* create 1 prio*/ - prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1); - return PTR_ERR_OR_ZERO(prio); + vport_ns = xa_erase(esw_acl_root_ns, vport_idx); + if (vport_ns) { + cleanup_root_ns(vport_ns->root_ns); + kfree(vport_ns); + } } -static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport) +static int +mlx5_fs_add_vport_acl_root_ns(struct mlx5_flow_steering *steering, + struct xarray *esw_acl_root_ns, + enum fs_flow_table_type table_type, + u16 vport_idx) { + struct mlx5_vport_acl_root_ns *vport_ns; struct fs_prio *prio; + int err; - steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL); - if (!steering->esw_ingress_root_ns[vport]) - return -ENOMEM; + /* sanity check, intended xarrays are used */ + if (WARN_ON(esw_acl_root_ns != &steering->esw_egress_root_ns && + esw_acl_root_ns != &steering->esw_ingress_root_ns)) + return -EINVAL; - /* create 1 prio*/ - prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1); - return PTR_ERR_OR_ZERO(prio); -} + if (table_type != FS_FT_ESW_EGRESS_ACL && + table_type != FS_FT_ESW_INGRESS_ACL) { + mlx5_core_err(steering->dev, + "Invalid table type %d for egress/ingress ACLs\n", + table_type); + return -EINVAL; + } -int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports) -{ - struct mlx5_flow_steering *steering = dev->priv.steering; - int err; - int i; + if (xa_load(esw_acl_root_ns, vport_idx)) + return -EEXIST; - steering->esw_egress_root_ns = - kcalloc(total_vports, - sizeof(*steering->esw_egress_root_ns), - GFP_KERNEL); - if (!steering->esw_egress_root_ns) + vport_ns = kzalloc(sizeof(*vport_ns), GFP_KERNEL); + if (!vport_ns) return -ENOMEM; - for (i = 0; i < total_vports; i++) { - err = init_egress_acl_root_ns(steering, i); - if (err) - goto cleanup_root_ns; + vport_ns->root_ns = create_root_ns(steering, table_type); + if (!vport_ns->root_ns) { + err = -ENOMEM; + goto kfree_vport_ns; + } + + /* create 1 prio*/ + prio = fs_create_prio(&vport_ns->root_ns->ns, 0, 1); + if (IS_ERR(prio)) { + err = PTR_ERR(prio); + goto cleanup_root_ns; } - steering->esw_egress_acl_vports = total_vports; + + vport_ns->vport_idx = vport_idx; + err = xa_insert(esw_acl_root_ns, vport_idx, vport_ns, GFP_KERNEL); + if (err) + goto cleanup_root_ns; return 0; cleanup_root_ns: - for (i--; i >= 0; i--) - cleanup_root_ns(steering->esw_egress_root_ns[i]); - kfree(steering->esw_egress_root_ns); - steering->esw_egress_root_ns = NULL; + cleanup_root_ns(vport_ns->root_ns); +kfree_vport_ns: + kfree(vport_ns); return err; } -void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev) +int mlx5_fs_vport_egress_acl_ns_add(struct mlx5_flow_steering *steering, + u16 vport_idx) { - struct mlx5_flow_steering *steering = dev->priv.steering; - int i; - - if (!steering->esw_egress_root_ns) - return; - - for (i = 0; i < steering->esw_egress_acl_vports; i++) - cleanup_root_ns(steering->esw_egress_root_ns[i]); - - kfree(steering->esw_egress_root_ns); - steering->esw_egress_root_ns = NULL; + return mlx5_fs_add_vport_acl_root_ns(steering, + &steering->esw_egress_root_ns, + FS_FT_ESW_EGRESS_ACL, vport_idx); } -int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports) +int mlx5_fs_vport_ingress_acl_ns_add(struct mlx5_flow_steering *steering, + u16 vport_idx) { - struct mlx5_flow_steering *steering = dev->priv.steering; - int err; - int i; - - steering->esw_ingress_root_ns = - kcalloc(total_vports, - sizeof(*steering->esw_ingress_root_ns), - GFP_KERNEL); - if (!steering->esw_ingress_root_ns) - return -ENOMEM; - - for (i = 0; i < total_vports; i++) { - err = init_ingress_acl_root_ns(steering, i); - if (err) - goto cleanup_root_ns; - } - steering->esw_ingress_acl_vports = total_vports; - return 0; - -cleanup_root_ns: - for (i--; i >= 0; i--) - cleanup_root_ns(steering->esw_ingress_root_ns[i]); - kfree(steering->esw_ingress_root_ns); - steering->esw_ingress_root_ns = NULL; - return err; + return mlx5_fs_add_vport_acl_root_ns(steering, + &steering->esw_ingress_root_ns, + FS_FT_ESW_INGRESS_ACL, vport_idx); } -void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev) +void mlx5_fs_vport_egress_acl_ns_remove(struct mlx5_flow_steering *steering, + int vport_idx) { - struct mlx5_flow_steering *steering = dev->priv.steering; - int i; - - if (!steering->esw_ingress_root_ns) - return; - - for (i = 0; i < steering->esw_ingress_acl_vports; i++) - cleanup_root_ns(steering->esw_ingress_root_ns[i]); + mlx5_fs_remove_vport_acl_root_ns(&steering->esw_egress_root_ns, + vport_idx); +} - kfree(steering->esw_ingress_root_ns); - steering->esw_ingress_root_ns = NULL; +void mlx5_fs_vport_ingress_acl_ns_remove(struct mlx5_flow_steering *steering, + int vport_idx) +{ + mlx5_fs_remove_vport_acl_root_ns(&steering->esw_ingress_root_ns, + vport_idx); } u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type) @@ -3818,6 +3804,11 @@ void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev) { struct mlx5_flow_steering *steering = dev->priv.steering; + WARN_ON(!xa_empty(&steering->esw_egress_root_ns)); + WARN_ON(!xa_empty(&steering->esw_ingress_root_ns)); + xa_destroy(&steering->esw_egress_root_ns); + xa_destroy(&steering->esw_ingress_root_ns); + cleanup_root_ns(steering->root_ns); cleanup_fdb_root_ns(steering); cleanup_root_ns(steering->port_sel_root_ns); @@ -3908,6 +3899,8 @@ int mlx5_fs_core_init(struct mlx5_core_dev *dev) goto err; } + xa_init(&steering->esw_egress_root_ns); + xa_init(&steering->esw_ingress_root_ns); return 0; err: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index e6a95b310b55..8458ce203dac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -151,16 +151,14 @@ struct mlx5_flow_steering { struct mlx5_flow_root_namespace *root_ns; struct mlx5_flow_root_namespace *fdb_root_ns; struct mlx5_flow_namespace **fdb_sub_ns; - struct mlx5_flow_root_namespace **esw_egress_root_ns; - struct mlx5_flow_root_namespace **esw_ingress_root_ns; + struct xarray esw_egress_root_ns; + struct xarray esw_ingress_root_ns; struct mlx5_flow_root_namespace *sniffer_tx_root_ns; struct mlx5_flow_root_namespace *sniffer_rx_root_ns; struct mlx5_flow_root_namespace *rdma_rx_root_ns; struct mlx5_flow_root_namespace *rdma_tx_root_ns; struct mlx5_flow_root_namespace *egress_root_ns; struct mlx5_flow_root_namespace *port_sel_root_ns; - int esw_egress_acl_vports; - int esw_ingress_acl_vports; struct mlx5_flow_root_namespace **rdma_transport_rx_root_ns; struct mlx5_flow_root_namespace **rdma_transport_tx_root_ns; int rdma_transport_rx_vports; @@ -379,10 +377,14 @@ void mlx5_fs_core_free(struct mlx5_core_dev *dev); int mlx5_fs_core_init(struct mlx5_core_dev *dev); void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev); -int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports); -void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev); -int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports); -void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev); +int mlx5_fs_vport_egress_acl_ns_add(struct mlx5_flow_steering *steering, + u16 vport_idx); +int mlx5_fs_vport_ingress_acl_ns_add(struct mlx5_flow_steering *steering, + u16 vport_idx); +void mlx5_fs_vport_egress_acl_ns_remove(struct mlx5_flow_steering *steering, + int vport_idx); +void mlx5_fs_vport_ingress_acl_ns_remove(struct mlx5_flow_steering *steering, + int vport_idx); u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index 57476487e31f..eeb4437975f2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -294,6 +294,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN(dev, psp)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_PSP); + if (err) + return err; + } + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 22995131824a..89e399606877 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -27,6 +27,7 @@ struct mlx5_fw_reset { struct work_struct reset_reload_work; struct work_struct reset_now_work; struct work_struct reset_abort_work; + struct delayed_work reset_timeout_work; unsigned long reset_flags; u8 reset_method; struct timer_list timer; @@ -259,6 +260,8 @@ static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool return -EALREADY; } + if (current_work() != &fw_reset->reset_timeout_work.work) + cancel_delayed_work(&fw_reset->reset_timeout_work); mlx5_stop_sync_reset_poll(dev); if (poll_health) mlx5_start_health_poll(dev); @@ -330,6 +333,11 @@ static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev) } mlx5_stop_health_poll(dev, true); mlx5_start_sync_reset_poll(dev); + + if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, + &fw_reset->reset_flags)) + schedule_delayed_work(&fw_reset->reset_timeout_work, + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_SYNC_UPDATE))); return 0; } @@ -739,6 +747,19 @@ static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct } } +static void mlx5_sync_reset_timeout_work(struct work_struct *work) +{ + struct delayed_work *dwork = container_of(work, struct delayed_work, + work); + struct mlx5_fw_reset *fw_reset = + container_of(dwork, struct mlx5_fw_reset, reset_timeout_work); + struct mlx5_core_dev *dev = fw_reset->dev; + + if (mlx5_sync_reset_clear_reset_requested(dev, true)) + return; + mlx5_core_warn(dev, "PCI Sync FW Update Reset Timeout.\n"); +} + static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data) { struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb); @@ -822,6 +843,7 @@ void mlx5_drain_fw_reset(struct mlx5_core_dev *dev) cancel_work_sync(&fw_reset->reset_reload_work); cancel_work_sync(&fw_reset->reset_now_work); cancel_work_sync(&fw_reset->reset_abort_work); + cancel_delayed_work(&fw_reset->reset_timeout_work); } static const struct devlink_param mlx5_fw_reset_devlink_params[] = { @@ -865,6 +887,8 @@ int mlx5_fw_reset_init(struct mlx5_core_dev *dev) INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work); INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event); INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event); + INIT_DELAYED_WORK(&fw_reset->reset_timeout_work, + mlx5_sync_reset_timeout_work); init_completion(&fw_reset->done); return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index cf7a1edd0530..aeeb136f5ebc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -669,57 +669,64 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) } } +#define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000 +#define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000 +#define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000 +#define MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD \ + MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD + +static +const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ecpf_ops = { + .name = "fw_fatal", + .recover = mlx5_fw_fatal_reporter_recover, + .dump = mlx5_fw_fatal_reporter_dump, + .default_graceful_period = + MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD, +}; + static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_pf_ops = { .name = "fw_fatal", .recover = mlx5_fw_fatal_reporter_recover, .dump = mlx5_fw_fatal_reporter_dump, + .default_graceful_period = MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD, }; static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { .name = "fw_fatal", .recover = mlx5_fw_fatal_reporter_recover, + .default_graceful_period = + MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD, }; -#define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000 -#define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000 -#define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000 -#define MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD - void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) { const struct devlink_health_reporter_ops *fw_fatal_ops; struct mlx5_core_health *health = &dev->priv.health; const struct devlink_health_reporter_ops *fw_ops; struct devlink *devlink = priv_to_devlink(dev); - u64 grace_period; - fw_fatal_ops = &mlx5_fw_fatal_reporter_pf_ops; fw_ops = &mlx5_fw_reporter_pf_ops; if (mlx5_core_is_ecpf(dev)) { - grace_period = MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD; + fw_fatal_ops = &mlx5_fw_fatal_reporter_ecpf_ops; } else if (mlx5_core_is_pf(dev)) { - grace_period = MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD; + fw_fatal_ops = &mlx5_fw_fatal_reporter_pf_ops; } else { /* VF or SF */ - grace_period = MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD; fw_fatal_ops = &mlx5_fw_fatal_reporter_ops; fw_ops = &mlx5_fw_reporter_ops; } - health->fw_reporter = - devl_health_reporter_create(devlink, fw_ops, 0, dev); + health->fw_reporter = devl_health_reporter_create(devlink, fw_ops, dev); if (IS_ERR(health->fw_reporter)) - mlx5_core_warn(dev, "Failed to create fw reporter, err = %ld\n", - PTR_ERR(health->fw_reporter)); - - health->fw_fatal_reporter = - devl_health_reporter_create(devlink, - fw_fatal_ops, - grace_period, - dev); + mlx5_core_warn(dev, "Failed to create fw reporter, err = %pe\n", + health->fw_reporter); + + health->fw_fatal_reporter = devl_health_reporter_create(devlink, + fw_fatal_ops, + dev); if (IS_ERR(health->fw_fatal_reporter)) - mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %ld\n", - PTR_ERR(health->fw_fatal_reporter)); + mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %pe\n", + health->fw_fatal_reporter); } static void mlx5_fw_reporters_destroy(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c b/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c index 82d3c2568244..14d339eceb92 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c @@ -150,8 +150,8 @@ mlx5_irq_affinity_request(struct mlx5_core_dev *dev, struct mlx5_irq_pool *pool, if (IS_ERR(new_irq)) { if (!least_loaded_irq) { /* We failed to create an IRQ and we didn't find an IRQ */ - mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n", - PTR_ERR(new_irq)); + mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %pe\n", + new_irq); mutex_unlock(&pool->lock); return new_irq; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index d058cbb4a00c..59c00c911275 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -35,6 +35,7 @@ #include <linux/mlx5/driver.h> #include <linux/mlx5/eswitch.h> #include <linux/mlx5/vport.h> +#include "lib/mlx5.h" #include "lib/devcom.h" #include "mlx5_core.h" #include "eswitch.h" @@ -231,9 +232,13 @@ static void mlx5_do_bond_work(struct work_struct *work); static void mlx5_ldev_free(struct kref *ref) { struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref); + struct net *net; + + if (ldev->nb.notifier_call) { + net = read_pnet(&ldev->net); + unregister_netdevice_notifier_net(net, &ldev->nb); + } - if (ldev->nb.notifier_call) - unregister_netdevice_notifier_net(&init_net, &ldev->nb); mlx5_lag_mp_cleanup(ldev); cancel_delayed_work_sync(&ldev->bond_work); destroy_workqueue(ldev->wq); @@ -271,7 +276,8 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev) INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work); ldev->nb.notifier_call = mlx5_lag_netdev_event; - if (register_netdevice_notifier_net(&init_net, &ldev->nb)) { + write_pnet(&ldev->net, mlx5_core_net(dev)); + if (register_netdevice_notifier_net(read_pnet(&ldev->net), &ldev->nb)) { ldev->nb.notifier_call = NULL; mlx5_core_err(dev, "Failed to register LAG netdev notifier\n"); } @@ -1404,6 +1410,36 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) return 0; } +static void mlx5_lag_unregister_hca_devcom_comp(struct mlx5_core_dev *dev) +{ + mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp); +} + +static int mlx5_lag_register_hca_devcom_comp(struct mlx5_core_dev *dev) +{ + struct mlx5_devcom_match_attr attr = { + .key.val = mlx5_query_nic_system_image_guid(dev), + .flags = MLX5_DEVCOM_MATCH_FLAGS_NS, + .net = mlx5_core_net(dev), + }; + + /* This component is use to sync adding core_dev to lag_dev and to sync + * changes of mlx5_adev_devices between LAG layer and other layers. + */ + dev->priv.hca_devcom_comp = + mlx5_devcom_register_component(dev->priv.devc, + MLX5_DEVCOM_HCA_PORTS, + &attr, NULL, dev); + if (IS_ERR(dev->priv.hca_devcom_comp)) { + mlx5_core_err(dev, + "Failed to register devcom HCA component, err: %ld\n", + PTR_ERR(dev->priv.hca_devcom_comp)); + return PTR_ERR(dev->priv.hca_devcom_comp); + } + + return 0; +} + void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; @@ -1425,6 +1461,7 @@ recheck: } mlx5_ldev_remove_mdev(ldev, dev); mutex_unlock(&ldev->lock); + mlx5_lag_unregister_hca_devcom_comp(dev); mlx5_ldev_put(ldev); } @@ -1435,7 +1472,7 @@ void mlx5_lag_add_mdev(struct mlx5_core_dev *dev) if (!mlx5_lag_is_supported(dev)) return; - if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp)) + if (mlx5_lag_register_hca_devcom_comp(dev)) return; recheck: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index c2f256bb2bc2..4918eee2b3da 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -67,6 +67,7 @@ struct mlx5_lag { struct workqueue_struct *wq; struct delayed_work bond_work; struct notifier_block nb; + possible_net_t net; struct lag_mp lag_mp; struct mlx5_lag_port_sel port_sel; /* Protect lag fields/state changes */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c index 58bd749b5e4d..129725159a93 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -100,7 +100,7 @@ static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data) MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); - MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index); MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); @@ -129,7 +129,7 @@ static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node, return -ENOMEM; MLX5_SET(cqc, cqc_data, log_cq_size, 1); - MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.bfreg.up->index); if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD); @@ -163,7 +163,7 @@ static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node, struct mlx5_wq_param param; int err; - sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + sq->uar_map = mdev->priv.bfreg.map; param.db_numa_node = numa_node; param.buf_numa_node = numa_node; @@ -203,7 +203,7 @@ static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn, MLX5_SET(sqc, sqc, ts_format, ts_format); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); - MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); + MLX5_SET(wq, wq, uar_page, mdev->priv.bfreg.index); MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index 214d732d18e9..d0ba83d77cd1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -247,27 +247,24 @@ static bool mlx5_is_ptm_source_time_available(struct mlx5_core_dev *dev) return !!MLX5_GET(mtptm_reg, out, psta); } -static int mlx5_mtctr_syncdevicetime(ktime_t *device_time, - struct system_counterval_t *sys_counterval, - void *ctx) +static int mlx5_mtctr_read(struct mlx5_core_dev *mdev, + bool real_time_mode, + struct system_counterval_t *sys_counterval, + u64 *device) { u32 out[MLX5_ST_SZ_DW(mtctr_reg)] = {0}; u32 in[MLX5_ST_SZ_DW(mtctr_reg)] = {0}; - struct mlx5_core_dev *mdev = ctx; - bool real_time_mode; - u64 host, device; + u64 host; int err; - real_time_mode = mlx5_real_time_mode(mdev); - MLX5_SET(mtctr_reg, in, first_clock_timestamp_request, MLX5_MTCTR_REQUEST_PTM_ROOT_CLOCK); MLX5_SET(mtctr_reg, in, second_clock_timestamp_request, real_time_mode ? MLX5_MTCTR_REQUEST_REAL_TIME_CLOCK : - MLX5_MTCTR_REQUEST_FREE_RUNNING_COUNTER); + MLX5_MTCTR_REQUEST_FREE_RUNNING_COUNTER); - err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), MLX5_REG_MTCTR, - 0, 0); + err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), + MLX5_REG_MTCTR, 0, 0); if (err) return err; @@ -281,8 +278,26 @@ static int mlx5_mtctr_syncdevicetime(ktime_t *device_time, .cs_id = CSID_X86_ART, .use_nsecs = true, }; + *device = MLX5_GET64(mtctr_reg, out, second_clock_timestamp); + + return 0; +} + +static int mlx5_mtctr_syncdevicetime(ktime_t *device_time, + struct system_counterval_t *sys_counterval, + void *ctx) +{ + struct mlx5_core_dev *mdev = ctx; + bool real_time_mode; + u64 device; + int err; + + real_time_mode = mlx5_real_time_mode(mdev); + + err = mlx5_mtctr_read(mdev, real_time_mode, sys_counterval, &device); + if (err) + return err; - device = MLX5_GET64(mtctr_reg, out, second_clock_timestamp); if (real_time_mode) *device_time = ns_to_ktime(REAL_TIME_TO_NS(device >> 32, device & U32_MAX)); else @@ -291,6 +306,23 @@ static int mlx5_mtctr_syncdevicetime(ktime_t *device_time, return 0; } +static int +mlx5_mtctr_syncdevicecyclestime(ktime_t *device_time, + struct system_counterval_t *sys_counterval, + void *ctx) +{ + struct mlx5_core_dev *mdev = ctx; + u64 device; + int err; + + err = mlx5_mtctr_read(mdev, false, sys_counterval, &device); + if (err) + return err; + *device_time = ns_to_ktime(device); + + return 0; +} + static int mlx5_ptp_getcrosststamp(struct ptp_clock_info *ptp, struct system_device_crosststamp *cts) { @@ -315,6 +347,32 @@ unlock: mlx5_clock_unlock(clock); return err; } + +static int mlx5_ptp_getcrosscycles(struct ptp_clock_info *ptp, + struct system_device_crosststamp *cts) +{ + struct mlx5_clock *clock = + container_of(ptp, struct mlx5_clock, ptp_info); + struct system_time_snapshot history_begin = {0}; + struct mlx5_core_dev *mdev; + int err; + + mlx5_clock_lock(clock); + mdev = mlx5_clock_mdev_get(clock); + + if (!mlx5_is_ptm_source_time_available(mdev)) { + err = -EBUSY; + goto unlock; + } + + ktime_get_snapshot(&history_begin); + + err = get_device_system_crosststamp(mlx5_mtctr_syncdevicecyclestime, + mdev, &history_begin, cts); +unlock: + mlx5_clock_unlock(clock); + return err; +} #endif /* CONFIG_X86 */ static u64 mlx5_read_time(struct mlx5_core_dev *dev, @@ -513,6 +571,24 @@ out: return 0; } +static int mlx5_ptp_getcyclesx(struct ptp_clock_info *ptp, + struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, + ptp_info); + struct mlx5_core_dev *mdev; + u64 cycles; + + mlx5_clock_lock(clock); + mdev = mlx5_clock_mdev_get(clock); + + cycles = mlx5_read_time(mdev, sts, false); + *ts = ns_to_timespec64(cycles); + mlx5_clock_unlock(clock); + return 0; +} + static int mlx5_ptp_adjtime_real_time(struct mlx5_core_dev *mdev, s64 delta) { u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {}; @@ -1229,6 +1305,7 @@ static void mlx5_init_timer_max_freq_adjustment(struct mlx5_core_dev *mdev) static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) { struct mlx5_clock *clock = mdev->clock; + bool expose_cycles; /* Configure the PHC */ clock->ptp_info = mlx5_ptp_clock_info; @@ -1236,12 +1313,22 @@ static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) if (MLX5_CAP_MCAM_REG(mdev, mtutc)) mlx5_init_timer_max_freq_adjustment(mdev); + expose_cycles = !MLX5_CAP_GEN(mdev, disciplined_fr_counter) || + !mlx5_real_time_mode(mdev); + #ifdef CONFIG_X86 if (MLX5_CAP_MCAM_REG3(mdev, mtptm) && - MLX5_CAP_MCAM_REG3(mdev, mtctr) && boot_cpu_has(X86_FEATURE_ART)) + MLX5_CAP_MCAM_REG3(mdev, mtctr) && boot_cpu_has(X86_FEATURE_ART)) { clock->ptp_info.getcrosststamp = mlx5_ptp_getcrosststamp; + if (expose_cycles) + clock->ptp_info.getcrosscycles = + mlx5_ptp_getcrosscycles; + } #endif /* CONFIG_X86 */ + if (expose_cycles) + clock->ptp_info.getcyclesx64 = mlx5_ptp_getcyclesx; + mlx5_timecounter_init(mdev); mlx5_init_clock_info(mdev); mlx5_init_overflow_period(mdev); @@ -1278,9 +1365,9 @@ static void mlx5_init_clock_dev(struct mlx5_core_dev *mdev) clock->ptp = ptp_clock_register(&clock->ptp_info, clock->shared ? NULL : &mdev->pdev->dev); if (IS_ERR(clock->ptp)) { - mlx5_core_warn(mdev, "%sptp_clock_register failed %ld\n", + mlx5_core_warn(mdev, "%sptp_clock_register failed %pe\n", clock->shared ? "shared clock " : "", - PTR_ERR(clock->ptp)); + clock->ptp); clock->ptp = NULL; } @@ -1348,14 +1435,20 @@ static int mlx5_clock_alloc(struct mlx5_core_dev *mdev, bool shared) static void mlx5_shared_clock_register(struct mlx5_core_dev *mdev, u64 key) { struct mlx5_core_dev *peer_dev, *next = NULL; + struct mlx5_devcom_match_attr attr = { + .key.val = key, + }; + struct mlx5_devcom_comp_dev *compd; struct mlx5_devcom_comp_dev *pos; - mdev->clock_state->compdev = mlx5_devcom_register_component(mdev->priv.devc, - MLX5_DEVCOM_SHARED_CLOCK, - key, NULL, mdev); - if (IS_ERR(mdev->clock_state->compdev)) + compd = mlx5_devcom_register_component(mdev->priv.devc, + MLX5_DEVCOM_SHARED_CLOCK, + &attr, NULL, mdev); + if (IS_ERR(compd)) return; + mdev->clock_state->compdev = compd; + mlx5_devcom_comp_lock(mdev->clock_state->compdev); mlx5_devcom_for_each_peer_entry(mdev->clock_state->compdev, peer_dev, pos) { if (peer_dev->clock) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h index c819c047bb9c..4821163a547f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -8,6 +8,7 @@ enum { MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS, MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC, MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC, + MLX5_ACCEL_OBJ_PSP_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_PSP, MLX5_ACCEL_OBJ_TYPE_KEY_NUM, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 7b0766c89f4c..faa2833602c8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -4,6 +4,7 @@ #include <linux/mlx5/vport.h> #include <linux/list.h> #include "lib/devcom.h" +#include "lib/mlx5.h" #include "mlx5_core.h" static LIST_HEAD(devcom_dev_list); @@ -22,11 +23,17 @@ struct mlx5_devcom_dev { struct kref ref; }; +struct mlx5_devcom_key { + u32 flags; + union mlx5_devcom_match_key key; + possible_net_t net; +}; + struct mlx5_devcom_comp { struct list_head comp_list; enum mlx5_devcom_component id; - u64 key; struct list_head comp_dev_list_head; + struct mlx5_devcom_key key; mlx5_devcom_event_handler_t handler; struct kref ref; bool ready; @@ -108,7 +115,8 @@ void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc) } static struct mlx5_devcom_comp * -mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler) +mlx5_devcom_comp_alloc(u64 id, const struct mlx5_devcom_match_attr *attr, + mlx5_devcom_event_handler_t handler) { struct mlx5_devcom_comp *comp; @@ -117,7 +125,10 @@ mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler) return ERR_PTR(-ENOMEM); comp->id = id; - comp->key = key; + comp->key.key = attr->key; + comp->key.flags = attr->flags; + if (attr->flags & MLX5_DEVCOM_MATCH_FLAGS_NS) + write_pnet(&comp->key.net, attr->net); comp->handler = handler; init_rwsem(&comp->sem); lockdep_register_key(&comp->lock_key); @@ -180,21 +191,34 @@ devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) static bool devcom_component_equal(struct mlx5_devcom_comp *devcom, enum mlx5_devcom_component id, - u64 key) + const struct mlx5_devcom_match_attr *attr) { - return devcom->id == id && devcom->key == key; + if (devcom->id != id) + return false; + + if (devcom->key.flags != attr->flags) + return false; + + if (memcmp(&devcom->key.key, &attr->key, sizeof(devcom->key.key))) + return false; + + if (devcom->key.flags & MLX5_DEVCOM_MATCH_FLAGS_NS && + !net_eq(read_pnet(&devcom->key.net), attr->net)) + return false; + + return true; } static struct mlx5_devcom_comp * devcom_component_get(struct mlx5_devcom_dev *devc, enum mlx5_devcom_component id, - u64 key, + const struct mlx5_devcom_match_attr *attr, mlx5_devcom_event_handler_t handler) { struct mlx5_devcom_comp *comp; devcom_for_each_component(comp) { - if (devcom_component_equal(comp, id, key)) { + if (devcom_component_equal(comp, id, attr)) { if (handler == comp->handler) { kref_get(&comp->ref); return comp; @@ -212,7 +236,7 @@ devcom_component_get(struct mlx5_devcom_dev *devc, struct mlx5_devcom_comp_dev * mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, enum mlx5_devcom_component id, - u64 key, + const struct mlx5_devcom_match_attr *attr, mlx5_devcom_event_handler_t handler, void *data) { @@ -223,14 +247,14 @@ mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, return ERR_PTR(-EINVAL); mutex_lock(&comp_list_lock); - comp = devcom_component_get(devc, id, key, handler); + comp = devcom_component_get(devc, id, attr, handler); if (IS_ERR(comp)) { devcom = ERR_PTR(-EINVAL); goto out_unlock; } if (!comp) { - comp = mlx5_devcom_comp_alloc(id, key, handler); + comp = mlx5_devcom_comp_alloc(id, attr, handler); if (IS_ERR(comp)) { devcom = ERR_CAST(comp); goto out_unlock; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index c79699b94a02..609c85f47917 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -6,6 +6,20 @@ #include <linux/mlx5/driver.h> +enum mlx5_devom_match_flags { + MLX5_DEVCOM_MATCH_FLAGS_NS = BIT(0), +}; + +union mlx5_devcom_match_key { + u64 val; +}; + +struct mlx5_devcom_match_attr { + u32 flags; + union mlx5_devcom_match_key key; + struct net *net; +}; + enum mlx5_devcom_component { MLX5_DEVCOM_ESW_OFFLOADS, MLX5_DEVCOM_MPV, @@ -25,7 +39,7 @@ void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc); struct mlx5_devcom_comp_dev * mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, enum mlx5_devcom_component id, - u64 key, + const struct mlx5_devcom_match_attr *attr, mlx5_devcom_event_handler_t handler, void *data); void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c index ca9ecec358b2..7adad784ad46 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c @@ -9,7 +9,7 @@ #include "mlx5_core.h" #include "lib/fs_ttc.h" -#define MLX5_TTC_MAX_NUM_GROUPS 4 +#define MLX5_TTC_MAX_NUM_GROUPS 7 #define MLX5_TTC_GROUP_TCPUDP_SIZE (MLX5_TT_IPV6_UDP + 1) struct mlx5_fs_ttc_groups { @@ -31,10 +31,14 @@ static int mlx5_fs_ttc_table_size(const struct mlx5_fs_ttc_groups *groups) /* L3/L4 traffic type classifier */ struct mlx5_ttc_table { int num_groups; + const struct mlx5_fs_ttc_groups *groups; + struct mlx5_core_dev *mdev; struct mlx5_flow_table *t; struct mlx5_flow_group **g; struct mlx5_ttc_rule rules[MLX5_NUM_TT]; struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT]; + u32 refcnt; + struct mutex mutex; /* Protect adding rules for ipsec crypto offload */ }; struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc) @@ -163,6 +167,8 @@ static struct mlx5_etype_proto ttc_tunnel_rules[] = { enum TTC_GROUP_TYPE { TTC_GROUPS_DEFAULT = 0, TTC_GROUPS_USE_L4_TYPE = 1, + TTC_GROUPS_DEFAULT_ESP = 2, + TTC_GROUPS_USE_L4_TYPE_ESP = 3, }; static const struct mlx5_fs_ttc_groups ttc_groups[] = { @@ -184,6 +190,31 @@ static const struct mlx5_fs_ttc_groups ttc_groups[] = { BIT(0), }, }, + [TTC_GROUPS_DEFAULT_ESP] = { + .num_groups = 6, + .group_size = { + MLX5_TTC_GROUP_TCPUDP_SIZE + BIT(1) + + MLX5_NUM_TUNNEL_TT, + BIT(2), /* decrypted outer L4 */ + BIT(2), /* decrypted inner L4 */ + BIT(1), /* ESP */ + BIT(1), + BIT(0), + }, + }, + [TTC_GROUPS_USE_L4_TYPE_ESP] = { + .use_l4_type = true, + .num_groups = 7, + .group_size = { + MLX5_TTC_GROUP_TCPUDP_SIZE, + BIT(1) + MLX5_NUM_TUNNEL_TT, + BIT(2), /* decrypted outer L4 */ + BIT(2), /* decrypted inner L4 */ + BIT(1), /* ESP */ + BIT(1), + BIT(0), + }, + }, }; static const struct mlx5_fs_ttc_groups inner_ttc_groups[] = { @@ -207,6 +238,23 @@ static const struct mlx5_fs_ttc_groups inner_ttc_groups[] = { }, }; +static const struct mlx5_fs_ttc_groups * +mlx5_ttc_get_fs_groups(bool use_l4_type, bool ipsec_rss) +{ + if (!ipsec_rss) + return use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE] : + &ttc_groups[TTC_GROUPS_DEFAULT]; + + return use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE_ESP] : + &ttc_groups[TTC_GROUPS_DEFAULT_ESP]; +} + +bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc) +{ + return ttc->groups == &ttc_groups[TTC_GROUPS_DEFAULT_ESP] || + ttc->groups == &ttc_groups[TTC_GROUPS_USE_L4_TYPE_ESP]; +} + u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt) { return ttc_tunnel_rules[tt].proto; @@ -257,6 +305,31 @@ static u8 mlx5_etype_to_ipv(u16 ethertype) return 0; } +static void mlx5_fs_ttc_set_match_ipv_outer(struct mlx5_core_dev *mdev, + struct mlx5_flow_spec *spec, + u16 etype) +{ + int match_ipv_outer = + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, + ft_field_support.outer_ip_version); + u8 ipv; + + ipv = mlx5_etype_to_ipv(etype); + if (match_ipv_outer && ipv) { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.ip_version, ipv); + } else { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.ethertype); + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.ethertype, etype); + } + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; +} + static void mlx5_fs_ttc_set_match_proto(void *headers_c, void *headers_v, u8 proto, bool use_l4_type) { @@ -279,16 +352,12 @@ static void mlx5_fs_ttc_set_match_proto(void *headers_c, void *headers_v, static struct mlx5_flow_handle * mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, struct mlx5_flow_destination *dest, u16 etype, u8 proto, - bool use_l4_type) + bool use_l4_type, bool ipsec_rss) { - int match_ipv_outer = - MLX5_CAP_FLOWTABLE_NIC_RX(dev, - ft_field_support.outer_ip_version); MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; int err = 0; - u8 ipv; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) @@ -305,15 +374,15 @@ mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, proto, use_l4_type); } - ipv = mlx5_etype_to_ipv(etype); - if (match_ipv_outer && ipv) { - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv); - } else if (etype) { - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype); + if (etype) + mlx5_fs_ttc_set_match_ipv_outer(dev, spec, etype); + + if (ipsec_rss && proto == IPPROTO_ESP) { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + misc_parameters_2.ipsec_next_header); + MLX5_SET(fte_match_param, spec->match_value, + misc_parameters_2.ipsec_next_header, 0); + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; } rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); @@ -342,12 +411,16 @@ static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev, for (tt = 0; tt < MLX5_NUM_TT; tt++) { struct mlx5_ttc_rule *rule = &rules[tt]; + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + if (test_bit(tt, params->ignore_dests)) continue; rule->rule = mlx5_generate_ttc_rule(dev, ft, ¶ms->dests[tt], ttc_rules[tt].etype, ttc_rules[tt].proto, - use_l4_type); + use_l4_type, + params->ipsec_rss); if (IS_ERR(rule->rule)) { err = PTR_ERR(rule->rule); rule->rule = NULL; @@ -370,7 +443,7 @@ static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev, ¶ms->tunnel_dests[tt], ttc_tunnel_rules[tt].etype, ttc_tunnel_rules[tt].proto, - use_l4_type); + use_l4_type, false); if (IS_ERR(trules[tt])) { err = PTR_ERR(trules[tt]); trules[tt] = NULL; @@ -385,10 +458,78 @@ del_rules: return err; } +static int mlx5_create_ttc_table_ipsec_groups(struct mlx5_ttc_table *ttc, + bool use_ipv, + u32 *in, int *next_ix) +{ + u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + const struct mlx5_fs_ttc_groups *groups = ttc->groups; + int ix = *next_ix; + + MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0); + + /* decrypted ESP outer group */ + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.l4_type_ext); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += groups->group_size[ttc->num_groups]; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in); + if (IS_ERR(ttc->g[ttc->num_groups])) + goto err; + ttc->num_groups++; + + MLX5_SET(fte_match_param, mc, outer_headers.l4_type_ext, 0); + + /* decrypted ESP inner group */ + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS); + if (use_ipv) + MLX5_SET(fte_match_param, mc, outer_headers.ip_version, 0); + else + MLX5_SET(fte_match_param, mc, outer_headers.ethertype, 0); + MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version); + MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.l4_type_ext); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += groups->group_size[ttc->num_groups]; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in); + if (IS_ERR(ttc->g[ttc->num_groups])) + goto err; + ttc->num_groups++; + + MLX5_SET(fte_match_param, mc, inner_headers.ip_version, 0); + MLX5_SET(fte_match_param, mc, inner_headers.l4_type_ext, 0); + + /* undecrypted ESP group */ + MLX5_SET_CFG(in, match_criteria_enable, + MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2); + if (use_ipv) + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version); + else + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); + MLX5_SET_TO_ONES(fte_match_param, mc, + misc_parameters_2.ipsec_next_header); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += groups->group_size[ttc->num_groups]; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in); + if (IS_ERR(ttc->g[ttc->num_groups])) + goto err; + ttc->num_groups++; + + *next_ix = ix; + + return 0; + +err: + return PTR_ERR(ttc->g[ttc->num_groups]); +} + static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc, - bool use_ipv, - const struct mlx5_fs_ttc_groups *groups) + bool use_ipv) { + const struct mlx5_fs_ttc_groups *groups = ttc->groups; int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); int ix = 0; u32 *in; @@ -436,8 +577,18 @@ static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc, goto err; ttc->num_groups++; + if (mlx5_ttc_has_esp_flow_group(ttc)) { + err = mlx5_create_ttc_table_ipsec_groups(ttc, use_ipv, in, &ix); + if (err) + goto err; + + MLX5_SET(fte_match_param, mc, + misc_parameters_2.ipsec_next_header, 0); + } + /* L3 Group */ MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0); + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); MLX5_SET_CFG(in, start_flow_index, ix); ix += groups->group_size[ttc->num_groups]; MLX5_SET_CFG(in, end_flow_index, ix - 1); @@ -527,6 +678,9 @@ static int mlx5_generate_inner_ttc_table_rules(struct mlx5_core_dev *dev, for (tt = 0; tt < MLX5_NUM_TT; tt++) { struct mlx5_ttc_rule *rule = &rules[tt]; + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + if (test_bit(tt, params->ignore_dests)) continue; rule->rule = mlx5_generate_inner_ttc_rule(dev, ft, @@ -700,6 +854,7 @@ void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc) kfree(ttc->g); mlx5_destroy_flow_table(ttc->t); + mutex_destroy(&ttc->mutex); kvfree(ttc); } @@ -709,7 +864,6 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, bool match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_field_support.outer_ip_version); - const struct mlx5_fs_ttc_groups *groups; struct mlx5_flow_namespace *ns; struct mlx5_ttc_table *ttc; bool use_l4_type; @@ -738,11 +892,10 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, return ERR_PTR(-EOPNOTSUPP); } - groups = use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE] : - &ttc_groups[TTC_GROUPS_DEFAULT]; + ttc->groups = mlx5_ttc_get_fs_groups(use_l4_type, params->ipsec_rss); WARN_ON_ONCE(params->ft_attr.max_fte); - params->ft_attr.max_fte = mlx5_fs_ttc_table_size(groups); + params->ft_attr.max_fte = mlx5_fs_ttc_table_size(ttc->groups); ttc->t = mlx5_create_flow_table(ns, ¶ms->ft_attr); if (IS_ERR(ttc->t)) { err = PTR_ERR(ttc->t); @@ -750,7 +903,7 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, return ERR_PTR(err); } - err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer, groups); + err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer); if (err) goto destroy_ft; @@ -758,6 +911,9 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, if (err) goto destroy_ft; + ttc->mdev = dev; + mutex_init(&ttc->mutex); + return ttc; destroy_ft: @@ -791,3 +947,194 @@ int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc, return mlx5_ttc_fwd_dest(ttc, type, &dest); } + +static void _mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc) +{ + enum mlx5_traffic_types i; + + for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP; + i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) { + if (!ttc->rules[i].rule) + continue; + + mlx5_del_flow_rules(ttc->rules[i].rule); + ttc->rules[i].rule = NULL; + } +} + +void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc) +{ + if (!mlx5_ttc_has_esp_flow_group(ttc)) + return; + + mutex_lock(&ttc->mutex); + if (--ttc->refcnt) + goto unlock; + + _mlx5_ttc_destroy_ipsec_rules(ttc); +unlock: + mutex_unlock(&ttc->mutex); +} + +static int mlx5_ttc_get_tt_attrs(enum mlx5_traffic_types type, + u16 *etype, int *l4_type_ext, + enum mlx5_traffic_types *tir_tt) +{ + switch (type) { + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP: + *etype = ETH_P_IP; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP; + *tir_tt = MLX5_TT_IPV4_TCP; + break; + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP: + *etype = ETH_P_IPV6; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP; + *tir_tt = MLX5_TT_IPV6_TCP; + break; + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP: + *etype = ETH_P_IP; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP; + *tir_tt = MLX5_TT_IPV4_UDP; + break; + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP: + *etype = ETH_P_IPV6; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP; + *tir_tt = MLX5_TT_IPV6_UDP; + break; + default: + return -EINVAL; + } + + return 0; +} + +static struct mlx5_flow_handle * +mlx5_ttc_create_ipsec_outer_rule(struct mlx5_ttc_table *ttc, + enum mlx5_traffic_types type) +{ + struct mlx5_flow_destination dest; + MLX5_DECLARE_FLOW_ACT(flow_act); + enum mlx5_traffic_types tir_tt; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int l4_type_ext; + u16 etype; + int err; + + err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt); + if (err) + return ERR_PTR(err); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + mlx5_fs_ttc_set_match_ipv_outer(ttc->mdev, spec, etype); + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.l4_type_ext); + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.l4_type_ext, l4_type_ext); + + dest = mlx5_ttc_get_default_dest(ttc, tir_tt); + + rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__); + } + + kvfree(spec); + return err ? ERR_PTR(err) : rule; +} + +static struct mlx5_flow_handle * +mlx5_ttc_create_ipsec_inner_rule(struct mlx5_ttc_table *ttc, + struct mlx5_ttc_table *inner_ttc, + enum mlx5_traffic_types type) +{ + struct mlx5_flow_destination dest; + MLX5_DECLARE_FLOW_ACT(flow_act); + enum mlx5_traffic_types tir_tt; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int l4_type_ext; + u16 etype; + int err; + + err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt); + if (err) + return ERR_PTR(err); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + inner_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, + inner_headers.ip_version, mlx5_etype_to_ipv(etype)); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + inner_headers.l4_type_ext); + MLX5_SET(fte_match_param, spec->match_value, + inner_headers.l4_type_ext, l4_type_ext); + + dest = mlx5_ttc_get_default_dest(inner_ttc, tir_tt); + + spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS; + + rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__); + } + + kvfree(spec); + return err ? ERR_PTR(err) : rule; +} + +int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc, + struct mlx5_ttc_table *inner_ttc) +{ + struct mlx5_flow_handle *rule; + enum mlx5_traffic_types i; + + if (!mlx5_ttc_has_esp_flow_group(ttc)) + return 0; + + mutex_lock(&ttc->mutex); + if (ttc->refcnt) + goto skip; + + for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP; + i <= MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP; i++) { + rule = mlx5_ttc_create_ipsec_outer_rule(ttc, i); + if (IS_ERR(rule)) + goto err_out; + + ttc->rules[i].rule = rule; + } + + for (i = MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP; + i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) { + rule = mlx5_ttc_create_ipsec_inner_rule(ttc, inner_ttc, i); + if (IS_ERR(rule)) + goto err_out; + + ttc->rules[i].rule = rule; + } + +skip: + ttc->refcnt++; + mutex_unlock(&ttc->mutex); + return 0; + +err_out: + _mlx5_ttc_destroy_ipsec_rules(ttc); + mutex_unlock(&ttc->mutex); + return PTR_ERR(rule); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h index ab9434fe3ae6..95f6e56724a2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h @@ -18,6 +18,14 @@ enum mlx5_traffic_types { MLX5_TT_IPV4, MLX5_TT_IPV6, MLX5_TT_ANY, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP, MLX5_NUM_TT, MLX5_NUM_INDIR_TIRS = MLX5_TT_ANY, }; @@ -47,6 +55,7 @@ struct ttc_params { bool inner_ttc; DECLARE_BITMAP(ignore_tunnel_dests, MLX5_NUM_TUNNEL_TT); struct mlx5_flow_destination tunnel_dests[MLX5_NUM_TUNNEL_TT]; + bool ipsec_rss; }; const char *mlx5_ttc_get_name(enum mlx5_traffic_types tt); @@ -70,4 +79,14 @@ int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc, bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev); u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt); +bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc); +int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc, + struct mlx5_ttc_table *inner_ttc); +void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc); +static inline bool mlx5_ttc_is_decrypted_esp_tt(enum mlx5_traffic_types tt) +{ + return tt >= MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP && + tt <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; +} + #endif /* __MLX5_FS_TTC_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c index b7d4b1a2baf2..d524f0220513 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c @@ -164,6 +164,8 @@ ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev, roce->rule = rule; memset(spec, 0, sizeof(*spec)); + if (default_dst->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) + flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -178,6 +180,8 @@ ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev, goto out; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + if (default_dst->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) + flow_act.flags &= ~FLOW_ACT_IGNORE_FLOW_LEVEL; dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; dst.ft = roce->ft_rdma; rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.c index 762d55ba9e51..e6be2f01daf4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.c @@ -45,11 +45,7 @@ #define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8 #define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN) -/* MACsec RX flow steering */ -#define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E - /* MACsec fs_id handling for steering */ -#define macsec_fs_set_tx_fs_id(fs_id) (MLX5_ETH_WQE_FT_META_MACSEC | (fs_id) << 2) #define macsec_fs_set_rx_fs_id(fs_id) ((fs_id) | BIT(30)) struct mlx5_sectag_header { @@ -597,7 +593,7 @@ static int macsec_fs_tx_setup_fte(struct mlx5_macsec_fs *macsec_fs, MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_MACSEC_MASK); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, - macsec_fs_set_tx_fs_id(id)); + MLX5_MACSEC_TX_METADATA(id)); *fs_id = id; flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; @@ -2219,9 +2215,11 @@ static int mlx5_macsec_fs_add_roce_rule_tx(struct mlx5_macsec_fs *macsec_fs, u32 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_A); - MLX5_SET(set_action_in, action, data, macsec_fs_set_tx_fs_id(fs_id)); - MLX5_SET(set_action_in, action, offset, 0); - MLX5_SET(set_action_in, action, length, 32); + MLX5_SET(set_action_in, action, data, + mlx5_macsec_fs_set_tx_fs_id(fs_id)); + MLX5_SET(set_action_in, action, offset, + MLX5_ETH_WQE_FT_META_MACSEC_SHIFT); + MLX5_SET(set_action_in, action, length, 8); modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC, 1, action); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.h index 34b80c3ef6a5..15acaff43641 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/macsec_fs.h @@ -12,6 +12,21 @@ #define MLX5_MACSEC_METADATA_MARKER(metadata) ((((metadata) >> 30) & 0x3) == 0x1) #define MLX5_MACSEC_RX_METADAT_HANDLE(metadata) ((metadata) & MLX5_MACSEC_RX_FS_ID_MASK) +/* MACsec TX flow steering */ +#define MLX5_ETH_WQE_FT_META_MACSEC_MASK \ + (MLX5_ETH_WQE_FT_META_MACSEC | MLX5_ETH_WQE_FT_META_MACSEC_FS_ID_MASK) +#define MLX5_ETH_WQE_FT_META_MACSEC_SHIFT MLX5_ETH_WQE_FT_META_SHIFT + +/* MACsec fs_id handling for steering */ +#define mlx5_macsec_fs_set_tx_fs_id(fs_id) \ + (((MLX5_ETH_WQE_FT_META_MACSEC) >> MLX5_ETH_WQE_FT_META_MACSEC_SHIFT) \ + | ((fs_id) << 2)) + +#define MLX5_MACSEC_TX_METADATA(fs_id) \ + (mlx5_macsec_fs_set_tx_fs_id(fs_id) << \ + MLX5_ETH_WQE_FT_META_MACSEC_SHIFT) + +/* MACsec fs_id uses 4 bits, supports up to 16 interfaces */ #define MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES 16 struct mlx5_macsec_fs; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/nv_param.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/nv_param.c new file mode 100644 index 000000000000..459a0b4d08e6 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/nv_param.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include "nv_param.h" +#include "mlx5_core.h" + +enum { + MLX5_CLASS_0_CTRL_ID_NV_GLOBAL_PCI_CONF = 0x80, + MLX5_CLASS_0_CTRL_ID_NV_GLOBAL_PCI_CAP = 0x81, + MLX5_CLASS_0_CTRL_ID_NV_SW_OFFLOAD_CONFIG = 0x10a, + + MLX5_CLASS_3_CTRL_ID_NV_PF_PCI_CONF = 0x80, +}; + +struct mlx5_ifc_configuration_item_type_class_global_bits { + u8 type_class[0x8]; + u8 parameter_index[0x18]; +}; + +struct mlx5_ifc_configuration_item_type_class_per_host_pf_bits { + u8 type_class[0x8]; + u8 pf_index[0x6]; + u8 pci_bus_index[0x8]; + u8 parameter_index[0xa]; +}; + +union mlx5_ifc_config_item_type_auto_bits { + struct mlx5_ifc_configuration_item_type_class_global_bits + configuration_item_type_class_global; + struct mlx5_ifc_configuration_item_type_class_per_host_pf_bits + configuration_item_type_class_per_host_pf; + u8 reserved_at_0[0x20]; +}; + +struct mlx5_ifc_config_item_bits { + u8 valid[0x2]; + u8 priority[0x2]; + u8 header_type[0x2]; + u8 ovr_en[0x1]; + u8 rd_en[0x1]; + u8 access_mode[0x2]; + u8 reserved_at_a[0x1]; + u8 writer_id[0x5]; + u8 version[0x4]; + u8 reserved_at_14[0x2]; + u8 host_id_valid[0x1]; + u8 length[0x9]; + + union mlx5_ifc_config_item_type_auto_bits type; + + u8 reserved_at_40[0x10]; + u8 crc16[0x10]; +}; + +struct mlx5_ifc_mnvda_reg_bits { + struct mlx5_ifc_config_item_bits configuration_item_header; + + u8 configuration_item_data[64][0x20]; +}; + +struct mlx5_ifc_nv_global_pci_conf_bits { + u8 sriov_valid[0x1]; + u8 reserved_at_1[0x10]; + u8 per_pf_total_vf[0x1]; + u8 reserved_at_12[0xe]; + + u8 sriov_en[0x1]; + u8 reserved_at_21[0xf]; + u8 total_vfs[0x10]; + + u8 reserved_at_40[0x20]; +}; + +struct mlx5_ifc_nv_global_pci_cap_bits { + u8 max_vfs_per_pf_valid[0x1]; + u8 reserved_at_1[0x13]; + u8 per_pf_total_vf_supported[0x1]; + u8 reserved_at_15[0xb]; + + u8 sriov_support[0x1]; + u8 reserved_at_21[0xf]; + u8 max_vfs_per_pf[0x10]; + + u8 reserved_at_40[0x60]; +}; + +struct mlx5_ifc_nv_pf_pci_conf_bits { + u8 reserved_at_0[0x9]; + u8 pf_total_vf_en[0x1]; + u8 reserved_at_a[0x16]; + + u8 reserved_at_20[0x20]; + + u8 reserved_at_40[0x10]; + u8 total_vf[0x10]; + + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_nv_sw_offload_conf_bits { + u8 ip_over_vxlan_port[0x10]; + u8 tunnel_ecn_copy_offload_disable[0x1]; + u8 pci_atomic_mode[0x3]; + u8 sr_enable[0x1]; + u8 ptp_cyc2realtime[0x1]; + u8 vector_calc_disable[0x1]; + u8 uctx_en[0x1]; + u8 prio_tag_required_en[0x1]; + u8 esw_fdb_ipv4_ttl_modify_enable[0x1]; + u8 mkey_by_name[0x1]; + u8 ip_over_vxlan_en[0x1]; + u8 one_qp_per_recovery[0x1]; + u8 cqe_compression[0x3]; + u8 tunnel_udp_entropy_proto_disable[0x1]; + u8 reserved_at_21[0x1]; + u8 ar_enable[0x1]; + u8 log_max_outstanding_wqe[0x5]; + u8 vf_migration[0x2]; + u8 log_tx_psn_win[0x6]; + u8 lro_log_timeout3[0x4]; + u8 lro_log_timeout2[0x4]; + u8 lro_log_timeout1[0x4]; + u8 lro_log_timeout0[0x4]; +}; + +#define MNVDA_HDR_SZ \ + (MLX5_ST_SZ_BYTES(mnvda_reg) - \ + MLX5_BYTE_OFF(mnvda_reg, configuration_item_data)) + +#define MLX5_SET_CFG_ITEM_TYPE(_cls_name, _mnvda_ptr, _field, _val) \ + MLX5_SET(mnvda_reg, _mnvda_ptr, \ + configuration_item_header.type.configuration_item_type_class_##_cls_name._field, \ + _val) + +#define MLX5_SET_CFG_HDR_LEN(_mnvda_ptr, _cls_name) \ + MLX5_SET(mnvda_reg, _mnvda_ptr, configuration_item_header.length, \ + MLX5_ST_SZ_BYTES(_cls_name)) + +#define MLX5_GET_CFG_HDR_LEN(_mnvda_ptr) \ + MLX5_GET(mnvda_reg, _mnvda_ptr, configuration_item_header.length) + +static int mlx5_nv_param_read(struct mlx5_core_dev *dev, void *mnvda, + size_t len) +{ + u32 param_idx, type_class; + u32 header_len; + void *cls_ptr; + int err; + + if (WARN_ON(len > MLX5_ST_SZ_BYTES(mnvda_reg)) || len < MNVDA_HDR_SZ) + return -EINVAL; /* A caller bug */ + + err = mlx5_core_access_reg(dev, mnvda, len, mnvda, len, MLX5_REG_MNVDA, + 0, 0); + if (!err) + return 0; + + cls_ptr = MLX5_ADDR_OF(mnvda_reg, mnvda, + configuration_item_header.type.configuration_item_type_class_global); + + type_class = MLX5_GET(configuration_item_type_class_global, cls_ptr, + type_class); + param_idx = MLX5_GET(configuration_item_type_class_global, cls_ptr, + parameter_index); + header_len = MLX5_GET_CFG_HDR_LEN(mnvda); + + mlx5_core_warn(dev, "Failed to read mnvda reg: type_class 0x%x, param_idx 0x%x, header_len %u, err %d\n", + type_class, param_idx, header_len, err); + + return -EOPNOTSUPP; +} + +static int mlx5_nv_param_write(struct mlx5_core_dev *dev, void *mnvda, + size_t len) +{ + if (WARN_ON(len > MLX5_ST_SZ_BYTES(mnvda_reg)) || len < MNVDA_HDR_SZ) + return -EINVAL; + + if (WARN_ON(MLX5_GET_CFG_HDR_LEN(mnvda) == 0)) + return -EINVAL; + + return mlx5_core_access_reg(dev, mnvda, len, mnvda, len, MLX5_REG_MNVDA, + 0, 1); +} + +static int +mlx5_nv_param_read_sw_offload_conf(struct mlx5_core_dev *dev, void *mnvda, + size_t len) +{ + MLX5_SET_CFG_ITEM_TYPE(global, mnvda, type_class, 0); + MLX5_SET_CFG_ITEM_TYPE(global, mnvda, parameter_index, + MLX5_CLASS_0_CTRL_ID_NV_SW_OFFLOAD_CONFIG); + MLX5_SET_CFG_HDR_LEN(mnvda, nv_sw_offload_conf); + + return mlx5_nv_param_read(dev, mnvda, len); +} + +static const char *const + cqe_compress_str[] = { "balanced", "aggressive" }; + +static int +mlx5_nv_param_devlink_cqe_compress_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 mnvda[MLX5_ST_SZ_DW(mnvda_reg)] = {}; + u8 value = U8_MAX; + void *data; + int err; + + err = mlx5_nv_param_read_sw_offload_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + value = MLX5_GET(nv_sw_offload_conf, data, cqe_compression); + + if (value >= ARRAY_SIZE(cqe_compress_str)) + return -EOPNOTSUPP; + + strscpy(ctx->val.vstr, cqe_compress_str[value], sizeof(ctx->val.vstr)); + return 0; +} + +static int +mlx5_nv_param_devlink_cqe_compress_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(cqe_compress_str); i++) { + if (!strcmp(val.vstr, cqe_compress_str[i])) + return 0; + } + + NL_SET_ERR_MSG_MOD(extack, + "Invalid value, supported values are balanced/aggressive"); + return -EOPNOTSUPP; +} + +static int +mlx5_nv_param_devlink_cqe_compress_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 mnvda[MLX5_ST_SZ_DW(mnvda_reg)] = {}; + int err = 0; + void *data; + u8 value; + + if (!strcmp(ctx->val.vstr, "aggressive")) + value = 1; + else /* balanced: can't be anything else already validated above */ + value = 0; + + err = mlx5_nv_param_read_sw_offload_conf(dev, mnvda, sizeof(mnvda)); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to read sw_offload_conf mnvda reg"); + return err; + } + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + MLX5_SET(nv_sw_offload_conf, data, cqe_compression, value); + + return mlx5_nv_param_write(dev, mnvda, sizeof(mnvda)); +} + +static int mlx5_nv_param_read_global_pci_conf(struct mlx5_core_dev *dev, + void *mnvda, size_t len) +{ + MLX5_SET_CFG_ITEM_TYPE(global, mnvda, type_class, 0); + MLX5_SET_CFG_ITEM_TYPE(global, mnvda, parameter_index, + MLX5_CLASS_0_CTRL_ID_NV_GLOBAL_PCI_CONF); + MLX5_SET_CFG_HDR_LEN(mnvda, nv_global_pci_conf); + + return mlx5_nv_param_read(dev, mnvda, len); +} + +static int mlx5_nv_param_read_global_pci_cap(struct mlx5_core_dev *dev, + void *mnvda, size_t len) +{ + MLX5_SET_CFG_ITEM_TYPE(global, mnvda, type_class, 0); + MLX5_SET_CFG_ITEM_TYPE(global, mnvda, parameter_index, + MLX5_CLASS_0_CTRL_ID_NV_GLOBAL_PCI_CAP); + MLX5_SET_CFG_HDR_LEN(mnvda, nv_global_pci_cap); + + return mlx5_nv_param_read(dev, mnvda, len); +} + +static int mlx5_nv_param_read_per_host_pf_conf(struct mlx5_core_dev *dev, + void *mnvda, size_t len) +{ + MLX5_SET_CFG_ITEM_TYPE(per_host_pf, mnvda, type_class, 3); + MLX5_SET_CFG_ITEM_TYPE(per_host_pf, mnvda, parameter_index, + MLX5_CLASS_3_CTRL_ID_NV_PF_PCI_CONF); + MLX5_SET_CFG_HDR_LEN(mnvda, nv_pf_pci_conf); + + return mlx5_nv_param_read(dev, mnvda, len); +} + +static int mlx5_devlink_enable_sriov_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 mnvda[MLX5_ST_SZ_DW(mnvda_reg)] = {}; + bool sriov_en = false; + void *data; + int err; + + err = mlx5_nv_param_read_global_pci_cap(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + if (!MLX5_GET(nv_global_pci_cap, data, sriov_support)) { + ctx->val.vbool = false; + return 0; + } + + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_global_pci_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + sriov_en = MLX5_GET(nv_global_pci_conf, data, sriov_en); + if (!MLX5_GET(nv_global_pci_conf, data, per_pf_total_vf)) { + ctx->val.vbool = sriov_en; + return 0; + } + + /* SRIOV is per PF */ + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_per_host_pf_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + ctx->val.vbool = sriov_en && + MLX5_GET(nv_pf_pci_conf, data, pf_total_vf_en); + return 0; +} + +static int mlx5_devlink_enable_sriov_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 mnvda[MLX5_ST_SZ_DW(mnvda_reg)] = {}; + bool per_pf_support; + void *cap, *data; + int err; + + err = mlx5_nv_param_read_global_pci_cap(dev, mnvda, sizeof(mnvda)); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to read global PCI capability"); + return err; + } + + cap = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + per_pf_support = MLX5_GET(nv_global_pci_cap, cap, + per_pf_total_vf_supported); + + if (!MLX5_GET(nv_global_pci_cap, cap, sriov_support)) { + NL_SET_ERR_MSG_MOD(extack, + "SRIOV is not supported on this device"); + return -EOPNOTSUPP; + } + + if (!per_pf_support) { + /* We don't allow global SRIOV setting on per PF devlink */ + NL_SET_ERR_MSG_MOD(extack, + "SRIOV is not per PF on this device"); + return -EOPNOTSUPP; + } + + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_global_pci_conf(dev, mnvda, sizeof(mnvda)); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Unable to read global PCI configuration"); + return err; + } + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + + /* setup per PF sriov mode */ + MLX5_SET(nv_global_pci_conf, data, sriov_valid, 1); + MLX5_SET(nv_global_pci_conf, data, sriov_en, 1); + MLX5_SET(nv_global_pci_conf, data, per_pf_total_vf, 1); + + err = mlx5_nv_param_write(dev, mnvda, sizeof(mnvda)); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Unable to write global PCI configuration"); + return err; + } + + /* enable/disable sriov on this PF */ + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_per_host_pf_conf(dev, mnvda, sizeof(mnvda)); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Unable to read per host PF configuration"); + return err; + } + MLX5_SET(nv_pf_pci_conf, data, pf_total_vf_en, ctx->val.vbool); + return mlx5_nv_param_write(dev, mnvda, sizeof(mnvda)); +} + +static int mlx5_devlink_total_vfs_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 mnvda[MLX5_ST_SZ_DW(mnvda_reg)] = {}; + void *data; + int err; + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + + err = mlx5_nv_param_read_global_pci_cap(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + if (!MLX5_GET(nv_global_pci_cap, data, sriov_support)) { + ctx->val.vu32 = 0; + return 0; + } + + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_global_pci_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + if (!MLX5_GET(nv_global_pci_conf, data, per_pf_total_vf)) { + ctx->val.vu32 = MLX5_GET(nv_global_pci_conf, data, total_vfs); + return 0; + } + + /* SRIOV is per PF */ + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_per_host_pf_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + ctx->val.vu32 = MLX5_GET(nv_pf_pci_conf, data, total_vf); + + return 0; +} + +static int mlx5_devlink_total_vfs_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 mnvda[MLX5_ST_SZ_DW(mnvda_reg)]; + void *data; + int err; + + err = mlx5_nv_param_read_global_pci_cap(dev, mnvda, sizeof(mnvda)); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to read global pci cap"); + return err; + } + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + if (!MLX5_GET(nv_global_pci_cap, data, sriov_support)) { + NL_SET_ERR_MSG_MOD(extack, "Not configurable on this device"); + return -EOPNOTSUPP; + } + + if (!MLX5_GET(nv_global_pci_cap, data, per_pf_total_vf_supported)) { + /* We don't allow global SRIOV setting on per PF devlink */ + NL_SET_ERR_MSG_MOD(extack, + "SRIOV is not per PF on this device"); + return -EOPNOTSUPP; + } + + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_global_pci_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + MLX5_SET(nv_global_pci_conf, data, sriov_valid, 1); + MLX5_SET(nv_global_pci_conf, data, per_pf_total_vf, 1); + + err = mlx5_nv_param_write(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + memset(mnvda, 0, sizeof(mnvda)); + err = mlx5_nv_param_read_per_host_pf_conf(dev, mnvda, sizeof(mnvda)); + if (err) + return err; + + data = MLX5_ADDR_OF(mnvda_reg, mnvda, configuration_item_data); + MLX5_SET(nv_pf_pci_conf, data, total_vf, ctx->val.vu32); + return mlx5_nv_param_write(dev, mnvda, sizeof(mnvda)); +} + +static int mlx5_devlink_total_vfs_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + u32 cap[MLX5_ST_SZ_DW(mnvda_reg)]; + void *data; + u16 max; + int err; + + data = MLX5_ADDR_OF(mnvda_reg, cap, configuration_item_data); + + err = mlx5_nv_param_read_global_pci_cap(dev, cap, sizeof(cap)); + if (err) + return err; + + if (!MLX5_GET(nv_global_pci_cap, data, max_vfs_per_pf_valid)) + return 0; /* optimistic, but set might fail later */ + + max = MLX5_GET(nv_global_pci_cap, data, max_vfs_per_pf); + if (val.vu16 > max) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Max allowed by device is %u", max); + return -EINVAL; + } + + return 0; +} + +static const struct devlink_param mlx5_nv_param_devlink_params[] = { + DEVLINK_PARAM_GENERIC(ENABLE_SRIOV, BIT(DEVLINK_PARAM_CMODE_PERMANENT), + mlx5_devlink_enable_sriov_get, + mlx5_devlink_enable_sriov_set, NULL), + DEVLINK_PARAM_GENERIC(TOTAL_VFS, BIT(DEVLINK_PARAM_CMODE_PERMANENT), + mlx5_devlink_total_vfs_get, + mlx5_devlink_total_vfs_set, + mlx5_devlink_total_vfs_validate), + DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_CQE_COMPRESSION_TYPE, + "cqe_compress_type", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_PERMANENT), + mlx5_nv_param_devlink_cqe_compress_get, + mlx5_nv_param_devlink_cqe_compress_set, + mlx5_nv_param_devlink_cqe_compress_validate), +}; + +int mlx5_nv_param_register_dl_params(struct devlink *devlink) +{ + if (!mlx5_core_is_pf(devlink_priv(devlink))) + return 0; + + return devl_params_register(devlink, mlx5_nv_param_devlink_params, + ARRAY_SIZE(mlx5_nv_param_devlink_params)); +} + +void mlx5_nv_param_unregister_dl_params(struct devlink *devlink) +{ + if (!mlx5_core_is_pf(devlink_priv(devlink))) + return; + + devl_params_unregister(devlink, mlx5_nv_param_devlink_params, + ARRAY_SIZE(mlx5_nv_param_devlink_params)); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/nv_param.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/nv_param.h new file mode 100644 index 000000000000..9f4922ff7745 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/nv_param.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_NV_PARAM_H +#define __MLX5_NV_PARAM_H + +#include <linux/mlx5/driver.h> +#include "devlink.h" + +int mlx5_nv_param_register_dl_params(struct devlink *devlink); +void mlx5_nv_param_unregister_dl_params(struct devlink *devlink); + +#endif + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c index eeb0b7ea05f1..f5c2701f6e87 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c @@ -210,13 +210,17 @@ static void sd_cleanup(struct mlx5_core_dev *dev) static int sd_register(struct mlx5_core_dev *dev) { struct mlx5_devcom_comp_dev *devcom, *pos; + struct mlx5_devcom_match_attr attr = {}; struct mlx5_core_dev *peer, *primary; struct mlx5_sd *sd, *primary_sd; int err, i; sd = mlx5_get_sd(dev); + attr.key.val = sd->group_id; + attr.flags = MLX5_DEVCOM_MATCH_FLAGS_NS; + attr.net = mlx5_core_net(dev); devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, - sd->group_id, NULL, dev); + &attr, NULL, dev); if (IS_ERR(devcom)) return PTR_ERR(devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 8517d4e5d5ef..df93625c9dfa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -973,36 +973,14 @@ static void mlx5_pci_close(struct mlx5_core_dev *dev) mlx5_pci_disable_device(dev); } -static void mlx5_register_hca_devcom_comp(struct mlx5_core_dev *dev) -{ - /* This component is use to sync adding core_dev to lag_dev and to sync - * changes of mlx5_adev_devices between LAG layer and other layers. - */ - if (!mlx5_lag_is_supported(dev)) - return; - - dev->priv.hca_devcom_comp = - mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_HCA_PORTS, - mlx5_query_nic_system_image_guid(dev), - NULL, dev); - if (IS_ERR(dev->priv.hca_devcom_comp)) - mlx5_core_err(dev, "Failed to register devcom HCA component\n"); -} - -static void mlx5_unregister_hca_devcom_comp(struct mlx5_core_dev *dev) -{ - mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp); -} - static int mlx5_init_once(struct mlx5_core_dev *dev) { int err; dev->priv.devc = mlx5_devcom_register_device(dev); if (IS_ERR(dev->priv.devc)) - mlx5_core_warn(dev, "failed to register devcom device %ld\n", - PTR_ERR(dev->priv.devc)); - mlx5_register_hca_devcom_comp(dev); + mlx5_core_warn(dev, "failed to register devcom device %pe\n", + dev->priv.devc); err = mlx5_query_board_id(dev); if (err) { @@ -1140,7 +1118,6 @@ err_eq_cleanup: err_irq_cleanup: mlx5_irq_table_cleanup(dev); err_devcom: - mlx5_unregister_hca_devcom_comp(dev); mlx5_devcom_unregister_device(dev->priv.devc); return err; @@ -1171,7 +1148,6 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_events_cleanup(dev); mlx5_eq_table_cleanup(dev); mlx5_irq_table_cleanup(dev); - mlx5_unregister_hca_devcom_comp(dev); mlx5_devcom_unregister_device(dev->priv.devc); } @@ -1340,10 +1316,9 @@ static int mlx5_load(struct mlx5_core_dev *dev) { int err; - dev->priv.uar = mlx5_get_uars_page(dev); - if (IS_ERR(dev->priv.uar)) { - mlx5_core_err(dev, "Failed allocating uar, aborting\n"); - err = PTR_ERR(dev->priv.uar); + err = mlx5_alloc_bfreg(dev, &dev->priv.bfreg, false, false); + if (err) { + mlx5_core_err(dev, "Failed allocating bfreg, %d\n", err); return err; } @@ -1454,7 +1429,7 @@ err_eq_table: err_irq_table: mlx5_pagealloc_stop(dev); mlx5_events_stop(dev); - mlx5_put_uars_page(dev, dev->priv.uar); + mlx5_free_bfreg(dev, &dev->priv.bfreg); return err; } @@ -1479,7 +1454,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev) mlx5_irq_table_destroy(dev); mlx5_pagealloc_stop(dev); mlx5_events_stop(dev); - mlx5_put_uars_page(dev, dev->priv.uar); + mlx5_free_bfreg(dev, &dev->priv.bfreg); } int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev) @@ -1798,6 +1773,7 @@ static const int types[] = { MLX5_CAP_VDPA_EMULATION, MLX5_CAP_IPSEC, MLX5_CAP_PORT_SELECTION, + MLX5_CAP_PSP, MLX5_CAP_MACSEC, MLX5_CAP_ADV_VIRTUALIZATION, MLX5_CAP_CRYPTO, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 9d3504f5abfa..082259b56816 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -449,8 +449,6 @@ int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap #define mlx5_vport_get_other_func_general_cap(dev, vport, out) \ mlx5_vport_get_other_func_cap(dev, vport, out, MLX5_CAP_GENERAL) -int mlx5_vport_get_vhca_id(struct mlx5_core_dev *dev, u16 vport, u16 *vhca_id); - static inline u32 mlx5_sriov_get_vf_total_msix(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index 9bc9bd83c232..cd68c4b2c0bf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -489,9 +489,12 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev, u32 func_id; u32 npages; u32 i = 0; + int err; - if (!mlx5_cmd_is_down(dev)) - return mlx5_cmd_do(dev, in, in_size, out, out_size); + err = mlx5_cmd_do(dev, in, in_size, out, out_size); + /* If FW is gone (-ENXIO), proceed to forceful reclaim */ + if (err != -ENXIO) + return err; /* No hard feelings, we want our pages back! */ npages = MLX5_GET(manage_pages_in, in, input_num_entries); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 692ef9c2f729..e18a850c615c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -54,7 +54,7 @@ static int mlx5_core_func_to_vport(const struct mlx5_core_dev *dev, /** * mlx5_get_default_msix_vec_count - Get the default number of MSI-X vectors - * to be ssigned to each VF. + * to be assigned to each VF. * @dev: PF to work on * @num_vfs: Number of enabled VFs */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/diag/dev_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/diag/dev_tracepoint.h index 0537de86f981..9b0f44253f33 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/diag/dev_tracepoint.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/diag/dev_tracepoint.h @@ -28,7 +28,7 @@ DECLARE_EVENT_CLASS(mlx5_sf_dev_template, __entry->hw_fn_id = sfdev->fn_id; __entry->sfnum = sfdev->sfnum; ), - TP_printk("(%s) sfdev=%pK aux_id=%d hw_id=0x%x sfnum=%u\n", + TP_printk("(%s) sfdev=%p aux_id=%d hw_id=0x%x sfnum=%u\n", __get_str(devname), __entry->sfdev, __entry->aux_id, __entry->hw_fn_id, __entry->sfnum) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c index adeccc588e5d..6ef0c4be27e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c @@ -51,9 +51,6 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, u8 size_log_rx, u8 size_log_tx, struct mlx5hws_matcher_attr *attr) { - struct mlx5hws_bwc_matcher *first_matcher = - bwc_matcher->complex_first_bwc_matcher; - memset(attr, 0, sizeof(*attr)); attr->priority = priority; @@ -66,9 +63,6 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX].rule.num_log = size_log_tx; attr->resizable = true; attr->max_num_of_at_attach = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM; - - attr->isolated_matcher_end_ft_id = - first_matcher ? first_matcher->matcher->end_ft_id : 0; } static int @@ -171,10 +165,16 @@ hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher) static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher) { - if (!bwc_matcher->complex) + switch (bwc_matcher->matcher_type) { + case MLX5HWS_BWC_MATCHER_SIMPLE: return hws_bwc_matcher_move_all_simple(bwc_matcher); - - return mlx5hws_bwc_matcher_move_all_complex(bwc_matcher); + case MLX5HWS_BWC_MATCHER_COMPLEX_FIRST: + return mlx5hws_bwc_matcher_complex_move_first(bwc_matcher); + case MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER: + return mlx5hws_bwc_matcher_complex_move(bwc_matcher); + default: + return -EINVAL; + } } static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher) @@ -249,6 +249,7 @@ int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher, bwc_matcher->tx_size.size_log, &attr); + bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_SIMPLE; bwc_matcher->priority = priority; bwc_matcher->size_of_at_array = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM; @@ -393,7 +394,7 @@ int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) "BWC matcher destroy: matcher still has %u RX and %u TX rules\n", rx_rules, tx_rules); - if (bwc_matcher->complex) + if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST) mlx5hws_bwc_matcher_destroy_complex(bwc_matcher); else mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); @@ -651,7 +652,8 @@ int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule) int mlx5hws_bwc_rule_destroy(struct mlx5hws_bwc_rule *bwc_rule) { - bool is_complex = !!bwc_rule->bwc_matcher->complex; + bool is_complex = bwc_rule->bwc_matcher->matcher_type == + MLX5HWS_BWC_MATCHER_COMPLEX_FIRST; int ret = 0; if (is_complex) @@ -1147,7 +1149,7 @@ mlx5hws_bwc_rule_create(struct mlx5hws_bwc_matcher *bwc_matcher, bwc_queue_idx = hws_bwc_gen_queue_idx(ctx); - if (bwc_matcher->complex) + if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST) ret = mlx5hws_bwc_rule_create_complex(bwc_rule, params, flow_source, @@ -1216,10 +1218,9 @@ int mlx5hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule, return -EINVAL; } - /* For complex rule, the update should happen on the second matcher */ - if (bwc_rule->isolated_bwc_rule) - return hws_bwc_rule_action_update(bwc_rule->isolated_bwc_rule, - rule_actions); - else - return hws_bwc_rule_action_update(bwc_rule, rule_actions); + /* For complex rules, the update should happen on the last subrule. */ + while (bwc_rule->next_subrule) + bwc_rule = bwc_rule->next_subrule; + + return hws_bwc_rule_action_update(bwc_rule, rule_actions); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h index af391d70c14f..b905511f5c53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h @@ -18,6 +18,21 @@ #define MLX5HWS_BWC_POLLING_TIMEOUT 60 +enum mlx5hws_bwc_matcher_type { + /* Standalone bwc matcher. */ + MLX5HWS_BWC_MATCHER_SIMPLE, + /* The first matcher of a complex matcher. When rules are inserted into + * a matcher of this type, they are split into subrules and inserted + * into their corresponding submatchers. + */ + MLX5HWS_BWC_MATCHER_COMPLEX_FIRST, + /* A submatcher that is part of a complex matcher. For most purposes + * these are treated as simple matchers, except when it comes to moving + * rules during resize. + */ + MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER, +}; + struct mlx5hws_bwc_matcher_complex_data; struct mlx5hws_bwc_matcher_size { @@ -31,9 +46,9 @@ struct mlx5hws_bwc_matcher { struct mlx5hws_match_template *mt; struct mlx5hws_action_template **at; struct mlx5hws_bwc_matcher_complex_data *complex; - struct mlx5hws_bwc_matcher *complex_first_bwc_matcher; u8 num_of_at; u8 size_of_at_array; + enum mlx5hws_bwc_matcher_type matcher_type; u32 priority; struct mlx5hws_bwc_matcher_size rx_size; struct mlx5hws_bwc_matcher_size tx_size; @@ -43,8 +58,8 @@ struct mlx5hws_bwc_matcher { struct mlx5hws_bwc_rule { struct mlx5hws_bwc_matcher *bwc_matcher; struct mlx5hws_rule *rule; - struct mlx5hws_bwc_rule *isolated_bwc_rule; - struct mlx5hws_bwc_complex_rule_hash_node *complex_hash_node; + struct mlx5hws_bwc_rule *next_subrule; + struct mlx5hws_bwc_complex_subrule_data *subrule_data; u32 flow_source; u16 bwc_queue_idx; bool skip_rx; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c index 14e79579c719..660630f18ce9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c @@ -3,25 +3,27 @@ #include "internal.h" -#define HWS_CLEAR_MATCH_PARAM(mask, field) \ - MLX5_SET(fte_match_param, (mask)->match_buf, field, 0) - -#define HWS_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4) - -static const struct rhashtable_params hws_refcount_hash = { - .key_len = sizeof_field(struct mlx5hws_bwc_complex_rule_hash_node, - match_buf), - .key_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node, - match_buf), - .head_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node, - hash_node), - .automatic_shrinking = true, - .min_size = 1, +/* We chain submatchers by applying three rules on a subrule: modify header (to + * set register C6), jump to table (to the next submatcher) and the mandatory + * last rule. + */ +#define HWS_NUM_CHAIN_ACTIONS 3 + +static const struct rhashtable_params hws_rules_hash_params = { + .key_len = sizeof_field(struct mlx5hws_bwc_complex_subrule_data, + match_tag), + .key_offset = + offsetof(struct mlx5hws_bwc_complex_subrule_data, match_tag), + .head_offset = + offsetof(struct mlx5hws_bwc_complex_subrule_data, hash_node), + .automatic_shrinking = true, .min_size = 1, }; -bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, - u8 match_criteria_enable, - struct mlx5hws_match_parameters *mask) +static bool +hws_match_params_exceeds_definer(struct mlx5hws_context *ctx, + u8 match_criteria_enable, + struct mlx5hws_match_parameters *mask, + bool allow_jumbo) { struct mlx5hws_definer match_layout = {0}; struct mlx5hws_match_template *mt; @@ -36,11 +38,11 @@ bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, mask->match_sz, match_criteria_enable); if (!mt) { - mlx5hws_err(ctx, "BWC: failed creating match template\n"); + mlx5hws_err(ctx, "Complex matcher: failed creating match template\n"); return false; } - ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout); + ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, allow_jumbo); if (ret) { /* The only case that we're interested in is E2BIG, * which means that the match parameters need to be @@ -64,825 +66,481 @@ bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, return is_complex; } -static void -hws_bwc_matcher_complex_params_clear_fld(struct mlx5hws_context *ctx, - enum mlx5hws_definer_fname fname, +bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, + u8 match_criteria_enable, struct mlx5hws_match_parameters *mask) { - struct mlx5hws_cmd_query_caps *caps = ctx->caps; - - switch (fname) { - case MLX5HWS_DEFINER_FNAME_ETH_TYPE_O: - case MLX5HWS_DEFINER_FNAME_ETH_TYPE_I: - case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_O: - case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_I: - case MLX5HWS_DEFINER_FNAME_IP_VERSION_O: - case MLX5HWS_DEFINER_FNAME_IP_VERSION_I: - /* Because of the strict requirements for IP address matching - * that require ethtype/ip_version matching as well, don't clear - * these fields - have them in both parts of the complex matcher - */ - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_CFI_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_CFI_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_ID_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_vid); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_ID_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_vid); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_O: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.outer_second_cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.outer_second_svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_I: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.inner_second_cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.inner_second_svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_vid); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_vid); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_IHL_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ipv4_ihl); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_IHL_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ipv4_ihl); - break; - case MLX5HWS_DEFINER_FNAME_IP_DSCP_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_dscp); - break; - case MLX5HWS_DEFINER_FNAME_IP_DSCP_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_dscp); - break; - case MLX5HWS_DEFINER_FNAME_IP_ECN_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_ecn); - break; - case MLX5HWS_DEFINER_FNAME_IP_ECN_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_ecn); - break; - case MLX5HWS_DEFINER_FNAME_IP_TTL_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ttl_hoplimit); - break; - case MLX5HWS_DEFINER_FNAME_IP_TTL_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ttl_hoplimit); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_DST_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_SRC_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_DST_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_SRC_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IP_FRAG_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.frag); - break; - case MLX5HWS_DEFINER_FNAME_IP_FRAG_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.frag); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_O: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.outer_ipv6_flow_label); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_I: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.inner_ipv6_flow_label); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_protocol); - break; - case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_protocol); - break; - case MLX5HWS_DEFINER_FNAME_L4_SPORT_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_sport); - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_sport); - break; - case MLX5HWS_DEFINER_FNAME_L4_SPORT_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport); - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport); - break; - case MLX5HWS_DEFINER_FNAME_L4_DPORT_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_dport); - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_dport); - break; - case MLX5HWS_DEFINER_FNAME_L4_DPORT_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport); - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport); - break; - case MLX5HWS_DEFINER_FNAME_TCP_FLAGS_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_flags); - break; - case MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM: - case MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_tcp_seq_num); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_tcp_ack_num); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.inner_tcp_seq_num); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.inner_tcp_ack_num); - break; - case MLX5HWS_DEFINER_FNAME_GTP_TEID: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_teid); - break; - case MLX5HWS_DEFINER_FNAME_GTP_MSG_TYPE: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_type); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_flags); - break; - case MLX5HWS_DEFINER_FNAME_GTPU_FIRST_EXT_DW0: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.gtpu_first_ext_dw_0); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_0); - break; - case MLX5HWS_DEFINER_FNAME_GTPU_DW2: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_2); - break; - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_2.outer_first_mpls_over_gre); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_2.outer_first_mpls_over_udp); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.geneve_tlv_option_0_data); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_id_0); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_0); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_1); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_id_2); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_2); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_id_3); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_3); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_VNI: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.vxlan_vni); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_FLAGS: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_vxlan_gpe_flags); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_RSVD0: - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_PROTO: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_vxlan_gpe_next_protocol); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_VNI: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_vxlan_gpe_vni); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_OPT_LEN: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_opt_len); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_OAM: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_oam); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_PROTO: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.geneve_protocol_type); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_VNI: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_vni); - break; - case MLX5HWS_DEFINER_FNAME_SOURCE_QP: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_sqn); - break; - case MLX5HWS_DEFINER_FNAME_SOURCE_GVMI: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_port); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.source_eswitch_owner_vhca_id); - break; - case MLX5HWS_DEFINER_FNAME_REG_0: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_0); - break; - case MLX5HWS_DEFINER_FNAME_REG_1: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_1); - break; - case MLX5HWS_DEFINER_FNAME_REG_2: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_2); - break; - case MLX5HWS_DEFINER_FNAME_REG_3: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_3); - break; - case MLX5HWS_DEFINER_FNAME_REG_4: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_4); - break; - case MLX5HWS_DEFINER_FNAME_REG_5: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_5); - break; - case MLX5HWS_DEFINER_FNAME_REG_7: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_7); - break; - case MLX5HWS_DEFINER_FNAME_REG_A: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_a); - break; - case MLX5HWS_DEFINER_FNAME_GRE_C: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_c_present); - break; - case MLX5HWS_DEFINER_FNAME_GRE_K: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_k_present); - break; - case MLX5HWS_DEFINER_FNAME_GRE_S: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_s_present); - break; - case MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_protocol); - break; - case MLX5HWS_DEFINER_FNAME_GRE_OPT_KEY: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_key.key); - break; - case MLX5HWS_DEFINER_FNAME_ICMP_DW1: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_header_data); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_type); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_code); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.icmpv6_header_data); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_type); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_code); - break; - case MLX5HWS_DEFINER_FNAME_MPLS0_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.outer_first_mpls); - break; - case MLX5HWS_DEFINER_FNAME_MPLS0_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.inner_first_mpls); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_0: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_0); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_1: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_1); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_2: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_2); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_3: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_3); - break; - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK: - /* assuming this is flex parser for geneve option */ - if ((fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 0) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 1) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 2) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 3) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 4) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 5) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 6) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 7)) { - mlx5hws_err(ctx, - "Complex params: unsupported field %s (%d), flex parser ID for geneve is %d\n", - mlx5hws_definer_fname_to_str(fname), fname, - caps->flex_parser_id_geneve_tlv_option_0); - break; - } - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.geneve_tlv_option_0_exist); - break; - case MLX5HWS_DEFINER_FNAME_REG_6: - default: - mlx5hws_err(ctx, "Complex params: unsupported field %s (%d)\n", - mlx5hws_definer_fname_to_str(fname), fname); - break; - } + return hws_match_params_exceeds_definer(ctx, match_criteria_enable, + mask, true); } -static bool -hws_bwc_matcher_complex_params_comb_is_valid(struct mlx5hws_definer_fc *fc, - int fc_sz, - u32 combination_num) +static int +hws_get_last_set_dword_idx(const struct mlx5hws_match_parameters *mask) { - bool m1[MLX5HWS_DEFINER_FNAME_MAX] = {0}; - bool m2[MLX5HWS_DEFINER_FNAME_MAX] = {0}; - bool is_first_matcher; int i; - for (i = 0; i < fc_sz; i++) { - is_first_matcher = !(combination_num & BIT(i)); - if (is_first_matcher) - m1[fc[i].fname] = true; - else - m2[fc[i].fname] = true; - } - - /* Not all the fields can be split into separate matchers. - * Some should be together on the same matcher. - * For example, IPv6 parts - the whole IPv6 address should be on the - * same matcher in order for us to deduce if it's IPv6 or IPv4 address. - */ - if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] && - (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] || - m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O])) - return false; - - if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] && - (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] || - m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O])) - return false; + for (i = mask->match_sz / 4 - 1; i >= 0; i--) + if (mask->match_buf[i]) + return i; - if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] && - (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] || - m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I])) - return false; + return -1; +} - if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] && - (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] || - m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I])) - return false; +static bool hws_match_mask_is_empty(const struct mlx5hws_match_parameters *mask) +{ + return hws_get_last_set_dword_idx(mask) == -1; +} - /* Don't split outer IPv6 dest address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O])) - return false; +static bool hws_dword_is_inner_ipaddr_off(int dword_off) +{ + /* IPv4 and IPv6 addresses share the same entry via a union, and the + * source and dest addresses are contiguous in the fte_match_param. So + * we need to check 8 words. + */ + static const int inner_ip_dword_off = + __mlx5_dw_off(fte_match_param, inner_headers.src_ipv4_src_ipv6); - /* Don't split outer IPv6 source address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O])) - return false; + return dword_off >= inner_ip_dword_off && + dword_off < inner_ip_dword_off + 8; +} - /* Don't split inner IPv6 dest address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I])) - return false; +static bool hws_dword_is_outer_ipaddr_off(int dword_off) +{ + static const int outer_ip_dword_off = + __mlx5_dw_off(fte_match_param, outer_headers.src_ipv4_src_ipv6); - /* Don't split inner IPv6 source address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I])) - return false; + return dword_off >= outer_ip_dword_off && + dword_off < outer_ip_dword_off + 8; +} - /* Don't split GRE parameters. */ - if ((m1[MLX5HWS_DEFINER_FNAME_GRE_C] || - m1[MLX5HWS_DEFINER_FNAME_GRE_K] || - m1[MLX5HWS_DEFINER_FNAME_GRE_S] || - m1[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL]) && - (m2[MLX5HWS_DEFINER_FNAME_GRE_C] || - m2[MLX5HWS_DEFINER_FNAME_GRE_K] || - m2[MLX5HWS_DEFINER_FNAME_GRE_S] || - m2[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL])) - return false; +static void hws_add_dword_to_mask(struct mlx5hws_match_parameters *mask, + const struct mlx5hws_match_parameters *orig, + int dword_idx, bool *added_inner_ipv, + bool *added_outer_ipv) +{ + mask->match_buf[dword_idx] |= orig->match_buf[dword_idx]; - /* Don't split TCP ack/seq numbers. */ - if ((m1[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] || - m1[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM]) && - (m2[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] || - m2[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM])) - return false; + *added_inner_ipv = false; + *added_outer_ipv = false; - /* Don't split flex parser. */ - if ((m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7]) && - (m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7])) - return false; + /* Any IP address fragment must be accompanied by a match on IP version. + * Use the `added_ipv` variables to keep track if we added IP versions + * specifically for this dword, so that we can roll them back if the + * match params become too large to fit into a definer. + */ + if (hws_dword_is_inner_ipaddr_off(dword_idx) && + !MLX5_GET(fte_match_param, mask->match_buf, + inner_headers.ip_version)) { + MLX5_SET(fte_match_param, mask->match_buf, + inner_headers.ip_version, 0xf); + *added_inner_ipv = true; + } + if (hws_dword_is_outer_ipaddr_off(dword_idx) && + !MLX5_GET(fte_match_param, mask->match_buf, + outer_headers.ip_version)) { + MLX5_SET(fte_match_param, mask->match_buf, + outer_headers.ip_version, 0xf); + *added_outer_ipv = true; + } +} - return true; +static void hws_remove_dword_from_mask(struct mlx5hws_match_parameters *mask, + int dword_idx, bool added_inner_ipv, + bool added_outer_ipv) +{ + mask->match_buf[dword_idx] = 0; + if (added_inner_ipv) + MLX5_SET(fte_match_param, mask->match_buf, + inner_headers.ip_version, 0); + if (added_outer_ipv) + MLX5_SET(fte_match_param, mask->match_buf, + outer_headers.ip_version, 0); } -static void -hws_bwc_matcher_complex_params_comb_create(struct mlx5hws_context *ctx, - struct mlx5hws_match_parameters *m, - struct mlx5hws_match_parameters *m1, - struct mlx5hws_match_parameters *m2, - struct mlx5hws_definer_fc *fc, - int fc_sz, - u32 combination_num) +/* Avoid leaving a single lower dword in `mask` if there are others present in + * `orig`. Splitting IPv6 addresses like this causes them to be interpreted as + * IPv4. + */ +static void hws_avoid_ipv6_split_of(struct mlx5hws_match_parameters *orig, + struct mlx5hws_match_parameters *mask, + int off) { - bool is_first_matcher; - int i; + /* Masks are allocated to a full fte_match_param, but it can't hurt to + * double check. + */ + if (orig->match_sz <= off + 3 || mask->match_sz <= off + 3) + return; - memcpy(m1->match_buf, m->match_buf, m->match_sz); - memcpy(m2->match_buf, m->match_buf, m->match_sz); + /* Lower dword is not set, nothing to do. */ + if (!mask->match_buf[off + 3]) + return; - for (i = 0; i < fc_sz; i++) { - is_first_matcher = !(combination_num & BIT(i)); - hws_bwc_matcher_complex_params_clear_fld(ctx, - fc[i].fname, - is_first_matcher ? - m2 : m1); - } + /* Higher dwords also present in `mask`, no ambiguity. */ + if (mask->match_buf[off] || mask->match_buf[off + 1] || + mask->match_buf[off + 2]) + return; + + /* There are no higher dwords in `orig`, i.e. we match on IPv4. */ + if (!orig->match_buf[off] && !orig->match_buf[off + 1] && + !orig->match_buf[off + 2]) + return; - MLX5_SET(fte_match_param, m2->match_buf, - misc_parameters_2.metadata_reg_c_6, -1); + /* Put the lower dword back in `orig`. It is always safe to do this, the + * dword will just be picked up in the next submask. + */ + orig->match_buf[off + 3] = mask->match_buf[off + 3]; + mask->match_buf[off + 3] = 0; } -static void -hws_bwc_matcher_complex_params_destroy(struct mlx5hws_match_parameters *mask_1, - struct mlx5hws_match_parameters *mask_2) +static void hws_avoid_ipv6_split(struct mlx5hws_match_parameters *orig, + struct mlx5hws_match_parameters *mask) { - kfree(mask_1->match_buf); - kfree(mask_2->match_buf); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + outer_headers.src_ipv4_src_ipv6)); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + outer_headers.dst_ipv4_dst_ipv6)); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + inner_headers.src_ipv4_src_ipv6)); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + inner_headers.dst_ipv4_dst_ipv6)); } -static int -hws_bwc_matcher_complex_params_create(struct mlx5hws_context *ctx, - u8 match_criteria, - struct mlx5hws_match_parameters *mask, - struct mlx5hws_match_parameters *mask_1, - struct mlx5hws_match_parameters *mask_2) +/* Build a subset of the `orig` match parameters into `mask`. This subset is + * guaranteed to fit in a single definer an as such is a candidate for being a + * part of a complex matcher. Upon successful execution, the match params that + * go into `mask` are cleared from `orig`. + */ +static int hws_get_simple_params(struct mlx5hws_context *ctx, u8 match_criteria, + struct mlx5hws_match_parameters *orig, + struct mlx5hws_match_parameters *mask) { - struct mlx5hws_definer_fc *fc; - u32 num_of_combinations; - int fc_sz = 0; - int res = 0; - u32 i; - - if (MLX5_GET(fte_match_param, mask->match_buf, - misc_parameters_2.metadata_reg_c_6)) { - mlx5hws_err(ctx, "Complex matcher: REG_C_6 matching is reserved\n"); - res = -EINVAL; - goto out; - } + bool added_inner_ipv, added_outer_ipv; + int dword_idx; + u32 *backup; + int ret; - mask_1->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), - GFP_KERNEL); - mask_2->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), - GFP_KERNEL); - if (!mask_1->match_buf || !mask_2->match_buf) { - mlx5hws_err(ctx, "Complex matcher: failed to allocate match_param\n"); - res = -ENOMEM; - goto free_params; - } + dword_idx = hws_get_last_set_dword_idx(orig); + /* Nothing to do, we consumed all of the match params before. */ + if (dword_idx == -1) + return 0; - mask_1->match_sz = mask->match_sz; - mask_2->match_sz = mask->match_sz; + backup = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); + if (!backup) + return -ENOMEM; - fc = mlx5hws_definer_conv_match_params_to_compressed_fc(ctx, - match_criteria, - mask->match_buf, - &fc_sz); - if (!fc) { - res = -ENOMEM; - goto free_params; - } + while (1) { + dword_idx = hws_get_last_set_dword_idx(orig); + /* Nothing to do, we consumed all of the original match params + * into this subset, which still fits into a single matcher. + */ + if (dword_idx == -1) { + ret = 0; + goto free_backup; + } - if (fc_sz >= sizeof(num_of_combinations) * BITS_PER_BYTE) { - mlx5hws_err(ctx, - "Complex matcher: too many match parameters (%d)\n", - fc_sz); - res = -EINVAL; - goto free_fc; + memcpy(backup, mask->match_buf, mask->match_sz); + + /* Try to add this dword to the current subset. */ + hws_add_dword_to_mask(mask, orig, dword_idx, &added_inner_ipv, + &added_outer_ipv); + + if (hws_match_params_exceeds_definer(ctx, match_criteria, mask, + false)) { + /* We just added a match param that makes the definer + * too large. Revert and return what we had before. + * Note that we can't just zero out the affected fields, + * because it's possible that the dword we're looking at + * wasn't zero before (e.g. it included auto-added + * matches in IP version. This is why we employ the + * rather cumbersome memcpy for backing up. + */ + memcpy(mask->match_buf, backup, mask->match_sz); + /* Possible future improvement: We can't add any more + * dwords, but it may be possible to squeeze in + * individual bytes, as definers have special slots for + * those. + * + * For now, keep the code simple. This results in an + * extra submatcher in some cases, but it's good enough. + */ + ret = 0; + break; + } + + /* The current subset of match params still fits in a single + * definer. Remove the dword from the original mask. + * + * Also remove any explicit match on IP version if we just + * included one here. We will still automatically add it to + * accompany any IP address fragment, but do not need to + * consider it by itself. + */ + hws_remove_dword_from_mask(orig, dword_idx, added_inner_ipv, + added_outer_ipv); } - /* We have list of all the match fields from the match parameter. - * Now try all the possibilities of splitting them into two match - * buffers and look for the supported combination. + /* Make sure we have not picked up a single lower dword of an IPv6 + * address, as the firmware will erroneously treat it as an IPv4 + * address. */ - num_of_combinations = 1 << fc_sz; + hws_avoid_ipv6_split(orig, mask); - /* Start from combination at index 1 - we know that 0 is unsupported */ - for (i = 1; i < num_of_combinations; i++) { - if (!hws_bwc_matcher_complex_params_comb_is_valid(fc, fc_sz, i)) - continue; +free_backup: + kfree(backup); - hws_bwc_matcher_complex_params_comb_create(ctx, - mask, mask_1, mask_2, - fc, fc_sz, i); - /* We now have two separate sets of match params. - * Check if each of them can be used in its own matcher. + return ret; +} + +static int +hws_bwc_matcher_split_mask(struct mlx5hws_context *ctx, u8 match_criteria, + const struct mlx5hws_match_parameters *mask, + struct mlx5hws_match_parameters *submasks, + int *num_submasks) +{ + struct mlx5hws_match_parameters mask_copy; + int ret, i = 0; + + mask_copy.match_sz = MLX5_ST_SZ_BYTES(fte_match_param); + mask_copy.match_buf = kzalloc(mask_copy.match_sz, GFP_KERNEL); + if (!mask_copy.match_buf) + return -ENOMEM; + + memcpy(mask_copy.match_buf, mask->match_buf, mask->match_sz); + + while (!hws_match_mask_is_empty(&mask_copy)) { + if (i >= MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS) { + mlx5hws_err(ctx, + "Complex matcher: mask too large for %d matchers\n", + MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS); + ret = -E2BIG; + goto free_copy; + } + /* All but the first matcher need to match on register C6 to + * connect pieces of the complex rule together. */ - if (!mlx5hws_bwc_match_params_is_complex(ctx, - match_criteria, - mask_1) && - !mlx5hws_bwc_match_params_is_complex(ctx, - match_criteria, - mask_2)) - break; + if (i > 0) { + MLX5_SET(fte_match_param, submasks[i].match_buf, + misc_parameters_2.metadata_reg_c_6, -1); + match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2; + } + ret = hws_get_simple_params(ctx, match_criteria, &mask_copy, + &submasks[i]); + if (ret < 0) + goto free_copy; + i++; } - if (i == num_of_combinations) { - /* We've scanned all the combinations, but to no avail */ - mlx5hws_err(ctx, "Complex matcher: couldn't find match params combination\n"); - res = -EINVAL; - goto free_fc; - } + *num_submasks = i; + ret = 0; - kfree(fc); - return 0; +free_copy: + kfree(mask_copy.match_buf); -free_fc: - kfree(fc); -free_params: - hws_bwc_matcher_complex_params_destroy(mask_1, mask_2); -out: - return res; + return ret; } -static int -hws_bwc_isolated_table_create(struct mlx5hws_bwc_matcher *bwc_matcher, - struct mlx5hws_table *table) +static struct mlx5hws_table * +hws_isolated_table_create(const struct mlx5hws_bwc_matcher *cmatcher) { + struct mlx5hws_bwc_complex_submatcher *first_subm; struct mlx5hws_cmd_ft_modify_attr ft_attr = {0}; - struct mlx5hws_context *ctx = table->ctx; struct mlx5hws_table_attr tbl_attr = {0}; - struct mlx5hws_table *isolated_tbl; - int ret = 0; + struct mlx5hws_table *orig_tbl; + struct mlx5hws_context *ctx; + struct mlx5hws_table *tbl; + int ret; - tbl_attr.type = table->type; - tbl_attr.level = table->level; + first_subm = &cmatcher->complex->submatchers[0]; + orig_tbl = first_subm->tbl; + ctx = orig_tbl->ctx; - bwc_matcher->complex->isolated_tbl = - mlx5hws_table_create(ctx, &tbl_attr); - isolated_tbl = bwc_matcher->complex->isolated_tbl; - if (!isolated_tbl) - return -EINVAL; + tbl_attr.type = orig_tbl->type; + tbl_attr.level = orig_tbl->level; + tbl = mlx5hws_table_create(ctx, &tbl_attr); + if (!tbl) + return ERR_PTR(-EINVAL); - /* Set the default miss of the isolated table to - * point to the end anchor of the original matcher. + /* Set the default miss of the isolated table to point + * to the end anchor of the original matcher. */ - mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, - isolated_tbl->fw_ft_type, - isolated_tbl->type, - &ft_attr); - ft_attr.table_miss_id = bwc_matcher->matcher->end_ft_id; - - ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, - &ft_attr, - isolated_tbl->ft_id); + mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, tbl->fw_ft_type, + tbl->type, &ft_attr); + ft_attr.table_miss_id = first_subm->bwc_matcher->matcher->end_ft_id; + + ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, &ft_attr, tbl->ft_id); if (ret) { - mlx5hws_err(ctx, "Failed setting isolated tbl default miss\n"); + mlx5hws_err(ctx, "Complex matcher: failed to set isolated tbl default miss\n"); goto destroy_tbl; } - return 0; + return tbl; destroy_tbl: - mlx5hws_table_destroy(isolated_tbl); - return ret; + mlx5hws_table_destroy(tbl); + + return ERR_PTR(ret); } -static void hws_bwc_isolated_table_destroy(struct mlx5hws_table *isolated_tbl) +static int hws_submatcher_init_first(struct mlx5hws_bwc_matcher *cmatcher, + struct mlx5hws_table *table, u32 priority, + u8 match_criteria, + struct mlx5hws_match_parameters *mask) { - /* This table is isolated - no table is pointing to it, no need to - * disconnect it from anywhere, it won't affect any other table's miss. + enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS]; + struct mlx5hws_bwc_complex_submatcher *subm; + int ret; + + subm = &cmatcher->complex->submatchers[0]; + + /* The first submatcher lives in the original table and does not have an + * associated jump to table action. It also points to the outer complex + * matcher. */ - mlx5hws_table_destroy(isolated_tbl); + subm->tbl = table; + subm->action_tbl = NULL; + subm->bwc_matcher = cmatcher; + + action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR; + action_types[1] = MLX5HWS_ACTION_TYP_TBL; + action_types[2] = MLX5HWS_ACTION_TYP_LAST; + + ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl, + priority, match_criteria, mask, + action_types); + if (ret) + return ret; + + subm->bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_COMPLEX_FIRST; + + ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params); + if (ret) + goto destroy_matcher; + mutex_init(&subm->hash_lock); + ida_init(&subm->chain_ida); + + return 0; + +destroy_matcher: + mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher); + + return ret; } -static int -hws_bwc_isolated_matcher_create(struct mlx5hws_bwc_matcher *bwc_matcher, - struct mlx5hws_table *table, - u8 match_criteria_enable, - struct mlx5hws_match_parameters *mask) +static int hws_submatcher_init(struct mlx5hws_bwc_matcher *cmatcher, int idx, + struct mlx5hws_table *table, u32 priority, + u8 match_criteria, + struct mlx5hws_match_parameters *mask) { - struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; - struct mlx5hws_context *ctx = table->ctx; + enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS]; + struct mlx5hws_bwc_complex_submatcher *subm; + bool is_last; int ret; - isolated_bwc_matcher = kzalloc(sizeof(*bwc_matcher), GFP_KERNEL); - if (!isolated_bwc_matcher) - return -ENOMEM; + if (!idx) + return hws_submatcher_init_first(cmatcher, table, priority, + match_criteria, mask); + + subm = &cmatcher->complex->submatchers[idx]; + is_last = idx == cmatcher->complex->num_submatchers - 1; + + subm->tbl = hws_isolated_table_create(cmatcher); + if (IS_ERR(subm->tbl)) + return PTR_ERR(subm->tbl); + + subm->action_tbl = + mlx5hws_action_create_dest_table(subm->tbl->ctx, subm->tbl, + MLX5HWS_ACTION_FLAG_HWS_FDB); + if (!subm->action_tbl) { + ret = -EINVAL; + goto destroy_tbl; + } + + subm->bwc_matcher = kzalloc(sizeof(*subm->bwc_matcher), GFP_KERNEL); + if (!subm->bwc_matcher) { + ret = -ENOMEM; + goto destroy_action; + } - bwc_matcher->complex->isolated_bwc_matcher = isolated_bwc_matcher; + /* Every matcher other than the first also matches of register C6 to + * bind subrules together in the complex rule using the chain ids. + */ + match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2; - /* Isolated BWC matcher needs access to the first BWC matcher */ - isolated_bwc_matcher->complex_first_bwc_matcher = bwc_matcher; + action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR; + action_types[1] = MLX5HWS_ACTION_TYP_TBL; + action_types[2] = MLX5HWS_ACTION_TYP_LAST; - /* Isolated matcher needs to match on REG_C_6, - * so make sure its criteria bit is on. + /* Every matcher other than the last sets register C6 and jumps to the + * next submatcher's table. The final submatcher will use the + * user-supplied actions and will attach an action template at rule + * insertion time. */ - match_criteria_enable |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2; - - ret = mlx5hws_bwc_matcher_create_simple(isolated_bwc_matcher, - isolated_tbl, - 0, - match_criteria_enable, - mask, - NULL); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated BWC matcher\n"); + ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl, + priority, match_criteria, mask, + is_last ? NULL : action_types); + if (ret) goto free_matcher; - } + + subm->bwc_matcher->matcher_type = + MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER; + + ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params); + if (ret) + goto destroy_matcher; + mutex_init(&subm->hash_lock); + ida_init(&subm->chain_ida); return 0; +destroy_matcher: + mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher); free_matcher: - kfree(bwc_matcher->complex->isolated_bwc_matcher); + kfree(subm->bwc_matcher); +destroy_action: + mlx5hws_action_destroy(subm->action_tbl); +destroy_tbl: + mlx5hws_table_destroy(subm->tbl); + return ret; } -static void -hws_bwc_isolated_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) +static void hws_submatcher_destroy(struct mlx5hws_bwc_matcher *cmatcher, + int idx) { - mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); - kfree(bwc_matcher); + struct mlx5hws_bwc_complex_submatcher *subm; + + subm = &cmatcher->complex->submatchers[idx]; + + ida_destroy(&subm->chain_ida); + mutex_destroy(&subm->hash_lock); + rhashtable_destroy(&subm->rules_hash); + + if (subm->bwc_matcher) { + mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher); + if (idx) + kfree(subm->bwc_matcher); + } + + /* We own all of the isolated tables, but not the original one. */ + if (idx) { + mlx5hws_action_destroy(subm->action_tbl); + mlx5hws_table_destroy(subm->tbl); + } } static int -hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher, - struct mlx5hws_table *table) +hws_complex_data_actions_init(struct mlx5hws_bwc_matcher_complex_data *cdata) { - struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl; + struct mlx5hws_context *ctx = cdata->submatchers[0].tbl->ctx; u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0}; - struct mlx5hws_context *ctx = table->ctx; struct mlx5hws_action_mh_pattern ptrn; int ret = 0; - /* Create action to jump to isolated table */ - - bwc_matcher->complex->action_go_to_tbl = - mlx5hws_action_create_dest_table(ctx, - isolated_tbl, - MLX5HWS_ACTION_FLAG_HWS_FDB); - if (!bwc_matcher->complex->action_go_to_tbl) { - mlx5hws_err(ctx, "Complex matcher: failed to create go-to-tbl action\n"); - return -EINVAL; - } - /* Create modify header action to set REG_C_6 */ - MLX5_SET(set_action_in, modify_hdr_action, action_type, MLX5_MODIFICATION_TYPE_SET); MLX5_SET(set_action_in, modify_hdr_action, @@ -895,19 +553,18 @@ hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher, ptrn.data = (void *)modify_hdr_action; ptrn.sz = MLX5HWS_ACTION_DOUBLE_SIZE; - bwc_matcher->complex->action_metadata = + cdata->action_metadata = mlx5hws_action_create_modify_header(ctx, 1, &ptrn, 0, MLX5HWS_ACTION_FLAG_HWS_FDB); - if (!bwc_matcher->complex->action_metadata) { - ret = -EINVAL; - goto destroy_action_go_to_tbl; + if (!cdata->action_metadata) { + mlx5hws_err(ctx, "Complex matcher: failed to create set reg C6 action\n"); + return -EINVAL; } /* Create last action */ - - bwc_matcher->complex->action_last = + cdata->action_last = mlx5hws_action_create_last(ctx, MLX5HWS_ACTION_FLAG_HWS_FDB); - if (!bwc_matcher->complex->action_last) { + if (!cdata->action_last) { mlx5hws_err(ctx, "Complex matcher: failed to create last action\n"); ret = -EINVAL; goto destroy_action_metadata; @@ -916,196 +573,130 @@ hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher, return 0; destroy_action_metadata: - mlx5hws_action_destroy(bwc_matcher->complex->action_metadata); -destroy_action_go_to_tbl: - mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl); + mlx5hws_action_destroy(cdata->action_metadata); + return ret; } static void -hws_bwc_isolated_actions_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) +hws_complex_data_actions_destroy(struct mlx5hws_bwc_matcher_complex_data *cdata) { - mlx5hws_action_destroy(bwc_matcher->complex->action_last); - mlx5hws_action_destroy(bwc_matcher->complex->action_metadata); - mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl); + mlx5hws_action_destroy(cdata->action_last); + mlx5hws_action_destroy(cdata->action_metadata); } int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher, struct mlx5hws_table *table, - u32 priority, - u8 match_criteria_enable, + u32 priority, u8 match_criteria_enable, struct mlx5hws_match_parameters *mask) { - enum mlx5hws_action_type complex_init_action_types[3]; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; - struct mlx5hws_match_parameters mask_1 = {0}; - struct mlx5hws_match_parameters mask_2 = {0}; + struct mlx5hws_match_parameters + submasks[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0}; + struct mlx5hws_bwc_matcher_complex_data *cdata; struct mlx5hws_context *ctx = table->ctx; - int ret; - - ret = hws_bwc_matcher_complex_params_create(table->ctx, - match_criteria_enable, - mask, &mask_1, &mask_2); - if (ret) - goto err; - - bwc_matcher->complex = - kzalloc(sizeof(*bwc_matcher->complex), GFP_KERNEL); - if (!bwc_matcher->complex) { - ret = -ENOMEM; - goto free_masks; - } + int num_submatchers; + int i, ret; - ret = rhashtable_init(&bwc_matcher->complex->refcount_hash, - &hws_refcount_hash); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed to initialize rhashtable\n"); - goto free_complex; + for (i = 0; i < ARRAY_SIZE(submasks); i++) { + submasks[i].match_sz = MLX5_ST_SZ_BYTES(fte_match_param); + submasks[i].match_buf = kzalloc(submasks[i].match_sz, + GFP_KERNEL); + if (!submasks[i].match_buf) { + ret = -ENOMEM; + goto free_submasks; + } } - mutex_init(&bwc_matcher->complex->hash_lock); - ida_init(&bwc_matcher->complex->metadata_ida); - - /* Create initial action template for the first matcher. - * Usually the initial AT is just dummy, but in case of complex - * matcher we know exactly which actions should it have. - */ - - complex_init_action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR; - complex_init_action_types[1] = MLX5HWS_ACTION_TYP_TBL; - complex_init_action_types[2] = MLX5HWS_ACTION_TYP_LAST; - - /* Create the first matcher */ - - ret = mlx5hws_bwc_matcher_create_simple(bwc_matcher, - table, - priority, - match_criteria_enable, - &mask_1, - complex_init_action_types); + ret = hws_bwc_matcher_split_mask(ctx, match_criteria_enable, mask, + submasks, &num_submatchers); if (ret) - goto destroy_ida; - - /* Create isolated table to hold the second isolated matcher */ + goto free_submasks; - ret = hws_bwc_isolated_table_create(bwc_matcher, table); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated table\n"); - goto destroy_first_matcher; + cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); + if (!cdata) { + ret = -ENOMEM; + goto free_submasks; } - /* Now create the second BWC matcher - the isolated one */ + bwc_matcher->complex = cdata; + cdata->num_submatchers = num_submatchers; - ret = hws_bwc_isolated_matcher_create(bwc_matcher, table, - match_criteria_enable, &mask_2); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated matcher\n"); - goto destroy_isolated_tbl; + for (i = 0; i < num_submatchers; i++) { + ret = hws_submatcher_init(bwc_matcher, i, table, priority, + match_criteria_enable, &submasks[i]); + if (ret) + goto destroy_submatchers; } - /* Create action for isolated matcher's rules */ - - ret = hws_bwc_isolated_actions_create(bwc_matcher, table); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated actions\n"); - goto destroy_isolated_matcher; - } + ret = hws_complex_data_actions_init(cdata); + if (ret) + goto destroy_submatchers; - hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2); - return 0; + ret = 0; + goto free_submasks; -destroy_isolated_matcher: - isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher; - hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher); -destroy_isolated_tbl: - hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl); -destroy_first_matcher: - mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); -destroy_ida: - ida_destroy(&bwc_matcher->complex->metadata_ida); - mutex_destroy(&bwc_matcher->complex->hash_lock); - rhashtable_destroy(&bwc_matcher->complex->refcount_hash); -free_complex: - kfree(bwc_matcher->complex); +destroy_submatchers: + while (i--) + hws_submatcher_destroy(bwc_matcher, i); + kfree(cdata); bwc_matcher->complex = NULL; -free_masks: - hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2); -err: + +free_submasks: + for (i = 0; i < ARRAY_SIZE(submasks); i++) + kfree(submasks[i].match_buf); + return ret; } void mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher) { - struct mlx5hws_bwc_matcher *isolated_bwc_matcher = - bwc_matcher->complex->isolated_bwc_matcher; - - hws_bwc_isolated_actions_destroy(bwc_matcher); - hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher); - hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl); - mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); - ida_destroy(&bwc_matcher->complex->metadata_ida); - mutex_destroy(&bwc_matcher->complex->hash_lock); - rhashtable_destroy(&bwc_matcher->complex->refcount_hash); + int i; + + hws_complex_data_actions_destroy(bwc_matcher->complex); + for (i = 0; i < bwc_matcher->complex->num_submatchers; i++) + hws_submatcher_destroy(bwc_matcher, i); kfree(bwc_matcher->complex); bwc_matcher->complex = NULL; } -static void -hws_bwc_matcher_complex_hash_lock(struct mlx5hws_bwc_matcher *bwc_matcher) -{ - mutex_lock(&bwc_matcher->complex->hash_lock); -} - -static void -hws_bwc_matcher_complex_hash_unlock(struct mlx5hws_bwc_matcher *bwc_matcher) -{ - mutex_unlock(&bwc_matcher->complex->hash_lock); -} - static int -hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule *bwc_rule, - struct mlx5hws_match_parameters *params) +hws_complex_get_subrule_data(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_bwc_complex_submatcher *subm, + u32 *match_params) +__must_hold(&subm->hash_lock) { - struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; - struct mlx5hws_bwc_complex_rule_hash_node *node, *old_node; - struct rhashtable *refcount_hash; - int ret, i; - - bwc_rule->complex_hash_node = NULL; + struct mlx5hws_bwc_matcher *bwc_matcher = subm->bwc_matcher; + struct mlx5hws_bwc_complex_subrule_data *sr_data, *old_data; + struct mlx5hws_match_template *mt; + int ret; - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (unlikely(!node)) + sr_data = kzalloc(sizeof(*sr_data), GFP_KERNEL); + if (!sr_data) return -ENOMEM; - ret = ida_alloc(&bwc_matcher->complex->metadata_ida, GFP_KERNEL); + ret = ida_alloc(&subm->chain_ida, GFP_KERNEL); if (ret < 0) - goto err_free_node; - node->tag = ret; + goto free_sr_data; + sr_data->chain_id = ret; - refcount_set(&node->refcount, 1); + refcount_set(&sr_data->refcount, 1); - /* Clear match buffer - turn off all the unrelated fields - * in accordance with the match params mask for the first - * matcher out of the two parts of the complex matcher. - * The resulting mask is the key for the hash. - */ - for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++) - node->match_buf[i] = params->match_buf[i] & - bwc_matcher->mt->match_param[i]; - - refcount_hash = &bwc_matcher->complex->refcount_hash; - old_node = rhashtable_lookup_get_insert_fast(refcount_hash, - &node->hash_node, - hws_refcount_hash); - if (IS_ERR(old_node)) { - ret = PTR_ERR(old_node); - goto err_free_ida; + mt = bwc_matcher->matcher->mt; + mlx5hws_definer_create_tag(match_params, mt->fc, mt->fc_sz, + (u8 *)&sr_data->match_tag); + + old_data = rhashtable_lookup_get_insert_fast(&subm->rules_hash, + &sr_data->hash_node, + hws_rules_hash_params); + if (IS_ERR(old_data)) { + ret = PTR_ERR(old_data); + goto free_ida; } - if (old_node) { + if (old_data) { /* Rule with the same tag already exists - update refcount */ - refcount_inc(&old_node->refcount); + refcount_inc(&old_data->refcount); /* Let the new rule use the same tag as the existing rule. * Note that we don't have any indication for the rule creation * process that a rule with similar matching params already @@ -1114,247 +705,281 @@ hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule *bwc_rule, * There's some performance advantage in skipping such cases, * so this is left for future optimizations. */ - ida_free(&bwc_matcher->complex->metadata_ida, node->tag); - kfree(node); - node = old_node; + bwc_rule->subrule_data = old_data; + ret = 0; + goto free_ida; } - bwc_rule->complex_hash_node = node; + bwc_rule->subrule_data = sr_data; return 0; -err_free_ida: - ida_free(&bwc_matcher->complex->metadata_ida, node->tag); -err_free_node: - kfree(node); +free_ida: + ida_free(&subm->chain_ida, sr_data->chain_id); +free_sr_data: + kfree(sr_data); + return ret; } static void -hws_bwc_rule_complex_hash_node_put(struct mlx5hws_bwc_rule *bwc_rule, - bool *is_last_rule) +hws_complex_put_subrule_data(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_bwc_complex_submatcher *subm, + bool *is_last_rule) +__must_hold(&subm->hash_lock) { - struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; - struct mlx5hws_bwc_complex_rule_hash_node *node; + struct mlx5hws_bwc_complex_subrule_data *sr_data; if (is_last_rule) *is_last_rule = false; - node = bwc_rule->complex_hash_node; - if (refcount_dec_and_test(&node->refcount)) { - rhashtable_remove_fast(&bwc_matcher->complex->refcount_hash, - &node->hash_node, - hws_refcount_hash); - ida_free(&bwc_matcher->complex->metadata_ida, node->tag); - kfree(node); + sr_data = bwc_rule->subrule_data; + if (refcount_dec_and_test(&sr_data->refcount)) { + rhashtable_remove_fast(&subm->rules_hash, + &sr_data->hash_node, + hws_rules_hash_params); + ida_free(&subm->chain_ida, sr_data->chain_id); + kfree(sr_data); if (is_last_rule) *is_last_rule = true; } - bwc_rule->complex_hash_node = NULL; + bwc_rule->subrule_data = NULL; } -int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule, - struct mlx5hws_match_parameters *params, - u32 flow_source, - struct mlx5hws_rule_action rule_actions[], - u16 bwc_queue_idx) +static int hws_complex_subrule_create(struct mlx5hws_bwc_matcher *cmatcher, + struct mlx5hws_bwc_rule *subrule, + u32 *match_params, u32 flow_source, + int bwc_queue_idx, int subm_idx, + struct mlx5hws_rule_action *actions, + u32 *chain_id) { - struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; - struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; + struct mlx5hws_rule_action chain_actions[HWS_NUM_CHAIN_ACTIONS] = {0}; u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0}; - struct mlx5hws_rule_action rule_actions_1[3] = {0}; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; - u32 *match_buf_2; - u32 metadata_val; - int ret = 0; + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_bwc_complex_submatcher *subm; + int ret; - isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher; - bwc_rule->isolated_bwc_rule = - mlx5hws_bwc_rule_alloc(isolated_bwc_matcher); - if (unlikely(!bwc_rule->isolated_bwc_rule)) - return -ENOMEM; + cdata = cmatcher->complex; + subm = &cdata->submatchers[subm_idx]; - hws_bwc_matcher_complex_hash_lock(bwc_matcher); + mutex_lock(&subm->hash_lock); - /* Get a new hash node for this complex rule. - * If this is a unique set of match params for the first matcher, - * we will get a new hash node with newly allocated IDA. - * Otherwise we will get an existing node with IDA and updated refcount. - */ - ret = hws_bwc_rule_complex_hash_node_get(bwc_rule, params); - if (unlikely(ret)) { - mlx5hws_err(ctx, "Complex rule: failed getting RHT node for this rule\n"); - goto free_isolated_rule; + ret = hws_complex_get_subrule_data(subrule, subm, match_params); + if (ret) + goto unlock; + + *chain_id = subrule->subrule_data->chain_id; + + if (!actions) { + MLX5_SET(set_action_in, modify_hdr_action, data, *chain_id); + chain_actions[0].action = cdata->action_metadata; + chain_actions[0].modify_header.data = modify_hdr_action; + chain_actions[1].action = + cdata->submatchers[subm_idx + 1].action_tbl; + chain_actions[2].action = cdata->action_last; + actions = chain_actions; } - /* No need to clear match buffer's fields in accordance to what - * will actually be matched on first and second matchers. - * Both matchers were created with the appropriate masks - * and each of them holds the appropriate field copy array, - * so rule creation will use only the fields that will be copied - * in accordance with setters in field copy array. - * We do, however, need to temporary allocate match buffer - * for the second (isolated) rule in order to not modify - * user's match params buffer. - */ - - match_buf_2 = kmemdup(params->match_buf, - MLX5_ST_SZ_BYTES(fte_match_param), - GFP_KERNEL); - if (unlikely(!match_buf_2)) { - mlx5hws_err(ctx, "Complex rule: failed allocating match_buf\n"); - ret = -ENOMEM; - goto hash_node_put; - } + ret = mlx5hws_bwc_rule_create_simple(subrule, match_params, actions, + flow_source, bwc_queue_idx); + if (ret) + goto put_subrule_data; - /* On 2nd matcher, use unique 32-bit ID as a matching tag */ - metadata_val = bwc_rule->complex_hash_node->tag; - MLX5_SET(fte_match_param, match_buf_2, - misc_parameters_2.metadata_reg_c_6, metadata_val); - - /* Isolated rule's rule_actions contain all the original actions */ - ret = mlx5hws_bwc_rule_create_simple(bwc_rule->isolated_bwc_rule, - match_buf_2, - rule_actions, - flow_source, - bwc_queue_idx); - kfree(match_buf_2); - if (unlikely(ret)) { - mlx5hws_err(ctx, - "Complex rule: failed creating isolated BWC rule (%d)\n", - ret); - goto hash_node_put; - } + ret = 0; + goto unlock; - /* First rule's rule_actions contain setting metadata and - * jump to isolated table that contains the second matcher. - * Set metadata value to a unique value for this rule. - */ +put_subrule_data: + hws_complex_put_subrule_data(subrule, subm, NULL); +unlock: + mutex_unlock(&subm->hash_lock); - MLX5_SET(set_action_in, modify_hdr_action, - action_type, MLX5_MODIFICATION_TYPE_SET); - MLX5_SET(set_action_in, modify_hdr_action, - field, MLX5_MODI_META_REG_C_6); - MLX5_SET(set_action_in, modify_hdr_action, - length, 0); /* zero means length of 32 */ - MLX5_SET(set_action_in, modify_hdr_action, - offset, 0); - MLX5_SET(set_action_in, modify_hdr_action, - data, metadata_val); + return ret; +} - rule_actions_1[0].action = bwc_matcher->complex->action_metadata; - rule_actions_1[0].modify_header.offset = 0; - rule_actions_1[0].modify_header.data = modify_hdr_action; +static int hws_complex_subrule_destroy(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_bwc_matcher *cmatcher, + int subm_idx) +{ + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_bwc_complex_submatcher *subm; + struct mlx5hws_context *ctx; + bool is_last_rule; + int ret = 0; - rule_actions_1[1].action = bwc_matcher->complex->action_go_to_tbl; - rule_actions_1[2].action = bwc_matcher->complex->action_last; + cdata = cmatcher->complex; + subm = &cdata->submatchers[subm_idx]; + ctx = subm->tbl->ctx; - ret = mlx5hws_bwc_rule_create_simple(bwc_rule, - params->match_buf, - rule_actions_1, - flow_source, - bwc_queue_idx); + mutex_lock(&subm->hash_lock); - if (unlikely(ret)) { + hws_complex_put_subrule_data(bwc_rule, subm, &is_last_rule); + bwc_rule->rule->skip_delete = !is_last_rule; + ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule); + if (unlikely(ret)) mlx5hws_err(ctx, - "Complex rule: failed creating first BWC rule (%d)\n", - ret); - goto destroy_isolated_rule; - } + "Complex rule: failed to delete subrule %d (%d)\n", + subm_idx, ret); - hws_bwc_matcher_complex_hash_unlock(bwc_matcher); + if (subm_idx) + mlx5hws_bwc_rule_free(bwc_rule); - return 0; + mutex_unlock(&subm->hash_lock); -destroy_isolated_rule: - mlx5hws_bwc_rule_destroy_simple(bwc_rule->isolated_bwc_rule); -hash_node_put: - hws_bwc_rule_complex_hash_node_put(bwc_rule, NULL); -free_isolated_rule: - hws_bwc_matcher_complex_hash_unlock(bwc_matcher); - mlx5hws_bwc_rule_free(bwc_rule->isolated_bwc_rule); return ret; } -int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule) +int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_match_parameters *params, + u32 flow_source, + struct mlx5hws_rule_action rule_actions[], + u16 bwc_queue_idx) { - struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx; - struct mlx5hws_bwc_rule *isolated_bwc_rule; - int ret_isolated, ret; - bool is_last_rule; + struct mlx5hws_bwc_rule + *subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0}; + struct mlx5hws_bwc_matcher *cmatcher = bwc_rule->bwc_matcher; + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_rule_action *subrule_actions; + struct mlx5hws_bwc_complex_submatcher *subm; + struct mlx5hws_bwc_rule *subrule; + u32 *match_params; + u32 chain_id; + int i, ret; - hws_bwc_matcher_complex_hash_lock(bwc_rule->bwc_matcher); + cdata = cmatcher->complex; + if (!cdata) + return -EINVAL; - hws_bwc_rule_complex_hash_node_put(bwc_rule, &is_last_rule); - bwc_rule->rule->skip_delete = !is_last_rule; + /* Duplicate user data because we will modify it to set register C6 + * values. For the same reason, make sure that we allocate a full + * match_param even if the user gave us fewer bytes. We need to ensure + * there is space for the match on C6. + */ + match_params = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); + if (!match_params) + return -ENOMEM; - ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule); - if (unlikely(ret)) - mlx5hws_err(ctx, "BWC complex rule: failed destroying first rule\n"); + memcpy(match_params, params->match_buf, params->match_sz); + + ret = hws_complex_subrule_create(cmatcher, bwc_rule, match_params, + flow_source, bwc_queue_idx, 0, + NULL, &chain_id); + if (ret) + goto free_match_params; + subrules[0] = bwc_rule; + + for (i = 1; i < cdata->num_submatchers; i++) { + subm = &cdata->submatchers[i]; + subrule = mlx5hws_bwc_rule_alloc(subm->bwc_matcher); + if (!subrule) { + ret = -ENOMEM; + goto destroy_subrules; + } + + /* Match on the previous subrule's chain_id. This is how + * subrules are connected in steering. + */ + MLX5_SET(fte_match_param, match_params, + misc_parameters_2.metadata_reg_c_6, chain_id); + + /* The last subrule uses the complex rule's user-specified + * actions. Everything else uses the chaining rules based on the + * next table and chain_id. + */ + subrule_actions = + i == cdata->num_submatchers - 1 ? rule_actions : NULL; + + ret = hws_complex_subrule_create(cmatcher, subrule, + match_params, flow_source, + bwc_queue_idx, i, + subrule_actions, &chain_id); + if (ret) { + mlx5hws_bwc_rule_free(subrule); + goto destroy_subrules; + } + + subrules[i] = subrule; + } + + for (i = 0; i < cdata->num_submatchers - 1; i++) + subrules[i]->next_subrule = subrules[i + 1]; - isolated_bwc_rule = bwc_rule->isolated_bwc_rule; - ret_isolated = mlx5hws_bwc_rule_destroy_simple(isolated_bwc_rule); - if (unlikely(ret_isolated)) - mlx5hws_err(ctx, "BWC complex rule: failed destroying second (isolated) rule\n"); + kfree(match_params); - hws_bwc_matcher_complex_hash_unlock(bwc_rule->bwc_matcher); + return 0; - mlx5hws_bwc_rule_free(isolated_bwc_rule); +destroy_subrules: + while (i--) + hws_complex_subrule_destroy(subrules[i], cmatcher, i); +free_match_params: + kfree(match_params); - return ret || ret_isolated; + return ret; } -static void -hws_bwc_matcher_clear_hash_rtcs(struct mlx5hws_bwc_matcher *bwc_matcher) +int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule) { - struct mlx5hws_bwc_complex_rule_hash_node *node; - struct rhashtable_iter iter; + struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; + struct mlx5hws_bwc_rule + *subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0}; + struct mlx5hws_bwc_matcher_complex_data *cdata; + int i, err, ret_val; + + cdata = bwc_matcher->complex; + + /* Construct a list of all the subrules we need to destroy. */ + subrules[0] = bwc_rule; + for (i = 1; i < cdata->num_submatchers; i++) + subrules[i] = subrules[i - 1]->next_subrule; + + ret_val = 0; + for (i = 0; i < cdata->num_submatchers; i++) { + err = hws_complex_subrule_destroy(subrules[i], bwc_matcher, i); + /* If something goes wrong, plow along to destroy all of the + * subrules but return an error upstack. + */ + if (unlikely(err)) + ret_val = err; + } - rhashtable_walk_enter(&bwc_matcher->complex->refcount_hash, &iter); - rhashtable_walk_start(&iter); + return ret_val; +} - while ((node = rhashtable_walk_next(&iter)) != NULL) { - if (IS_ERR(node)) +static void +hws_bwc_matcher_init_move(struct mlx5hws_bwc_matcher *bwc_matcher) +{ + struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; + u16 bwc_queues = mlx5hws_bwc_queues(ctx); + struct mlx5hws_bwc_rule *bwc_rule; + struct list_head *rules_list; + int i; + + for (i = 0; i < bwc_queues; i++) { + rules_list = &bwc_matcher->rules[i]; + if (list_empty(rules_list)) continue; - node->rtc_valid = false; - } - rhashtable_walk_stop(&iter); - rhashtable_walk_exit(&iter); + list_for_each_entry(bwc_rule, rules_list, list_node) { + if (!bwc_rule->subrule_data) + continue; + bwc_rule->subrule_data->was_moved = false; + } + } } -int -mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) +int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher) { struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; struct mlx5hws_matcher *matcher = bwc_matcher->matcher; u16 bwc_queues = mlx5hws_bwc_queues(ctx); struct mlx5hws_bwc_rule *tmp_bwc_rule; struct mlx5hws_rule_attr rule_attr; - struct mlx5hws_table *isolated_tbl; int move_error = 0, poll_error = 0; struct mlx5hws_rule *tmp_rule; struct list_head *rules_list; u32 expected_completions = 1; - u32 end_ft_id; - int i, ret; + int i, ret = 0; - /* We are rehashing the matcher that is the first part of the complex - * matcher. Need to update the isolated matcher to point to the end_ft - * of this new matcher. This needs to be done before moving any rules - * to prevent possible steering loops. - */ - isolated_tbl = bwc_matcher->complex->isolated_tbl; - end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id; - ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl, end_ft_id); - if (ret) { - mlx5hws_err(ctx, - "Failed updating end_ft of isolated matcher (%d)\n", - ret); - return ret; - } - - hws_bwc_matcher_clear_hash_rtcs(bwc_matcher); + hws_bwc_matcher_init_move(bwc_matcher); mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr); @@ -1369,15 +994,15 @@ mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) /* Check if a rule with similar tag has already * been moved. */ - if (tmp_bwc_rule->complex_hash_node->rtc_valid) { - /* This rule is a duplicate of rule with similar - * tag that has already been moved earlier. - * Just update this rule's RTCs. + if (tmp_bwc_rule->subrule_data->was_moved) { + /* This rule is a duplicate of rule with + * identical tag that has already been moved + * earlier. Just update this rule's RTCs. */ tmp_bwc_rule->rule->rtc_0 = - tmp_bwc_rule->complex_hash_node->rtc_0; + tmp_bwc_rule->subrule_data->rtc_0; tmp_bwc_rule->rule->rtc_1 = - tmp_bwc_rule->complex_hash_node->rtc_1; + tmp_bwc_rule->subrule_data->rtc_1; tmp_bwc_rule->rule->matcher = tmp_bwc_rule->rule->matcher->resize_dst; continue; @@ -1425,12 +1050,12 @@ mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) /* Done moving the rule to the new matcher, * now update RTCs for all the duplicated rules. */ - tmp_bwc_rule->complex_hash_node->rtc_0 = + tmp_bwc_rule->subrule_data->rtc_0 = tmp_bwc_rule->rule->rtc_0; - tmp_bwc_rule->complex_hash_node->rtc_1 = + tmp_bwc_rule->subrule_data->rtc_1 = tmp_bwc_rule->rule->rtc_1; - tmp_bwc_rule->complex_hash_node->rtc_valid = true; + tmp_bwc_rule->subrule_data->was_moved = true; } } @@ -1442,3 +1067,35 @@ mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) return ret; } + +int +mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher) +{ + struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_table *isolated_tbl; + u32 end_ft_id; + int i, ret; + + cdata = bwc_matcher->complex; + + /* We are rehashing the first submatcher. We need to update the + * subsequent submatchers to point to the end_ft of this new matcher. + * This needs to be done before moving any rules to prevent possible + * steering loops. + */ + end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id; + for (i = 1; i < cdata->num_submatchers; i++) { + isolated_tbl = cdata->submatchers[i].tbl; + ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl, + end_ft_id); + if (ret) { + mlx5hws_err(ctx, + "Complex matcher: failed updating end_ft of isolated matcher (%d)\n", + ret); + return ret; + } + } + + return mlx5hws_bwc_matcher_complex_move(bwc_matcher); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h index a6887c7e39d5..d07de631ce9f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h @@ -4,25 +4,60 @@ #ifndef HWS_BWC_COMPLEX_H_ #define HWS_BWC_COMPLEX_H_ -struct mlx5hws_bwc_complex_rule_hash_node { - u32 match_buf[MLX5_ST_SZ_DW_MATCH_PARAM]; - u32 tag; +#define MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS 4 + +/* A matcher can't contain two rules with the same match tag, but it is possible + * that two different complex rules' subrules have the same match tag. In that + * case, those subrules correspond to a single rule, and we need to refcount. + */ +struct mlx5hws_bwc_complex_subrule_data { + struct mlx5hws_rule_match_tag match_tag; refcount_t refcount; - bool rtc_valid; + /* The chain_id is what glues individual subrules into larger complex + * rules. It is the value that this subrule writes to register C6, and + * that the next subrule matches against. + */ + u32 chain_id; u32 rtc_0; u32 rtc_1; + /* During rehash we iterate through all the subrules to move them. But + * two or more subrules can share the same physical rule in the + * submatcher, so we use `was_moved` to keep track if a given rule was + * already moved. + */ + bool was_moved; struct rhash_head hash_node; }; +struct mlx5hws_bwc_complex_submatcher { + /* Isolated table that the matcher lives in. Not set for the first + * matcher, which lives in the original table. + */ + struct mlx5hws_table *tbl; + /* Match a rule with this action to go to `tbl`. This is set in all + * submatchers but the first. + */ + struct mlx5hws_action *action_tbl; + /* This submatcher's simple matcher. The first submatcher points to the + * outer (complex) matcher. + */ + struct mlx5hws_bwc_matcher *bwc_matcher; + struct rhashtable rules_hash; + struct ida chain_ida; + struct mutex hash_lock; /* Protect the hash and ida. */ +}; + struct mlx5hws_bwc_matcher_complex_data { - struct mlx5hws_table *isolated_tbl; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; + struct mlx5hws_bwc_complex_submatcher + submatchers[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS]; + int num_submatchers; + /* Actions used by all but the last submatcher to point to the next + * submatcher in the chain. The last submatcher uses the action template + * from the complex matcher, to perform the actions that the user + * originally requested. + */ struct mlx5hws_action *action_metadata; - struct mlx5hws_action *action_go_to_tbl; struct mlx5hws_action *action_last; - struct rhashtable refcount_hash; - struct mutex hash_lock; /* Protect the refcount rhashtable */ - struct ida metadata_ida; }; bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, @@ -37,7 +72,10 @@ int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher, void mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher); -int mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher); +int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher); + +int +mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher); int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule, struct mlx5hws_match_parameters *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c index 0bdcab2e5cf3..f22eaf506d28 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c @@ -1200,34 +1200,20 @@ out: int mlx5hws_cmd_query_gvmi(struct mlx5_core_dev *mdev, bool other_function, u16 vport_number, u16 *gvmi) { - bool ec_vf_func = other_function ? mlx5_core_is_ec_vf_vport(mdev, vport_number) : false; - u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; - int out_size; - void *out; int err; - out_size = MLX5_ST_SZ_BYTES(query_hca_cap_out); - out = kzalloc(out_size, GFP_KERNEL); - if (!out) - return -ENOMEM; - - MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); - MLX5_SET(query_hca_cap_in, in, other_function, other_function); - MLX5_SET(query_hca_cap_in, in, function_id, - mlx5_vport_to_func_id(mdev, vport_number, ec_vf_func)); - MLX5_SET(query_hca_cap_in, in, ec_vf_function, ec_vf_func); - MLX5_SET(query_hca_cap_in, in, op_mod, - MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1 | HCA_CAP_OPMOD_GET_CUR); + if (!other_function) { + /* self vhca_id */ + *gvmi = MLX5_CAP_GEN(mdev, vhca_id); + return 0; + } - err = mlx5_cmd_exec_inout(mdev, query_hca_cap, in, out); + err = mlx5_vport_get_vhca_id(mdev, vport_number, gvmi); if (err) { - kfree(out); + mlx5_core_err(mdev, "Failed to get vport vhca id for vport %d\n", + vport_number); return err; } - *gvmi = MLX5_GET(query_hca_cap_out, out, capability.cmd_hca_cap.vhca_id); - - kfree(out); - return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c index c6436c3a7a83..82fd122d4284 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c @@ -1280,7 +1280,7 @@ hws_definer_conv_misc2(struct mlx5hws_definer_conv_data *cd, struct mlx5hws_definer_fc *fc = cd->fc; struct mlx5hws_definer_fc *curr_fc; - if (HWS_IS_FLD_SET_SZ(match_param, misc_parameters_2.reserved_at_1a0, 0x8) || + if (HWS_IS_FLD_SET_SZ(match_param, misc_parameters_2.psp_syndrome, 0x8) || HWS_IS_FLD_SET_SZ(match_param, misc_parameters_2.ipsec_next_header, 0x8) || HWS_IS_FLD_SET_SZ(match_param, misc_parameters_2.reserved_at_1c0, 0x40) || @@ -1831,80 +1831,6 @@ err_free_fc: return ret; } -struct mlx5hws_definer_fc * -mlx5hws_definer_conv_match_params_to_compressed_fc(struct mlx5hws_context *ctx, - u8 match_criteria_enable, - u32 *match_param, - int *fc_sz) -{ - struct mlx5hws_definer_fc *compressed_fc = NULL; - struct mlx5hws_definer_conv_data cd = {0}; - struct mlx5hws_definer_fc *fc; - int ret; - - fc = hws_definer_alloc_fc(ctx, MLX5HWS_DEFINER_FNAME_MAX); - if (!fc) - return NULL; - - cd.fc = fc; - cd.ctx = ctx; - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_OUTER) { - ret = hws_definer_conv_outer(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_INNER) { - ret = hws_definer_conv_inner(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC) { - ret = hws_definer_conv_misc(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2) { - ret = hws_definer_conv_misc2(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC3) { - ret = hws_definer_conv_misc3(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC4) { - ret = hws_definer_conv_misc4(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC5) { - ret = hws_definer_conv_misc5(&cd, match_param); - if (ret) - goto err_free_fc; - } - - /* Allocate fc array on mt */ - compressed_fc = hws_definer_alloc_compressed_fc(fc); - if (!compressed_fc) { - mlx5hws_err(ctx, - "Convert to compressed fc: failed to set field copy to match template\n"); - goto err_free_fc; - } - *fc_sz = hws_definer_get_fc_size(fc); - -err_free_fc: - kfree(fc); - return compressed_fc; -} - static int hws_definer_find_byte_in_tag(struct mlx5hws_definer *definer, u32 hl_byte_off, @@ -2067,7 +1993,7 @@ hws_definer_copy_sel_ctrl(struct mlx5hws_definer_sel_ctrl *ctrl, static int hws_definer_find_best_match_fit(struct mlx5hws_context *ctx, struct mlx5hws_definer *definer, - u8 *hl) + u8 *hl, bool allow_jumbo) { struct mlx5hws_definer_sel_ctrl ctrl = {0}; bool found; @@ -2084,6 +2010,9 @@ hws_definer_find_best_match_fit(struct mlx5hws_context *ctx, return 0; } + if (!allow_jumbo) + return -E2BIG; + /* Try to create a full/limited jumbo definer */ ctrl.allowed_full_dw = ctx->caps->full_dw_jumbo_support ? DW_SELECTORS : DW_SELECTORS_MATCH; @@ -2160,7 +2089,8 @@ int mlx5hws_definer_compare(struct mlx5hws_definer *definer_a, int mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, struct mlx5hws_match_template *mt, - struct mlx5hws_definer *match_definer) + struct mlx5hws_definer *match_definer, + bool allow_jumbo) { u8 *match_hl; int ret; @@ -2182,7 +2112,8 @@ mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, } /* Find the match definer layout for header layout match union */ - ret = hws_definer_find_best_match_fit(ctx, match_definer, match_hl); + ret = hws_definer_find_best_match_fit(ctx, match_definer, match_hl, + allow_jumbo); if (ret) { if (ret == -E2BIG) mlx5hws_dbg(ctx, @@ -2370,7 +2301,7 @@ int mlx5hws_definer_mt_init(struct mlx5hws_context *ctx, struct mlx5hws_definer match_layout = {0}; int ret; - ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout); + ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, true); if (ret) { mlx5hws_err(ctx, "Failed to calculate matcher definer layout\n"); return ret; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h index 62da55389331..141f3eb2e307 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h @@ -823,13 +823,8 @@ void mlx5hws_definer_free(struct mlx5hws_context *ctx, int mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, struct mlx5hws_match_template *mt, - struct mlx5hws_definer *match_definer); - -struct mlx5hws_definer_fc * -mlx5hws_definer_conv_match_params_to_compressed_fc(struct mlx5hws_context *ctx, - u8 match_criteria_enable, - u32 *match_param, - int *fc_sz); + struct mlx5hws_definer *match_definer, + bool allow_jumbo); const char *mlx5hws_definer_fname_to_str(enum mlx5hws_definer_fname fname); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c index b0595c9b09e4..24ef7d66fa8a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c @@ -690,7 +690,7 @@ static int hws_send_ring_alloc_sq(struct mlx5_core_dev *mdev, size_t buf_sz; int err; - sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + sq->uar_map = mdev->priv.bfreg.map; sq->mdev = mdev; param.db_numa_node = numa_node; @@ -764,7 +764,7 @@ static int hws_send_ring_create_sq(struct mlx5_core_dev *mdev, u32 pdn, MLX5_SET(sqc, sqc, ts_format, ts_format); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); - MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); + MLX5_SET(wq, wq, uar_page, mdev->priv.bfreg.index); MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); @@ -940,7 +940,7 @@ static int hws_send_ring_create_cq(struct mlx5_core_dev *mdev, (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); - MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index); MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); @@ -963,7 +963,7 @@ static int hws_send_ring_open_cq(struct mlx5_core_dev *mdev, if (!cqc_data) return -ENOMEM; - MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.bfreg.up->index); MLX5_SET(cqc, cqc_data, log_cq_size, ilog2(queue->num_entries)); err = hws_send_ring_alloc_cq(mdev, numa_node, queue, cqc_data, cq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_cmd.c index baefb9a3fa05..1ebb2b15c080 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_cmd.c @@ -2,6 +2,7 @@ /* Copyright (c) 2019 Mellanox Technologies. */ #include "dr_types.h" +#include "eswitch.h" int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev, bool other_vport, @@ -34,34 +35,21 @@ int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev, int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev, bool other_vport, u16 vport_number, u16 *gvmi) { - bool ec_vf_func = other_vport ? mlx5_core_is_ec_vf_vport(mdev, vport_number) : false; - u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; - int out_size; - void *out; int err; - out_size = MLX5_ST_SZ_BYTES(query_hca_cap_out); - out = kzalloc(out_size, GFP_KERNEL); - if (!out) - return -ENOMEM; - - MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); - MLX5_SET(query_hca_cap_in, in, other_function, other_vport); - MLX5_SET(query_hca_cap_in, in, function_id, mlx5_vport_to_func_id(mdev, vport_number, ec_vf_func)); - MLX5_SET(query_hca_cap_in, in, ec_vf_function, ec_vf_func); - MLX5_SET(query_hca_cap_in, in, op_mod, - MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1 | - HCA_CAP_OPMOD_GET_CUR); + if (!other_vport) { + /* self vhca_id */ + *gvmi = MLX5_CAP_GEN(mdev, vhca_id); + return 0; + } - err = mlx5_cmd_exec_inout(mdev, query_hca_cap, in, out); + err = mlx5_vport_get_vhca_id(mdev, vport_number, gvmi); if (err) { - kfree(out); + mlx5_core_err(mdev, "Failed to get vport vhca id for vport %d\n", + vport_number); return err; } - *gvmi = MLX5_GET(query_hca_cap_out, out, capability.cmd_hca_cap.vhca_id); - - kfree(out); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c index 4fd4e8483382..077a77fde670 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c @@ -1131,7 +1131,6 @@ static struct mlx5dr_cq *dr_create_cq(struct mlx5_core_dev *mdev, *cq->mcq.arm_db = cpu_to_be32(2 << 28); cq->mcq.vector = 0; - cq->mcq.uar = uar; cq->mdev = mdev; return cq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index da5c24fc7b30..2ed2e530b07d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -36,6 +36,7 @@ #include <linux/mlx5/vport.h> #include <linux/mlx5/eswitch.h> #include "mlx5_core.h" +#include "eswitch.h" #include "sf/sf.h" /* Mutex to hold while enabling or disabling RoCE */ @@ -1189,18 +1190,44 @@ u64 mlx5_query_nic_system_image_guid(struct mlx5_core_dev *mdev) } EXPORT_SYMBOL_GPL(mlx5_query_nic_system_image_guid); +static bool mlx5_vport_use_vhca_id_as_func_id(struct mlx5_core_dev *dev, + u16 vport_num, u16 *vhca_id) +{ + if (!MLX5_CAP_GEN_2(dev, function_id_type_vhca_id)) + return false; + + return mlx5_esw_vport_vhca_id(dev->priv.eswitch, vport_num, vhca_id); +} + int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 vport, void *out, u16 opmod) { - bool ec_vf_func = mlx5_core_is_ec_vf_vport(dev, vport); u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)] = {}; + u16 vhca_id = 0, function_id = 0; + bool ec_vf_func = false; + + /* if this vport is referring to a vport on the ec PF (embedded cpu ) + * let the FW know which domain we are querying since vport numbers or + * function_ids are not unique across the different PF domains, + * unless we use vhca_id as the function_id below. + */ + ec_vf_func = mlx5_core_is_ec_vf_vport(dev, vport); + function_id = mlx5_vport_to_func_id(dev, vport, ec_vf_func); + + if (mlx5_vport_use_vhca_id_as_func_id(dev, vport, &vhca_id)) { + MLX5_SET(query_hca_cap_in, in, function_id_type, 1); + function_id = vhca_id; + ec_vf_func = false; + mlx5_core_dbg(dev, "%s using vhca_id as function_id for vport %d vhca_id 0x%x\n", + __func__, vport, vhca_id); + } opmod = (opmod << 1) | (HCA_CAP_OPMOD_GET_MAX & 0x01); MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); MLX5_SET(query_hca_cap_in, in, op_mod, opmod); - MLX5_SET(query_hca_cap_in, in, function_id, mlx5_vport_to_func_id(dev, vport, ec_vf_func)); MLX5_SET(query_hca_cap_in, in, other_function, true); MLX5_SET(query_hca_cap_in, in, ec_vf_function, ec_vf_func); + MLX5_SET(query_hca_cap_in, in, function_id, function_id); return mlx5_cmd_exec_inout(dev, query_hca_cap, in, out); } EXPORT_SYMBOL_GPL(mlx5_vport_get_other_func_cap); @@ -1212,7 +1239,9 @@ int mlx5_vport_get_vhca_id(struct mlx5_core_dev *dev, u16 vport, u16 *vhca_id) void *hca_caps; int err; - *vhca_id = 0; + /* try get vhca_id via eswitch */ + if (mlx5_esw_vport_vhca_id(dev->priv.eswitch, vport, vhca_id)) + return 0; query_ctx = kzalloc(query_out_sz, GFP_KERNEL); if (!query_ctx) @@ -1229,12 +1258,14 @@ out_free: kfree(query_ctx); return err; } +EXPORT_SYMBOL_GPL(mlx5_vport_get_vhca_id); int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap, u16 vport, u16 opmod) { - bool ec_vf_func = mlx5_core_is_ec_vf_vport(dev, vport); int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); + u16 vhca_id = 0, function_id = 0; + bool ec_vf_func = false; void *set_hca_cap; void *set_ctx; int ret; @@ -1243,14 +1274,29 @@ int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap if (!set_ctx) return -ENOMEM; + /* if this vport is referring to a vport on the ec PF (embedded cpu ) + * let the FW know which domain we are querying since vport numbers or + * function_ids are not unique across the different PF domains, + * unless we use vhca_id as the function_id below. + */ + ec_vf_func = mlx5_core_is_ec_vf_vport(dev, vport); + function_id = mlx5_vport_to_func_id(dev, vport, ec_vf_func); + + if (mlx5_vport_use_vhca_id_as_func_id(dev, vport, &vhca_id)) { + MLX5_SET(set_hca_cap_in, set_ctx, function_id_type, 1); + function_id = vhca_id; + ec_vf_func = false; + mlx5_core_dbg(dev, "%s using vhca_id as function_id for vport %d vhca_id 0x%x\n", + __func__, vport, vhca_id); + } + MLX5_SET(set_hca_cap_in, set_ctx, opcode, MLX5_CMD_OP_SET_HCA_CAP); MLX5_SET(set_hca_cap_in, set_ctx, op_mod, opmod << 1); set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); memcpy(set_hca_cap, hca_cap, MLX5_ST_SZ_BYTES(cmd_hca_cap)); - MLX5_SET(set_hca_cap_in, set_ctx, function_id, - mlx5_vport_to_func_id(dev, vport, ec_vf_func)); MLX5_SET(set_hca_cap_in, set_ctx, other_function, true); MLX5_SET(set_hca_cap_in, set_ctx, ec_vf_function, ec_vf_func); + MLX5_SET(set_hca_cap_in, set_ctx, function_id, function_id); ret = mlx5_cmd_exec_in(dev, set_hca_cap, set_ctx); kfree(set_ctx); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 2f0316616fa4..c281153bd411 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -7,6 +7,10 @@ #include "mlx5_core.h" #include "wq.h" +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && IS_ENABLED(CONFIG_ARM64) +#include <asm/neon.h> +#endif + #define TEST_WC_NUM_WQES 255 #define TEST_WC_LOG_CQ_SZ (order_base_2(TEST_WC_NUM_WQES)) #define TEST_WC_SQ_LOG_WQ_SZ TEST_WC_LOG_CQ_SZ @@ -94,7 +98,7 @@ static int create_wc_cq(struct mlx5_wc_cq *cq, void *cqc_data) MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); - MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index); MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); @@ -116,7 +120,7 @@ static int mlx5_wc_create_cq(struct mlx5_core_dev *mdev, struct mlx5_wc_cq *cq) return -ENOMEM; MLX5_SET(cqc, cqc, log_cq_size, TEST_WC_LOG_CQ_SZ); - MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index); if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD); @@ -255,7 +259,29 @@ static void mlx5_wc_destroy_sq(struct mlx5_wc_sq *sq) mlx5_wq_destroy(&sq->wq_ctrl); } -static void mlx5_wc_post_nop(struct mlx5_wc_sq *sq, bool signaled) +static void mlx5_iowrite64_copy(struct mlx5_wc_sq *sq, __be32 mmio_wqe[16], + size_t mmio_wqe_size, unsigned int offset) +{ +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && IS_ENABLED(CONFIG_ARM64) + if (cpu_has_neon()) { + kernel_neon_begin(); + asm volatile + (".arch_extension simd;\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%0]\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%1]" + : + : "r"(mmio_wqe), "r"(sq->bfreg.map + offset) + : "memory", "v0", "v1", "v2", "v3"); + kernel_neon_end(); + return; + } +#endif + __iowrite64_copy(sq->bfreg.map + offset, mmio_wqe, + mmio_wqe_size / 8); +} + +static void mlx5_wc_post_nop(struct mlx5_wc_sq *sq, unsigned int *offset, + bool signaled) { int buf_size = (1 << MLX5_CAP_GEN(sq->cq.mdev, log_bf_reg_size)) / 2; struct mlx5_wqe_ctrl_seg *ctrl; @@ -288,10 +314,9 @@ static void mlx5_wc_post_nop(struct mlx5_wc_sq *sq, bool signaled) */ wmb(); - __iowrite64_copy(sq->bfreg.map + sq->bfreg.offset, mmio_wqe, - sizeof(mmio_wqe) / 8); + mlx5_iowrite64_copy(sq, mmio_wqe, sizeof(mmio_wqe), *offset); - sq->bfreg.offset ^= buf_size; + *offset ^= buf_size; } static int mlx5_wc_poll_cq(struct mlx5_wc_sq *sq) @@ -332,6 +357,7 @@ static int mlx5_wc_poll_cq(struct mlx5_wc_sq *sq) static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) { + unsigned int offset = 0; unsigned long expires; struct mlx5_wc_sq *sq; int i, err; @@ -358,9 +384,9 @@ static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) goto err_create_sq; for (i = 0; i < TEST_WC_NUM_WQES - 1; i++) - mlx5_wc_post_nop(sq, false); + mlx5_wc_post_nop(sq, &offset, false); - mlx5_wc_post_nop(sq, true); + mlx5_wc_post_nop(sq, &offset, true); expires = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES; do { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 2bb2b77351bd..83c7cf3bbea3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -886,7 +886,7 @@ static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core) if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX)) return 0; - emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0); + emad_wq = alloc_workqueue("mlxsw_core_emad", WQ_PERCPU, 0); if (!emad_wq) return -ENOMEM; mlxsw_core->emad_wq = emad_wq; @@ -2043,7 +2043,7 @@ static int mlxsw_core_health_init(struct mlxsw_core *mlxsw_core) return 0; fw_fatal = devl_health_reporter_create(devlink, &mlxsw_core_health_fw_fatal_ops, - 0, mlxsw_core); + mlxsw_core); if (IS_ERR(fw_fatal)) { dev_err(mlxsw_core->bus_info->dev, "Failed to create fw fatal reporter"); return PTR_ERR(fw_fatal); @@ -3381,7 +3381,7 @@ static int __init mlxsw_core_module_init(void) if (err) return err; - mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0); + mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, WQ_PERCPU, 0); if (!mlxsw_wq) { err = -ENOMEM; goto err_alloc_workqueue; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c index 50e591420bd9..b1094aaffa5f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c @@ -170,8 +170,7 @@ void mlxsw_sp_counter_pool_fini(struct mlxsw_sp *mlxsw_sp) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); mlxsw_sp_counter_sub_pools_fini(mlxsw_sp); - WARN_ON(find_first_bit(pool->usage, pool->pool_size) != - pool->pool_size); + WARN_ON(!bitmap_empty(pool->usage, pool->pool_size)); WARN_ON(atomic_read(&pool->active_entries_count)); bitmap_free(pool->usage); devl_resource_occ_get_unregister(devlink, |