mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
Merge branch 'net-mlx5e-shampo-allow-high-order-pages-in-zerocopy-mode'
Tariq Toukan says: ==================== net/mlx5e: SHAMPO, Allow high order pages in zerocopy mode This series adds support for high order pages when io_uring/devmem zero copy is used. See detailed description by Dragos below. The first patches are moving code around to allow using queue specific parameters that are not just for XSK. They are a bit large as they touch a lot of functions. The middle part of the series is updating various formulas to remove remaining hardcoded use of PAGE_SIZE/PAGE_SHIFT. The last part adds support for high order pages by implementing the queue configuration functions and allowing larger rx_page_size configurations when in zero-copy mode. Results show an increase in BW and a decrease in CPU usage. The benchmark was done with the zcrx samples from liburing [0]. rx_buf_len=4K, oncpu [1]: packets=3358832 (MB=820027), rps=55794 (MB/s=13621) Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle Average: 9 1.56 0.00 18.09 13.42 0.00 66.80 0.00 0.00 0.00 0.12 rx_buf_len=128K, oncpu [2]: packets=3781376 (MB=923187), rps=62813 (MB/s=15335) Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle Average: 9 0.33 0.00 7.61 18.86 0.00 73.08 0.00 0.00 0.00 0.12 rx_buf_len=4K, offcpu [3]: packets=3460368 (MB=844816), rps=57481 (MB/s=14033) Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle Average: 9 0.00 0.00 0.26 0.00 0.00 92.63 0.00 0.00 0.00 7.11 Average: 11 3.04 0.00 68.09 28.87 0.00 0.00 0.00 0.00 0.00 0.00 rx_buf_len=128K, offcpu [4]: packets=4119840 (MB=1005820), rps=68435 (MB/s=16707) Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle Average: 9 0.00 0.00 0.87 0.00 0.00 63.77 0.00 0.00 0.00 35.36 Average: 11 1.96 0.00 43.68 54.37 0.00 0.00 0.00 0.00 0.00 0.00 [0] https://github.com/isilence/liburing/tree/zcrx/rx-buf-len [1] commands: $> taskset -c 9 ./zcrx 6 -i eth2 -q 9 -A 1 -B 4096 -S 33554432 $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000 [2] commands: $> taskset -c 9 ./zcrx 6 -i eth2 -q 9 -A 1 -B 131072 -S 33554432 $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000 [3] commands: $> taskset -c 11 ./zcrx 6 -i eth2 -q 9 -A 1 -B 4096 -S 33554432 $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000 [4] commands: $> taskset -c 11 ./zcrx 6 -i eth2 -q 9 -A 1 -B 131072 -S 33554432 $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000 ==================== Link: https://patch.msgid.link/20260223204155.1783580-1-tariqt@nvidia.com Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni
@@ -80,6 +80,7 @@ struct page_pool;
|
||||
#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
|
||||
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
|
||||
|
||||
#define MLX5E_PAGECNT_BIAS_MAX U16_MAX
|
||||
#define MLX5E_RX_MAX_HEAD (256)
|
||||
#define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8)
|
||||
#define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \
|
||||
@@ -1060,13 +1061,15 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv);
|
||||
struct mlx5e_xsk_param;
|
||||
|
||||
struct mlx5e_rq_param;
|
||||
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
|
||||
struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
|
||||
struct mlx5e_rq_opt_param;
|
||||
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
|
||||
struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
|
||||
struct mlx5e_rq *rq);
|
||||
#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
|
||||
int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
|
||||
void mlx5e_close_rq(struct mlx5e_rq *rq);
|
||||
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter);
|
||||
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
|
||||
u16 q_counter);
|
||||
void mlx5e_destroy_rq(struct mlx5e_rq *rq);
|
||||
|
||||
bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,11 @@ struct mlx5e_xsk_param {
|
||||
bool unaligned;
|
||||
};
|
||||
|
||||
struct mlx5e_rq_opt_param {
|
||||
struct mlx5e_xsk_param *xsk;
|
||||
struct netdev_queue_config *qcfg;
|
||||
};
|
||||
|
||||
struct mlx5e_cq_param {
|
||||
u32 cqc[MLX5_ST_SZ_DW(cqc)];
|
||||
struct mlx5_wq_param wq;
|
||||
@@ -38,6 +43,7 @@ struct mlx5e_sq_param {
|
||||
|
||||
struct mlx5e_channel_param {
|
||||
struct mlx5e_rq_param rq;
|
||||
struct mlx5e_rq_opt_param rq_opt;
|
||||
struct mlx5e_sq_param txq_sq;
|
||||
struct mlx5e_sq_param xdp_sq;
|
||||
struct mlx5e_sq_param icosq;
|
||||
@@ -56,9 +62,11 @@ struct mlx5e_create_sq_param {
|
||||
|
||||
/* Striding RQ dynamic parameters */
|
||||
|
||||
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
|
||||
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
enum mlx5e_mpwrq_umr_mode
|
||||
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
|
||||
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode);
|
||||
u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
|
||||
enum mlx5e_mpwrq_umr_mode umr_mode);
|
||||
@@ -80,22 +88,22 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
|
||||
bool slow_pci_heuristic(struct mlx5_core_dev *mdev);
|
||||
int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
|
||||
int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
|
||||
void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
|
||||
void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
|
||||
|
||||
u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_rq_param *rq_param);
|
||||
@@ -105,21 +113,22 @@ u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev,
|
||||
u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout);
|
||||
u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz);
|
||||
u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev);
|
||||
|
||||
/* Build queue parameters */
|
||||
|
||||
void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c);
|
||||
int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk,
|
||||
struct mlx5e_rq_opt_param *rqo,
|
||||
struct mlx5e_rq_param *param);
|
||||
void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_rq_param *param);
|
||||
@@ -136,13 +145,19 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_sq_param *param);
|
||||
int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct netdev_queue_config *qcfg,
|
||||
struct mlx5e_channel_param *cparam);
|
||||
|
||||
void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk,
|
||||
struct mlx5e_channel_param *cparam);
|
||||
|
||||
u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
|
||||
int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
|
||||
bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk);
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
|
||||
static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params)
|
||||
@@ -158,4 +173,10 @@ static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
|
||||
"enhanced" : "basic");
|
||||
};
|
||||
|
||||
static inline struct mlx5e_xsk_param *
|
||||
mlx5e_rqo_xsk_param(struct mlx5e_rq_opt_param *rqo)
|
||||
{
|
||||
return rqo ? rqo->xsk : NULL;
|
||||
}
|
||||
|
||||
#endif /* __MLX5_EN_PARAMS_H__ */
|
||||
|
||||
@@ -660,13 +660,13 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev,
|
||||
struct net_device *netdev,
|
||||
struct mlx5e_ptp_params *ptp_params)
|
||||
{
|
||||
struct mlx5e_rq_param *rq_params = &ptp_params->rq_param;
|
||||
struct mlx5e_rq_param *rq_param = &ptp_params->rq_param;
|
||||
struct mlx5e_params *params = &ptp_params->params;
|
||||
|
||||
params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC;
|
||||
mlx5e_init_rq_type_params(mdev, params);
|
||||
params->sw_mtu = netdev->max_mtu;
|
||||
mlx5e_build_rq_param(mdev, params, NULL, rq_params);
|
||||
mlx5e_build_rq_param(mdev, params, NULL, rq_param);
|
||||
}
|
||||
|
||||
static void mlx5e_ptp_build_params(struct mlx5e_ptp *c,
|
||||
|
||||
@@ -37,9 +37,10 @@
|
||||
#include <linux/bitfield.h>
|
||||
#include <net/page_pool/helpers.h>
|
||||
|
||||
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
|
||||
int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
|
||||
struct mlx5e_rq_opt_param *rqo)
|
||||
{
|
||||
int hr = mlx5e_get_linear_rq_headroom(params, xsk);
|
||||
int hr = mlx5e_get_linear_rq_headroom(params, rqo);
|
||||
|
||||
/* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)).
|
||||
* The condition checked in mlx5e_rx_is_linear_skb is:
|
||||
|
||||
@@ -96,7 +96,8 @@ union mlx5e_xdp_info {
|
||||
};
|
||||
|
||||
struct mlx5e_xsk_param;
|
||||
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
|
||||
int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
|
||||
struct mlx5e_rq_opt_param *rqo);
|
||||
bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
|
||||
struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
|
||||
void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
|
||||
|
||||
@@ -79,6 +79,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
|
||||
struct xsk_buff_pool *pool, u16 ix)
|
||||
{
|
||||
struct mlx5e_params *params = &priv->channels.params;
|
||||
struct mlx5e_channel_param *cparam;
|
||||
enum mlx5e_mpwrq_umr_mode umr_mode;
|
||||
struct mlx5e_xsk_param xsk;
|
||||
struct mlx5e_channel *c;
|
||||
int err;
|
||||
@@ -89,18 +91,24 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
|
||||
if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
|
||||
return -EINVAL;
|
||||
|
||||
cparam = kvzalloc_obj(*cparam, GFP_KERNEL);
|
||||
if (!cparam)
|
||||
return -ENOMEM;
|
||||
|
||||
err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool);
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
goto err_free_cparam;
|
||||
|
||||
err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
|
||||
if (unlikely(err))
|
||||
goto err_unmap_pool;
|
||||
|
||||
mlx5e_build_xsk_param(pool, &xsk);
|
||||
mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam);
|
||||
|
||||
umr_mode = mlx5e_mpwrq_umr_mode(priv->mdev, &cparam->rq_opt);
|
||||
if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
|
||||
mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
|
||||
umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
|
||||
const char *recommendation = is_power_of_2(xsk.chunk_size) ?
|
||||
"Upgrade firmware" : "Disable striding RQ";
|
||||
|
||||
@@ -122,7 +130,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
|
||||
|
||||
c = priv->channels.c[ix];
|
||||
|
||||
err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
|
||||
err = mlx5e_open_xsk(priv, params, cparam, pool, c);
|
||||
if (unlikely(err))
|
||||
goto err_remove_pool;
|
||||
|
||||
@@ -138,6 +146,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
|
||||
mlx5e_deactivate_rq(&c->rq);
|
||||
mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);
|
||||
|
||||
kvfree(cparam);
|
||||
|
||||
return 0;
|
||||
|
||||
err_remove_pool:
|
||||
@@ -146,17 +156,22 @@ err_remove_pool:
|
||||
err_unmap_pool:
|
||||
mlx5e_xsk_unmap_pool(priv, pool);
|
||||
|
||||
err_free_cparam:
|
||||
kvfree(cparam);
|
||||
|
||||
return err;
|
||||
|
||||
validate_closed:
|
||||
/* Check the configuration in advance, rather than fail at a later stage
|
||||
* (in mlx5e_xdp_set or on open) and end up with no channels.
|
||||
*/
|
||||
if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
|
||||
if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) {
|
||||
err = -EINVAL;
|
||||
goto err_remove_pool;
|
||||
}
|
||||
|
||||
kvfree(cparam);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
|
||||
static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk)
|
||||
struct mlx5e_rq_opt_param *rqo)
|
||||
{
|
||||
if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
|
||||
if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) {
|
||||
mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
@@ -25,9 +25,14 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
|
||||
#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
|
||||
|
||||
bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk,
|
||||
struct mlx5e_rq_opt_param *rqo,
|
||||
struct mlx5_core_dev *mdev)
|
||||
{
|
||||
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
|
||||
|
||||
if (WARN_ON(!xsk))
|
||||
return false;
|
||||
|
||||
/* AF_XDP doesn't support frames larger than PAGE_SIZE,
|
||||
* and xsk->chunk_size is limited to 65535 bytes.
|
||||
*/
|
||||
@@ -42,21 +47,12 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
|
||||
*/
|
||||
switch (params->rq_wq_type) {
|
||||
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
|
||||
return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
|
||||
return !mlx5e_mpwrq_validate_xsk(mdev, params, rqo);
|
||||
default: /* MLX5_WQ_TYPE_CYCLIC */
|
||||
return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk);
|
||||
return !mlx5e_legacy_rq_validate_xsk(mdev, params, rqo);
|
||||
}
|
||||
}
|
||||
|
||||
static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk,
|
||||
struct mlx5e_channel_param *cparam)
|
||||
{
|
||||
mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq);
|
||||
mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
|
||||
}
|
||||
|
||||
static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
|
||||
struct mlx5e_params *params,
|
||||
struct xsk_buff_pool *pool,
|
||||
@@ -90,19 +86,23 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
|
||||
return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id);
|
||||
}
|
||||
|
||||
static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
|
||||
struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
|
||||
struct mlx5e_xsk_param *xsk)
|
||||
static int mlx5e_open_xsk_rq(struct mlx5e_channel *c,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_channel_param *cparam,
|
||||
struct xsk_buff_pool *pool)
|
||||
{
|
||||
struct mlx5e_rq_param *rq_param = &cparam->rq;
|
||||
struct mlx5e_rq_opt_param *rqo = &cparam->rq_opt;
|
||||
u16 q_counter = c->priv->q_counter[c->sd_ix];
|
||||
struct mlx5e_rq *xskrq = &c->xskrq;
|
||||
int err;
|
||||
|
||||
err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
|
||||
err = mlx5e_init_xsk_rq(c, params, pool, rqo->xsk, xskrq);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq);
|
||||
err = mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu),
|
||||
q_counter, xskrq);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
@@ -111,30 +111,24 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param
|
||||
}
|
||||
|
||||
int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
|
||||
struct mlx5e_channel_param *cparam,
|
||||
struct xsk_buff_pool *pool,
|
||||
struct mlx5e_channel *c)
|
||||
{
|
||||
struct mlx5e_channel_param *cparam;
|
||||
struct mlx5e_create_cq_param ccp;
|
||||
int err;
|
||||
|
||||
mlx5e_build_create_cq_param(&ccp, c);
|
||||
|
||||
if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
|
||||
if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev))
|
||||
return -EINVAL;
|
||||
|
||||
cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL);
|
||||
if (!cparam)
|
||||
return -ENOMEM;
|
||||
|
||||
mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam);
|
||||
|
||||
err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
|
||||
&c->xskrq.cq);
|
||||
if (unlikely(err))
|
||||
goto err_free_cparam;
|
||||
return err;
|
||||
|
||||
err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk);
|
||||
err = mlx5e_open_xsk_rq(c, params, cparam, pool);
|
||||
if (unlikely(err))
|
||||
goto err_close_rx_cq;
|
||||
|
||||
@@ -153,8 +147,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
|
||||
if (unlikely(err))
|
||||
goto err_close_tx_cq;
|
||||
|
||||
kvfree(cparam);
|
||||
|
||||
set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
|
||||
|
||||
return 0;
|
||||
@@ -168,9 +160,6 @@ err_close_rq:
|
||||
err_close_rx_cq:
|
||||
mlx5e_close_cq(&c->xskrq.cq);
|
||||
|
||||
err_free_cparam:
|
||||
kvfree(cparam);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,10 +9,12 @@
|
||||
struct mlx5e_xsk_param;
|
||||
|
||||
bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk,
|
||||
struct mlx5e_rq_opt_param *rqo,
|
||||
struct mlx5_core_dev *mdev);
|
||||
struct mlx5e_channel_param;
|
||||
int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
|
||||
struct mlx5e_channel_param *cparam,
|
||||
struct xsk_buff_pool *pool,
|
||||
struct mlx5e_channel *c);
|
||||
void mlx5e_close_xsk(struct mlx5e_channel *c);
|
||||
void mlx5e_activate_xsk(struct mlx5e_channel *c);
|
||||
|
||||
@@ -636,14 +636,18 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work)
|
||||
|
||||
static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
|
||||
{
|
||||
rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
|
||||
/* xsk can have page_shift < PAGE_SHIFT */
|
||||
u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0);
|
||||
u32 page_size = BIT(PAGE_SHIFT + page_order);
|
||||
|
||||
rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order);
|
||||
if (!rq->wqe_overflow.page)
|
||||
return -ENOMEM;
|
||||
|
||||
rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
|
||||
PAGE_SIZE, rq->buff.map_dir);
|
||||
page_size, rq->buff.map_dir);
|
||||
if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
|
||||
__free_page(rq->wqe_overflow.page);
|
||||
__free_pages(rq->wqe_overflow.page, page_order);
|
||||
return -ENOMEM;
|
||||
}
|
||||
return 0;
|
||||
@@ -651,9 +655,12 @@ static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
|
||||
|
||||
static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
|
||||
{
|
||||
dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
|
||||
rq->buff.map_dir);
|
||||
__free_page(rq->wqe_overflow.page);
|
||||
u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0);
|
||||
u32 page_size = BIT(PAGE_SHIFT + page_order);
|
||||
|
||||
dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size,
|
||||
rq->buff.map_dir);
|
||||
__free_pages(rq->wqe_overflow.page, page_order);
|
||||
}
|
||||
|
||||
static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
|
||||
@@ -780,7 +787,7 @@ static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev,
|
||||
|
||||
static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_rq_param *rqp,
|
||||
struct mlx5e_rq_param *rq_param,
|
||||
struct mlx5e_rq *rq,
|
||||
int node)
|
||||
{
|
||||
@@ -791,7 +798,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
|
||||
if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
|
||||
return 0;
|
||||
|
||||
hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp);
|
||||
hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param);
|
||||
hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE);
|
||||
nentries = hd_buf_size / PAGE_SIZE;
|
||||
if (!nentries) {
|
||||
@@ -851,19 +858,19 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
|
||||
}
|
||||
|
||||
static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
struct mlx5e_xsk_param *xsk,
|
||||
struct mlx5e_rq_param *rqp,
|
||||
struct mlx5e_rq_param *rq_param,
|
||||
struct mlx5e_rq_opt_param *rqo,
|
||||
int node, struct mlx5e_rq *rq)
|
||||
{
|
||||
void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
|
||||
struct mlx5_core_dev *mdev = rq->mdev;
|
||||
void *rqc = rqp->rqc;
|
||||
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
|
||||
u32 pool_order = 0;
|
||||
u32 pool_size;
|
||||
int wq_sz;
|
||||
int err;
|
||||
int i;
|
||||
|
||||
rqp->wq.db_numa_node = node;
|
||||
rq_param->wq.db_numa_node = node;
|
||||
INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
|
||||
INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work);
|
||||
|
||||
@@ -872,28 +879,28 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
|
||||
|
||||
rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
|
||||
rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
|
||||
rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo);
|
||||
pool_size = 1 << params->log_rq_mtu_frames;
|
||||
|
||||
rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
|
||||
|
||||
switch (rq->wq_type) {
|
||||
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
|
||||
err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
|
||||
&rq->wq_ctrl);
|
||||
err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq,
|
||||
&rq->mpwqe.wq, &rq->wq_ctrl);
|
||||
if (err)
|
||||
goto err_rq_xdp_prog;
|
||||
|
||||
err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
|
||||
if (err)
|
||||
goto err_rq_wq_destroy;
|
||||
|
||||
rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
|
||||
|
||||
wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
|
||||
|
||||
rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
|
||||
rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
|
||||
rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
|
||||
err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
|
||||
if (err)
|
||||
goto err_rq_wq_destroy;
|
||||
|
||||
rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
|
||||
rq->mpwqe.pages_per_wqe =
|
||||
mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
|
||||
rq->mpwqe.umr_mode);
|
||||
@@ -905,14 +912,18 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
rq->mpwqe.umr_mode);
|
||||
|
||||
pool_size = rq->mpwqe.pages_per_wqe <<
|
||||
mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
|
||||
mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo);
|
||||
pool_order = rq->mpwqe.page_shift - PAGE_SHIFT;
|
||||
|
||||
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog)
|
||||
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) &&
|
||||
params->xdp_prog)
|
||||
pool_size *= 2; /* additional page per packet for the linear part */
|
||||
|
||||
rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
|
||||
rq->mpwqe.log_stride_sz =
|
||||
mlx5e_mpwqe_get_log_stride_size(mdev, params,
|
||||
rqo);
|
||||
rq->mpwqe.num_strides =
|
||||
BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
|
||||
BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo));
|
||||
rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz);
|
||||
|
||||
rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
|
||||
@@ -925,14 +936,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
if (err)
|
||||
goto err_rq_mkey;
|
||||
|
||||
err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node);
|
||||
err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node);
|
||||
if (err)
|
||||
goto err_free_mpwqe_info;
|
||||
|
||||
break;
|
||||
default: /* MLX5_WQ_TYPE_CYCLIC */
|
||||
err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
|
||||
&rq->wq_ctrl);
|
||||
err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq,
|
||||
&rq->wqe.wq, &rq->wq_ctrl);
|
||||
if (err)
|
||||
goto err_rq_xdp_prog;
|
||||
|
||||
@@ -940,7 +951,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
|
||||
wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
|
||||
|
||||
rq->wqe.info = rqp->frags_info;
|
||||
rq->wqe.info = rq_param->frags_info;
|
||||
rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
|
||||
|
||||
err = mlx5e_init_wqe_alloc_info(rq, node);
|
||||
@@ -948,7 +959,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
goto err_rq_wq_destroy;
|
||||
}
|
||||
|
||||
if (xsk) {
|
||||
if (mlx5e_rqo_xsk_param(rqo)) {
|
||||
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
|
||||
MEM_TYPE_XSK_BUFF_POOL, NULL);
|
||||
if (err)
|
||||
@@ -958,7 +969,13 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
/* Create a page_pool and register it with rxq */
|
||||
struct page_pool_params pp_params = { 0 };
|
||||
|
||||
pp_params.order = 0;
|
||||
if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 >
|
||||
MLX5E_PAGECNT_BIAS_MAX)) {
|
||||
err = -E2BIG;
|
||||
goto err_free_by_rq_type;
|
||||
}
|
||||
|
||||
pp_params.order = pool_order;
|
||||
pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
|
||||
pp_params.pool_size = pool_size;
|
||||
pp_params.nid = node;
|
||||
@@ -966,7 +983,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
|
||||
pp_params.napi = rq->cq.napi;
|
||||
pp_params.netdev = rq->netdev;
|
||||
pp_params.dma_dir = rq->buff.map_dir;
|
||||
pp_params.max_len = PAGE_SIZE;
|
||||
pp_params.max_len = BIT(PAGE_SHIFT + pool_order);
|
||||
pp_params.queue_idx = rq->ix;
|
||||
|
||||
/* Shampo header data split allow for unreadable netmem */
|
||||
@@ -1085,7 +1102,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
|
||||
xdp_rxq_info_unreg(&rq->xdp_rxq);
|
||||
}
|
||||
|
||||
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter)
|
||||
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
|
||||
u16 q_counter)
|
||||
{
|
||||
struct mlx5_core_dev *mdev = rq->mdev;
|
||||
u8 ts_format;
|
||||
@@ -1107,7 +1125,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou
|
||||
rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
|
||||
wq = MLX5_ADDR_OF(rqc, rqc, wq);
|
||||
|
||||
memcpy(rqc, param->rqc, sizeof(param->rqc));
|
||||
memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc));
|
||||
|
||||
MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
|
||||
MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
|
||||
@@ -1323,8 +1341,8 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
|
||||
|
||||
}
|
||||
|
||||
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
|
||||
struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
|
||||
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
|
||||
struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
|
||||
struct mlx5e_rq *rq)
|
||||
{
|
||||
struct mlx5_core_dev *mdev = rq->mdev;
|
||||
@@ -1333,11 +1351,11 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
|
||||
if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
|
||||
__set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
|
||||
|
||||
err = mlx5e_alloc_rq(params, xsk, param, node, rq);
|
||||
err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = mlx5e_create_rq(rq, param, q_counter);
|
||||
err = mlx5e_create_rq(rq, rq_param, q_counter);
|
||||
if (err)
|
||||
goto err_free_rq;
|
||||
|
||||
@@ -2506,17 +2524,20 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
|
||||
return err;
|
||||
}
|
||||
|
||||
static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
|
||||
struct mlx5e_rq_param *rq_params)
|
||||
static int mlx5e_open_rxq_rq(struct mlx5e_channel *c,
|
||||
struct mlx5e_params *params,
|
||||
struct mlx5e_rq_param *rq_param,
|
||||
struct mlx5e_rq_opt_param *rqo)
|
||||
{
|
||||
u16 q_counter = c->priv->q_counter[c->sd_ix];
|
||||
int err;
|
||||
|
||||
err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq);
|
||||
err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq);
|
||||
return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu),
|
||||
q_counter, &c->rq);
|
||||
}
|
||||
|
||||
static struct mlx5e_icosq *
|
||||
@@ -2619,7 +2640,7 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
|
||||
if (err)
|
||||
goto err_close_icosq;
|
||||
|
||||
err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
|
||||
err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt);
|
||||
if (err)
|
||||
goto err_close_sqs;
|
||||
|
||||
@@ -2764,6 +2785,7 @@ static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c)
|
||||
|
||||
static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
|
||||
struct mlx5e_params *params,
|
||||
struct netdev_queue_config *qcfg,
|
||||
struct xsk_buff_pool *xsk_pool,
|
||||
struct mlx5e_channel **cp)
|
||||
{
|
||||
@@ -2797,7 +2819,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
|
||||
goto err_free;
|
||||
}
|
||||
|
||||
err = mlx5e_build_channel_param(mdev, params, cparam);
|
||||
err = mlx5e_build_channel_param(mdev, params, qcfg, cparam);
|
||||
if (err)
|
||||
goto err_free;
|
||||
|
||||
@@ -2828,7 +2850,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
|
||||
|
||||
if (xsk_pool) {
|
||||
mlx5e_build_xsk_param(xsk_pool, &xsk);
|
||||
err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
|
||||
mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam);
|
||||
err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c);
|
||||
if (unlikely(err))
|
||||
goto err_close_queues;
|
||||
}
|
||||
@@ -2921,7 +2944,8 @@ int mlx5e_open_channels(struct mlx5e_priv *priv,
|
||||
if (chs->params.xdp_prog)
|
||||
xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
|
||||
|
||||
err = mlx5e_open_channel(priv, i, &chs->params, xsk_pool, &chs->c[i]);
|
||||
err = mlx5e_open_channel(priv, i, &chs->params, NULL,
|
||||
xsk_pool, &chs->c[i]);
|
||||
if (err)
|
||||
goto err_close_channels;
|
||||
}
|
||||
@@ -3577,15 +3601,14 @@ static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
|
||||
|
||||
static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_rq *rq,
|
||||
struct mlx5e_rq_param *param)
|
||||
struct mlx5e_rq_param *rq_param)
|
||||
{
|
||||
void *rqc = param->rqc;
|
||||
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
|
||||
void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
|
||||
int err;
|
||||
|
||||
param->wq.db_numa_node = param->wq.buf_numa_node;
|
||||
rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node;
|
||||
|
||||
err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq,
|
||||
err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq,
|
||||
&rq->wq_ctrl);
|
||||
if (err)
|
||||
return err;
|
||||
@@ -4586,6 +4609,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
|
||||
for (ix = 0; ix < chs->params.num_channels; ix++) {
|
||||
struct xsk_buff_pool *xsk_pool =
|
||||
mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
|
||||
struct mlx5e_rq_opt_param rqo = {0};
|
||||
struct mlx5e_xsk_param xsk;
|
||||
int max_xdp_mtu;
|
||||
|
||||
@@ -4593,12 +4617,13 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
|
||||
continue;
|
||||
|
||||
mlx5e_build_xsk_param(xsk_pool, &xsk);
|
||||
max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk);
|
||||
rqo.xsk = &xsk;
|
||||
max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo);
|
||||
|
||||
/* Validate XSK params and XDP MTU in advance */
|
||||
if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) ||
|
||||
if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) ||
|
||||
new_params->sw_mtu > max_xdp_mtu) {
|
||||
u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
|
||||
u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo);
|
||||
int max_mtu_frame, max_mtu_page, max_mtu;
|
||||
|
||||
/* Two criteria must be met:
|
||||
@@ -5549,9 +5574,52 @@ static const struct netdev_stat_ops mlx5e_stat_ops = {
|
||||
|
||||
struct mlx5_qmgmt_data {
|
||||
struct mlx5e_channel *c;
|
||||
struct mlx5e_channel_param cparam;
|
||||
};
|
||||
|
||||
static void mlx5e_queue_default_qcfg(struct net_device *dev,
|
||||
struct netdev_queue_config *qcfg)
|
||||
{
|
||||
qcfg->rx_page_size = PAGE_SIZE;
|
||||
}
|
||||
|
||||
static int mlx5e_queue_validate_qcfg(struct net_device *dev,
|
||||
struct netdev_queue_config *qcfg,
|
||||
struct netlink_ext_ack *extack)
|
||||
{
|
||||
struct mlx5e_priv *priv = netdev_priv(dev);
|
||||
struct mlx5_core_dev *mdev = priv->mdev;
|
||||
u32 max;
|
||||
|
||||
if (!is_power_of_2(qcfg->rx_page_size)) {
|
||||
netdev_err(priv->netdev, "rx_page_size not power of 2: %u",
|
||||
qcfg->rx_page_size);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
max = mlx5e_mpwrq_max_page_size(mdev);
|
||||
if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) {
|
||||
netdev_err(priv->netdev,
|
||||
"Selected rx_page_size %u not in supported range [%lu, %u]\n",
|
||||
qcfg->rx_page_size, PAGE_SIZE, max);
|
||||
return -ERANGE;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool mlx5e_queue_validate_page_size(struct net_device *dev,
|
||||
struct netdev_queue_config *qcfg,
|
||||
int queue_index)
|
||||
{
|
||||
if (qcfg->rx_page_size == PAGE_SIZE)
|
||||
return true;
|
||||
|
||||
if (!netif_rxq_has_unreadable_mp(dev, queue_index))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static int mlx5e_queue_mem_alloc(struct net_device *dev,
|
||||
struct netdev_queue_config *qcfg,
|
||||
void *newq, int queue_index)
|
||||
@@ -5560,7 +5628,6 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev,
|
||||
struct mlx5e_priv *priv = netdev_priv(dev);
|
||||
struct mlx5e_channels *chs = &priv->channels;
|
||||
struct mlx5e_params params = chs->params;
|
||||
struct mlx5_core_dev *mdev;
|
||||
int err;
|
||||
|
||||
mutex_lock(&priv->state_lock);
|
||||
@@ -5584,12 +5651,14 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev,
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, queue_index);
|
||||
err = mlx5e_build_channel_param(mdev, ¶ms, &new->cparam);
|
||||
if (err)
|
||||
if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) {
|
||||
netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n");
|
||||
err = -EINVAL;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
err = mlx5e_open_channel(priv, queue_index, ¶ms, NULL, &new->c);
|
||||
err = mlx5e_open_channel(priv, queue_index, ¶ms, qcfg, NULL,
|
||||
&new->c);
|
||||
unlock:
|
||||
mutex_unlock(&priv->state_lock);
|
||||
return err;
|
||||
@@ -5668,6 +5737,9 @@ static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = {
|
||||
.ndo_queue_start = mlx5e_queue_start,
|
||||
.ndo_queue_stop = mlx5e_queue_stop,
|
||||
.ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev,
|
||||
.ndo_default_qcfg = mlx5e_queue_default_qcfg,
|
||||
.ndo_validate_qcfg = mlx5e_queue_validate_qcfg,
|
||||
.supported_params = QCFG_RX_PAGE_SIZE,
|
||||
};
|
||||
|
||||
static void mlx5e_build_nic_netdev(struct net_device *netdev)
|
||||
|
||||
@@ -272,8 +272,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
|
||||
return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem);
|
||||
}
|
||||
|
||||
#define MLX5E_PAGECNT_BIAS_MAX (PAGE_SIZE / 64)
|
||||
|
||||
static int mlx5e_page_alloc_fragmented(struct page_pool *pp,
|
||||
struct mlx5e_frag_page *frag_page)
|
||||
{
|
||||
@@ -1847,11 +1845,14 @@ mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
|
||||
struct mlx5e_frag_page *frag_page,
|
||||
u32 data_bcnt, u32 data_offset)
|
||||
{
|
||||
u32 page_size = BIT(rq->mpwqe.page_shift);
|
||||
|
||||
net_prefetchw(skb->data);
|
||||
|
||||
do {
|
||||
/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
|
||||
u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt);
|
||||
u32 pg_consumed_bytes = min_t(u32, page_size - data_offset,
|
||||
data_bcnt);
|
||||
unsigned int truesize = pg_consumed_bytes;
|
||||
|
||||
mlx5e_add_skb_frag(rq, skb, frag_page, data_offset,
|
||||
@@ -1872,6 +1873,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
|
||||
u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
|
||||
struct mlx5e_frag_page *head_page = frag_page;
|
||||
struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf;
|
||||
u32 page_size = BIT(rq->mpwqe.page_shift);
|
||||
u32 frag_offset = head_offset;
|
||||
u32 byte_cnt = cqe_bcnt;
|
||||
struct skb_shared_info *sinfo;
|
||||
@@ -1926,9 +1928,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
|
||||
linear_hr = skb_headroom(skb);
|
||||
linear_data_len = headlen;
|
||||
linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb));
|
||||
if (unlikely(frag_offset >= PAGE_SIZE)) {
|
||||
if (unlikely(frag_offset >= page_size)) {
|
||||
frag_page++;
|
||||
frag_offset -= PAGE_SIZE;
|
||||
frag_offset -= page_size;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1940,7 +1942,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
|
||||
while (byte_cnt) {
|
||||
/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
|
||||
pg_consumed_bytes =
|
||||
min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
|
||||
min_t(u32, page_size - frag_offset, byte_cnt);
|
||||
|
||||
if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
|
||||
truesize += pg_consumed_bytes;
|
||||
@@ -1978,7 +1980,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
|
||||
nr_frags_free = old_nr_frags - sinfo->nr_frags;
|
||||
if (unlikely(nr_frags_free)) {
|
||||
frag_page -= nr_frags_free;
|
||||
truesize -= (nr_frags_free - 1) * PAGE_SIZE +
|
||||
truesize -= (nr_frags_free - 1) * page_size +
|
||||
ALIGN(pg_consumed_bytes,
|
||||
BIT(rq->mpwqe.log_stride_sz));
|
||||
}
|
||||
@@ -2166,15 +2168,16 @@ mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
|
||||
rq->hw_gro_data->skb = NULL;
|
||||
}
|
||||
|
||||
static bool
|
||||
mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt)
|
||||
static bool mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb,
|
||||
u16 data_bcnt,
|
||||
u32 page_size)
|
||||
{
|
||||
int nr_frags = skb_shinfo(skb)->nr_frags;
|
||||
|
||||
if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE)
|
||||
if (page_size >= GRO_LEGACY_MAX_SIZE)
|
||||
return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE;
|
||||
else
|
||||
return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
|
||||
return page_size * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
|
||||
}
|
||||
|
||||
static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
|
||||
@@ -2183,18 +2186,19 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
|
||||
u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe);
|
||||
u32 wqe_offset = be32_to_cpu(cqe->shampo.data_offset);
|
||||
u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
|
||||
u32 data_offset = wqe_offset & (PAGE_SIZE - 1);
|
||||
u32 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
|
||||
u16 wqe_id = be16_to_cpu(cqe->wqe_id);
|
||||
u32 page_idx = wqe_offset >> PAGE_SHIFT;
|
||||
u16 head_size = cqe->shampo.header_size;
|
||||
struct sk_buff **skb = &rq->hw_gro_data->skb;
|
||||
bool flush = cqe->shampo.flush;
|
||||
bool match = cqe->shampo.match;
|
||||
u32 page_size = BIT(rq->mpwqe.page_shift);
|
||||
struct mlx5e_rq_stats *stats = rq->stats;
|
||||
struct mlx5e_rx_wqe_ll *wqe;
|
||||
struct mlx5e_mpw_info *wi;
|
||||
struct mlx5_wq_ll *wq;
|
||||
u32 data_offset;
|
||||
u32 page_idx;
|
||||
|
||||
wi = mlx5e_get_mpw_info(rq, wqe_id);
|
||||
wi->consumed_strides += cstrides;
|
||||
@@ -2210,7 +2214,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
|
||||
goto mpwrq_cqe_out;
|
||||
}
|
||||
|
||||
if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) {
|
||||
data_offset = wqe_offset & (page_size - 1);
|
||||
page_idx = wqe_offset >> rq->mpwqe.page_shift;
|
||||
if (*skb &&
|
||||
!(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt,
|
||||
page_size))) {
|
||||
match = false;
|
||||
mlx5e_shampo_flush_skb(rq, cqe, match);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user