linuxmsm/laptops-kernel
0
0
Fork 0
mirror of https://github.com/linux-msm/laptops-kernel.git synced 2026-08-13 14:19:53 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dpaa2-eth-AF_XDP-zc'

Browse Source 
Ioana Ciornei says:

====================
net: dpaa2-eth: AF_XDP zero-copy support

This patch set adds support for AF_XDP zero-copy in the dpaa2-eth
driver. The support is available on the LX2160A SoC and its variants and
only on interfaces (DPNIs) with a maximum of 8 queues (HW limitations
are the root cause).

We are first implementing the .get_channels() callback since this a
dependency for further work.

Patches 2-3 are working on making the necessary changes for multiple
buffer pools on a single interface. By default, without an AF_XDP socket
attached, only a single buffer pool will be used and shared between all
the queues. The changes in the functions are made in this patch, but the
actual allocation and setup of a new BP is done in patch#10.

Patches 4-5 are improving the information exposed in debugfs. We are
exposing a new file to show which buffer pool is used by what channels
and how many buffers it currently has.

The 6th patch updates the dpni_set_pools() firmware API so that we are
capable of setting up a different buffer per queue in later patches.

In the 7th patch the generic dev_open/close APIs are used instead of the
dpaa2-eth internal ones.

Patches 8-9 are rearranging the existing code in dpaa2-eth.c in order to
create new functions which will be used in the XSK implementation in
dpaa2-xsk.c

Finally, the last 3 patches are adding the actual support for both the
Rx and Tx path of AF_XDP zero-copy and some associated tracepoints.
Details on the implementation can be found in the actual patch.

Changes in v2:
 - 3/12:  Export dpaa2_eth_allocate_dpbp/dpaa2_eth_free_dpbp in this
   patch to avoid a build warning. The functions will be used in next
   patches.
 - 6/12:  Use __le16 instead of u16 for the dpbp_id field.
 - 12/12: Use xdp_buff->data_hard_start when tracing the BP seeding.

Changes in v3:
 - 3/12: fix leaking of bp on error path
====================

Acked-by: Björn Töpel <bjorn@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
2022-10-24 09:22:12 +01:00
parent 3bd5549bd4 3817b2ac71
commit 225480f040
11 changed files with 1089 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MAINTAINERS
+1
View File
@@ -6326,6 +6326,7 @@ F: drivers/net/ethernet/freescale/dpaa2/Kconfig
F: drivers/net/ethernet/freescale/dpaa2/Makefile
F: drivers/net/ethernet/freescale/dpaa2/dpaa2-eth*
F: drivers/net/ethernet/freescale/dpaa2/dpaa2-mac*
F: drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk*
F: drivers/net/ethernet/freescale/dpaa2/dpkg.h
F: drivers/net/ethernet/freescale/dpaa2/dpmac*
F: drivers/net/ethernet/freescale/dpaa2/dpni*
drivers/net/ethernet/freescale/dpaa2/Makefile
+1 -1
View File
@@ -7,7 +7,7 @@ obj-$(CONFIG_FSL_DPAA2_ETH) += fsl-dpaa2-eth.o
obj-$(CONFIG_FSL_DPAA2_PTP_CLOCK) += fsl-dpaa2-ptp.o
obj-$(CONFIG_FSL_DPAA2_SWITCH) += fsl-dpaa2-switch.o
fsl-dpaa2-eth-objs := dpaa2-eth.o dpaa2-ethtool.o dpni.o dpaa2-mac.o dpmac.o dpaa2-eth-devlink.o
fsl-dpaa2-eth-objs := dpaa2-eth.o dpaa2-ethtool.o dpni.o dpaa2-mac.o dpmac.o dpaa2-eth-devlink.o dpaa2-xsk.o
fsl-dpaa2-eth-${CONFIG_FSL_DPAA2_ETH_DCB} += dpaa2-eth-dcb.o
fsl-dpaa2-eth-${CONFIG_DEBUG_FS} += dpaa2-eth-debugfs.o
fsl-dpaa2-ptp-objs := dpaa2-ptp.o dprtc.o
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.c
+53 -4
View File
@@ -98,14 +98,14 @@ static int dpaa2_dbg_ch_show(struct seq_file *file, void *offset)
int i;
seq_printf(file, "Channel stats for %s:\n", priv->net_dev->name);
seq_printf(file, "%s%16s%16s%16s%16s%16s%16s\n",
"CHID", "CPU", "Deq busy", "Frames", "CDANs",
seq_printf(file, "%s %5s%16s%16s%16s%16s%16s%16s\n",
"IDX", "CHID", "CPU", "Deq busy", "Frames", "CDANs",
"Avg Frm/CDAN", "Buf count");
for (i = 0; i < priv->num_channels; i++) {
ch = priv->channel[i];
seq_printf(file, "%4d%16d%16llu%16llu%16llu%16llu%16d\n",
ch->ch_id,
seq_printf(file, "%3s%d%6d%16d%16llu%16llu%16llu%16llu%16d\n",
"CH#", i, ch->ch_id,
ch->nctx.desired_cpu,
ch->stats.dequeue_portal_busy,
ch->stats.frames,
@@ -119,6 +119,51 @@ static int dpaa2_dbg_ch_show(struct seq_file *file, void *offset)
DEFINE_SHOW_ATTRIBUTE(dpaa2_dbg_ch);
static int dpaa2_dbg_bp_show(struct seq_file *file, void *offset)
{
struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)file->private;
int i, j, num_queues, buf_cnt;
struct dpaa2_eth_bp *bp;
char ch_name[10];
int err;
/* Print out the header */
seq_printf(file, "Buffer pool info for %s:\n", priv->net_dev->name);
seq_printf(file, "%s %10s%15s", "IDX", "BPID", "Buf count");
num_queues = dpaa2_eth_queue_count(priv);
for (i = 0; i < num_queues; i++) {
snprintf(ch_name, sizeof(ch_name), "CH#%d", i);
seq_printf(file, "%10s", ch_name);
}
seq_printf(file, "\n");
/* For each buffer pool, print out its BPID, the number of buffers in
* that buffer pool and the channels which are using it.
*/
for (i = 0; i < priv->num_bps; i++) {
bp = priv->bp[i];
err = dpaa2_io_query_bp_count(NULL, bp->bpid, &buf_cnt);
if (err) {
netdev_warn(priv->net_dev, "Buffer count query error %d\n", err);
return err;
}
seq_printf(file, "%3s%d%10d%15d", "BP#", i, bp->bpid, buf_cnt);
for (j = 0; j < num_queues; j++) {
if (priv->channel[j]->bp == bp)
seq_printf(file, "%10s", "x");
else
seq_printf(file, "%10s", "");
}
seq_printf(file, "\n");
}
return 0;
}
DEFINE_SHOW_ATTRIBUTE(dpaa2_dbg_bp);
void dpaa2_dbg_add(struct dpaa2_eth_priv *priv)
{
struct fsl_mc_device *dpni_dev;
@@ -139,6 +184,10 @@ void dpaa2_dbg_add(struct dpaa2_eth_priv *priv)
/* per-fq stats file */
debugfs_create_file("ch_stats", 0444, dir, priv, &dpaa2_dbg_ch_fops);
/* per buffer pool stats file */
debugfs_create_file("bp_stats", 0444, dir, priv, &dpaa2_dbg_bp_fops);
}
void dpaa2_dbg_remove(struct dpaa2_eth_priv *priv)
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-trace.h
+87 -47
View File
@@ -73,6 +73,14 @@ DEFINE_EVENT(dpaa2_eth_fd, dpaa2_tx_fd,
TP_ARGS(netdev, fd)
);
/* Tx (egress) XSK fd */
DEFINE_EVENT(dpaa2_eth_fd, dpaa2_tx_xsk_fd,
TP_PROTO(struct net_device *netdev,
const struct dpaa2_fd *fd),
TP_ARGS(netdev, fd)
);
/* Rx fd */
DEFINE_EVENT(dpaa2_eth_fd, dpaa2_rx_fd,
TP_PROTO(struct net_device *netdev,
@@ -81,6 +89,14 @@ DEFINE_EVENT(dpaa2_eth_fd, dpaa2_rx_fd,
TP_ARGS(netdev, fd)
);
/* Rx XSK fd */
DEFINE_EVENT(dpaa2_eth_fd, dpaa2_rx_xsk_fd,
TP_PROTO(struct net_device *netdev,
const struct dpaa2_fd *fd),
TP_ARGS(netdev, fd)
);
/* Tx confirmation fd */
DEFINE_EVENT(dpaa2_eth_fd, dpaa2_tx_conf_fd,
TP_PROTO(struct net_device *netdev,
@@ -90,57 +106,81 @@ DEFINE_EVENT(dpaa2_eth_fd, dpaa2_tx_conf_fd,
);
/* Log data about raw buffers. Useful for tracing DPBP content. */
TRACE_EVENT(dpaa2_eth_buf_seed,
/* Trace function prototype */
TP_PROTO(struct net_device *netdev,
/* virtual address and size */
void *vaddr,
size_t size,
/* dma map address and size */
dma_addr_t dma_addr,
size_t map_size,
/* buffer pool id, if relevant */
u16 bpid),
DECLARE_EVENT_CLASS(dpaa2_eth_buf,
/* Trace function prototype */
TP_PROTO(struct net_device *netdev,
/* virtual address and size */
void *vaddr,
size_t size,
/* dma map address and size */
dma_addr_t dma_addr,
size_t map_size,
/* buffer pool id, if relevant */
u16 bpid),
/* Repeat argument list here */
TP_ARGS(netdev, vaddr, size, dma_addr, map_size, bpid),
/* Repeat argument list here */
TP_ARGS(netdev, vaddr, size, dma_addr, map_size, bpid),
/* A structure containing the relevant information we want
* to record. Declare name and type for each normal element,
* name, type and size for arrays. Use __string for variable
* length strings.
*/
TP_STRUCT__entry(
__field(void *, vaddr)
__field(size_t, size)
__field(dma_addr_t, dma_addr)
__field(size_t, map_size)
__field(u16, bpid)
__string(name, netdev->name)
),
/* A structure containing the relevant information we want
* to record. Declare name and type for each normal element,
* name, type and size for arrays. Use __string for variable
* length strings.
*/
TP_STRUCT__entry(
__field(void *, vaddr)
__field(size_t, size)
__field(dma_addr_t, dma_addr)
__field(size_t, map_size)
__field(u16, bpid)
__string(name, netdev->name)
),
/* The function that assigns values to the above declared
* fields
*/
TP_fast_assign(
__entry->vaddr = vaddr;
__entry->size = size;
__entry->dma_addr = dma_addr;
__entry->map_size = map_size;
__entry->bpid = bpid;
__assign_str(name, netdev->name);
),
/* The function that assigns values to the above declared
* fields
*/
TP_fast_assign(
__entry->vaddr = vaddr;
__entry->size = size;
__entry->dma_addr = dma_addr;
__entry->map_size = map_size;
__entry->bpid = bpid;
__assign_str(name, netdev->name);
),
/* This is what gets printed when the trace event is
* triggered.
*/
TP_printk(TR_BUF_FMT,
__get_str(name),
__entry->vaddr,
__entry->size,
&__entry->dma_addr,
__entry->map_size,
__entry->bpid)
/* This is what gets printed when the trace event is
* triggered.
*/
TP_printk(TR_BUF_FMT,
__get_str(name),
__entry->vaddr,
__entry->size,
&__entry->dma_addr,
__entry->map_size,
__entry->bpid)
);
/* Main memory buff seeding */
DEFINE_EVENT(dpaa2_eth_buf, dpaa2_eth_buf_seed,
TP_PROTO(struct net_device *netdev,
void *vaddr,
size_t size,
dma_addr_t dma_addr,
size_t map_size,
u16 bpid),
TP_ARGS(netdev, vaddr, size, dma_addr, map_size, bpid)
);
/* UMEM buff seeding on AF_XDP fast path */
DEFINE_EVENT(dpaa2_eth_buf, dpaa2_xsk_buf_seed,
TP_PROTO(struct net_device *netdev,
void *vaddr,
size_t size,
dma_addr_t dma_addr,
size_t map_size,
u16 bpid),
TP_ARGS(netdev, vaddr, size, dma_addr, map_size, bpid)
);
/* If only one event of a certain type needs to be declared, use TRACE_EVENT().
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+332 -153
View File
File diff suppressed because it is too large Load Diff
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
+93 -8
View File
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
/* Copyright 2014-2016 Freescale Semiconductor Inc.
* Copyright 2016-2020 NXP
* Copyright 2016-2022 NXP
*/
#ifndef __DPAA2_ETH_H
@@ -53,6 +53,12 @@
*/
#define DPAA2_ETH_TXCONF_PER_NAPI 256
/* Maximum number of Tx frames to be processed in a single NAPI
* call when AF_XDP is running. Bind it to DPAA2_ETH_TXCONF_PER_NAPI
* to maximize the throughput.
*/
#define DPAA2_ETH_TX_ZC_PER_NAPI DPAA2_ETH_TXCONF_PER_NAPI
/* Buffer qouta per channel. We want to keep in check number of ingress frames
* in flight: for small sized frames, congestion group taildrop may kick in
* first; for large sizes, Rx FQ taildrop threshold will ensure only a
@@ -109,6 +115,14 @@
#define DPAA2_ETH_RX_BUF_ALIGN_REV1 256
#define DPAA2_ETH_RX_BUF_ALIGN 64
/* The firmware allows assigning multiple buffer pools to a single DPNI -
* maximum 8 DPBP objects. By default, only the first DPBP (idx 0) is used for
* all queues. Thus, when enabling AF_XDP we must accommodate up to 9 DPBPs
* object: the default and 8 other distinct buffer pools, one for each queue.
*/
#define DPAA2_ETH_DEFAULT_BP_IDX 0
#define DPAA2_ETH_MAX_BPS 9
/* We are accommodating a skb backpointer and some S/G info
* in the frame's software annotation. The hardware
* options are either 0 or 64, so we choose the latter.
@@ -122,6 +136,7 @@ enum dpaa2_eth_swa_type {
DPAA2_ETH_SWA_SINGLE,
DPAA2_ETH_SWA_SG,
DPAA2_ETH_SWA_XDP,
DPAA2_ETH_SWA_XSK,
DPAA2_ETH_SWA_SW_TSO,
};
@@ -143,6 +158,10 @@ struct dpaa2_eth_swa {
int dma_size;
struct xdp_frame *xdpf;
} xdp;
struct {
struct xdp_buff *xdp_buff;
int sgt_size;
} xsk;
struct {
struct sk_buff *skb;
int num_sg;
@@ -421,12 +440,19 @@ enum dpaa2_eth_fq_type {
};
struct dpaa2_eth_priv;
struct dpaa2_eth_channel;
struct dpaa2_eth_fq;
struct dpaa2_eth_xdp_fds {
struct dpaa2_fd fds[DEV_MAP_BULK_SIZE];
ssize_t num;
};
typedef void dpaa2_eth_consume_cb_t(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
const struct dpaa2_fd *fd,
struct dpaa2_eth_fq *fq);
struct dpaa2_eth_fq {
u32 fqid;
u32 tx_qdbin;
@@ -439,10 +465,7 @@ struct dpaa2_eth_fq {
struct dpaa2_eth_channel *channel;
enum dpaa2_eth_fq_type type;
void (*consume)(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
const struct dpaa2_fd *fd,
struct dpaa2_eth_fq *fq);
dpaa2_eth_consume_cb_t *consume;
struct dpaa2_eth_fq_stats stats;
struct dpaa2_eth_xdp_fds xdp_redirect_fds;
@@ -454,6 +477,11 @@ struct dpaa2_eth_ch_xdp {
unsigned int res;
};
struct dpaa2_eth_bp {
struct fsl_mc_device *dev;
int bpid;
};
struct dpaa2_eth_channel {
struct dpaa2_io_notification_ctx nctx;
struct fsl_mc_device *dpcon;
@@ -472,6 +500,11 @@ struct dpaa2_eth_channel {
/* Buffers to be recycled back in the buffer pool */
u64 recycled_bufs[DPAA2_ETH_BUFS_PER_CMD];
int recycled_bufs_cnt;
bool xsk_zc;
int xsk_tx_pkts_sent;
struct xsk_buff_pool *xsk_pool;
struct dpaa2_eth_bp *bp;
};
struct dpaa2_eth_dist_fields {
@@ -506,7 +539,7 @@ struct dpaa2_eth_trap_data {
#define DPAA2_ETH_DEFAULT_COPYBREAK 512
#define DPAA2_ETH_ENQUEUE_MAX_FDS 200
#define DPAA2_ETH_ENQUEUE_MAX_FDS 256
struct dpaa2_eth_fds {
struct dpaa2_fd array[DPAA2_ETH_ENQUEUE_MAX_FDS];
};
@@ -535,14 +568,16 @@ struct dpaa2_eth_priv {
u8 ptp_correction_off;
void (*dpaa2_set_onestep_params_cb)(struct dpaa2_eth_priv *priv,
u32 offset, u8 udp);
struct fsl_mc_device *dpbp_dev;
u16 rx_buf_size;
u16 bpid;
struct iommu_domain *iommu_domain;
enum hwtstamp_tx_types tx_tstamp_type; /* Tx timestamping type */
bool rx_tstamp; /* Rx timestamping enabled */
/* Buffer pool management */
struct dpaa2_eth_bp *bp[DPAA2_ETH_MAX_BPS];
int num_bps;
u16 tx_qdid;
struct fsl_mc_io *mc_io;
/* Cores which have an affine DPIO/DPCON.
@@ -771,4 +806,54 @@ void dpaa2_eth_dl_traps_unregister(struct dpaa2_eth_priv *priv);
struct dpaa2_eth_trap_item *dpaa2_eth_dl_get_trap(struct dpaa2_eth_priv *priv,
struct dpaa2_fapr *fapr);
struct dpaa2_eth_bp *dpaa2_eth_allocate_dpbp(struct dpaa2_eth_priv *priv);
void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv, struct dpaa2_eth_bp *bp);
struct sk_buff *dpaa2_eth_alloc_skb(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
const struct dpaa2_fd *fd, u32 fd_length,
void *fd_vaddr);
void dpaa2_eth_receive_skb(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
const struct dpaa2_fd *fd, void *vaddr,
struct dpaa2_eth_fq *fq,
struct rtnl_link_stats64 *percpu_stats,
struct sk_buff *skb);
void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
const struct dpaa2_fd *fd,
struct dpaa2_eth_fq *fq);
struct dpaa2_eth_bp *dpaa2_eth_allocate_dpbp(struct dpaa2_eth_priv *priv);
void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_bp *bp);
void *dpaa2_iova_to_virt(struct iommu_domain *domain, dma_addr_t iova_addr);
void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
dma_addr_t addr);
void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
struct dpaa2_fd *fd,
void *buf_start, u16 queue_id);
int dpaa2_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags);
int dpaa2_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid);
void dpaa2_eth_free_tx_fd(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
struct dpaa2_eth_fq *fq,
const struct dpaa2_fd *fd, bool in_napi);
bool dpaa2_xsk_tx(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch);
/* SGT (Scatter-Gather Table) cache management */
void *dpaa2_eth_sgt_get(struct dpaa2_eth_priv *priv);
void dpaa2_eth_sgt_recycle(struct dpaa2_eth_priv *priv, void *sgt_buf);
#endif /* __DPAA2_H */
drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c
+41 -17
View File
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/* Copyright 2014-2016 Freescale Semiconductor Inc.
* Copyright 2016 NXP
* Copyright 2020 NXP
* Copyright 2016-2022 NXP
*/
#include <linux/net_tstamp.h>
@@ -227,17 +226,8 @@ static void dpaa2_eth_get_ethtool_stats(struct net_device *net_dev,
struct ethtool_stats *stats,
u64 *data)
{
int i = 0;
int j, k, err;
int num_cnt;
union dpni_statistics dpni_stats;
u32 fcnt, bcnt;
u32 fcnt_rx_total = 0, fcnt_tx_total = 0;
u32 bcnt_rx_total = 0, bcnt_tx_total = 0;
u32 buf_cnt;
struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
struct dpaa2_eth_drv_stats *extras;
struct dpaa2_eth_ch_stats *ch_stats;
union dpni_statistics dpni_stats;
int dpni_stats_page_size[DPNI_STATISTICS_CNT] = {
sizeof(dpni_stats.page_0),
sizeof(dpni_stats.page_1),
@@ -247,6 +237,13 @@ static void dpaa2_eth_get_ethtool_stats(struct net_device *net_dev,
sizeof(dpni_stats.page_5),
sizeof(dpni_stats.page_6),
};
u32 fcnt_rx_total = 0, fcnt_tx_total = 0;
u32 bcnt_rx_total = 0, bcnt_tx_total = 0;
struct dpaa2_eth_ch_stats *ch_stats;
struct dpaa2_eth_drv_stats *extras;
u32 buf_cnt, buf_cnt_total = 0;
int j, k, err, num_cnt, i = 0;
u32 fcnt, bcnt;
memset(data, 0,
sizeof(u64) * (DPAA2_ETH_NUM_STATS + DPAA2_ETH_NUM_EXTRA_STATS));
@@ -308,12 +305,15 @@ static void dpaa2_eth_get_ethtool_stats(struct net_device *net_dev,
*(data + i++) = fcnt_tx_total;
*(data + i++) = bcnt_tx_total;
err = dpaa2_io_query_bp_count(NULL, priv->bpid, &buf_cnt);
if (err) {
netdev_warn(net_dev, "Buffer count query error %d\n", err);
return;
for (j = 0; j < priv->num_bps; j++) {
err = dpaa2_io_query_bp_count(NULL, priv->bp[j]->bpid, &buf_cnt);
if (err) {
netdev_warn(net_dev, "Buffer count query error %d\n", err);
return;
}
buf_cnt_total += buf_cnt;
}
*(data + i++) = buf_cnt;
*(data + i++) = buf_cnt_total;
if (dpaa2_eth_has_mac(priv))
dpaa2_mac_get_ethtool_stats(priv->mac, data + i);
@@ -876,6 +876,29 @@ restore_rx_usecs:
return err;
}
static void dpaa2_eth_get_channels(struct net_device *net_dev,
struct ethtool_channels *channels)
{
struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
int queue_count = dpaa2_eth_queue_count(priv);
channels->max_rx = queue_count;
channels->max_tx = queue_count;
channels->rx_count = queue_count;
channels->tx_count = queue_count;
/* Tx confirmation and Rx error */
channels->max_other = queue_count + 1;
channels->max_combined = channels->max_rx +
channels->max_tx +
channels->max_other;
/* Tx conf and Rx err */
channels->other_count = queue_count + 1;
channels->combined_count = channels->rx_count +
channels->tx_count +
channels->other_count;
}
const struct ethtool_ops dpaa2_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
@@ -896,4 +919,5 @@ const struct ethtool_ops dpaa2_ethtool_ops = {
.set_tunable = dpaa2_eth_set_tunable,
.get_coalesce = dpaa2_eth_get_coalesce,
.set_coalesce = dpaa2_eth_set_coalesce,
.get_channels = dpaa2_eth_get_channels,
};
drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c
+454
View File
@@ -0,0 +1,454 @@
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/* Copyright 2022 NXP
*/
#include <linux/filter.h>
#include <linux/compiler.h>
#include <linux/bpf_trace.h>
#include <net/xdp.h>
#include <net/xdp_sock_drv.h>
#include "dpaa2-eth.h"
static void dpaa2_eth_setup_consume_func(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
enum dpaa2_eth_fq_type type,
dpaa2_eth_consume_cb_t *consume)
{
struct dpaa2_eth_fq *fq;
int i;
for (i = 0; i < priv->num_fqs; i++) {
fq = &priv->fq[i];
if (fq->type != type)
continue;
if (fq->channel != ch)
continue;
fq->consume = consume;
}
}
static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
struct dpaa2_eth_fq *rx_fq,
struct dpaa2_fd *fd, void *vaddr)
{
dma_addr_t addr = dpaa2_fd_get_addr(fd);
struct bpf_prog *xdp_prog;
struct xdp_buff *xdp_buff;
struct dpaa2_eth_swa *swa;
u32 xdp_act = XDP_PASS;
int err;
xdp_prog = READ_ONCE(ch->xdp.prog);
if (!xdp_prog)
goto out;
swa = (struct dpaa2_eth_swa *)(vaddr + DPAA2_ETH_RX_HWA_SIZE +
ch->xsk_pool->umem->headroom);
xdp_buff = swa->xsk.xdp_buff;
xdp_buff->data_hard_start = vaddr;
xdp_buff->data = vaddr + dpaa2_fd_get_offset(fd);
xdp_buff->data_end = xdp_buff->data + dpaa2_fd_get_len(fd);
xdp_set_data_meta_invalid(xdp_buff);
xdp_buff->rxq = &ch->xdp_rxq;
xsk_buff_dma_sync_for_cpu(xdp_buff, ch->xsk_pool);
xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff);
/* xdp.data pointer may have changed */
dpaa2_fd_set_offset(fd, xdp_buff->data - vaddr);
dpaa2_fd_set_len(fd, xdp_buff->data_end - xdp_buff->data);
if (likely(xdp_act == XDP_REDIRECT)) {
err = xdp_do_redirect(priv->net_dev, xdp_buff, xdp_prog);
if (unlikely(err)) {
ch->stats.xdp_drop++;
dpaa2_eth_recycle_buf(priv, ch, addr);
} else {
ch->buf_count--;
ch->stats.xdp_redirect++;
}
goto xdp_redir;
}
switch (xdp_act) {
case XDP_PASS:
break;
case XDP_TX:
dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);
break;
default:
bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
fallthrough;
case XDP_DROP:
dpaa2_eth_recycle_buf(priv, ch, addr);
ch->stats.xdp_drop++;
break;
}
xdp_redir:
ch->xdp.res |= xdp_act;
out:
return xdp_act;
}
/* Rx frame processing routine for the AF_XDP fast path */
static void dpaa2_xsk_rx(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
const struct dpaa2_fd *fd,
struct dpaa2_eth_fq *fq)
{
dma_addr_t addr = dpaa2_fd_get_addr(fd);
u8 fd_format = dpaa2_fd_get_format(fd);
struct rtnl_link_stats64 *percpu_stats;
u32 fd_length = dpaa2_fd_get_len(fd);
struct sk_buff *skb;
void *vaddr;
u32 xdp_act;
trace_dpaa2_rx_xsk_fd(priv->net_dev, fd);
vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
percpu_stats = this_cpu_ptr(priv->percpu_stats);
if (fd_format != dpaa2_fd_single) {
WARN_ON(priv->xdp_prog);
/* AF_XDP doesn't support any other formats */
goto err_frame_format;
}
xdp_act = dpaa2_xsk_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);
if (xdp_act != XDP_PASS) {
percpu_stats->rx_packets++;
percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
return;
}
/* Build skb */
skb = dpaa2_eth_alloc_skb(priv, ch, fd, fd_length, vaddr);
if (!skb)
/* Nothing else we can do, recycle the buffer and
* drop the frame.
*/
goto err_alloc_skb;
/* Send the skb to the Linux networking stack */
dpaa2_eth_receive_skb(priv, ch, fd, vaddr, fq, percpu_stats, skb);
return;
err_alloc_skb:
dpaa2_eth_recycle_buf(priv, ch, addr);
err_frame_format:
percpu_stats->rx_dropped++;
}
static void dpaa2_xsk_set_bp_per_qdbin(struct dpaa2_eth_priv *priv,
struct dpni_pools_cfg *pools_params)
{
int curr_bp = 0, i, j;
pools_params->pool_options = DPNI_POOL_ASSOC_QDBIN;
for (i = 0; i < priv->num_bps; i++) {
for (j = 0; j < priv->num_channels; j++)
if (priv->bp[i] == priv->channel[j]->bp)
pools_params->pools[curr_bp].priority_mask |= (1 << j);
if (!pools_params->pools[curr_bp].priority_mask)
continue;
pools_params->pools[curr_bp].dpbp_id = priv->bp[i]->bpid;
pools_params->pools[curr_bp].buffer_size = priv->rx_buf_size;
pools_params->pools[curr_bp++].backup_pool = 0;
}
pools_params->num_dpbp = curr_bp;
}
static int dpaa2_xsk_disable_pool(struct net_device *dev, u16 qid)
{
struct xsk_buff_pool *pool = xsk_get_pool_from_qid(dev, qid);
struct dpaa2_eth_priv *priv = netdev_priv(dev);
struct dpni_pools_cfg pools_params = { 0 };
struct dpaa2_eth_channel *ch;
int err;
bool up;
ch = priv->channel[qid];
if (!ch->xsk_pool)
return -EINVAL;
up = netif_running(dev);
if (up)
dev_close(dev);
xsk_pool_dma_unmap(pool, 0);
err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq,
MEM_TYPE_PAGE_ORDER0, NULL);
if (err)
netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed (err = %d)\n",
err);
dpaa2_eth_free_dpbp(priv, ch->bp);
ch->xsk_zc = false;
ch->xsk_pool = NULL;
ch->xsk_tx_pkts_sent = 0;
ch->bp = priv->bp[DPAA2_ETH_DEFAULT_BP_IDX];
dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_eth_rx);
dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);
err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
if (err)
netdev_err(dev, "dpni_set_pools() failed\n");
if (up) {
err = dev_open(dev, NULL);
if (err)
return err;
}
return 0;
}
static int dpaa2_xsk_enable_pool(struct net_device *dev,
struct xsk_buff_pool *pool,
u16 qid)
{
struct dpaa2_eth_priv *priv = netdev_priv(dev);
struct dpni_pools_cfg pools_params = { 0 };
struct dpaa2_eth_channel *ch;
int err, err2;
bool up;
if (priv->dpni_attrs.wriop_version < DPAA2_WRIOP_VERSION(3, 0, 0)) {
netdev_err(dev, "AF_XDP zero-copy not supported on devices <= WRIOP(3, 0, 0)\n");
return -EOPNOTSUPP;
}
if (priv->dpni_attrs.num_queues > 8) {
netdev_err(dev, "AF_XDP zero-copy not supported on DPNI with more then 8 queues\n");
return -EOPNOTSUPP;
}
up = netif_running(dev);
if (up)
dev_close(dev);
err = xsk_pool_dma_map(pool, priv->net_dev->dev.parent, 0);
if (err) {
netdev_err(dev, "xsk_pool_dma_map() failed (err = %d)\n",
err);
goto err_dma_unmap;
}
ch = priv->channel[qid];
err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL);
if (err) {
netdev_err(dev, "xdp_rxq_info_reg_mem_model() failed (err = %d)\n", err);
goto err_mem_model;
}
xsk_pool_set_rxq_info(pool, &ch->xdp_rxq);
priv->bp[priv->num_bps] = dpaa2_eth_allocate_dpbp(priv);
if (IS_ERR(priv->bp[priv->num_bps])) {
err = PTR_ERR(priv->bp[priv->num_bps]);
goto err_bp_alloc;
}
ch->xsk_zc = true;
ch->xsk_pool = pool;
ch->bp = priv->bp[priv->num_bps++];
dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_xsk_rx);
dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);
err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
if (err) {
netdev_err(dev, "dpni_set_pools() failed\n");
goto err_set_pools;
}
if (up) {
err = dev_open(dev, NULL);
if (err)
return err;
}
return 0;
err_set_pools:
err2 = dpaa2_xsk_disable_pool(dev, qid);
if (err2)
netdev_err(dev, "dpaa2_xsk_disable_pool() failed %d\n", err2);
err_bp_alloc:
err2 = xdp_rxq_info_reg_mem_model(&priv->channel[qid]->xdp_rxq,
MEM_TYPE_PAGE_ORDER0, NULL);
if (err2)
netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed with %d)\n", err2);
err_mem_model:
xsk_pool_dma_unmap(pool, 0);
err_dma_unmap:
if (up)
dev_open(dev, NULL);
return err;
}
int dpaa2_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
{
return pool ? dpaa2_xsk_enable_pool(dev, pool, qid) :
dpaa2_xsk_disable_pool(dev, qid);
}
int dpaa2_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
{
struct dpaa2_eth_priv *priv = netdev_priv(dev);
struct dpaa2_eth_channel *ch = priv->channel[qid];
if (!priv->link_state.up)
return -ENETDOWN;
if (!priv->xdp_prog)
return -EINVAL;
if (!ch->xsk_zc)
return -EINVAL;
/* We do not have access to a per channel SW interrupt, so instead we
* schedule a NAPI instance.
*/
if (!napi_if_scheduled_mark_missed(&ch->napi))
napi_schedule(&ch->napi);
return 0;
}
static int dpaa2_xsk_tx_build_fd(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch,
struct dpaa2_fd *fd,
struct xdp_desc *xdp_desc)
{
struct device *dev = priv->net_dev->dev.parent;
struct dpaa2_sg_entry *sgt;
struct dpaa2_eth_swa *swa;
void *sgt_buf = NULL;
dma_addr_t sgt_addr;
int sgt_buf_size;
dma_addr_t addr;
int err = 0;
/* Prepare the HW SGT structure */
sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry);
sgt_buf = dpaa2_eth_sgt_get(priv);
if (unlikely(!sgt_buf))
return -ENOMEM;
sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);
/* Get the address of the XSK Tx buffer */
addr = xsk_buff_raw_get_dma(ch->xsk_pool, xdp_desc->addr);
xsk_buff_raw_dma_sync_for_device(ch->xsk_pool, addr, xdp_desc->len);
/* Fill in the HW SGT structure */
dpaa2_sg_set_addr(sgt, addr);
dpaa2_sg_set_len(sgt, xdp_desc->len);
dpaa2_sg_set_final(sgt, true);
/* Store the necessary info in the SGT buffer */
swa = (struct dpaa2_eth_swa *)sgt_buf;
swa->type = DPAA2_ETH_SWA_XSK;
swa->xsk.sgt_size = sgt_buf_size;
/* Separately map the SGT buffer */
sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);
if (unlikely(dma_mapping_error(dev, sgt_addr))) {
err = -ENOMEM;
goto sgt_map_failed;
}
/* Initialize FD fields */
memset(fd, 0, sizeof(struct dpaa2_fd));
dpaa2_fd_set_offset(fd, priv->tx_data_offset);
dpaa2_fd_set_format(fd, dpaa2_fd_sg);
dpaa2_fd_set_addr(fd, sgt_addr);
dpaa2_fd_set_len(fd, xdp_desc->len);
dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
return 0;
sgt_map_failed:
dpaa2_eth_sgt_recycle(priv, sgt_buf);
return err;
}
bool dpaa2_xsk_tx(struct dpaa2_eth_priv *priv,
struct dpaa2_eth_channel *ch)
{
struct xdp_desc *xdp_descs = ch->xsk_pool->tx_descs;
struct dpaa2_eth_drv_stats *percpu_extras;
struct rtnl_link_stats64 *percpu_stats;
int budget = DPAA2_ETH_TX_ZC_PER_NAPI;
int total_enqueued, enqueued;
int retries, max_retries;
struct dpaa2_eth_fq *fq;
struct dpaa2_fd *fds;
int batch, i, err;
percpu_stats = this_cpu_ptr(priv->percpu_stats);
percpu_extras = this_cpu_ptr(priv->percpu_extras);
fds = (this_cpu_ptr(priv->fd))->array;
/* Use the FQ with the same idx as the affine CPU */
fq = &priv->fq[ch->nctx.desired_cpu];
batch = xsk_tx_peek_release_desc_batch(ch->xsk_pool, budget);
if (!batch)
return false;
/* Create a FD for each XSK frame to be sent */
for (i = 0; i < batch; i++) {
err = dpaa2_xsk_tx_build_fd(priv, ch, &fds[i], &xdp_descs[i]);
if (err) {
batch = i;
break;
}
trace_dpaa2_tx_xsk_fd(priv->net_dev, &fds[i]);
}
/* Enqueue all the created FDs */
max_retries = batch * DPAA2_ETH_ENQUEUE_RETRIES;
total_enqueued = 0;
enqueued = 0;
retries = 0;
while (total_enqueued < batch && retries < max_retries) {
err = priv->enqueue(priv, fq, &fds[total_enqueued], 0,
batch - total_enqueued, &enqueued);
if (err == -EBUSY) {
retries++;
continue;
}
total_enqueued += enqueued;
}
percpu_extras->tx_portal_busy += retries;
/* Update statistics */
percpu_stats->tx_packets += total_enqueued;
for (i = 0; i < total_enqueued; i++)
percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);
for (i = total_enqueued; i < batch; i++) {
dpaa2_eth_free_tx_fd(priv, ch, fq, &fds[i], false);
percpu_stats->tx_errors++;
}
xsk_tx_release(ch->xsk_pool);
return total_enqueued == budget ? true : false;
}
drivers/net/ethernet/freescale/dpaa2/dpni-cmd.h
+13 -6
View File
@@ -13,10 +13,12 @@
#define DPNI_VER_MINOR 0
#define DPNI_CMD_BASE_VERSION 1
#define DPNI_CMD_2ND_VERSION 2
#define DPNI_CMD_3RD_VERSION 3
#define DPNI_CMD_ID_OFFSET 4
#define DPNI_CMD(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_BASE_VERSION)
#define DPNI_CMD_V2(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_2ND_VERSION)
#define DPNI_CMD_V3(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_3RD_VERSION)
#define DPNI_CMDID_OPEN DPNI_CMD(0x801)
#define DPNI_CMDID_CLOSE DPNI_CMD(0x800)
@@ -39,7 +41,7 @@
#define DPNI_CMDID_GET_IRQ_STATUS DPNI_CMD(0x016)
#define DPNI_CMDID_CLEAR_IRQ_STATUS DPNI_CMD(0x017)
#define DPNI_CMDID_SET_POOLS DPNI_CMD(0x200)
#define DPNI_CMDID_SET_POOLS DPNI_CMD_V3(0x200)
#define DPNI_CMDID_SET_ERRORS_BEHAVIOR DPNI_CMD(0x20B)
#define DPNI_CMDID_GET_QDID DPNI_CMD(0x210)
@@ -115,14 +117,19 @@ struct dpni_cmd_open {
};
#define DPNI_BACKUP_POOL(val, order) (((val) & 0x1) << (order))
struct dpni_cmd_pool {
__le16 dpbp_id;
u8 priority_mask;
u8 pad;
};
struct dpni_cmd_set_pools {
/* cmd word 0 */
u8 num_dpbp;
u8 backup_pool_mask;
__le16 pad;
/* cmd word 0..4 */
__le32 dpbp_id[DPNI_MAX_DPBP];
/* cmd word 4..6 */
u8 pad;
u8 pool_options;
struct dpni_cmd_pool pool[DPNI_MAX_DPBP];
__le16 buffer_size[DPNI_MAX_DPBP];
};
drivers/net/ethernet/freescale/dpaa2/dpni.c
+5 -1
View File
@@ -173,8 +173,12 @@ int dpni_set_pools(struct fsl_mc_io *mc_io,
token);
cmd_params = (struct dpni_cmd_set_pools *)cmd.params;
cmd_params->num_dpbp = cfg->num_dpbp;
cmd_params->pool_options = cfg->pool_options;
for (i = 0; i < DPNI_MAX_DPBP; i++) {
cmd_params->dpbp_id[i] = cpu_to_le32(cfg->pools[i].dpbp_id);
cmd_params->pool[i].dpbp_id =
cpu_to_le16(cfg->pools[i].dpbp_id);
cmd_params->pool[i].priority_mask =
cfg->pools[i].priority_mask;
cmd_params->buffer_size[i] =
cpu_to_le16(cfg->pools[i].buffer_size);
cmd_params->backup_pool_mask |=
drivers/net/ethernet/freescale/dpaa2/dpni.h
+9
View File
@@ -92,19 +92,28 @@ int dpni_close(struct fsl_mc_io *mc_io,
u32 cmd_flags,
u16 token);
#define DPNI_POOL_ASSOC_QPRI 0
#define DPNI_POOL_ASSOC_QDBIN 1
/**
* struct dpni_pools_cfg - Structure representing buffer pools configuration
* @num_dpbp: Number of DPBPs
* @pool_options: Buffer assignment options.
* This field is a combination of DPNI_POOL_ASSOC_flags
* @pools: Array of buffer pools parameters; The number of valid entries
* must match 'num_dpbp' value
* @pools.dpbp_id: DPBP object ID
* @pools.priority: Priority mask that indicates TC's used with this buffer.
* If set to 0x00 MC will assume value 0xff.
* @pools.buffer_size: Buffer size
* @pools.backup_pool: Backup pool
*/
struct dpni_pools_cfg {
u8 num_dpbp;
u8 pool_options;
struct {
int dpbp_id;
u8 priority_mask;
u16 buffer_size;
int backup_pool;
} pools[DPNI_MAX_DPBP];