armbian/linux-cix
0
0
Fork 0
mirror of https://github.com/armbian/linux-cix.git synced 2026-01-06 12:30:45 -08:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

Browse Source 
Pull virtio updates from Michael Tsirkin:
 "virtio,vhost,vdpa: features, fixes, and cleanups:

   - reduction in interrupt rate in virtio

   - perf improvement for VDUSE

   - scalability for vhost-scsi

   - non power of 2 ring support for packed rings

   - better management for mlx5 vdpa

   - suspend for snet

   - VIRTIO_F_NOTIFICATION_DATA

   - shared backend with vdpa-sim-blk

   - user VA support in vdpa-sim

   - better struct packing for virtio

  and fixes, cleanups all over the place"

* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (52 commits)
  vhost_vdpa: fix unmap process in no-batch mode
  MAINTAINERS: make me a reviewer of VIRTIO CORE AND NET DRIVERS
  tools/virtio: fix build caused by virtio_ring changes
  virtio_ring: add a struct device forward declaration
  vdpa_sim_blk: support shared backend
  vdpa_sim: move buffer allocation in the devices
  vdpa/snet: use likely/unlikely macros in hot functions
  vdpa/snet: implement kick_vq_with_data callback
  virtio-vdpa: add VIRTIO_F_NOTIFICATION_DATA feature support
  virtio: add VIRTIO_F_NOTIFICATION_DATA feature support
  vdpa/snet: support the suspend vDPA callback
  vdpa/snet: support getting and setting VQ state
  MAINTAINERS: add vringh.h to Virtio Core and Net Drivers
  vringh: address kdoc warnings
  vdpa: address kdoc warnings
  virtio_ring: don't update event idx on get_buf
  vdpa_sim: add support for user VA
  vdpa_sim: replace the spinlock with a mutex to protect the state
  vdpa_sim: use kthread worker
  vdpa_sim: make devices agnostic for work management
  ...
This commit is contained in:
Linus Torvalds
2023-04-27 17:05:34 -07:00
parent 0835b5ee87 c82729e066
commit 8ccd54fe45
32 changed files with 1760 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MAINTAINERS
View File

@@ -22212,6 +22212,7 @@ F: include/uapi/linux/virtio_console.h
VIRTIO CORE AND NET DRIVERS
M: "Michael S. Tsirkin" <mst@redhat.com>
M: Jason Wang <jasowang@redhat.com>
R: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
L: virtualization@lists.linux-foundation.org
S: Maintained
F: Documentation/ABI/testing/sysfs-bus-vdpa
@@ -22225,6 +22226,7 @@ F: drivers/vdpa/
F: drivers/virtio/
F: include/linux/vdpa.h
F: include/linux/virtio*.h
F: include/linux/vringh.h
F: include/uapi/linux/virtio_*.h
F: tools/virtio/

22
drivers/s390/virtio/virtio_ccw.c
View File

@@ -391,7 +391,7 @@ static void virtio_ccw_drop_indicator(struct virtio_ccw_device *vcdev,
ccw_device_dma_free(vcdev->cdev, thinint_area, sizeof(*thinint_area));
}
static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
static inline bool virtio_ccw_do_kvm_notify(struct virtqueue *vq, u32 data)
{
struct virtio_ccw_vq_info *info = vq->priv;
struct virtio_ccw_device *vcdev;
@@ -402,12 +402,22 @@ static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
BUILD_BUG_ON(sizeof(struct subchannel_id) != sizeof(unsigned int));
info->cookie = kvm_hypercall3(KVM_S390_VIRTIO_CCW_NOTIFY,
*((unsigned int *)&schid),
vq->index, info->cookie);
data, info->cookie);
if (info->cookie < 0)
return false;
return true;
}
static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
{
return virtio_ccw_do_kvm_notify(vq, vq->index);
}
static bool virtio_ccw_kvm_notify_with_data(struct virtqueue *vq)
{
return virtio_ccw_do_kvm_notify(vq, vring_notification_data(vq));
}
static int virtio_ccw_read_vq_conf(struct virtio_ccw_device *vcdev,
struct ccw1 *ccw, int index)
{
@@ -495,6 +505,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
struct ccw1 *ccw)
{
struct virtio_ccw_device *vcdev = to_vc_device(vdev);
bool (*notify)(struct virtqueue *vq);
int err;
struct virtqueue *vq = NULL;
struct virtio_ccw_vq_info *info;
@@ -502,6 +513,11 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
unsigned long flags;
bool may_reduce;
if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA))
notify = virtio_ccw_kvm_notify_with_data;
else
notify = virtio_ccw_kvm_notify;
/* Allocate queue. */
info = kzalloc(sizeof(struct virtio_ccw_vq_info), GFP_KERNEL);
if (!info) {
@@ -524,7 +540,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
may_reduce = vcdev->revision > 0;
vq = vring_create_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN,
vdev, true, may_reduce, ctx,
virtio_ccw_kvm_notify, callback, name);
notify, callback, name);
if (!vq) {
/* For now, we fail if we can't get the requested size. */

261
drivers/vdpa/mlx5/net/mlx5_vnet.c
View File

@@ -778,12 +778,28 @@ static bool vq_is_tx(u16 idx)
return idx % 2;
}
static u16 get_features_12_3(u64 features)
enum {
MLX5_VIRTIO_NET_F_MRG_RXBUF = 2,
MLX5_VIRTIO_NET_F_HOST_ECN = 4,
MLX5_VIRTIO_NET_F_GUEST_ECN = 6,
MLX5_VIRTIO_NET_F_GUEST_TSO6 = 7,
MLX5_VIRTIO_NET_F_GUEST_TSO4 = 8,
MLX5_VIRTIO_NET_F_GUEST_CSUM = 9,
MLX5_VIRTIO_NET_F_CSUM = 10,
MLX5_VIRTIO_NET_F_HOST_TSO6 = 11,
MLX5_VIRTIO_NET_F_HOST_TSO4 = 12,
};
static u16 get_features(u64 features)
{
return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) |
(!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) |
(!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) |
(!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6);
return (!!(features & BIT_ULL(VIRTIO_NET_F_MRG_RXBUF)) << MLX5_VIRTIO_NET_F_MRG_RXBUF) |
(!!(features & BIT_ULL(VIRTIO_NET_F_HOST_ECN)) << MLX5_VIRTIO_NET_F_HOST_ECN) |
(!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_ECN)) << MLX5_VIRTIO_NET_F_GUEST_ECN) |
(!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_TSO6)) << MLX5_VIRTIO_NET_F_GUEST_TSO6) |
(!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_TSO4)) << MLX5_VIRTIO_NET_F_GUEST_TSO4) |
(!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << MLX5_VIRTIO_NET_F_CSUM) |
(!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << MLX5_VIRTIO_NET_F_HOST_TSO6) |
(!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << MLX5_VIRTIO_NET_F_HOST_TSO4);
}
static bool counters_supported(const struct mlx5_vdpa_dev *mvdev)
@@ -797,6 +813,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
void *obj_context;
u16 mlx_features;
void *cmd_hdr;
void *vq_ctx;
void *in;
@@ -812,6 +829,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
goto err_alloc;
}
mlx_features = get_features(ndev->mvdev.actual_features);
cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr);
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
@@ -822,7 +840,9 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
get_features_12_3(ndev->mvdev.actual_features));
mlx_features >> 3);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0,
mlx_features & 7);
vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context);
MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev));
@@ -2171,23 +2191,27 @@ static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx)
return MLX5_VDPA_DATAVQ_GROUP;
}
enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9,
MLX5_VIRTIO_NET_F_CSUM = 1 << 10,
MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11,
MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12,
};
static u64 mlx_to_vritio_features(u16 dev_features)
{
u64 result = 0;
if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM)
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_MRG_RXBUF))
result |= BIT_ULL(VIRTIO_NET_F_MRG_RXBUF);
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_HOST_ECN))
result |= BIT_ULL(VIRTIO_NET_F_HOST_ECN);
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_ECN))
result |= BIT_ULL(VIRTIO_NET_F_GUEST_ECN);
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_TSO6))
result |= BIT_ULL(VIRTIO_NET_F_GUEST_TSO6);
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_TSO4))
result |= BIT_ULL(VIRTIO_NET_F_GUEST_TSO4);
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_CSUM))
result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM);
if (dev_features & MLX5_VIRTIO_NET_F_CSUM)
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_CSUM))
result |= BIT_ULL(VIRTIO_NET_F_CSUM);
if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6)
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_HOST_TSO6))
result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6);
if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4)
if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_HOST_TSO4))
result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4);
return result;
@@ -2298,6 +2322,113 @@ static void update_cvq_info(struct mlx5_vdpa_dev *mvdev)
}
}
static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
{
u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {};
u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {};
int err;
MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE);
MLX5_SET(query_vport_state_in, in, op_mod, opmod);
MLX5_SET(query_vport_state_in, in, vport_number, vport);
if (vport)
MLX5_SET(query_vport_state_in, in, other_vport, 1);
err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out);
if (err)
return 0;
return MLX5_GET(query_vport_state_out, out, state);
}
static bool get_link_state(struct mlx5_vdpa_dev *mvdev)
{
if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) ==
VPORT_STATE_UP)
return true;
return false;
}
static void update_carrier(struct work_struct *work)
{
struct mlx5_vdpa_wq_ent *wqent;
struct mlx5_vdpa_dev *mvdev;
struct mlx5_vdpa_net *ndev;
wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
mvdev = wqent->mvdev;
ndev = to_mlx5_vdpa_ndev(mvdev);
if (get_link_state(mvdev))
ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
else
ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
if (ndev->config_cb.callback)
ndev->config_cb.callback(ndev->config_cb.private);
kfree(wqent);
}
static int queue_link_work(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_wq_ent *wqent;
wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
if (!wqent)
return -ENOMEM;
wqent->mvdev = &ndev->mvdev;
INIT_WORK(&wqent->work, update_carrier);
queue_work(ndev->mvdev.wq, &wqent->work);
return 0;
}
static int event_handler(struct notifier_block *nb, unsigned long event, void *param)
{
struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb);
struct mlx5_eqe *eqe = param;
int ret = NOTIFY_DONE;
if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
switch (eqe->sub_type) {
case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
if (queue_link_work(ndev))
return NOTIFY_DONE;
ret = NOTIFY_OK;
break;
default:
return NOTIFY_DONE;
}
return ret;
}
return ret;
}
static void register_link_notifier(struct mlx5_vdpa_net *ndev)
{
if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS)))
return;
ndev->nb.notifier_call = event_handler;
mlx5_notifier_register(ndev->mvdev.mdev, &ndev->nb);
ndev->nb_registered = true;
queue_link_work(ndev);
}
static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
{
if (!ndev->nb_registered)
return;
ndev->nb_registered = false;
mlx5_notifier_unregister(ndev->mvdev.mdev, &ndev->nb);
if (ndev->mvdev.wq)
flush_workqueue(ndev->mvdev.wq);
}
static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -2567,10 +2698,11 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n");
goto err_setup;
}
register_link_notifier(ndev);
err = setup_driver(mvdev);
if (err) {
mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
goto err_setup;
goto err_driver;
}
} else {
mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
@@ -2582,6 +2714,8 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
up_write(&ndev->reslock);
return;
err_driver:
unregister_link_notifier(ndev);
err_setup:
mlx5_vdpa_destroy_mr(&ndev->mvdev);
ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
@@ -2607,6 +2741,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
mlx5_vdpa_info(mvdev, "performing device reset\n");
down_write(&ndev->reslock);
unregister_link_notifier(ndev);
teardown_driver(ndev);
clear_vqs_ready(ndev);
mlx5_vdpa_destroy_mr(&ndev->mvdev);
@@ -2861,9 +2996,7 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev)
mlx5_vdpa_info(mvdev, "suspending device\n");
down_write(&ndev->reslock);
ndev->nb_registered = false;
mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
flush_workqueue(ndev->mvdev.wq);
unregister_link_notifier(ndev);
for (i = 0; i < ndev->cur_num_vqs; i++) {
mvq = &ndev->vqs[i];
suspend_vq(ndev, mvq);
@@ -3000,84 +3133,6 @@ struct mlx5_vdpa_mgmtdev {
struct mlx5_vdpa_net *ndev;
};
static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
{
u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {};
u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {};
int err;
MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE);
MLX5_SET(query_vport_state_in, in, op_mod, opmod);
MLX5_SET(query_vport_state_in, in, vport_number, vport);
if (vport)
MLX5_SET(query_vport_state_in, in, other_vport, 1);
err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out);
if (err)
return 0;
return MLX5_GET(query_vport_state_out, out, state);
}
static bool get_link_state(struct mlx5_vdpa_dev *mvdev)
{
if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) ==
VPORT_STATE_UP)
return true;
return false;
}
static void update_carrier(struct work_struct *work)
{
struct mlx5_vdpa_wq_ent *wqent;
struct mlx5_vdpa_dev *mvdev;
struct mlx5_vdpa_net *ndev;
wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
mvdev = wqent->mvdev;
ndev = to_mlx5_vdpa_ndev(mvdev);
if (get_link_state(mvdev))
ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
else
ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
if (ndev->nb_registered && ndev->config_cb.callback)
ndev->config_cb.callback(ndev->config_cb.private);
kfree(wqent);
}
static int event_handler(struct notifier_block *nb, unsigned long event, void *param)
{
struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb);
struct mlx5_eqe *eqe = param;
int ret = NOTIFY_DONE;
struct mlx5_vdpa_wq_ent *wqent;
if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS)))
return NOTIFY_DONE;
switch (eqe->sub_type) {
case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
if (!wqent)
return NOTIFY_DONE;
wqent->mvdev = &ndev->mvdev;
INIT_WORK(&wqent->work, update_carrier);
queue_work(ndev->mvdev.wq, &wqent->work);
ret = NOTIFY_OK;
break;
default:
return NOTIFY_DONE;
}
return ret;
}
return ret;
}
static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu)
{
int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
@@ -3127,6 +3182,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
return -EINVAL;
}
device_features &= add_config->device_features;
} else {
device_features &= ~BIT_ULL(VIRTIO_NET_F_MRG_RXBUF);
}
if (!(device_features & BIT_ULL(VIRTIO_F_VERSION_1) &&
device_features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) {
@@ -3258,9 +3315,6 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_res2;
}
ndev->nb.notifier_call = event_handler;
mlx5_notifier_register(mdev, &ndev->nb);
ndev->nb_registered = true;
mvdev->vdev.mdev = &mgtdev->mgtdev;
err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1);
if (err)
@@ -3294,10 +3348,7 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *
mlx5_vdpa_remove_debugfs(ndev->debugfs);
ndev->debugfs = NULL;
if (ndev->nb_registered) {
ndev->nb_registered = false;
mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
}
unregister_link_notifier(ndev);
wq = mvdev->wq;
mvdev->wq = NULL;
destroy_workqueue(wq);

1
drivers/vdpa/solidrun/Makefile
View File

@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_SNET_VDPA) += snet_vdpa.o
snet_vdpa-$(CONFIG_SNET_VDPA) += snet_main.o
snet_vdpa-$(CONFIG_SNET_VDPA) += snet_ctrl.o
ifdef CONFIG_HWMON
snet_vdpa-$(CONFIG_SNET_VDPA) += snet_hwmon.o
endif

330
drivers/vdpa/solidrun/snet_ctrl.c Normal file
View File

@@ -0,0 +1,330 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* SolidRun DPU driver for control plane
*
* Copyright (C) 2022-2023 SolidRun
*
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
*
*/
#include <linux/iopoll.h>
#include "snet_vdpa.h"
enum snet_ctrl_opcodes {
SNET_CTRL_OP_DESTROY = 1,
SNET_CTRL_OP_READ_VQ_STATE,
SNET_CTRL_OP_SUSPEND,
};
#define SNET_CTRL_TIMEOUT 2000000
#define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF
#define SNET_CTRL_IN_PROCESS_MASK 0x00010000
#define SNET_CTRL_CHUNK_RDY_MASK 0x00020000
#define SNET_CTRL_ERROR_MASK 0x0FFC0000
#define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
#define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \
!((val) & SNET_CTRL_IN_PROCESS_MASK))
#define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
/* Control register used to read data from the DPU */
struct snet_ctrl_reg_ctrl {
/* Chunk size in 4B words */
u16 data_size;
/* We are in the middle of a command */
u16 in_process:1;
/* A data chunk is ready and can be consumed */
u16 chunk_ready:1;
/* Error code */
u16 error:10;
/* Saved for future usage */
u16 rsvd:4;
};
/* Opcode register */
struct snet_ctrl_reg_op {
u16 opcode;
/* Only if VQ index is relevant for the command */
u16 vq_idx;
};
struct snet_ctrl_regs {
struct snet_ctrl_reg_op op;
struct snet_ctrl_reg_ctrl ctrl;
u32 rsvd;
u32 data[];
};
static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
{
return snet->bar + snet->psnet->cfg.ctrl_off;
}
static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
{
u32 val;
return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
SNET_CTRL_TIMEOUT);
}
static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
{
u32 val;
return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
}
static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
{
u32 val;
return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
SNET_CTRL_TIMEOUT);
}
static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
{
return ioread32(&ctrl_regs->data[word_idx]);
}
static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
{
return ioread32(&ctrl_regs->ctrl);
}
static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
{
iowrite32(val, &ctrl_regs->ctrl);
}
static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
{
iowrite32(val, &ctrl_regs->op);
}
static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
{
/* Wait until the DPU finishes completely.
* It will clear the opcode register.
*/
return snet_wait_for_empty_op(ctrl_regs);
}
/* Reading ctrl from the DPU:
* buf_size must be 4B aligned
*
* Steps:
*
* (1) Verify that the DPU is not in the middle of another operation by
* reading the in_process and error bits in the control register.
* (2) Write the request opcode and the VQ idx in the opcode register
* and write the buffer size in the control register.
* (3) Start readind chunks of data, chunk_ready bit indicates that a
* data chunk is available, we signal that we read the data by clearing the bit.
* (4) Detect that the transfer is completed when the in_process bit
* in the control register is cleared or when the an error appears.
*/
static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
u32 buf_size)
{
struct pci_dev *pdev = snet->pdev;
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
u32 *bfr_ptr = (u32 *)buffer;
u32 val;
u16 buf_words;
int ret;
u16 words, i, tot_words = 0;
/* Supported for config 2+ */
if (!SNET_CFG_VER(snet, 2))
return -EOPNOTSUPP;
if (!IS_ALIGNED(buf_size, 4))
return -EINVAL;
mutex_lock(&snet->ctrl_lock);
buf_words = buf_size / 4;
/* Make sure control register is empty */
ret = snet_wait_for_empty_ctrl(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
goto exit;
}
/* We need to write the buffer size in the control register, and the opcode + vq index in
* the opcode register.
* We use a spinlock to serialize the writes.
*/
spin_lock(&snet->ctrl_spinlock);
snet_write_ctrl(regs, buf_words);
snet_write_op(regs, opcode | (vq_idx << 16));
spin_unlock(&snet->ctrl_spinlock);
while (buf_words != tot_words) {
ret = snet_wait_for_data(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for control data\n");
goto exit;
}
val = snet_read_ctrl(regs);
/* Error? */
if (val & SNET_CTRL_ERROR_MASK) {
ret = SNET_VAL_TO_ERR(val);
SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
goto exit;
}
words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
for (i = 0; i < words; i++) {
*bfr_ptr = snet_read32_word(regs, i);
bfr_ptr++;
}
tot_words += words;
/* Is the job completed? */
if (!(val & SNET_CTRL_IN_PROCESS_MASK))
break;
/* Clear the chunk ready bit and continue */
val &= ~SNET_CTRL_CHUNK_RDY_MASK;
snet_write_ctrl(regs, val);
}
ret = snet_wait_for_dpu_completion(regs);
if (ret)
SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
exit:
mutex_unlock(&snet->ctrl_lock);
return ret;
}
/* Send a control message to the DPU using the old mechanism
* used with config version 1.
*/
static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
{
struct pci_dev *pdev = snet->pdev;
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
int ret;
mutex_lock(&snet->ctrl_lock);
/* Old mechanism uses just 1 register, the opcode register.
* Make sure that the opcode register is empty, and that the DPU isn't
* processing an old message.
*/
ret = snet_wait_for_empty_op(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
goto exit;
}
/* Write the message */
snet_write_op(regs, opcode);
/* DPU ACKs the message by clearing the opcode register */
ret = snet_wait_for_empty_op(regs);
if (ret)
SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
exit:
mutex_unlock(&snet->ctrl_lock);
return ret;
}
/* Send a control message to the DPU.
* A control message is a message without payload.
*/
static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
{
struct pci_dev *pdev = snet->pdev;
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
u32 val;
int ret;
/* If config version is not 2+, use the old mechanism */
if (!SNET_CFG_VER(snet, 2))
return snet_send_ctrl_msg_old(snet, opcode);
mutex_lock(&snet->ctrl_lock);
/* Make sure control register is empty */
ret = snet_wait_for_empty_ctrl(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
goto exit;
}
/* We need to clear the control register and write the opcode + vq index in the opcode
* register.
* We use a spinlock to serialize the writes.
*/
spin_lock(&snet->ctrl_spinlock);
snet_write_ctrl(regs, 0);
snet_write_op(regs, opcode | (vq_idx << 16));
spin_unlock(&snet->ctrl_spinlock);
/* The DPU ACKs control messages by setting the chunk ready bit
* without data.
*/
ret = snet_wait_for_data(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
goto exit;
}
/* Check for errors */
val = snet_read_ctrl(regs);
ret = SNET_VAL_TO_ERR(val);
/* Clear the chunk ready bit */
val &= ~SNET_CTRL_CHUNK_RDY_MASK;
snet_write_ctrl(regs, val);
ret = snet_wait_for_dpu_completion(regs);
if (ret)
SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
ret);
exit:
mutex_unlock(&snet->ctrl_lock);
return ret;
}
void snet_ctrl_clear(struct snet *snet)
{
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
snet_write_op(regs, 0);
}
int snet_destroy_dev(struct snet *snet)
{
return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
}
int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
{
return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
sizeof(*state));
}
int snet_suspend_dev(struct snet *snet)
{
return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
}

2
drivers/vdpa/solidrun/snet_hwmon.c
View File

@@ -2,7 +2,7 @@
/*
* SolidRun DPU driver for control plane
*
* Copyright (C) 2022 SolidRun
* Copyright (C) 2022-2023 SolidRun
*
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
*

146
drivers/vdpa/solidrun/snet_main.c
View File

@@ -2,7 +2,7 @@
/*
* SolidRun DPU driver for control plane
*
* Copyright (C) 2022 SolidRun
* Copyright (C) 2022-2023 SolidRun
*
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
*
@@ -16,14 +16,12 @@
/* SNET signature */
#define SNET_SIGNATURE 0xD0D06363
/* Max. config version that we can work with */
#define SNET_CFG_VERSION 0x1
#define SNET_CFG_VERSION 0x2
/* Queue align */
#define SNET_QUEUE_ALIGNMENT PAGE_SIZE
/* Kick value to notify that new data is available */
#define SNET_KICK_VAL 0x1
#define SNET_CONFIG_OFF 0x0
/* ACK timeout for a message */
#define SNET_ACK_TIMEOUT 2000000
/* How long we are willing to wait for a SNET device */
#define SNET_DETECT_TIMEOUT 5000000
/* How long should we wait for the DPU to read our config */
@@ -32,61 +30,16 @@
#define SNET_GENERAL_CFG_LEN 36
#define SNET_GENERAL_CFG_VQ_LEN 40
enum snet_msg {
SNET_MSG_DESTROY = 1,
};
static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)
{
return container_of(vdpa, struct snet, vdpa);
}
static int snet_wait_for_msg_ack(struct snet *snet)
{
struct pci_dev *pdev = snet->pdev;
int ret;
u32 val;
/* The DPU will clear the messages offset once messages
* are processed.
*/
ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.msg_off,
val, !val, 10, SNET_ACK_TIMEOUT);
if (ret)
SNET_WARN(pdev, "Timeout waiting for message ACK\n");
return ret;
}
/* Sends a message to the DPU.
* If blocking is set, the function will return once the
* message was processed by the DPU (or timeout).
*/
static int snet_send_msg(struct snet *snet, u32 msg, bool blocking)
{
int ret = 0;
/* Make sure the DPU acked last message before issuing a new one */
ret = snet_wait_for_msg_ack(snet);
if (ret)
return ret;
/* Write the message */
snet_write32(snet, snet->psnet->cfg.msg_off, msg);
if (blocking)
ret = snet_wait_for_msg_ack(snet);
else /* If non-blocking, flush the write by issuing a read */
snet_read32(snet, snet->psnet->cfg.msg_off);
return ret;
}
static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)
{
struct snet *snet = data;
/* Call callback if any */
if (snet->cb.callback)
if (likely(snet->cb.callback))
return snet->cb.callback(snet->cb.private);
return IRQ_HANDLED;
@@ -96,7 +49,7 @@ static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)
{
struct snet_vq *vq = data;
/* Call callback if any */
if (vq->cb.callback)
if (likely(vq->cb.callback))
return vq->cb.callback(vq->cb.private);
return IRQ_HANDLED;
@@ -153,12 +106,24 @@ static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)
{
struct snet *snet = vdpa_to_snet(vdev);
/* not ready - ignore */
if (!snet->vqs[idx]->ready)
if (unlikely(!snet->vqs[idx]->ready))
return;
iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
}
static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)
{
struct snet *snet = vdpa_to_snet(vdev);
u16 idx = data & 0xFFFF;
/* not ready - ignore */
if (unlikely(!snet->vqs[idx]->ready))
return;
iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
}
static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
{
struct snet *snet = vdpa_to_snet(vdev);
@@ -181,33 +146,48 @@ static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)
return snet->vqs[idx]->ready;
}
static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)
{
struct snet *snet = vdpa_to_snet(vdev);
/* Setting the VQ state is not supported.
* If the asked state is the same as the initial one
* we can ignore it.
*/
if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {
const struct vdpa_vq_state_packed *p = &state->packed;
if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&
p->last_avail_idx == 0 && p->last_used_idx == 0)
return 0;
return true;
} else {
const struct vdpa_vq_state_split *s = &state->split;
if (s->avail_index == 0)
return 0;
return true;
}
return false;
}
static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
{
struct snet *snet = vdpa_to_snet(vdev);
/* We can set any state for config version 2+ */
if (SNET_CFG_VER(snet, 2)) {
memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));
return 0;
}
/* Older config - we can't set the VQ state.
* Return 0 only if this is the initial state we use in the DPU.
*/
if (snet_vq_state_is_initial(snet, state))
return 0;
return -EOPNOTSUPP;
}
static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
{
/* Not supported */
return -EOPNOTSUPP;
struct snet *snet = vdpa_to_snet(vdev);
return snet_read_vq_state(snet, idx, state);
}
static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)
@@ -232,9 +212,9 @@ static int snet_reset_dev(struct snet *snet)
if (!snet->status)
return 0;
/* If DPU started, send a destroy message */
/* If DPU started, destroy it */
if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)
ret = snet_send_msg(snet, SNET_MSG_DESTROY, true);
ret = snet_destroy_dev(snet);
/* Clear VQs */
for (i = 0; i < snet->cfg->vq_num; i++) {
@@ -258,7 +238,7 @@ static int snet_reset_dev(struct snet *snet)
snet->dpu_ready = false;
if (ret)
SNET_WARN(pdev, "Incomplete reset to SNET[%u] device\n", snet->sid);
SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);
else
SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);
@@ -356,7 +336,7 @@ static int snet_write_conf(struct snet *snet)
* | DESC AREA |
* | DEVICE AREA |
* | DRIVER AREA |
* | RESERVED |
* | VQ STATE (CFG 2+) | RSVD |
*
* Magic number should be written last, this is the DPU indication that the data is ready
*/
@@ -391,12 +371,15 @@ static int snet_write_conf(struct snet *snet)
off += 8;
snet_write64(snet, off, snet->vqs[i]->driver_area);
off += 8;
/* Write VQ state if config version is 2+ */
if (SNET_CFG_VER(snet, 2))
snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);
off += 4;
/* Ignore reserved */
off += 8;
off += 4;
}
/* Clear snet messages address for this device */
snet_write32(snet, snet->psnet->cfg.msg_off, 0);
/* Write magic number - data is ready */
snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);
@@ -512,10 +495,25 @@ static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,
iowrite8(*buf_ptr++, cfg_ptr + i);
}
static int snet_suspend(struct vdpa_device *vdev)
{
struct snet *snet = vdpa_to_snet(vdev);
int ret;
ret = snet_suspend_dev(snet);
if (ret)
SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);
else
SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);
return ret;
}
static const struct vdpa_config_ops snet_config_ops = {
.set_vq_address = snet_set_vq_address,
.set_vq_num = snet_set_vq_num,
.kick_vq = snet_kick_vq,
.kick_vq_with_data = snet_kick_vq_with_data,
.set_vq_cb = snet_set_vq_cb,
.set_vq_ready = snet_set_vq_ready,
.get_vq_ready = snet_get_vq_ready,
@@ -537,6 +535,7 @@ static const struct vdpa_config_ops snet_config_ops = {
.set_status = snet_set_status,
.get_config = snet_get_config,
.set_config = snet_set_config,
.suspend = snet_suspend,
};
static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)
@@ -697,7 +696,7 @@ static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)
off += 4;
cfg->hwmon_off = psnet_read32(psnet, off);
off += 4;
cfg->msg_off = psnet_read32(psnet, off);
cfg->ctrl_off = psnet_read32(psnet, off);
off += 4;
cfg->flags = psnet_read32(psnet, off);
off += 4;
@@ -997,6 +996,10 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev)
goto free_irqs;
}
/* Init control mutex and spinlock */
mutex_init(&snet->ctrl_lock);
spin_lock_init(&snet->ctrl_spinlock);
/* Save pci device pointer */
snet->pdev = pdev;
snet->psnet = psnet;
@@ -1013,6 +1016,9 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev)
/* Create a VirtIO config pointer */
snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;
/* Clear control registers */
snet_ctrl_clear(snet);
pci_set_master(pdev);
pci_set_drvdata(pdev, snet);

20
drivers/vdpa/solidrun/snet_vdpa.h
View File

@@ -2,7 +2,7 @@
/*
* SolidRun DPU driver for control plane
*
* Copyright (C) 2022 SolidRun
* Copyright (C) 2022-2023 SolidRun
*
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
*
@@ -20,10 +20,15 @@
#define SNET_INFO(pdev, fmt, ...) dev_info(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__)
#define SNET_DBG(pdev, fmt, ...) dev_dbg(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__)
#define SNET_HAS_FEATURE(s, f) ((s)->negotiated_features & BIT_ULL(f))
/* Check if negotiated config version is at least @ver */
#define SNET_CFG_VER(snet, ver) ((snet)->psnet->negotiated_cfg_ver >= (ver))
/* VQ struct */
struct snet_vq {
/* VQ callback */
struct vdpa_callback cb;
/* VQ state received from bus */
struct vdpa_vq_state vq_state;
/* desc base address */
u64 desc_area;
/* device base address */
@@ -51,6 +56,10 @@ struct snet {
struct vdpa_device vdpa;
/* Config callback */
struct vdpa_callback cb;
/* To lock the control mechanism */
struct mutex ctrl_lock;
/* Spinlock to protect critical parts in the control mechanism */
spinlock_t ctrl_spinlock;
/* array of virqueues */
struct snet_vq **vqs;
/* Used features */
@@ -117,8 +126,8 @@ struct snet_cfg {
u32 kick_off;
/* Offset in PCI BAR for HW monitoring */
u32 hwmon_off;
/* Offset in PCI BAR for SNET messages */
u32 msg_off;
/* Offset in PCI BAR for Control mechanism */
u32 ctrl_off;
/* Config general flags - enum snet_cfg_flags */
u32 flags;
/* Reserved for future usage */
@@ -191,4 +200,9 @@ static inline void snet_write64(struct snet *snet, u32 off, u64 val)
void psnet_create_hwmon(struct pci_dev *pdev);
#endif
void snet_ctrl_clear(struct snet *snet);
int snet_destroy_dev(struct snet *snet);
int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state);
int snet_suspend_dev(struct snet *snet);
#endif //_SNET_VDPA_H_

166
drivers/vdpa/vdpa_sim/vdpa_sim.c
View File

@@ -11,8 +11,8 @@
#include <linux/module.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/dma-map-ops.h>
#include <linux/vringh.h>
#include <linux/vdpa.h>
@@ -35,10 +35,44 @@ module_param(max_iotlb_entries, int, 0444);
MODULE_PARM_DESC(max_iotlb_entries,
"Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
static bool use_va = true;
module_param(use_va, bool, 0444);
MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");
#define VDPASIM_QUEUE_ALIGN PAGE_SIZE
#define VDPASIM_QUEUE_MAX 256
#define VDPASIM_VENDOR_ID 0
struct vdpasim_mm_work {
struct kthread_work work;
struct vdpasim *vdpasim;
struct mm_struct *mm_to_bind;
int ret;
};
static void vdpasim_mm_work_fn(struct kthread_work *work)
{
struct vdpasim_mm_work *mm_work =
container_of(work, struct vdpasim_mm_work, work);
struct vdpasim *vdpasim = mm_work->vdpasim;
mm_work->ret = 0;
//TODO: should we attach the cgroup of the mm owner?
vdpasim->mm_bound = mm_work->mm_to_bind;
}
static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim,
struct vdpasim_mm_work *mm_work)
{
struct kthread_work *work = &mm_work->work;
kthread_init_work(work, vdpasim_mm_work_fn);
kthread_queue_work(vdpasim->worker, work);
kthread_flush_work(work);
}
static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
{
return container_of(vdpa, struct vdpasim, vdpa);
@@ -59,13 +93,20 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
{
struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
uint16_t last_avail_idx = vq->vring.last_avail_idx;
struct vring_desc *desc = (struct vring_desc *)
(uintptr_t)vq->desc_addr;
struct vring_avail *avail = (struct vring_avail *)
(uintptr_t)vq->driver_addr;
struct vring_used *used = (struct vring_used *)
(uintptr_t)vq->device_addr;
vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, true,
(struct vring_desc *)(uintptr_t)vq->desc_addr,
(struct vring_avail *)
(uintptr_t)vq->driver_addr,
(struct vring_used *)
(uintptr_t)vq->device_addr);
if (use_va && vdpasim->mm_bound) {
vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num,
true, desc, avail, used);
} else {
vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num,
true, desc, avail, used);
}
vq->vring.last_avail_idx = last_avail_idx;
@@ -127,6 +168,25 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim)
static const struct vdpa_config_ops vdpasim_config_ops;
static const struct vdpa_config_ops vdpasim_batch_config_ops;
static void vdpasim_work_fn(struct kthread_work *work)
{
struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
struct mm_struct *mm = vdpasim->mm_bound;
if (use_va && mm) {
if (!mmget_not_zero(mm))
return;
kthread_use_mm(mm);
}
vdpasim->dev_attr.work_fn(vdpasim);
if (use_va && mm) {
kthread_unuse_mm(mm);
mmput(mm);
}
}
struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
const struct vdpa_dev_set_config *config)
{
@@ -155,7 +215,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
vdpa = __vdpa_alloc_device(NULL, ops,
dev_attr->ngroups, dev_attr->nas,
dev_attr->alloc_size,
dev_attr->name, false);
dev_attr->name, use_va);
if (IS_ERR(vdpa)) {
ret = PTR_ERR(vdpa);
goto err_alloc;
@@ -163,11 +223,17 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
vdpasim = vdpa_to_sim(vdpa);
vdpasim->dev_attr = *dev_attr;
INIT_WORK(&vdpasim->work, dev_attr->work_fn);
spin_lock_init(&vdpasim->lock);
dev = &vdpasim->vdpa.dev;
kthread_init_work(&vdpasim->work, vdpasim_work_fn);
vdpasim->worker = kthread_create_worker(0, "vDPA sim worker: %s",
dev_attr->name);
if (IS_ERR(vdpasim->worker))
goto err_iommu;
mutex_init(&vdpasim->mutex);
spin_lock_init(&vdpasim->iommu_lock);
dev = &vdpasim->vdpa.dev;
dev->dma_mask = &dev->coherent_dma_mask;
if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
goto err_iommu;
@@ -195,10 +261,6 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
for (i = 0; i < vdpasim->dev_attr.nas; i++)
vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL);
if (!vdpasim->buffer)
goto err_iommu;
for (i = 0; i < dev_attr->nvqs; i++)
vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
&vdpasim->iommu_lock);
@@ -214,6 +276,12 @@ err_alloc:
}
EXPORT_SYMBOL_GPL(vdpasim_create);
void vdpasim_schedule_work(struct vdpasim *vdpasim)
{
kthread_queue_work(vdpasim->worker, &vdpasim->work);
}
EXPORT_SYMBOL_GPL(vdpasim_schedule_work);
static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,
u64 desc_area, u64 driver_area,
u64 device_area)
@@ -248,7 +316,7 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
}
if (vq->ready)
schedule_work(&vdpasim->work);
vdpasim_schedule_work(vdpasim);
}
static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
@@ -267,13 +335,13 @@ static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)
struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
bool old_ready;
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
old_ready = vq->ready;
vq->ready = ready;
if (vq->ready && !old_ready) {
vdpasim_queue_ready(vdpasim, idx);
}
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
}
static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
@@ -291,9 +359,9 @@ static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
struct vringh *vrh = &vq->vring;
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
vrh->last_avail_idx = state->split.avail_index;
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
return 0;
}
@@ -390,9 +458,9 @@ static u8 vdpasim_get_status(struct vdpa_device *vdpa)
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
u8 status;
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
status = vdpasim->status;
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
return status;
}
@@ -401,19 +469,19 @@ static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
vdpasim->status = status;
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
}
static int vdpasim_reset(struct vdpa_device *vdpa)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
vdpasim->status = 0;
vdpasim_do_reset(vdpasim);
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
return 0;
}
@@ -422,9 +490,9 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
vdpasim->running = false;
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
return 0;
}
@@ -434,7 +502,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
int i;
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
vdpasim->running = true;
if (vdpasim->pending_kick) {
@@ -445,7 +513,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
vdpasim->pending_kick = false;
}
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
return 0;
}
@@ -517,14 +585,14 @@ static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group,
iommu = &vdpasim->iommu[asid];
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
for (i = 0; i < vdpasim->dev_attr.nvqs; i++)
if (vdpasim_get_vq_group(vdpa, i) == group)
vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,
&vdpasim->iommu_lock);
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
return 0;
}
@@ -563,6 +631,30 @@ err:
return ret;
}
static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
struct vdpasim_mm_work mm_work;
mm_work.vdpasim = vdpasim;
mm_work.mm_to_bind = mm;
vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
return mm_work.ret;
}
static void vdpasim_unbind_mm(struct vdpa_device *vdpa)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
struct vdpasim_mm_work mm_work;
mm_work.vdpasim = vdpasim;
mm_work.mm_to_bind = NULL;
vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
}
static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,
u64 iova, u64 size,
u64 pa, u32 perm, void *opaque)
@@ -610,14 +702,16 @@ static void vdpasim_free(struct vdpa_device *vdpa)
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
int i;
cancel_work_sync(&vdpasim->work);
kthread_cancel_work_sync(&vdpasim->work);
kthread_destroy_worker(vdpasim->worker);
for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov);
vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);
}
kvfree(vdpasim->buffer);
vdpasim->dev_attr.free(vdpasim);
for (i = 0; i < vdpasim->dev_attr.nas; i++)
vhost_iotlb_reset(&vdpasim->iommu[i]);
kfree(vdpasim->iommu);
@@ -658,6 +752,8 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
.set_group_asid = vdpasim_set_group_asid,
.dma_map = vdpasim_dma_map,
.dma_unmap = vdpasim_dma_unmap,
.bind_mm = vdpasim_bind_mm,
.unbind_mm = vdpasim_unbind_mm,
.free = vdpasim_free,
};
@@ -692,6 +788,8 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
.get_iova_range = vdpasim_get_iova_range,
.set_group_asid = vdpasim_set_group_asid,
.set_map = vdpasim_set_map,
.bind_mm = vdpasim_bind_mm,
.unbind_mm = vdpasim_unbind_mm,
.free = vdpasim_free,
};

14
drivers/vdpa/vdpa_sim/vdpa_sim.h
View File

@@ -39,33 +39,34 @@ struct vdpasim_dev_attr {
u64 supported_features;
size_t alloc_size;
size_t config_size;
size_t buffer_size;
int nvqs;
u32 id;
u32 ngroups;
u32 nas;
work_func_t work_fn;
void (*work_fn)(struct vdpasim *vdpasim);
void (*get_config)(struct vdpasim *vdpasim, void *config);
void (*set_config)(struct vdpasim *vdpasim, const void *config);
int (*get_stats)(struct vdpasim *vdpasim, u16 idx,
struct sk_buff *msg,
struct netlink_ext_ack *extack);
void (*free)(struct vdpasim *vdpasim);
};
/* State of each vdpasim device */
struct vdpasim {
struct vdpa_device vdpa;
struct vdpasim_virtqueue *vqs;
struct work_struct work;
struct kthread_worker *worker;
struct kthread_work work;
struct mm_struct *mm_bound;
struct vdpasim_dev_attr dev_attr;
/* spinlock to synchronize virtqueue state */
spinlock_t lock;
/* mutex to synchronize virtqueue state */
struct mutex mutex;
/* virtio config according to device type */
void *config;
struct vhost_iotlb *iommu;
bool *iommu_pt;
void *buffer;
u32 status;
u32 generation;
u64 features;
@@ -78,6 +79,7 @@ struct vdpasim {
struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
const struct vdpa_dev_set_config *config);
void vdpasim_schedule_work(struct vdpasim *vdpasim);
/* TODO: cross-endian support */
static inline bool vdpasim_is_little_endian(struct vdpasim *vdpasim)

93
drivers/vdpa/vdpa_sim/vdpa_sim_blk.c
View File

@@ -11,7 +11,6 @@
#include <linux/module.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/blkdev.h>
#include <linux/vringh.h>
#include <linux/vdpa.h>
@@ -44,8 +43,39 @@
#define VDPASIM_BLK_AS_NUM 1
#define VDPASIM_BLK_GROUP_NUM 1
struct vdpasim_blk {
struct vdpasim vdpasim;
void *buffer;
bool shared_backend;
};
static struct vdpasim_blk *sim_to_blk(struct vdpasim *vdpasim)
{
return container_of(vdpasim, struct vdpasim_blk, vdpasim);
}
static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
static bool shared_backend;
module_param(shared_backend, bool, 0444);
MODULE_PARM_DESC(shared_backend, "Enable the shared backend between virtio-blk devices");
static void *shared_buffer;
/* mutex to synchronize shared_buffer access */
static DEFINE_MUTEX(shared_buffer_mutex);
static void vdpasim_blk_buffer_lock(struct vdpasim_blk *blk)
{
if (blk->shared_backend)
mutex_lock(&shared_buffer_mutex);
}
static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk)
{
if (blk->shared_backend)
mutex_unlock(&shared_buffer_mutex);
}
static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
u64 num_sectors, u64 max_sectors)
{
@@ -79,6 +109,7 @@ static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
struct vdpasim_virtqueue *vq)
{
struct vdpasim_blk *blk = sim_to_blk(vdpasim);
size_t pushed = 0, to_pull, to_push;
struct virtio_blk_outhdr hdr;
bool handled = false;
@@ -144,9 +175,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
break;
}
vdpasim_blk_buffer_lock(blk);
bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
vdpasim->buffer + offset,
to_push);
blk->buffer + offset, to_push);
vdpasim_blk_buffer_unlock(blk);
if (bytes < 0) {
dev_dbg(&vdpasim->vdpa.dev,
"vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
@@ -166,9 +198,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
break;
}
vdpasim_blk_buffer_lock(blk);
bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov,
vdpasim->buffer + offset,
to_pull);
blk->buffer + offset, to_pull);
vdpasim_blk_buffer_unlock(blk);
if (bytes < 0) {
dev_dbg(&vdpasim->vdpa.dev,
"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
@@ -248,8 +281,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
}
if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
memset(vdpasim->buffer + offset, 0,
vdpasim_blk_buffer_lock(blk);
memset(blk->buffer + offset, 0,
num_sectors << SECTOR_SHIFT);
vdpasim_blk_buffer_unlock(blk);
}
break;
@@ -286,13 +321,12 @@ err:
return handled;
}
static void vdpasim_blk_work(struct work_struct *work)
static void vdpasim_blk_work(struct vdpasim *vdpasim)
{
struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
bool reschedule = false;
int i;
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
goto out;
@@ -323,10 +357,10 @@ static void vdpasim_blk_work(struct work_struct *work)
}
}
out:
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
if (reschedule)
schedule_work(&vdpasim->work);
vdpasim_schedule_work(vdpasim);
}
static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
@@ -355,6 +389,14 @@ static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
}
static void vdpasim_blk_free(struct vdpasim *vdpasim)
{
struct vdpasim_blk *blk = sim_to_blk(vdpasim);
if (!blk->shared_backend)
kvfree(blk->buffer);
}
static void vdpasim_blk_mgmtdev_release(struct device *dev)
{
}
@@ -368,6 +410,7 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
const struct vdpa_dev_set_config *config)
{
struct vdpasim_dev_attr dev_attr = {};
struct vdpasim_blk *blk;
struct vdpasim *simdev;
int ret;
@@ -378,16 +421,30 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
dev_attr.nas = VDPASIM_BLK_AS_NUM;
dev_attr.alloc_size = sizeof(struct vdpasim);
dev_attr.alloc_size = sizeof(struct vdpasim_blk);
dev_attr.config_size = sizeof(struct virtio_blk_config);
dev_attr.get_config = vdpasim_blk_get_config;
dev_attr.work_fn = vdpasim_blk_work;
dev_attr.buffer_size = VDPASIM_BLK_CAPACITY << SECTOR_SHIFT;
dev_attr.free = vdpasim_blk_free;
simdev = vdpasim_create(&dev_attr, config);
if (IS_ERR(simdev))
return PTR_ERR(simdev);
blk = sim_to_blk(simdev);
blk->shared_backend = shared_backend;
if (blk->shared_backend) {
blk->buffer = shared_buffer;
} else {
blk->buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
GFP_KERNEL);
if (!blk->buffer) {
ret = -ENOMEM;
goto put_dev;
}
}
ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM);
if (ret)
goto put_dev;
@@ -437,6 +494,15 @@ static int __init vdpasim_blk_init(void)
if (ret)
goto parent_err;
if (shared_backend) {
shared_buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
GFP_KERNEL);
if (!shared_buffer) {
ret = -ENOMEM;
goto parent_err;
}
}
return 0;
parent_err:
@@ -446,6 +512,7 @@ parent_err:
static void __exit vdpasim_blk_exit(void)
{
kvfree(shared_buffer);
vdpa_mgmtdev_unregister(&mgmt_dev);
device_unregister(&vdpasim_blk_mgmtdev);
}

38
drivers/vdpa/vdpa_sim/vdpa_sim_net.c
View File

@@ -11,7 +11,6 @@
#include <linux/module.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/etherdevice.h>
#include <linux/vringh.h>
#include <linux/vdpa.h>
@@ -59,6 +58,7 @@ struct vdpasim_net{
struct vdpasim_dataq_stats tx_stats;
struct vdpasim_dataq_stats rx_stats;
struct vdpasim_cq_stats cq_stats;
void *buffer;
};
static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim)
@@ -88,14 +88,15 @@ static bool receive_filter(struct vdpasim *vdpasim, size_t len)
size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) :
sizeof(struct virtio_net_hdr);
struct virtio_net_config *vio_config = vdpasim->config;
struct vdpasim_net *net = sim_to_net(vdpasim);
if (len < ETH_ALEN + hdr_len)
return false;
if (is_broadcast_ether_addr(vdpasim->buffer + hdr_len) ||
is_multicast_ether_addr(vdpasim->buffer + hdr_len))
if (is_broadcast_ether_addr(net->buffer + hdr_len) ||
is_multicast_ether_addr(net->buffer + hdr_len))
return true;
if (!strncmp(vdpasim->buffer + hdr_len, vio_config->mac, ETH_ALEN))
if (!strncmp(net->buffer + hdr_len, vio_config->mac, ETH_ALEN))
return true;
return false;
@@ -192,9 +193,8 @@ static void vdpasim_handle_cvq(struct vdpasim *vdpasim)
u64_stats_update_end(&net->cq_stats.syncp);
}
static void vdpasim_net_work(struct work_struct *work)
static void vdpasim_net_work(struct vdpasim *vdpasim)
{
struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
struct vdpasim_net *net = sim_to_net(vdpasim);
@@ -203,7 +203,7 @@ static void vdpasim_net_work(struct work_struct *work)
u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0;
int err;
spin_lock(&vdpasim->lock);
mutex_lock(&vdpasim->mutex);
if (!vdpasim->running)
goto out;
@@ -227,8 +227,7 @@ static void vdpasim_net_work(struct work_struct *work)
++tx_pkts;
read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov,
vdpasim->buffer,
PAGE_SIZE);
net->buffer, PAGE_SIZE);
tx_bytes += read;
@@ -247,7 +246,7 @@ static void vdpasim_net_work(struct work_struct *work)
}
write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov,
vdpasim->buffer, read);
net->buffer, read);
if (write <= 0) {
++rx_errors;
break;
@@ -260,13 +259,13 @@ static void vdpasim_net_work(struct work_struct *work)
vdpasim_net_complete(rxq, write);
if (tx_pkts > 4) {
schedule_work(&vdpasim->work);
vdpasim_schedule_work(vdpasim);
goto out;
}
}
out:
spin_unlock(&vdpasim->lock);
mutex_unlock(&vdpasim->mutex);
u64_stats_update_begin(&net->tx_stats.syncp);
net->tx_stats.pkts += tx_pkts;
@@ -429,6 +428,13 @@ static void vdpasim_net_setup_config(struct vdpasim *vdpasim,
vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500);
}
static void vdpasim_net_free(struct vdpasim *vdpasim)
{
struct vdpasim_net *net = sim_to_net(vdpasim);
kvfree(net->buffer);
}
static void vdpasim_net_mgmtdev_release(struct device *dev)
{
}
@@ -458,7 +464,7 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
dev_attr.get_config = vdpasim_net_get_config;
dev_attr.work_fn = vdpasim_net_work;
dev_attr.get_stats = vdpasim_net_get_stats;
dev_attr.buffer_size = PAGE_SIZE;
dev_attr.free = vdpasim_net_free;
simdev = vdpasim_create(&dev_attr, config);
if (IS_ERR(simdev))
@@ -472,6 +478,12 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
u64_stats_init(&net->rx_stats.syncp);
u64_stats_init(&net->cq_stats.syncp);
net->buffer = kvmalloc(PAGE_SIZE, GFP_KERNEL);
if (!net->buffer) {
ret = -ENOMEM;
goto reg_err;
}
/*
* Initialization must be completed before this call, since it can
* connect the device to the vDPA bus, so requests can arrive after

414
drivers/vdpa/vdpa_user/vduse_dev.c
View File

File diff suppressed because it is too large Load Diff

102
drivers/vhost/scsi.c
View File

@@ -229,7 +229,10 @@ struct vhost_scsi_ctx {
struct iov_iter out_iter;
};
/* Global spinlock to protect vhost_scsi TPG list for vhost IOCTL access */
/*
* Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
* configfs management operations.
*/
static DEFINE_MUTEX(vhost_scsi_mutex);
static LIST_HEAD(vhost_scsi_list);
@@ -1501,7 +1504,7 @@ out:
* vhost_scsi_tpg with an active struct vhost_scsi_nexus
*
* The lock nesting rule is:
* vhost_scsi_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
* vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
*/
static int
vhost_scsi_set_endpoint(struct vhost_scsi *vs,
@@ -1515,7 +1518,6 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
int index, ret, i, len;
bool match = false;
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&vs->dev.mutex);
/* Verify that ring has been setup correctly. */
@@ -1536,6 +1538,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
if (vs->vs_tpg)
memcpy(vs_tpg, vs->vs_tpg, len);
mutex_lock(&vhost_scsi_mutex);
list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
mutex_lock(&tpg->tv_tpg_mutex);
if (!tpg->tpg_nexus) {
@@ -1551,6 +1554,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
mutex_unlock(&tpg->tv_tpg_mutex);
mutex_unlock(&vhost_scsi_mutex);
ret = -EEXIST;
goto undepend;
}
@@ -1565,6 +1569,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
if (ret) {
pr_warn("target_depend_item() failed: %d\n", ret);
mutex_unlock(&tpg->tv_tpg_mutex);
mutex_unlock(&vhost_scsi_mutex);
goto undepend;
}
tpg->tv_tpg_vhost_count++;
@@ -1574,6 +1579,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
}
mutex_unlock(&tpg->tv_tpg_mutex);
}
mutex_unlock(&vhost_scsi_mutex);
if (match) {
memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
@@ -1629,7 +1635,6 @@ undepend:
kfree(vs_tpg);
out:
mutex_unlock(&vs->dev.mutex);
mutex_unlock(&vhost_scsi_mutex);
return ret;
}
@@ -1645,7 +1650,6 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
int index, ret, i;
u8 target;
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&vs->dev.mutex);
/* Verify that ring has been setup correctly. */
for (index = 0; index < vs->dev.nvqs; ++index) {
@@ -1666,11 +1670,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
if (!tpg)
continue;
mutex_lock(&tpg->tv_tpg_mutex);
tv_tport = tpg->tport;
if (!tv_tport) {
ret = -ENODEV;
goto err_tpg;
goto err_dev;
}
if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
@@ -1679,35 +1682,51 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
tv_tport->tport_name, tpg->tport_tpgt,
t->vhost_wwpn, t->vhost_tpgt);
ret = -EINVAL;
goto err_tpg;
goto err_dev;
}
match = true;
}
if (!match)
goto free_vs_tpg;
/* Prevent new cmds from starting and accessing the tpgs/sessions */
for (i = 0; i < vs->dev.nvqs; i++) {
vq = &vs->vqs[i].vq;
mutex_lock(&vq->mutex);
vhost_vq_set_backend(vq, NULL);
mutex_unlock(&vq->mutex);
}
/* Make sure cmds are not running before tearing them down. */
vhost_scsi_flush(vs);
for (i = 0; i < vs->dev.nvqs; i++) {
vq = &vs->vqs[i].vq;
vhost_scsi_destroy_vq_cmds(vq);
}
/*
* We can now release our hold on the tpg and sessions and userspace
* can free them after this point.
*/
for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
target = i;
tpg = vs->vs_tpg[target];
if (!tpg)
continue;
mutex_lock(&tpg->tv_tpg_mutex);
tpg->tv_tpg_vhost_count--;
tpg->vhost_scsi = NULL;
vs->vs_tpg[target] = NULL;
match = true;
mutex_unlock(&tpg->tv_tpg_mutex);
/*
* Release se_tpg->tpg_group.cg_item configfs dependency now
* to allow vhost-scsi WWPN se_tpg->tpg_group shutdown to occur.
*/
se_tpg = &tpg->se_tpg;
target_undepend_item(&se_tpg->tpg_group.cg_item);
}
if (match) {
for (i = 0; i < vs->dev.nvqs; i++) {
vq = &vs->vqs[i].vq;
mutex_lock(&vq->mutex);
vhost_vq_set_backend(vq, NULL);
mutex_unlock(&vq->mutex);
}
/* Make sure cmds are not running before tearing them down. */
vhost_scsi_flush(vs);
for (i = 0; i < vs->dev.nvqs; i++) {
vq = &vs->vqs[i].vq;
vhost_scsi_destroy_vq_cmds(vq);
}
}
free_vs_tpg:
/*
* Act as synchronize_rcu to make sure access to
* old vs->vs_tpg is finished.
@@ -1717,14 +1736,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
vs->vs_tpg = NULL;
WARN_ON(vs->vs_events_nr);
mutex_unlock(&vs->dev.mutex);
mutex_unlock(&vhost_scsi_mutex);
return 0;
err_tpg:
mutex_unlock(&tpg->tv_tpg_mutex);
err_dev:
mutex_unlock(&vs->dev.mutex);
mutex_unlock(&vhost_scsi_mutex);
return ret;
}
@@ -1965,8 +1980,6 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
if (!vs)
return;
mutex_lock(&vs->dev.mutex);
if (plug)
reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
else
@@ -1974,11 +1987,18 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
mutex_lock(&vq->mutex);
/*
* We can't queue events if the backend has been cleared, because
* we could end up queueing an event after the flush.
*/
if (!vhost_vq_get_backend(vq))
goto unlock;
if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
vhost_scsi_send_evt(vs, tpg, lun,
VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
unlock:
mutex_unlock(&vq->mutex);
mutex_unlock(&vs->dev.mutex);
}
static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
@@ -1997,15 +2017,10 @@ static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
struct vhost_scsi_tpg *tpg = container_of(se_tpg,
struct vhost_scsi_tpg, se_tpg);
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&tpg->tv_tpg_mutex);
tpg->tv_tpg_port_count++;
mutex_unlock(&tpg->tv_tpg_mutex);
vhost_scsi_hotplug(tpg, lun);
mutex_unlock(&vhost_scsi_mutex);
mutex_unlock(&tpg->tv_tpg_mutex);
return 0;
}
@@ -2016,15 +2031,10 @@ static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
struct vhost_scsi_tpg *tpg = container_of(se_tpg,
struct vhost_scsi_tpg, se_tpg);
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&tpg->tv_tpg_mutex);
tpg->tv_tpg_port_count--;
mutex_unlock(&tpg->tv_tpg_mutex);
vhost_scsi_hotunplug(tpg, lun);
mutex_unlock(&vhost_scsi_mutex);
mutex_unlock(&tpg->tv_tpg_mutex);
}
static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(

44
drivers/vhost/vdpa.c
View File

@@ -219,6 +219,28 @@ static int vhost_vdpa_reset(struct vhost_vdpa *v)
return vdpa_reset(vdpa);
}
static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
{
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
if (!vdpa->use_va || !ops->bind_mm)
return 0;
return ops->bind_mm(vdpa, v->vdev.mm);
}
static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
{
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
if (!vdpa->use_va || !ops->unbind_mm)
return;
ops->unbind_mm(vdpa);
}
static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
{
struct vdpa_device *vdpa = v->vdpa;
@@ -599,9 +621,11 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
if (vq->call_ctx.ctx) {
cb.callback = vhost_vdpa_virtqueue_cb;
cb.private = vq;
cb.trigger = vq->call_ctx.ctx;
} else {
cb.callback = NULL;
cb.private = NULL;
cb.trigger = NULL;
}
ops->set_vq_cb(vdpa, idx, &cb);
vhost_vdpa_setup_vq_irq(v, idx);
@@ -716,6 +740,17 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
break;
}
if (r)
goto out;
switch (cmd) {
case VHOST_SET_OWNER:
r = vhost_vdpa_bind_mm(v);
if (r)
vhost_dev_reset_owner(d, NULL);
break;
}
out:
mutex_unlock(&d->mutex);
return r;
}
@@ -851,11 +886,7 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
if (!v->in_batch)
ops->set_map(vdpa, asid, iotlb);
}
/* If we are in the middle of batch processing, delay the free
* of AS until BATCH_END.
*/
if (!v->in_batch && !iotlb->nmaps)
vhost_vdpa_remove_as(v, asid);
}
static int vhost_vdpa_va_map(struct vhost_vdpa *v,
@@ -1112,8 +1143,6 @@ static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
if (v->in_batch && ops->set_map)
ops->set_map(vdpa, asid, iotlb);
v->in_batch = false;
if (!iotlb->nmaps)
vhost_vdpa_remove_as(v, asid);
break;
default:
r = -EINVAL;
@@ -1287,6 +1316,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
vhost_vdpa_clean_irq(v);
vhost_vdpa_reset(v);
vhost_dev_stop(&v->vdev);
vhost_vdpa_unbind_mm(v);
vhost_vdpa_config_put(v);
vhost_vdpa_cleanup(v);
mutex_unlock(&d->mutex);

6
drivers/vhost/vhost.c
View File

@@ -434,8 +434,7 @@ static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
size_t event __maybe_unused =
vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return sizeof(*vq->avail) +
sizeof(*vq->avail->ring) * num + event;
return size_add(struct_size(vq->avail, ring, num), event);
}
static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
@@ -444,8 +443,7 @@ static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
size_t event __maybe_unused =
vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return sizeof(*vq->used) +
sizeof(*vq->used->ring) * num + event;
return size_add(struct_size(vq->used, ring, num), event);
}
static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,

191
drivers/vhost/vringh.c
View File

@@ -636,9 +636,9 @@ static inline int xfer_to_user(const struct vringh *vrh,
* @features: the feature bits for this ring.
* @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userpace descriptor pointer.
* @avail: the userpace avail pointer.
* @used: the userpace used pointer.
* @desc: the userspace descriptor pointer.
* @avail: the userspace avail pointer.
* @used: the userspace used pointer.
*
* Returns an error if num is invalid: you should check pointers
* yourself!
@@ -911,9 +911,9 @@ static inline int kern_xfer(const struct vringh *vrh, void *dst,
* @features: the feature bits for this ring.
* @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userpace descriptor pointer.
* @avail: the userpace avail pointer.
* @used: the userpace used pointer.
* @desc: the userspace descriptor pointer.
* @avail: the userspace avail pointer.
* @used: the userspace used pointer.
*
* Returns an error if num is invalid.
*/
@@ -1094,10 +1094,17 @@ EXPORT_SYMBOL(vringh_need_notify_kern);
#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
struct iotlb_vec {
union {
struct iovec *iovec;
struct bio_vec *bvec;
} iov;
size_t count;
};
static int iotlb_translate(const struct vringh *vrh,
u64 addr, u64 len, u64 *translated,
struct bio_vec iov[],
int iov_size, u32 perm)
struct iotlb_vec *ivec, u32 perm)
{
struct vhost_iotlb_map *map;
struct vhost_iotlb *iotlb = vrh->iotlb;
@@ -1107,9 +1114,11 @@ static int iotlb_translate(const struct vringh *vrh,
spin_lock(vrh->iotlb_lock);
while (len > s) {
u64 size, pa, pfn;
uintptr_t io_addr;
size_t io_len;
u64 size;
if (unlikely(ret >= iov_size)) {
if (unlikely(ret >= ivec->count)) {
ret = -ENOBUFS;
break;
}
@@ -1124,10 +1133,22 @@ static int iotlb_translate(const struct vringh *vrh,
}
size = map->size - addr + map->start;
pa = map->addr + addr - map->start;
pfn = pa >> PAGE_SHIFT;
bvec_set_page(&iov[ret], pfn_to_page(pfn), min(len - s, size),
pa & (PAGE_SIZE - 1));
io_len = min(len - s, size);
io_addr = map->addr - map->start + addr;
if (vrh->use_va) {
struct iovec *iovec = ivec->iov.iovec;
iovec[ret].iov_len = io_len;
iovec[ret].iov_base = (void __user *)io_addr;
} else {
u64 pfn = io_addr >> PAGE_SHIFT;
struct bio_vec *bvec = ivec->iov.bvec;
bvec_set_page(&bvec[ret], pfn_to_page(pfn), io_len,
io_addr & (PAGE_SIZE - 1));
}
s += size;
addr += size;
++ret;
@@ -1141,26 +1162,41 @@ static int iotlb_translate(const struct vringh *vrh,
return ret;
}
#define IOTLB_IOV_STRIDE 16
static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
void *src, size_t len)
{
struct iotlb_vec ivec;
union {
struct iovec iovec[IOTLB_IOV_STRIDE];
struct bio_vec bvec[IOTLB_IOV_STRIDE];
} iov;
u64 total_translated = 0;
ivec.iov.iovec = iov.iovec;
ivec.count = IOTLB_IOV_STRIDE;
while (total_translated < len) {
struct bio_vec iov[16];
struct iov_iter iter;
u64 translated;
int ret;
ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
len - total_translated, &translated,
iov, ARRAY_SIZE(iov), VHOST_MAP_RO);
&ivec, VHOST_MAP_RO);
if (ret == -ENOBUFS)
ret = ARRAY_SIZE(iov);
ret = IOTLB_IOV_STRIDE;
else if (ret < 0)
return ret;
iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, translated);
if (vrh->use_va) {
iov_iter_init(&iter, ITER_SOURCE, ivec.iov.iovec, ret,
translated);
} else {
iov_iter_bvec(&iter, ITER_SOURCE, ivec.iov.bvec, ret,
translated);
}
ret = copy_from_iter(dst, translated, &iter);
if (ret < 0)
@@ -1177,23 +1213,36 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
void *src, size_t len)
{
struct iotlb_vec ivec;
union {
struct iovec iovec[IOTLB_IOV_STRIDE];
struct bio_vec bvec[IOTLB_IOV_STRIDE];
} iov;
u64 total_translated = 0;
ivec.iov.iovec = iov.iovec;
ivec.count = IOTLB_IOV_STRIDE;
while (total_translated < len) {
struct bio_vec iov[16];
struct iov_iter iter;
u64 translated;
int ret;
ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
len - total_translated, &translated,
iov, ARRAY_SIZE(iov), VHOST_MAP_WO);
&ivec, VHOST_MAP_WO);
if (ret == -ENOBUFS)
ret = ARRAY_SIZE(iov);
ret = IOTLB_IOV_STRIDE;
else if (ret < 0)
return ret;
iov_iter_bvec(&iter, ITER_DEST, iov, ret, translated);
if (vrh->use_va) {
iov_iter_init(&iter, ITER_DEST, ivec.iov.iovec, ret,
translated);
} else {
iov_iter_bvec(&iter, ITER_DEST, ivec.iov.bvec, ret,
translated);
}
ret = copy_to_iter(src, translated, &iter);
if (ret < 0)
@@ -1210,20 +1259,36 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
static inline int getu16_iotlb(const struct vringh *vrh,
u16 *val, const __virtio16 *p)
{
struct bio_vec iov;
void *kaddr, *from;
struct iotlb_vec ivec;
union {
struct iovec iovec[1];
struct bio_vec bvec[1];
} iov;
__virtio16 tmp;
int ret;
ivec.iov.iovec = iov.iovec;
ivec.count = 1;
/* Atomic read is needed for getu16 */
ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
&iov, 1, VHOST_MAP_RO);
ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
NULL, &ivec, VHOST_MAP_RO);
if (ret < 0)
return ret;
kaddr = kmap_atomic(iov.bv_page);
from = kaddr + iov.bv_offset;
*val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));
kunmap_atomic(kaddr);
if (vrh->use_va) {
ret = __get_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
if (ret)
return ret;
} else {
void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page);
void *from = kaddr + ivec.iov.bvec[0].bv_offset;
tmp = READ_ONCE(*(__virtio16 *)from);
kunmap_local(kaddr);
}
*val = vringh16_to_cpu(vrh, tmp);
return 0;
}
@@ -1231,20 +1296,36 @@ static inline int getu16_iotlb(const struct vringh *vrh,
static inline int putu16_iotlb(const struct vringh *vrh,
__virtio16 *p, u16 val)
{
struct bio_vec iov;
void *kaddr, *to;
struct iotlb_vec ivec;
union {
struct iovec iovec;
struct bio_vec bvec;
} iov;
__virtio16 tmp;
int ret;
ivec.iov.iovec = &iov.iovec;
ivec.count = 1;
/* Atomic write is needed for putu16 */
ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
&iov, 1, VHOST_MAP_WO);
ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
NULL, &ivec, VHOST_MAP_RO);
if (ret < 0)
return ret;
kaddr = kmap_atomic(iov.bv_page);
to = kaddr + iov.bv_offset;
WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));
kunmap_atomic(kaddr);
tmp = cpu_to_vringh16(vrh, val);
if (vrh->use_va) {
ret = __put_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
if (ret)
return ret;
} else {
void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page);
void *to = kaddr + ivec.iov.bvec[0].bv_offset;
WRITE_ONCE(*(__virtio16 *)to, tmp);
kunmap_local(kaddr);
}
return 0;
}
@@ -1306,9 +1387,9 @@ static inline int putused_iotlb(const struct vringh *vrh,
* @features: the feature bits for this ring.
* @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userpace descriptor pointer.
* @avail: the userpace avail pointer.
* @used: the userpace used pointer.
* @desc: the userspace descriptor pointer.
* @avail: the userspace avail pointer.
* @used: the userspace used pointer.
*
* Returns an error if num is invalid.
*/
@@ -1318,11 +1399,39 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features,
struct vring_avail *avail,
struct vring_used *used)
{
vrh->use_va = false;
return vringh_init_kern(vrh, features, num, weak_barriers,
desc, avail, used);
}
EXPORT_SYMBOL(vringh_init_iotlb);
/**
* vringh_init_iotlb_va - initialize a vringh for a ring with IOTLB containing
* user VA.
* @vrh: the vringh to initialize.
* @features: the feature bits for this ring.
* @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userspace descriptor pointer.
* @avail: the userspace avail pointer.
* @used: the userspace used pointer.
*
* Returns an error if num is invalid.
*/
int vringh_init_iotlb_va(struct vringh *vrh, u64 features,
unsigned int num, bool weak_barriers,
struct vring_desc *desc,
struct vring_avail *avail,
struct vring_used *used)
{
vrh->use_va = true;
return vringh_init_kern(vrh, features, num, weak_barriers,
desc, avail, used);
}
EXPORT_SYMBOL(vringh_init_iotlb_va);
/**
* vringh_set_iotlb - initialize a vringh for a ring with IOTLB.
* @vrh: the vring

18
drivers/virtio/virtio_mmio.c
View File

@@ -286,6 +286,16 @@ static bool vm_notify(struct virtqueue *vq)
return true;
}
static bool vm_notify_with_data(struct virtqueue *vq)
{
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
u32 data = vring_notification_data(vq);
writel(data, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
return true;
}
/* Notify all virtqueues on an interrupt. */
static irqreturn_t vm_interrupt(int irq, void *opaque)
{
@@ -364,12 +374,18 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in
const char *name, bool ctx)
{
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
bool (*notify)(struct virtqueue *vq);
struct virtio_mmio_vq_info *info;
struct virtqueue *vq;
unsigned long flags;
unsigned int num;
int err;
if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA))
notify = vm_notify_with_data;
else
notify = vm_notify;
if (!name)
return NULL;
@@ -398,7 +414,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in
/* Create the vring */
vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev,
true, true, ctx, vm_notify, callback, name);
true, true, ctx, notify, callback, name);
if (!vq) {
err = -ENOMEM;
goto error_new_virtqueue;

22
drivers/virtio/virtio_pci_modern.c
View File

@@ -288,6 +288,15 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
return vp_modern_config_vector(&vp_dev->mdev, vector);
}
static bool vp_notify_with_data(struct virtqueue *vq)
{
u32 data = vring_notification_data(vq);
iowrite32(data, (void __iomem *)vq->priv);
return true;
}
static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
struct virtio_pci_vq_info *info,
unsigned int index,
@@ -298,10 +307,16 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
{
struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
bool (*notify)(struct virtqueue *vq);
struct virtqueue *vq;
u16 num;
int err;
if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA))
notify = vp_notify_with_data;
else
notify = vp_notify;
if (index >= vp_modern_get_num_queues(mdev))
return ERR_PTR(-EINVAL);
@@ -310,18 +325,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
if (!num || vp_modern_get_queue_enable(mdev, index))
return ERR_PTR(-ENOENT);
if (!is_power_of_2(num)) {
dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);
return ERR_PTR(-EINVAL);
}
info->msix_vector = msix_vec;
/* create the vring */
vq = vring_create_virtqueue(index, num,
SMP_CACHE_BYTES, &vp_dev->vdev,
true, true, ctx,
vp_notify, callback, name);
notify, callback, name);
if (!vq)
return ERR_PTR(-ENOMEM);

89
drivers/virtio/virtio_ring.c
View File

@@ -231,10 +231,10 @@ static void vring_free(struct virtqueue *_vq);
* Helpers.
*/
#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
unsigned int total_sg)
static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
unsigned int total_sg)
{
/*
* If the host supports indirect descriptor tables, and we have multiple
@@ -269,7 +269,7 @@ static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
* unconditionally on data path.
*/
static bool vring_use_dma_api(struct virtio_device *vdev)
static bool vring_use_dma_api(const struct virtio_device *vdev)
{
if (!virtio_has_dma_quirk(vdev))
return true;
@@ -289,7 +289,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev)
return false;
}
size_t virtio_max_dma_size(struct virtio_device *vdev)
size_t virtio_max_dma_size(const struct virtio_device *vdev)
{
size_t max_segment_size = SIZE_MAX;
@@ -349,7 +349,7 @@ static void vring_free_queue(struct virtio_device *vdev, size_t size,
* making all of the arch DMA ops work on the vring device itself
* is a mess.
*/
static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
{
return vq->dma_dev;
}
@@ -423,7 +423,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
*/
static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
struct vring_desc *desc)
const struct vring_desc *desc)
{
u16 flags;
@@ -784,7 +784,7 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
}
}
static inline bool more_used_split(const struct vring_virtqueue *vq)
static bool more_used_split(const struct vring_virtqueue *vq)
{
return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
vq->split.vring.used->idx);
@@ -854,6 +854,14 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
/*
* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
if (vq->event)
/* TODO: this is a hack. Figure out a cleaner value to write. */
vring_used_event(&vq->split.vring) = 0x0;
@@ -1172,18 +1180,18 @@ err:
/*
* Packed ring specific functions - *_packed().
*/
static inline bool packed_used_wrap_counter(u16 last_used_idx)
static bool packed_used_wrap_counter(u16 last_used_idx)
{
return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
}
static inline u16 packed_last_used(u16 last_used_idx)
static u16 packed_last_used(u16 last_used_idx)
{
return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
}
static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
struct vring_desc_extra *extra)
const struct vring_desc_extra *extra)
{
u16 flags;
@@ -1206,7 +1214,7 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
}
static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
struct vring_packed_desc *desc)
const struct vring_packed_desc *desc)
{
u16 flags;
@@ -1612,7 +1620,7 @@ static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
return avail == used && used == used_wrap_counter;
}
static inline bool more_used_packed(const struct vring_virtqueue *vq)
static bool more_used_packed(const struct vring_virtqueue *vq)
{
u16 last_used;
u16 last_used_idx;
@@ -1699,6 +1707,14 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
/*
* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
vq->packed.vring.driver->flags =
cpu_to_le16(vq->packed.event_flags_shadow);
}
@@ -2330,12 +2346,6 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
/* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
if (vq->packed_ring)
virtqueue_disable_cb_packed(_vq);
else
@@ -2752,6 +2762,23 @@ void vring_del_virtqueue(struct virtqueue *_vq)
}
EXPORT_SYMBOL_GPL(vring_del_virtqueue);
u32 vring_notification_data(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 next;
if (vq->packed_ring)
next = (vq->packed.next_avail_idx &
~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
vq->packed.avail_wrap_counter <<
VRING_PACKED_EVENT_F_WRAP_CTR;
else
next = vq->split.avail_idx_shadow;
return next << 16 | _vq->index;
}
EXPORT_SYMBOL_GPL(vring_notification_data);
/* Manipulates transport-specific feature bits. */
void vring_transport_features(struct virtio_device *vdev)
{
@@ -2771,6 +2798,8 @@ void vring_transport_features(struct virtio_device *vdev)
break;
case VIRTIO_F_ORDER_PLATFORM:
break;
case VIRTIO_F_NOTIFICATION_DATA:
break;
default:
/* We don't understand this bit. */
__virtio_clear_bit(vdev, i);
@@ -2786,10 +2815,10 @@ EXPORT_SYMBOL_GPL(vring_transport_features);
* Returns the size of the vring. This is mainly used for boasting to
* userspace. Unlike other operations, this need not be serialized.
*/
unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
const struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
}
@@ -2819,9 +2848,9 @@ void __virtqueue_unbreak(struct virtqueue *_vq)
}
EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
bool virtqueue_is_broken(struct virtqueue *_vq)
bool virtqueue_is_broken(const struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
const struct vring_virtqueue *vq = to_vvq(_vq);
return READ_ONCE(vq->broken);
}
@@ -2868,9 +2897,9 @@ void __virtio_unbreak_device(struct virtio_device *dev)
}
EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring);
@@ -2881,9 +2910,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
}
EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring);
@@ -2895,9 +2924,9 @@ dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
}
EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring);
@@ -2910,7 +2939,7 @@ dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
/* Only available for split ring */
const struct vring *virtqueue_get_vring(struct virtqueue *vq)
const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
{
return &to_vvq(vq)->split.vring;
}

Some files were not shown because too many files have changed in this diff Show More