izzy/xemu
0
0
Fork 0
mirror of https://github.com/izzy2lost/xemu.git synced 2026-07-06 00:20:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

hw/rdma: Remove deprecated pvrdma device and rdmacm-mux helper

Browse Source 
The whole RDMA subsystem was deprecated in commit e9a54265f5
("hw/rdma: Deprecate the pvrdma device and the rdma subsystem")
released in v8.2.

Remove:
 - PVRDMA device
 - generated vmw_pvrdma/ directory from linux-headers
 - rdmacm-mux tool from contrib/

Cc: Yuval Shaia <yuval.shaia.ml@gmail.com>
Cc: Marcel Apfelbaum <marcel.apfelbaum@gmail.com>
Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Message-Id: <20240328130255.52257-2-philmd@linaro.org>
This commit is contained in:
Philippe Mathieu-Daudé
2024-04-24 16:03:38 +02:00
parent a60e53fa8f
commit 1dfd42c426
51 changed files with 5 additions and 7977 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Kconfig.host
-3
View File
@@ -35,9 +35,6 @@ config VHOST_KERNEL
config VIRTFS
bool
config PVRDMA
bool
config MULTIPROCESS_ALLOWED
bool
imply MULTIPROCESS
MAINTAINERS
-10
View File
@@ -4044,16 +4044,6 @@ F: block/replication.c
F: tests/unit/test-replication.c
F: docs/block-replication.txt
PVRDMA
M: Yuval Shaia <yuval.shaia.ml@gmail.com>
M: Marcel Apfelbaum <marcel.apfelbaum@gmail.com>
S: Odd Fixes
F: hw/rdma/*
F: hw/rdma/vmw/*
F: docs/pvrdma.txt
F: contrib/rdmacm-mux/*
F: qapi/rdma.json
Semihosting
M: Alex Bennée <alex.bennee@linaro.org>
S: Maintained
contrib/rdmacm-mux/main.c
-831
View File
File diff suppressed because it is too large Load Diff
contrib/rdmacm-mux/meson.build
-7
View File
@@ -1,7 +0,0 @@
if have_pvrdma
# FIXME: broken on big endian architectures
executable('rdmacm-mux', files('main.c'), genh,
dependencies: [glib, libumad],
build_by_default: false,
install: false)
endif
contrib/rdmacm-mux/rdmacm-mux.h
-61
View File
@@ -1,61 +0,0 @@
/*
* QEMU paravirtual RDMA - rdmacm-mux declarations
*
* Copyright (C) 2018 Oracle
* Copyright (C) 2018 Red Hat Inc
*
* Authors:
* Yuval Shaia <yuval.shaia@oracle.com>
* Marcel Apfelbaum <marcel@redhat.com>
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*
*/
#ifndef RDMACM_MUX_H
#define RDMACM_MUX_H
#include "linux/if.h"
#include <infiniband/verbs.h>
#include <infiniband/umad.h>
#include <rdma/rdma_user_cm.h>
typedef enum RdmaCmMuxMsgType {
RDMACM_MUX_MSG_TYPE_REQ = 0,
RDMACM_MUX_MSG_TYPE_RESP = 1,
} RdmaCmMuxMsgType;
typedef enum RdmaCmMuxOpCode {
RDMACM_MUX_OP_CODE_REG = 0,
RDMACM_MUX_OP_CODE_UNREG = 1,
RDMACM_MUX_OP_CODE_MAD = 2,
} RdmaCmMuxOpCode;
typedef enum RdmaCmMuxErrCode {
RDMACM_MUX_ERR_CODE_OK = 0,
RDMACM_MUX_ERR_CODE_EINVAL = 1,
RDMACM_MUX_ERR_CODE_EEXIST = 2,
RDMACM_MUX_ERR_CODE_EACCES = 3,
RDMACM_MUX_ERR_CODE_ENOTFOUND = 4,
} RdmaCmMuxErrCode;
typedef struct RdmaCmMuxHdr {
RdmaCmMuxMsgType msg_type;
RdmaCmMuxOpCode op_code;
union ibv_gid sgid;
RdmaCmMuxErrCode err_code;
} RdmaCmUHdr;
typedef struct RdmaCmUMad {
struct ib_user_mad hdr;
char mad[RDMA_MAX_PRIVATE_DATA];
} RdmaCmUMad;
typedef struct RdmaCmMuxMsg {
RdmaCmUHdr hdr;
int umad_len;
RdmaCmUMad umad;
} RdmaCmMuxMsg;
#endif
docs/about/deprecated.rst
-9
View File
@@ -365,15 +365,6 @@ recommending to switch to their stable counterparts:
- "Zve64f" should be replaced with "zve64f"
- "Zve64d" should be replaced with "zve64d"
``-device pvrdma`` and the rdma subsystem (since 8.2)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The pvrdma device and the whole rdma subsystem are in a bad shape and
without active maintenance. The QEMU project intends to remove this
device and subsystem from the code base in a future release without
replacement unless somebody steps up and improves the situation.
Block device options
''''''''''''''''''''
docs/about/removed-features.rst
+4
View File
@@ -925,6 +925,10 @@ contains native support for this feature and thus use of the option
ROM approach was obsolete. The native SeaBIOS support can be activated
by using ``-machine graphics=off``.
``pvrdma`` and the RDMA subsystem (removed in 9.1)
''''''''''''''''''''''''''''''''''''''''''''''''''
The 'pvrdma' device and the whole RDMA subsystem have been removed.
Related binaries
----------------
docs/pvrdma.txt
-345
View File
@@ -1,345 +0,0 @@
Paravirtualized RDMA Device (PVRDMA)
====================================
1. Description
===============
PVRDMA is the QEMU implementation of VMware's paravirtualized RDMA device.
It works with its Linux Kernel driver AS IS, no need for any special guest
modifications.
While it complies with the VMware device, it can also communicate with bare
metal RDMA-enabled machines as peers.
It does not require an RDMA HCA in the host, it can work with Soft-RoCE (rxe).
It does not require the whole guest RAM to be pinned allowing memory
over-commit and, even if not implemented yet, migration support will be
possible with some HW assistance.
A project presentation accompany this document:
- https://blog.linuxplumbersconf.org/2017/ocw/system/presentations/4730/original/lpc-2017-pvrdma-marcel-apfelbaum-yuval-shaia.pdf
2. Setup
========
2.1 Guest setup
===============
Fedora 27+ kernels work out of the box, older distributions
require updating the kernel to 4.14 to include the pvrdma driver.
However the libpvrdma library needed by User Level Software is still
not available as part of the distributions, so the rdma-core library
needs to be compiled and optionally installed.
Please follow the instructions at:
https://github.com/linux-rdma/rdma-core.git
2.2 Host Setup
==============
The pvrdma backend is an ibdevice interface that can be exposed
either by a Soft-RoCE(rxe) device on machines with no RDMA device,
or an HCA SRIOV function(VF/PF).
Note that ibdevice interfaces can't be shared between pvrdma devices,
each one requiring a separate instance (rxe or SRIOV VF).
2.2.1 Soft-RoCE backend(rxe)
===========================
A stable version of rxe is required, Fedora 27+ or a Linux
Kernel 4.14+ is preferred.
The rdma_rxe module is part of the Linux Kernel but not loaded by default.
Install the User Level library (librxe) following the instructions from:
https://github.com/SoftRoCE/rxe-dev/wiki/rxe-dev:-Home
Associate an ETH interface with rxe by running:
rxe_cfg add eth0
An rxe0 ibdevice interface will be created and can be used as pvrdma backend.
2.2.2 RDMA device Virtual Function backend
==========================================
Nothing special is required, the pvrdma device can work not only with
Ethernet Links, but also Infinibands Links.
All is needed is an ibdevice with an active port, for Mellanox cards
will be something like mlx5_6 which can be the backend.
2.2.3 QEMU setup
================
Configure QEMU with --enable-rdma flag, installing
the required RDMA libraries.
3. Usage
========
3.1 VM Memory settings
======================
Currently the device is working only with memory backed RAM
and it must be mark as "shared":
-m 1G \
-object memory-backend-ram,id=mb1,size=1G,share \
-numa node,memdev=mb1 \
3.2 MAD Multiplexer
===================
MAD Multiplexer is a service that exposes MAD-like interface for VMs in
order to overcome the limitation where only single entity can register with
MAD layer to send and receive RDMA-CM MAD packets.
To build rdmacm-mux run
# make rdmacm-mux
Before running the rdmacm-mux make sure that both ib_cm and rdma_cm kernel
modules aren't loaded, otherwise the rdmacm-mux service will fail to start.
The application accepts 3 command line arguments and exposes a UNIX socket
to pass control and data to it.
-d rdma-device-name Name of RDMA device to register with
-s unix-socket-path Path to unix socket to listen (default /var/run/rdmacm-mux)
-p rdma-device-port Port number of RDMA device to register with (default 1)
The final UNIX socket file name is a concatenation of the 3 arguments so
for example for device mlx5_0 on port 2 this /var/run/rdmacm-mux-mlx5_0-2
will be created.
pvrdma requires this service.
Please refer to contrib/rdmacm-mux for more details.
3.3 Service exposed by libvirt daemon
=====================================
The control over the RDMA device's GID table is done by updating the
device's Ethernet function addresses.
Usually the first GID entry is determined by the MAC address, the second by
the first IPv6 address and the third by the IPv4 address. Other entries can
be added by adding more IP addresses. The opposite is the same, i.e.
whenever an address is removed, the corresponding GID entry is removed.
The process is done by the network and RDMA stacks. Whenever an address is
added the ib_core driver is notified and calls the device driver add_gid
function which in turn update the device.
To support this in pvrdma device the device hooks into the create_bind and
destroy_bind HW commands triggered by pvrdma driver in guest.
Whenever changed is made to the pvrdma port's GID table a special QMP
messages is sent to be processed by libvirt to update the address of the
backend Ethernet device.
pvrdma requires that libvirt service will be up.
3.4 PCI devices settings
========================
RoCE device exposes two functions - an Ethernet and RDMA.
To support it, pvrdma device is composed of two PCI functions, an Ethernet
device of type vmxnet3 on PCI slot 0 and a PVRDMA device on PCI slot 1. The
Ethernet function can be used for other Ethernet purposes such as IP.
3.5 Device parameters
=====================
- netdev: Specifies the Ethernet device function name on the host for
example enp175s0f0. For Soft-RoCE device (rxe) this would be the Ethernet
device used to create it.
- ibdev: The IB device name on host for example rxe0, mlx5_0 etc.
- mad-chardev: The name of the MAD multiplexer char device.
- ibport: In case of multi-port device (such as Mellanox's HCA) this
specify the port to use. If not set 1 will be used.
- dev-caps-max-mr-size: The maximum size of MR.
- dev-caps-max-qp: Maximum number of QPs.
- dev-caps-max-cq: Maximum number of CQs.
- dev-caps-max-mr: Maximum number of MRs.
- dev-caps-max-pd: Maximum number of PDs.
- dev-caps-max-ah: Maximum number of AHs.
Notes:
- The first 3 parameters are mandatory settings, the rest have their
defaults.
- The last 8 parameters (the ones that prefixed by dev-caps) defines the top
limits but the final values is adjusted by the backend device limitations.
- netdev can be extracted from ibdev's sysfs
(/sys/class/infiniband/<ibdev>/device/net/)
3.6 Example
===========
Define bridge device with vmxnet3 network backend:
<interface type='bridge'>
<mac address='56:b4:44:e9:62:dc'/>
<source bridge='bridge1'/>
<model type='vmxnet3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x10' function='0x0' multifunction='on'/>
</interface>
Define pvrdma device:
<qemu:commandline>
<qemu:arg value='-object'/>
<qemu:arg value='memory-backend-ram,id=mb1,size=1G,share'/>
<qemu:arg value='-numa'/>
<qemu:arg value='node,memdev=mb1'/>
<qemu:arg value='-chardev'/>
<qemu:arg value='socket,path=/var/run/rdmacm-mux-rxe0-1,id=mads'/>
<qemu:arg value='-device'/>
<qemu:arg value='pvrdma,addr=10.1,ibdev=rxe0,netdev=bridge0,mad-chardev=mads'/>
</qemu:commandline>
4. Implementation details
=========================
4.1 Overview
============
The device acts like a proxy between the Guest Driver and the host
ibdevice interface.
On configuration path:
- For every hardware resource request (PD/QP/CQ/...) the pvrdma will request
a resource from the backend interface, maintaining a 1-1 mapping
between the guest and host.
On data path:
- Every post_send/receive received from the guest will be converted into
a post_send/receive for the backend. The buffers data will not be touched
or copied resulting in near bare-metal performance for large enough buffers.
- Completions from the backend interface will result in completions for
the pvrdma device.
4.2 PCI BARs
============
PCI Bars:
BAR 0 - MSI-X
MSI-X vectors:
(0) Command - used when execution of a command is completed.
(1) Async - not in use.
(2) Completion - used when a completion event is placed in
device's CQ ring.
BAR 1 - Registers
--------------------------------------------------------
| VERSION | DSR | CTL | REQ | ERR | ICR | IMR | MAC |
--------------------------------------------------------
DSR - Address of driver/device shared memory used
for the command channel, used for passing:
- General info such as driver version
- Address of 'command' and 'response'
- Address of async ring
- Address of device's CQ ring
- Device capabilities
CTL - Device control operations (activate, reset etc)
IMG - Set interrupt mask
REQ - Command execution register
ERR - Operation status
BAR 2 - UAR
---------------------------------------------------------
| QP_NUM | SEND/RECV Flag || CQ_NUM | ARM/POLL Flag |
---------------------------------------------------------
- Offset 0 used for QP operations (send and recv)
- Offset 4 used for CQ operations (arm and poll)
4.3 Major flows
===============
4.3.1 Create CQ
===============
- Guest driver
- Allocates pages for CQ ring
- Creates page directory (pdir) to hold CQ ring's pages
- Initializes CQ ring
- Initializes 'Create CQ' command object (cqe, pdir etc)
- Copies the command to 'command' address
- Writes 0 into REQ register
- Device
- Reads the request object from the 'command' address
- Allocates CQ object and initialize CQ ring based on pdir
- Creates the backend CQ
- Writes operation status to ERR register
- Posts command-interrupt to guest
- Guest driver
- Reads the HW response code from ERR register
4.3.2 Create QP
===============
- Guest driver
- Allocates pages for send and receive rings
- Creates page directory(pdir) to hold the ring's pages
- Initializes 'Create QP' command object (max_send_wr,
send_cq_handle, recv_cq_handle, pdir etc)
- Copies the object to 'command' address
- Write 0 into REQ register
- Device
- Reads the request object from 'command' address
- Allocates the QP object and initialize
- Send and recv rings based on pdir
- Send and recv ring state
- Creates the backend QP
- Writes the operation status to ERR register
- Posts command-interrupt to guest
- Guest driver
- Reads the HW response code from ERR register
4.3.3 Post receive
==================
- Guest driver
- Initializes a wqe and place it on recv ring
- Write to qpn|qp_recv_bit (31) to QP offset in UAR
- Device
- Extracts qpn from UAR
- Walks through the ring and does the following for each wqe
- Prepares the backend CQE context to be used when
receiving completion from backend (wr_id, op_code, emu_cq_num)
- For each sge prepares backend sge
- Calls backend's post_recv
4.3.4 Process backend events
============================
- Done by a dedicated thread used to process backend events;
at initialization is attached to the device and creates
the communication channel.
- Thread main loop:
- Polls for completions
- Extracts QEMU _cq_num, wr_id and op_code from context
- Writes CQE to CQ ring
- Writes CQ number to device CQ
- Sends completion-interrupt to guest
- Deallocates context
- Acks the event to backend
5. Limitations
==============
- The device obviously is limited by the Guest Linux Driver features implementation
of the VMware device API.
- Memory registration mechanism requires mremap for every page in the buffer in order
to map it to a contiguous virtual address range. Since this is not the data path
it should not matter much. If the default max mr size is increased, be aware that
memory registration can take up to 0.5 seconds for 1GB of memory.
- The device requires target page size to be the same as the host page size,
otherwise it will fail to init.
- QEMU cannot map guest RAM from a file descriptor if a pvrdma device is attached,
so it can't work with huge pages. The limitation will be addressed in the future,
however QEMU allocates Guest RAM with MADV_HUGEPAGE so if there are enough huge
pages available, QEMU will use them. QEMU will fail to init if the requirements
are not met.
6. Performance
==============
By design the pvrdma device exits on each post-send/receive, so for small buffers
the performance is affected; however for medium buffers it will became close to
bare metal and from 1MB buffers and up it reaches bare metal performance.
(tested with 2 VMs, the pvrdma devices connected to 2 VFs of the same device)
All the above assumes no memory registration is done on data path.
docs/system/loongarch/virt.rst
+1 -1
View File
@@ -39,7 +39,7 @@ can be accessed by following steps.
.. code-block:: bash
./configure --disable-rdma --disable-pvrdma --prefix=/usr \
./configure --disable-rdma --prefix=/usr \
--target-list="loongarch64-softmmu" \
--disable-libiscsi --disable-libnfs --disable-libpmem \
--disable-glusterfs --enable-libusb --enable-usb-redir \
hmp-commands-info.hx
-13
View File
@@ -182,19 +182,6 @@ SRST
Show PIC state.
ERST
{
.name = "rdma",
.args_type = "",
.params = "",
.help = "show RDMA state",
.cmd_info_hrt = qmp_x_query_rdma,
},
SRST
``info rdma``
Show RDMA state.
ERST
{
.name = "pci",
.args_type = "",
hw/Kconfig
-1
View File
@@ -29,7 +29,6 @@ source pci-bridge/Kconfig
source pci-host/Kconfig
source pcmcia/Kconfig
source pci/Kconfig
source rdma/Kconfig
source remote/Kconfig
source rtc/Kconfig
source scsi/Kconfig
hw/core/machine-qmp-cmds.c
-32
View File
@@ -12,7 +12,6 @@
#include "hw/boards.h"
#include "hw/intc/intc.h"
#include "hw/mem/memory-device.h"
#include "hw/rdma/rdma.h"
#include "qapi/error.h"
#include "qapi/qapi-builtin-visit.h"
#include "qapi/qapi-commands-machine.h"
@@ -291,37 +290,6 @@ MemoryInfo *qmp_query_memory_size_summary(Error **errp)
return mem_info;
}
static int qmp_x_query_rdma_foreach(Object *obj, void *opaque)
{
RdmaProvider *rdma;
RdmaProviderClass *k;
GString *buf = opaque;
if (object_dynamic_cast(obj, INTERFACE_RDMA_PROVIDER)) {
rdma = RDMA_PROVIDER(obj);
k = RDMA_PROVIDER_GET_CLASS(obj);
if (k->format_statistics) {
k->format_statistics(rdma, buf);
} else {
g_string_append_printf(buf,
"RDMA statistics not available for %s.\n",
object_get_typename(obj));
}
}
return 0;
}
HumanReadableText *qmp_x_query_rdma(Error **errp)
{
g_autoptr(GString) buf = g_string_new("");
object_child_foreach_recursive(object_get_root(),
qmp_x_query_rdma_foreach, buf);
return human_readable_text_from_str(buf);
}
HumanReadableText *qmp_x_query_ramblock(Error **errp)
{
g_autoptr(GString) buf = ram_block_format();
hw/meson.build
-1
View File
@@ -28,7 +28,6 @@ subdir('pci')
subdir('pci-bridge')
subdir('pci-host')
subdir('pcmcia')
subdir('rdma')
subdir('rtc')
subdir('scsi')
subdir('sd')
hw/rdma/Kconfig
-3
View File
@@ -1,3 +0,0 @@
config VMW_PVRDMA
default y if PCI_DEVICES
depends on PVRDMA && MSI_NONBROKEN && VMXNET3_PCI
hw/rdma/meson.build
-12
View File
@@ -1,12 +0,0 @@
system_ss.add(when: 'CONFIG_VMW_PVRDMA', if_true: files(
'rdma.c',
'rdma_backend.c',
'rdma_utils.c',
'vmw/pvrdma_qp_ops.c',
))
specific_ss.add(when: 'CONFIG_VMW_PVRDMA', if_true: files(
'rdma_rm.c',
'vmw/pvrdma_cmd.c',
'vmw/pvrdma_dev_ring.c',
'vmw/pvrdma_main.c',
))
hw/rdma/rdma.c
-30
View File
@@ -1,30 +0,0 @@
/*
* RDMA device interface
*
* Copyright (C) 2018 Oracle
* Copyright (C) 2018 Red Hat Inc
*
* Authors:
* Yuval Shaia <yuval.shaia@oracle.com>
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*
*/
#include "qemu/osdep.h"
#include "hw/rdma/rdma.h"
#include "qemu/module.h"
static const TypeInfo rdma_hmp_info = {
.name = INTERFACE_RDMA_PROVIDER,
.parent = TYPE_INTERFACE,
.class_size = sizeof(RdmaProviderClass),
};
static void rdma_register_types(void)
{
type_register_static(&rdma_hmp_info);
}
type_init(rdma_register_types)
hw/rdma/rdma_backend.c
-1401
View File
File diff suppressed because it is too large Load Diff
hw/rdma/rdma_backend.h
-129
View File
@@ -1,129 +0,0 @@
/*
* RDMA device: Definitions of Backend Device functions
*
* Copyright (C) 2018 Oracle
* Copyright (C) 2018 Red Hat Inc
*
* Authors:
* Yuval Shaia <yuval.shaia@oracle.com>
* Marcel Apfelbaum <marcel@redhat.com>
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*
*/
#ifndef RDMA_BACKEND_H
#define RDMA_BACKEND_H
#include "qapi/error.h"
#include "chardev/char-fe.h"
#include "rdma_rm_defs.h"
#include "rdma_backend_defs.h"
/* Vendor Errors */
#define VENDOR_ERR_FAIL_BACKEND 0x201
#define VENDOR_ERR_TOO_MANY_SGES 0x202
#define VENDOR_ERR_NOMEM 0x203
#define VENDOR_ERR_QP0 0x204
#define VENDOR_ERR_INV_NUM_SGE 0x205
#define VENDOR_ERR_MAD_SEND 0x206
#define VENDOR_ERR_INVLKEY 0x207
#define VENDOR_ERR_MR_SMALL 0x208
#define VENDOR_ERR_INV_MAD_BUFF 0x209
#define VENDOR_ERR_INV_GID_IDX 0x210
/* Add definition for QP0 and QP1 as there is no userspace enums for them */
enum ibv_special_qp_type {
IBV_QPT_SMI = 0,
IBV_QPT_GSI = 1,
};
static inline uint32_t rdma_backend_qpn(const RdmaBackendQP *qp)
{
return qp->ibqp ? qp->ibqp->qp_num : 1;
}
static inline uint32_t rdma_backend_mr_lkey(const RdmaBackendMR *mr)
{
return mr->ibmr ? mr->ibmr->lkey : 0;
}
static inline uint32_t rdma_backend_mr_rkey(const RdmaBackendMR *mr)
{
return mr->ibmr ? mr->ibmr->rkey : 0;
}
int rdma_backend_init(RdmaBackendDev *backend_dev, PCIDevice *pdev,
RdmaDeviceResources *rdma_dev_res,
const char *backend_device_name, uint8_t port_num,
struct ibv_device_attr *dev_attr,
CharBackend *mad_chr_be);
void rdma_backend_fini(RdmaBackendDev *backend_dev);
int rdma_backend_add_gid(RdmaBackendDev *backend_dev, const char *ifname,
union ibv_gid *gid);
int rdma_backend_del_gid(RdmaBackendDev *backend_dev, const char *ifname,
union ibv_gid *gid);
int rdma_backend_get_gid_index(RdmaBackendDev *backend_dev,
union ibv_gid *gid);
void rdma_backend_start(RdmaBackendDev *backend_dev);
void rdma_backend_stop(RdmaBackendDev *backend_dev);
void rdma_backend_register_comp_handler(void (*handler)(void *ctx,
struct ibv_wc *wc));
void rdma_backend_unregister_comp_handler(void);
int rdma_backend_query_port(RdmaBackendDev *backend_dev,
struct ibv_port_attr *port_attr);
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd);
void rdma_backend_destroy_pd(RdmaBackendPD *pd);
int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, void *addr,
size_t length, uint64_t guest_start, int access);
void rdma_backend_destroy_mr(RdmaBackendMR *mr);
int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,
int cqe);
void rdma_backend_destroy_cq(RdmaBackendCQ *cq);
void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq);
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
uint32_t max_send_wr, uint32_t max_recv_wr,
uint32_t max_send_sge, uint32_t max_recv_sge);
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey);
int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint8_t sgid_idx,
union ibv_gid *dgid, uint32_t dqpn,
uint32_t rq_psn, uint32_t qkey, bool use_qkey);
int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
uint32_t sq_psn, uint32_t qkey, bool use_qkey);
int rdma_backend_query_qp(RdmaBackendQP *qp, struct ibv_qp_attr *attr,
int attr_mask, struct ibv_qp_init_attr *init_attr);
void rdma_backend_destroy_qp(RdmaBackendQP *qp, RdmaDeviceResources *dev_res);
void rdma_backend_post_send(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp, uint8_t qp_type,
struct ibv_sge *sge, uint32_t num_sge,
uint8_t sgid_idx, union ibv_gid *sgid,
union ibv_gid *dgid, uint32_t dqpn, uint32_t dqkey,
void *ctx);
void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp, uint8_t qp_type,
struct ibv_sge *sge, uint32_t num_sge, void *ctx);
int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
uint32_t max_wr, uint32_t max_sge,
uint32_t srq_limit);
int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr);
int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
int srq_attr_mask);
void rdma_backend_destroy_srq(RdmaBackendSRQ *srq,
RdmaDeviceResources *dev_res);
void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
RdmaBackendSRQ *srq, struct ibv_sge *sge,
uint32_t num_sge, void *ctx);
#endif
hw/rdma/rdma_backend_defs.h
-76
View File
@@ -1,76 +0,0 @@
/*
* RDMA device: Definitions of Backend Device structures
*
* Copyright (C) 2018 Oracle
* Copyright (C) 2018 Red Hat Inc
*
* Authors:
* Yuval Shaia <yuval.shaia@oracle.com>
* Marcel Apfelbaum <marcel@redhat.com>
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*
*/
#ifndef RDMA_BACKEND_DEFS_H
#define RDMA_BACKEND_DEFS_H
#include "qemu/thread.h"
#include "chardev/char-fe.h"
#include <infiniband/verbs.h>
#include "contrib/rdmacm-mux/rdmacm-mux.h"
#include "rdma_utils.h"
typedef struct RdmaDeviceResources RdmaDeviceResources;
typedef struct RdmaBackendThread {
QemuThread thread;
bool run; /* Set by thread manager to let thread know it should exit */
bool is_running; /* Set by the thread to report its status */
} RdmaBackendThread;
typedef struct RdmaCmMux {
CharBackend *chr_be;
int can_receive;
} RdmaCmMux;
typedef struct RdmaBackendDev {
RdmaBackendThread comp_thread;
PCIDevice *dev;
RdmaDeviceResources *rdma_dev_res;
struct ibv_device *ib_dev;
struct ibv_context *context;
struct ibv_comp_channel *channel;
uint8_t port_num;
RdmaProtectedGQueue recv_mads_list;
RdmaCmMux rdmacm_mux;
} RdmaBackendDev;
typedef struct RdmaBackendPD {
struct ibv_pd *ibpd;
} RdmaBackendPD;
typedef struct RdmaBackendMR {
struct ibv_pd *ibpd;
struct ibv_mr *ibmr;
} RdmaBackendMR;
typedef struct RdmaBackendCQ {
RdmaBackendDev *backend_dev;
struct ibv_cq *ibcq;
} RdmaBackendCQ;
typedef struct RdmaBackendQP {
struct ibv_pd *ibpd;
struct ibv_qp *ibqp;
uint8_t sgid_idx;
RdmaProtectedGSList cqe_ctx_list;
} RdmaBackendQP;
typedef struct RdmaBackendSRQ {
struct ibv_srq *ibsrq;
RdmaProtectedGSList cqe_ctx_list;
} RdmaBackendSRQ;
#endif
hw/rdma/rdma_rm.c
-812
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More