mirror of
https://github.com/izzy2lost/xemu.git
synced 2026-07-06 00:20:22 -07:00
hw/rdma: Implementation of generic rdma device layers
This layer is composed of two sub-modules, backend and resource manager. Backend sub-module is responsible for all the interaction with IB layers such as ibverbs and umad (external libraries). Resource manager is a collection of functions and structures to manage RDMA resources such as QPs, CQs and MRs. Reviewed-by: Dotan Barak <dotanb@mellanox.com> Reviewed-by: Zhu Yanjun <yanjun.zhu@oracle.com> Signed-off-by: Yuval Shaia <yuval.shaia@oracle.com> Signed-off-by: Marcel Apfelbaum <marcel@redhat.com>
This commit is contained in:
Yuval Shaia
committed by
Marcel Apfelbaum
Marcel Apfelbaum
parent
b3a9227769
commit
ef6d4ccdc9
@@ -130,6 +130,7 @@ trace-events-subdirs += hw/block/dataplane
|
||||
trace-events-subdirs += hw/char
|
||||
trace-events-subdirs += hw/intc
|
||||
trace-events-subdirs += hw/net
|
||||
trace-events-subdirs += hw/rdma
|
||||
trace-events-subdirs += hw/virtio
|
||||
trace-events-subdirs += hw/audio
|
||||
trace-events-subdirs += hw/misc
|
||||
|
||||
@@ -1572,7 +1572,7 @@ disabled with --disable-FEATURE, default is enabled if available:
|
||||
hax HAX acceleration support
|
||||
hvf Hypervisor.framework acceleration support
|
||||
whpx Windows Hypervisor Platform acceleration support
|
||||
rdma RDMA-based migration support
|
||||
rdma Enable RDMA-based migration and PVRDMA support
|
||||
vde support for vde network
|
||||
netmap support for netmap network
|
||||
linux-aio Linux AIO support
|
||||
@@ -2923,15 +2923,16 @@ if test "$rdma" != "no" ; then
|
||||
#include <rdma/rdma_cma.h>
|
||||
int main(void) { return 0; }
|
||||
EOF
|
||||
rdma_libs="-lrdmacm -libverbs"
|
||||
rdma_libs="-lrdmacm -libverbs -libumad"
|
||||
if compile_prog "" "$rdma_libs" ; then
|
||||
rdma="yes"
|
||||
libs_softmmu="$libs_softmmu $rdma_libs"
|
||||
else
|
||||
if test "$rdma" = "yes" ; then
|
||||
error_exit \
|
||||
" OpenFabrics librdmacm/libibverbs not present." \
|
||||
" OpenFabrics librdmacm/libibverbs/libibumad not present." \
|
||||
" Your options:" \
|
||||
" (1) Fast: Install infiniband packages from your distro." \
|
||||
" (1) Fast: Install infiniband packages (devel) from your distro." \
|
||||
" (2) Cleanest: Install libraries from www.openfabrics.org" \
|
||||
" (3) Also: Install softiwarp if you don't have RDMA hardware"
|
||||
fi
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
ifeq ($(CONFIG_RDMA),y)
|
||||
obj-$(CONFIG_PCI) += rdma_utils.o
|
||||
obj-$(CONFIG_PCI) += rdma_utils.o rdma_backend.o rdma_rm.o
|
||||
endif
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* RDMA device: Definitions of Backend Device functions
|
||||
*
|
||||
* Copyright (C) 2018 Oracle
|
||||
* Copyright (C) 2018 Red Hat Inc
|
||||
*
|
||||
* Authors:
|
||||
* Yuval Shaia <yuval.shaia@oracle.com>
|
||||
* Marcel Apfelbaum <marcel@redhat.com>
|
||||
*
|
||||
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
||||
* See the COPYING file in the top-level directory.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef RDMA_BACKEND_H
|
||||
#define RDMA_BACKEND_H
|
||||
|
||||
#include <qapi/error.h>
|
||||
#include "rdma_rm_defs.h"
|
||||
#include "rdma_backend_defs.h"
|
||||
|
||||
/* Add definition for QP0 and QP1 as there is no userspace enums for them */
|
||||
enum ibv_special_qp_type {
|
||||
IBV_QPT_SMI = 0,
|
||||
IBV_QPT_GSI = 1,
|
||||
};
|
||||
|
||||
static inline union ibv_gid *rdma_backend_gid(RdmaBackendDev *dev)
|
||||
{
|
||||
return &dev->gid;
|
||||
}
|
||||
|
||||
static inline uint32_t rdma_backend_qpn(const RdmaBackendQP *qp)
|
||||
{
|
||||
return qp->ibqp ? qp->ibqp->qp_num : 0;
|
||||
}
|
||||
|
||||
static inline uint32_t rdma_backend_mr_lkey(const RdmaBackendMR *mr)
|
||||
{
|
||||
return mr->ibmr ? mr->ibmr->lkey : 0;
|
||||
}
|
||||
|
||||
static inline uint32_t rdma_backend_mr_rkey(const RdmaBackendMR *mr)
|
||||
{
|
||||
return mr->ibmr ? mr->ibmr->rkey : 0;
|
||||
}
|
||||
|
||||
int rdma_backend_init(RdmaBackendDev *backend_dev,
|
||||
RdmaDeviceResources *rdma_dev_res,
|
||||
const char *backend_device_name, uint8_t port_num,
|
||||
uint8_t backend_gid_idx, struct ibv_device_attr *dev_attr,
|
||||
Error **errp);
|
||||
void rdma_backend_fini(RdmaBackendDev *backend_dev);
|
||||
void rdma_backend_register_comp_handler(void (*handler)(int status,
|
||||
unsigned int vendor_err, void *ctx));
|
||||
void rdma_backend_unregister_comp_handler(void);
|
||||
|
||||
int rdma_backend_query_port(RdmaBackendDev *backend_dev,
|
||||
struct ibv_port_attr *port_attr);
|
||||
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd);
|
||||
void rdma_backend_destroy_pd(RdmaBackendPD *pd);
|
||||
|
||||
int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, uint64_t addr,
|
||||
size_t length, int access);
|
||||
void rdma_backend_destroy_mr(RdmaBackendMR *mr);
|
||||
|
||||
int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,
|
||||
int cqe);
|
||||
void rdma_backend_destroy_cq(RdmaBackendCQ *cq);
|
||||
void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq);
|
||||
|
||||
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
|
||||
RdmaBackendPD *pd, RdmaBackendCQ *scq,
|
||||
RdmaBackendCQ *rcq, uint32_t max_send_wr,
|
||||
uint32_t max_recv_wr, uint32_t max_send_sge,
|
||||
uint32_t max_recv_sge);
|
||||
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
|
||||
uint8_t qp_type, uint32_t qkey);
|
||||
int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
|
||||
uint8_t qp_type, union ibv_gid *dgid,
|
||||
uint32_t dqpn, uint32_t rq_psn, uint32_t qkey,
|
||||
bool use_qkey);
|
||||
int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
|
||||
uint32_t sq_psn, uint32_t qkey, bool use_qkey);
|
||||
void rdma_backend_destroy_qp(RdmaBackendQP *qp);
|
||||
|
||||
void rdma_backend_post_send(RdmaBackendDev *backend_dev,
|
||||
RdmaBackendQP *qp, uint8_t qp_type,
|
||||
struct ibv_sge *sge, uint32_t num_sge,
|
||||
union ibv_gid *dgid, uint32_t dqpn, uint32_t dqkey,
|
||||
void *ctx);
|
||||
void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
|
||||
RdmaDeviceResources *rdma_dev_res,
|
||||
RdmaBackendQP *qp, uint8_t qp_type,
|
||||
struct ibv_sge *sge, uint32_t num_sge, void *ctx);
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* RDMA device: Definitions of Resource Manager functions
|
||||
*
|
||||
* Copyright (C) 2018 Oracle
|
||||
* Copyright (C) 2018 Red Hat Inc
|
||||
*
|
||||
* Authors:
|
||||
* Yuval Shaia <yuval.shaia@oracle.com>
|
||||
* Marcel Apfelbaum <marcel@redhat.com>
|
||||
*
|
||||
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
||||
* See the COPYING file in the top-level directory.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef RDMA_RM_H
|
||||
#define RDMA_RM_H
|
||||
|
||||
#include <qapi/error.h>
|
||||
#include "rdma_backend_defs.h"
|
||||
#include "rdma_rm_defs.h"
|
||||
|
||||
int rdma_rm_init(RdmaDeviceResources *dev_res, struct ibv_device_attr *dev_attr,
|
||||
Error **errp);
|
||||
void rdma_rm_fini(RdmaDeviceResources *dev_res);
|
||||
|
||||
int rdma_rm_alloc_pd(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
|
||||
uint32_t *pd_handle, uint32_t ctx_handle);
|
||||
RdmaRmPD *rdma_rm_get_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle);
|
||||
void rdma_rm_dealloc_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle);
|
||||
|
||||
int rdma_rm_alloc_mr(RdmaDeviceResources *dev_res, uint32_t pd_handle,
|
||||
uint64_t guest_start, size_t guest_length, void *host_virt,
|
||||
int access_flags, uint32_t *mr_handle, uint32_t *lkey,
|
||||
uint32_t *rkey);
|
||||
RdmaRmMR *rdma_rm_get_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle);
|
||||
void rdma_rm_dealloc_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle);
|
||||
|
||||
int rdma_rm_alloc_uc(RdmaDeviceResources *dev_res, uint32_t pfn,
|
||||
uint32_t *uc_handle);
|
||||
RdmaRmUC *rdma_rm_get_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle);
|
||||
void rdma_rm_dealloc_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle);
|
||||
|
||||
int rdma_rm_alloc_cq(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
|
||||
uint32_t cqe, uint32_t *cq_handle, void *opaque);
|
||||
RdmaRmCQ *rdma_rm_get_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle);
|
||||
void rdma_rm_req_notify_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle,
|
||||
bool notify);
|
||||
void rdma_rm_dealloc_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle);
|
||||
|
||||
int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle,
|
||||
uint8_t qp_type, uint32_t max_send_wr,
|
||||
uint32_t max_send_sge, uint32_t send_cq_handle,
|
||||
uint32_t max_recv_wr, uint32_t max_recv_sge,
|
||||
uint32_t recv_cq_handle, void *opaque, uint32_t *qpn);
|
||||
RdmaRmQP *rdma_rm_get_qp(RdmaDeviceResources *dev_res, uint32_t qpn);
|
||||
int rdma_rm_modify_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
|
||||
uint32_t qp_handle, uint32_t attr_mask,
|
||||
union ibv_gid *dgid, uint32_t dqpn,
|
||||
enum ibv_qp_state qp_state, uint32_t qkey,
|
||||
uint32_t rq_psn, uint32_t sq_psn);
|
||||
void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle);
|
||||
|
||||
int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id,
|
||||
void *ctx);
|
||||
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
|
||||
void rdma_rm_dealloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,5 @@
|
||||
# See docs/tracing.txt for syntax documentation.
|
||||
|
||||
#hw/rdma/rdma_backend.c
|
||||
create_ah_cache_hit(uint64_t subnet, uint64_t net_id) "subnet = 0x%"PRIx64" net_id = 0x%"PRIx64
|
||||
create_ah_cache_miss(uint64_t subnet, uint64_t net_id) "subnet = 0x%"PRIx64" net_id = 0x%"PRIx64
|
||||
Reference in New Issue
Block a user