Merge tag 'for_upstream' of git://git.kernel.org/pub/scm/virt/kvm/mst/qemu into staging

virtio,pc,pci: fixes,cleanups,features

most of CXL support
fixes, cleanups all over the place

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

# -----BEGIN PGP SIGNATURE-----
#
# iQFDBAABCAAtFiEEXQn9CHHI+FuUyooNKB8NuNKNVGkFAmKCuLIPHG1zdEByZWRo
# YXQuY29tAAoJECgfDbjSjVRpdDUH/12SmWaAo+0+SdIHgWFFxsmg3t/EdcO38fgi
# MV+GpYdbp6TlU3jdQhrMZYmFdkVVydBdxk93ujCLbFS0ixTsKj31j0IbZMfdcGgv
# SLqnV+E3JdHqnGP39q9a9rdwYWyqhkgHoldxilIFW76ngOSapaZVvnwnOMAMkf77
# 1LieL4/Xq7N9Ho86Zrs3IczQcf0czdJRDaFaSIu8GaHl8ELyuPhlSm6CSqqrEEWR
# PA/COQsLDbLOMxbfCi5v88r5aaxmGNZcGbXQbiH9qVHw65nlHyLH9UkNTdJn1du1
# f2GYwwa7eekfw/LCvvVwxO1znJrj02sfFai7aAtQYbXPvjvQiqA=
# =xdSk
# -----END PGP SIGNATURE-----
# gpg: Signature made Mon 16 May 2022 01:48:50 PM PDT
# gpg:                using RSA key 5D09FD0871C8F85B94CA8A0D281F0DB8D28D5469
# gpg:                issuer "mst@redhat.com"
# gpg: Good signature from "Michael S. Tsirkin <mst@kernel.org>" [undefined]
# gpg:                 aka "Michael S. Tsirkin <mst@redhat.com>" [undefined]
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 0270 606B 6F3C DF3D 0B17  0970 C350 3912 AFBE 8E67
#      Subkey fingerprint: 5D09 FD08 71C8 F85B 94CA  8A0D 281F 0DB8 D28D 5469

* tag 'for_upstream' of git://git.kernel.org/pub/scm/virt/kvm/mst/qemu: (86 commits)
  vhost-user-scsi: avoid unlink(NULL) with fd passing
  virtio-net: don't handle mq request in userspace handler for vhost-vdpa
  vhost-vdpa: change name and polarity for vhost_vdpa_one_time_request()
  vhost-vdpa: backend feature should set only once
  vhost-net: fix improper cleanup in vhost_net_start
  vhost-vdpa: fix improper cleanup in net_init_vhost_vdpa
  virtio-net: align ctrl_vq index for non-mq guest for vhost_vdpa
  virtio-net: setup vhost_dev and notifiers for cvq only when feature is negotiated
  hw/i386/amd_iommu: Fix IOMMU event log encoding errors
  hw/i386: Make pic a property of common x86 base machine type
  hw/i386: Make pit a property of common x86 base machine type
  include/hw/pci/pcie_host: Correct PCIE_MMCFG_SIZE_MAX
  include/hw/pci/pcie_host: Correct PCIE_MMCFG_BUS_MASK
  docs/vhost-user: Clarifications for VHOST_USER_ADD/REM_MEM_REG
  vhost-user: more master/slave things
  virtio: add vhost support for virtio devices
  virtio: drop name parameter for virtio_init()
  virtio/vhost-user: dynamically assign VhostUserHostNotifiers
  hw/virtio/vhost-user: don't suppress F_CONFIG when supported
  include/hw: start documenting the vhost API
  ...

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson
2022-05-16 16:31:01 -07:00
131 changed files with 5352 additions and 577 deletions
+7
View File
@@ -2545,6 +2545,13 @@ F: qapi/block*.json
F: qapi/transaction.json
T: git https://repo.or.cz/qemu/armbru.git block-next
Compute Express Link
M: Ben Widawsky <ben.widawsky@intel.com>
M: Jonathan Cameron <jonathan.cameron@huawei.com>
S: Supported
F: hw/cxl/
F: include/hw/cxl/
Dirty Bitmaps
M: Eric Blake <eblake@redhat.com>
M: Vladimir Sementsov-Ogievskiy <v.sementsov-og@mail.ru>
+4 -1
View File
@@ -433,13 +433,16 @@ out:
if (vdev_scsi) {
g_main_loop_unref(vdev_scsi->loop);
g_free(vdev_scsi);
unlink(opt_socket_path);
}
if (csock >= 0) {
close(csock);
}
if (lsock >= 0) {
close(lsock);
if (opt_socket_path) {
unlink(opt_socket_path);
}
}
g_free(opt_socket_path);
g_free(iscsi_uri);
+1
View File
@@ -18,3 +18,4 @@ Details about QEMU's various subsystems including how to add features to them.
tracing
vfio-migration
writing-monitor-commands
virtio-backends
+214
View File
@@ -0,0 +1,214 @@
..
Copyright (c) 2022, Linaro Limited
Written by Alex Bennée
Writing VirtIO backends for QEMU
================================
This document attempts to outline the information a developer needs to
know to write device emulations in QEMU. It is specifically focused on
implementing VirtIO devices. For VirtIO the frontend is the driver
running on the guest. The backend is the everything that QEMU needs to
do to handle the emulation of the VirtIO device. This can be done
entirely in QEMU, divided between QEMU and the kernel (vhost) or
handled by a separate process which is configured by QEMU
(vhost-user).
VirtIO Transports
-----------------
VirtIO supports a number of different transports. While the details of
the configuration and operation of the device will generally be the
same QEMU represents them as different devices depending on the
transport they use. For example -device virtio-foo represents the foo
device using mmio and -device virtio-foo-pci is the same class of
device using the PCI transport.
Using the QEMU Object Model (QOM)
---------------------------------
Generally all devices in QEMU are super classes of ``TYPE_DEVICE``
however VirtIO devices should be based on ``TYPE_VIRTIO_DEVICE`` which
itself is derived from the base class. For example:
.. code:: c
static const TypeInfo virtio_blk_info = {
.name = TYPE_VIRTIO_BLK,
.parent = TYPE_VIRTIO_DEVICE,
.instance_size = sizeof(VirtIOBlock),
.instance_init = virtio_blk_instance_init,
.class_init = virtio_blk_class_init,
};
The author may decide to have a more expansive class hierarchy to
support multiple device types. For example the Virtio GPU device:
.. code:: c
static const TypeInfo virtio_gpu_base_info = {
.name = TYPE_VIRTIO_GPU_BASE,
.parent = TYPE_VIRTIO_DEVICE,
.instance_size = sizeof(VirtIOGPUBase),
.class_size = sizeof(VirtIOGPUBaseClass),
.class_init = virtio_gpu_base_class_init,
.abstract = true
};
static const TypeInfo vhost_user_gpu_info = {
.name = TYPE_VHOST_USER_GPU,
.parent = TYPE_VIRTIO_GPU_BASE,
.instance_size = sizeof(VhostUserGPU),
.instance_init = vhost_user_gpu_instance_init,
.instance_finalize = vhost_user_gpu_instance_finalize,
.class_init = vhost_user_gpu_class_init,
};
static const TypeInfo virtio_gpu_info = {
.name = TYPE_VIRTIO_GPU,
.parent = TYPE_VIRTIO_GPU_BASE,
.instance_size = sizeof(VirtIOGPU),
.class_size = sizeof(VirtIOGPUClass),
.class_init = virtio_gpu_class_init,
};
defines a base class for the VirtIO GPU and then specialises two
versions, one for the internal implementation and the other for the
vhost-user version.
VirtIOPCIProxy
^^^^^^^^^^^^^^
[AJB: the following is supposition and welcomes more informed
opinions]
Probably due to legacy from the pre-QOM days PCI VirtIO devices don't
follow the normal hierarchy. Instead the a standalone object is based
on the VirtIOPCIProxy class and the specific VirtIO instance is
manually instantiated:
.. code:: c
/*
* virtio-blk-pci: This extends VirtioPCIProxy.
*/
#define TYPE_VIRTIO_BLK_PCI "virtio-blk-pci-base"
DECLARE_INSTANCE_CHECKER(VirtIOBlkPCI, VIRTIO_BLK_PCI,
TYPE_VIRTIO_BLK_PCI)
struct VirtIOBlkPCI {
VirtIOPCIProxy parent_obj;
VirtIOBlock vdev;
};
static Property virtio_blk_pci_properties[] = {
DEFINE_PROP_UINT32("class", VirtIOPCIProxy, class_code, 0),
DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
DEV_NVECTORS_UNSPECIFIED),
DEFINE_PROP_END_OF_LIST(),
};
static void virtio_blk_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
{
VirtIOBlkPCI *dev = VIRTIO_BLK_PCI(vpci_dev);
DeviceState *vdev = DEVICE(&dev->vdev);
...
qdev_realize(vdev, BUS(&vpci_dev->bus), errp);
}
static void virtio_blk_pci_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
device_class_set_props(dc, virtio_blk_pci_properties);
k->realize = virtio_blk_pci_realize;
pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_BLOCK;
pcidev_k->revision = VIRTIO_PCI_ABI_VERSION;
pcidev_k->class_id = PCI_CLASS_STORAGE_SCSI;
}
static void virtio_blk_pci_instance_init(Object *obj)
{
VirtIOBlkPCI *dev = VIRTIO_BLK_PCI(obj);
virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
TYPE_VIRTIO_BLK);
object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
"bootindex");
}
static const VirtioPCIDeviceTypeInfo virtio_blk_pci_info = {
.base_name = TYPE_VIRTIO_BLK_PCI,
.generic_name = "virtio-blk-pci",
.transitional_name = "virtio-blk-pci-transitional",
.non_transitional_name = "virtio-blk-pci-non-transitional",
.instance_size = sizeof(VirtIOBlkPCI),
.instance_init = virtio_blk_pci_instance_init,
.class_init = virtio_blk_pci_class_init,
};
Here you can see the instance_init has to manually instantiate the
underlying ``TYPE_VIRTIO_BLOCK`` object and link an alias for one of
it's properties to the PCI device.
Back End Implementations
------------------------
There are a number of places where the implementation of the backend
can be done:
* in QEMU itself
* in the host kernel (a.k.a vhost)
* in a separate process (a.k.a. vhost-user)
vhost_ops vs TYPE_VHOST_USER_BACKEND
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are two choices to how to implement vhost code. Most of the code
which has to work with either vhost or vhost-user uses
``vhost_dev_init()`` to instantiate the appropriate backend. This
means including a ``struct vhost_dev`` in the main object structure.
For vhost-user devices you also need to add code to track the
initialisation of the ``chardev`` device used for the control socket
between QEMU and the external vhost-user process.
If you only need to implement a vhost-user backed the other option is
a use a QOM-ified version of vhost-user.
.. code:: c
static void
vhost_user_gpu_instance_init(Object *obj)
{
VhostUserGPU *g = VHOST_USER_GPU(obj);
g->vhost = VHOST_USER_BACKEND(object_new(TYPE_VHOST_USER_BACKEND));
object_property_add_alias(obj, "chardev",
OBJECT(g->vhost), "chardev");
}
static const TypeInfo vhost_user_gpu_info = {
.name = TYPE_VHOST_USER_GPU,
.parent = TYPE_VIRTIO_GPU_BASE,
.instance_size = sizeof(VhostUserGPU),
.instance_init = vhost_user_gpu_instance_init,
.instance_finalize = vhost_user_gpu_instance_finalize,
.class_init = vhost_user_gpu_class_init,
};
Using it this way entails adding a ``struct VhostUserBackend`` to your
core object structure and manually instantiating the backend. This
sub-structure tracks both the ``vhost_dev`` and ``CharDev`` types
needed for the connection. Instead of calling ``vhost_dev_init`` you
would call ``vhost_user_backend_dev_init`` which does what is needed
on your behalf.
+5 -5
View File
@@ -13,10 +13,10 @@ Introduction
============
The vhost-user-gpu protocol is aiming at sharing the rendering result
of a virtio-gpu, done from a vhost-user slave process to a vhost-user
master process (such as QEMU). It bears a resemblance to a display
of a virtio-gpu, done from a vhost-user back-end process to a vhost-user
front-end process (such as QEMU). It bears a resemblance to a display
server protocol, if you consider QEMU as the display server and the
slave as the client, but in a very limited way. Typically, it will
back-end as the client, but in a very limited way. Typically, it will
work by setting a scanout/display configuration, before sending flush
events for the display updates. It will also update the cursor shape
and position.
@@ -26,8 +26,8 @@ socket ancillary data to share opened file descriptors (DMABUF fds or
shared memory). The socket is usually obtained via
``VHOST_USER_GPU_SET_SOCKET``.
Requests are sent by the *slave*, and the optional replies by the
*master*.
Requests are sent by the *back-end*, and the optional replies by the
*front-end*.
Wire format
===========
File diff suppressed because it is too large Load Diff
+1
View File
@@ -84,6 +84,7 @@ Emulated Devices
devices/can.rst
devices/ccid.rst
devices/cxl.rst
devices/ivshmem.rst
devices/net.rst
devices/nvme.rst
+302
View File
@@ -0,0 +1,302 @@
Compute Express Link (CXL)
==========================
From the view of a single host, CXL is an interconnect standard that
targets accelerators and memory devices attached to a CXL host.
This description will focus on those aspects visible either to
software running on a QEMU emulated host or to the internals of
functional emulation. As such, it will skip over many of the
electrical and protocol elements that would be more of interest
for real hardware and will dominate more general introductions to CXL.
It will also completely ignore the fabric management aspects of CXL
by considering only a single host and a static configuration.
CXL shares many concepts and much of the infrastructure of PCI Express,
with CXL Host Bridges, which have CXL Root Ports which may be directly
attached to CXL or PCI End Points. Alternatively there may be CXL Switches
with CXL and PCI Endpoints attached below them. In many cases additional
control and capabilities are exposed via PCI Express interfaces.
This sharing of interfaces and hence emulation code is is reflected
in how the devices are emulated in QEMU. In most cases the various
CXL elements are built upon an equivalent PCIe devices.
CXL devices support the following interfaces:
* Most conventional PCIe interfaces
- Configuration space access
- BAR mapped memory accesses used for registers and mailboxes.
- MSI/MSI-X
- AER
- DOE mailboxes
- IDE
- Many other PCI express defined interfaces..
* Memory operations
- Equivalent of accessing DRAM / NVDIMMs. Any access / feature
supported by the host for normal memory should also work for
CXL attached memory devices.
* Cache operations. The are mostly irrelevant to QEMU emulation as
QEMU is not emulating a coherency protocol. Any emulation related
to these will be device specific and is out of the scope of this
document.
CXL 2.0 Device Types
--------------------
CXL 2.0 End Points are often categorized into three types.
**Type 1:** These support coherent caching of host memory. Example might
be a crypto accelerators. May also have device private memory accessible
via means such as PCI memory reads and writes to BARs.
**Type 2:** These support coherent caching of host memory and host
managed device memory (HDM) for which the coherency protocol is managed
by the host. This is a complex topic, so for more information on CXL
coherency see the CXL 2.0 specification.
**Type 3 Memory devices:** These devices act as a means of attaching
additional memory (HDM) to a CXL host including both volatile and
persistent memory. The CXL topology may support interleaving across a
number of Type 3 memory devices using HDM Decoders in the host, host
bridge, switch upstream port and endpoints.
Scope of CXL emulation in QEMU
------------------------------
The focus of CXL emulation is CXL revision 2.0 and later. Earlier CXL
revisions defined a smaller set of features, leaving much of the control
interface as implementation defined or device specific, making generic
emulation challenging with host specific firmware being responsible
for setup and the Endpoints being presented to operating systems
as Root Complex Integrated End Points. CXL rev 2.0 looks a lot
more like PCI Express, with fully specified discoverability
of the CXL topology.
CXL System components
----------------------
A CXL system is made up a Host with a number of 'standard components'
the control and capabilities of which are discoverable by system software
using means described in the CXL 2.0 specification.
CXL Fixed Memory Windows (CFMW)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A CFMW consists of a particular range of Host Physical Address space
which is routed to particular CXL Host Bridges. At time of generic
software initialization it will have a particularly interleaving
configuration and associated Quality of Serice Throtling Group (QTG).
This information is available to system software, when making
decisions about how to configure interleave across available CXL
memory devices. It is provide as CFMW Structures (CFMWS) in
the CXL Early Discovery Table, an ACPI table.
Note: QTG 0 is the only one currently supported in QEMU.
CXL Host Bridge (CXL HB)
~~~~~~~~~~~~~~~~~~~~~~~~
A CXL host bridge is similar to the PCIe equivalent, but with a
specification defined register interface called CXL Host Bridge
Component Registers (CHBCR). The location of this CHBCR MMIO
space is described to system software via a CXL Host Bridge
Structure (CHBS) in the CEDT ACPI table. The actual interfaces
are identical to those used for other parts of the CXL heirarchy
as CXL Component Registers in PCI BARs.
Interfaces provided include:
* Configuration of HDM Decoders to route CXL Memory accesses with
a particularly Host Physical Address range to the target port
below which the CXL device servicing that address lies. This
may be a mapping to a single Root Port (RP) or across a set of
target RPs.
CXL Root Ports (CXL RP)
~~~~~~~~~~~~~~~~~~~~~~~
A CXL Root Port servers te same purpose as a PCIe Root Port.
There are a number of CXL specific Designated Vendor Specific
Extended Capabilities (DVSEC) in PCIe Configuration Space
and associated component register access via PCI bars.
CXL Switch
~~~~~~~~~~
Not yet implemented in QEMU.
Here we consider a simple CXL switch with only a single
virtual hierarchy. Whilst more complex devices exist, their
visibility to a particular host is generally the same as for
a simple switch design. Hosts often have no awareness
of complex rerouting and device pooling, they simply see
devices being hot added or hot removed.
A CXL switch has a similar architecture to those in PCIe,
with a single upstream port, internal PCI bus and multiple
downstream ports.
Both the CXL upstream and downstream ports have CXL specific
DVSECs in configuration space, and component registers in PCI
BARs. The Upstream Port has the configuration interfaces for
the HDM decoders which route incoming memory accesses to the
appropriate downstream port.
CXL Memory Devices - Type 3
~~~~~~~~~~~~~~~~~~~~~~~~~~~
CXL type 3 devices use a PCI class code and are intended to be supported
by a generic operating system driver. They have HDM decoders
though in these EP devices, the decoder is reponsible not for
routing but for translation of the incoming host physical address (HPA)
into a Device Physical Address (DPA).
CXL Memory Interleave
---------------------
To understand the interaction of different CXL hardware components which
are emulated in QEMU, let us consider a memory read in a fully configured
CXL topology. Note that system software is responsible for configuration
of all components with the exception of the CFMWs. System software is
responsible for allocating appropriate ranges from within the CFMWs
and exposing those via normal memory configurations as would be done
for system RAM.
Example system Topology. x marks the match in each decoder level::
|<------------------SYSTEM PHYSICAL ADDRESS MAP (1)----------------->|
| __________ __________________________________ __________ |
| | | | | | | |
| | CFMW 0 | | CXL Fixed Memory Window 1 | | CFMW 1 | |
| | HB0 only | | Configured to interleave memory | | HB1 only | |
| | | | memory accesses across HB0/HB1 | | | |
| |__________| |_____x____________________________| |__________| |
| | | |
| | | |
| | | |
| Interleave Decoder | |
| Matches this HB | |
\_____________| |_____________/
__________|__________ _____|_______________
| | | |
(2) | CXL HB 0 | | CXL HB 1 |
| HB IntLv Decoders | | HB IntLv Decoders |
| PCI/CXL Root Bus 0c | | PCI/CXL Root Bus 0d |
| | | |
|___x_________________| |_____________________|
| | | |
| | | |
A HB 0 HDM Decoder | | |
matches this Port | | |
| | | |
___________|___ __________|__ __|_________ ___|_________
(3)| Root Port 0 | | Root Port 1 | | Root Port 2| | Root Port 3 |
| Appears in | | Appears in | | Appears in | | Appear in |
| PCI topology | | PCI Topology| | PCI Topo | | PCI Topo |
| As 0c:00.0 | | as 0c:01.0 | | as de:00.0 | | as de:01.0 |
|_______________| |_____________| |____________| |_____________|
| | | |
| | | |
_____|_________ ______|______ ______|_____ ______|_______
(4)| x | | | | | | |
| CXL Type3 0 | | CXL Type3 1 | | CXL type3 2| | CLX Type 3 3 |
| | | | | | | |
| PMEM0(Vol LSA)| | PMEM1 (...) | | PMEM2 (...)| | PMEM3 (...) |
| Decoder to go | | | | | | |
| from host PA | | PCI 0e:00.0 | | PCI df:00.0| | PCI e0:00.0 |
| to device PA | | | | | | |
| PCI as 0d:00.0| | | | | | |
|_______________| |_____________| |____________| |______________|
Notes:
(1) **3 CXL Fixed Memory Windows (CFMW)** corresponding to different
ranges of the system physical address map. Each CFMW has
particular interleave setup across the CXL Host Bridges (HB)
CFMW0 provides uninterleaved access to HB0, CFW2 provides
uninterleaved acess to HB1. CFW1 provides interleaved memory access
across HB0 and HB1.
(2) **Two CXL Host Bridges**. Each of these has 2 CXL Root Ports and
programmable HDM decoders to route memory accesses either to
a single port or interleave them across multiple ports.
A complex configuration here, might be to use the following HDM
decoders in HB0. HDM0 routes CFMW0 requests to RP0 and hence
part of CXL Type3 0. HDM1 routes CFMW0 requests from a
different region of the CFMW0 PA range to RP2 and hence part
of CXL Type 3 1. HDM2 routes yet another PA range from within
CFMW0 to be interleaved across RP0 and RP1, providing 2 way
interleave of part of the memory provided by CXL Type3 0 and
CXL Type 3 1. HDM3 routes those interleaved accesses from
CFMW1 that target HB0 to RP 0 and another part of the memory of
CXL Type 3 0 (as part of a 2 way interleave at the system level
across for example CXL Type3 0 and CXL Type3 2.
HDM4 is used to enable system wide 4 way interleave across all
the present CXL type3 devices, by interleaving those (interleaved)
requests that HB0 receives from from CFMW1 across RP 0 and
RP 1 and hence to yet more regions of the memory of the
attached Type3 devices. Note this is a representative subset
of the full range of possible HDM decoder configurations in this
topology.
(3) **Four CXL Root Ports.** In this case the CXL Type 3 devices are
directly attached to these ports.
(4) **Four CXL Type3 memory expansion devices.** These will each have
HDM decoders, but in this case rather than performing interleave
they will take the Host Physical Addresses of accesses and map
them to their own local Device Physical Address Space (DPA).
Example command lines
---------------------
A very simple setup with just one directly attached CXL Type 3 device::
qemu-system-aarch64 -M virt,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max \
...
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=256M \
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=256M \
-device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
-device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0 \
-cxl-fixed-memory-window targets.0=cxl.1,size=4G
A setup suitable for 4 way interleave. Only one fixed window provided, to enable 2 way
interleave across 2 CXL host bridges. Each host bridge has 2 CXL Root Ports, with
the CXL Type3 device directly attached (no switches).::
qemu-system-aarch64 -M virt,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max \
...
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=256M \
-object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=256M \
-object memory-backend-file,id=cxl-mem3,share=on,mem-path=/tmp/cxltest3.raw,size=256M \
-object memory-backend-file,id=cxl-mem4,share=on,mem-path=/tmp/cxltest4.raw,size=256M \
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=256M \
-object memory-backend-file,id=cxl-lsa2,share=on,mem-path=/tmp/lsa2.raw,size=256M \
-object memory-backend-file,id=cxl-lsa3,share=on,mem-path=/tmp/lsa3.raw,size=256M \
-object memory-backend-file,id=cxl-lsa4,share=on,mem-path=/tmp/lsa4.raw,size=256M \
-device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
-device pxb-cxl,bus_nr=222,bus=pcie.0,id=cxl.2 \
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
-device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0 \
-device cxl-rp,port=1,bus=cxl.1,id=root_port14,chassis=0,slot=3 \
-device cxl-type3,bus=root_port14,memdev=cxl-mem2,lsa=cxl-lsa2,id=cxl-pmem1 \
-device cxl-rp,port=0,bus=cxl.2,id=root_port15,chassis=0,slot=5 \
-device cxl-type3,bus=root_port15,memdev=cxl-mem3,lsa=cxl-lsa3,id=cxl-pmem2 \
-device cxl-rp,port=1,bus=cxl.2,id=root_port16,chassis=0,slot=6 \
-device cxl-type3,bus=root_port16,memdev=cxl-mem4,lsa=cxl-lsa4,id=cxl-pmem3 \
-cxl-fixed-memory-window targets.0=cxl.1,targets.1=cxl.2,size=4G,interleave-granularity=8k
Kernel Configuration Options
----------------------------
In Linux 5.18 the followings options are necessary to make use of
OS management of CXL memory devices as described here.
* CONFIG_CXL_BUS
* CONFIG_CXL_PCI
* CONFIG_CXL_ACPI
* CONFIG_CXL_PMEM
* CONFIG_CXL_MEM
* CONFIG_CXL_PORT
* CONFIG_CXL_REGION
References
----------
- Consortium website for specifications etc:
http://www.computeexpresslink.org
- Compute Express link Revision 2 specification, October 2020
- CEDT CFMWS & QTG _DSM ECN May 2021
+1 -1
View File
@@ -216,7 +216,7 @@ static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
}
v->config_size = sizeof(struct virtio_9p_config) + strlen(s->fsconf.tag);
virtio_init(vdev, "virtio-9p", VIRTIO_ID_9P, v->config_size);
virtio_init(vdev, VIRTIO_ID_9P, v->config_size);
v->vq = virtio_add_queue(vdev, MAX_REQ, handle_9p_output);
}
+1
View File
@@ -6,6 +6,7 @@ source audio/Kconfig
source block/Kconfig
source char/Kconfig
source core/Kconfig
source cxl/Kconfig
source display/Kconfig
source dma/Kconfig
source gpio/Kconfig
+5
View File
@@ -5,6 +5,7 @@ config ACPI_X86
bool
select ACPI
select ACPI_NVDIMM
select ACPI_CXL
select ACPI_CPU_HOTPLUG
select ACPI_MEMORY_HOTPLUG
select ACPI_HMAT
@@ -66,3 +67,7 @@ config ACPI_ERST
bool
default y
depends on ACPI && PCI
config ACPI_CXL
bool
depends on ACPI
+12
View File
@@ -0,0 +1,12 @@
/*
* Stubs for ACPI platforms that don't support CXl
*/
#include "qemu/osdep.h"
#include "hw/acpi/aml-build.h"
#include "hw/acpi/cxl.h"
void build_cxl_osc_method(Aml *dev)
{
g_assert_not_reached();
}
+257
View File
@@ -0,0 +1,257 @@
/*
* CXL ACPI Implementation
*
* Copyright(C) 2020 Intel Corporation.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>
*/
#include "qemu/osdep.h"
#include "hw/sysbus.h"
#include "hw/pci/pci_bridge.h"
#include "hw/pci/pci_host.h"
#include "hw/cxl/cxl.h"
#include "hw/mem/memory-device.h"
#include "hw/acpi/acpi.h"
#include "hw/acpi/aml-build.h"
#include "hw/acpi/bios-linker-loader.h"
#include "hw/acpi/cxl.h"
#include "qapi/error.h"
#include "qemu/uuid.h"
static void cedt_build_chbs(GArray *table_data, PXBDev *cxl)
{
SysBusDevice *sbd = SYS_BUS_DEVICE(cxl->cxl.cxl_host_bridge);
struct MemoryRegion *mr = sbd->mmio[0].memory;
/* Type */
build_append_int_noprefix(table_data, 0, 1);
/* Reserved */
build_append_int_noprefix(table_data, 0, 1);
/* Record Length */
build_append_int_noprefix(table_data, 32, 2);
/* UID - currently equal to bus number */
build_append_int_noprefix(table_data, cxl->bus_nr, 4);
/* Version */
build_append_int_noprefix(table_data, 1, 4);
/* Reserved */
build_append_int_noprefix(table_data, 0, 4);
/* Base - subregion within a container that is in PA space */
build_append_int_noprefix(table_data, mr->container->addr + mr->addr, 8);
/* Length */
build_append_int_noprefix(table_data, memory_region_size(mr), 8);
}
/*
* CFMWS entries in CXL 2.0 ECN: CEDT CFMWS & QTG _DSM.
* Interleave ways encoding in CXL 2.0 ECN: 3, 6, 12 and 16-way memory
* interleaving.
*/
static void cedt_build_cfmws(GArray *table_data, MachineState *ms)
{
CXLState *cxls = ms->cxl_devices_state;
GList *it;
for (it = cxls->fixed_windows; it; it = it->next) {
CXLFixedWindow *fw = it->data;
int i;
/* Type */
build_append_int_noprefix(table_data, 1, 1);
/* Reserved */
build_append_int_noprefix(table_data, 0, 1);
/* Record Length */
build_append_int_noprefix(table_data, 36 + 4 * fw->num_targets, 2);
/* Reserved */
build_append_int_noprefix(table_data, 0, 4);
/* Base HPA */
build_append_int_noprefix(table_data, fw->mr.addr, 8);
/* Window Size */
build_append_int_noprefix(table_data, fw->size, 8);
/* Host Bridge Interleave Ways */
build_append_int_noprefix(table_data, fw->enc_int_ways, 1);
/* Host Bridge Interleave Arithmetic */
build_append_int_noprefix(table_data, 0, 1);
/* Reserved */
build_append_int_noprefix(table_data, 0, 2);
/* Host Bridge Interleave Granularity */
build_append_int_noprefix(table_data, fw->enc_int_gran, 4);
/* Window Restrictions */
build_append_int_noprefix(table_data, 0x0f, 2); /* No restrictions */
/* QTG ID */
build_append_int_noprefix(table_data, 0, 2);
/* Host Bridge List (list of UIDs - currently bus_nr) */
for (i = 0; i < fw->num_targets; i++) {
g_assert(fw->target_hbs[i]);
build_append_int_noprefix(table_data, fw->target_hbs[i]->bus_nr, 4);
}
}
}
static int cxl_foreach_pxb_hb(Object *obj, void *opaque)
{
Aml *cedt = opaque;
if (object_dynamic_cast(obj, TYPE_PXB_CXL_DEVICE)) {
cedt_build_chbs(cedt->buf, PXB_CXL_DEV(obj));
}
return 0;
}
void cxl_build_cedt(MachineState *ms, GArray *table_offsets, GArray *table_data,
BIOSLinker *linker, const char *oem_id,
const char *oem_table_id)
{
Aml *cedt;
AcpiTable table = { .sig = "CEDT", .rev = 1, .oem_id = oem_id,
.oem_table_id = oem_table_id };
acpi_add_table(table_offsets, table_data);
acpi_table_begin(&table, table_data);
cedt = init_aml_allocator();
/* reserve space for CEDT header */
object_child_foreach_recursive(object_get_root(), cxl_foreach_pxb_hb, cedt);
cedt_build_cfmws(cedt->buf, ms);
/* copy AML table into ACPI tables blob and patch header there */
g_array_append_vals(table_data, cedt->buf->data, cedt->buf->len);
free_aml_allocator();
acpi_table_end(linker, &table);
}
static Aml *__build_cxl_osc_method(void)
{
Aml *method, *if_uuid, *else_uuid, *if_arg1_not_1, *if_cxl, *if_caps_masked;
Aml *a_ctrl = aml_local(0);
Aml *a_cdw1 = aml_name("CDW1");
method = aml_method("_OSC", 4, AML_NOTSERIALIZED);
/* CDW1 is used for the return value so is present whether or not a match occurs */
aml_append(method, aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1"));
/*
* Generate shared section between:
* CXL 2.0 - 9.14.2.1.4 and
* PCI Firmware Specification 3.0
* 4.5.1. _OSC Interface for PCI Host Bridge Devices
* The _OSC interface for a PCI/PCI-X/PCI Express hierarchy is
* identified by the Universal Unique IDentifier (UUID)
* 33DB4D5B-1FF7-401C-9657-7441C03DD766
* The _OSC interface for a CXL Host bridge is
* identified by the UUID 68F2D50B-C469-4D8A-BD3D-941A103FD3FC
* A CXL Host bridge is compatible with a PCI host bridge so
* for the shared section match both.
*/
if_uuid = aml_if(
aml_lor(aml_equal(aml_arg(0),
aml_touuid("33DB4D5B-1FF7-401C-9657-7441C03DD766")),
aml_equal(aml_arg(0),
aml_touuid("68F2D50B-C469-4D8A-BD3D-941A103FD3FC"))));
aml_append(if_uuid, aml_create_dword_field(aml_arg(3), aml_int(4), "CDW2"));
aml_append(if_uuid, aml_create_dword_field(aml_arg(3), aml_int(8), "CDW3"));
aml_append(if_uuid, aml_store(aml_name("CDW3"), a_ctrl));
/*
*
* Allows OS control for all 5 features:
* PCIeHotplug SHPCHotplug PME AER PCIeCapability
*/
aml_append(if_uuid, aml_and(a_ctrl, aml_int(0x1F), a_ctrl));
/*
* Check _OSC revision.
* PCI Firmware specification 3.3 and CXL 2.0 both use revision 1
* Unknown Revision is CDW1 - BIT (3)
*/
if_arg1_not_1 = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(0x1))));
aml_append(if_arg1_not_1, aml_or(a_cdw1, aml_int(0x08), a_cdw1));
aml_append(if_uuid, if_arg1_not_1);
if_caps_masked = aml_if(aml_lnot(aml_equal(aml_name("CDW3"), a_ctrl)));
/* Capability bits were masked */
aml_append(if_caps_masked, aml_or(a_cdw1, aml_int(0x10), a_cdw1));
aml_append(if_uuid, if_caps_masked);
aml_append(if_uuid, aml_store(aml_name("CDW2"), aml_name("SUPP")));
aml_append(if_uuid, aml_store(aml_name("CDW3"), aml_name("CTRL")));
/* Update DWORD3 (the return value) */
aml_append(if_uuid, aml_store(a_ctrl, aml_name("CDW3")));
/* CXL only section as per CXL 2.0 - 9.14.2.1.4 */
if_cxl = aml_if(aml_equal(
aml_arg(0), aml_touuid("68F2D50B-C469-4D8A-BD3D-941A103FD3FC")));
/* CXL support field */
aml_append(if_cxl, aml_create_dword_field(aml_arg(3), aml_int(12), "CDW4"));
/* CXL capabilities */
aml_append(if_cxl, aml_create_dword_field(aml_arg(3), aml_int(16), "CDW5"));
aml_append(if_cxl, aml_store(aml_name("CDW4"), aml_name("SUPC")));
aml_append(if_cxl, aml_store(aml_name("CDW5"), aml_name("CTRC")));
/* CXL 2.0 Port/Device Register access */
aml_append(if_cxl,
aml_or(aml_name("CDW5"), aml_int(0x1), aml_name("CDW5")));
aml_append(if_uuid, if_cxl);
aml_append(if_uuid, aml_return(aml_arg(3)));
aml_append(method, if_uuid);
/*
* If no UUID matched, return Unrecognized UUID via Arg3 DWord 1
* ACPI 6.4 - 6.2.11
* Unrecognised UUID - BIT(2)
*/
else_uuid = aml_else();
aml_append(else_uuid,
aml_or(aml_name("CDW1"), aml_int(0x4), aml_name("CDW1")));
aml_append(else_uuid, aml_return(aml_arg(3)));
aml_append(method, else_uuid);
return method;
}
void build_cxl_osc_method(Aml *dev)
{
aml_append(dev, aml_name_decl("SUPP", aml_int(0)));
aml_append(dev, aml_name_decl("CTRL", aml_int(0)));
aml_append(dev, aml_name_decl("SUPC", aml_int(0)));
aml_append(dev, aml_name_decl("CTRC", aml_int(0)));
aml_append(dev, __build_cxl_osc_method());
}
+3 -1
View File
@@ -13,6 +13,7 @@ acpi_ss.add(when: 'CONFIG_ACPI_MEMORY_HOTPLUG', if_false: files('acpi-mem-hotplu
acpi_ss.add(when: 'CONFIG_ACPI_NVDIMM', if_true: files('nvdimm.c'))
acpi_ss.add(when: 'CONFIG_ACPI_NVDIMM', if_false: files('acpi-nvdimm-stub.c'))
acpi_ss.add(when: 'CONFIG_ACPI_PCI', if_true: files('pci.c'))
acpi_ss.add(when: 'CONFIG_ACPI_CXL', if_true: files('cxl.c'), if_false: files('cxl-stub.c'))
acpi_ss.add(when: 'CONFIG_ACPI_VMGENID', if_true: files('vmgenid.c'))
acpi_ss.add(when: 'CONFIG_ACPI_HW_REDUCED', if_true: files('generic_event_device.c'))
acpi_ss.add(when: 'CONFIG_ACPI_HMAT', if_true: files('hmat.c'))
@@ -33,4 +34,5 @@ softmmu_ss.add_all(when: 'CONFIG_ACPI', if_true: acpi_ss)
softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('acpi-stub.c', 'aml-build-stub.c',
'acpi-x86-stub.c', 'ipmi-stub.c', 'ghes-stub.c',
'acpi-mem-hotplug-stub.c', 'acpi-cpu-hotplug-stub.c',
'acpi-pci-hotplug-stub.c', 'acpi-nvdimm-stub.c'))
'acpi-pci-hotplug-stub.c', 'acpi-nvdimm-stub.c',
'cxl-stub.c'))
+1
View File
@@ -29,6 +29,7 @@ config ARM_VIRT
select ACPI_APEI
select ACPI_VIOT
select VIRTIO_MEM_SUPPORTED
select ACPI_CXL
config CHEETAH
bool
+8 -1
View File
@@ -491,7 +491,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
return;
}
virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
virtio_init(vdev, VIRTIO_ID_BLOCK,
sizeof(struct virtio_blk_config));
s->virtqs = g_new(VirtQueue *, s->num_queues);
@@ -569,6 +569,12 @@ static void vhost_user_blk_instance_init(Object *obj)
"/disk@0,0", DEVICE(obj));
}
static struct vhost_dev *vhost_user_blk_get_vhost(VirtIODevice *vdev)
{
VHostUserBlk *s = VHOST_USER_BLK(vdev);
return &s->dev;
}
static const VMStateDescription vmstate_vhost_user_blk = {
.name = "vhost-user-blk",
.minimum_version_id = 1,
@@ -603,6 +609,7 @@ static void vhost_user_blk_class_init(ObjectClass *klass, void *data)
vdc->get_features = vhost_user_blk_get_features;
vdc->set_status = vhost_user_blk_set_status;
vdc->reset = vhost_user_blk_reset;
vdc->get_vhost = vhost_user_blk_get_vhost;
}
static const TypeInfo vhost_user_blk_info = {
+1 -1
View File
@@ -1206,7 +1206,7 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
virtio_blk_set_config_size(s, s->host_features);
virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK, s->config_size);
virtio_init(vdev, VIRTIO_ID_BLOCK, s->config_size);
s->blk = conf->conf.blk;
s->rq = NULL;
+1 -2
View File
@@ -1044,8 +1044,7 @@ static void virtio_serial_device_realize(DeviceState *dev, Error **errp)
VIRTIO_CONSOLE_F_EMERG_WRITE)) {
config_size = offsetof(struct virtio_console_config, emerg_wr);
}
virtio_init(vdev, "virtio-serial", VIRTIO_ID_CONSOLE,
config_size);
virtio_init(vdev, VIRTIO_ID_CONSOLE, config_size);
/* Spawn a new virtio-serial bus on which the ports will ride as devices */
qbus_init(&vser->bus, sizeof(vser->bus), TYPE_VIRTIO_SERIAL_BUS,
+28
View File
@@ -33,6 +33,7 @@
#include "sysemu/qtest.h"
#include "hw/pci/pci.h"
#include "hw/mem/nvdimm.h"
#include "hw/cxl/cxl.h"
#include "migration/global_state.h"
#include "migration/vmstate.h"
#include "exec/confidential-guest-support.h"
@@ -625,6 +626,20 @@ static void machine_set_nvdimm_persistence(Object *obj, const char *value,
nvdimms_state->persistence_string = g_strdup(value);
}
static bool machine_get_cxl(Object *obj, Error **errp)
{
MachineState *ms = MACHINE(obj);
return ms->cxl_devices_state->is_enabled;
}
static void machine_set_cxl(Object *obj, bool value, Error **errp)
{
MachineState *ms = MACHINE(obj);
ms->cxl_devices_state->is_enabled = value;
}
void machine_class_allow_dynamic_sysbus_dev(MachineClass *mc, const char *type)
{
QAPI_LIST_PREPEND(mc->allowed_dynamic_sysbus_devices, g_strdup(type));
@@ -911,6 +926,8 @@ static void machine_class_init(ObjectClass *oc, void *data)
mc->default_ram_size = 128 * MiB;
mc->rom_file_has_mr = true;
/* Few machines support CXL, so default to off */
mc->cxl_supported = false;
/* numa node memory size aligned on 8MB by default.
* On Linux, each node's border has to be 8MB aligned
*/
@@ -1071,6 +1088,16 @@ static void machine_initfn(Object *obj)
"Valid values are cpu, mem-ctrl");
}
if (mc->cxl_supported) {
Object *obj = OBJECT(ms);
ms->cxl_devices_state = g_new0(CXLState, 1);
object_property_add_bool(obj, "cxl", machine_get_cxl, machine_set_cxl);
object_property_set_description(obj, "cxl",
"Set on/off to enable/disable "
"CXL instantiation");
}
if (mc->cpu_index_to_instance_props && mc->get_default_cpu_node_id) {
ms->numa_state = g_new0(NumaState, 1);
object_property_add_bool(obj, "hmat",
@@ -1108,6 +1135,7 @@ static void machine_finalize(Object *obj)
g_free(ms->device_memory);
g_free(ms->nvdimms_state);
g_free(ms->numa_state);
g_free(ms->cxl_devices_state);
}
bool machine_usb(MachineState *machine)
+3
View File
@@ -0,0 +1,3 @@
config CXL
bool
default y if PCI_EXPRESS

Some files were not shown because too many files have changed in this diff Show More