izzy/xemu
0
0
Fork 0
mirror of https://github.com/izzy2lost/xemu.git synced 2026-07-06 00:20:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'pull-request-2025-04-23' of https://gitlab.com/thuth/qemu into staging

Browse Source 
* Remove the obsolete s390-ccw-virtio-2.9 machine type
* Prepare the dump-skeys QMP command for the universal binary project
* Add compat machine types for 10.1
* Convert the remaining Avocado tests to the functional framework
* Some more small fixes for the functional tests

# -----BEGIN PGP SIGNATURE-----
#
# iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmgIghARHHRodXRoQHJl
# ZGhhdC5jb20ACgkQLtnXdP5wLbXFeA//ayM/KF9so5DbatHQ+TD4wReNd5zlcYxz
# hjx5FAQPl+QmMNcBhkfBWJ6DqypzzrDOrRevwKe1xcJYyUthDJ6Bl//phmfTKzbo
# lAW5XmQhFGi+yolhRMdd6N+5Ky/c0jPzZjuJvi21RmSWi+ZqBiwUmaH1wutiI6A5
# peytvpt1R+DPJsZtFViEW9AwRtSkQh4I5+not33bvKdFWTGHE+iSYnVTXDiEbY5p
# TNIXaN9t9ZnojknswwHvyF+QbXn/pVLSA+XLDZuops2IEiPCsCTXUdjSwlfPReKM
# G0tl5DOXJFUMwmU1wRtqn0TdkP2AL1KfMr6ZVsE1zD1/XMj9uDF6BxxjB2si2UrC
# DTXCn+K535IyEa3qf25mhv9ueDr2ItiaDk4CBpb+Z/lg3p73jVW8G1mAU0mG9XlH
# KhJgOqTelnSuwlF6PdhtOxiH/rmgozMcrlppAMBbNgKBvzrT7Iwcq3OS9r+ZAWXO
# 7ouY/zYlbe1fnRgkgaapWx5AXhbkyxqgMkmZk0JE5XcsOHwMUd4O/Sk2kE6zVObe
# 7ReNqJdxj4c+lfqof5UwMUeqb465NTEAJXBGpiGDcP5qCzGBwnaIv5D71o1A+KoB
# 8g3E6woUN2i3ZPq8W4L8PbMR+zHNGUIVdnmH3EtNd6X3cBN9w4UYVN1MdzKSzQsi
# aCG/VdfcZsw=
# =yO4v
# -----END PGP SIGNATURE-----
# gpg: Signature made Wed 23 Apr 2025 02:00:48 EDT
# gpg:                using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5
# gpg:                issuer "thuth@redhat.com"
# gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [full]
# gpg:                 aka "Thomas Huth <thuth@redhat.com>" [full]
# gpg:                 aka "Thomas Huth <huth@tuxfamily.org>" [full]
# gpg:                 aka "Thomas Huth <th.huth@posteo.de>" [unknown]
# Primary key fingerprint: 27B8 8847 EEE0 2501 18F3  EAB9 2ED9 D774 FE70 2DB5

* tag 'pull-request-2025-04-23' of https://gitlab.com/thuth/qemu: (29 commits)
  MAINTAINERS: Add functional tests that are not covered yet
  tests/functional: Remove unnecessary import statements
  tests/functional: Remove semicolons at the end of lines
  Remove the remainders of the Avocado tests
  docs/devel/testing: Dissolve the ci-definitions.rst.inc file
  gitlab-ci: Update QEMU_JOB_AVOCADO and QEMU_CI_AVOCADO_TESTING
  tests/functional: Convert the SMMU test to the functional framework
  tests/functional: Use the tuxrun kernel for the aarch64 replay test
  tests/functional: Use the tuxrun kernel for the x86 replay test
  tests/avocado: Remove the boot_linux.py tests
  tests/functional: Convert the 64-bit big endian Wheezy mips test
  tests/functional: Convert the 64-bit little endian Wheezy mips test
  tests/functional: Convert the 32-bit little endian Wheezy mips test
  tests/functional: Convert the 32-bit big endian Wheezy mips test
  tests/avocado: Remove the LinuxKernelTest class
  tests/functional: Convert the i386 replay avocado test
  tests/functional: Convert reverse_debugging tests to the functional framework
  tests/functional: Move the check for the parameters from avocado to functional
  gitlab-ci: Remove the avocado tests from the CI pipelines
  tests/functional/test_vnc: skip test if no crypto backend available
  ...

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
This commit is contained in:
Stefan Hajnoczi
2025-04-23 09:29:23 -04:00
parent eebba0536a 12c6b61530
commit 55f5bf716a
91 changed files with 981 additions and 2797 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitlab-ci.d/base.yml
+2 -6
View File
@@ -69,10 +69,6 @@ variables:
- if: '$QEMU_CI != "1" && $QEMU_CI != "2" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
when: never
# Avocado jobs don't run in forks unless $QEMU_CI_AVOCADO_TESTING is set
- if: '$QEMU_JOB_AVOCADO && $QEMU_CI_AVOCADO_TESTING != "1" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
when: never
#############################################################
# Stage 2: fine tune execution of jobs in specific scenarios
@@ -101,8 +97,8 @@ variables:
when: manual
allow_failure: true
# Avocado jobs can be manually start in forks if $QEMU_CI_AVOCADO_TESTING is unset
- if: '$QEMU_JOB_AVOCADO && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
# Functional jobs can be manually started in forks
- if: '$QEMU_JOB_FUNCTIONAL && $QEMU_CI_FUNCTIONAL != "1" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
when: manual
allow_failure: true
.gitlab-ci.d/buildtest-template.yml
+1 -12
View File
@@ -95,7 +95,6 @@
cache:
key: "${CI_JOB_NAME}-cache"
paths:
- ${CI_PROJECT_DIR}/avocado-cache
- ${CI_PROJECT_DIR}/functional-cache
policy: pull-push
artifacts:
@@ -109,20 +108,10 @@
reports:
junit: build/tests/results/latest/results.xml
before_script:
- mkdir -p ~/.config/avocado
- echo "[datadir.paths]" > ~/.config/avocado/avocado.conf
- echo "cache_dirs = ['${CI_PROJECT_DIR}/avocado-cache']"
>> ~/.config/avocado/avocado.conf
- echo -e '[job.output.testlogs]\nstatuses = ["FAIL", "INTERRUPT"]'
>> ~/.config/avocado/avocado.conf
- if [ -d ${CI_PROJECT_DIR}/avocado-cache ]; then
du -chs ${CI_PROJECT_DIR}/*-cache ;
fi
- export AVOCADO_ALLOW_UNTRUSTED_CODE=1
- export QEMU_TEST_ALLOW_UNTRUSTED_CODE=1
- export QEMU_TEST_CACHE_DIR=${CI_PROJECT_DIR}/functional-cache
after_script:
- cd build
- du -chs ${CI_PROJECT_DIR}/*-cache
variables:
QEMU_JOB_AVOCADO: 1
QEMU_JOB_FUNCTIONAL: 1
.gitlab-ci.d/buildtest.yml
+12 -21
View File
@@ -29,8 +29,7 @@ functional-system-alpine:
artifacts: true
variables:
IMAGE: alpine
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:avr arch:loongarch64 arch:mips64 arch:mipsel
MAKE_CHECK_ARGS: check-functional
build-system-ubuntu:
extends:
@@ -60,8 +59,7 @@ functional-system-ubuntu:
artifacts: true
variables:
IMAGE: ubuntu2204
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:alpha arch:microblazeel arch:mips64el
MAKE_CHECK_ARGS: check-functional
build-system-debian:
extends:
@@ -92,8 +90,7 @@ functional-system-debian:
artifacts: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:arm arch:i386 arch:riscv64 arch:sh4 arch:sparc arch:xtensa
MAKE_CHECK_ARGS: check-functional
crash-test-debian:
extends: .native_test_job_template
@@ -155,9 +152,7 @@ functional-system-fedora:
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:microblaze arch:mips arch:xtensa arch:m68k
arch:riscv32 arch:ppc arch:sparc64
MAKE_CHECK_ARGS: check-functional
crash-test-fedora:
extends: .native_test_job_template
@@ -278,9 +273,7 @@ functional-system-centos:
artifacts: true
variables:
IMAGE: centos9
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:ppc64 arch:or1k arch:s390x arch:x86_64 arch:rx
arch:sh4
MAKE_CHECK_ARGS: check-functional
build-system-opensuse:
extends:
@@ -309,8 +302,7 @@ functional-system-opensuse:
artifacts: true
variables:
IMAGE: opensuse-leap
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:s390x arch:x86_64 arch:aarch64
MAKE_CHECK_ARGS: check-functional
#
# Flaky tests. We don't run these by default and they are allow fail
@@ -338,10 +330,9 @@ functional-system-flaky:
allow_failure: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado check-functional
MAKE_CHECK_ARGS: check-functional
QEMU_JOB_OPTIONAL: 1
QEMU_TEST_FLAKY_TESTS: 1
AVOCADO_TAGS: flaky
# This jobs explicitly disable TCG (--disable-tcg), KVM is detected by
# the configure script. The container doesn't contain Xen headers so
@@ -482,8 +473,8 @@ clang-user:
# Since slirp callbacks are used in QEMU Timers, we cannot use libslirp with
# CFI builds, and thus have to disable it here.
#
# Split in three sets of build/check/avocado to limit the execution time of each
# job
# Split in three sets of build/check/functional to limit the execution time
# of each job
build-cfi-aarch64:
extends:
- .native_build_job_template
@@ -520,7 +511,7 @@ functional-cfi-aarch64:
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado check-functional
MAKE_CHECK_ARGS: check-functional
build-cfi-ppc64-s390x:
extends:
@@ -558,7 +549,7 @@ functional-cfi-ppc64-s390x:
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado check-functional
MAKE_CHECK_ARGS: check-functional
build-cfi-x86_64:
extends:
@@ -592,7 +583,7 @@ functional-cfi-x86_64:
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado check-functional
MAKE_CHECK_ARGS: check-functional
tsan-build:
extends: .native_build_job_template
MAINTAINERS
+14 -14
View File
@@ -211,7 +211,7 @@ L: qemu-arm@nongnu.org
S: Maintained
F: hw/arm/smmu*
F: include/hw/arm/smmu*
F: tests/avocado/smmu.py
F: tests/functional/test_aarch64_smmu.py
AVR TCG CPUs
M: Michael Rolnik <mrolnik@gmail.com>
@@ -475,6 +475,7 @@ F: docs/system/i386/sgx.rst
F: target/i386/kvm/
F: target/i386/sev*
F: scripts/kvm/vmxcap
F: tests/functional/test_x86_64_hotplug_cpu.py
Xen emulation on X86 KVM CPUs
M: David Woodhouse <dwmw2@infradead.org>
@@ -634,6 +635,7 @@ S: Maintained
F: hw/alpha/
F: hw/isa/smc37c669-superio.c
F: tests/tcg/alpha/system/
F: tests/functional/test_alpha_clipper.py
ARM Machines
------------
@@ -958,7 +960,7 @@ F: hw/misc/sbsa_ec.c
F: hw/watchdog/sbsa_gwdt.c
F: include/hw/watchdog/sbsa_gwdt.h
F: docs/system/arm/sbsa.rst
F: tests/functional/test_aarch64_sbsaref*.py
F: tests/functional/test_aarch64_*sbsaref*.py
Sharp SL-5500 (Collie) PDA
M: Peter Maydell <peter.maydell@linaro.org>
@@ -1027,9 +1029,10 @@ S: Maintained
F: hw/arm/virt*
F: include/hw/arm/virt.h
F: docs/system/arm/virt.rst
F: tests/functional/test_aarch64_virt*.py
F: tests/functional/test_aarch64_*virt*.py
F: tests/functional/test_aarch64_tuxrun.py
F: tests/functional/test_arm_tuxrun.py
F: tests/functional/test_arm_virt.py
Xilinx Zynq
M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -1270,6 +1273,7 @@ F: hw/m68k/mcf_intc.c
F: hw/char/mcf_uart.c
F: hw/net/mcf_fec.c
F: include/hw/m68k/mcf*.h
F: tests/functional/test_m68k_mcf5208evb.py
NeXTcube
M: Thomas Huth <huth@tuxfamily.org>
@@ -1363,7 +1367,6 @@ F: hw/acpi/piix4.c
F: hw/mips/malta.c
F: hw/pci-host/gt64120.c
F: include/hw/southbridge/piix.h
F: tests/avocado/linux_ssh_mips_malta.py
F: tests/functional/test_mips*_malta.py
F: tests/functional/test_mips*_tuxrun.py
@@ -1415,6 +1418,7 @@ S: Maintained
F: docs/system/openrisc/or1k-sim.rst
F: hw/intc/ompic.c
F: hw/openrisc/openrisc_sim.c
F: tests/functional/test_or1k_sim.py
PowerPC Machines
----------------
@@ -1836,6 +1840,7 @@ F: include/hw/isa/apm.h
F: tests/unit/test-x86-topo.c
F: tests/qtest/test-x86-cpuid-compat.c
F: tests/functional/test_i386_tuxrun.py
F: tests/functional/test_linux_initrd.py
F: tests/functional/test_mem_addr_space.py
F: tests/functional/test_pc_cpu_hotplug_props.py
F: tests/functional/test_x86_64_tuxrun.py
@@ -2082,7 +2087,7 @@ S: Supported
F: hw/acpi/viot.c
F: hw/acpi/viot.h
ACPI/AVOCADO/BIOSBITS
ACPI/FUNCTIONAL/BIOSBITS
M: Ani Sinha <anisinha@redhat.com>
M: Michael S. Tsirkin <mst@redhat.com>
S: Supported
@@ -3159,6 +3164,7 @@ F: include/ui/
F: qapi/ui.json
F: util/drm.c
F: docs/devel/ui.rst
F: tests/functional/test_vnc.py
Cocoa graphics
M: Peter Maydell <peter.maydell@linaro.org>
@@ -3677,9 +3683,7 @@ F: include/system/replay.h
F: docs/devel/replay.rst
F: docs/system/replay.rst
F: stubs/replay.c
F: tests/avocado/replay_kernel.py
F: tests/avocado/replay_linux.py
F: tests/avocado/reverse_debugging.py
F: tests/functional/*reverse_debug*.py
F: tests/functional/*replay*.py
F: qapi/replay.json
@@ -3826,6 +3830,7 @@ F: configs/targets/*linux-user.mak
F: scripts/qemu-binfmt-conf.sh
F: scripts/update-syscalltbl.sh
F: scripts/update-mips-syscall-args.sh
F: tests/functional/test_arm_bflt.py
Tiny Code Generator (TCG)
-------------------------
@@ -4198,6 +4203,7 @@ F: hw/remote/vfio-user-obj.c
F: include/hw/remote/vfio-user-obj.h
F: hw/remote/iommu.c
F: include/hw/remote/iommu.h
F: tests/functional/test_multiprocess.py
EBPF:
M: Jason Wang <jasowang@redhat.com>
@@ -4257,12 +4263,6 @@ R: Philippe Mathieu-Daudé <philmd@linaro.org>
S: Maintained
F: tests/tcg/Makefile.target
Integration Testing with the Avocado framework
W: https://trello.com/b/6Qi1pxVn/avocado-qemu
R: Cleber Rosa <crosa@redhat.com>
S: Odd Fixes
F: tests/avocado/
GitLab custom runner (Works On Arm Sponsored)
M: Alex Bennée <alex.bennee@linaro.org>
M: Philippe Mathieu-Daudé <philmd@linaro.org>
configure
Vendored
+1 -1
View File
@@ -1685,7 +1685,7 @@ LINKS="$LINKS pc-bios/optionrom/Makefile"
LINKS="$LINKS pc-bios/s390-ccw/Makefile"
LINKS="$LINKS pc-bios/vof/Makefile"
LINKS="$LINKS .gdbinit scripts" # scripts needed by relative path in .gdbinit
LINKS="$LINKS tests/avocado tests/data"
LINKS="$LINKS tests/data"
LINKS="$LINKS tests/qemu-iotests/check tests/qemu-iotests/Makefile"
LINKS="$LINKS python"
for f in $LINKS ; do
docs/about/build-platforms.rst
+5 -5
View File
@@ -123,11 +123,11 @@ Rust build dependencies
to build QEMU.
Optional build dependencies
Build components whose absence does not affect the ability to build
QEMU may not be available in distros, or may be too old for QEMU's
requirements. Many of these, such as the Avocado testing framework
or various linters, are written in Python and therefore can also
be installed using ``pip``. Cross compilers are another example
Build components whose absence does not affect the ability to build QEMU
may not be available in distros, or may be too old for our requirements.
Many of these, such as additional modules for the functional testing
framework or various linters, are written in Python and therefore can
also be installed using ``pip``. Cross compilers are another example
of optional build-time dependency; in this case it is possible to
download them from repositories such as EPEL, to use container-based
cross compilation using ``docker`` or ``podman``, or to use pre-built
docs/devel/build-system.rst
+3 -8
View File
@@ -134,7 +134,7 @@ in how the build process runs Python code.
At this stage, ``configure`` also queries the chosen Python interpreter
about QEMU's build dependencies. Note that the build process does *not*
look for ``meson``, ``sphinx-build`` or ``avocado`` binaries in the PATH;
look for ``meson`` or ``sphinx-build`` binaries in the PATH;
likewise, there are no options such as ``--meson`` or ``--sphinx-build``.
This avoids a potential mismatch, where Meson and Sphinx binaries on the
PATH might operate in a different Python environment than the one chosen
@@ -151,7 +151,7 @@ virtual environment with ``pip``, either from wheels in ``python/wheels``
or by downloading the package with PyPI. Downloading can be disabled with
``--disable-download``; and anyway, it only happens when a ``configure``
option (currently, only ``--enable-docs``) is explicitly enabled but
the dependencies are not present\ [#pip]_.
the dependencies are not present.
.. [#distlib] The scripts are created based on the package's metadata,
specifically the ``console_script`` entry points. This is the
@@ -164,10 +164,6 @@ the dependencies are not present\ [#pip]_.
because the Python Packaging Authority provides a package
``distlib.scripts`` to perform this task.
.. [#pip] ``pip`` might also be used when running ``make check-avocado``
if downloading is enabled, to ensure that Avocado is
available.
The required versions of the packages are stored in a configuration file
``pythondeps.toml``. The format is custom to QEMU, but it is documented
at the top of the file itself and it should be easy to understand. The
@@ -497,8 +493,7 @@ number of dynamically created files listed later.
``pyvenv/bin``, and calling ``pip`` to install dependencies.
``tests/Makefile.include``
Rules for external test harnesses. These include the TCG tests
and the Avocado-based integration tests.
Rules for external test harnesses like the TCG tests.
``tests/docker/Makefile.include``
Rules for Docker tests. Like ``tests/Makefile.include``, this file is
docs/devel/codebase.rst
-5
View File
@@ -175,11 +175,6 @@ yet, so sometimes the source code is all you have.
* `tests <https://gitlab.com/qemu-project/qemu/-/tree/master/tests>`_:
QEMU `test <testing>` suite
- `avocado <https://gitlab.com/qemu-project/qemu/-/tree/master/tests/avocado>`_:
Functional tests booting full VM using `Avocado framework <checkavocado-ref>`.
Those tests will be transformed and moved into
`tests/functional <https://gitlab.com/qemu-project/qemu/-/tree/master/tests/functional>`_
in the future.
- `data <https://gitlab.com/qemu-project/qemu/-/tree/master/tests/data>`_:
Data for various tests.
- `decode <https://gitlab.com/qemu-project/qemu/-/tree/master/tests/decode>`_:
docs/devel/testing/avocado.rst
-581
View File
File diff suppressed because it is too large Load Diff
docs/devel/testing/ci-definitions.rst.inc
-121
View File
@@ -1,121 +0,0 @@
Definition of terms
===================
This section defines the terms used in this document and correlates them with
what is currently used on QEMU.
Automated tests
---------------
An automated test is written on a test framework using its generic test
functions/classes. The test framework can run the tests and report their
success or failure [1]_.
An automated test has essentially three parts:
1. The test initialization of the parameters, where the expected parameters,
like inputs and expected results, are set up;
2. The call to the code that should be tested;
3. An assertion, comparing the result from the previous call with the expected
result set during the initialization of the parameters. If the result
matches the expected result, the test has been successful; otherwise, it has
failed.
Unit testing
------------
A unit test is responsible for exercising individual software components as a
unit, like interfaces, data structures, and functionality, uncovering errors
within the boundaries of a component. The verification effort is in the
smallest software unit and focuses on the internal processing logic and data
structures. A test case of unit tests should be designed to uncover errors due
to erroneous computations, incorrect comparisons, or improper control flow [2]_.
On QEMU, unit testing is represented by the 'check-unit' target from 'make'.
Functional testing
------------------
A functional test focuses on the functional requirement of the software.
Deriving sets of input conditions, the functional tests should fully exercise
all the functional requirements for a program. Functional testing is
complementary to other testing techniques, attempting to find errors like
incorrect or missing functions, interface errors, behavior errors, and
initialization and termination errors [3]_.
On QEMU, functional testing is represented by the 'check-qtest' target from
'make'.
System testing
--------------
System tests ensure all application elements mesh properly while the overall
functionality and performance are achieved [4]_. Some or all system components
are integrated to create a complete system to be tested as a whole. System
testing ensures that components are compatible, interact correctly, and
transfer the right data at the right time across their interfaces. As system
testing focuses on interactions, use case-based testing is a practical approach
to system testing [5]_. Note that, in some cases, system testing may require
interaction with third-party software, like operating system images, databases,
networks, and so on.
On QEMU, system testing is represented by the 'check-avocado' target from
'make'.
Flaky tests
-----------
A flaky test is defined as a test that exhibits both a passing and a failing
result with the same code on different runs. Some usual reasons for an
intermittent/flaky test are async wait, concurrency, and test order dependency
[6]_.
Gating
------
A gate restricts the move of code from one stage to another on a
test/deployment pipeline. The step move is granted with approval. The approval
can be a manual intervention or a set of tests succeeding [7]_.
On QEMU, the gating process happens during the pull request. The approval is
done by the project leader running its own set of tests. The pull request gets
merged when the tests succeed.
Continuous Integration (CI)
---------------------------
Continuous integration (CI) requires the builds of the entire application and
the execution of a comprehensive set of automated tests every time there is a
need to commit any set of changes [8]_. The automated tests can be composed of
the unit, functional, system, and other tests.
Keynotes about continuous integration (CI) [9]_:
1. System tests may depend on external software (operating system images,
firmware, database, network).
2. It may take a long time to build and test. It may be impractical to build
the system being developed several times per day.
3. If the development platform is different from the target platform, it may
not be possible to run system tests in the developer’s private workspace.
There may be differences in hardware, operating system, or installed
software. Therefore, more time is required for testing the system.
References
----------
.. [1] Sommerville, Ian (2016). Software Engineering. p. 233.
.. [2] Pressman, Roger S. & Maxim, Bruce R. (2020). Software Engineering,
A Practitioner’s Approach. p. 48, 376, 378, 381.
.. [3] Pressman, Roger S. & Maxim, Bruce R. (2020). Software Engineering,
A Practitioner’s Approach. p. 388.
.. [4] Pressman, Roger S. & Maxim, Bruce R. (2020). Software Engineering,
A Practitioner’s Approach. Software Engineering, p. 377.
.. [5] Sommerville, Ian (2016). Software Engineering. p. 59, 232, 240.
.. [6] Luo, Qingzhou, et al. An empirical analysis of flaky tests.
Proceedings of the 22nd ACM SIGSOFT International Symposium on
Foundations of Software Engineering. 2014.
.. [7] Humble, Jez & Farley, David (2010). Continuous Delivery:
Reliable Software Releases Through Build, Test, and Deployment, p. 122.
.. [8] Humble, Jez & Farley, David (2010). Continuous Delivery:
Reliable Software Releases Through Build, Test, and Deployment, p. 55.
.. [9] Sommerville, Ian (2016). Software Engineering. p. 743.
docs/devel/testing/ci-jobs.rst.inc
+9 -10
View File
@@ -126,10 +126,10 @@ QEMU_JOB_PUBLISH
The job is for publishing content after a branch has been
merged into the upstream default branch.
QEMU_JOB_AVOCADO
~~~~~~~~~~~~~~~~
QEMU_JOB_FUNCTIONAL
~~~~~~~~~~~~~~~~~~~
The job runs the Avocado integration test suite
The job runs the functional test suite
Contributor controlled runtime variables
----------------------------------------
@@ -149,13 +149,12 @@ the jobs to be manually started from the UI
Set this variable to 2 to create the pipelines and run all
the jobs immediately, as was the historical behaviour
QEMU_CI_AVOCADO_TESTING
~~~~~~~~~~~~~~~~~~~~~~~
By default, tests using the Avocado framework are not run automatically in
the pipelines (because multiple artifacts have to be downloaded, and if
these artifacts are not already cached, downloading them make the jobs
reach the timeout limit). Set this variable to have the tests using the
Avocado framework run automatically.
QEMU_CI_FUNCTIONAL
~~~~~~~~~~~~~~~~~~
By default, tests using the functional framework are not run automatically
in the pipelines (because multiple artifacts have to be downloaded, which
might cause a lot of network traffic). Set this variable to have the tests
using the functional framework run automatically.
Other misc variables
--------------------
docs/devel/testing/ci.rst
+24 -4
View File
@@ -1,14 +1,34 @@
.. _ci:
==
CI
==
Continuous Integration (CI)
===========================
Continuous integration (CI) requires the builds of the entire application and
the execution of a comprehensive set of automated tests every time there is a
need to commit any set of changes [1]_. The automated tests are composed
of unit, functional and other tests.
Most of QEMU's CI is run on GitLab's infrastructure although a number
of other CI services are used for specialised purposes. The most up to
date information about them and their status can be found on the
`project wiki testing page <https://wiki.qemu.org/Testing/CI>`_.
.. include:: ci-definitions.rst.inc
These tests are also used as gating tests before merging pull requests.
A gating test restricts the move of code from one stage to another on a
test/deployment pipeline. The step move is granted with approval. The approval
can be a manual intervention or a set of tests succeeding [2]_.
On QEMU, the gating process happens during the pull request. The approval is
done by the project leader running its own set of tests. The pull request gets
merged when the tests succeed.
.. include:: ci-jobs.rst.inc
.. include:: ci-runners.rst.inc
References
----------
.. [1] Humble, Jez & Farley, David (2010). Continuous Delivery:
Reliable Software Releases Through Build, Test, and Deployment, p. 55.
.. [2] Humble, Jez & Farley, David (2010). Continuous Delivery:
Reliable Software Releases Through Build, Test, and Deployment, p. 122.
docs/devel/testing/functional.rst
-3
View File
@@ -6,9 +6,6 @@ Functional testing with Python
The ``tests/functional`` directory hosts functional tests written in
Python. They are usually higher level tests, and may interact with
external resources and with various guest operating systems.
The functional tests have initially evolved from the Avocado tests, so there
is a lot of similarity to those tests here (see :ref:`checkavocado-ref` for
details about the Avocado tests).
The tests should be written in the style of the Python `unittest`_ framework,
using stdio for the TAP protocol. The folder ``tests/functional/qemu_test``
docs/devel/testing/index.rst
-1
View File
@@ -10,7 +10,6 @@ testing infrastructure.
main
qtest
functional
avocado
acpi-bits
ci
fuzzing
docs/devel/testing/main.rst
+57 -23
View File
@@ -5,19 +5,32 @@ Testing in QEMU
QEMU's testing infrastructure is fairly complex as it covers
everything from unit testing and exercising specific sub-systems all
the way to full blown acceptance tests. To get an overview of the
the way to full blown functional tests. To get an overview of the
tests you can run ``make check-help`` from either the source or build
tree.
Most (but not all) tests are also integrated into the meson build
system so can be run directly from the build tree, for example:
.. code::
Most (but not all) tests are also integrated as an automated test into
the meson build system so can be run directly from the build tree,
for example::
[./pyvenv/bin/]meson test --suite qemu:softfloat
will run just the softfloat tests.
An automated test is written with one of the test frameworks using its
generic test functions/classes. The test framework can run the tests and
report their success or failure [1]_.
An automated test has essentially three parts:
1. The test initialization of the parameters, where the expected parameters,
like inputs and expected results, are set up;
2. The call to the code that should be tested;
3. An assertion, comparing the result from the previous call with the expected
result set during the initialization of the parameters. If the result
matches the expected result, the test has been successful; otherwise, it has
failed.
The rest of this document will cover the details for specific test
groups.
@@ -44,9 +57,17 @@ cannot find them.
Unit tests
~~~~~~~~~~
Unit tests, which can be invoked with ``make check-unit``, are simple C tests
that typically link to individual QEMU object files and exercise them by
calling exported functions.
A unit test is responsible for exercising individual software components as a
unit, like interfaces, data structures, and functionality, uncovering errors
within the boundaries of a component. The verification effort is in the
smallest software unit and focuses on the internal processing logic and data
structures. A test case of unit tests should be designed to uncover errors
due to erroneous computations, incorrect comparisons, or improper control
flow [2]_.
In QEMU, unit tests can be invoked with ``make check-unit``. They are
simple C tests that typically link to individual QEMU object files and
exercise them by calling exported functions.
If you are writing new code in QEMU, consider adding a unit test, especially
for utility modules that are relatively stateless or have few dependencies. To
@@ -885,6 +906,10 @@ changing the ``-c`` option.
Functional tests using Python
-----------------------------
A functional test focuses on the functional requirement of the software,
attempting to find errors like incorrect functions, interface errors,
behavior errors, and initialization and termination errors [3]_.
The ``tests/functional`` directory hosts functional tests written in
Python. You can run the functional tests simply by executing:
@@ -894,21 +919,6 @@ Python. You can run the functional tests simply by executing:
See :ref:`checkfunctional-ref` for more details.
Integration tests using the Avocado Framework
---------------------------------------------
The ``tests/avocado`` directory hosts integration tests. They're usually
higher level tests, and may interact with external resources and with
various guest operating systems.
You can run the avocado tests simply by executing:
.. code::
make check-avocado
See :ref:`checkavocado-ref` for more details.
.. _checktcg-ref:
Testing with "make check-tcg"
@@ -1023,3 +1033,27 @@ coverage-html`` which will create
Further analysis can be conducted by running the ``gcov`` command
directly on the various .gcda output files. Please read the ``gcov``
documentation for more information.
Flaky tests
-----------
A flaky test is defined as a test that exhibits both a passing and a failing
result with the same code on different runs. Some usual reasons for an
intermittent/flaky test are async wait, concurrency, and test order dependency
[4]_.
In QEMU, tests that are identified to be flaky are normally disabled by
default. Set the QEMU_TEST_FLAKY_TESTS environment variable before running
the tests to enable them.
References
----------
.. [1] Sommerville, Ian (2016). Software Engineering. p. 233.
.. [2] Pressman, Roger S. & Maxim, Bruce R. (2020). Software Engineering,
A Practitioner’s Approach. p. 48, 376, 378, 381.
.. [3] Pressman, Roger S. & Maxim, Bruce R. (2020). Software Engineering,
A Practitioner’s Approach. p. 388.
.. [4] Luo, Qingzhou, et al. An empirical analysis of flaky tests.
Proceedings of the 22nd ACM SIGSOFT International Symposium on
Foundations of Software Engineering. 2014.
hw/arm/virt.c
+9 -2
View File
@@ -3408,10 +3408,17 @@ static void machvirt_machine_init(void)
}
type_init(machvirt_machine_init);
static void virt_machine_10_0_options(MachineClass *mc)
static void virt_machine_10_1_options(MachineClass *mc)
{
}
DEFINE_VIRT_MACHINE_AS_LATEST(10, 0)
DEFINE_VIRT_MACHINE_AS_LATEST(10, 1)
static void virt_machine_10_0_options(MachineClass *mc)
{
virt_machine_10_1_options(mc);
compat_props_add(mc->compat_props, hw_compat_10_0, hw_compat_10_0_len);
}
DEFINE_VIRT_MACHINE(10, 0)
static void virt_machine_9_2_options(MachineClass *mc)
{
hw/core/machine-qmp-cmds.c
+14
View File
@@ -25,6 +25,7 @@
#include "system/numa.h"
#include "system/runstate.h"
#include "system/system.h"
#include "hw/s390x/storage-keys.h"
/*
* fast means: we NEVER interrupt vCPU threads to retrieve
@@ -406,3 +407,16 @@ GuidInfo *qmp_query_vm_generation_id(Error **errp)
info->guid = qemu_uuid_unparse_strdup(&vms->guid);
return info;
}
void qmp_dump_skeys(const char *filename, Error **errp)
{
ObjectClass *mc = object_get_class(qdev_get_machine());
ObjectClass *oc = object_class_dynamic_cast(mc, TYPE_DUMP_SKEYS_INTERFACE);
if (!oc) {
error_setg(errp, "Storage keys information not available"
" for this architecture");
return;
}
DUMP_SKEYS_INTERFACE_CLASS(oc)->qmp_dump_skeys(filename, errp);
}
hw/core/machine.c
+3
View File
@@ -37,6 +37,9 @@
#include "hw/virtio/virtio-iommu.h"
#include "audio/audio.h"
GlobalProperty hw_compat_10_0[] = {};
const size_t hw_compat_10_0_len = G_N_ELEMENTS(hw_compat_10_0);
GlobalProperty hw_compat_9_2[] = {
{"arm-cpu", "backcompat-pauth-default-use-qarma5", "true"},
{ "virtio-balloon-pci", "vectors", "0" },
hw/i386/pc.c
+3
View File
@@ -79,6 +79,9 @@
{ "qemu64-" TYPE_X86_CPU, "model-id", "QEMU Virtual CPU version " v, },\
{ "athlon-" TYPE_X86_CPU, "model-id", "QEMU Virtual CPU version " v, },
GlobalProperty pc_compat_10_0[] = {};
const size_t pc_compat_10_0_len = G_N_ELEMENTS(pc_compat_10_0);
GlobalProperty pc_compat_9_2[] = {};
const size_t pc_compat_9_2_len = G_N_ELEMENTS(pc_compat_9_2);
hw/i386/pc_piix.c
+11 -2
View File
@@ -479,12 +479,21 @@ static void pc_i440fx_machine_options(MachineClass *m)
"Use a different south bridge than PIIX3");
}
static void pc_i440fx_machine_10_0_options(MachineClass *m)
static void pc_i440fx_machine_10_1_options(MachineClass *m)
{
pc_i440fx_machine_options(m);
}
DEFINE_I440FX_MACHINE_AS_LATEST(10, 0);
DEFINE_I440FX_MACHINE_AS_LATEST(10, 1);
static void pc_i440fx_machine_10_0_options(MachineClass *m)
{
pc_i440fx_machine_10_1_options(m);
compat_props_add(m->compat_props, hw_compat_10_0, hw_compat_10_0_len);
compat_props_add(m->compat_props, pc_compat_10_0, pc_compat_10_0_len);
}
DEFINE_I440FX_MACHINE(10, 0);
static void pc_i440fx_machine_9_2_options(MachineClass *m)
{

Some files were not shown because too many files have changed in this diff Show More