izzy/xemu
0
0
Fork 0
mirror of https://github.com/izzy2lost/xemu.git synced 2026-07-06 00:20:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'pull-request-2024-09-04' of https://gitlab.com/thuth/qemu into staging

Browse Source 
* Bump Avocado to version 103
* Introduce new functional test framework for Python-based tests
* Convert many Avocado tests to the new functional test framework

# -----BEGIN PGP SIGNATURE-----
#
# iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmbYOEsRHHRodXRoQHJl
# ZGhhdC5jb20ACgkQLtnXdP5wLbUDAA/+Kdlak/nCrK5gXDyDasfy86IxgMD0QlDR
# U0MOpQyfXbM2EJjwCUhmgo8pui8qV23dKzfCwbDmkjB7mJ+yKi2ZdiFEp6onq/ke
# aAdaaZwENtWcFglRD80TOSQX6oyeNmE/PuvJGG0BfwWXyyhaEa6kCdytEPORipQs
# lZ+ZndHgXtcM3roXtgI3kp2V1nY5LLCJ044UrasKRq2xWfD/Ken90uWP5/nMLV7f
# 7YLRUIb0sgV7IdjZiT1UkXJZRB7MatV7+OsojYbG8BPbQEvXqpryXMIeygHVR9a0
# yxNDUpTZR6JoS1IaLKkHh1mTM+L1JpFltKadKkXa0zqJHHSur7Tp0xVO/GeqCek4
# 9N8K4zw2CoO/AKmN8JjW5i4GnMrFMdcvxxNwLdRoVgYt4YA731wnHrbosXZOXcuv
# H0z8Tm6ueKvfBtrQErdvqsGrP/8FUYRqZP4H6XaaC+wEis++7OmVR2nlQ/gAyr6/
# mMJtmxqVHCIcEVjDu1jYltrW3BN2CcxN2M9gxyOScq2/Xmzqtaeb4iyjxeCUjIBW
# Pc4LXlSafIg3hPrdH3EKN275ev8cx/5jp8oEgXD5We25Mj3W930zde6/STXoX318
# NVNlbrIQjGjQN7rN5oxTFxTlIN8ax2tuuzpQDFvS/4bLyMYXcZ4I5gUrM5tvWTGv
# +0UN45pJ7Nk=
# =l6Ki
# -----END PGP SIGNATURE-----
# gpg: Signature made Wed 04 Sep 2024 11:36:59 BST
# gpg:                using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5
# gpg:                issuer "thuth@redhat.com"
# gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [full]
# gpg:                 aka "Thomas Huth <thuth@redhat.com>" [full]
# gpg:                 aka "Thomas Huth <huth@tuxfamily.org>" [full]
# gpg:                 aka "Thomas Huth <th.huth@posteo.de>" [unknown]
# Primary key fingerprint: 27B8 8847 EEE0 2501 18F3  EAB9 2ED9 D774 FE70 2DB5

* tag 'pull-request-2024-09-04' of https://gitlab.com/thuth/qemu: (42 commits)
  docs/devel/testing: Add documentation for functional tests
  docs/devel/testing: Rename avocado_qemu.Test class
  docs/devel/testing: Split the Avocado documentation into a separate file
  docs/devel: Split testing docs from the build docs and move to separate folder
  gitlab-ci: Add "check-functional" to the build tests
  tests/avocado: Remove unused QemuUserTest class
  tests/functional: Convert ARM bFLT linux-user avocado test
  tests/functional: Add QemuUserTest class
  tests/functional: Convert mips64el Fuloong2e avocado test (1/2)
  tests/functional: Convert Aarch64 Virt machine avocado tests
  tests/functional: Convert Aarch64 SBSA-Ref avocado tests
  tests/functional: Convert ARM Integrator/CP avocado tests
  tests/functional: Convert the linux_initrd avocado test into a standalone test
  tests/functional: Convert the rx_gdbsim avocado test into a standalone test
  tests/functional: Convert the acpi-bits test into a standalone test
  tests/functional: Convert the m68k nextcube test with tesseract
  tests/functional: Convert the ppc_hv avocado test into a standalone test
  tests/functional: Convert the ppc_amiga avocado test into a standalone test
  tests/functional: Convert most ppc avocado tests into standalone tests
  tests/functional: Convert the virtio_gpu avocado test into a standalone test
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell
2024-09-05 18:01:51 +01:00
parent 7b87a25f49 c3e24cff2b
commit eabebca69b
83 changed files with 2915 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitlab-ci.d/buildtest-template.yml
+11 -3
View File
@@ -59,6 +59,10 @@
- cd build
- find . -type f -exec touch {} +
# Avoid recompiling by hiding ninja with NINJA=":"
# We also have to pre-cache the functional tests manually in this case
- if [ "x${QEMU_TEST_CACHE_DIR}" != "x" ]; then
$MAKE precache-functional ;
fi
- $MAKE NINJA=":" $MAKE_CHECK_ARGS
.native_test_job_template:
@@ -72,12 +76,13 @@
reports:
junit: build/meson-logs/testlog.junit.xml
.avocado_test_job_template:
.functional_test_job_template:
extends: .common_test_job_template
cache:
key: "${CI_JOB_NAME}-cache"
paths:
- ${CI_PROJECT_DIR}/avocado-cache
- ${CI_PROJECT_DIR}/functional-cache
policy: pull-push
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
@@ -86,6 +91,7 @@
paths:
- build/tests/results/latest/results.xml
- build/tests/results/latest/test-results
- build/tests/functional/*/*/*.log
reports:
junit: build/tests/results/latest/results.xml
before_script:
@@ -96,11 +102,13 @@
- echo -e '[job.output.testlogs]\nstatuses = ["FAIL", "INTERRUPT"]'
>> ~/.config/avocado/avocado.conf
- if [ -d ${CI_PROJECT_DIR}/avocado-cache ]; then
du -chs ${CI_PROJECT_DIR}/avocado-cache ;
du -chs ${CI_PROJECT_DIR}/*-cache ;
fi
- export AVOCADO_ALLOW_UNTRUSTED_CODE=1
- export QEMU_TEST_ALLOW_UNTRUSTED_CODE=1
- export QEMU_TEST_CACHE_DIR=${CI_PROJECT_DIR}/functional-cache
after_script:
- cd build
- du -chs ${CI_PROJECT_DIR}/avocado-cache
- du -chs ${CI_PROJECT_DIR}/*-cache
variables:
QEMU_JOB_AVOCADO: 1
.gitlab-ci.d/buildtest.yml
+30 -30
View File
@@ -22,14 +22,14 @@ check-system-alpine:
IMAGE: alpine
MAKE_CHECK_ARGS: check-unit check-qtest
avocado-system-alpine:
extends: .avocado_test_job_template
functional-system-alpine:
extends: .functional_test_job_template
needs:
- job: build-system-alpine
artifacts: true
variables:
IMAGE: alpine
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:avr arch:loongarch64 arch:mips64 arch:mipsel
build-system-ubuntu:
@@ -53,14 +53,14 @@ check-system-ubuntu:
IMAGE: ubuntu2204
MAKE_CHECK_ARGS: check
avocado-system-ubuntu:
extends: .avocado_test_job_template
functional-system-ubuntu:
extends: .functional_test_job_template
needs:
- job: build-system-ubuntu
artifacts: true
variables:
IMAGE: ubuntu2204
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:alpha arch:microblazeel arch:mips64el
build-system-debian:
@@ -85,14 +85,14 @@ check-system-debian:
IMAGE: debian
MAKE_CHECK_ARGS: check
avocado-system-debian:
extends: .avocado_test_job_template
functional-system-debian:
extends: .functional_test_job_template
needs:
- job: build-system-debian
artifacts: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:arm arch:i386 arch:riscv64 arch:sh4 arch:sparc arch:xtensa
crash-test-debian:
@@ -129,14 +129,14 @@ check-system-fedora:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-system-fedora:
extends: .avocado_test_job_template
functional-system-fedora:
extends: .functional_test_job_template
needs:
- job: build-system-fedora
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:microblaze arch:mips arch:xtensa arch:m68k
arch:riscv32 arch:ppc arch:sparc64
@@ -243,14 +243,14 @@ check-system-centos:
IMAGE: centos9
MAKE_CHECK_ARGS: check
avocado-system-centos:
extends: .avocado_test_job_template
functional-system-centos:
extends: .functional_test_job_template
needs:
- job: build-system-centos
artifacts: true
variables:
IMAGE: centos9
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:ppc64 arch:or1k arch:s390x arch:x86_64 arch:rx
arch:sh4
@@ -274,14 +274,14 @@ check-system-opensuse:
IMAGE: opensuse-leap
MAKE_CHECK_ARGS: check
avocado-system-opensuse:
extends: .avocado_test_job_template
functional-system-opensuse:
extends: .functional_test_job_template
needs:
- job: build-system-opensuse
artifacts: true
variables:
IMAGE: opensuse-leap
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
AVOCADO_TAGS: arch:s390x arch:x86_64 arch:aarch64
#
@@ -302,15 +302,15 @@ build-system-flaky:
ppc64-softmmu rx-softmmu s390x-softmmu sh4-softmmu x86_64-softmmu
MAKE_CHECK_ARGS: check-build
avocado-system-flaky:
extends: .avocado_test_job_template
functional-system-flaky:
extends: .functional_test_job_template
needs:
- job: build-system-flaky
artifacts: true
allow_failure: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
QEMU_JOB_OPTIONAL: 1
QEMU_TEST_FLAKY_TESTS: 1
AVOCADO_TAGS: flaky
@@ -485,14 +485,14 @@ check-cfi-aarch64:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-cfi-aarch64:
extends: .avocado_test_job_template
functional-cfi-aarch64:
extends: .functional_test_job_template
needs:
- job: build-cfi-aarch64
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
build-cfi-ppc64-s390x:
extends:
@@ -523,14 +523,14 @@ check-cfi-ppc64-s390x:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-cfi-ppc64-s390x:
extends: .avocado_test_job_template
functional-cfi-ppc64-s390x:
extends: .functional_test_job_template
needs:
- job: build-cfi-ppc64-s390x
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
build-cfi-x86_64:
extends:
@@ -557,14 +557,14 @@ check-cfi-x86_64:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-cfi-x86_64:
extends: .avocado_test_job_template
functional-cfi-x86_64:
extends: .functional_test_job_template
needs:
- job: build-cfi-x86_64
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
MAKE_CHECK_ARGS: check-avocado check-functional
tsan-build:
extends: .native_build_job_template
MAINTAINERS
+40 -23
View File
@@ -222,7 +222,7 @@ S: Maintained
F: docs/system/target-avr.rst
F: gdb-xml/avr-cpu.xml
F: target/avr/
F: tests/avocado/machine_avr6.py
F: tests/functional/test_avr_mega2560.py
CRIS TCG CPUs
M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -266,7 +266,7 @@ M: Song Gao <gaosong@loongson.cn>
S: Maintained
F: target/loongarch/
F: tests/tcg/loongarch64/
F: tests/avocado/machine_loongarch.py
F: tests/functional/test_loongarch64_virt.py
M68K TCG CPUs
M: Laurent Vivier <laurent@vivier.eu>
@@ -318,6 +318,7 @@ F: configs/devices/ppc*
F: docs/system/ppc/embedded.rst
F: docs/system/target-ppc.rst
F: tests/tcg/ppc*/*
F: tests/functional/test_ppc_74xx.py
RISC-V TCG CPUs
M: Palmer Dabbelt <palmer@dabbelt.com>
@@ -734,7 +735,7 @@ S: Odd Fixes
F: include/hw/arm/digic.h
F: hw/*/digic*
F: include/hw/*/digic*
F: tests/avocado/machine_arm_canona1100.py
F: tests/functional/test_arm_canona1100.py
F: docs/system/arm/digic.rst
Goldfish RTC
@@ -785,7 +786,7 @@ S: Maintained
F: hw/arm/integratorcp.c
F: hw/misc/arm_integrator_debug.c
F: include/hw/misc/arm_integrator_debug.h
F: tests/avocado/machine_arm_integratorcp.py
F: tests/functional/test_arm_integratorcp.py
F: docs/system/arm/integratorcp.rst
MCIMX6UL EVK / i.MX6ul
@@ -971,7 +972,7 @@ F: hw/misc/sbsa_ec.c
F: hw/watchdog/sbsa_gwdt.c
F: include/hw/watchdog/sbsa_gwdt.h
F: docs/system/arm/sbsa.rst
F: tests/avocado/machine_aarch64_sbsaref.py
F: tests/functional/test_aarch64_sbsaref.py
Sharp SL-5500 (Collie) PDA
M: Peter Maydell <peter.maydell@linaro.org>
@@ -1023,7 +1024,7 @@ S: Maintained
F: hw/arm/virt*
F: include/hw/arm/virt.h
F: docs/system/arm/virt.rst
F: tests/avocado/machine_aarch64_virt.py
F: tests/functional/test_aarch64_virt.py
Xilinx Zynq
M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -1283,6 +1284,7 @@ S: Odd Fixes
F: hw/m68k/next-*.c
F: hw/display/next-fb.c
F: include/hw/m68k/next-cube.h
F: tests/functional/test_m68k_nextcube.py
q800
M: Laurent Vivier <laurent@vivier.eu>
@@ -1330,7 +1332,7 @@ M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
S: Maintained
F: hw/microblaze/petalogix_s3adsp1800_mmu.c
F: include/hw/char/xilinx_uartlite.h
F: tests/avocado/machine_microblaze.py
F: tests/functional/test_microblaze*.py
petalogix_ml605
M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -1383,7 +1385,7 @@ S: Odd Fixes
F: hw/mips/fuloong2e.c
F: hw/pci-host/bonito.c
F: include/hw/pci-host/bonito.h
F: tests/avocado/machine_mips_fuloong2e.py
F: tests/functional/test_mips64el_fuloong2e.py
Loongson-3 virtual platforms
M: Huacai Chen <chenhuacai@kernel.org>
@@ -1398,7 +1400,7 @@ F: hw/mips/loongson3_virt.c
F: include/hw/intc/loongson_ipi_common.h
F: include/hw/intc/loongson_ipi.h
F: include/hw/intc/loongson_liointc.h
F: tests/avocado/machine_mips_loongson3v.py
F: tests/functional/test_mips64el_loongson3v.py
Boston
M: Paul Burton <paulburton@kernel.org>
@@ -1424,14 +1426,14 @@ PowerPC Machines
L: qemu-ppc@nongnu.org
S: Orphan
F: hw/ppc/ppc405*
F: tests/avocado/ppc_405.py
F: tests/functional/test_ppc_405.py
Bamboo
L: qemu-ppc@nongnu.org
S: Orphan
F: hw/ppc/ppc440_bamboo.c
F: hw/pci-host/ppc4xx_pci.c
F: tests/avocado/ppc_bamboo.py
F: tests/functional/test_ppc_bamboo.py
e500
L: qemu-ppc@nongnu.org
@@ -1454,7 +1456,7 @@ L: qemu-ppc@nongnu.org
S: Orphan
F: hw/ppc/mpc8544ds.c
F: hw/ppc/mpc8544_guts.c
F: tests/avocado/ppc_mpc8544ds.py
F: tests/functional/test_ppc_mpc8544ds.py
New World (mac99)
M: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk>
@@ -1507,7 +1509,7 @@ F: hw/dma/i82374.c
F: hw/rtc/m48t59-isa.c
F: include/hw/isa/pc87312.h
F: include/hw/rtc/m48t59.h
F: tests/avocado/ppc_prep_40p.py
F: tests/functional/test_ppc_40p.py
sPAPR (pseries)
M: Nicholas Piggin <npiggin@gmail.com>
@@ -1531,8 +1533,8 @@ F: tests/qtest/spapr*
F: tests/qtest/libqos/*spapr*
F: tests/qtest/rtas*
F: tests/qtest/libqos/rtas*
F: tests/avocado/ppc_pseries.py
F: tests/avocado/ppc_hv_tests.py
F: tests/functional/test_ppc64_pseries.py
F: tests/functional/test_ppc64_hv.py
PowerNV (Non-Virtualized)
M: Cédric Le Goater <clg@kaod.org>
@@ -1549,6 +1551,7 @@ F: include/hw/ppc/pnv*
F: include/hw/pci-host/pnv*
F: pc-bios/skiboot.lid
F: tests/qtest/pnv*
F: tests/functional/test_ppc64_powernv.py
pca955x
M: Glenn Miles <milesg@linux.ibm.com>
@@ -1563,7 +1566,7 @@ M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
L: qemu-ppc@nongnu.org
S: Odd Fixes
F: hw/ppc/virtex_ml507.c
F: tests/avocado/ppc_virtex_ml507.py
F: tests/functional/test_ppc_virtex_ml507.py
sam460ex
M: BALATON Zoltan <balaton@eik.bme.hu>
@@ -1596,6 +1599,7 @@ S: Maintained
F: hw/ppc/amigaone.c
F: hw/pci-host/articia.c
F: include/hw/pci-host/articia.h
F: tests/functional/test_ppc_amiga.py
Virtual Open Firmware (VOF)
M: Alexey Kardashevskiy <aik@ozlabs.ru>
@@ -1662,7 +1666,7 @@ R: Yoshinori Sato <ysato@users.sourceforge.jp>
S: Orphan
F: docs/system/target-rx.rst
F: hw/rx/rx-gdbsim.c
F: tests/avocado/machine_rx_gdbsim.py
F: tests/functional/test_rx_gdbsim.py
SH4 Machines
------------
@@ -1717,7 +1721,7 @@ F: include/hw/pci-host/sabre.h
F: hw/pci-bridge/simba.c
F: include/hw/pci-bridge/simba.h
F: pc-bios/openbios-sparc64
F: tests/avocado/machine_sparc64_sun4u.py
F: tests/functional/test_sparc64_sun4u.py
Sun4v
M: Artyom Tarasenko <atar4qemu@gmail.com>
@@ -1744,7 +1748,7 @@ S: Supported
F: hw/s390x/
F: include/hw/s390x/
F: configs/devices/s390x-softmmu/default.mak
F: tests/avocado/machine_s390_ccw_virtio.py
F: tests/functional/test_s390x_ccw_virtio.py
T: git https://github.com/borntraeger/qemu.git s390-next
L: qemu-s390x@nongnu.org
@@ -1807,7 +1811,7 @@ F: hw/s390x/cpu-topology.c
F: target/s390x/kvm/stsi-topology.c
F: docs/devel/s390-cpu-topology.rst
F: docs/system/s390x/cpu-topology.rst
F: tests/avocado/s390_topology.py
F: tests/functional/test_s390x_topology.py
X86 Machines
------------
@@ -1835,6 +1839,9 @@ F: hw/isa/apm.c
F: include/hw/isa/apm.h
F: tests/unit/test-x86-topo.c
F: tests/qtest/test-x86-cpuid-compat.c
F: tests/functional/test_mem_addr_space.py
F: tests/functional/test_pc_cpu_hotplug_props.py
F: tests/functional/test_x86_cpu_model_versions.py
PC Chipset
M: Michael S. Tsirkin <mst@redhat.com>
@@ -1901,6 +1908,8 @@ F: include/hw/boards.h
F: include/hw/core/cpu.h
F: include/hw/cpu/cluster.h
F: include/sysemu/numa.h
F: tests/functional/test_cpu_queries.py
F: tests/functional/test_empty_cpu_model.py
F: tests/unit/test-smp-parse.c
T: git https://gitlab.com/ehabkost/qemu.git machine-next
@@ -2067,8 +2076,8 @@ ACPI/AVOCADO/BIOSBITS
M: Ani Sinha <anisinha@redhat.com>
M: Michael S. Tsirkin <mst@redhat.com>
S: Supported
F: tests/avocado/acpi-bits/*
F: tests/avocado/acpi-bits.py
F: tests/functional/acpi-bits/*
F: tests/functional/test_acpi_bits.py
F: docs/devel/acpi-bits.rst
ACPI/HEST/GHES
@@ -2105,6 +2114,7 @@ S: Odd Fixes
F: hw/net/
F: include/hw/net/
F: tests/qtest/virtio-net-test.c
F: tests/functional/test_info_usernet.py
F: docs/virtio-net-failover.rst
T: git https://github.com/jasowang/qemu.git net
@@ -2240,6 +2250,7 @@ F: net/vhost-user.c
F: include/hw/virtio/
F: docs/devel/virtio*
F: docs/devel/migration/virtio.rst
F: tests/functional/test_virtio_version.py
virtio-balloon
M: Michael S. Tsirkin <mst@redhat.com>
@@ -2490,7 +2501,7 @@ R: Sriram Yagnaraman <sriram.yagnaraman@ericsson.com>
S: Maintained
F: docs/system/devices/igb.rst
F: hw/net/igb*
F: tests/avocado/netdev-ethtool.py
F: tests/functional/test_netdev_ethtool.py
F: tests/qtest/igb-test.c
F: tests/qtest/libqos/igb.c
@@ -2973,6 +2984,7 @@ S: Supported
F: include/qemu/option.h
F: tests/unit/test-keyval.c
F: tests/unit/test-qemu-opts.c
F: tests/functional/test_version.py
F: util/keyval.c
F: util/qemu-option.c
@@ -4148,6 +4160,11 @@ F: .gitlab-ci.d/cirrus/freebsd*
F: tests/vm/freebsd
W: https://cirrus-ci.com/github/qemu/qemu
Functional testing framework
M: Thomas Huth <thuth@redhat.com>
R: Philippe Mathieu-Daudé <philmd@linaro.org>
F: tests/functional/qemu_test/
Windows Hosted Continuous Integration
M: Yonggang Luo <luoyonggang@gmail.com>
S: Maintained
docs/devel/index-build.rst
+4 -10
View File
@@ -1,9 +1,8 @@
QEMU Build and Test System
--------------------------
QEMU Build System
-----------------
Details about how QEMU's build system works and how it is integrated
into our testing infrastructure. You will need to understand some of
the basics if you are adding new files and targets to the build.
Details about how QEMU's build system works. You will need to understand
some of the basics if you are adding new files and targets to the build.
.. toctree::
:maxdepth: 3
@@ -11,10 +10,5 @@ the basics if you are adding new files and targets to the build.
build-system
kconfig
docs
testing
acpi-bits
qtest
ci
qapi-code-gen
fuzzing
control-flow-integrity
docs/devel/index.rst
+1
View File
@@ -31,6 +31,7 @@ the :ref:`tcg_internals`.
index-process
index-build
testing/index
index-api
index-internals
index-tcg
docs/devel/acpi-bits.rst → docs/devel/testing/acpi-bits.rst
+25 -37
View File
@@ -1,6 +1,6 @@
=============================================================================
ACPI/SMBIOS avocado tests using biosbits
=============================================================================
==================================
ACPI/SMBIOS testing using biosbits
==================================
************
Introduction
************
@@ -35,7 +35,7 @@ for developing biosbits and its real life uses can be found in [#a]_ and [#b]_.
For QEMU, we maintain a fork of bios bits in gitlab along with all the
dependent submodules `here <https://gitlab.com/qemu-project/biosbits-bits>`__.
This fork contains numerous fixes, a newer acpica and changes specific to
running this avocado QEMU tests using bits. The author of this document
running these functional QEMU tests using bits. The author of this document
is the sole maintainer of the QEMU fork of bios bits repository. For more
information, please see author's `FOSDEM talk on this bios-bits based test
framework <https://fosdem.org/2024/schedule/event/fosdem-2024-2262-exercising-qemu-generated-acpi-smbios-tables-using-biosbits-from-within-a-guest-vm-/>`__.
@@ -44,12 +44,12 @@ framework <https://fosdem.org/2024/schedule/event/fosdem-2024-2262-exercising-qe
Description of the test framework
*********************************
Under the directory ``tests/avocado/``, ``acpi-bits.py`` is a QEMU avocado
test that drives all this.
Under the directory ``tests/functional/``, ``test_acpi_bits.py`` is a QEMU
functional test that drives all this.
A brief description of the various test files follows.
Under ``tests/avocado/`` as the root we have:
Under ``tests/functional/`` as the root we have:
::
@@ -60,12 +60,12 @@ Under ``tests/avocado/`` as the root we have:
│ ├── smbios.py2
│ ├── testacpi.py2
│ └── testcpuid.py2
├── acpi-bits.py
├── test_acpi_bits.py
* ``tests/avocado``:
* ``tests/functional``:
``acpi-bits.py``:
This is the main python avocado test script that generates a
``test_acpi_bits.py``:
This is the main python functional test script that generates a
biosbits iso. It then spawns a QEMU VM with it, collects the log and reports
test failures. This is the script one would be interested in if they wanted
to add or change some component of the log parsing, add a new command line
@@ -79,35 +79,22 @@ Under ``tests/avocado/`` as the root we have:
you to inspect and run the specific commands manually.
In order to run this test, please perform the following steps from the QEMU
build directory:
build directory (assuming that the sources are in ".."):
::
$ make check-venv (needed only the first time to create the venv)
$ ./pyvenv/bin/avocado run -t acpi tests/avocado
$ export PYTHONPATH=../python:../tests/functional
$ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
$ python3 ../tests/functional/test_acpi_bits.py
The above will run all acpi avocado tests including this one.
In order to run the individual tests, perform the following:
::
The above will run all acpi-bits functional tests (producing output in
tap format).
$ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py --tap -
You can inspect the log files in tests/functional/x86_64/test_acpi_bits.*/
for more information about the run or in order to diagnoze issues.
If you pass V=1 in the environment, more diagnostic logs will be put into
the test log.
The above will produce output in tap format. You can omit "--tap -" in the
end and it will produce output like the following:
::
$ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py
Fetching asset from tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits
JOB ID : eab225724da7b64c012c65705dc2fa14ab1defef
JOB LOG : /home/anisinha/avocado/job-results/job-2022-10-10T17.58-eab2257/job.log
(1/1) tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits: PASS (33.09 s)
RESULTS : PASS 1 | ERROR 0 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0
JOB TIME : 39.22 s
You can inspect the log file for more information about the run or in order
to diagnoze issues. If you pass V=1 in the environment, more diagnostic logs
would be found in the test log.
* ``tests/avocado/acpi-bits/bits-config``:
* ``tests/functional/acpi-bits/bits-config``:
This location contains biosbits configuration files that determine how the
software runs the tests.
@@ -117,7 +104,7 @@ Under ``tests/avocado/`` as the root we have:
or actions are performed by bits. The description of the config options are
provided in the file itself.
* ``tests/avocado/acpi-bits/bits-tests``:
* ``tests/functional/acpi-bits/bits-tests``:
This directory contains biosbits python based tests that are run from within
the biosbits environment in the spawned VM. New additions of test cases can
@@ -155,7 +142,8 @@ Under ``tests/avocado/`` as the root we have:
(a) They are python2.7 based scripts and not python 3 scripts.
(b) They are run from within the bios bits VM and is not subjected to QEMU
build/test python script maintenance and dependency resolutions.
(c) They need not be loaded by avocado framework when running tests.
(c) They need not be loaded by the test framework by accident when running
tests.
Author: Ani Sinha <anisinha@redhat.com>
docs/devel/testing/avocado.rst
+581
View File
File diff suppressed because it is too large Load Diff
docs/devel/ci-definitions.rst.inc → docs/devel/testing/ci-definitions.rst.inc
View File
docs/devel/ci-jobs.rst.inc → docs/devel/testing/ci-jobs.rst.inc
View File
docs/devel/ci-runners.rst.inc → docs/devel/testing/ci-runners.rst.inc
View File
docs/devel/ci.rst → docs/devel/testing/ci.rst
View File
docs/devel/testing/functional.rst
+338
View File
@@ -0,0 +1,338 @@
.. _checkfunctional-ref:
Functional testing with Python
==============================
The ``tests/functional`` directory hosts functional tests written in
Python. They are usually higher level tests, and may interact with
external resources and with various guest operating systems.
The functional tests have initially evolved from the Avocado tests, so there
is a lot of similarity to those tests here (see :ref:`checkavocado-ref` for
details about the Avocado tests).
The tests should be written in the style of the Python `unittest`_ framework,
using stdio for the TAP protocol. The folder ``tests/functional/qemu_test``
provides classes (e.g. the ``QemuBaseTest``, ``QemuUserTest`` and the
``QemuSystemTest`` classes) and utility functions that help to get your test
into the right shape, e.g. by replacing the 'stdout' python object to redirect
the normal output of your test to stderr instead.
Note that if you don't use one of the QemuBaseTest based classes for your
test, or if you spawn subprocesses from your test, you have to make sure
that there is no TAP-incompatible output written to stdio, e.g. either by
prefixing every line with a "# " to mark the output as a TAP comment, or
e.g. by capturing the stdout output of subprocesses (redirecting it to
stderr is OK).
Tests based on ``qemu_test.QemuSystemTest`` can easily:
* Customize the command line arguments given to the convenience
``self.vm`` attribute (a QEMUMachine instance)
* Interact with the QEMU monitor, send QMP commands and check
their results
* Interact with the guest OS, using the convenience console device
(which may be useful to assert the effectiveness and correctness of
command line arguments or QMP commands)
* Download (and cache) remote data files, such as firmware and kernel
images
Running tests
-------------
You can run the functional tests simply by executing:
.. code::
make check-functional
It is also possible to run tests for a certain target only, for example
the following line will only run the tests for the x86_64 target:
.. code::
make check-functional-x86_64
To run a single test file without the meson test runner, you can also
execute the file directly by specifying two environment variables first,
the PYTHONPATH that has to include the python folder and the tests/functional
folder of the source tree, and QEMU_TEST_QEMU_BINARY that has to point
to the QEMU binary that should be used for the test, for example::
$ export PYTHONPATH=../python:../tests/functional
$ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
$ python3 ../tests/functional/test_file.py
Overview
--------
The ``tests/functional/qemu_test`` directory provides the ``qemu_test``
Python module, containing the ``qemu_test.QemuSystemTest`` class.
Here is a simple usage example:
.. code::
#!/usr/bin/env python3
from qemu_test import QemuSystemTest
class Version(QemuSystemTest):
def test_qmp_human_info_version(self):
self.vm.launch()
res = self.vm.cmd('human-monitor-command',
command_line='info version')
self.assertRegex(res, r'^(\d+\.\d+\.\d)')
if __name__ == '__main__':
QemuSystemTest.main()
By providing the "hash bang" line at the beginning of the script, marking
the file as executable and by calling into QemuSystemTest.main(), the test
can also be run stand-alone, without a test runner. OTOH when run via a test
runner, the QemuSystemTest.main() function takes care of running the test
functions in the right fassion (e.g. with TAP output that is required by the
meson test runner).
The ``qemu_test.QemuSystemTest`` base test class
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The ``qemu_test.QemuSystemTest`` class has a number of characteristics
that are worth being mentioned.
First of all, it attempts to give each test a ready to use QEMUMachine
instance, available at ``self.vm``. Because many tests will tweak the
QEMU command line, launching the QEMUMachine (by using ``self.vm.launch()``)
is left to the test writer.
The base test class has also support for tests with more than one
QEMUMachine. The way to get machines is through the ``self.get_vm()``
method which will return a QEMUMachine instance. The ``self.get_vm()``
method accepts arguments that will be passed to the QEMUMachine creation
and also an optional ``name`` attribute so you can identify a specific
machine and get it more than once through the tests methods. A simple
and hypothetical example follows:
.. code::
from qemu_test import QemuSystemTest
class MultipleMachines(QemuSystemTest):
def test_multiple_machines(self):
first_machine = self.get_vm()
second_machine = self.get_vm()
self.get_vm(name='third_machine').launch()
first_machine.launch()
second_machine.launch()
first_res = first_machine.cmd(
'human-monitor-command',
command_line='info version')
second_res = second_machine.cmd(
'human-monitor-command',
command_line='info version')
third_res = self.get_vm(name='third_machine').cmd(
'human-monitor-command',
command_line='info version')
self.assertEqual(first_res, second_res, third_res)
At test "tear down", ``qemu_test.QemuSystemTest`` handles all the QEMUMachines
shutdown.
QEMUMachine
-----------
The QEMUMachine API is already widely used in the Python iotests,
device-crash-test and other Python scripts. It's a wrapper around the
execution of a QEMU binary, giving its users:
* the ability to set command line arguments to be given to the QEMU
binary
* a ready to use QMP connection and interface, which can be used to
send commands and inspect its results, as well as asynchronous
events
* convenience methods to set commonly used command line arguments in
a more succinct and intuitive way
QEMU binary selection
^^^^^^^^^^^^^^^^^^^^^
The QEMU binary used for the ``self.vm`` QEMUMachine instance will
primarily depend on the value of the ``qemu_bin`` class attribute.
If it is not explicitly set by the test code, its default value will
be the result the QEMU_TEST_QEMU_BINARY environment variable.
Attribute reference
-------------------
QemuBaseTest
^^^^^^^^^^^^
The following attributes are available on any ``qemu_test.QemuBaseTest``
instance.
arch
""""
The target architecture of the QEMU binary.
Tests are also free to use this attribute value, for their own needs.
A test may, for instance, use this value when selecting the architecture
of a kernel or disk image to boot a VM with.
qemu_bin
""""""""
The preserved value of the ``QEMU_TEST_QEMU_BINARY`` environment
variable.
QemuUserTest
^^^^^^^^^^^^
The QemuUserTest class can be used for running an executable via the
usermode emulation binaries.
QemuSystemTest
^^^^^^^^^^^^^^
The QemuSystemTest class can be used for running tests via one of the
qemu-system-* binaries.
vm
""
A QEMUMachine instance, initially configured according to the given
``qemu_bin`` parameter.
cpu
"""
The cpu model that will be set to all QEMUMachine instances created
by the test.
machine
"""""""
The machine type that will be set to all QEMUMachine instances created
by the test. By using the set_machine() function of the QemuSystemTest
class to set this attribute, you can automatically check whether the
machine is available to skip the test in case it is not built into the
QEMU binary.
Asset handling
--------------
Many functional tests download assets (e.g. Linux kernels, initrds,
firmware images, etc.) from the internet to be able to run tests with
them. This imposes additional challenges to the test framework.
First there is the the problem that some people might not have an
unconstrained internet connection, so such tests should not be run by
default when running ``make check``. To accomplish this situation,
the tests that download files should only be added to the "thorough"
speed mode in the meson.build file, while the "quick" speed mode is
fine for functional tests that can be run without downloading files.
``make check`` then only runs the quick functional tests along with
the other quick tests from the other test suites. If you choose to
run only run ``make check-functional``, the "thorough" tests will be
executed, too. And to run all functional tests along with the others,
you can use something like::
make -j$(nproc) check SPEED=thorough
The second problem with downloading files from the internet are time
constraints. The time for downloading files should not be taken into
account when the test is running and the timeout of the test is ticking
(since downloading can be very slow, depending on the network bandwidth).
This problem is solved by downloading the assets ahead of time, before
the tests are run. This pre-caching is done with the qemu_test.Asset
class. To use it in your test, declare an asset in your test class with
its URL and SHA256 checksum like this::
ASSET_somename = (
('https://www.qemu.org/assets/images/qemu_head_200.png'),
'34b74cad46ea28a2966c1d04e102510daf1fd73e6582b6b74523940d5da029dd')
In your test function, you can then get the file name of the cached
asset like this::
def test_function(self):
file_path = self.ASSET_somename.fetch()
The pre-caching will be done automatically when running
``make check-functional`` (but not when running e.g.
``make check-functional-<target>``). In case you just want to download
the assets without running the tests, you can do so by running::
make precache-functional
The cache is populated in the ``~/.cache/qemu/download`` directory by
default, but the location can be changed by setting the
``QEMU_TEST_CACHE_DIR`` environment variable.
Skipping tests
--------------
Since the test framework is based on the common Python unittest framework,
you can use the usual Python decorators which allow for easily skipping
tests running under certain conditions, for example, on the lack of a binary
on the test system or when the running environment is a CI system. For further
information about those decorators, please refer to:
https://docs.python.org/3/library/unittest.html#skipping-tests-and-expected-failures
While the conditions for skipping tests are often specifics of each one, there
are recurring scenarios identified by the QEMU developers and the use of
environment variables became a kind of standard way to enable/disable tests.
Here is a list of the most used variables:
QEMU_TEST_ALLOW_LARGE_STORAGE
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Tests which are going to fetch or produce assets considered *large* are not
going to run unless that ``QEMU_TEST_ALLOW_LARGE_STORAGE=1`` is exported on
the environment.
The definition of *large* is a bit arbitrary here, but it usually means an
asset which occupies at least 1GB of size on disk when uncompressed.
QEMU_TEST_ALLOW_UNTRUSTED_CODE
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are tests which will boot a kernel image or firmware that can be
considered not safe to run on the developer's workstation, thus they are
skipped by default. The definition of *not safe* is also arbitrary but
usually it means a blob which either its source or build process aren't
public available.
You should export ``QEMU_TEST_ALLOW_UNTRUSTED_CODE=1`` on the environment in
order to allow tests which make use of those kind of assets.
QEMU_TEST_FLAKY_TESTS
^^^^^^^^^^^^^^^^^^^^^
Some tests are not working reliably and thus are disabled by default.
This includes tests that don't run reliably on GitLab's CI which
usually expose real issues that are rarely seen on developer machines
due to the constraints of the CI environment. If you encounter a
similar situation then raise a bug and then mark the test as shown on
the code snippet below:
.. code::
# See https://gitlab.com/qemu-project/qemu/-/issues/nnnn
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test(self):
do_something()
Tests should not live in this state forever and should either be fixed
or eventually removed.
.. _unittest: https://docs.python.org/3/library/unittest.html
docs/devel/fuzzing.rst → docs/devel/testing/fuzzing.rst
View File
docs/devel/testing/index.rst
+16
View File
@@ -0,0 +1,16 @@
Testing QEMU
------------
Details about how to test QEMU and how it is integrated into our CI
testing infrastructure.
.. toctree::
:maxdepth: 3
main
qtest
functional
avocado
acpi-bits
ci
fuzzing
docs/devel/testing.rst → docs/devel/testing/main.rst
+13 -564
View File
File diff suppressed because it is too large Load Diff
docs/devel/qgraph.rst → docs/devel/testing/qgraph.rst
View File
docs/devel/qtest.rst → docs/devel/testing/qtest.rst
View File
python/wheels/pycotap-1.3.1-py3-none-any.whl
BIN
View File
Binary file not shown.
pythondeps.toml
+2 -1
View File
@@ -20,6 +20,7 @@
[meson]
# The install key should match the version in python/wheels/
meson = { accepted = ">=1.1.0", installed = "1.2.3", canary = "meson" }
pycotap = { accepted = ">=1.1.0", installed = "1.3.1" }
[docs]
# Please keep the installed versions in sync with docs/requirements.txt
@@ -30,5 +31,5 @@ sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" }
# Note that qemu.git/python/ is always implicitly installed.
# Prefer an LTS version when updating the accepted versions of
# avocado-framework, for example right now the limit is 92.x.
avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" }
avocado-framework = { accepted = "(>=103.0, <104.0)", installed = "103.0", canary = "avocado" }
pycdlib = { accepted = ">=1.11.0" }
tests/Makefile.include
+28 -16
View File
@@ -3,28 +3,30 @@
.PHONY: check-help
check-help:
@echo "Regression testing targets:"
@echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
@echo " $(MAKE) bench Run speed tests"
@echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
@echo " $(MAKE) bench Run speed tests"
@echo
@echo "Individual test suites:"
@echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
@echo " $(MAKE) check-qtest Run qtest tests"
@echo " $(MAKE) check-unit Run qobject tests"
@echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
@echo " $(MAKE) check-block Run block tests"
@echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
@echo " $(MAKE) check-qtest Run qtest tests"
@echo " $(MAKE) check-functional Run python-based functional tests"
@echo " $(MAKE) check-functional-TARGET Run functional tests for a given target"
@echo " $(MAKE) check-unit Run qobject tests"
@echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
@echo " $(MAKE) check-block Run block tests"
ifneq ($(filter $(all-check-targets), check-softfloat),)
@echo " $(MAKE) check-tcg Run TCG tests"
@echo " $(MAKE) check-softfloat Run FPU emulation tests"
@echo " $(MAKE) check-tcg Run TCG tests"
@echo " $(MAKE) check-softfloat Run FPU emulation tests"
endif
@echo " $(MAKE) check-avocado Run avocado (integration) tests for currently configured targets"
@echo " $(MAKE) check-avocado Run avocado (integration) tests for currently configured targets"
@echo
@echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report"
@echo " $(MAKE) check-venv Creates a Python venv for tests"
@echo " $(MAKE) check-clean Clean the tests and related data"
@echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report"
@echo " $(MAKE) check-venv Creates a Python venv for tests"
@echo " $(MAKE) check-clean Clean the tests and related data"
@echo
@echo "The following are useful for CI builds"
@echo " $(MAKE) check-build Build most test binaries"
@echo " $(MAKE) get-vm-images Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)"
@echo " $(MAKE) check-build Build most test binaries"
@echo " $(MAKE) get-vm-images Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)"
@echo
@echo
@echo "The variable SPEED can be set to control the gtester speed setting."
@@ -141,7 +143,7 @@ check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images
--show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
$(if $(AVOCADO_TAGS),, --filter-by-tags-include-empty \
--filter-by-tags-include-empty-key) \
$(AVOCADO_CMDLINE_TAGS) \
$(AVOCADO_CMDLINE_TAGS) --max-parallel-tasks=1 \
$(if $(GITLAB_CI),,--failfast) $(AVOCADO_TESTS), \
"AVOCADO", "tests/avocado")
@@ -152,6 +154,16 @@ check-acceptance-deprecated-warning:
check-acceptance: check-acceptance-deprecated-warning | check-avocado
FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS)))
.PHONY: $(FUNCTIONAL_TARGETS)
$(FUNCTIONAL_TARGETS):
@$(MAKE) SPEED=thorough $(subst -functional,-func,$@)
.PHONY: check-functional
check-functional:
@$(NINJA) precache-functional
@QEMU_TEST_NO_DOWNLOAD=1 $(MAKE) SPEED=thorough check-func check-func-quick
# Consolidated targets
.PHONY: check check-clean get-vm-images

Some files were not shown because too many files have changed in this diff Show More