Merge pull request #32352 from DaanDeMeyer/test

test: Various mkosi integration test improvements
This commit is contained in:
Daan De Meyer
2024-04-23 11:27:23 +02:00
committed by GitHub
12 changed files with 281 additions and 234 deletions

View File

@@ -69,12 +69,9 @@ jobs:
- distro: centos
release: "9"
env:
SYSTEMD_LOG_LEVEL: debug
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
- uses: systemd/mkosi@6ab7d9f09f8f2633f4b7c777a04e62e109486e2f
- uses: systemd/mkosi@8cbde8a4ed20a078ad5c70fe38c0dd2294a68bb1
# Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
# immediately, we remove the files in the background. However, we first move them to a different location
@@ -85,6 +82,18 @@ jobs:
sudo mv /opt/hostedtoolcache /opt/hostedtoolcache.trash
sudo systemd-run rm -rf /usr/local.trash /opt/hostedtoolcache.trash
- name: Btrfs
run: |
truncate --size=100G btrfs.raw
mkfs.btrfs btrfs.raw
sudo mkdir /mnt/mkosi
LOOP="$(sudo losetup --find --show --direct-io=on btrfs.raw)"
sudo mount "$LOOP" /mnt/mkosi --options compress=zstd:1,user_subvol_rm_allowed,noatime,discard=async,space_cache=v2
sudo chown "$(id -u):$(id -g)" /mnt/mkosi
mkdir /mnt/mkosi/tmp
echo "TMPDIR=/mnt/mkosi/tmp" >>"$GITHUB_ENV"
ln -s /mnt/mkosi/build build
- name: Configure
run: |
tee mkosi.local.conf <<EOF
@@ -95,6 +104,19 @@ jobs:
[Output]
# Build a disk image in CI as this logic is much more prone to breakage.
Format=disk
UseSubvolumes=yes
WorkspaceDirectory=$TMPDIR
PackageCacheDirectory=$TMPDIR/cache
[Content]
Environment=
# mkfs.erofs is extremely noisy when not connected to a tty.
SYSTEMD_REPART_MKFS_OPTIONS_EROFS="--quiet"
# Build debuginfo packages since we'll be publishing the packages as artifacts.
WITH_DEBUG=1
# Enabling optimizations significantly speeds up integration tests.
OPTIMIZATION=g
[Host]
ToolsTree=default
@@ -103,47 +125,82 @@ jobs:
QemuKvm=yes
# TODO: Drop once https://bugs.launchpad.net/ubuntu/+source/linux/+bug/2038777 is fixed in Github Actions
QemuFirmware=uefi
Ephemeral=yes
EOF
# These should override the options from mkosi.conf so we put them in a dropin that's ordered later
# instead.
tee mkosi.conf.d/99-ci.conf <<EOF
[Host]
KernelCommandLineExtra=systemd.unit=mkosi-check-and-shutdown.service
systemd.log_level=debug
systemd.journald.max_level_console=debug
# udev's debug log output is very verbose, so up it to info in CI.
udev.log_level=info
# Root device can take a long time to appear, so let's bump the timeout.
systemd.default_device_timeout_sec=180
KernelCommandLineExtra=
# udev's debug log output is very verbose, so up it to info in CI.
udev.log_level=info
# Root device can take a long time to appear, so let's bump the timeout.
systemd.default_device_timeout_sec=180
EOF
# The emergency shell is not useful in the CI, as it just blocks for a long time before the job
# eventually times out. Override it to just shutdown immediately.
mkdir -p mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/
mkdir -p mkosi.images/system/mkosi.extra/usr/lib/systemd/system/emergency.service.d/
tee mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf <<EOF
[Unit]
FailureAction=exit
[Service]
ExecStartPre=
ExecStart=
ExecStart=false
EOF
cp mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf mkosi.images/system/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf
- name: Generate secure boot key
run: mkosi --debug genkey
- name: Show image summary
run: mkosi summary
- name: Build
run: mkosi --debug
- name: Install build dependencies
run: |
sudo apt-get install \
meson \
gperf \
libfdisk-dev \
libtss2-dev \
libblkid-dev \
libmicrohttpd-dev \
libcap-dev \
libcurl4-openssl-dev \
libcryptsetup-dev \
erofs-utils \
dosfstools \
python3-pefile \
sbsigntool \
mtools
- name: Boot systemd-nspawn
run: test "$(sudo mkosi --debug boot 1>&2; echo $?)" -eq 123
- name: Configure meson
run: |
meson setup build \
--buildtype=debugoptimized \
-Dintegration-tests=true \
-Dremote=enabled \
-Dopenssl=enabled \
-Dblkid=enabled \
-Dtpm2=enabled \
-Dlibcryptsetup=enabled \
-Dlibcurl=enabled \
-Drepart=enabled \
-Dfirstboot=true \
-Dsysusers=true \
-Dtmpfiles=true \
-Dhwdb=true \
-Dvmspawn=enabled
- name: Boot QEMU
run: timeout -k 30 10m test "$(mkosi --debug qemu 1>&2; echo $?)" -eq 123
- name: Build image
run: meson compile -C build mkosi
- name: Run integration tests
run: meson test -C build --no-rebuild --suite integration-tests --print-errorlogs --no-stdsplit
- name: Archive failed test journals
uses: actions/upload-artifact@v4
if: failure()
with:
name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-failed-test-journals
path: |
build/test/journal/*.journal
- name: Archive packages
uses: actions/upload-artifact@v4
if: success() || failure()
with:
name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-packages
path: |
build/mkosi.output/*.rpm
build/mkosi.output/*.deb
build/mkosi.output/*.ddeb
build/mkosi.output/*.tar.pkg

View File

@@ -2573,13 +2573,27 @@ endif
#####################################################################
if get_option('integration-tests') != false
system_mkosi = custom_target('system_mkosi',
mkosi = find_program('mkosi', required : false)
if mkosi.found()
custom_target('mkosi',
build_always_stale : true,
output : 'system',
build_by_default: false,
console : true,
command : ['mkosi', '-C', meson.project_source_root(), '--image=system', '--format=disk', '--output-dir', meson.project_build_root() / '@OUTPUT@', '--without-tests', '-fi', 'build'],
depends : [executables_by_name['bootctl'], executables_by_name['systemd-measure'], executables_by_name['systemd-repart'], ukify],
output : '.',
command : [
'mkosi',
'--directory', meson.current_source_dir(),
'--output-dir', meson.current_build_dir() / 'mkosi.output',
'--cache-dir', meson.current_build_dir() / 'mkosi.cache',
'--build-dir', meson.current_build_dir() / 'mkosi.builddir',
'--force',
'build'
],
depends : public_programs + [
executables_by_name['systemd-journal-remote'],
executables_by_name['systemd-measure'],
ukify,
],
)
endif

View File

@@ -5,9 +5,9 @@
MinimumVersion=23~devel
[Output]
@OutputDirectory=mkosi.output
@BuildDirectory=mkosi.builddir
@CacheDirectory=mkosi.cache
@OutputDirectory=build/mkosi.output
@BuildDirectory=build/mkosi.builddir
@CacheDirectory=build/mkosi.cache
[Content]
# Prevent ASAN warnings when building the image and ship the real ASAN options prefixed with MKOSI_.
@@ -20,8 +20,6 @@ BuildSourcesEphemeral=yes
KernelCommandLine=systemd.crash_shell
systemd.log_level=debug,console:info
systemd.log_ratelimit_kmsg=0
systemd.journald.forward_to_console
systemd.journald.max_level_console=warning
# Disable the kernel's ratelimiting on userspace logging to kmsg.
printk.devkmsg=on
# Make sure /sysroot is mounted rw in the initrd.

View File

@@ -34,7 +34,7 @@ SRCDEST="/usr/src/debug/systemd-$VERSION-${RELEASE}${DIST}.$ARCH"
# TODO: Drop -U_FORTIFY_SOURCE when we switch to CentOS Stream 10.
CFLAGS="$(rpm --define "_fortify_level 0" --undefine _lto_cflags --eval %build_cflags) -O${OPTIMIZATION:-0} -Wp,-U_FORTIFY_SOURCE"
if ((WITH_DEBUG)); then
CFLAGS="$CFLAGS -ffile-prefix-map=../src=$SRCDEST"
CFLAGS="$CFLAGS -fdebug-prefix-map=../src=$SRCDEST"
fi
IFS=

View File

@@ -45,7 +45,7 @@ build() {
DEB_BUILD_OPTIONS=$(awk '$1=$1' <<<"\
$( ((WITH_TESTS)) || echo nocheck) \
$( ((WITH_DOCS)) || echo nodoc) \
$( ((WITH_DEBUG)) || echo nostrip) \
$( ((WITH_DEBUG)) && echo debug || echo nostrip) \
terse \
optimize=-lto \
hardening=-fortify \
@@ -100,5 +100,8 @@ if ! build; then
build
fi
cp ../*.deb "$PACKAGEDIR"
cp ../*.deb "$OUTPUTDIR"
(
shopt -s nullglob
cp ../*.deb ../*.ddeb "$PACKAGEDIR"
cp ../*.deb ../*.ddeb "$OUTPUTDIR"
)

View File

@@ -40,7 +40,7 @@ SRCDEST="/usr/src/debug/systemd-$VERSION-${RELEASE}${DIST}.$ARCH"
# EXTRA_CFLAGS="-O${OPTIMIZATION:-0} -Wp,-U_FORTIFY_SOURCE"
EXTRA_CFLAGS=""
if ((WITH_DEBUG)); then
EXTRA_CFLAGS="$EXTRA_CFLAGS -ffile-prefix-map=../src=$SRCDEST"
EXTRA_CFLAGS="$EXTRA_CFLAGS -fdebug-prefix-map=../src=$SRCDEST"
fi
build() {

View File

@@ -1,20 +0,0 @@
#!/bin/bash -eux
# SPDX-License-Identifier: LGPL-2.1-or-later
systemctl --failed --no-legend | tee /failed-services
# Check that secure boot keys were properly enrolled.
if ! systemd-detect-virt --container && \
cmp /sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c <(printf '\6\0\0\0\1')
then
cmp /sys/firmware/efi/efivars/SetupMode-8be4df61-93ca-11d2-aa0d-00e098032b8c <(printf '\6\0\0\0\0')
if command -v sbsign &>/dev/null; then
cat /proc/cmdline
grep -q this_should_be_here /proc/cmdline
(! grep -q this_should_not_be_here /proc/cmdline)
fi
fi
# Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
[[ ! -s /failed-services ]]

View File

@@ -1,15 +0,0 @@
# SPDX-License-Identifier: LGPL-2.1-or-later
[Unit]
Description=Check if any service failed and then shutdown the machine
After=multi-user.target network-online.target
Requires=multi-user.target
Wants=systemd-resolved.service systemd-networkd.service network-online.target
SuccessAction=exit
FailureAction=exit
# On success, exit with 123 so that we can check that we receive the actual exit code from the script on the
# host.
SuccessActionExitStatus=123
[Service]
Type=oneshot
ExecStart=/usr/lib/systemd/mkosi-check-and-shutdown.sh

View File

@@ -33,14 +33,24 @@ enable integration tests and options for required commands with the following:
$ meson configure build -Dintegration-tests=true -Dremote=enabled -Dopenssl=enabled -Dblkid=enabled -Dtpm2=enabled
Once enabled the integration tests can be run with:
Once enabled, first build the integration test image:
$ sudo meson test -C build/ --suite integration-tests --num-processes "$((nproc / 2))"
$ meson compile -C build mkosi
After the image has been built, the integration tests can be run with:
$ meson test -C build/ --suite integration-tests --num-processes "$(($(nproc) / 2))"
As usual, specific tests can be run in meson by appending the name of the test
which is usually the name of the directory e.g.
$ sudo meson test -C build/ --suite integration-tests --num-processes "$((nproc / 2))" TEST-01-BASIC
$ meson test -C build/ -v TEST-01-BASIC
Due to limitations in meson, the integration tests do not yet depend on the mkosi target, which means the
mkosi target has to be manually rebuilt before running the integration tests. To rebuild the image and rerun
a test, the following command can be used:
$ meson compile -C build mkosi && meson test -C build -v TEST-01-BASIC
See `meson introspect build --tests` for a list of tests.

130
test/integration-test-wrapper.py Executable file
View File

@@ -0,0 +1,130 @@
#!/usr/bin/python3
# SPDX-License-Identifier: LGPL-2.1-or-later
'''Test wrapper command for driving integration tests.
Note: This is deliberately rough and only intended to drive existing tests
with the expectation that as part of formally defining the API it will be tidy.
'''
import argparse
import os
import shlex
import subprocess
import sys
import textwrap
from pathlib import Path
EMERGENCY_EXIT_DROPIN = """\
[Unit]
Wants=emergency-exit.service
"""
EMERGENCY_EXIT_SERVICE = """\
[Unit]
DefaultDependencies=no
Conflicts=shutdown.target
Conflicts=rescue.service
Before=shutdown.target
Before=rescue.service
FailureAction=exit
[Service]
ExecStart=false
"""
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--meson-source-dir', required=True, type=Path)
parser.add_argument('--meson-build-dir', required=True, type=Path)
parser.add_argument('--test-name', required=True)
parser.add_argument('--test-number', required=True)
parser.add_argument('mkosi_args', nargs="*")
args = parser.parse_args()
test_unit = f"testsuite-{args.test_number}.service"
dropin = textwrap.dedent(
"""\
[Unit]
After=multi-user.target network.target
Requires=multi-user.target
[Service]
StandardOutput=journal+console
"""
)
if not sys.stderr.isatty():
dropin += textwrap.dedent(
"""
[Unit]
SuccessAction=exit
FailureAction=exit
"""
)
journal_file = (args.meson_build_dir / (f"test/journal/{args.test_name}.journal")).absolute()
journal_file.unlink(missing_ok=True)
else:
journal_file = None
cmd = [
'mkosi',
'--directory', os.fspath(args.meson_source_dir),
'--output-dir', os.fspath(args.meson_build_dir / 'mkosi.output'),
'--extra-search-path', os.fspath(args.meson_build_dir),
'--machine', args.test_name,
'--ephemeral',
*(['--forward-journal', journal_file] if journal_file else []),
*(
[
'--credential',
f"systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}",
'--credential',
f"systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}",
'--kernel-command-line-extra=systemd.mask=serial-getty@.service',
]
if not sys.stderr.isatty()
else []
),
'--credential',
f"systemd.unit-dropin.{test_unit}={shlex.quote(dropin)}",
'--append',
'--kernel-command-line-extra',
' '.join([
'systemd.hostname=H',
f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-{args.test_number}.units:/usr/lib/systemd/tests/testdata/units:",
f"systemd.unit={test_unit}",
]),
*args.mkosi_args,
'qemu',
]
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
if e.returncode != 77 and journal_file:
cmd = [
'journalctl',
'--no-hostname',
'-o', 'short-monotonic',
'--file', journal_file,
'-u', test_unit,
'-p', 'info',
]
print("Test failed, relevant logs can be viewed with: \n\n"
f"{shlex.join(str(a) for a in cmd)}\n", file=sys.stderr)
exit(e.returncode)
# Do not keep journal files for tests that don't fail.
if journal_file:
journal_file.unlink(missing_ok=True)
if __name__ == '__main__':
main()

View File

@@ -1,134 +0,0 @@
#!/usr/bin/python3
# SPDX-License-Identifier: LGPL-2.1-or-later
'''Test wrapper command for driving integration tests.
Note: This is deliberately rough and only intended to drive existing tests
with the expectation that as part of formally defining the API it will be tidy.
'''
import argparse
import logging
import os
from pathlib import Path
import shlex
import subprocess
TEST_EXIT_DROPIN = """\
[Unit]
SuccessAction=exit
FailureAction=exit
"""
EMERGENCY_EXIT_DROPIN = """\
[Unit]
Wants=emergency-exit.service
"""
EMERGENCY_EXIT_SERVICE = """\
[Unit]
DefaultDependencies=no
Conflicts=shutdown.target
Conflicts=rescue.service
Before=shutdown.target
Before=rescue.service
FailureAction=exit
[Service]
ExecStart=false
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--test-name', required=True)
parser.add_argument('--mkosi-image-name', required=True)
parser.add_argument('--mkosi-output-path', required=True, type=Path)
parser.add_argument('--test-number', required=True)
parser.add_argument('--no-emergency-exit',
dest='emergency_exit', default=True, action='store_false',
help="Disable emergency exit drop-ins for interactive debugging")
parser.add_argument('mkosi_args', nargs="*")
def main():
logging.basicConfig(level=logging.DEBUG)
args = parser.parse_args()
test_unit_name = f"testsuite-{args.test_number}.service"
# Machine names shouldn't have / since it's used as a file name
# and it must be a valid hostname so 64 chars max
machine_name = args.test_name.replace('/', '_')[:64]
logging.debug(f"test name: {args.test_name}\n"
f"test number: {args.test_number}\n"
f"image: {args.mkosi_image_name}\n"
f"mkosi output path: {args.mkosi_output_path}\n"
f"mkosi args: {args.mkosi_args}\n"
f"emergency exit: {args.emergency_exit}")
journal_file = Path(f"{machine_name}.journal").absolute()
logging.info(f"Capturing journal to {journal_file}")
mkosi_args = [
'mkosi',
'--directory', Path('..').resolve(),
'--output-dir', args.mkosi_output_path.absolute(),
'--machine', machine_name,
'--image', args.mkosi_image_name,
'--format=disk',
'--runtime-build-sources=no',
'--ephemeral',
'--forward-journal', journal_file,
*(
[
'--credential',
f"systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)} "
f"systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}",
]
if args.emergency_exit
else []
),
f"--credential=systemd.unit-dropin.{test_unit_name}={shlex.quote(TEST_EXIT_DROPIN)}",
'--append',
'--kernel-command-line-extra',
' '.join([
'systemd.hostname=H',
f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-{args.test_number}.units:/usr/lib/systemd/tests/testdata/units:",
'systemd.unit=testsuite.target',
f"systemd.wants={test_unit_name}",
]),
*args.mkosi_args,
]
mkosi_args += ['qemu']
logging.debug(f"Running {shlex.join(os.fspath(a) for a in mkosi_args)}")
try:
subprocess.run(mkosi_args, check=True)
except subprocess.CalledProcessError as e:
if e.returncode not in (0, 77):
suggested_command = [
'journalctl',
'--all',
'--no-hostname',
'-o', 'short-monotonic',
'--file', journal_file,
f"_SYSTEMD_UNIT={test_unit_name}",
'+', f"SYSLOG_IDENTIFIER=testsuite-{args.test_number}.sh",
'+', 'PRIORITY=4',
'+', 'PRIORITY=3',
'+', 'PRIORITY=2',
'+', 'PRIORITY=1',
'+', 'PRIORITY=0',
]
logging.info("Test failed, relevant logs can be viewed with: "
f"{shlex.join(os.fspath(a) for a in suggested_command)}")
exit(e.returncode)
if __name__ == '__main__':
main()

View File

@@ -334,21 +334,19 @@ endif
############################################################
if get_option('integration-tests') != false
integration_test_wrapper = find_program('integration_test_wrapper.py')
if get_option('integration-tests')
if not mkosi.found()
error('Could not find mkosi which is required to run the integration tests')
endif
integration_test_wrapper = find_program('integration-test-wrapper.py')
integration_tests = {
'01': 'TEST-01-BASIC',
'02': 'TEST-02-UNITTESTS',
}
foreach test_number, dirname : integration_tests
test_unit_name = f'testsuite-@test_number@.service'
test_params = {
'test_name' : dirname,
'mkosi_image_name' : 'system',
'mkosi_output_path' : system_mkosi,
'test_number' : test_number,
'mkosi_args' : [],
'depends' : [system_mkosi],
'timeout' : 600,
}
@@ -358,16 +356,22 @@ if get_option('integration-tests') != false
if fs.exists(dirname / 'meson.build')
subdir(dirname)
endif
args = ['--test-name', test_params['test_name'],
'--mkosi-image-name', test_params['mkosi_image_name'],
'--mkosi-output-path', test_params['mkosi_output_path'],
'--test-number', test_params['test_number']]
args += ['--'] + test_params['mkosi_args']
test(test_params['test_name'],
args = [
'--meson-source-dir', meson.project_source_root(),
'--meson-build-dir', meson.project_build_root(),
'--test-name', dirname,
'--test-number', test_number,
'--',
] + test_params['mkosi_args']
# We don't explicitly depend on the "mkosi" target because that means the image is rebuilt
# on every "ninja -C build". Instead, the mkosi target has to be rebuilt manually before
# running the integration tests with mkosi.
test(dirname,
integration_test_wrapper,
env: test_env,
args : args,
depends : test_params['depends'],
timeout : test_params['timeout'],
suite : 'integration-tests')
endforeach