From 4326586be28a2a41be2fa1fe8abe4941566a9610 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 17 Nov 2020 16:10:29 -0500 Subject: [PATCH 1/8] test: remove unused 'basedir' var from integration test makefiles --- test/TEST-01-BASIC/Makefile | 2 +- test/TEST-52-HONORFIRSTSHUTDOWN/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/TEST-01-BASIC/Makefile b/test/TEST-01-BASIC/Makefile index 79fe9688b8..bee382b8b1 100644 --- a/test/TEST-01-BASIC/Makefile +++ b/test/TEST-01-BASIC/Makefile @@ -1,6 +1,6 @@ BUILD_DIR=$(shell ../../tools/find-build-dir.sh) all setup run clean clean-again: - @basedir=../.. TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./test.sh --$@ + @TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./test.sh --$@ .PHONY: all setup run clean clean-again diff --git a/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile b/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile index a06599081b..46a28ac810 100644 --- a/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile +++ b/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile @@ -1,7 +1,7 @@ BUILD_DIR=$(shell ../../tools/find-build-dir.sh) all setup run clean clean-again: - @basedir=../.. TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./test.sh --$@ + @TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./test.sh --$@ # finish option is used to run checks that can only be run outside of # the test execution. Example case, honor first shutdown, proof is obtained @@ -11,6 +11,6 @@ all setup run clean clean-again: # the test will loop and will be terminated via a command timeout. # This just provides concrete confirmation. finish: - @basedir=../.. TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./fini.sh --$@ + @TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./fini.sh --$@ .PHONY: all setup run clean clean-again finish From 1918406900c2b687cca867d32441ec98f05b16ea Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 17 Nov 2020 16:02:14 -0500 Subject: [PATCH 2/8] test/test-functions: move var assignment This makes no code change, only moves a small block of vars higher in the file. This makes the next commit a bit easier to read. --- test/test-functions | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test-functions b/test/test-functions index 837082890a..f6a824ae69 100644 --- a/test/test-functions +++ b/test/test-functions @@ -50,6 +50,11 @@ PATH_TO_INIT=$ROOTLIBDIR/systemd [ "$SYSTEMD_NSPAWN" ] || SYSTEMD_NSPAWN=$(which -a $BUILD_DIR/systemd-nspawn systemd-nspawn 2>/dev/null | grep '^/' -m1) [ "$JOURNALCTL" ] || JOURNALCTL=$(which -a $BUILD_DIR/journalctl journalctl 2>/dev/null | grep '^/' -m1) +STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))" +STATEFILE="$STATEDIR/.testdir" +IMAGESTATEDIR="$STATEDIR/.." +TESTLOG="$STATEDIR/test.log" + BASICTOOLS=( awk basename @@ -148,11 +153,6 @@ DEBUGTOOLS=( vi ) -STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))" -STATEFILE="$STATEDIR/.testdir" -IMAGESTATEDIR="$STATEDIR/.." -TESTLOG="$STATEDIR/test.log" - is_built_with_asan() { if ! type -P objdump >/dev/null; then ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN." From 42f3b48c975bd45b489dff08df5e374d3e3bfb5f Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 26 Jan 2021 16:46:10 -0500 Subject: [PATCH 3/8] test/test-functions: add variables for several dir locations --- test/test-functions | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/test-functions b/test/test-functions index f6a824ae69..928d3fc0ad 100644 --- a/test/test-functions +++ b/test/test-functions @@ -43,6 +43,12 @@ if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then ROOTLIBDIR=/usr/lib/systemd fi +# The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it +TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath $(dirname "$BASH_SOURCE"))} +TEST_UNITS_DIR="$TEST_BASE_DIR/units" +SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..") +TOOLS_DIR="$SOURCE_DIR/tools" + PATH_TO_INIT=$ROOTLIBDIR/systemd [ "$SYSTEMD_JOURNALD" ] || SYSTEMD_JOURNALD=$(which -a $BUILD_DIR/systemd-journald $ROOTLIBDIR/systemd-journald 2>/dev/null | grep '^/' -m1) [ "$SYSTEMD_JOURNAL_REMOTE" ] || SYSTEMD_JOURNAL_REMOTE=$(which -a $BUILD_DIR/systemd-journal-remote $ROOTLIBDIR/systemd-journal-remote 2>/dev/null | grep '^/' -m1) @@ -50,7 +56,13 @@ PATH_TO_INIT=$ROOTLIBDIR/systemd [ "$SYSTEMD_NSPAWN" ] || SYSTEMD_NSPAWN=$(which -a $BUILD_DIR/systemd-nspawn systemd-nspawn 2>/dev/null | grep '^/' -m1) [ "$JOURNALCTL" ] || JOURNALCTL=$(which -a $BUILD_DIR/journalctl journalctl 2>/dev/null | grep '^/' -m1) -STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))" +TESTFILE=${BASH_SOURCE[1]} +if [ -z "$TESTFILE" ]; then + echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2 + exit 1 +fi +TESTNAME=$(basename $(dirname $(realpath $TESTFILE))) +STATEDIR="$BUILD_DIR/test/$TESTNAME" STATEFILE="$STATEDIR/.testdir" IMAGESTATEDIR="$STATEDIR/.." TESTLOG="$STATEDIR/test.log" From 12d31e4ea50583c2aa39573f3d544f08492d5d61 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 17 Nov 2020 15:39:37 -0500 Subject: [PATCH 4/8] test: find $BUILD_DIR in test-functions, remove from other scripts The $BUILD_DIR is only used in test-functions, and doesn't need to be specified in any other scripts. Additionally, to be able to allow the integration test suite to be run against locally installed binaries, instead of built binaries, moving BUILD_DIR logic completely into test-functions allows later patches to be simpler. --- test/README.testsuite | 4 ++-- test/TEST-01-BASIC/Makefile | 4 +--- test/TEST-52-HONORFIRSTSHUTDOWN/Makefile | 6 ++---- test/run-integration-tests.sh | 6 +++--- test/test-functions | 6 ++++++ 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/test/README.testsuite b/test/README.testsuite index 1b4736013d..a05a9936a6 100644 --- a/test/README.testsuite +++ b/test/README.testsuite @@ -8,7 +8,7 @@ $ sudo test/run-integration-tests.sh ninja: Entering directory `/home/zbyszek/src/systemd/build' ninja: no work to do. --x-- Running TEST-01-BASIC --x-- -+ make -C TEST-01-BASIC BUILD_DIR=/home/zbyszek/src/systemd/build clean setup run ++ make -C TEST-01-BASIC clean setup run make: Entering directory '/home/zbyszek/src/systemd/test/TEST-01-BASIC' TEST-01-BASIC CLEANUP: Basic systemd setup TEST-01-BASIC SETUP: Basic systemd setup @@ -17,7 +17,7 @@ TEST-01-BASIC RUN: Basic systemd setup [OK] make: Leaving directory '/home/zbyszek/src/systemd/test/TEST-01-BASIC' --x-- Result of TEST-01-BASIC: 0 --x-- --x-- Running TEST-02-CRYPTSETUP --x-- -+ make -C TEST-02-CRYPTSETUP BUILD_DIR=/home/zbyszek/src/systemd/build clean setup run ++ make -C TEST-02-CRYPTSETUP clean setup run If one of the tests fails, then $subdir/test.log contains the log file of the test. diff --git a/test/TEST-01-BASIC/Makefile b/test/TEST-01-BASIC/Makefile index bee382b8b1..9b90a1c2bc 100644 --- a/test/TEST-01-BASIC/Makefile +++ b/test/TEST-01-BASIC/Makefile @@ -1,6 +1,4 @@ -BUILD_DIR=$(shell ../../tools/find-build-dir.sh) - all setup run clean clean-again: - @TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./test.sh --$@ + @TEST_BASE_DIR=../ ./test.sh --$@ .PHONY: all setup run clean clean-again diff --git a/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile b/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile index 46a28ac810..71487d7076 100644 --- a/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile +++ b/test/TEST-52-HONORFIRSTSHUTDOWN/Makefile @@ -1,7 +1,5 @@ -BUILD_DIR=$(shell ../../tools/find-build-dir.sh) - all setup run clean clean-again: - @TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./test.sh --$@ + @TEST_BASE_DIR=../ ./test.sh --$@ # finish option is used to run checks that can only be run outside of # the test execution. Example case, honor first shutdown, proof is obtained @@ -11,6 +9,6 @@ all setup run clean clean-again: # the test will loop and will be terminated via a command timeout. # This just provides concrete confirmation. finish: - @TEST_BASE_DIR=../ BUILD_DIR=$(BUILD_DIR) ./fini.sh --$@ + @TEST_BASE_DIR=../ ./fini.sh --$@ .PHONY: all setup run clean clean-again finish diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index e9307601ff..219216ac56 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -22,7 +22,7 @@ cd "$(dirname "$0")" # cache. if [ $do_clean = 1 ]; then for TEST in TEST-??-* ; do - ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean ) + ( set -x ; make -C "$TEST" clean ) done [ -n "$args_no_clean" ] || exit 0 @@ -46,7 +46,7 @@ for TEST in TEST-??-* ; do echo -e "\n--x-- Running $TEST --x--" set +e - ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" $args_no_clean ) + ( set -x ; make -C "$TEST" $args_no_clean ) RESULT=$? set -e echo "--x-- Result of $TEST: $RESULT --x--" @@ -59,7 +59,7 @@ done if [ $FAILURES -eq 0 -a $do_clean = 1 ]; then for TEST in ${!results[@]}; do - ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean-again ) + ( set -x ; make -C "$TEST" clean-again ) done fi diff --git a/test/test-functions b/test/test-functions index 928d3fc0ad..947ba22357 100644 --- a/test/test-functions +++ b/test/test-functions @@ -49,6 +49,12 @@ TEST_UNITS_DIR="$TEST_BASE_DIR/units" SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..") TOOLS_DIR="$SOURCE_DIR/tools" +# note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it +if ! BUILD_DIR=$($TOOLS_DIR/find-build-dir.sh); then + echo "ERROR: no build found, please set BUILD_DIR" >&2 + exit 1 +fi + PATH_TO_INIT=$ROOTLIBDIR/systemd [ "$SYSTEMD_JOURNALD" ] || SYSTEMD_JOURNALD=$(which -a $BUILD_DIR/systemd-journald $ROOTLIBDIR/systemd-journald 2>/dev/null | grep '^/' -m1) [ "$SYSTEMD_JOURNAL_REMOTE" ] || SYSTEMD_JOURNAL_REMOTE=$(which -a $BUILD_DIR/systemd-journal-remote $ROOTLIBDIR/systemd-journal-remote 2>/dev/null | grep '^/' -m1) From 8fa038085ebfebf6f1dd1aae7112fbb7e4d88700 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 17 Nov 2020 17:20:22 -0500 Subject: [PATCH 5/8] test/test-functions: allow installing systemd files from local system Add NO_BUILD var to allow testing with no local build, by installing local systemd files into the image. This only works for debian-like distros currently, that use the tools 'apt' and 'dpkg' for package management. --- test/test-functions | 46 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/test/test-functions b/test/test-functions index 947ba22357..a9341445d5 100644 --- a/test/test-functions +++ b/test/test-functions @@ -51,8 +51,12 @@ TOOLS_DIR="$SOURCE_DIR/tools" # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it if ! BUILD_DIR=$($TOOLS_DIR/find-build-dir.sh); then - echo "ERROR: no build found, please set BUILD_DIR" >&2 - exit 1 + if [ "$NO_BUILD" ]; then + BUILD_DIR=$SOURCE_DIR + else + echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2 + exit 1 + fi fi PATH_TO_INIT=$ROOTLIBDIR/systemd @@ -681,7 +685,7 @@ install_dmevent() { fi } -install_systemd() { +install_compiled_systemd() { ddebug "Install compiled systemd" local _ninja_bin=$(type -P ninja || type -P ninja-build) @@ -690,6 +694,42 @@ install_systemd() { exit 1 fi (set -x; DESTDIR=$initdir "$_ninja_bin" -C $BUILD_DIR install) +} + +install_debian_systemd() { + ddebug "Install debian systemd" + + local _systemd_pkgs=$(apt-cache showsrc systemd | grep -m 1 -E '^Binary:' | cut -d ':' -f 2 | tr -d ,) + local _files="" + for deb in $_systemd_pkgs; do + _files=$(dpkg-query -L $deb 2>/dev/null) || continue + ddebug "Install debian files from package $deb" + for file in $_files; do + [ -e "$file" ] || continue + [ -d "$file" ] && continue + inst $file + done + done +} + +install_distro_systemd() { + ddebug "Install distro systemd" + + if [ "$LOOKS_LIKE_DEBIAN" ]; then + install_debian_systemd + else + dfatal "NO_BUILD not supported for this distro" + exit 1 + fi +} + +install_systemd() { + if [ "$NO_BUILD" ]; then + install_distro_systemd + else + install_compiled_systemd + fi + # remove unneeded documentation rm -fr $initdir/usr/share/{man,doc} From 232add5c47a8486623c98158ed3a9a19f422b5d8 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 17 Nov 2020 15:55:47 -0500 Subject: [PATCH 6/8] test: allow run-integration-tests.sh to run without build --- test/README.testsuite | 8 ++++++++ test/run-integration-tests.sh | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/test/README.testsuite b/test/README.testsuite index a05a9936a6..da989c1e0d 100644 --- a/test/README.testsuite +++ b/test/README.testsuite @@ -41,6 +41,14 @@ $ sudo make -C test/TEST-01-BASIC BUILD_DIR=../../some-other-build/ ... Note that in the second case, the path is relative to the test case directory. An absolute path may also be used in both cases. +Testing installed binaries instead of built +=========================================== + +To run the extended testsuite using the systemd installed on the system instead +of the systemd from a build, use the NO_BUILD=1: + +$ sudo NO_BUILD=1 test/run-integration-tests + Configuration variables ======================= diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index 219216ac56..34fa4f56a2 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -1,7 +1,15 @@ #!/usr/bin/env bash set -e -BUILD_DIR="$($(dirname "$0")/../tools/find-build-dir.sh)" +if [ "$NO_BUILD" ]; then + BUILD_DIR="" +elif BUILD_DIR="$($(dirname "$0")/../tools/find-build-dir.sh)"; then + ninja -C "$BUILD_DIR" +else + echo "No build found, please set BUILD_DIR or NO_BUILD" >&2 + exit 1 +fi + if [ $# -gt 0 ]; then args="$@" else From b3e43406646bee75a2c0d36fb7abe7e33a1139f8 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 18 Nov 2020 07:30:11 -0500 Subject: [PATCH 7/8] test/run-integration-tests.sh: adjust arg processing The script currently parses either 'clean' or 'clean-again' as wanting to clean both before and after running tests. This fixes that to split the action up; clean runs before tests, clean-again after; and also verifies the parameter(s) before passing them to make. --- test/run-integration-tests.sh | 82 +++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index 34fa4f56a2..036c075eef 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -11,12 +11,36 @@ else fi if [ $# -gt 0 ]; then - args="$@" + args="$*" else args="setup run clean-again" fi -args_no_clean=$(sed -r 's/\bclean.*\b//g' <<<$args) -do_clean=$( [ "$args" = "$args_no_clean" ]; echo $? ) + +VALID_TARGETS="all setup run clean clean-again" + +is_valid_target() { + for target in $VALID_TARGETS; do + [ "$1" = "$target" ] && return 0 + done + return 1 +} + +# reject invalid make targets in $args +for arg in $args; do + if ! is_valid_target "$arg"; then + echo "Invalid target: $arg" >&2 + exit 1 + fi +done + +CLEAN=0 +CLEANAGAIN=0 + +# separate 'clean' and 'clean-again' operations +[[ "$args" =~ "clean-again" ]] && CLEANAGAIN=1 +args=${args/clean-again} +[[ "$args" =~ "clean" ]] && CLEAN=1 +args=${args/clean} declare -A results declare -A times @@ -26,16 +50,6 @@ FAILURES=0 cd "$(dirname "$0")" -# Let's always do the cleaning operation first, because it destroys the image -# cache. -if [ $do_clean = 1 ]; then - for TEST in TEST-??-* ; do - ( set -x ; make -C "$TEST" clean ) - done - - [ -n "$args_no_clean" ] || exit 0 -fi - pass_deny_list() { for marker in $DENY_LIST_MARKERS $BLACKLIST_MARKERS; do if [ -f "$1/$marker" ]; then @@ -46,26 +60,38 @@ pass_deny_list() { return 0 } -for TEST in TEST-??-* ; do - COUNT=$(($COUNT+1)) +# Let's always do the cleaning operation first, because it destroys the image +# cache. +if [ $CLEAN = 1 ]; then + for TEST in TEST-??-* ; do + ( set -x ; make -C "$TEST" clean ) + done +fi - pass_deny_list $TEST || continue - start=$(date +%s) +# Run actual tests (if requested) +if [[ $args =~ [a-z] ]]; then + for TEST in TEST-??-* ; do + COUNT=$(($COUNT+1)) - echo -e "\n--x-- Running $TEST --x--" - set +e - ( set -x ; make -C "$TEST" $args_no_clean ) - RESULT=$? - set -e - echo "--x-- Result of $TEST: $RESULT --x--" + pass_deny_list $TEST || continue + start=$(date +%s) - results["$TEST"]="$RESULT" - times["$TEST"]=$(( $(date +%s) - $start )) + echo -e "\n--x-- Running $TEST --x--" + set +e + ( set -x ; make -C "$TEST" $args ) + RESULT=$? + set -e + echo "--x-- Result of $TEST: $RESULT --x--" - [ "$RESULT" -ne "0" ] && FAILURES=$(($FAILURES+1)) -done + results["$TEST"]="$RESULT" + times["$TEST"]=$(( $(date +%s) - $start )) -if [ $FAILURES -eq 0 -a $do_clean = 1 ]; then + [ "$RESULT" -ne "0" ] && FAILURES=$(($FAILURES+1)) + done +fi + +# Run clean-again, if requested, and if no tests failed +if [ $FAILURES -eq 0 -a $CLEANAGAIN = 1 ]; then for TEST in ${!results[@]}; do ( set -x ; make -C "$TEST" clean-again ) done From 59b8cb3c45c1975e015680193c02cfb62bdb8bd5 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 18 Nov 2020 09:39:50 -0500 Subject: [PATCH 8/8] test/TEST-01-BASIC: also install testsuite.target This test doesn't require the tests to be installed, so it must manually install required test services and targets itself, including the default target of testsuite.target Also use $TEST_UNITS_DIR which is set by test-functions instead of calculating the path --- test/TEST-01-BASIC/test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/TEST-01-BASIC/test.sh b/test/TEST-01-BASIC/test.sh index 606b0dbfaf..66c35fe2b7 100755 --- a/test/TEST-01-BASIC/test.sh +++ b/test/TEST-01-BASIC/test.sh @@ -9,8 +9,9 @@ TEST_REQUIRE_INSTALL_TESTS=0 test_append_files() { # install tests manually so the test is functional even when -Dinstall-tests=false - mkdir -p $1/usr/lib/systemd/tests/testdata/units/ - cp -v $(dirname $0)/../units/{testsuite-01,end}.service $1/usr/lib/systemd/tests/testdata/units/ + local dst="$1/usr/lib/systemd/tests/testdata/units/" + mkdir -p "$dst" + cp -v $TEST_UNITS_DIR/{testsuite-01,end}.service $TEST_UNITS_DIR/testsuite.target "$dst" } do_test "$@" 01