snapd/tests/core/failover/task.yaml

summary: check that the core and kernel snaps roll back correctly after a failed upgrade

details: |
    This test ensures that the system can survive to a failed upgrade of a fundamental
    snap, rolling back to the last good known version.

    The logic common to all the scenarios unpacks the target snap, injects the failure,
    repacks and installs it. Then it checks that all is set for installed the snap with
    the failure and executes a reboot. The test checks that after the reboot (in fact two
    reboots, one for trying the upgrade and another for rolling back) the installed
    fundamental snap is the good one and the boot environment variables are correctly set.

# TODO: enable for UC20 ?
systems: [ubuntu-core-16-*, ubuntu-core-18-*]

# Start early as it takes a long time.
priority: 100

environment:
    INJECT_FAILURE/rclocalcrash: inject_rclocalcrash_failure
    INJECT_FAILURE/emptysystemd: inject_emptysystemd_failure
    # FIXME: disabled until we find what to do!
    # fails with:  ERROR cannot replace signed kernel snap with an unasserted one
    INJECT_FAILURE/emptyinitrd: inject_emptyinitrd_failure
    TARGET_SNAP/rclocalcrash: core
    TARGET_SNAP/emptysystemd: core
    TARGET_SNAP/emptyinitrd: kernel
    BUILD_DIR: /home/tmp
    # uploading the core or otherwise large snap triggers OOM
    SNAPD_NO_MEMORY_LIMIT: 1
    BLOB_DIR: $(pwd)/fake-store-blobdir

    SNAP_ID_core20: DLqre5XGLbDqg9jPtiAhRRjDuPVa5X1q
    SNAP_ID_core18: CSO04Jhav2yK0uz97cr0ipQRyqg0qQL6
    SNAP_ID_core: 99T7MUlRhtI3U0QFgl5mXXESAiSwt776
    # pc-kernel snap is specific to i386/x86-64
    SNAP_ID_pc_kernel: pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza

prepare: |
    if [ "$TARGET_SNAP" = "kernel" ] && os.query is-arm; then
        echo "variant not supported on ARM architectures"
        exit 0
    fi

    if [ "$TRUST_TEST_KEYS" = "false" ]; then
        echo "This test needs test keys to be trusted"
        exit
    fi
    snap install jq
    snap install remarshal

    snap ack "$TESTSLIB/assertions/testrootorg-store.account-key"
    snap ack "$TESTSLIB/assertions/developer1.account"
    snap ack "$TESTSLIB/assertions/developer1.account-key"
    "$TESTSTOOLS"/store-state setup-fake-store "$BLOB_DIR"

    mkdir -p "$BUILD_DIR"

restore: |
    if [ "$TARGET_SNAP" = "kernel" ] && os.query is-arm; then
        echo "variant not supported on ARM architectures"
        exit 0
    fi

    if [ "$TRUST_TEST_KEYS" = "false" ]; then
        echo "This test needs test keys to be trusted"
        exit
    fi
    "$TESTSTOOLS"/store-state teardown-fake-store "$BLOB_DIR"

    rm -rf "$BUILD_DIR"

debug: |
    snap debug boot-vars || true
    snap list || true
    snap changes || true

execute: |
    if [ "$TARGET_SNAP" = "kernel" ] && os.query is-arm; then
        echo "variant not supported on ARM architectures"
        exit 0
    fi

    if [ "$TRUST_TEST_KEYS" = "false" ]; then
        echo "This test needs test keys to be trusted"
        exit
    fi

    inject_rclocalcrash_failure(){
        chmod a+x "$BUILD_DIR/unpack/etc/rc.local"
        cat <<EOF > "$BUILD_DIR/unpack/etc/rc.local"
    #!bin/sh
    printf c > /proc/sysrq-trigger
    EOF
    }

    inject_emptysystemd_failure(){
        truncate -s 0 "$BUILD_DIR/unpack/lib/systemd/systemd"
    }

    inject_emptyinitrd_failure(){
        truncate -s 0 "$BUILD_DIR/unpack/initrd.img"
    }

    if os.query is-core18 && [ "$SPREAD_VARIANT" = "rclocalcrash" ]; then
        # there is no /etc/rc.local on core18
        echo "scenario isn't supported on core18"
        exit 0
    fi

    #shellcheck source=tests/lib/snaps.sh
    . "$TESTSLIB"/snaps.sh

    core_name="core"
    core_snap_id="$SNAP_ID_core"
    if os.query is-core18; then
        core_name="core18"
        core_snap_id="$SNAP_ID_core18"
    elif os.query is-core20; then
        core_name="core20"
        core_snap_id="$SNAP_ID_core20"
    fi
    TARGET_SNAP_NAME="$core_name"
    SNAP_ID="$core_snap_id"

    if [ "$TARGET_SNAP" = kernel ]; then
        TARGET_SNAP_NAME=pc-kernel
        SNAP_ID="$SNAP_ID_pc_kernel"
    fi

    if [ "$SPREAD_REBOOT" = 0 ]; then
        # first pass, save current target snap revision
        snap list | awk "/^${TARGET_SNAP_NAME} / {print(\$3)}" > prevBoot
        # it is possible that the previous revision of the snap was unasserted
        START_REVISION=9999
        PREV_REV="$(cat prevBoot)"
        if [ "${PREV_REV##x}" = "$PREV_REV" ]; then
            # the previous revision has no x prefix, meaning the snap was
            # asserted, so just bump the revision number
            START_REVISION=$((PREV_REV + 1))
        fi
        # unpack current target snap
        unsquashfs -no-progress -d "$BUILD_DIR/unpack" "/var/lib/snapd/snaps/${TARGET_SNAP_NAME}_$(cat prevBoot).snap"

        # set failure condition
        eval "${INJECT_FAILURE}"

        # repack new target snap
        mksnap_fast "$BUILD_DIR/unpack" failing.snap
        cat <<EOF > decl-headers.json
    {"snap-id": "$SNAP_ID", "snap-name": "$TARGET_SNAP_NAME"}
    EOF
        cat <<EOF > rev-headers.json
    {"snap-id": "$SNAP_ID", "snap-revision": "$START_REVISION"}
    EOF

        p=$(fakestore new-snap-declaration --dir "$BLOB_DIR" failing.snap --snap-decl-json decl-headers.json)
        snap ack "$p"
        p=$(fakestore new-snap-revision --dir "$BLOB_DIR" failing.snap --snap-rev-json rev-headers.json)
        snap ack "$p"

        # use journalctl wrapper to grep only the logs collected while the test is running
        if "$TESTSTOOLS"/journal-state get-log | MATCH "Waiting for system reboot"; then
            echo "Already waiting for system reboot, exiting..."
            exit 1
        fi

        # install new target snap
        snap install --no-wait failing.snap

        # use journalctl wrapper to grep only the logs collected while the test is running
        # waiting up to 100s to reach waiting for reboot
        retry -n 20 --wait 2 "$TESTSTOOLS"/journal-state match-log "Waiting for system reboot"

        # check boot env vars
        readlink "/snap/$TARGET_SNAP_NAME/current" > failBoot
        snap debug boot-vars > before-reboot.bootenv
        if [ "$TARGET_SNAP" = kernel ]; then
            MATCH "snap_kernel=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$"  < before-reboot.bootenv
            MATCH "snap_try_kernel=${TARGET_SNAP_NAME}_$(cat failBoot).snap\$"  < before-reboot.bootenv
        else
            MATCH "snap_core=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$"  < before-reboot.bootenv
            MATCH "snap_try_core=${TARGET_SNAP_NAME}_$(cat failBoot).snap\$"  < before-reboot.bootenv
        fi
        REBOOT
    fi

    # since the change fails snap watch will have non-0 exit status too, so we
    # need to ignore the exit status
    snap watch --last=install? || true

    # after rollback, we have the original target snap for a while
    # wait until the kernel and core snap revisions are in place
    #shellcheck disable=SC2016
    retry -n 60 --wait 1 --env TARGET_SNAP_NAME="$TARGET_SNAP_NAME" sh -c 'test $(snap list | awk "/^${TARGET_SNAP_NAME} / {print(\$3)}") = $(cat prevBoot)'

    # ensure the last install change failed as expected
    snap change --last=install | MATCH "cannot finish $TARGET_SNAP_NAME installation, there was a rollback across reboot"
    snap change --last=install | MATCH "^Error.*Automatically connect"

    # and the boot env vars are correctly set
    echo "Waiting for snapd to clean snap_mode"
    #shellcheck disable=SC2148
    #shellcheck disable=SC2016
    retry -n 200 --wait 1 sh -c 'snap debug boot-vars | MATCH "snap_mode=\$"'

    snap debug boot-vars > after-reboot.bootenv
    if [ "$TARGET_SNAP" = kernel ]; then
        MATCH 'snap_try_kernel=$' < after-reboot.bootenv
        MATCH "snap_kernel=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$" < after-reboot.bootenv
    else
        MATCH 'snap_try_core=$' < after-reboot.bootenv
        MATCH "snap_core=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$" < after-reboot.bootenv
    fi