diffutils/tests/run-upstream-testsuite.sh

#!/bin/bash

# Run the GNU upstream test suite for diffutils against a local build of the
# Rust implementation, print out a summary of the test results, and writes a
# JSON file ('test-results.json') containing detailed information about the
# test run.

# The JSON file contains metadata about the test run, and for each test the
# result as well as the contents of stdout, stderr, and of all the files
# written by the test script, if any (excluding subdirectories).

# The script takes a shortcut to fetch only the test suite from the upstream
# repository and carefully avoids running the autotools machinery which is
# time-consuming and resource-intensive, and doesn't offer the option to not
# build the upstream binaries. As a consequence, the environment in which the
# tests are run might not match exactly that used when the upstream tests are
# run through the autotools.

# Exit codes: 0 if all tests passed, 1 if at least one test failed, and 2 if
# the test suite could not be run at all (e.g. the upstream repository couldn't
# be fetched). Callers must treat 2 as an infrastructure error: no meaningful
# result was produced.

# By default it expects a release build of the diffutils binary, but a
# different build profile can be specified as an argument
# (e.g. 'dev' or 'test').
# Unless overridden by the $TESTS environment variable, all tests in the test
# suite will be run. Tests targeting a command that is not yet implemented
# (e.g. diff3 or sdiff) are skipped.

scriptpath=$(dirname "$(readlink -f "$0")")
rev=$(git rev-parse HEAD)

# Report an infrastructure error: the test suite could not be run at all
die() {
  echo "ERROR: $*" >&2
  exit 2
}

# Allow passing a specific profile as parameter (default to "release")
profile="release"
[[ -n $1 ]] && profile="$1"

# Verify that the diffutils binary was built for the requested profile
binary="$scriptpath/../target/$profile/diffutils"
if [[ ! -x "$binary" ]]
then
  die "Missing build for profile $profile"
fi

# Work in a temporary directory
tempdir=$(mktemp -d)
trap 'rm -rf "$tempdir"' EXIT
cd "$tempdir" || die "Cannot enter temporary directory $tempdir"

# Check out the upstream test suite. git.savannah.gnu.org is regularly
# unavailable or slow, so retry a few times before giving up.
gitserver="https://git.savannah.gnu.org"
testsuite="$gitserver/git/diffutils.git"
attempts=3
for (( attempt = 1; attempt <= attempts; attempt++ ))
do
  echo "Fetching upstream test suite from $testsuite (attempt $attempt/$attempts)"
  rm -rf diffutils
  git clone -n --depth=1 --filter=tree:0 "$testsuite" && break
  (( attempt < attempts )) && sleep $(( attempt * 10 ))
done
[[ -d diffutils ]] || die "Failed to fetch the upstream test suite from $testsuite"
cd diffutils || die "Failed to fetch the upstream test suite from $testsuite"
git sparse-checkout set --no-cone tests &> /dev/null || die "Cannot sparse-checkout the upstream tests"
git checkout &> /dev/null || die "Cannot check out the upstream tests"
upstreamrev=$(git rev-parse HEAD)
[[ -d tests ]] || die "The upstream checkout contains no tests directory"

# Ensure that calling `diff` invokes the built `diffutils` binary instead of
# the upstream `diff` binary that is most likely installed on the system
mkdir src
cd src || die "Cannot create the directory holding the diff and cmp symlinks"
ln -s "$binary" diff
ln -s "$binary" cmp
cd ../tests || die "Cannot enter the upstream tests directory"

# Fetch tests/init.sh from the gnulib repository (needed since
# https://git.savannah.gnu.org/cgit/diffutils.git/commit/tests?id=1d2456f539)
# The savannah gitweb interface is often rate-limited or unavailable, so fall
# back to the official gnulib mirror on GitHub
initsh_urls=(
  "$gitserver/gitweb/?p=gnulib.git;a=blob_plain;f=tests/init.sh;hb=HEAD"
  "https://raw.githubusercontent.com/coreutils/gnulib/master/tests/init.sh"
)
for url in "${initsh_urls[@]}"
do
  echo "Fetching tests/init.sh from $url"
  curl -sSL --fail --retry 3 --retry-delay 5 --retry-all-errors \
    --connect-timeout 30 --max-time 300 "$url" -o init.sh && [[ -s init.sh ]] && break
  rm -f init.sh
done
[[ -s init.sh ]] || die "Failed to fetch tests/init.sh from the gnulib repository"

if [[ -n "$TESTS" ]]
then
  tests="$TESTS"
else
  # Get a list of all upstream tests (default if $TESTS isn't set)
  echo -e '\n\nprinttests:\n\t@echo "${TESTS}"' >> Makefile.am
  tests=$(make -f Makefile.am printtests)
fi
total=$(echo "$tests" | wc -w)
(( total > 0 )) || die "No test to run: the upstream test list is empty"
echo "Running $total tests"
export LC_ALL=C
export KEEP=yes
timestamp=$(date -Iseconds)
urlroot="$gitserver/cgit/diffutils.git/tree/tests/"
passed=0
failed=0
skipped=0
normal="$(tput sgr0)"
for test in $tests
do
  result="FAIL"
  url="$urlroot$test?id=$upstreamrev"
  # Run only the tests that invoke `diff` or `cmp`,
  # because other binaries aren't implemented yet
  if ! grep -E -s -q "(diff3|sdiff)" "$test"
  then
    sh "$test" 1> stdout.txt 2> stderr.txt && result="PASS"
    if [[ $? = 77 ]]
    then
      result="SKIP"
    else
      json+="{\"test\":\"$test\",\"result\":\"$result\","
      json+="\"url\":\"$url\","
      json+="\"stdout\":\"$(base64 -w0 < stdout.txt)\","
      json+="\"stderr\":\"$(base64 -w0 < stderr.txt)\","
      json+="\"files\":{"
      cd gt-$test.*
      # Note: this doesn't include the contents of subdirectories,
      # but there isn't much value added in doing so
      for file in *
      do
        # Encode the name with jq: some tests create files whose name contains
        # quotes or control characters, which would produce invalid JSON
        [[ -f "$file" ]] && json+="$(jq -Rn --arg name "$file" '$name'):\"$(base64 -w0 < "$file")\","
      done
      json="${json%,}}},"
      cd - > /dev/null
      [[ "$result" = "PASS" ]] && (( passed++ ))
      [[ "$result" = "FAIL" ]] && (( failed++ ))
    fi
  else
    result="SKIP"
  fi
  color=2 # green
  [[ "$result" = "FAIL" ]] && color=1 # red
  if [[ $result = "SKIP" ]]
  then
    (( skipped++ ))
    json+="{\"test\":\"$test\",\"url\":\"$url\",\"result\":\"$result\"},"
    color=3 # yellow
  fi
  printf "  %-40s $(tput setaf $color)$result$(tput sgr0)\n" "$test"
done
echo ""
echo -n "Summary: TOTAL: $total / "
echo -n "$(tput setaf 2)PASS$normal: $passed / "
echo -n "$(tput setaf 1)FAIL$normal: $failed / "
echo "$(tput setaf 3)SKIP$normal: $skipped"
echo ""

json="\"tests\":[${json%,}]"
metadata="\"timestamp\":\"$timestamp\","
metadata+="\"revision\":\"$rev\","
metadata+="\"upstream-revision\":\"$upstreamrev\","
if [[ -n "$GITHUB_ACTIONS" ]]
then
  metadata+="\"branch\":\"$GITHUB_REF\","
fi
json="{$metadata $json}"

# Clean up
cd "$scriptpath"

# Write the results out only once they are known to be valid JSON, so that a
# malformed (or truncated) file is never left behind for the caller to consume
resultsfile="test-results.json"
if ! echo "$json" | jq > "$resultsfile.tmp"
then
  rm -f "$resultsfile.tmp"
  die "Generated invalid JSON results"
fi
mv "$resultsfile.tmp" "$resultsfile"
echo "Results written to $scriptpath/$resultsfile"

(( failed > 0 )) && exit 1
exit 0