diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml index a86b5fb..2493086 100644 --- a/.github/workflows/GnuComment.yml +++ b/.github/workflows/GnuComment.yml @@ -24,44 +24,43 @@ jobs: uses: actions/github-script@v7 with: script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, run_id: ${{ github.event.workflow_run.id }}, }); - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "comment" - })[0]; + const matchArtifact = artifacts.data.artifacts.find(a => a.name === "comment"); if (!matchArtifact) { - return core.setFailed("No 'comment' artifact found"); + return core.info("No 'comment' artifact found"); } - var download = await github.rest.actions.downloadArtifact({ + const download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifact.id, archive_format: 'zip', }); - var fs = require('fs'); + const fs = require('fs'); fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); - - run: unzip comment.zip + - name: 'Unzip artifact' + run: unzip comment.zip - name: 'Comment on PR' uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - var fs = require('fs'); + const fs = require('fs'); if (!fs.existsSync('./NR') || !fs.existsSync('./result.txt')) { - return; + return core.info("No NR or result.txt found, skipping comment"); } - var issue_number = Number(fs.readFileSync('./NR')); - var content = fs.readFileSync('./result.txt'); - if (content.toString().trim().length > 7) { + const issue_number = Number(fs.readFileSync('./NR', 'utf8').trim()); + const content = fs.readFileSync('./result.txt', 'utf8').trim(); + if (content.length > 0) { await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: issue_number, - body: 'GNU testsuite comparison:\n```\n' + content + '```' + body: `### GNU Testsuite Comparison\n\n\`\`\`\n${content}\n\`\`\`` }); } diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 92ed7d5..2c87db8 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -16,11 +16,16 @@ concurrency: env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} TEST_FULL_SUMMARY_FILE: 'gnu-full-result.json' + TEST_SUMMARY_FILE: 'gnu-result.json' + AGGREGATED_SUMMARY_FILE: 'aggregated-result.json' jobs: native: name: Run GNU tests (native) runs-on: ubuntu-24.04 + defaults: + run: + shell: bash steps: - name: Checkout code (uutils) uses: actions/checkout@v4 @@ -38,43 +43,36 @@ jobs: workspaces: "./uutils -> target" - name: Install dependencies - shell: bash run: | sudo apt-get update sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libacl1-dev libattr1-dev libcap-dev libselinux1-dev attr quilt gettext - name: Build binaries and GNU tar - shell: bash run: | cd 'uutils' - # This script will clone GNU tar and build everything bash util/build-gnu.sh --release-build - name: Run GNU tests - shell: bash continue-on-error: true run: | cd 'uutils' bash util/run-gnu-test.sh - - name: Extract testing info from individual logs into JSON - shell: bash + - name: Extract info into JSON run : | python uutils/util/gnu-json-result.py gnu/tests > ${{ env.TEST_FULL_SUMMARY_FILE }} - - name: Upload full json results + - name: Upload full results uses: actions/upload-artifact@v4 with: name: gnu-full-result path: ${{ env.TEST_FULL_SUMMARY_FILE }} - - name: Compress test logs - shell: bash + - name: Compress logs run : | - # Compress logs before upload (fails otherwise) gzip gnu/tests/*/*.log - - name: Upload test logs + - name: Upload logs uses: actions/upload-artifact@v4 with: name: test-logs @@ -90,17 +88,11 @@ jobs: pull-requests: read name: Aggregate GNU test results runs-on: ubuntu-24.04 + defaults: + run: + shell: bash steps: - - name: Initialize workflow variables - id: vars - shell: bash - run: | - TEST_SUMMARY_FILE='gnu-result.json' - AGGREGATED_SUMMARY_FILE='aggregated-result.json' - echo "TEST_SUMMARY_FILE=${TEST_SUMMARY_FILE}" >> $GITHUB_OUTPUT - echo "AGGREGATED_SUMMARY_FILE=${AGGREGATED_SUMMARY_FILE}" >> $GITHUB_OUTPUT - - - name: Checkout code (uutils) + - name: Checkout code uses: actions/checkout@v4 with: path: 'uutils' @@ -115,21 +107,18 @@ jobs: workflow_conclusion: completed path: "reference" - - name: Download full json results + - name: Download results uses: actions/download-artifact@v4 with: name: gnu-full-result path: results - - name: Extract/summarize testing info + - name: Summarize results id: summary - shell: bash run: | - # Look at all individual results and summarize - eval $(python3 uutils/util/analyze-gnu-results.py -o=${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} results/*.json) - - output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" - echo "${output}" + eval $(python3 uutils/util/analyze-gnu-results.py -o=${{ env.AGGREGATED_SUMMARY_FILE }} results/*.json) + + echo "GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" jq -n \ --arg date "$(date --rfc-email)" \ @@ -140,37 +129,50 @@ jobs: --arg fail "$FAIL" \ --arg xpass "$XPASS" \ --arg error "$ERROR" \ - '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' - - HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) - echo "HASH=${HASH}" >> $GITHUB_OUTPUT + '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ env.TEST_SUMMARY_FILE }}' - - name: Upload test results summary - uses: actions/upload-artifact@v4 - with: - name: test-summary - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + - name: Prepare comparison + if: github.event_name == 'pull_request' + run: | + mkdir -p comment + echo "${{ github.event.pull_request.number }}" > comment/NR - - name: Upload aggregated json results - uses: actions/upload-artifact@v4 - with: - name: aggregated-result - path: ${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} - - - name: Compare test failures VS reference - shell: bash + - name: Compare results continue-on-error: true run: | - ## Compare test failures VS reference using JSON files - # This step will fail if no reference is found, which is expected for the first run - REF_SUMMARY_FILE='reference/aggregated-result/aggregated-result.json' - CURRENT_SUMMARY_FILE='${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}' + REF_SUMMARY_FILE='reference/aggregated-result/${{ env.AGGREGATED_SUMMARY_FILE }}' + CURRENT_SUMMARY_FILE='${{ env.AGGREGATED_SUMMARY_FILE }}' IGNORE_INTERMITTENT="uutils/.github/workflows/ignore-intermittent.txt" + + OUTPUT_ARG="" + if [ "${{ github.event_name }}" == "pull_request" ]; then + OUTPUT_ARG="--output comment/result.txt" + fi if [ -f "${REF_SUMMARY_FILE}" ]; then python3 uutils/util/compare_test_results.py \ --ignore-file "${IGNORE_INTERMITTENT}" \ + ${OUTPUT_ARG} \ "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" else echo "No reference summary found. Skipping comparison." fi + + - name: Upload summary + uses: actions/upload-artifact@v4 + with: + name: test-summary + path: "${{ env.TEST_SUMMARY_FILE }}" + + - name: Upload aggregated results + uses: actions/upload-artifact@v4 + with: + name: aggregated-result + path: ${{ env.AGGREGATED_SUMMARY_FILE }} + + - name: Upload comment artifact + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: comment + path: comment diff --git a/util/build-gnu.sh b/util/build-gnu.sh index f91490a..0b7d826 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -38,13 +38,10 @@ if test ! -d "${path_GNU}/.git"; then echo "Cloning GNU tar..." git clone --recurse-submodules https://git.savannah.gnu.org/git/tar.git "${path_GNU}" cd "${path_GNU}" - # Checkout v1.35 git checkout v1.35 + git submodule update --init --recursive - # Bootstrap requires gnulib. - # If git clone --recurse-submodules was used it might be there, but standard bootstrap fetches it. - # We need to make sure we have dependencies. - # FORCE_UNSAFE_CONFIGURE=1 might be needed if running as root in some envs, but usually fine. + # Bootstrap requires gnulib and generates the configure script ./bootstrap --skip-po fi diff --git a/util/compare_gnu_result.py b/util/compare_gnu_result.py deleted file mode 100755 index 924e2bb..0000000 --- a/util/compare_gnu_result.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -""" -Compare the current results to the last results gathered from the main branch to -highlight if a PR is making the results better/worse. -Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. -""" - -import json -import sys -from os import environ - -REPO_DEFAULT_BRANCH = environ.get("REPO_DEFAULT_BRANCH", "main") -ONLY_INTERMITTENT = environ.get("ONLY_INTERMITTENT", "false") - -NEW = json.load(open("gnu-result.json")) -OLD = json.load(open("main-gnu-result.json")) - -# Extract the specific results from the dicts -last = OLD[list(OLD.keys())[0]] -current = NEW[list(NEW.keys())[0]] - - -pass_d = int(current["pass"]) - int(last["pass"]) -fail_d = int(current["fail"]) - int(last["fail"]) -error_d = int(current["error"]) - int(last["error"]) -skip_d = int(current["skip"]) - int(last["skip"]) - -# Get an annotation to highlight changes -print( - f"""::warning ::Changes from '{REPO_DEFAULT_BRANCH}': PASS {pass_d:+d} / FAIL {fail_d:+d} / ERROR {error_d:+d} / SKIP {skip_d:+d}""" -) - -# If results are worse, check if we should fail the job -if pass_d < 0: - print( - f"""::error ::PASS count is reduced from '{REPO_DEFAULT_BRANCH}': PASS {pass_d:+d}""" - ) - - # Check if all failing tests are intermittent based on the environment variable - only_intermittent = ONLY_INTERMITTENT.lower() == "true" - - if only_intermittent: - print("::notice ::All failing tests are in the ignored intermittent list") - print("::notice ::Not failing the build") - else: - print("::error ::Found non-ignored failing tests") - sys.exit(1) diff --git a/util/gnu-json-result.py b/util/gnu-json-result.py index 3b2603a..37622b4 100755 --- a/util/gnu-json-result.py +++ b/util/gnu-json-result.py @@ -24,7 +24,7 @@ for filepath in test_dir.glob("**/*.log"): if path.name == "testsuite.log": # Handle Autotest testsuite.log try: - with open(path, errors="ignore") as f: + with open(path, "r", errors="ignore") as f: for line in f: # Look for lines like: " 1: basic functionality ok" # or " 10: ... FAILED (basic.at:123)" @@ -38,13 +38,18 @@ for filepath in test_dir.glob("**/*.log"): print(f"Error processing testsuite.log {path}: {e}", file=sys.stderr) continue + # Handle individual Automake-style .log files current = out for key in path.parent.relative_to(test_dir).parts: if key not in current: current[key] = {} current = current[key] try: - with open(path, errors="ignore") as f: + with open(path, "r", errors="ignore") as f: + # Only read the end of the file where the result is usually located + f.seek(0, 2) + size = f.tell() + f.seek(max(0, size - 1000), 0) content = f.read() result = re.search( r"(PASS|FAIL|SKIP|ERROR) [^ ]+ \(exit status: \d+\)$", content diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 770c796..6e0d93f 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -46,7 +46,6 @@ export TAR="${UU_BUILD_DIR}/tar" echo "Running GNU tar tests..." # Run with timeout and make check -# We use TAR to set the tar binary for the testsuite -# Then we use $* to pass any additional user arguments (specific tests) +# We use $* to pass any additional user arguments (e.g. TESTSUITEFLAGS="1-5") cp "${TAR}" src/tar -timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check +timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check "$@"