feat: add gnu testsuite tracking workflows

.github/workflows/GnuComment.yml

@@ -0,0 +1,100 @@
+name: GnuComment
+
+on:
+  workflow_run:
+    workflows: ["GnuTests"]
+    types:
+      - completed
+
+permissions: {}
+
+jobs:
+  post-comment:
+    permissions:
+      actions: read  #  to list workflow runs artifacts
+      pull-requests: write  #  to comment on pr
+
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: 'Download artifact'
+        uses: actions/github-script@v9
+        with:
+          script: |
+            // List all artifacts from GnuTests
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: ${{ github.event.workflow_run.id }},
+            });
+
+            // Download the "comment" artifact, which contains a PR number (NR) and result.txt
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "comment"
+            })[0];
+
+            if (!matchArtifact) {
+              console.log('No comment artifact found');
+              return;
+            }
+
+            var download = await github.rest.actions.downloadArtifact({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: matchArtifact.id,
+              archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
+
+      - run: unzip comment.zip || echo "Failed to unzip comment artifact"
+
+      - name: 'Comment on PR'
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            var fs = require('fs');
+
+            // Check if files exist
+            if (!fs.existsSync('./NR')) {
+              console.log('No NR file found, skipping comment');
+              return;
+            }
+            if (!fs.existsSync('./result.txt')) {
+              console.log('No result.txt file found, skipping comment');
+              return;
+            }
+
+            var issue_number = Number(fs.readFileSync('./NR'));
+            var content = fs.readFileSync('./result.txt');
+
+            if (content.toString().trim().length > 7) { // 7 because we have backquote + \n
+              // Update existing comment if present, otherwise create a new one
+              var marker = '<!-- gnu-tests-bot -->';
+              var body = marker + '\nGNU diffutils testsuite comparison:\n```\n' + content + '```';
+              var comments = await github.rest.issues.listComments({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue_number,
+              });
+              var existing = comments.data.filter(c => c.body.includes(marker))[0];
+              if (existing) {
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: existing.id,
+                  body: body,
+                });
+              } else {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: issue_number,
+                  body: body,
+                });
+              }
+            } else {
+              console.log('Comment content too short, skipping');
+            }

.github/workflows/GnuTests.yml

@@ -0,0 +1,231 @@
+name: GnuTests
+
+# Run GNU diffutils testsuite against the Rust diffutils implementation
+# and compare results against the main branch to catch regressions
+
+on:
+  pull_request:
+  push:
+    branches:
+      - '*'
+
+permissions:
+  contents: write # Publish diffutils instead of discarding
+
+# End the current execution if there is a new changeset in the PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+  TEST_FULL_SUMMARY_FILE: 'diffutils-gnu-full-result.json'
+
+jobs:
+  native:
+    name: Run GNU diffutils testsuite
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      ### Build
+      - name: Build Rust diffutils binary
+        shell: bash
+        run: |
+          ## Build Rust diffutils binary
+          cargo build --config=profile.release.strip=true --profile=release
+          zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst
+
+      - name: Publish latest commit
+        uses: softprops/action-gh-release@v3
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          tag_name: latest-commit
+          body: |
+            commit: ${{ github.sha }}
+          draft: false
+          prerelease: true
+          files: |
+            diffutils-x86_64-unknown-linux-gnu.zst
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      ### Run tests
+      - name: Run GNU diffutils testsuite
+        shell: bash
+        run: |
+          ## Run GNU diffutils testsuite
+          ./tests/run-upstream-testsuite.sh release || true
+        env:
+          TERM: xterm
+
+      - name: Upload full json results
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffutils-gnu-full-result
+          path: tests/test-results.json
+          if-no-files-found: warn
+
+  aggregate:
+    needs: [native]
+    permissions:
+      actions: read
+      contents: read
+      pull-requests: read
+    name: Aggregate GNU test results
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Initialize workflow variables
+        id: vars
+        shell: bash
+        run: |
+          ## VARs setup
+          outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
+
+          TEST_SUMMARY_FILE='diffutils-gnu-result.json'
+          outputs TEST_SUMMARY_FILE
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Retrieve reference artifacts
+        uses: dawidd6/action-download-artifact@v20
+        continue-on-error: true
+        with:
+          workflow: GnuTests.yml
+          branch: "${{ env.DEFAULT_BRANCH }}"
+          workflow_conclusion: completed
+          path: "reference"
+          if_no_artifact_found: warn
+
+      - name: Download full json results
+        uses: actions/download-artifact@v4
+        with:
+          name: diffutils-gnu-full-result
+          path: results
+
+      - name: Extract/summarize testing info
+        id: summary
+        shell: bash
+        run: |
+          ## Extract/summarize testing info
+          outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
+
+          RESULT_FILE="results/test-results.json"
+          if [[ ! -f "$RESULT_FILE" ]]; then
+            echo "::error ::Missing test results at $RESULT_FILE"
+            exit 1
+          fi
+
+          TOTAL=$(jq '[.tests[]] | length' "$RESULT_FILE")
+          PASS=$(jq '[.tests[] | select(.result=="PASS")] | length' "$RESULT_FILE")
+          FAIL=$(jq '[.tests[] | select(.result=="FAIL")] | length' "$RESULT_FILE")
+          SKIP=$(jq '[.tests[] | select(.result=="SKIP")] | length' "$RESULT_FILE")
+          ERROR=0
+
+          output="GNU diffutils tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP"
+          echo "${output}"
+
+          if [[ "$FAIL" -gt 0 ]]; then
+            echo "::warning ::${output}"
+          fi
+
+          jq -n \
+            --arg date "$(date --rfc-email)" \
+            --arg sha "$GITHUB_SHA" \
+            --arg total "$TOTAL" \
+            --arg pass "$PASS" \
+            --arg skip "$SKIP" \
+            --arg fail "$FAIL" \
+            --arg error "$ERROR" \
+            '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
+
+          HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
+          outputs HASH TOTAL PASS FAIL SKIP
+
+      - name: Upload SHA1/ID of 'test-summary'
+        uses: actions/upload-artifact@v4
+        with:
+          name: "${{ steps.summary.outputs.HASH }}"
+          path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
+
+      - name: Upload test results summary
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summary
+          path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
+
+      - name: Compare test failures VS reference
+        shell: bash
+        run: |
+          ## Compare test failures VS reference
+          REF_SUMMARY_FILE='reference/diffutils-gnu-full-result/test-results.json'
+          CURRENT_SUMMARY_FILE="results/test-results.json"
+
+          IGNORE_INTERMITTENT=".github/workflows/ignore-intermittent.txt"
+
+          COMMENT_DIR="reference/comment"
+          mkdir -p ${COMMENT_DIR}
+          echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
+          COMMENT_LOG="${COMMENT_DIR}/result.txt"
+
+          COMPARISON_RESULT=0
+          if test -f "${CURRENT_SUMMARY_FILE}"; then
+            if test -f "${REF_SUMMARY_FILE}"; then
+              echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
+              echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")"
+
+              python3 util/compare_test_results.py \
+                --ignore-file "${IGNORE_INTERMITTENT}" \
+                --output "${COMMENT_LOG}" \
+                "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
+
+              COMPARISON_RESULT=$?
+            else
+              echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'."
+            fi
+          else
+            echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early"
+            exit 1
+          fi
+
+          if [ ${COMPARISON_RESULT} -eq 1 ]; then
+            echo "::error ::Found new non-intermittent test failures"
+            exit 1
+          else
+            echo "::notice ::No new test failures detected"
+          fi
+
+      - name: Upload comparison log (for GnuComment workflow)
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: comment
+          path: reference/comment/
+
+      - name: Report test results
+        if: success() || failure()
+        shell: bash
+        run: |
+          ## Report final results
+          echo "::notice ::GNU diffutils testsuite results:"
+          echo "::notice ::  Total tests: ${{ steps.summary.outputs.TOTAL }}"
+          echo "::notice ::  Passed: ${{ steps.summary.outputs.PASS }}"
+          echo "::notice ::  Failed: ${{ steps.summary.outputs.FAIL }}"
+          echo "::notice ::  Skipped: ${{ steps.summary.outputs.SKIP }}"
+
+          if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then
+            PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) ))
+            echo "::notice ::  Pass rate: ${PASS_RATE}%"
+          fi

.github/workflows/ci.yml

@@ -56,37 +56,6 @@ jobs:
       - uses: actions/checkout@v4
       - run: cargo clippy -- -D warnings
 
-  gnu-testsuite:
-    permissions:
-      contents: write # Publish diffutils instead of discarding
-    name: GNU test suite
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          cargo build --config=profile.release.strip=true --profile=release #-fast
-          zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst
-      # do not fail, the report is merely informative (at least until all tests pass reliably)
-      - run: ./tests/run-upstream-testsuite.sh release || true
-        env:
-          TERM: xterm
-      - uses: actions/upload-artifact@v4
-        with:
-          name: test-results.json
-          path: tests/test-results.json
-      - run: ./tests/print-test-results.sh tests/test-results.json
-      - name: Publish latest commit
-        uses: softprops/action-gh-release@v3
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        with:
-          tag_name: latest-commit
-          draft: false
-          prerelease: true
-          files: |
-            diffutils-x86_64-unknown-linux-gnu.zst
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
   coverage:
     name: Code Coverage
     env:

util/compare_test_results.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+"""
+Compare the current GNU test results to the last results gathered from the main branch to
+highlight if a PR is making the results better/worse.
+Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
+"""
+
+import json
+import sys
+import argparse
+from pathlib import Path
+
+
+def load_ignore_list(ignore_file):
+    """Load list of intermittent test names to ignore from file."""
+    ignore_set = set()
+    if ignore_file and Path(ignore_file).exists():
+        with open(ignore_file, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    ignore_set.add(line)
+    return ignore_set
+
+
+def extract_test_results(json_data):
+    """Extract test results from a diffutils test-results.json.
+
+    Note: unlike sed, diffutils JSON has no 'summary' object — results are
+    computed from the 'tests' array using the 'result' and 'test' fields.
+    """
+    tests = json_data.get("tests", [])
+    passed  = sum(1 for t in tests if t.get("result") == "PASS")
+    failed  = sum(1 for t in tests if t.get("result") == "FAIL")
+    skipped = sum(1 for t in tests if t.get("result") == "SKIP")
+    summary = {"total": len(tests), "passed": passed, "failed": failed, "skipped": skipped}
+    failed_tests = [t["test"] for t in tests if t.get("result") == "FAIL"]
+    return summary, failed_tests
+
+
+def compare_results(current_file, reference_file, ignore_file=None, output_file=None):
+    """Compare current results with reference results."""
+    ignore_set = load_ignore_list(ignore_file)
+
+    try:
+        with open(current_file, "r") as f:
+            current_data = json.load(f)
+        current_summary, current_failed = extract_test_results(current_data)
+    except Exception as e:
+        print(f"Error loading current results: {e}")
+        return 1
+
+    try:
+        with open(reference_file, "r") as f:
+            reference_data = json.load(f)
+        reference_summary, reference_failed = extract_test_results(reference_data)
+    except Exception as e:
+        print(f"Error loading reference results: {e}")
+        return 1
+
+    # Calculate differences
+    pass_diff  = int(current_summary.get("passed",  0)) - int(reference_summary.get("passed",  0))
+    fail_diff  = int(current_summary.get("failed",  0)) - int(reference_summary.get("failed",  0))
+    total_diff = int(current_summary.get("total",   0)) - int(reference_summary.get("total",   0))
+
+    # Find new failures and improvements
+    current_failed_set   = set(current_failed)
+    reference_failed_set = set(reference_failed)
+
+    new_failures  = current_failed_set   - reference_failed_set
+    improvements  = reference_failed_set - current_failed_set
+
+    # Filter out intermittent failures
+    non_intermittent_new_failures = new_failures - ignore_set
+
+    # Check if results are identical (no changes)
+    no_changes = (
+        pass_diff == 0
+        and fail_diff == 0
+        and total_diff == 0
+        and not new_failures
+        and not improvements
+    )
+
+    # If no changes, write empty output to prevent comment posting
+    if no_changes:
+        if output_file:
+            with open(output_file, "w") as f:
+                f.write("")
+        return 0
+
+    # Prepare output message
+    output_lines = []
+
+    output_lines.append("Test results comparison:")
+    output_lines.append(
+        f"  Current:   TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}"
+    )
+    output_lines.append(
+        f"  Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}"
+    )
+    output_lines.append("")
+
+    if pass_diff != 0 or fail_diff != 0 or total_diff != 0:
+        output_lines.append("Changes from main branch:")
+        output_lines.append(f"  TOTAL: {total_diff:+d}")
+        output_lines.append(f"  PASSED: {pass_diff:+d}")
+        output_lines.append(f"  FAILED: {fail_diff:+d}")
+        output_lines.append("")
+
+    if new_failures:
+        output_lines.append(f"New test failures ({len(new_failures)}):")
+        for test in sorted(new_failures):
+            if test in ignore_set:
+                output_lines.append(f"  - {test} (intermittent)")
+            else:
+                output_lines.append(f"  - {test}")
+        output_lines.append("")
+
+    if improvements:
+        output_lines.append(f"Test improvements ({len(improvements)}):")
+        for test in sorted(improvements):
+            output_lines.append(f"  + {test}")
+        output_lines.append("")
+
+    output_text = "\n".join(output_lines)
+    if output_file:
+        with open(output_file, "w") as f:
+            f.write(output_text)
+    else:
+        print(output_text)
+
+    if non_intermittent_new_failures:
+        print(
+            f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures"
+        )
+        return 1
+
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare GNU diffutils test results")
+    parser.add_argument("current",   help="Current test results JSON file")
+    parser.add_argument("reference", help="Reference test results JSON file")
+    parser.add_argument(
+        "--ignore-file", help="File containing intermittent test names to ignore"
+    )
+    parser.add_argument("--output",  help="Output file for comparison results")
+
+    args = parser.parse_args()
+
+    return compare_results(args.current, args.reference, args.ignore_file, args.output)
+
+
+if __name__ == "__main__":
+    sys.exit(main())