diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml new file mode 100644 index 0000000..5f5913e --- /dev/null +++ b/.github/workflows/GnuTests.yml @@ -0,0 +1,180 @@ +name: GnuTests + +# Run the upstream GNU grep testsuite against the Rust grep implementation to +# track and guard byte-for-byte compatibility. See util/run-gnu-testsuite.sh. + +on: + pull_request: + push: + branches: + - '*' + +# End the current execution if there is a new changeset in the PR. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + TEST_FULL_SUMMARY_FILE: 'grep-gnu-full-result.json' + +jobs: + native: + name: Run GNU grep testsuite + runs-on: ubuntu-24.04 + steps: + - name: Checkout code (grep) + uses: actions/checkout@v4 + with: + path: 'grep' + persist-credentials: false + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "./grep -> target" + + - name: Fetch GNU grep testsuite + shell: bash + run: | + ## Download and extract the upstream GNU grep release tarball + mkdir -p gnu.grep + cd gnu.grep + bash ../grep/util/fetch-gnu.sh + + - name: Build Rust grep binary + shell: bash + run: | + cd 'grep' + cargo build --release + + - name: Run GNU grep testsuite + shell: bash + run: | + cd 'grep' + export GNU_GREP_DIR="../gnu.grep" + ./util/run-gnu-testsuite.sh --json-output "${{ env.TEST_FULL_SUMMARY_FILE }}" || true + + - name: Upload full json results + uses: actions/upload-artifact@v4 + with: + name: grep-gnu-full-result + path: grep/${{ env.TEST_FULL_SUMMARY_FILE }} + if-no-files-found: warn + + aggregate: + needs: [native] + permissions: + actions: read + contents: read + pull-requests: read + name: Aggregate GNU test results + runs-on: ubuntu-24.04 + steps: + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + TEST_SUMMARY_FILE='grep-gnu-result.json' + outputs TEST_SUMMARY_FILE + + - name: Checkout code (grep) + uses: actions/checkout@v4 + with: + path: 'grep' + persist-credentials: false + + - name: Retrieve reference artifacts + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: GnuTests.yml + branch: "${{ env.DEFAULT_BRANCH }}" + workflow_conclusion: completed + path: "reference" + if_no_artifact_found: warn + + - name: Download full json results + uses: actions/download-artifact@v4 + with: + name: grep-gnu-full-result + path: results + + - name: Extract/summarize testing info + id: summary + shell: bash + run: | + ## Extract/summarize testing info + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + RESULT_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + if [[ ! -f "$RESULT_FILE" ]]; then + echo "::error ::Result file $RESULT_FILE not found" + find results -type f || true + exit 1 + fi + + TOTAL=$(jq -r '.summary.total // 0' "$RESULT_FILE") + PASS=$(jq -r '.summary.passed // 0' "$RESULT_FILE") + FAIL=$(jq -r '.summary.failed // 0' "$RESULT_FILE") + SKIP=$(jq -r '.summary.skipped // 0' "$RESULT_FILE") + + output="GNU grep tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" + echo "${output}" + if [[ "$FAIL" -gt 0 ]]; then + echo "::warning ::${output}" + fi + + outputs TOTAL PASS FAIL SKIP + + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare current results against the reference summary from the default branch + REF_SUMMARY_FILE='reference/grep-gnu-full-result/${{ env.TEST_FULL_SUMMARY_FILE }}' + CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + IGNORE_INTERMITTENT="grep/.github/workflows/ignore-intermittent.txt" + + # Set up comment directory for the GnuComment workflow. + COMMENT_DIR="reference/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + : > "${COMMENT_LOG}" + + COMPARISON_RESULT=0 + if test -f "${REF_SUMMARY_FILE}"; then + python3 grep/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" || COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary at '${REF_SUMMARY_FILE}'." + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "::error ::Found new non-intermittent test failures" + UPLOAD_EXIT=1 + else + echo "::notice ::No new test failures detected" + UPLOAD_EXIT=0 + fi + echo "UPLOAD_EXIT=${UPLOAD_EXIT}" >> $GITHUB_ENV + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: comment + path: reference/comment/ + + - name: Report test results + if: success() || failure() + shell: bash + run: | + echo "::notice ::GNU grep testsuite: TOTAL ${{ steps.summary.outputs.TOTAL }} / PASS ${{ steps.summary.outputs.PASS }} / FAIL ${{ steps.summary.outputs.FAIL }} / SKIP ${{ steps.summary.outputs.SKIP }}" + # Fail the job if the comparison found new non-intermittent regressions. + exit "${UPLOAD_EXIT:-0}" diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt new file mode 100644 index 0000000..7889f3d --- /dev/null +++ b/.github/workflows/ignore-intermittent.txt @@ -0,0 +1,7 @@ +# List of intermittent test names to ignore in result comparisons +# Format: one test name per line, lines starting with # are comments +# +# Add test names that are known to be flaky or environment-dependent +# Example: +# basic_substitution +# line_address_test diff --git a/util/compare_test_results.py b/util/compare_test_results.py new file mode 100755 index 0000000..7ceb3ff --- /dev/null +++ b/util/compare_test_results.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from JSON data.""" + if not json_data or "summary" not in json_data: + return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, [] + + summary = json_data["summary"] + tests = json_data.get("tests", []) + + # Extract failed test names + failed_tests = [] + for test in tests: + if test.get("status") == "FAIL": + failed_tests.append(test.get("name", "unknown")) + + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + # Load ignore list + ignore_set = load_ignore_list(ignore_file) + + # Load JSON files + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 1 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 1 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int( + reference_summary.get("passed", 0) + ) + fail_diff = int(current_summary.get("failed", 0)) - int( + reference_summary.get("failed", 0) + ) + total_diff = int(current_summary.get("total", 0)) - int( + reference_summary.get("total", 0) + ) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Check if results are identical (no changes) + no_changes = ( + pass_diff == 0 + and fail_diff == 0 + and total_diff == 0 + and not new_failures + and not improvements + ) + + # If no changes, write empty output to prevent comment posting + if no_changes: + with open(output_file, "w") as f: + f.write("") + return 0 + + # Prepare output message + output_lines = [] + + # Show current vs reference numbers for debugging + output_lines.append("Test results comparison:") + output_lines.append( + f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}" + ) + output_lines.append( + f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}" + ) + output_lines.append("") + + # Summary of changes + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + # New failures + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + # Improvements + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + # Write output + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + # Return appropriate exit code + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main())