mirror of
https://github.com/uutils/grep.git
synced 2026-06-10 16:15:11 -07:00
ci: add GnuTests workflow running the GNU grep testsuite
Add a GnuTests workflow that, on push and PR, fetches the GNU grep release tarball, builds the Rust grep binary, runs util/run-gnu-testsuite.sh, and uploads the JSON results. An aggregate job compares the run against the reference summary from the default branch (util/compare_test_results.py, borrowed from ../sed) and fails only on new, non-intermittent regressions; known-flaky tests are listed in .github/workflows/ignore-intermittent.txt.
This commit is contained in:
Sylvestre Ledru
@@ -0,0 +1,180 @@
|
||||
name: GnuTests
|
||||
|
||||
# Run the upstream GNU grep testsuite against the Rust grep implementation to
|
||||
# track and guard byte-for-byte compatibility. See util/run-gnu-testsuite.sh.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- '*'
|
||||
|
||||
# End the current execution if there is a new changeset in the PR.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEST_FULL_SUMMARY_FILE: 'grep-gnu-full-result.json'
|
||||
|
||||
jobs:
|
||||
native:
|
||||
name: Run GNU grep testsuite
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code (grep)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: 'grep'
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: "./grep -> target"
|
||||
|
||||
- name: Fetch GNU grep testsuite
|
||||
shell: bash
|
||||
run: |
|
||||
## Download and extract the upstream GNU grep release tarball
|
||||
mkdir -p gnu.grep
|
||||
cd gnu.grep
|
||||
bash ../grep/util/fetch-gnu.sh
|
||||
|
||||
- name: Build Rust grep binary
|
||||
shell: bash
|
||||
run: |
|
||||
cd 'grep'
|
||||
cargo build --release
|
||||
|
||||
- name: Run GNU grep testsuite
|
||||
shell: bash
|
||||
run: |
|
||||
cd 'grep'
|
||||
export GNU_GREP_DIR="../gnu.grep"
|
||||
./util/run-gnu-testsuite.sh --json-output "${{ env.TEST_FULL_SUMMARY_FILE }}" || true
|
||||
|
||||
- name: Upload full json results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: grep-gnu-full-result
|
||||
path: grep/${{ env.TEST_FULL_SUMMARY_FILE }}
|
||||
if-no-files-found: warn
|
||||
|
||||
aggregate:
|
||||
needs: [native]
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
name: Aggregate GNU test results
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Initialize workflow variables
|
||||
id: vars
|
||||
shell: bash
|
||||
run: |
|
||||
## VARs setup
|
||||
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
|
||||
TEST_SUMMARY_FILE='grep-gnu-result.json'
|
||||
outputs TEST_SUMMARY_FILE
|
||||
|
||||
- name: Checkout code (grep)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: 'grep'
|
||||
persist-credentials: false
|
||||
|
||||
- name: Retrieve reference artifacts
|
||||
uses: dawidd6/action-download-artifact@v6
|
||||
continue-on-error: true
|
||||
with:
|
||||
workflow: GnuTests.yml
|
||||
branch: "${{ env.DEFAULT_BRANCH }}"
|
||||
workflow_conclusion: completed
|
||||
path: "reference"
|
||||
if_no_artifact_found: warn
|
||||
|
||||
- name: Download full json results
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: grep-gnu-full-result
|
||||
path: results
|
||||
|
||||
- name: Extract/summarize testing info
|
||||
id: summary
|
||||
shell: bash
|
||||
run: |
|
||||
## Extract/summarize testing info
|
||||
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
|
||||
|
||||
RESULT_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}"
|
||||
if [[ ! -f "$RESULT_FILE" ]]; then
|
||||
echo "::error ::Result file $RESULT_FILE not found"
|
||||
find results -type f || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOTAL=$(jq -r '.summary.total // 0' "$RESULT_FILE")
|
||||
PASS=$(jq -r '.summary.passed // 0' "$RESULT_FILE")
|
||||
FAIL=$(jq -r '.summary.failed // 0' "$RESULT_FILE")
|
||||
SKIP=$(jq -r '.summary.skipped // 0' "$RESULT_FILE")
|
||||
|
||||
output="GNU grep tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP"
|
||||
echo "${output}"
|
||||
if [[ "$FAIL" -gt 0 ]]; then
|
||||
echo "::warning ::${output}"
|
||||
fi
|
||||
|
||||
outputs TOTAL PASS FAIL SKIP
|
||||
|
||||
- name: Compare test failures VS reference
|
||||
shell: bash
|
||||
run: |
|
||||
## Compare current results against the reference summary from the default branch
|
||||
REF_SUMMARY_FILE='reference/grep-gnu-full-result/${{ env.TEST_FULL_SUMMARY_FILE }}'
|
||||
CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}"
|
||||
IGNORE_INTERMITTENT="grep/.github/workflows/ignore-intermittent.txt"
|
||||
|
||||
# Set up comment directory for the GnuComment workflow.
|
||||
COMMENT_DIR="reference/comment"
|
||||
mkdir -p ${COMMENT_DIR}
|
||||
echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
|
||||
COMMENT_LOG="${COMMENT_DIR}/result.txt"
|
||||
: > "${COMMENT_LOG}"
|
||||
|
||||
COMPARISON_RESULT=0
|
||||
if test -f "${REF_SUMMARY_FILE}"; then
|
||||
python3 grep/util/compare_test_results.py \
|
||||
--ignore-file "${IGNORE_INTERMITTENT}" \
|
||||
--output "${COMMENT_LOG}" \
|
||||
"${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" || COMPARISON_RESULT=$?
|
||||
else
|
||||
echo "::warning ::Skipping test comparison; no prior reference summary at '${REF_SUMMARY_FILE}'."
|
||||
fi
|
||||
|
||||
if [ ${COMPARISON_RESULT} -eq 1 ]; then
|
||||
echo "::error ::Found new non-intermittent test failures"
|
||||
UPLOAD_EXIT=1
|
||||
else
|
||||
echo "::notice ::No new test failures detected"
|
||||
UPLOAD_EXIT=0
|
||||
fi
|
||||
echo "UPLOAD_EXIT=${UPLOAD_EXIT}" >> $GITHUB_ENV
|
||||
|
||||
- name: Upload comparison log (for GnuComment workflow)
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: comment
|
||||
path: reference/comment/
|
||||
|
||||
- name: Report test results
|
||||
if: success() || failure()
|
||||
shell: bash
|
||||
run: |
|
||||
echo "::notice ::GNU grep testsuite: TOTAL ${{ steps.summary.outputs.TOTAL }} / PASS ${{ steps.summary.outputs.PASS }} / FAIL ${{ steps.summary.outputs.FAIL }} / SKIP ${{ steps.summary.outputs.SKIP }}"
|
||||
# Fail the job if the comparison found new non-intermittent regressions.
|
||||
exit "${UPLOAD_EXIT:-0}"
|
||||
@@ -0,0 +1,7 @@
|
||||
# List of intermittent test names to ignore in result comparisons
|
||||
# Format: one test name per line, lines starting with # are comments
|
||||
#
|
||||
# Add test names that are known to be flaky or environment-dependent
|
||||
# Example:
|
||||
# basic_substitution
|
||||
# line_address_test
|
||||
Executable
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Compare the current GNU test results to the last results gathered from the main branch to
|
||||
highlight if a PR is making the results better/worse.
|
||||
Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_ignore_list(ignore_file):
|
||||
"""Load list of intermittent test names to ignore from file."""
|
||||
ignore_set = set()
|
||||
if ignore_file and Path(ignore_file).exists():
|
||||
with open(ignore_file, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
ignore_set.add(line)
|
||||
return ignore_set
|
||||
|
||||
|
||||
def extract_test_results(json_data):
|
||||
"""Extract test results from JSON data."""
|
||||
if not json_data or "summary" not in json_data:
|
||||
return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, []
|
||||
|
||||
summary = json_data["summary"]
|
||||
tests = json_data.get("tests", [])
|
||||
|
||||
# Extract failed test names
|
||||
failed_tests = []
|
||||
for test in tests:
|
||||
if test.get("status") == "FAIL":
|
||||
failed_tests.append(test.get("name", "unknown"))
|
||||
|
||||
return summary, failed_tests
|
||||
|
||||
|
||||
def compare_results(current_file, reference_file, ignore_file=None, output_file=None):
|
||||
"""Compare current results with reference results."""
|
||||
# Load ignore list
|
||||
ignore_set = load_ignore_list(ignore_file)
|
||||
|
||||
# Load JSON files
|
||||
try:
|
||||
with open(current_file, "r") as f:
|
||||
current_data = json.load(f)
|
||||
current_summary, current_failed = extract_test_results(current_data)
|
||||
except Exception as e:
|
||||
print(f"Error loading current results: {e}")
|
||||
return 1
|
||||
|
||||
try:
|
||||
with open(reference_file, "r") as f:
|
||||
reference_data = json.load(f)
|
||||
reference_summary, reference_failed = extract_test_results(reference_data)
|
||||
except Exception as e:
|
||||
print(f"Error loading reference results: {e}")
|
||||
return 1
|
||||
|
||||
# Calculate differences
|
||||
pass_diff = int(current_summary.get("passed", 0)) - int(
|
||||
reference_summary.get("passed", 0)
|
||||
)
|
||||
fail_diff = int(current_summary.get("failed", 0)) - int(
|
||||
reference_summary.get("failed", 0)
|
||||
)
|
||||
total_diff = int(current_summary.get("total", 0)) - int(
|
||||
reference_summary.get("total", 0)
|
||||
)
|
||||
|
||||
# Find new failures and improvements
|
||||
current_failed_set = set(current_failed)
|
||||
reference_failed_set = set(reference_failed)
|
||||
|
||||
new_failures = current_failed_set - reference_failed_set
|
||||
improvements = reference_failed_set - current_failed_set
|
||||
|
||||
# Filter out intermittent failures
|
||||
non_intermittent_new_failures = new_failures - ignore_set
|
||||
|
||||
# Check if results are identical (no changes)
|
||||
no_changes = (
|
||||
pass_diff == 0
|
||||
and fail_diff == 0
|
||||
and total_diff == 0
|
||||
and not new_failures
|
||||
and not improvements
|
||||
)
|
||||
|
||||
# If no changes, write empty output to prevent comment posting
|
||||
if no_changes:
|
||||
with open(output_file, "w") as f:
|
||||
f.write("")
|
||||
return 0
|
||||
|
||||
# Prepare output message
|
||||
output_lines = []
|
||||
|
||||
# Show current vs reference numbers for debugging
|
||||
output_lines.append("Test results comparison:")
|
||||
output_lines.append(
|
||||
f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}"
|
||||
)
|
||||
output_lines.append(
|
||||
f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}"
|
||||
)
|
||||
output_lines.append("")
|
||||
|
||||
# Summary of changes
|
||||
if pass_diff != 0 or fail_diff != 0 or total_diff != 0:
|
||||
output_lines.append("Changes from main branch:")
|
||||
output_lines.append(f" TOTAL: {total_diff:+d}")
|
||||
output_lines.append(f" PASSED: {pass_diff:+d}")
|
||||
output_lines.append(f" FAILED: {fail_diff:+d}")
|
||||
output_lines.append("")
|
||||
|
||||
# New failures
|
||||
if new_failures:
|
||||
output_lines.append(f"New test failures ({len(new_failures)}):")
|
||||
for test in sorted(new_failures):
|
||||
if test in ignore_set:
|
||||
output_lines.append(f" - {test} (intermittent)")
|
||||
else:
|
||||
output_lines.append(f" - {test}")
|
||||
output_lines.append("")
|
||||
|
||||
# Improvements
|
||||
if improvements:
|
||||
output_lines.append(f"Test improvements ({len(improvements)}):")
|
||||
for test in sorted(improvements):
|
||||
output_lines.append(f" + {test}")
|
||||
output_lines.append("")
|
||||
|
||||
# Write output
|
||||
output_text = "\n".join(output_lines)
|
||||
if output_file:
|
||||
with open(output_file, "w") as f:
|
||||
f.write(output_text)
|
||||
else:
|
||||
print(output_text)
|
||||
|
||||
# Return appropriate exit code
|
||||
if non_intermittent_new_failures:
|
||||
print(
|
||||
f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures"
|
||||
)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare GNU test results")
|
||||
parser.add_argument("current", help="Current test results JSON file")
|
||||
parser.add_argument("reference", help="Reference test results JSON file")
|
||||
parser.add_argument(
|
||||
"--ignore-file", help="File containing intermittent test names to ignore"
|
||||
)
|
||||
parser.add_argument("--output", help="Output file for comparison results")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
return compare_results(args.current, args.reference, args.ignore_file, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user