diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml new file mode 100644 index 0000000..7fc8880 --- /dev/null +++ b/.github/workflows/GnuComment.yml @@ -0,0 +1,80 @@ +name: GnuComment + +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +permissions: {} +jobs: + post-comment: + permissions: + actions: read # to list workflow runs artifacts + pull-requests: write # to comment on pr + + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/github-script@v8 + with: + script: | + // List all artifacts from GnuTests + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + // Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + + if (!matchArtifact) { + console.log('No comment artifact found'); + return; + } + + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); + - run: unzip comment.zip || echo "Failed to unzip comment artifact" + + - name: 'Comment on PR' + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + + // Check if files exist + if (!fs.existsSync('./NR')) { + console.log('No NR file found, skipping comment'); + return; + } + if (!fs.existsSync('./result.txt')) { + console.log('No result.txt file found, skipping comment'); + return; + } + + var issue_number = Number(fs.readFileSync('./NR')); + var content = fs.readFileSync('./result.txt'); + + if (content.toString().trim().length > 7) { // 7 because we have backquote + \n + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: 'GNU sed testsuite comparison:\n```\n' + content + '```' + }); + } else { + console.log('Comment content too short, skipping'); + } \ No newline at end of file diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 0646a96..0f00f3f 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -208,7 +208,61 @@ jobs: name: test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare test failures VS reference using JSON files + REF_SUMMARY_FILE='reference/sed-gnu-full-result/sed-gnu-full-result.json' + CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + REPO_DEFAULT_BRANCH='${{ env.DEFAULT_BRANCH }}' + + # Path to ignore file for intermittent issues + IGNORE_INTERMITTENT="sed/.github/workflows/ignore-intermittent.txt" + + # Set up comment directory + COMMENT_DIR="reference/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + + COMPARISON_RESULT=0 + if test -f "${CURRENT_SUMMARY_FILE}"; then + if test -f "${REF_SUMMARY_FILE}"; then + echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" + + python3 sed/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" + + COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'." + fi + else + echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" + exit 1 + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV + echo "::error ::Found new non-intermittent test failures" + exit 1 + else + echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV + echo "::notice ::No new test failures detected" + fi + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v6 + with: + name: comment + path: reference/comment/ + - name: Report test results + if: success() || failure() shell: bash run: | ## Report final results diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt new file mode 100644 index 0000000..9e812e9 --- /dev/null +++ b/.github/workflows/ignore-intermittent.txt @@ -0,0 +1,7 @@ +# List of intermittent test names to ignore in result comparisons +# Format: one test name per line, lines starting with # are comments +# +# Add test names that are known to be flaky or environment-dependent +# Example: +# basic_substitution +# line_address_test \ No newline at end of file diff --git a/util/__pycache__/compare_test_results.cpython-313.pyc b/util/__pycache__/compare_test_results.cpython-313.pyc new file mode 100644 index 0000000..01d5d3e Binary files /dev/null and b/util/__pycache__/compare_test_results.cpython-313.pyc differ diff --git a/util/__pycache__/test_compare_test_results.cpython-313-pytest-8.3.5.pyc b/util/__pycache__/test_compare_test_results.cpython-313-pytest-8.3.5.pyc new file mode 100644 index 0000000..e2e6cc2 Binary files /dev/null and b/util/__pycache__/test_compare_test_results.cpython-313-pytest-8.3.5.pyc differ diff --git a/util/compare_test_results.py b/util/compare_test_results.py new file mode 100755 index 0000000..03a34cc --- /dev/null +++ b/util/compare_test_results.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from JSON data.""" + if not json_data or "summary" not in json_data: + return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, [] + + summary = json_data["summary"] + tests = json_data.get("tests", []) + + # Extract failed test names + failed_tests = [] + for test in tests: + if test.get("status") == "FAIL": + failed_tests.append(test.get("name", "unknown")) + + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + # Load ignore list + ignore_set = load_ignore_list(ignore_file) + + # Load JSON files + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 1 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 1 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int( + reference_summary.get("passed", 0) + ) + fail_diff = int(current_summary.get("failed", 0)) - int( + reference_summary.get("failed", 0) + ) + total_diff = int(current_summary.get("total", 0)) - int( + reference_summary.get("total", 0) + ) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Prepare output message + output_lines = [] + + # Show current vs reference numbers for debugging + output_lines.append("Test results comparison:") + output_lines.append(f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}") + output_lines.append(f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}") + output_lines.append("") + + # Summary of changes + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + # New failures + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + # Improvements + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + # Write output + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + # Return appropriate exit code + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/util/test_compare_test_results.py b/util/test_compare_test_results.py new file mode 100644 index 0000000..c7c5a81 --- /dev/null +++ b/util/test_compare_test_results.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 + +""" +Unit tests for compare_test_results.py +""" + +import json +import tempfile +import unittest +from pathlib import Path +from compare_test_results import load_ignore_list, extract_test_results, compare_results + + +class TestCompareTestResults(unittest.TestCase): + def setUp(self): + """Set up test fixtures.""" + self.temp_dir = tempfile.TemporaryDirectory() + self.temp_path = Path(self.temp_dir.name) + + # Sample test data + self.current_data = { + "timestamp": "2025-01-11T10:00:00Z", + "summary": {"total": 50, "passed": 40, "failed": 8, "skipped": 2}, + "tests": [ + {"name": "test1", "status": "PASS"}, + {"name": "test2", "status": "FAIL"}, + {"name": "test3", "status": "PASS"}, + {"name": "new_test", "status": "FAIL"}, + ], + } + + self.reference_data = { + "timestamp": "2025-01-10T10:00:00Z", + "summary": {"total": 48, "passed": 42, "failed": 5, "skipped": 1}, + "tests": [ + {"name": "test1", "status": "PASS"}, + {"name": "old_failure", "status": "FAIL"}, + {"name": "test3", "status": "PASS"}, + ], + } + + def tearDown(self): + """Clean up test fixtures.""" + self.temp_dir.cleanup() + + def test_load_ignore_list_empty_file(self): + """Test loading ignore list from empty file.""" + ignore_file = self.temp_path / "empty_ignore.txt" + ignore_file.write_text("") + + result = load_ignore_list(str(ignore_file)) + self.assertEqual(result, set()) + + def test_load_ignore_list_with_content(self): + """Test loading ignore list with actual content.""" + ignore_file = self.temp_path / "ignore.txt" + ignore_file.write_text("# Comment\ntest1\ntest2\n\n# Another comment\ntest3") + + result = load_ignore_list(str(ignore_file)) + self.assertEqual(result, {"test1", "test2", "test3"}) + + def test_load_ignore_list_nonexistent_file(self): + """Test loading ignore list from nonexistent file.""" + result = load_ignore_list("nonexistent.txt") + self.assertEqual(result, set()) + + def test_extract_test_results_valid_data(self): + """Test extracting test results from valid JSON data.""" + summary, failed_tests = extract_test_results(self.current_data) + + self.assertEqual(summary["total"], 50) + self.assertEqual(summary["passed"], 40) + self.assertEqual(summary["failed"], 8) + self.assertEqual(summary["skipped"], 2) + self.assertEqual(set(failed_tests), {"test2", "new_test"}) + + def test_extract_test_results_missing_summary(self): + """Test extracting test results from data without summary.""" + data = {"tests": [{"name": "test1", "status": "PASS"}]} + summary, failed_tests = extract_test_results(data) + + self.assertEqual(summary["total"], 0) + self.assertEqual(failed_tests, []) + + def test_extract_test_results_empty_data(self): + """Test extracting test results from empty data.""" + summary, failed_tests = extract_test_results({}) + + self.assertEqual(summary["total"], 0) + self.assertEqual(failed_tests, []) + + def test_compare_results_improvements_and_new_failures(self): + """Test comparison showing both improvements and new failures.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + output_file = self.temp_path / "output.txt" + + current_file.write_text(json.dumps(self.current_data)) + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results( + str(current_file), str(reference_file), output_file=str(output_file) + ) + + # Should return 1 because there are new failures + self.assertEqual(result, 1) + + # Check output content + output_content = output_file.read_text() + self.assertIn("Test results comparison:", output_content) + self.assertIn("Changes from main branch:", output_content) + self.assertIn("TOTAL: +2", output_content) + self.assertIn("PASSED: -2", output_content) + self.assertIn("FAILED: +3", output_content) + self.assertIn("New test failures (2):", output_content) + self.assertIn("- test2", output_content) + self.assertIn("- new_test", output_content) + self.assertIn("Test improvements (1):", output_content) + self.assertIn("+ old_failure", output_content) + + def test_compare_results_with_ignore_list(self): + """Test comparison with ignored intermittent failures.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + ignore_file = self.temp_path / "ignore.txt" + output_file = self.temp_path / "output.txt" + + current_file.write_text(json.dumps(self.current_data)) + reference_file.write_text(json.dumps(self.reference_data)) + ignore_file.write_text("test2\nnew_test") # Ignore both new failures + + result = compare_results( + str(current_file), str(reference_file), str(ignore_file), str(output_file) + ) + + # Should return 0 because all new failures are ignored + self.assertEqual(result, 0) + + # Check that intermittent failures are marked + output_content = output_file.read_text() + self.assertIn("- test2 (intermittent)", output_content) + self.assertIn("- new_test (intermittent)", output_content) + + def test_compare_results_no_changes(self): + """Test comparison with identical results.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + output_file = self.temp_path / "output.txt" + + # Use same data for both files + current_file.write_text(json.dumps(self.reference_data)) + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results( + str(current_file), str(reference_file), output_file=str(output_file) + ) + + # Should return 0 for no new failures + self.assertEqual(result, 0) + + # Output should be minimal or empty + output_content = output_file.read_text() + self.assertNotIn("New test failures", output_content) + self.assertNotIn("Test improvements", output_content) + + def test_compare_results_only_improvements(self): + """Test comparison with only improvements (no new failures).""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + output_file = self.temp_path / "output.txt" + + # Create data where current has fewer failures than reference + improved_data = { + "summary": {"total": 48, "passed": 47, "failed": 0, "skipped": 1}, + "tests": [ + {"name": "test1", "status": "PASS"}, + {"name": "test3", "status": "PASS"}, + {"name": "old_failure", "status": "PASS"}, # Fixed the old failure! + ], + } + + current_file.write_text(json.dumps(improved_data)) + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results( + str(current_file), str(reference_file), output_file=str(output_file) + ) + + # Should return 0 for no new failures + self.assertEqual(result, 0) + + output_content = output_file.read_text() + self.assertIn("Test improvements", output_content) + # Should show improvement from old_failure + self.assertIn("+ old_failure", output_content) + # Should have no new failures + self.assertNotIn("New test failures", output_content) + + def test_compare_results_invalid_current_file(self): + """Test comparison with invalid current file.""" + reference_file = self.temp_path / "reference.json" + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results("nonexistent_current.json", str(reference_file)) + + # Should return 1 for error + self.assertEqual(result, 1) + + def test_compare_results_invalid_reference_file(self): + """Test comparison with invalid reference file.""" + current_file = self.temp_path / "current.json" + current_file.write_text(json.dumps(self.current_data)) + + result = compare_results(str(current_file), "nonexistent_reference.json") + + # Should return 1 for error + self.assertEqual(result, 1) + + def test_compare_results_malformed_json(self): + """Test comparison with malformed JSON files.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + + current_file.write_text("{ invalid json") + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results(str(current_file), str(reference_file)) + + # Should return 1 for error + self.assertEqual(result, 1) + + +if __name__ == "__main__": + unittest.main()