Merge pull request #247 from sylvestre/gnu-ci-info

Publish comments in case of changes with the GNU test suite

.github/workflows/GnuComment.yml

@@ -0,0 +1,80 @@
+name: GnuComment
+
+on:
+  workflow_run:
+    workflows: ["GnuTests"]
+    types:
+      - completed
+
+permissions: {}
+jobs:
+  post-comment:
+    permissions:
+      actions: read  #  to list workflow runs artifacts
+      pull-requests: write  #  to comment on pr
+
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: 'Download artifact'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            // List all artifacts from GnuTests
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: ${{ github.event.workflow_run.id }},
+            });
+
+            // Download the "comment" artifact, which contains a PR number (NR) and result.txt
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "comment"
+            })[0];
+            
+            if (!matchArtifact) {
+              console.log('No comment artifact found');
+              return;
+            }
+            
+            var download = await github.rest.actions.downloadArtifact({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: matchArtifact.id,
+              archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
+      - run: unzip comment.zip || echo "Failed to unzip comment artifact"
+
+      - name: 'Comment on PR'
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            var fs = require('fs');
+            
+            // Check if files exist
+            if (!fs.existsSync('./NR')) {
+              console.log('No NR file found, skipping comment');
+              return;
+            }
+            if (!fs.existsSync('./result.txt')) {
+              console.log('No result.txt file found, skipping comment');
+              return;
+            }
+            
+            var issue_number = Number(fs.readFileSync('./NR'));
+            var content = fs.readFileSync('./result.txt');
+            
+            if (content.toString().trim().length > 7) { // 7 because we have backquote + \n
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue_number,
+                body: 'GNU sed testsuite comparison:\n```\n' + content + '```'
+              });
+            } else {
+              console.log('Comment content too short, skipping');
+            }

.github/workflows/GnuTests.yml

@@ -208,7 +208,61 @@ jobs:
         name: test-summary
         path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
 
+    - name: Compare test failures VS reference
+      shell: bash
+      run: |
+        ## Compare test failures VS reference using JSON files
+        REF_SUMMARY_FILE='reference/sed-gnu-full-result/sed-gnu-full-result.json'
+        CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}"
+        REPO_DEFAULT_BRANCH='${{ env.DEFAULT_BRANCH }}'
+
+        # Path to ignore file for intermittent issues
+        IGNORE_INTERMITTENT="sed/.github/workflows/ignore-intermittent.txt"
+
+        # Set up comment directory
+        COMMENT_DIR="reference/comment"
+        mkdir -p ${COMMENT_DIR}
+        echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
+        COMMENT_LOG="${COMMENT_DIR}/result.txt"
+
+        COMPARISON_RESULT=0
+        if test -f "${CURRENT_SUMMARY_FILE}"; then
+          if test -f "${REF_SUMMARY_FILE}"; then
+            echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
+            echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")"
+
+            python3 sed/util/compare_test_results.py \
+              --ignore-file "${IGNORE_INTERMITTENT}" \
+              --output "${COMMENT_LOG}" \
+              "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
+
+            COMPARISON_RESULT=$?
+          else
+            echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'."
+          fi
+        else
+          echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early"
+          exit 1
+        fi
+
+        if [ ${COMPARISON_RESULT} -eq 1 ]; then
+          echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV
+          echo "::error ::Found new non-intermittent test failures"
+          exit 1
+        else
+          echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV
+          echo "::notice ::No new test failures detected"
+        fi
+    
+    - name: Upload comparison log (for GnuComment workflow)
+      if: success() || failure()
+      uses: actions/upload-artifact@v6
+      with:
+        name: comment
+        path: reference/comment/
+
     - name: Report test results
+      if: success() || failure()
       shell: bash
       run: |
         ## Report final results

.github/workflows/ignore-intermittent.txt

@@ -0,0 +1,7 @@
+# List of intermittent test names to ignore in result comparisons
+# Format: one test name per line, lines starting with # are comments
+#
+# Add test names that are known to be flaky or environment-dependent
+# Example:
+# basic_substitution
+# line_address_test

util/compare_test_results.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+"""
+Compare the current GNU test results to the last results gathered from the main branch to
+highlight if a PR is making the results better/worse.
+Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
+"""
+
+import json
+import sys
+import argparse
+from pathlib import Path
+
+
+def load_ignore_list(ignore_file):
+    """Load list of intermittent test names to ignore from file."""
+    ignore_set = set()
+    if ignore_file and Path(ignore_file).exists():
+        with open(ignore_file, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    ignore_set.add(line)
+    return ignore_set
+
+
+def extract_test_results(json_data):
+    """Extract test results from JSON data."""
+    if not json_data or "summary" not in json_data:
+        return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, []
+
+    summary = json_data["summary"]
+    tests = json_data.get("tests", [])
+
+    # Extract failed test names
+    failed_tests = []
+    for test in tests:
+        if test.get("status") == "FAIL":
+            failed_tests.append(test.get("name", "unknown"))
+
+    return summary, failed_tests
+
+
+def compare_results(current_file, reference_file, ignore_file=None, output_file=None):
+    """Compare current results with reference results."""
+    # Load ignore list
+    ignore_set = load_ignore_list(ignore_file)
+
+    # Load JSON files
+    try:
+        with open(current_file, "r") as f:
+            current_data = json.load(f)
+        current_summary, current_failed = extract_test_results(current_data)
+    except Exception as e:
+        print(f"Error loading current results: {e}")
+        return 1
+
+    try:
+        with open(reference_file, "r") as f:
+            reference_data = json.load(f)
+        reference_summary, reference_failed = extract_test_results(reference_data)
+    except Exception as e:
+        print(f"Error loading reference results: {e}")
+        return 1
+
+    # Calculate differences
+    pass_diff = int(current_summary.get("passed", 0)) - int(
+        reference_summary.get("passed", 0)
+    )
+    fail_diff = int(current_summary.get("failed", 0)) - int(
+        reference_summary.get("failed", 0)
+    )
+    total_diff = int(current_summary.get("total", 0)) - int(
+        reference_summary.get("total", 0)
+    )
+
+    # Find new failures and improvements
+    current_failed_set = set(current_failed)
+    reference_failed_set = set(reference_failed)
+
+    new_failures = current_failed_set - reference_failed_set
+    improvements = reference_failed_set - current_failed_set
+
+    # Filter out intermittent failures
+    non_intermittent_new_failures = new_failures - ignore_set
+
+    # Prepare output message
+    output_lines = []
+
+    # Show current vs reference numbers for debugging
+    output_lines.append("Test results comparison:")
+    output_lines.append(f"  Current:   TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}")
+    output_lines.append(f"  Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}")
+    output_lines.append("")
+
+    # Summary of changes
+    if pass_diff != 0 or fail_diff != 0 or total_diff != 0:
+        output_lines.append("Changes from main branch:")
+        output_lines.append(f"  TOTAL: {total_diff:+d}")
+        output_lines.append(f"  PASSED: {pass_diff:+d}")
+        output_lines.append(f"  FAILED: {fail_diff:+d}")
+        output_lines.append("")
+
+    # New failures
+    if new_failures:
+        output_lines.append(f"New test failures ({len(new_failures)}):")
+        for test in sorted(new_failures):
+            if test in ignore_set:
+                output_lines.append(f"  - {test} (intermittent)")
+            else:
+                output_lines.append(f"  - {test}")
+        output_lines.append("")
+
+    # Improvements
+    if improvements:
+        output_lines.append(f"Test improvements ({len(improvements)}):")
+        for test in sorted(improvements):
+            output_lines.append(f"  + {test}")
+        output_lines.append("")
+
+    # Write output
+    output_text = "\n".join(output_lines)
+    if output_file:
+        with open(output_file, "w") as f:
+            f.write(output_text)
+    else:
+        print(output_text)
+
+    # Return appropriate exit code
+    if non_intermittent_new_failures:
+        print(
+            f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures"
+        )
+        return 1
+
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare GNU test results")
+    parser.add_argument("current", help="Current test results JSON file")
+    parser.add_argument("reference", help="Reference test results JSON file")
+    parser.add_argument(
+        "--ignore-file", help="File containing intermittent test names to ignore"
+    )
+    parser.add_argument("--output", help="Output file for comparison results")
+
+    args = parser.parse_args()
+
+    return compare_results(args.current, args.reference, args.ignore_file, args.output)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

util/test_compare_test_results.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+
+"""
+Unit tests for compare_test_results.py
+"""
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from compare_test_results import load_ignore_list, extract_test_results, compare_results
+
+
+class TestCompareTestResults(unittest.TestCase):
+    def setUp(self):
+        """Set up test fixtures."""
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.temp_path = Path(self.temp_dir.name)
+
+        # Sample test data
+        self.current_data = {
+            "timestamp": "2025-01-11T10:00:00Z",
+            "summary": {"total": 50, "passed": 40, "failed": 8, "skipped": 2},
+            "tests": [
+                {"name": "test1", "status": "PASS"},
+                {"name": "test2", "status": "FAIL"},
+                {"name": "test3", "status": "PASS"},
+                {"name": "new_test", "status": "FAIL"},
+            ],
+        }
+
+        self.reference_data = {
+            "timestamp": "2025-01-10T10:00:00Z",
+            "summary": {"total": 48, "passed": 42, "failed": 5, "skipped": 1},
+            "tests": [
+                {"name": "test1", "status": "PASS"},
+                {"name": "old_failure", "status": "FAIL"},
+                {"name": "test3", "status": "PASS"},
+            ],
+        }
+
+    def tearDown(self):
+        """Clean up test fixtures."""
+        self.temp_dir.cleanup()
+
+    def test_load_ignore_list_empty_file(self):
+        """Test loading ignore list from empty file."""
+        ignore_file = self.temp_path / "empty_ignore.txt"
+        ignore_file.write_text("")
+
+        result = load_ignore_list(str(ignore_file))
+        self.assertEqual(result, set())
+
+    def test_load_ignore_list_with_content(self):
+        """Test loading ignore list with actual content."""
+        ignore_file = self.temp_path / "ignore.txt"
+        ignore_file.write_text("# Comment\ntest1\ntest2\n\n# Another comment\ntest3")
+
+        result = load_ignore_list(str(ignore_file))
+        self.assertEqual(result, {"test1", "test2", "test3"})
+
+    def test_load_ignore_list_nonexistent_file(self):
+        """Test loading ignore list from nonexistent file."""
+        result = load_ignore_list("nonexistent.txt")
+        self.assertEqual(result, set())
+
+    def test_extract_test_results_valid_data(self):
+        """Test extracting test results from valid JSON data."""
+        summary, failed_tests = extract_test_results(self.current_data)
+
+        self.assertEqual(summary["total"], 50)
+        self.assertEqual(summary["passed"], 40)
+        self.assertEqual(summary["failed"], 8)
+        self.assertEqual(summary["skipped"], 2)
+        self.assertEqual(set(failed_tests), {"test2", "new_test"})
+
+    def test_extract_test_results_missing_summary(self):
+        """Test extracting test results from data without summary."""
+        data = {"tests": [{"name": "test1", "status": "PASS"}]}
+        summary, failed_tests = extract_test_results(data)
+
+        self.assertEqual(summary["total"], 0)
+        self.assertEqual(failed_tests, [])
+
+    def test_extract_test_results_empty_data(self):
+        """Test extracting test results from empty data."""
+        summary, failed_tests = extract_test_results({})
+
+        self.assertEqual(summary["total"], 0)
+        self.assertEqual(failed_tests, [])
+
+    def test_compare_results_improvements_and_new_failures(self):
+        """Test comparison showing both improvements and new failures."""
+        current_file = self.temp_path / "current.json"
+        reference_file = self.temp_path / "reference.json"
+        output_file = self.temp_path / "output.txt"
+
+        current_file.write_text(json.dumps(self.current_data))
+        reference_file.write_text(json.dumps(self.reference_data))
+
+        result = compare_results(
+            str(current_file), str(reference_file), output_file=str(output_file)
+        )
+
+        # Should return 1 because there are new failures
+        self.assertEqual(result, 1)
+
+        # Check output content
+        output_content = output_file.read_text()
+        self.assertIn("Test results comparison:", output_content)
+        self.assertIn("Changes from main branch:", output_content)
+        self.assertIn("TOTAL: +2", output_content)
+        self.assertIn("PASSED: -2", output_content)
+        self.assertIn("FAILED: +3", output_content)
+        self.assertIn("New test failures (2):", output_content)
+        self.assertIn("- test2", output_content)
+        self.assertIn("- new_test", output_content)
+        self.assertIn("Test improvements (1):", output_content)
+        self.assertIn("+ old_failure", output_content)
+
+    def test_compare_results_with_ignore_list(self):
+        """Test comparison with ignored intermittent failures."""
+        current_file = self.temp_path / "current.json"
+        reference_file = self.temp_path / "reference.json"
+        ignore_file = self.temp_path / "ignore.txt"
+        output_file = self.temp_path / "output.txt"
+
+        current_file.write_text(json.dumps(self.current_data))
+        reference_file.write_text(json.dumps(self.reference_data))
+        ignore_file.write_text("test2\nnew_test")  # Ignore both new failures
+
+        result = compare_results(
+            str(current_file), str(reference_file), str(ignore_file), str(output_file)
+        )
+
+        # Should return 0 because all new failures are ignored
+        self.assertEqual(result, 0)
+
+        # Check that intermittent failures are marked
+        output_content = output_file.read_text()
+        self.assertIn("- test2 (intermittent)", output_content)
+        self.assertIn("- new_test (intermittent)", output_content)
+
+    def test_compare_results_no_changes(self):
+        """Test comparison with identical results."""
+        current_file = self.temp_path / "current.json"
+        reference_file = self.temp_path / "reference.json"
+        output_file = self.temp_path / "output.txt"
+
+        # Use same data for both files
+        current_file.write_text(json.dumps(self.reference_data))
+        reference_file.write_text(json.dumps(self.reference_data))
+
+        result = compare_results(
+            str(current_file), str(reference_file), output_file=str(output_file)
+        )
+
+        # Should return 0 for no new failures
+        self.assertEqual(result, 0)
+
+        # Output should be minimal or empty
+        output_content = output_file.read_text()
+        self.assertNotIn("New test failures", output_content)
+        self.assertNotIn("Test improvements", output_content)
+
+    def test_compare_results_only_improvements(self):
+        """Test comparison with only improvements (no new failures)."""
+        current_file = self.temp_path / "current.json"
+        reference_file = self.temp_path / "reference.json"
+        output_file = self.temp_path / "output.txt"
+
+        # Create data where current has fewer failures than reference
+        improved_data = {
+            "summary": {"total": 48, "passed": 47, "failed": 0, "skipped": 1},
+            "tests": [
+                {"name": "test1", "status": "PASS"},
+                {"name": "test3", "status": "PASS"},
+                {"name": "old_failure", "status": "PASS"},  # Fixed the old failure!
+            ],
+        }
+
+        current_file.write_text(json.dumps(improved_data))
+        reference_file.write_text(json.dumps(self.reference_data))
+
+        result = compare_results(
+            str(current_file), str(reference_file), output_file=str(output_file)
+        )
+
+        # Should return 0 for no new failures
+        self.assertEqual(result, 0)
+
+        output_content = output_file.read_text()
+        self.assertIn("Test improvements", output_content)
+        # Should show improvement from old_failure
+        self.assertIn("+ old_failure", output_content)
+        # Should have no new failures
+        self.assertNotIn("New test failures", output_content)
+
+    def test_compare_results_invalid_current_file(self):
+        """Test comparison with invalid current file."""
+        reference_file = self.temp_path / "reference.json"
+        reference_file.write_text(json.dumps(self.reference_data))
+
+        result = compare_results("nonexistent_current.json", str(reference_file))
+
+        # Should return 1 for error
+        self.assertEqual(result, 1)
+
+    def test_compare_results_invalid_reference_file(self):
+        """Test comparison with invalid reference file."""
+        current_file = self.temp_path / "current.json"
+        current_file.write_text(json.dumps(self.current_data))
+
+        result = compare_results(str(current_file), "nonexistent_reference.json")
+
+        # Should return 1 for error
+        self.assertEqual(result, 1)
+
+    def test_compare_results_malformed_json(self):
+        """Test comparison with malformed JSON files."""
+        current_file = self.temp_path / "current.json"
+        reference_file = self.temp_path / "reference.json"
+
+        current_file.write_text("{ invalid json")
+        reference_file.write_text(json.dumps(self.reference_data))
+
+        result = compare_results(str(current_file), str(reference_file))
+
+        # Should return 1 for error
+        self.assertEqual(result, 1)
+
+
+if __name__ == "__main__":
+    unittest.main()