Add test result comparison script

util/compare_test_results.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+"""
+Compare the current GNU test results to the last results gathered from the main branch to
+highlight if a PR is making the results better/worse.
+Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
+"""
+
+import json
+import sys
+import argparse
+from pathlib import Path
+
+
+def load_ignore_list(ignore_file):
+    """Load list of intermittent test names to ignore from file."""
+    ignore_set = set()
+    if ignore_file and Path(ignore_file).exists():
+        with open(ignore_file, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    ignore_set.add(line)
+    return ignore_set
+
+
+def extract_test_results(json_data):
+    """Extract test results from JSON data."""
+    if not json_data or "summary" not in json_data:
+        return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, []
+
+    summary = json_data["summary"]
+    tests = json_data.get("tests", [])
+
+    # Extract failed test names
+    failed_tests = []
+    for test in tests:
+        if test.get("status") == "FAIL":
+            failed_tests.append(test.get("name", "unknown"))
+
+    return summary, failed_tests
+
+
+def compare_results(current_file, reference_file, ignore_file=None, output_file=None):
+    """Compare current results with reference results."""
+    # Load ignore list
+    ignore_set = load_ignore_list(ignore_file)
+
+    # Load JSON files
+    try:
+        with open(current_file, "r") as f:
+            current_data = json.load(f)
+        current_summary, current_failed = extract_test_results(current_data)
+    except Exception as e:
+        print(f"Error loading current results: {e}")
+        return 1
+
+    try:
+        with open(reference_file, "r") as f:
+            reference_data = json.load(f)
+        reference_summary, reference_failed = extract_test_results(reference_data)
+    except Exception as e:
+        print(f"Error loading reference results: {e}")
+        return 1
+
+    # Calculate differences
+    pass_diff = int(current_summary.get("passed", 0)) - int(
+        reference_summary.get("passed", 0)
+    )
+    fail_diff = int(current_summary.get("failed", 0)) - int(
+        reference_summary.get("failed", 0)
+    )
+    total_diff = int(current_summary.get("total", 0)) - int(
+        reference_summary.get("total", 0)
+    )
+
+    # Find new failures and improvements
+    current_failed_set = set(current_failed)
+    reference_failed_set = set(reference_failed)
+
+    new_failures = current_failed_set - reference_failed_set
+    improvements = reference_failed_set - current_failed_set
+
+    # Filter out intermittent failures
+    non_intermittent_new_failures = new_failures - ignore_set
+
+    # Prepare output message
+    output_lines = []
+
+    # Summary of changes
+    if pass_diff != 0 or fail_diff != 0 or total_diff != 0:
+        output_lines.append("Test result changes from main branch:")
+        output_lines.append(f"  TOTAL: {total_diff:+d}")
+        output_lines.append(f"  PASSED: {pass_diff:+d}")
+        output_lines.append(f"  FAILED: {fail_diff:+d}")
+        output_lines.append("")
+
+    # New failures
+    if new_failures:
+        output_lines.append(f"New test failures ({len(new_failures)}):")
+        for test in sorted(new_failures):
+            if test in ignore_set:
+                output_lines.append(f"  - {test} (intermittent)")
+            else:
+                output_lines.append(f"  - {test}")
+        output_lines.append("")
+
+    # Improvements
+    if improvements:
+        output_lines.append(f"Test improvements ({len(improvements)}):")
+        for test in sorted(improvements):
+            output_lines.append(f"  + {test}")
+        output_lines.append("")
+
+    # Write output
+    output_text = "\n".join(output_lines)
+    if output_file:
+        with open(output_file, "w") as f:
+            f.write(output_text)
+    else:
+        print(output_text)
+
+    # Return appropriate exit code
+    if non_intermittent_new_failures:
+        print(
+            f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures"
+        )
+        return 1
+
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare GNU test results")
+    parser.add_argument("current", help="Current test results JSON file")
+    parser.add_argument("reference", help="Reference test results JSON file")
+    parser.add_argument(
+        "--ignore-file", help="File containing intermittent test names to ignore"
+    )
+    parser.add_argument("--output", help="Output file for comparison results")
+
+    args = parser.parse_args()
+
+    return compare_results(args.current, args.reference, args.ignore_file, args.output)
+
+
+if __name__ == "__main__":
+    sys.exit(main())