From 35ea457759b48a9a5b2a00bb22cee3c6a128ae7d Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 11 Jan 2026 15:44:25 +0100 Subject: [PATCH 1/6] Add test result comparison script --- util/compare_test_results.py | 148 +++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100755 util/compare_test_results.py diff --git a/util/compare_test_results.py b/util/compare_test_results.py new file mode 100755 index 0000000..8714bc9 --- /dev/null +++ b/util/compare_test_results.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from JSON data.""" + if not json_data or "summary" not in json_data: + return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, [] + + summary = json_data["summary"] + tests = json_data.get("tests", []) + + # Extract failed test names + failed_tests = [] + for test in tests: + if test.get("status") == "FAIL": + failed_tests.append(test.get("name", "unknown")) + + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + # Load ignore list + ignore_set = load_ignore_list(ignore_file) + + # Load JSON files + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 1 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 1 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int( + reference_summary.get("passed", 0) + ) + fail_diff = int(current_summary.get("failed", 0)) - int( + reference_summary.get("failed", 0) + ) + total_diff = int(current_summary.get("total", 0)) - int( + reference_summary.get("total", 0) + ) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Prepare output message + output_lines = [] + + # Summary of changes + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Test result changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + # New failures + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + # Improvements + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + # Write output + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + # Return appropriate exit code + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main()) From ce3e4b33b580307c867e43d55bcedce4e8ca0fba Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 11 Jan 2026 15:44:52 +0100 Subject: [PATCH 2/6] Add automatic PR comment workflow for test comparisons --- .github/workflows/GnuComment.yml | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/GnuComment.yml diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml new file mode 100644 index 0000000..7fc8880 --- /dev/null +++ b/.github/workflows/GnuComment.yml @@ -0,0 +1,80 @@ +name: GnuComment + +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +permissions: {} +jobs: + post-comment: + permissions: + actions: read # to list workflow runs artifacts + pull-requests: write # to comment on pr + + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/github-script@v8 + with: + script: | + // List all artifacts from GnuTests + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + // Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + + if (!matchArtifact) { + console.log('No comment artifact found'); + return; + } + + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); + - run: unzip comment.zip || echo "Failed to unzip comment artifact" + + - name: 'Comment on PR' + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + + // Check if files exist + if (!fs.existsSync('./NR')) { + console.log('No NR file found, skipping comment'); + return; + } + if (!fs.existsSync('./result.txt')) { + console.log('No result.txt file found, skipping comment'); + return; + } + + var issue_number = Number(fs.readFileSync('./NR')); + var content = fs.readFileSync('./result.txt'); + + if (content.toString().trim().length > 7) { // 7 because we have backquote + \n + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: 'GNU sed testsuite comparison:\n```\n' + content + '```' + }); + } else { + console.log('Comment content too short, skipping'); + } \ No newline at end of file From 5ff4a7c0c295963c63dcdda51c7cf14941611935 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 11 Jan 2026 16:14:02 +0100 Subject: [PATCH 3/6] Add ignore list for intermittent test failures --- .github/workflows/ignore-intermittent.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/workflows/ignore-intermittent.txt diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt new file mode 100644 index 0000000..9e812e9 --- /dev/null +++ b/.github/workflows/ignore-intermittent.txt @@ -0,0 +1,7 @@ +# List of intermittent test names to ignore in result comparisons +# Format: one test name per line, lines starting with # are comments +# +# Add test names that are known to be flaky or environment-dependent +# Example: +# basic_substitution +# line_address_test \ No newline at end of file From 69ddd044c552ff100c3705a3142cc8397882f536 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 11 Jan 2026 16:16:22 +0100 Subject: [PATCH 4/6] Integrate test result comparison in CI workflow --- .github/workflows/GnuTests.yml | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 0646a96..0f00f3f 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -208,7 +208,61 @@ jobs: name: test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare test failures VS reference using JSON files + REF_SUMMARY_FILE='reference/sed-gnu-full-result/sed-gnu-full-result.json' + CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + REPO_DEFAULT_BRANCH='${{ env.DEFAULT_BRANCH }}' + + # Path to ignore file for intermittent issues + IGNORE_INTERMITTENT="sed/.github/workflows/ignore-intermittent.txt" + + # Set up comment directory + COMMENT_DIR="reference/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + + COMPARISON_RESULT=0 + if test -f "${CURRENT_SUMMARY_FILE}"; then + if test -f "${REF_SUMMARY_FILE}"; then + echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" + + python3 sed/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" + + COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'." + fi + else + echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" + exit 1 + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV + echo "::error ::Found new non-intermittent test failures" + exit 1 + else + echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV + echo "::notice ::No new test failures detected" + fi + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v6 + with: + name: comment + path: reference/comment/ + - name: Report test results + if: success() || failure() shell: bash run: | ## Report final results From 270f3fdce88f34f05a33c80ab7338c192fc953ff Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 11 Jan 2026 16:43:46 +0100 Subject: [PATCH 5/6] Add comprehensive unit tests for comparison script --- util/test_compare_test_results.py | 233 ++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 util/test_compare_test_results.py diff --git a/util/test_compare_test_results.py b/util/test_compare_test_results.py new file mode 100644 index 0000000..dd85337 --- /dev/null +++ b/util/test_compare_test_results.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 + +""" +Unit tests for compare_test_results.py +""" + +import json +import tempfile +import unittest +from pathlib import Path +from compare_test_results import load_ignore_list, extract_test_results, compare_results + + +class TestCompareTestResults(unittest.TestCase): + def setUp(self): + """Set up test fixtures.""" + self.temp_dir = tempfile.TemporaryDirectory() + self.temp_path = Path(self.temp_dir.name) + + # Sample test data + self.current_data = { + "timestamp": "2025-01-11T10:00:00Z", + "summary": {"total": 50, "passed": 40, "failed": 8, "skipped": 2}, + "tests": [ + {"name": "test1", "status": "PASS"}, + {"name": "test2", "status": "FAIL"}, + {"name": "test3", "status": "PASS"}, + {"name": "new_test", "status": "FAIL"}, + ], + } + + self.reference_data = { + "timestamp": "2025-01-10T10:00:00Z", + "summary": {"total": 48, "passed": 42, "failed": 5, "skipped": 1}, + "tests": [ + {"name": "test1", "status": "PASS"}, + {"name": "old_failure", "status": "FAIL"}, + {"name": "test3", "status": "PASS"}, + ], + } + + def tearDown(self): + """Clean up test fixtures.""" + self.temp_dir.cleanup() + + def test_load_ignore_list_empty_file(self): + """Test loading ignore list from empty file.""" + ignore_file = self.temp_path / "empty_ignore.txt" + ignore_file.write_text("") + + result = load_ignore_list(str(ignore_file)) + self.assertEqual(result, set()) + + def test_load_ignore_list_with_content(self): + """Test loading ignore list with actual content.""" + ignore_file = self.temp_path / "ignore.txt" + ignore_file.write_text("# Comment\ntest1\ntest2\n\n# Another comment\ntest3") + + result = load_ignore_list(str(ignore_file)) + self.assertEqual(result, {"test1", "test2", "test3"}) + + def test_load_ignore_list_nonexistent_file(self): + """Test loading ignore list from nonexistent file.""" + result = load_ignore_list("nonexistent.txt") + self.assertEqual(result, set()) + + def test_extract_test_results_valid_data(self): + """Test extracting test results from valid JSON data.""" + summary, failed_tests = extract_test_results(self.current_data) + + self.assertEqual(summary["total"], 50) + self.assertEqual(summary["passed"], 40) + self.assertEqual(summary["failed"], 8) + self.assertEqual(summary["skipped"], 2) + self.assertEqual(set(failed_tests), {"test2", "new_test"}) + + def test_extract_test_results_missing_summary(self): + """Test extracting test results from data without summary.""" + data = {"tests": [{"name": "test1", "status": "PASS"}]} + summary, failed_tests = extract_test_results(data) + + self.assertEqual(summary["total"], 0) + self.assertEqual(failed_tests, []) + + def test_extract_test_results_empty_data(self): + """Test extracting test results from empty data.""" + summary, failed_tests = extract_test_results({}) + + self.assertEqual(summary["total"], 0) + self.assertEqual(failed_tests, []) + + def test_compare_results_improvements_and_new_failures(self): + """Test comparison showing both improvements and new failures.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + output_file = self.temp_path / "output.txt" + + current_file.write_text(json.dumps(self.current_data)) + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results( + str(current_file), str(reference_file), output_file=str(output_file) + ) + + # Should return 1 because there are new failures + self.assertEqual(result, 1) + + # Check output content + output_content = output_file.read_text() + self.assertIn("Test result changes from main branch:", output_content) + self.assertIn("TOTAL: +2", output_content) + self.assertIn("PASSED: -2", output_content) + self.assertIn("FAILED: +3", output_content) + self.assertIn("New test failures (2):", output_content) + self.assertIn("- test2", output_content) + self.assertIn("- new_test", output_content) + self.assertIn("Test improvements (1):", output_content) + self.assertIn("+ old_failure", output_content) + + def test_compare_results_with_ignore_list(self): + """Test comparison with ignored intermittent failures.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + ignore_file = self.temp_path / "ignore.txt" + output_file = self.temp_path / "output.txt" + + current_file.write_text(json.dumps(self.current_data)) + reference_file.write_text(json.dumps(self.reference_data)) + ignore_file.write_text("test2\nnew_test") # Ignore both new failures + + result = compare_results( + str(current_file), str(reference_file), str(ignore_file), str(output_file) + ) + + # Should return 0 because all new failures are ignored + self.assertEqual(result, 0) + + # Check that intermittent failures are marked + output_content = output_file.read_text() + self.assertIn("- test2 (intermittent)", output_content) + self.assertIn("- new_test (intermittent)", output_content) + + def test_compare_results_no_changes(self): + """Test comparison with identical results.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + output_file = self.temp_path / "output.txt" + + # Use same data for both files + current_file.write_text(json.dumps(self.reference_data)) + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results( + str(current_file), str(reference_file), output_file=str(output_file) + ) + + # Should return 0 for no new failures + self.assertEqual(result, 0) + + # Output should be minimal or empty + output_content = output_file.read_text() + self.assertNotIn("New test failures", output_content) + self.assertNotIn("Test improvements", output_content) + + def test_compare_results_only_improvements(self): + """Test comparison with only improvements (no new failures).""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + output_file = self.temp_path / "output.txt" + + # Create data where current has fewer failures than reference + improved_data = { + "summary": {"total": 48, "passed": 47, "failed": 0, "skipped": 1}, + "tests": [ + {"name": "test1", "status": "PASS"}, + {"name": "test3", "status": "PASS"}, + {"name": "old_failure", "status": "PASS"}, # Fixed the old failure! + ], + } + + current_file.write_text(json.dumps(improved_data)) + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results( + str(current_file), str(reference_file), output_file=str(output_file) + ) + + # Should return 0 for no new failures + self.assertEqual(result, 0) + + output_content = output_file.read_text() + self.assertIn("Test improvements", output_content) + # Should show improvement from old_failure + self.assertIn("+ old_failure", output_content) + # Should have no new failures + self.assertNotIn("New test failures", output_content) + + def test_compare_results_invalid_current_file(self): + """Test comparison with invalid current file.""" + reference_file = self.temp_path / "reference.json" + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results("nonexistent_current.json", str(reference_file)) + + # Should return 1 for error + self.assertEqual(result, 1) + + def test_compare_results_invalid_reference_file(self): + """Test comparison with invalid reference file.""" + current_file = self.temp_path / "current.json" + current_file.write_text(json.dumps(self.current_data)) + + result = compare_results(str(current_file), "nonexistent_reference.json") + + # Should return 1 for error + self.assertEqual(result, 1) + + def test_compare_results_malformed_json(self): + """Test comparison with malformed JSON files.""" + current_file = self.temp_path / "current.json" + reference_file = self.temp_path / "reference.json" + + current_file.write_text("{ invalid json") + reference_file.write_text(json.dumps(self.reference_data)) + + result = compare_results(str(current_file), str(reference_file)) + + # Should return 1 for error + self.assertEqual(result, 1) + + +if __name__ == "__main__": + unittest.main() From d89243bdb922c4423f2173e864abf09846b21d28 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 11 Jan 2026 17:12:41 +0100 Subject: [PATCH 6/6] Improve comparison output to show both current and reference numbers --- .../compare_test_results.cpython-313.pyc | Bin 0 -> 7470 bytes ...re_test_results.cpython-313-pytest-8.3.5.pyc | Bin 0 -> 12196 bytes util/compare_test_results.py | 8 +++++++- util/test_compare_test_results.py | 3 ++- 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 util/__pycache__/compare_test_results.cpython-313.pyc create mode 100644 util/__pycache__/test_compare_test_results.cpython-313-pytest-8.3.5.pyc diff --git a/util/__pycache__/compare_test_results.cpython-313.pyc b/util/__pycache__/compare_test_results.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01d5d3ed8c989b93b033c866e3860f442e581127 GIT binary patch literal 7470 zcmey&%ge>Uz`)>nIyv)(2m`}o5C?`?pp4Hg3=9lY8G;##7=jstnYj; zU|@)03KE0bfJ7-V#BeGxm_kKN86gY>hIFQMCQYuYS6t5dxdn+usR|_-sS3%ZMMbH3 zB?|6-p$a9b#U%W}ZS)QDR0~?r`o|j*g zs+*Ztl3J9TSyGakSE5%^QKFEOSzMyW^%CTDKTXD4ECGon8OhL)00}a%F)%PNGcYiG z_F)9YOf*~$7H)Zrp$r;er3?&-4Gav?Ak|=gC<81^gIK`=CtYC6OL~gp@M85?l*F-cv zvNK3ZeP?FiQ~AKfz$12nMYO}}0*mN(ZU#P;E4<1u!C%GV3=9l=q@6^V4@!%GNMlha z5#~c8Tu#dDD0u*sDnUW~`31OiLQ3y!3=F}Hp$yiH3=D~!3=F|cP{^#uqR(K$q{0A7 z$DqUy$|XVYyn{$r!7Ta=u=JD1qz_8pppvoh%7Rq~D=Zxbvw_Mta4MGqr*e=R z5k`YqILu%V=78A>GfjcP*A|khBZNUN*W|2HajhsRN=z<+q(*4eLQmc5O}WKhoSj)vkeYIf6_ikl zZ?P4ZB$kvG-(qod^z^yK0!r(**h}-W^YY8{G?{KOr>BsQDKjuINP`k$ z12`c*;})60xkB=?gwYL2sRfA}3=ari6MN~2<&w&^q3@Uv!BBxvL9i~NP{B|EFiSX8 zD3k@v647VK6Acv&5`~$Epo8JDr@(+i1vid(L~=JDLog2xRY2~Na0~% z2<3ya*%%l?`62AYLd=G?bS}m13cMM5;vgr4YPD zSA^3);?bbmADnnY;eO*|2$nELYUf3Rsu?sj!VDfJ3=DZ$ zV+fW)HqloBziQ;r^EHFi$2js5vX6z~J`TlczBH1Lg<-LY$H#Cr!VDhbU>{?TcNvf` zQQR#_th<9{gFw|VID`>Is3^>JdE%kWaG$B8^evc!)g)1$!P&m9e_XTeI5)rE?{ZIonSK&T0pM+!d?n7Mom!3xOno6eHXj=S~m&g{1V z()w3qhz9qDLDsW_`>1)GNOm!Ya`K_%BzcBlraVqZ&U6+CoyM5P05b_!U&Ec*Zyihx zUfWoyVVmDvh+UeBRr;`wGpy4LYdV6uAqqvQX{kl2dC94;F0@_|sL|$ECGQIE&w{!Q zpgt^`W-Eo4p!Rc>BC0A3)2tMVKz)61YqSWI$BRH!Rh2{ta;F{AXUQzi&$Fr$QBZJ( zbndMb6cj@ILmYjq6sout^c4aegM(dNpd3&$70lsL&{qid_6!IB@rooE7#KiC1i^e| z1vBCnLzSd+Mq*xiYB8kEj5G#dRmF+0ib*@Aic0}u8<&Ct+%jRm)N)Ak8`Rb?lx*x34^_vnOjhlUzVDinpXl+&!V89t#FHx>lR~q6|ZYh zkbjVsf?IxRUW!7MvO->Jxk6rk9?H-Fs*Rfb;GSC%s9|`E1=J9~#R7^Xa5MTAr)x!W zYC%b6e%>wCf}+g4l3UD~c_l@1pjJAl<5VOMYKJrDq~_gXE6y(}Nlm%Ml9ivCcZ;>W zD6=G0Q{om6G^QavQa*T$g843zw4AijrbbQUrH> zun>^m5I-c{r@}iz?hFhJI-riwV-`@4{0kd{kkSI}%Yu3hULV-lSOr17aGeFrS9n!c zFy4`tpKm$KazWs1n+_jP_geLdmfen;%UV7cIn+D&KX5_S>n%{e!mGAI@s6zG{K#36 z3kqk)bohbPt3T0mI$(EMFX$qN_5}{fFFXu_A`|?ti>O}|QD4EaLGiMP$z=i4J}Y*vQi)BswAfx~RrQQH>Si8x${#np_q{ zu_=8);bjg@m`y@#4PIZk7^Gxp#9fxuxxgZJM_TTJy#8frg9|JY4}?`OXxLm7wrTLW zA*_Bu3&Q-s#-M7r!Rn%l?FAM^Xvgpa8w0E0br$i9EaDd=jIOX4-(V4da;5ZE)L++k zx~T7T!2E>cWqrR3dj2P>FGOcukIuUoop&L>{6b~b<>=}QQ8gbJ7-|_km_9HtcrryW zeFsr7OdlcC2L^^%rc{tzI#UMIXOLhfi08qS1){Q;%0N^(lNH`CV zH**N{2L^^P=5XfEAi)R_uZ$@YL`g<5e_>!?jb=u6@CAK`>-sJi^<55xpGdr{A8f|+ALE{tQ22RXZzDFMW* zVM+v1l1X4^CnKCagXs#3^anNu4#5eI&2=**7bMP>TVb_BaJBsn4w35|(ib_TFUXi* z;jn;nXJoG6+F-Px9$h$Vh2#dK6?{YrJ6++B|Hj6^Ap$q|f}G(Nwppfq}skI+&%xpvmJ`B>`(&qEEzVGT!1&Ni9w; z%7ipV*pnfRx+*zn>kiVWfi*?J1DK#8aL}wrMruyMElzlSU8MkT&Ee2g#jC3e8QIna znNelvmYI{Pker`al9-tXYP=%X;t(@H1Dr^M`Bfacx{&(3O35GGxKRL^q>u)l(E-gn zfm)}qCbA~yEsn&Z^n%2q;?!Gwjz#IEptfBAh+A}vCov@@J`pN*iwmp>#4j!aRqsV! zpyrYfhyXXG*dVrQGJ!^Di;7c=Zm}d5r56{2QlSEL)RYC(R4oc&U|{G5H4bKjM@jGS zieBecxx}k-!O;66uTO*94Hhm)Z8n4Rf`-QhP(9~~5aYg}$cm4@#R8gZTgd>n0Mdfuu*uC& zDa}c>D~e%YU;uSUia|m9ftit!@go}xBg$Ef&4 wDTa}e_fs7cH)90j2N?zyo{p-^%#t@)*xNmtJUT+Q3d|+l^kpkNW0AuPrdH?_b literal 0 HcmV?d00001 diff --git a/util/__pycache__/test_compare_test_results.cpython-313-pytest-8.3.5.pyc b/util/__pycache__/test_compare_test_results.cpython-313-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2e6cc21b72aa9fae1a2f42bda617571de1cfcff GIT binary patch literal 12196 zcmey&%ge>Uz`*eNP;%yTRR)H~APx+(LK&a8FfcGoWe8>{VhCmoX7Xk%Vgl35UM$5- z3JhMXMQjQX8TKM}unb2L2bkt8;tXaCX7=Xt;x6I_t6}lx@!~Dw_2Mhy^Wrb!S74A} zP+|yXm0}2Hi{T7XfZB*enKDB-MS?Lx!R%HM3=D}v3=A=xL23|jWKxO26v|a#NM}lC z(&VVp;0n#lEKw*)EiNfmNXsu$NY2kKNGwW?2Z_WNr52awloaa~RC2up>GacNyu}ib zSdyX1d`loFKQSdfGd(ZAC^bGOv$*7zNNPn%QDSllir!oNFcYDMC4&rtVMZw9a~?Pd zV;G7UgBd`jwISu4O4h9AWP&j}70d{6INDPdFn7|YYfe7VL zh8(zjG&@XDC_@knjE$g!Kz1OwV3sM=4WSGb zT*2I6ng`0~h4A%Rg8B5ALm6_|zy>lfFa+}l3j_-W3k3_K=of*g4~FX3XNKy7yE;ofIJHEfv;dr36w)#)N=l1Ti}k8Rj0}uSbqx%44GlvK4Xg}6DC(sV0|UcL4G;nH z+DlMKtYo>xT9RLqm~)G*AhEbOHRTptT4H8SYRWD4;_S?Vg4C2-te}Kvc#9>#F*tZ7 z(=C>~#N5NuygVZjPQl5RUOJj=a=zaEjApzQtLRnVVW%l9*d?i@mrs zH#f1U5~8da$&Cg`ZZrTnPYXnV0{A5;yl-*m=cL4goK;$s>Zi$Zi=!kpw;(MuC-s(4 zNNR3Deod_1 zFfcSQJdseiE@6C0!uYa;X$MOW?;T0m87cE~XXaj)Gq@;ca9Pr@gQbV>0|$eY!2?O@ z8&Wbiq@-_1%Y5WymEilp#~`J+!tJ7f@eN4@G+KE@*+l`f51g!Gd>;k*WVt$6u5d_x zP$FcE+6J|Y0+ul2KCrWZwDZbyb+BCJkp2uxz$lrL36?1>z?m|XAqb=d4F`c@1x*a8 z5M@Hkn6cqFf3=9ek>CBoe zRhl5rD}btl%)E33NR6NXsu2{@it=+6QgaJRDiuH(NUur&%z@S#dL3{;C8)8vB$lzOS z&`badHU$L*Nc4b0DA=$@czI??hC*U;Noit^LUMjyNorn+-Y+gl zOo5`iN>5qAIX^cyHLrvV6v&2P(uj*oS-~+cza%3S+yH?~8{2^*v`B=3fuTqeL`X9* zF!*V*f)h-UIYyg7a)%eqL%tW-%!KpgETxNeC2^ewxhSTnY(xD+UGzO(t;0 zEV2a!38;jCF@R(sBoo_$+(dLNplkO8St}2Yi6?@J7nF=I3!0Fa?+$N_MbH z4niuCY6N(V5zHCPWr*bNbZ$+aDkD(bC_o$dpkf{rrwY(g7gElbCFW$ND0l_?`ze4L zJ9-5CLjwLvpqZIA^mIf!tlB36cZnXHdd|miR?- zpkki~)G|v=0k`;yi&1jA3dl|IP&a}Vz&bGTV7GzWT(_)2KEajB2xtan#o}g28dEUY z5OUqr?V_pMWd-+(9I_qk58%bS_yX~ZlA0@wu5jof%6#z!5*H=4RybYZ(8myTL{lkw zQBr$FAhOCi!ZTtnOKPnM-;i}d$MGtM6K074DhfdX^0@|FVt{&>pq_#rlRl#XV;&1C36elSX+x6*T;AMb0T~2N97W(V02B-g@Zw(+ccRG6 zEH2K>OOJ=zeanI5l!0bK1gHpd11FCM0^$qIH>6#Zvb`W_cY)u&gXIn=!CY3b!k$zN zuW;xhQiaTdqN^NQcw#sQ9K(nhHDJIfm9WJV_EHI(N+y&T1-0@Zl|D3sU@rl{Ei2^A z500}UUr-!+fWii36Fg?{WLL03zF~iIWX0c?> zV+myl0@WNaW1w^>OE4>XD7AZ3@FjT363T9|yLm@dMF)ux}7}Dm*P0Y+wNGeLqOU|&W;tcT*arCiL&^D^# z0u3Lzx>zab8dY(DMvFjfZR092ztnPYw-D|&1q~xjt15O~u&7ZLmoCWV;KoB0cz6)i zXBvi@R#m*(3dlo5E4dNfky|Vv58h%;Da|b?hBRZeK&2lj!yXPZ6G2UwijrF# zkp8V_-byw|bp{J6u>X-qj6glsqDYWwpceKmK4>sO`!K~ID?rsAtkrDy@XSmZJUeQHkTD_uW;Di;1Ijc zp?Hx)af!-utwmZZLaysNT-0^AtmJrw!|92j!W|y|2|Sm1WI;6FWghuE64EogFH2~A zWM>iL>R|i8$H2)8>wYf~yuzXUfsH{*^}3|qMM=HOk_H`oHy|T37sPCL)Lr55gK{Ti zFR;79VFcslEbzO+VT_e`K}>H0-xUsfm}KSxxhot7Fdm9Vbfx(V>aKE_VOIE{kyCJ$ z<3;N#2YYqJ0q!`9S525f`W&0kMieZN4foXnCrjfozgyl_a8gMU(O?iUN%xu!9T1 zB2de%C=q0l4Tyl3dPPYL3=GAfh8U!%gO2p6f^3G5io?n}kdKf@{%+ZT{7h`|hOWC6 zWb-?4W0Hgtjaua!d|dH@yulR?Lu|zz5B~)2%RDlm;txcB=41sIdjdokdnl#X1u?@7 zc38WapwI`GNuR*OB_pmG3@VeM%}5vp&+pJdIt2#ApgE#JjMNNJz<|*V#!|kZG=s4;lQEjX*wmmjgVQ-QIjht``3SS@NJ-5r$xKenfwt6&^^nR7 zP*G6?>K{YO2SJQ>*e^k_p~&qMKTR$~%c=-8=~@&6Dh;r-sET4iV?)rn0>Atc&%B~| zP<};k6v<;PDDv{-p{>eWW*}D(TSOrA%?DX_p47tO0~-UUD0+j(?xMEcWd-{y91iqo z@IV?w7bJC7w;oe4PcRpl<_+cp z(fZ86{CZ4zTXvju%93Erc>) zYn22G5urvHSxvf#rf3y-q6MSm$j{5EMC)he<%9ZR@LrlGMp*({3iDDAR2+cEkU_1E zB2ajsj~#+$j6l68P)`vw*azzC6iI=a8cdKeL-2qjxMa!%HD}nhVLd@@kPE;)L0kn9 zc$gYg#2~tM;DV?q4rD$?QwG$Exy1`D;ZnfETj0qO9E}>tv7Bk+XbMRDH;7%KoYH-udju=xO#%nD@+W`?OzU`Pi|<7$EGI*iPi znFpRMQGhO7flU7ufg1f)63BCIPZiy`A9c*pFCG2tdC{e${74Ga&^h9(G%H{9S+1GTk5Lk*u~edrz>UEsr^rAqZRX#~h3_!Jf{l$ySBE_{&YqNy{(FO@+>6g9crZ zJ2F*#)e6wqRsaq4LQ1}(5>SN?nU6#oy8tIa&=3%4umZ9sNf~Pzgj*U9GV_)dDQOQ? zFKC%uaXGd$hg!J7M_Ml^fW{NBRKgDU##ZqsOHCG}p>NOPHKGoEuQ#z(DE**41aulT4qskNls>7YF>VPd=a>nXAElU*?|Zr5a9|U zyg-CMhyX49D*~-7E=mJ&vq3~5h$saS6(FJ-)I$`Fk1sAsEXhocPb?`Z%1kOPNiB|# zU&#P2r@=8)1WKS_0@SW74rE|pXk+-qqRh(m$%~PRRpAo{8>_}A0X9~$v}bC=7gDQ;GkuTp%hDiGU1 z7J&i;)H02a*A%(M9v`2QpBx{5izBHtGp8goulN>^LqTOcoF^GyPzg#udXPDT%=|pP zqEzszxm%oYp~ z*un|O5+6{VRRk(tZgGH9W@2$FIE0F7K?bsbJW$jG3S-bx#v;hlNE=A(+7(S;U|;}s zuZlqt^?{j@k?|uN6C=xa4hBZX7KUc-+YJ1-8TjuqNZeZO mvU*<_K)eq*>