mirror of
https://github.com/uutils/findutils.git
synced 2026-10-16 02:48:31 -07:00
Sylvestre Ledru
bb6a5ba53e
The old comparison only commented when the PASS/FAIL/ERROR/SKIP *counts* changed vs main, so it was blind to net-zero changes (one test fixed, one broken). diff-gnu.sh also compared against an unextracted (empty) baseline, flagging every current failure as a regression, and the published baseline could be stale. Adopt the sed/grep design: - build-gnu.sh / build-bfs.sh now emit a per-test JSON summary (findutils-gnu-full-result.json / bfs-full-result.json) listing every test's name + PASS/FAIL/SKIP, via the new gnu_json_result.py / bfs_json_result.py. - compare_test_results.py diffs the set of failing test names against a reference, writing a human-readable result file (empty when nothing changed) and exiting non-zero only on new, non-intermittent failures. - compat.yml fetches the reference *-full-result.json from the default branch's latest run (dawidd6/action-download-artifact) and uploads a comparison artifact per suite. - comment.yml posts a PR comment whenever the comparison is non-empty, i.e. on any per-test improvement or regression regardless of net count. - ignore-intermittent.txt lists known-flaky tests, reported but not failing the job. Remove the superseded compare_gnu_result.py / compare_bfs_result.py / diff-gnu.sh / diff-bfs.sh.
77 lines
1.9 KiB
Python
Executable File
77 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""
|
|
Build a per-test JSON summary of a bfs testsuite run.
|
|
|
|
bfs writes one line per test to tests.log:
|
|
|
|
[PASS] posix/H
|
|
[FAIL] common/newermt
|
|
[SKIP] gnu/...
|
|
|
|
Output format matches util/gnu_json_result.py and is consumed by
|
|
compare_test_results.py:
|
|
|
|
{
|
|
"summary": {"total": N, "passed": P, "failed": F, "skipped": S},
|
|
"tests": [{"name": "...", "status": "PASS|FAIL|SKIP"}, ...]
|
|
}
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
RESULT_RE = re.compile(r"^\[(PASS|FAIL|SKIP)\] (\S+)\s*$")
|
|
|
|
|
|
def collect(log_file):
|
|
tests = {}
|
|
log = Path(log_file)
|
|
if log.is_file():
|
|
for line in log.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
m = RESULT_RE.match(line)
|
|
if m:
|
|
tests[m.group(2)] = m.group(1)
|
|
return tests
|
|
|
|
|
|
def build(log_file):
|
|
tests = collect(log_file)
|
|
passed = sum(1 for s in tests.values() if s == "PASS")
|
|
failed = sum(1 for s in tests.values() if s == "FAIL")
|
|
skipped = sum(1 for s in tests.values() if s == "SKIP")
|
|
return {
|
|
"summary": {
|
|
"total": len(tests),
|
|
"passed": passed,
|
|
"failed": failed,
|
|
"skipped": skipped,
|
|
},
|
|
"tests": [
|
|
{"name": name, "status": status}
|
|
for name, status in sorted(tests.items())
|
|
],
|
|
}
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) != 3:
|
|
print(f"usage: {sys.argv[0]} <bfs-tests.log> <output.json>", file=sys.stderr)
|
|
return 2
|
|
result = build(sys.argv[1])
|
|
with open(sys.argv[2], "w", encoding="utf-8") as f:
|
|
json.dump(result, f, indent=2, sort_keys=True)
|
|
f.write("\n")
|
|
s = result["summary"]
|
|
print(
|
|
f"bfs tests summary = TOTAL: {s['total']} / "
|
|
f"PASS: {s['passed']} / FAIL: {s['failed']} / SKIP: {s['skipped']}"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|