mirror of
https://github.com/uutils/tar.git
synced 2026-06-10 16:14:35 -07:00
ci: implement GNU test result aggregation and comparison
- Add extraction of GNU test results into JSON artifacts - Add aggregation job to summarize results and compare against reference - Enable continue-on-error for GNU tests to allow workflow completion on failures - Align log compression and upload behavior with coreutils CI
This commit is contained in:
Jeff Bailey
@@ -0,0 +1,67 @@
|
||||
name: GnuComment
|
||||
|
||||
# spell-checker:ignore zizmor backquote
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["GnuTests"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
post-comment:
|
||||
permissions:
|
||||
actions: read # to list workflow runs artifacts
|
||||
pull-requests: write # to comment on pr
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
if: >
|
||||
github.event.workflow_run.event == 'pull_request'
|
||||
steps:
|
||||
- name: 'Download artifact'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: ${{ github.event.workflow_run.id }},
|
||||
});
|
||||
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
|
||||
return artifact.name == "comment"
|
||||
})[0];
|
||||
if (!matchArtifact) {
|
||||
return core.setFailed("No 'comment' artifact found");
|
||||
}
|
||||
var download = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: matchArtifact.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
var fs = require('fs');
|
||||
fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
|
||||
|
||||
- run: unzip comment.zip
|
||||
|
||||
- name: 'Comment on PR'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
var fs = require('fs');
|
||||
if (!fs.existsSync('./NR') || !fs.existsSync('./result.txt')) {
|
||||
return;
|
||||
}
|
||||
var issue_number = Number(fs.readFileSync('./NR'));
|
||||
var content = fs.readFileSync('./result.txt');
|
||||
if (content.toString().trim().length > 7) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue_number,
|
||||
body: 'GNU testsuite comparison:\n```\n' + content + '```'
|
||||
});
|
||||
}
|
||||
@@ -13,6 +13,10 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEST_FULL_SUMMARY_FILE: 'gnu-full-result.json'
|
||||
|
||||
jobs:
|
||||
native:
|
||||
name: Run GNU tests (native)
|
||||
@@ -48,6 +52,125 @@ jobs:
|
||||
|
||||
- name: Run GNU tests
|
||||
shell: bash
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd 'uutils'
|
||||
bash util/run-gnu-test.sh
|
||||
|
||||
- name: Extract testing info from individual logs into JSON
|
||||
shell: bash
|
||||
run : |
|
||||
python uutils/util/gnu-json-result.py gnu/tests > ${{ env.TEST_FULL_SUMMARY_FILE }}
|
||||
|
||||
- name: Upload full json results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: gnu-full-result
|
||||
path: ${{ env.TEST_FULL_SUMMARY_FILE }}
|
||||
|
||||
- name: Compress test logs
|
||||
shell: bash
|
||||
run : |
|
||||
# Compress logs before upload (fails otherwise)
|
||||
gzip gnu/tests/*/*.log
|
||||
|
||||
- name: Upload test logs
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-logs
|
||||
path: |
|
||||
gnu/tests/*.log
|
||||
gnu/tests/*/*.log.gz
|
||||
|
||||
aggregate:
|
||||
needs: [native]
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
name: Aggregate GNU test results
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Initialize workflow variables
|
||||
id: vars
|
||||
shell: bash
|
||||
run: |
|
||||
TEST_SUMMARY_FILE='gnu-result.json'
|
||||
AGGREGATED_SUMMARY_FILE='aggregated-result.json'
|
||||
echo "TEST_SUMMARY_FILE=${TEST_SUMMARY_FILE}" >> $GITHUB_OUTPUT
|
||||
echo "AGGREGATED_SUMMARY_FILE=${AGGREGATED_SUMMARY_FILE}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout code (uutils)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: 'uutils'
|
||||
persist-credentials: false
|
||||
|
||||
- name: Retrieve reference artifacts
|
||||
uses: dawidd6/action-download-artifact@v9
|
||||
continue-on-error: true
|
||||
with:
|
||||
workflow: GnuTests.yml
|
||||
branch: "${{ env.DEFAULT_BRANCH }}"
|
||||
workflow_conclusion: completed
|
||||
path: "reference"
|
||||
|
||||
- name: Download full json results
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: gnu-full-result
|
||||
path: results
|
||||
|
||||
- name: Extract/summarize testing info
|
||||
id: summary
|
||||
shell: bash
|
||||
run: |
|
||||
# Look at all individual results and summarize
|
||||
eval $(python3 uutils/util/analyze-gnu-results.py -o=${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} results/*.json)
|
||||
|
||||
output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP"
|
||||
echo "${output}"
|
||||
|
||||
jq -n \
|
||||
--arg date "$(date --rfc-email)" \
|
||||
--arg sha "$GITHUB_SHA" \
|
||||
--arg total "$TOTAL" \
|
||||
--arg pass "$PASS" \
|
||||
--arg skip "$SKIP" \
|
||||
--arg fail "$FAIL" \
|
||||
--arg xpass "$XPASS" \
|
||||
--arg error "$ERROR" \
|
||||
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
|
||||
|
||||
HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
|
||||
echo "HASH=${HASH}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Upload test results summary
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-summary
|
||||
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
||||
|
||||
- name: Upload aggregated json results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: aggregated-result
|
||||
path: ${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}
|
||||
|
||||
- name: Compare test failures VS reference
|
||||
shell: bash
|
||||
continue-on-error: true
|
||||
run: |
|
||||
## Compare test failures VS reference using JSON files
|
||||
# This step will fail if no reference is found, which is expected for the first run
|
||||
REF_SUMMARY_FILE='reference/aggregated-result/aggregated-result.json'
|
||||
CURRENT_SUMMARY_FILE='${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}'
|
||||
IGNORE_INTERMITTENT="uutils/.github/workflows/ignore-intermittent.txt"
|
||||
|
||||
if [ -f "${REF_SUMMARY_FILE}" ]; then
|
||||
python3 uutils/util/compare_test_results.py \
|
||||
--ignore-file "${IGNORE_INTERMITTENT}" \
|
||||
"${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
|
||||
else
|
||||
echo "No reference summary found. Skipping comparison."
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# List of intermittent GNU tests to ignore during comparison
|
||||
# Add test paths here (one per line, without .log extension)
|
||||
# Example: tests/ls/stat-free-color
|
||||
Executable
+185
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
GNU Test Results Analyzer and Aggregator
|
||||
|
||||
This script analyzes and aggregates test results from the GNU test suite.
|
||||
It parses JSON files containing test results (PASS/FAIL/SKIP/ERROR) and:
|
||||
1. Counts the number of tests in each result category
|
||||
2. Can aggregate results from multiple JSON files with priority ordering
|
||||
3. Outputs shell export statements for use in GitHub Actions workflows
|
||||
|
||||
Priority order for aggregation (highest to lowest):
|
||||
- PASS: Takes precedence over all other results (best outcome)
|
||||
- FAIL: Takes precedence over ERROR and SKIP
|
||||
- ERROR: Takes precedence over SKIP
|
||||
- SKIP: Lowest priority
|
||||
|
||||
Usage:
|
||||
- Single file:
|
||||
python analyze-gnu-results.py test-results.json
|
||||
|
||||
- Multiple files (with aggregation):
|
||||
python analyze-gnu-results.py file1.json file2.json
|
||||
|
||||
- With output file for aggregated results:
|
||||
python analyze-gnu-results.py -o=output.json file1.json file2.json
|
||||
|
||||
Output:
|
||||
Prints shell export statements for TOTAL, PASS, FAIL, SKIP, XPASS, and ERROR
|
||||
that can be evaluated in a shell environment.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
def get_priority(result):
|
||||
"""Return a priority value for result status (lower is higher priority)"""
|
||||
priorities = {
|
||||
"PASS": 0, # PASS is highest priority (best result)
|
||||
"FAIL": 1, # FAIL is second priority
|
||||
"ERROR": 2, # ERROR is third priority
|
||||
"SKIP": 3, # SKIP is lowest priority
|
||||
}
|
||||
return priorities.get(result, 4) # Unknown states have lowest priority
|
||||
|
||||
|
||||
def aggregate_results(json_files):
|
||||
"""
|
||||
Aggregate test results from multiple JSON files.
|
||||
Prioritizes results in the order: SKIP > ERROR > FAIL > PASS
|
||||
"""
|
||||
# Combined results dictionary
|
||||
combined_results = defaultdict(dict)
|
||||
|
||||
# Process each JSON file
|
||||
for json_file in json_files:
|
||||
try:
|
||||
with open(json_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# For each utility and its tests
|
||||
for utility, tests in data.items():
|
||||
for test_name, result in tests.items():
|
||||
# If this test hasn't been seen yet, add it
|
||||
if test_name not in combined_results[utility]:
|
||||
combined_results[utility][test_name] = result
|
||||
else:
|
||||
# If it has been seen, apply priority rules
|
||||
current_priority = get_priority(
|
||||
combined_results[utility][test_name]
|
||||
)
|
||||
new_priority = get_priority(result)
|
||||
|
||||
# Lower priority value means higher precedence
|
||||
if new_priority < current_priority:
|
||||
combined_results[utility][test_name] = result
|
||||
except FileNotFoundError:
|
||||
print(f"Warning: File '{json_file}' not found.", file=sys.stderr)
|
||||
continue
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: '{json_file}' is not a valid JSON file.", file=sys.stderr)
|
||||
continue
|
||||
|
||||
return combined_results
|
||||
|
||||
|
||||
def analyze_test_results(json_data):
|
||||
"""
|
||||
Analyze test results from GNU test suite JSON data.
|
||||
Counts PASS, FAIL, SKIP results for all tests.
|
||||
"""
|
||||
# Counters for test results
|
||||
total_tests = 0
|
||||
pass_count = 0
|
||||
fail_count = 0
|
||||
skip_count = 0
|
||||
xpass_count = 0 # Not in JSON data but included for compatibility
|
||||
error_count = 0 # Not in JSON data but included for compatibility
|
||||
|
||||
# Analyze each utility's tests
|
||||
for utility, tests in json_data.items():
|
||||
for test_name, result in tests.items():
|
||||
total_tests += 1
|
||||
|
||||
match result:
|
||||
case "PASS":
|
||||
pass_count += 1
|
||||
case "FAIL":
|
||||
fail_count += 1
|
||||
case "SKIP":
|
||||
skip_count += 1
|
||||
case "ERROR":
|
||||
error_count += 1
|
||||
case "XPASS":
|
||||
xpass_count += 1
|
||||
|
||||
# Return the statistics
|
||||
return {
|
||||
"TOTAL": total_tests,
|
||||
"PASS": pass_count,
|
||||
"FAIL": fail_count,
|
||||
"SKIP": skip_count,
|
||||
"XPASS": xpass_count,
|
||||
"ERROR": error_count,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to process JSON files and export variables.
|
||||
Supports both single file analysis and multi-file aggregation.
|
||||
"""
|
||||
# Check if file arguments were provided
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python analyze-gnu-results.py <json> [json ...]")
|
||||
print(" For multiple files, results will be aggregated")
|
||||
print(" Priority SKIP > ERROR > FAIL > PASS")
|
||||
sys.exit(1)
|
||||
|
||||
json_files = sys.argv[1:]
|
||||
output_file = None
|
||||
|
||||
# Check if the first argument is an output file (starts with -)
|
||||
if json_files[0].startswith("-o="):
|
||||
output_file = json_files[0][3:]
|
||||
json_files = json_files[1:]
|
||||
|
||||
# Process the files
|
||||
if len(json_files) == 1:
|
||||
# Single file analysis
|
||||
try:
|
||||
with open(json_files[0], "r") as file:
|
||||
json_data = json.load(file)
|
||||
results = analyze_test_results(json_data)
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File '{json_files[0]}' not found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError:
|
||||
print(
|
||||
f"Error: '{json_files[0]}' is not a valid JSON file.", file=sys.stderr
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Multiple files - aggregate them
|
||||
json_data = aggregate_results(json_files)
|
||||
results = analyze_test_results(json_data)
|
||||
|
||||
# Save aggregated data if output file is specified
|
||||
if output_file:
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(json_data, f, indent=2)
|
||||
|
||||
# Print export statements for shell evaluation
|
||||
print(f"export TOTAL={results['TOTAL']}")
|
||||
print(f"export PASS={results['PASS']}")
|
||||
print(f"export SKIP={results['SKIP']}")
|
||||
print(f"export FAIL={results['FAIL']}")
|
||||
print(f"export XPASS={results['XPASS']}")
|
||||
print(f"export ERROR={results['ERROR']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Compare the current results to the last results gathered from the main branch to
|
||||
highlight if a PR is making the results better/worse.
|
||||
Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from os import environ
|
||||
|
||||
REPO_DEFAULT_BRANCH = environ.get("REPO_DEFAULT_BRANCH", "main")
|
||||
ONLY_INTERMITTENT = environ.get("ONLY_INTERMITTENT", "false")
|
||||
|
||||
NEW = json.load(open("gnu-result.json"))
|
||||
OLD = json.load(open("main-gnu-result.json"))
|
||||
|
||||
# Extract the specific results from the dicts
|
||||
last = OLD[list(OLD.keys())[0]]
|
||||
current = NEW[list(NEW.keys())[0]]
|
||||
|
||||
|
||||
pass_d = int(current["pass"]) - int(last["pass"])
|
||||
fail_d = int(current["fail"]) - int(last["fail"])
|
||||
error_d = int(current["error"]) - int(last["error"])
|
||||
skip_d = int(current["skip"]) - int(last["skip"])
|
||||
|
||||
# Get an annotation to highlight changes
|
||||
print(
|
||||
f"""::warning ::Changes from '{REPO_DEFAULT_BRANCH}': PASS {pass_d:+d} / FAIL {fail_d:+d} / ERROR {error_d:+d} / SKIP {skip_d:+d}"""
|
||||
)
|
||||
|
||||
# If results are worse, check if we should fail the job
|
||||
if pass_d < 0:
|
||||
print(
|
||||
f"""::error ::PASS count is reduced from '{REPO_DEFAULT_BRANCH}': PASS {pass_d:+d}"""
|
||||
)
|
||||
|
||||
# Check if all failing tests are intermittent based on the environment variable
|
||||
only_intermittent = ONLY_INTERMITTENT.lower() == "true"
|
||||
|
||||
if only_intermittent:
|
||||
print("::notice ::All failing tests are in the ignored intermittent list")
|
||||
print("::notice ::Not failing the build")
|
||||
else:
|
||||
print("::error ::Found non-ignored failing tests")
|
||||
sys.exit(1)
|
||||
Executable
+246
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compare GNU test results between current run and reference to identify regressions and fixes.
|
||||
|
||||
Arguments:
|
||||
CURRENT_JSON Path to the current run's aggregated results JSON file
|
||||
REFERENCE_JSON Path to the reference (main branch) aggregated results JSON file
|
||||
--ignore-file Path to file containing list of tests to ignore (for intermittent issues)
|
||||
--output Path to output file for GitHub comment content
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def flatten_test_results(results):
|
||||
"""Convert nested JSON test results to a flat dictionary of test paths to statuses."""
|
||||
flattened = {}
|
||||
for util, tests in results.items():
|
||||
for test_name, status in tests.items():
|
||||
# Build the full test path
|
||||
test_path = f"tests/{util}/{test_name}"
|
||||
# Remove the .log extension
|
||||
test_path = test_path.replace(".log", "")
|
||||
flattened[test_path] = status
|
||||
return flattened
|
||||
|
||||
|
||||
def load_ignore_list(ignore_file):
|
||||
"""Load list of tests to ignore from file."""
|
||||
if not os.path.exists(ignore_file):
|
||||
return set()
|
||||
|
||||
with open(ignore_file, "r") as f:
|
||||
return {line.strip() for line in f if line.strip() and not line.startswith("#")}
|
||||
|
||||
|
||||
def identify_test_changes(current_flat, reference_flat):
|
||||
"""
|
||||
Identify different categories of test changes between current and reference results.
|
||||
|
||||
Args:
|
||||
current_flat (dict): Flattened dictionary of current test results
|
||||
reference_flat (dict): Flattened dictionary of reference test results
|
||||
|
||||
Returns:
|
||||
tuple: Five lists containing regressions, fixes, newly_skipped, newly_passing, and newly_failing tests
|
||||
"""
|
||||
# Find regressions (tests that were passing but now failing)
|
||||
regressions = []
|
||||
for test_path, status in current_flat.items():
|
||||
if status in ("FAIL", "ERROR"):
|
||||
if test_path in reference_flat:
|
||||
if reference_flat[test_path] == "PASS":
|
||||
regressions.append(test_path)
|
||||
|
||||
# Find fixes (tests that were failing but now passing)
|
||||
fixes = []
|
||||
for test_path, status in reference_flat.items():
|
||||
if status in ("FAIL", "ERROR"):
|
||||
if test_path in current_flat:
|
||||
if current_flat[test_path] == "PASS":
|
||||
fixes.append(test_path)
|
||||
|
||||
# Find newly skipped tests (were passing, now skipped)
|
||||
newly_skipped = []
|
||||
for test_path, status in current_flat.items():
|
||||
if (
|
||||
status == "SKIP"
|
||||
and test_path in reference_flat
|
||||
and reference_flat[test_path] == "PASS"
|
||||
):
|
||||
newly_skipped.append(test_path)
|
||||
|
||||
# Find newly passing tests (were skipped, now passing)
|
||||
newly_passing = []
|
||||
for test_path, status in current_flat.items():
|
||||
if (
|
||||
status == "PASS"
|
||||
and test_path in reference_flat
|
||||
and reference_flat[test_path] == "SKIP"
|
||||
):
|
||||
newly_passing.append(test_path)
|
||||
|
||||
# Find newly failing tests (were skipped, now failing)
|
||||
newly_failing = []
|
||||
for test_path, status in current_flat.items():
|
||||
if (
|
||||
status in ("FAIL", "ERROR")
|
||||
and test_path in reference_flat
|
||||
and reference_flat[test_path] == "SKIP"
|
||||
):
|
||||
newly_failing.append(test_path)
|
||||
|
||||
return regressions, fixes, newly_skipped, newly_passing, newly_failing
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare GNU test results and identify regressions and fixes"
|
||||
)
|
||||
parser.add_argument("current_json", help="Path to current run JSON results")
|
||||
parser.add_argument("reference_json", help="Path to reference JSON results")
|
||||
parser.add_argument(
|
||||
"--ignore-file",
|
||||
required=True,
|
||||
help="Path to file with tests to ignore (for intermittent issues)",
|
||||
)
|
||||
parser.add_argument("--output", help="Path to output file for GitHub comment")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load test results
|
||||
try:
|
||||
with open(args.current_json, "r") as f:
|
||||
current_results = json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError) as e:
|
||||
sys.stderr.write(f"Error loading current results: {e}\n")
|
||||
return 1
|
||||
|
||||
try:
|
||||
with open(args.reference_json, "r") as f:
|
||||
reference_results = json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError) as e:
|
||||
sys.stderr.write(f"Error loading reference results: {e}\n")
|
||||
sys.stderr.write("Skipping comparison as reference is not available.\n")
|
||||
return 0
|
||||
|
||||
# Load ignore list (required)
|
||||
if not os.path.exists(args.ignore_file):
|
||||
sys.stderr.write(f"Error: Ignore file {args.ignore_file} does not exist\n")
|
||||
return 1
|
||||
|
||||
ignore_list = load_ignore_list(args.ignore_file)
|
||||
print(f"Loaded {len(ignore_list)} tests to ignore from {args.ignore_file}")
|
||||
|
||||
# Flatten result structures for easier comparison
|
||||
current_flat = flatten_test_results(current_results)
|
||||
reference_flat = flatten_test_results(reference_results)
|
||||
|
||||
# Identify different categories of test changes
|
||||
regressions, fixes, newly_skipped, newly_passing, newly_failing = (
|
||||
identify_test_changes(current_flat, reference_flat)
|
||||
)
|
||||
|
||||
# Filter out intermittent issues from regressions
|
||||
real_regressions = [r for r in regressions if r not in ignore_list]
|
||||
intermittent_regressions = [r for r in regressions if r in ignore_list]
|
||||
|
||||
# Filter out intermittent issues from fixes
|
||||
real_fixes = [f for f in fixes if f not in ignore_list]
|
||||
intermittent_fixes = [f for f in fixes if f in ignore_list]
|
||||
|
||||
# Filter out intermittent issues from newly failing
|
||||
real_newly_failing = [n for n in newly_failing if n not in ignore_list]
|
||||
intermittent_newly_failing = [n for n in newly_failing if n in ignore_list]
|
||||
|
||||
# Print summary stats
|
||||
print(f"Total tests in current run: {len(current_flat)}")
|
||||
print(f"Total tests in reference: {len(reference_flat)}")
|
||||
print(f"New regressions: {len(real_regressions)}")
|
||||
print(f"Intermittent regressions: {len(intermittent_regressions)}")
|
||||
print(f"Fixed tests: {len(real_fixes)}")
|
||||
print(f"Intermittent fixes: {len(intermittent_fixes)}")
|
||||
print(f"Newly skipped tests: {len(newly_skipped)}")
|
||||
print(f"Newly passing tests (previously skipped): {len(newly_passing)}")
|
||||
print(f"Newly failing tests (previously skipped): {len(real_newly_failing)}")
|
||||
print(f"Intermittent newly failing: {len(intermittent_newly_failing)}")
|
||||
|
||||
output_lines = []
|
||||
|
||||
# Report regressions
|
||||
if real_regressions:
|
||||
print("\nREGRESSIONS (non-intermittent failures):", file=sys.stderr)
|
||||
for test in sorted(real_regressions):
|
||||
msg = f"GNU test failed: {test}. {test} is passing on 'main'. Maybe you have to rebase?"
|
||||
print(f"::error ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report intermittent issues (regressions)
|
||||
if intermittent_regressions:
|
||||
print("\nINTERMITTENT ISSUES (ignored regressions):", file=sys.stderr)
|
||||
for test in sorted(intermittent_regressions):
|
||||
msg = f"Skip an intermittent issue {test} (fails in this run but passes in the 'main' branch)"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report intermittent issues (fixes)
|
||||
if intermittent_fixes:
|
||||
print("\nINTERMITTENT ISSUES (ignored fixes):", file=sys.stderr)
|
||||
for test in sorted(intermittent_fixes):
|
||||
msg = f"Skipping an intermittent issue {test} (passes in this run but fails in the 'main' branch)"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report fixes
|
||||
if real_fixes:
|
||||
print("\nFIXED TESTS:", file=sys.stderr)
|
||||
for test in sorted(real_fixes):
|
||||
msg = f"Congrats! The gnu test {test} is no longer failing!"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report newly skipped and passing tests
|
||||
if newly_skipped:
|
||||
print("\nNEWLY SKIPPED TESTS:", file=sys.stderr)
|
||||
for test in sorted(newly_skipped):
|
||||
msg = f"Note: The gnu test {test} is now being skipped but was previously passing."
|
||||
print(f"::warning ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
if newly_passing:
|
||||
print("\nNEWLY PASSING TESTS (previously skipped):", file=sys.stderr)
|
||||
for test in sorted(newly_passing):
|
||||
msg = f"Congrats! The gnu test {test} is now passing!"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report newly failing tests (were skipped, now failing)
|
||||
if real_newly_failing:
|
||||
print("\nNEWLY FAILING TESTS (previously skipped):", file=sys.stderr)
|
||||
for test in sorted(real_newly_failing):
|
||||
msg = f"Note: The gnu test {test} was skipped on 'main' but is now failing."
|
||||
print(f"::warning ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
if intermittent_newly_failing:
|
||||
print("\nINTERMITTENT NEWLY FAILING (ignored):", file=sys.stderr)
|
||||
for test in sorted(intermittent_newly_failing):
|
||||
msg = f"Skip an intermittent issue {test} (was skipped on 'main', now failing)"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
if args.output and output_lines:
|
||||
with open(args.output, "w") as f:
|
||||
for line in output_lines:
|
||||
f.write(f"{line}\n")
|
||||
|
||||
# Return exit code based on whether we found regressions
|
||||
return 1 if real_regressions else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
Extract the GNU logs into a JSON file.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
out = {}
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python gnu-json-result.py <gnu_test_directory>")
|
||||
sys.exit(1)
|
||||
|
||||
test_dir = Path(sys.argv[1])
|
||||
if not test_dir.is_dir():
|
||||
print(f"Directory {test_dir} does not exist.")
|
||||
sys.exit(1)
|
||||
|
||||
# Test all the logs from the test execution
|
||||
for filepath in test_dir.glob("**/*.log"):
|
||||
path = Path(filepath)
|
||||
if path.name == "testsuite.log":
|
||||
# Handle Autotest testsuite.log
|
||||
try:
|
||||
with open(path, errors="ignore") as f:
|
||||
for line in f:
|
||||
# Look for lines like: " 1: basic functionality ok"
|
||||
# or " 10: ... FAILED (basic.at:123)"
|
||||
match = re.match(r"^\s*(\d+):\s+(.*?)\s+(ok|FAILED|skipped)(?:\s+\(.*\))?$", line)
|
||||
if match:
|
||||
num, name, status = match.groups()
|
||||
# Map Autotest status to Automake-style status
|
||||
status_map = {"ok": "PASS", "FAILED": "FAIL", "skipped": "SKIP"}
|
||||
out[f"test {num}"] = {name.strip(): status_map.get(status, status)}
|
||||
except Exception as e:
|
||||
print(f"Error processing testsuite.log {path}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
current = out
|
||||
for key in path.parent.relative_to(test_dir).parts:
|
||||
if key not in current:
|
||||
current[key] = {}
|
||||
current = current[key]
|
||||
try:
|
||||
with open(path, errors="ignore") as f:
|
||||
content = f.read()
|
||||
result = re.search(
|
||||
r"(PASS|FAIL|SKIP|ERROR) [^ ]+ \(exit status: \d+\)$", content
|
||||
)
|
||||
if result:
|
||||
current[path.name] = result.group(1)
|
||||
except Exception as e:
|
||||
print(f"Error processing file {path}: {e}", file=sys.stderr)
|
||||
|
||||
print(json.dumps(out, indent=2, sort_keys=True))
|
||||
Reference in New Issue
Block a user