mirror of
https://github.com/uutils/tar.git
synced 2026-06-10 16:14:35 -07:00
Merge pull request #45 from kaladron/gnu-tar-check
GitHub Actions workflow for running GNU tar tests
This commit is contained in:
Sylvestre Ledru
GitHub
@@ -0,0 +1,66 @@
|
||||
name: GnuComment
|
||||
|
||||
# spell-checker:ignore zizmor backquote
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["GnuTests"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
post-comment:
|
||||
permissions:
|
||||
actions: read # to list workflow runs artifacts
|
||||
pull-requests: write # to comment on pr
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
if: >
|
||||
github.event.workflow_run.event == 'pull_request'
|
||||
steps:
|
||||
- name: 'Download artifact'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: ${{ github.event.workflow_run.id }},
|
||||
});
|
||||
const matchArtifact = artifacts.data.artifacts.find(a => a.name === "comment");
|
||||
if (!matchArtifact) {
|
||||
return core.info("No 'comment' artifact found");
|
||||
}
|
||||
const download = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: matchArtifact.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
const fs = require('fs');
|
||||
fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
|
||||
|
||||
- name: 'Unzip artifact'
|
||||
run: unzip comment.zip
|
||||
|
||||
- name: 'Comment on PR'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
if (!fs.existsSync('./NR') || !fs.existsSync('./result.txt')) {
|
||||
return core.info("No NR or result.txt found, skipping comment");
|
||||
}
|
||||
const issue_number = Number(fs.readFileSync('./NR', 'utf8').trim());
|
||||
const content = fs.readFileSync('./result.txt', 'utf8').trim();
|
||||
if (content.length > 0) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue_number,
|
||||
body: `### GNU Testsuite Comparison\n\n\`\`\`\n${content}\n\`\`\``
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
name: GnuTests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEST_FULL_SUMMARY_FILE: 'gnu-full-result.json'
|
||||
TEST_SUMMARY_FILE: 'gnu-result.json'
|
||||
AGGREGATED_SUMMARY_FILE: 'aggregated-result.json'
|
||||
|
||||
jobs:
|
||||
native:
|
||||
name: Run GNU tests (native)
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- name: Checkout code (uutils)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: 'uutils'
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: stable
|
||||
components: rustfmt
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: "./uutils -> target"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y autopoint gperf gdb python3-pyinotify valgrind libacl1-dev libattr1-dev libcap-dev libselinux1-dev attr quilt gettext
|
||||
|
||||
- name: Build binaries and GNU tar
|
||||
run: |
|
||||
cd 'uutils'
|
||||
bash util/build-gnu.sh --release-build
|
||||
|
||||
- name: Run GNU tests
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd 'uutils'
|
||||
bash util/run-gnu-test.sh
|
||||
|
||||
- name: Extract info into JSON
|
||||
run : |
|
||||
python uutils/util/gnu-json-result.py gnu/tests > ${{ env.TEST_FULL_SUMMARY_FILE }}
|
||||
|
||||
- name: Upload full results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: gnu-full-result
|
||||
path: ${{ env.TEST_FULL_SUMMARY_FILE }}
|
||||
|
||||
- name: Compress logs
|
||||
run : |
|
||||
find gnu/tests -mindepth 2 -name '*.log' -exec gzip {} \;
|
||||
|
||||
- name: Upload logs
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-logs
|
||||
path: |
|
||||
gnu/tests/testsuite.log
|
||||
gnu/tests/**/*.log.gz
|
||||
|
||||
aggregate:
|
||||
needs: [native]
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
name: Aggregate GNU test results
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: 'uutils'
|
||||
persist-credentials: false
|
||||
|
||||
- name: Retrieve reference artifacts
|
||||
uses: dawidd6/action-download-artifact@v9
|
||||
continue-on-error: true
|
||||
with:
|
||||
workflow: GnuTests.yml
|
||||
branch: "${{ env.DEFAULT_BRANCH }}"
|
||||
workflow_conclusion: completed
|
||||
path: "reference"
|
||||
|
||||
- name: Download results
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: gnu-full-result
|
||||
path: results
|
||||
|
||||
- name: Summarize results
|
||||
id: summary
|
||||
run: |
|
||||
eval $(python3 uutils/util/analyze-gnu-results.py -o=${{ env.AGGREGATED_SUMMARY_FILE }} results/*.json)
|
||||
|
||||
echo "GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP"
|
||||
|
||||
jq -n \
|
||||
--arg date "$(date --rfc-email)" \
|
||||
--arg sha "$GITHUB_SHA" \
|
||||
--arg total "$TOTAL" \
|
||||
--arg pass "$PASS" \
|
||||
--arg skip "$SKIP" \
|
||||
--arg fail "$FAIL" \
|
||||
--arg xpass "$XPASS" \
|
||||
--arg error "$ERROR" \
|
||||
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ env.TEST_SUMMARY_FILE }}'
|
||||
|
||||
- name: Prepare comparison
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
mkdir -p comment
|
||||
echo "${{ github.event.pull_request.number }}" > comment/NR
|
||||
|
||||
- name: Compare results
|
||||
continue-on-error: true
|
||||
run: |
|
||||
REF_SUMMARY_FILE='reference/aggregated-result/${{ env.AGGREGATED_SUMMARY_FILE }}'
|
||||
CURRENT_SUMMARY_FILE='${{ env.AGGREGATED_SUMMARY_FILE }}'
|
||||
IGNORE_INTERMITTENT="uutils/.github/workflows/ignore-intermittent.txt"
|
||||
|
||||
OUTPUT_ARG=""
|
||||
if [ "${{ github.event_name }}" == "pull_request" ]; then
|
||||
OUTPUT_ARG="--output comment/result.txt"
|
||||
fi
|
||||
|
||||
if [ -f "${REF_SUMMARY_FILE}" ]; then
|
||||
python3 uutils/util/compare_test_results.py \
|
||||
--ignore-file "${IGNORE_INTERMITTENT}" \
|
||||
${OUTPUT_ARG} \
|
||||
"${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
|
||||
else
|
||||
echo "No reference summary found. Skipping comparison."
|
||||
fi
|
||||
|
||||
- name: Upload summary
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-summary
|
||||
path: "${{ env.TEST_SUMMARY_FILE }}"
|
||||
|
||||
- name: Upload aggregated results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: aggregated-result
|
||||
path: ${{ env.AGGREGATED_SUMMARY_FILE }}
|
||||
|
||||
- name: Upload comment artifact
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: comment
|
||||
path: comment
|
||||
@@ -0,0 +1,3 @@
|
||||
# List of intermittent GNU tests to ignore during comparison
|
||||
# Add test paths here (one per line, without .log extension)
|
||||
# Example: tests/ls/stat-free-color
|
||||
Executable
+185
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
GNU Test Results Analyzer and Aggregator
|
||||
|
||||
This script analyzes and aggregates test results from the GNU test suite.
|
||||
It parses JSON files containing test results (PASS/FAIL/SKIP/ERROR) and:
|
||||
1. Counts the number of tests in each result category
|
||||
2. Can aggregate results from multiple JSON files with priority ordering
|
||||
3. Outputs shell export statements for use in GitHub Actions workflows
|
||||
|
||||
Priority order for aggregation (highest to lowest):
|
||||
- PASS: Takes precedence over all other results (best outcome)
|
||||
- FAIL: Takes precedence over ERROR and SKIP
|
||||
- ERROR: Takes precedence over SKIP
|
||||
- SKIP: Lowest priority
|
||||
|
||||
Usage:
|
||||
- Single file:
|
||||
python analyze-gnu-results.py test-results.json
|
||||
|
||||
- Multiple files (with aggregation):
|
||||
python analyze-gnu-results.py file1.json file2.json
|
||||
|
||||
- With output file for aggregated results:
|
||||
python analyze-gnu-results.py -o=output.json file1.json file2.json
|
||||
|
||||
Output:
|
||||
Prints shell export statements for TOTAL, PASS, FAIL, SKIP, XPASS, and ERROR
|
||||
that can be evaluated in a shell environment.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
def get_priority(result):
|
||||
"""Return a priority value for result status (lower is higher priority)"""
|
||||
priorities = {
|
||||
"PASS": 0, # PASS is highest priority (best result)
|
||||
"FAIL": 1, # FAIL is second priority
|
||||
"ERROR": 2, # ERROR is third priority
|
||||
"SKIP": 3, # SKIP is lowest priority
|
||||
}
|
||||
return priorities.get(result, 4) # Unknown states have lowest priority
|
||||
|
||||
|
||||
def aggregate_results(json_files):
|
||||
"""
|
||||
Aggregate test results from multiple JSON files.
|
||||
Prioritizes results in the order: SKIP > ERROR > FAIL > PASS
|
||||
"""
|
||||
# Combined results dictionary
|
||||
combined_results = defaultdict(dict)
|
||||
|
||||
# Process each JSON file
|
||||
for json_file in json_files:
|
||||
try:
|
||||
with open(json_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# For each utility and its tests
|
||||
for utility, tests in data.items():
|
||||
for test_name, result in tests.items():
|
||||
# If this test hasn't been seen yet, add it
|
||||
if test_name not in combined_results[utility]:
|
||||
combined_results[utility][test_name] = result
|
||||
else:
|
||||
# If it has been seen, apply priority rules
|
||||
current_priority = get_priority(
|
||||
combined_results[utility][test_name]
|
||||
)
|
||||
new_priority = get_priority(result)
|
||||
|
||||
# Lower priority value means higher precedence
|
||||
if new_priority < current_priority:
|
||||
combined_results[utility][test_name] = result
|
||||
except FileNotFoundError:
|
||||
print(f"Warning: File '{json_file}' not found.", file=sys.stderr)
|
||||
continue
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: '{json_file}' is not a valid JSON file.", file=sys.stderr)
|
||||
continue
|
||||
|
||||
return combined_results
|
||||
|
||||
|
||||
def analyze_test_results(json_data):
|
||||
"""
|
||||
Analyze test results from GNU test suite JSON data.
|
||||
Counts PASS, FAIL, SKIP results for all tests.
|
||||
"""
|
||||
# Counters for test results
|
||||
total_tests = 0
|
||||
pass_count = 0
|
||||
fail_count = 0
|
||||
skip_count = 0
|
||||
xpass_count = 0 # Not in JSON data but included for compatibility
|
||||
error_count = 0 # Not in JSON data but included for compatibility
|
||||
|
||||
# Analyze each utility's tests
|
||||
for utility, tests in json_data.items():
|
||||
for test_name, result in tests.items():
|
||||
total_tests += 1
|
||||
|
||||
match result:
|
||||
case "PASS":
|
||||
pass_count += 1
|
||||
case "FAIL":
|
||||
fail_count += 1
|
||||
case "SKIP":
|
||||
skip_count += 1
|
||||
case "ERROR":
|
||||
error_count += 1
|
||||
case "XPASS":
|
||||
xpass_count += 1
|
||||
|
||||
# Return the statistics
|
||||
return {
|
||||
"TOTAL": total_tests,
|
||||
"PASS": pass_count,
|
||||
"FAIL": fail_count,
|
||||
"SKIP": skip_count,
|
||||
"XPASS": xpass_count,
|
||||
"ERROR": error_count,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to process JSON files and export variables.
|
||||
Supports both single file analysis and multi-file aggregation.
|
||||
"""
|
||||
# Check if file arguments were provided
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python analyze-gnu-results.py <json> [json ...]")
|
||||
print(" For multiple files, results will be aggregated")
|
||||
print(" Priority SKIP > ERROR > FAIL > PASS")
|
||||
sys.exit(1)
|
||||
|
||||
json_files = sys.argv[1:]
|
||||
output_file = None
|
||||
|
||||
# Check if the first argument is an output file (starts with -)
|
||||
if json_files[0].startswith("-o="):
|
||||
output_file = json_files[0][3:]
|
||||
json_files = json_files[1:]
|
||||
|
||||
# Process the files
|
||||
if len(json_files) == 1:
|
||||
# Single file analysis
|
||||
try:
|
||||
with open(json_files[0], "r") as file:
|
||||
json_data = json.load(file)
|
||||
results = analyze_test_results(json_data)
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File '{json_files[0]}' not found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError:
|
||||
print(
|
||||
f"Error: '{json_files[0]}' is not a valid JSON file.", file=sys.stderr
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Multiple files - aggregate them
|
||||
json_data = aggregate_results(json_files)
|
||||
results = analyze_test_results(json_data)
|
||||
|
||||
# Save aggregated data if output file is specified
|
||||
if output_file:
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(json_data, f, indent=2)
|
||||
|
||||
# Print export statements for shell evaluation
|
||||
print(f"export TOTAL={results['TOTAL']}")
|
||||
print(f"export PASS={results['PASS']}")
|
||||
print(f"export SKIP={results['SKIP']}")
|
||||
print(f"export FAIL={results['FAIL']}")
|
||||
print(f"export XPASS={results['XPASS']}")
|
||||
print(f"export ERROR={results['ERROR']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
ME="${0}"
|
||||
ME_dir="$(dirname -- "$("${READLINK:-readlink}" -fm -- "${ME}")")"
|
||||
REPO_main_dir="$(dirname -- "${ME_dir}")"
|
||||
|
||||
# Default profile is 'debug'
|
||||
UU_MAKE_PROFILE='debug'
|
||||
|
||||
for arg in "$@"
|
||||
do
|
||||
if [ "$arg" == "--release-build" ]; then
|
||||
UU_MAKE_PROFILE='release'
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
path_UUTILS=${path_UUTILS:-${REPO_main_dir}}
|
||||
path_GNU="${path_GNU:-${path_UUTILS}/../gnu}"
|
||||
|
||||
echo "Building uutils tar..."
|
||||
cd "${path_UUTILS}"
|
||||
cargo build --profile "${UU_MAKE_PROFILE}" --bin tarapp
|
||||
|
||||
if [[ ! -z "$CARGO_TARGET_DIR" ]]; then
|
||||
UU_BUILD_DIR="${CARGO_TARGET_DIR}/${UU_MAKE_PROFILE}"
|
||||
else
|
||||
UU_BUILD_DIR="${path_UUTILS}/target/${UU_MAKE_PROFILE}"
|
||||
fi
|
||||
|
||||
# Symlink tarapp to tar so tests find it as 'tar'
|
||||
ln -sf "${UU_BUILD_DIR}/tarapp" "${UU_BUILD_DIR}/tar"
|
||||
echo "Created symlink ${UU_BUILD_DIR}/tar -> tarapp"
|
||||
|
||||
# Clone GNU tar if needed
|
||||
if test ! -d "${path_GNU}/.git"; then
|
||||
echo "Cloning GNU tar..."
|
||||
git clone --recurse-submodules https://git.savannah.gnu.org/git/tar.git "${path_GNU}"
|
||||
cd "${path_GNU}"
|
||||
git checkout v1.35
|
||||
git submodule update --init --recursive
|
||||
|
||||
# Bootstrap requires gnulib and generates the configure script
|
||||
./bootstrap --skip-po
|
||||
fi
|
||||
|
||||
cd "${path_GNU}"
|
||||
|
||||
if [ ! -f Makefile ]; then
|
||||
echo "Configuring GNU tar..."
|
||||
# Configure to build native tar (needed for test suite generation)
|
||||
./configure --quiet
|
||||
fi
|
||||
|
||||
echo "Building GNU tar (for test suite)..."
|
||||
make -j$(nproc)
|
||||
Executable
+246
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compare GNU test results between current run and reference to identify regressions and fixes.
|
||||
|
||||
Arguments:
|
||||
CURRENT_JSON Path to the current run's aggregated results JSON file
|
||||
REFERENCE_JSON Path to the reference (main branch) aggregated results JSON file
|
||||
--ignore-file Path to file containing list of tests to ignore (for intermittent issues)
|
||||
--output Path to output file for GitHub comment content
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def flatten_test_results(results):
|
||||
"""Convert nested JSON test results to a flat dictionary of test paths to statuses."""
|
||||
flattened = {}
|
||||
for util, tests in results.items():
|
||||
for test_name, status in tests.items():
|
||||
# Build the full test path
|
||||
test_path = f"tests/{util}/{test_name}"
|
||||
# Remove the .log extension
|
||||
test_path = test_path.replace(".log", "")
|
||||
flattened[test_path] = status
|
||||
return flattened
|
||||
|
||||
|
||||
def load_ignore_list(ignore_file):
|
||||
"""Load list of tests to ignore from file."""
|
||||
if not os.path.exists(ignore_file):
|
||||
return set()
|
||||
|
||||
with open(ignore_file, "r") as f:
|
||||
return {line.strip() for line in f if line.strip() and not line.startswith("#")}
|
||||
|
||||
|
||||
def identify_test_changes(current_flat, reference_flat):
|
||||
"""
|
||||
Identify different categories of test changes between current and reference results.
|
||||
|
||||
Args:
|
||||
current_flat (dict): Flattened dictionary of current test results
|
||||
reference_flat (dict): Flattened dictionary of reference test results
|
||||
|
||||
Returns:
|
||||
tuple: Five lists containing regressions, fixes, newly_skipped, newly_passing, and newly_failing tests
|
||||
"""
|
||||
# Find regressions (tests that were passing but now failing)
|
||||
regressions = []
|
||||
for test_path, status in current_flat.items():
|
||||
if status in ("FAIL", "ERROR"):
|
||||
if test_path in reference_flat:
|
||||
if reference_flat[test_path] == "PASS":
|
||||
regressions.append(test_path)
|
||||
|
||||
# Find fixes (tests that were failing but now passing)
|
||||
fixes = []
|
||||
for test_path, status in reference_flat.items():
|
||||
if status in ("FAIL", "ERROR"):
|
||||
if test_path in current_flat:
|
||||
if current_flat[test_path] == "PASS":
|
||||
fixes.append(test_path)
|
||||
|
||||
# Find newly skipped tests (were passing, now skipped)
|
||||
newly_skipped = []
|
||||
for test_path, status in current_flat.items():
|
||||
if (
|
||||
status == "SKIP"
|
||||
and test_path in reference_flat
|
||||
and reference_flat[test_path] == "PASS"
|
||||
):
|
||||
newly_skipped.append(test_path)
|
||||
|
||||
# Find newly passing tests (were skipped, now passing)
|
||||
newly_passing = []
|
||||
for test_path, status in current_flat.items():
|
||||
if (
|
||||
status == "PASS"
|
||||
and test_path in reference_flat
|
||||
and reference_flat[test_path] == "SKIP"
|
||||
):
|
||||
newly_passing.append(test_path)
|
||||
|
||||
# Find newly failing tests (were skipped, now failing)
|
||||
newly_failing = []
|
||||
for test_path, status in current_flat.items():
|
||||
if (
|
||||
status in ("FAIL", "ERROR")
|
||||
and test_path in reference_flat
|
||||
and reference_flat[test_path] == "SKIP"
|
||||
):
|
||||
newly_failing.append(test_path)
|
||||
|
||||
return regressions, fixes, newly_skipped, newly_passing, newly_failing
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare GNU test results and identify regressions and fixes"
|
||||
)
|
||||
parser.add_argument("current_json", help="Path to current run JSON results")
|
||||
parser.add_argument("reference_json", help="Path to reference JSON results")
|
||||
parser.add_argument(
|
||||
"--ignore-file",
|
||||
required=True,
|
||||
help="Path to file with tests to ignore (for intermittent issues)",
|
||||
)
|
||||
parser.add_argument("--output", help="Path to output file for GitHub comment")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load test results
|
||||
try:
|
||||
with open(args.current_json, "r") as f:
|
||||
current_results = json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError) as e:
|
||||
sys.stderr.write(f"Error loading current results: {e}\n")
|
||||
return 1
|
||||
|
||||
try:
|
||||
with open(args.reference_json, "r") as f:
|
||||
reference_results = json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError) as e:
|
||||
sys.stderr.write(f"Error loading reference results: {e}\n")
|
||||
sys.stderr.write("Skipping comparison as reference is not available.\n")
|
||||
return 0
|
||||
|
||||
# Load ignore list (required)
|
||||
if not os.path.exists(args.ignore_file):
|
||||
sys.stderr.write(f"Error: Ignore file {args.ignore_file} does not exist\n")
|
||||
return 1
|
||||
|
||||
ignore_list = load_ignore_list(args.ignore_file)
|
||||
print(f"Loaded {len(ignore_list)} tests to ignore from {args.ignore_file}")
|
||||
|
||||
# Flatten result structures for easier comparison
|
||||
current_flat = flatten_test_results(current_results)
|
||||
reference_flat = flatten_test_results(reference_results)
|
||||
|
||||
# Identify different categories of test changes
|
||||
regressions, fixes, newly_skipped, newly_passing, newly_failing = (
|
||||
identify_test_changes(current_flat, reference_flat)
|
||||
)
|
||||
|
||||
# Filter out intermittent issues from regressions
|
||||
real_regressions = [r for r in regressions if r not in ignore_list]
|
||||
intermittent_regressions = [r for r in regressions if r in ignore_list]
|
||||
|
||||
# Filter out intermittent issues from fixes
|
||||
real_fixes = [f for f in fixes if f not in ignore_list]
|
||||
intermittent_fixes = [f for f in fixes if f in ignore_list]
|
||||
|
||||
# Filter out intermittent issues from newly failing
|
||||
real_newly_failing = [n for n in newly_failing if n not in ignore_list]
|
||||
intermittent_newly_failing = [n for n in newly_failing if n in ignore_list]
|
||||
|
||||
# Print summary stats
|
||||
print(f"Total tests in current run: {len(current_flat)}")
|
||||
print(f"Total tests in reference: {len(reference_flat)}")
|
||||
print(f"New regressions: {len(real_regressions)}")
|
||||
print(f"Intermittent regressions: {len(intermittent_regressions)}")
|
||||
print(f"Fixed tests: {len(real_fixes)}")
|
||||
print(f"Intermittent fixes: {len(intermittent_fixes)}")
|
||||
print(f"Newly skipped tests: {len(newly_skipped)}")
|
||||
print(f"Newly passing tests (previously skipped): {len(newly_passing)}")
|
||||
print(f"Newly failing tests (previously skipped): {len(real_newly_failing)}")
|
||||
print(f"Intermittent newly failing: {len(intermittent_newly_failing)}")
|
||||
|
||||
output_lines = []
|
||||
|
||||
# Report regressions
|
||||
if real_regressions:
|
||||
print("\nREGRESSIONS (non-intermittent failures):", file=sys.stderr)
|
||||
for test in sorted(real_regressions):
|
||||
msg = f"GNU test failed: {test}. {test} is passing on 'main'. Maybe you have to rebase?"
|
||||
print(f"::error ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report intermittent issues (regressions)
|
||||
if intermittent_regressions:
|
||||
print("\nINTERMITTENT ISSUES (ignored regressions):", file=sys.stderr)
|
||||
for test in sorted(intermittent_regressions):
|
||||
msg = f"Skip an intermittent issue {test} (fails in this run but passes in the 'main' branch)"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report intermittent issues (fixes)
|
||||
if intermittent_fixes:
|
||||
print("\nINTERMITTENT ISSUES (ignored fixes):", file=sys.stderr)
|
||||
for test in sorted(intermittent_fixes):
|
||||
msg = f"Skipping an intermittent issue {test} (passes in this run but fails in the 'main' branch)"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report fixes
|
||||
if real_fixes:
|
||||
print("\nFIXED TESTS:", file=sys.stderr)
|
||||
for test in sorted(real_fixes):
|
||||
msg = f"Congrats! The gnu test {test} is no longer failing!"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report newly skipped and passing tests
|
||||
if newly_skipped:
|
||||
print("\nNEWLY SKIPPED TESTS:", file=sys.stderr)
|
||||
for test in sorted(newly_skipped):
|
||||
msg = f"Note: The gnu test {test} is now being skipped but was previously passing."
|
||||
print(f"::warning ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
if newly_passing:
|
||||
print("\nNEWLY PASSING TESTS (previously skipped):", file=sys.stderr)
|
||||
for test in sorted(newly_passing):
|
||||
msg = f"Congrats! The gnu test {test} is now passing!"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
# Report newly failing tests (were skipped, now failing)
|
||||
if real_newly_failing:
|
||||
print("\nNEWLY FAILING TESTS (previously skipped):", file=sys.stderr)
|
||||
for test in sorted(real_newly_failing):
|
||||
msg = f"Note: The gnu test {test} was skipped on 'main' but is now failing."
|
||||
print(f"::warning ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
if intermittent_newly_failing:
|
||||
print("\nINTERMITTENT NEWLY FAILING (ignored):", file=sys.stderr)
|
||||
for test in sorted(intermittent_newly_failing):
|
||||
msg = f"Skip an intermittent issue {test} (was skipped on 'main', now failing)"
|
||||
print(f"::notice ::{msg}", file=sys.stderr)
|
||||
output_lines.append(msg)
|
||||
|
||||
if args.output and output_lines:
|
||||
with open(args.output, "w") as f:
|
||||
for line in output_lines:
|
||||
f.write(f"{line}\n")
|
||||
|
||||
# Return exit code based on whether we found regressions
|
||||
return 1 if real_regressions else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Extract the GNU logs into a JSON file.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
out = {}
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python gnu-json-result.py <gnu_test_directory>")
|
||||
sys.exit(1)
|
||||
|
||||
test_dir = Path(sys.argv[1])
|
||||
if not test_dir.is_dir():
|
||||
print(f"Directory {test_dir} does not exist.")
|
||||
sys.exit(1)
|
||||
|
||||
# Test all the logs from the test execution
|
||||
for filepath in test_dir.glob("**/*.log"):
|
||||
path = Path(filepath)
|
||||
if path.name == "testsuite.log":
|
||||
# Handle Autotest testsuite.log
|
||||
try:
|
||||
with open(path, "r", errors="ignore") as f:
|
||||
for line in f:
|
||||
# Look for lines like: " 1: basic functionality ok"
|
||||
# or " 10: ... FAILED (basic.at:123)"
|
||||
match = re.match(r"^\s*(\d+):\s+(.*?)\s+(ok|FAILED|skipped)(?:\s+\(.*\))?$", line)
|
||||
if match:
|
||||
num, name, status = match.groups()
|
||||
# Map Autotest status to Automake-style status
|
||||
status_map = {"ok": "PASS", "FAILED": "FAIL", "skipped": "SKIP"}
|
||||
out[f"test {num}"] = {name.strip(): status_map.get(status, status)}
|
||||
except Exception as e:
|
||||
print(f"Error processing testsuite.log {path}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
# Handle individual Automake-style .log files
|
||||
current = out
|
||||
for key in path.parent.relative_to(test_dir).parts:
|
||||
if key not in current:
|
||||
current[key] = {}
|
||||
current = current[key]
|
||||
try:
|
||||
with open(path, "r", errors="ignore") as f:
|
||||
# Only read the end of the file where the result is usually located
|
||||
f.seek(0, 2)
|
||||
size = f.tell()
|
||||
f.seek(max(0, size - 1000), 0)
|
||||
content = f.read()
|
||||
result = re.search(
|
||||
r"(PASS|FAIL|SKIP|ERROR) [^ ]+ \(exit status: \d+\)$", content
|
||||
)
|
||||
if result:
|
||||
current[path.name] = result.group(1)
|
||||
except Exception as e:
|
||||
print(f"Error processing file {path}: {e}", file=sys.stderr)
|
||||
|
||||
print(json.dumps(out, indent=2, sort_keys=True))
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# Use GNU version for make, nproc, readlink on *BSD
|
||||
MAKE=$(command -v gmake || command -v make)
|
||||
NPROC=$(command -v gnproc || command -v nproc)
|
||||
READLINK=$(command -v greadlink || command -v readlink)
|
||||
|
||||
ME="${0}"
|
||||
ME_dir="$(dirname -- "$("${READLINK:-readlink}" -fm -- "${ME}")")"
|
||||
REPO_main_dir="$(dirname -- "${ME_dir}")"
|
||||
|
||||
path_UUTILS=${path_UUTILS:-${REPO_main_dir}}
|
||||
path_GNU="${path_GNU:-${path_UUTILS}/../gnu}"
|
||||
|
||||
# Determine profile
|
||||
if [[ -d "${path_UUTILS}/target/release" ]]; then
|
||||
UU_BUILD_DIR="${path_UUTILS}/target/release"
|
||||
elif [[ -d "${path_UUTILS}/target/debug" ]]; then
|
||||
UU_BUILD_DIR="${path_UUTILS}/target/debug"
|
||||
else
|
||||
echo "Could not find build directory in ${path_UUTILS}/target"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using uutils tar from: ${UU_BUILD_DIR}"
|
||||
|
||||
cd "${path_GNU}"
|
||||
|
||||
export RUST_BACKTRACE=1
|
||||
|
||||
# The GNU tar testsuite usually looks for 'tar' in the path or uses the one in src/
|
||||
# We force it to use ours by putting it first in PATH.
|
||||
export PATH="${UU_BUILD_DIR}:$PATH"
|
||||
export TAR="${UU_BUILD_DIR}/tar"
|
||||
|
||||
echo "Running GNU tar tests..."
|
||||
|
||||
# Run with timeout and make check
|
||||
# We use $* to pass any additional user arguments (e.g. TESTSUITEFLAGS="1-5")
|
||||
cp "${TAR}" src/tar
|
||||
timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check "$@"
|
||||
Reference in New Issue
Block a user