mirror of
https://github.com/uutils/diffutils.git
synced 2026-06-10 15:48:59 -07:00
feat: add gnu testsuite tracking workflows
This commit is contained in:
pocopepe
committed by
Sylvestre Ledru
Sylvestre Ledru
parent
9a7a727da4
commit
2c47ea9f04
@@ -0,0 +1,100 @@
|
||||
name: GnuComment
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["GnuTests"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
post-comment:
|
||||
permissions:
|
||||
actions: read # to list workflow runs artifacts
|
||||
pull-requests: write # to comment on pr
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
if: >
|
||||
github.event.workflow_run.event == 'pull_request'
|
||||
steps:
|
||||
- name: 'Download artifact'
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
// List all artifacts from GnuTests
|
||||
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: ${{ github.event.workflow_run.id }},
|
||||
});
|
||||
|
||||
// Download the "comment" artifact, which contains a PR number (NR) and result.txt
|
||||
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
|
||||
return artifact.name == "comment"
|
||||
})[0];
|
||||
|
||||
if (!matchArtifact) {
|
||||
console.log('No comment artifact found');
|
||||
return;
|
||||
}
|
||||
|
||||
var download = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: matchArtifact.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
var fs = require('fs');
|
||||
fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
|
||||
|
||||
- run: unzip comment.zip || echo "Failed to unzip comment artifact"
|
||||
|
||||
- name: 'Comment on PR'
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
var fs = require('fs');
|
||||
|
||||
// Check if files exist
|
||||
if (!fs.existsSync('./NR')) {
|
||||
console.log('No NR file found, skipping comment');
|
||||
return;
|
||||
}
|
||||
if (!fs.existsSync('./result.txt')) {
|
||||
console.log('No result.txt file found, skipping comment');
|
||||
return;
|
||||
}
|
||||
|
||||
var issue_number = Number(fs.readFileSync('./NR'));
|
||||
var content = fs.readFileSync('./result.txt');
|
||||
|
||||
if (content.toString().trim().length > 7) { // 7 because we have backquote + \n
|
||||
// Update existing comment if present, otherwise create a new one
|
||||
var marker = '<!-- gnu-tests-bot -->';
|
||||
var body = marker + '\nGNU diffutils testsuite comparison:\n```\n' + content + '```';
|
||||
var comments = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue_number,
|
||||
});
|
||||
var existing = comments.data.filter(c => c.body.includes(marker))[0];
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body: body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue_number,
|
||||
body: body,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.log('Comment content too short, skipping');
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
name: GnuTests
|
||||
|
||||
# Run GNU diffutils testsuite against the Rust diffutils implementation
|
||||
# and compare results against the main branch to catch regressions
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write # Publish diffutils instead of discarding
|
||||
|
||||
# End the current execution if there is a new changeset in the PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEST_FULL_SUMMARY_FILE: 'diffutils-gnu-full-result.json'
|
||||
|
||||
jobs:
|
||||
native:
|
||||
name: Run GNU diffutils testsuite
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
### Build
|
||||
- name: Build Rust diffutils binary
|
||||
shell: bash
|
||||
run: |
|
||||
## Build Rust diffutils binary
|
||||
cargo build --config=profile.release.strip=true --profile=release
|
||||
zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst
|
||||
|
||||
- name: Publish latest commit
|
||||
uses: softprops/action-gh-release@v3
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
with:
|
||||
tag_name: latest-commit
|
||||
body: |
|
||||
commit: ${{ github.sha }}
|
||||
draft: false
|
||||
prerelease: true
|
||||
files: |
|
||||
diffutils-x86_64-unknown-linux-gnu.zst
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
### Run tests
|
||||
- name: Run GNU diffutils testsuite
|
||||
shell: bash
|
||||
run: |
|
||||
## Run GNU diffutils testsuite
|
||||
./tests/run-upstream-testsuite.sh release || true
|
||||
env:
|
||||
TERM: xterm
|
||||
|
||||
- name: Upload full json results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: diffutils-gnu-full-result
|
||||
path: tests/test-results.json
|
||||
if-no-files-found: warn
|
||||
|
||||
aggregate:
|
||||
needs: [native]
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
name: Aggregate GNU test results
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Initialize workflow variables
|
||||
id: vars
|
||||
shell: bash
|
||||
run: |
|
||||
## VARs setup
|
||||
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
|
||||
|
||||
TEST_SUMMARY_FILE='diffutils-gnu-result.json'
|
||||
outputs TEST_SUMMARY_FILE
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Retrieve reference artifacts
|
||||
uses: dawidd6/action-download-artifact@v20
|
||||
continue-on-error: true
|
||||
with:
|
||||
workflow: GnuTests.yml
|
||||
branch: "${{ env.DEFAULT_BRANCH }}"
|
||||
workflow_conclusion: completed
|
||||
path: "reference"
|
||||
if_no_artifact_found: warn
|
||||
|
||||
- name: Download full json results
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: diffutils-gnu-full-result
|
||||
path: results
|
||||
|
||||
- name: Extract/summarize testing info
|
||||
id: summary
|
||||
shell: bash
|
||||
run: |
|
||||
## Extract/summarize testing info
|
||||
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
|
||||
|
||||
RESULT_FILE="results/test-results.json"
|
||||
if [[ ! -f "$RESULT_FILE" ]]; then
|
||||
echo "::error ::Missing test results at $RESULT_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOTAL=$(jq '[.tests[]] | length' "$RESULT_FILE")
|
||||
PASS=$(jq '[.tests[] | select(.result=="PASS")] | length' "$RESULT_FILE")
|
||||
FAIL=$(jq '[.tests[] | select(.result=="FAIL")] | length' "$RESULT_FILE")
|
||||
SKIP=$(jq '[.tests[] | select(.result=="SKIP")] | length' "$RESULT_FILE")
|
||||
ERROR=0
|
||||
|
||||
output="GNU diffutils tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP"
|
||||
echo "${output}"
|
||||
|
||||
if [[ "$FAIL" -gt 0 ]]; then
|
||||
echo "::warning ::${output}"
|
||||
fi
|
||||
|
||||
jq -n \
|
||||
--arg date "$(date --rfc-email)" \
|
||||
--arg sha "$GITHUB_SHA" \
|
||||
--arg total "$TOTAL" \
|
||||
--arg pass "$PASS" \
|
||||
--arg skip "$SKIP" \
|
||||
--arg fail "$FAIL" \
|
||||
--arg error "$ERROR" \
|
||||
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
|
||||
|
||||
HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
|
||||
outputs HASH TOTAL PASS FAIL SKIP
|
||||
|
||||
- name: Upload SHA1/ID of 'test-summary'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "${{ steps.summary.outputs.HASH }}"
|
||||
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
||||
|
||||
- name: Upload test results summary
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-summary
|
||||
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
||||
|
||||
- name: Compare test failures VS reference
|
||||
shell: bash
|
||||
run: |
|
||||
## Compare test failures VS reference
|
||||
REF_SUMMARY_FILE='reference/diffutils-gnu-full-result/test-results.json'
|
||||
CURRENT_SUMMARY_FILE="results/test-results.json"
|
||||
|
||||
IGNORE_INTERMITTENT=".github/workflows/ignore-intermittent.txt"
|
||||
|
||||
COMMENT_DIR="reference/comment"
|
||||
mkdir -p ${COMMENT_DIR}
|
||||
echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
|
||||
COMMENT_LOG="${COMMENT_DIR}/result.txt"
|
||||
|
||||
COMPARISON_RESULT=0
|
||||
if test -f "${CURRENT_SUMMARY_FILE}"; then
|
||||
if test -f "${REF_SUMMARY_FILE}"; then
|
||||
echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
|
||||
echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")"
|
||||
|
||||
python3 util/compare_test_results.py \
|
||||
--ignore-file "${IGNORE_INTERMITTENT}" \
|
||||
--output "${COMMENT_LOG}" \
|
||||
"${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
|
||||
|
||||
COMPARISON_RESULT=$?
|
||||
else
|
||||
echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'."
|
||||
fi
|
||||
else
|
||||
echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ${COMPARISON_RESULT} -eq 1 ]; then
|
||||
echo "::error ::Found new non-intermittent test failures"
|
||||
exit 1
|
||||
else
|
||||
echo "::notice ::No new test failures detected"
|
||||
fi
|
||||
|
||||
- name: Upload comparison log (for GnuComment workflow)
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: comment
|
||||
path: reference/comment/
|
||||
|
||||
- name: Report test results
|
||||
if: success() || failure()
|
||||
shell: bash
|
||||
run: |
|
||||
## Report final results
|
||||
echo "::notice ::GNU diffutils testsuite results:"
|
||||
echo "::notice :: Total tests: ${{ steps.summary.outputs.TOTAL }}"
|
||||
echo "::notice :: Passed: ${{ steps.summary.outputs.PASS }}"
|
||||
echo "::notice :: Failed: ${{ steps.summary.outputs.FAIL }}"
|
||||
echo "::notice :: Skipped: ${{ steps.summary.outputs.SKIP }}"
|
||||
|
||||
if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then
|
||||
PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) ))
|
||||
echo "::notice :: Pass rate: ${PASS_RATE}%"
|
||||
fi
|
||||
@@ -56,37 +56,6 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- run: cargo clippy -- -D warnings
|
||||
|
||||
gnu-testsuite:
|
||||
permissions:
|
||||
contents: write # Publish diffutils instead of discarding
|
||||
name: GNU test suite
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: |
|
||||
cargo build --config=profile.release.strip=true --profile=release #-fast
|
||||
zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst
|
||||
# do not fail, the report is merely informative (at least until all tests pass reliably)
|
||||
- run: ./tests/run-upstream-testsuite.sh release || true
|
||||
env:
|
||||
TERM: xterm
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-results.json
|
||||
path: tests/test-results.json
|
||||
- run: ./tests/print-test-results.sh tests/test-results.json
|
||||
- name: Publish latest commit
|
||||
uses: softprops/action-gh-release@v3
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
with:
|
||||
tag_name: latest-commit
|
||||
draft: false
|
||||
prerelease: true
|
||||
files: |
|
||||
diffutils-x86_64-unknown-linux-gnu.zst
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
coverage:
|
||||
name: Code Coverage
|
||||
env:
|
||||
|
||||
Reference in New Issue
Block a user