diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 7a05be272..e5bad3e1d 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -127,37 +127,18 @@ jobs: shell: bash command: | ## `cargo clippy` lint testing - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * convert any warnings to GHA UI annotations; ref: - if [[ "${{ matrix.job.features }}" == "all" ]]; then - extra="--all-features" - else - extra="--features ${{ matrix.job.features }}" + ARGS="--features ${{ matrix.job.features }}" + ARGS="${ARGS} --fault-type ${{ steps.vars.outputs.FAULT_TYPE }}" + if [[ "${{ matrix.job.workspace }}" =~ ^(1|t|true|y|yes)$ ]]; then + ARGS="${ARGS} --workspace" fi - case '${{ matrix.job.workspace }}' in - 1|t|true|y|yes) - extra="${extra} --workspace" - ;; - esac if [[ -n "${{ matrix.job.target }}" ]]; then - extra="${extra} --no-default-features --target ${{ matrix.job.target }}" + ARGS="${ARGS} --target ${{ matrix.job.target }}" fi - # * determine sub-crate utility list (similar to FreeBSD workflow) - if [[ -n "${{ matrix.job.target }}" ]]; then - # For cross-compilation targets, just check -pcoreutils (show-utils.sh over-resolves due to default features) - S=$(cargo clippy $extra -pcoreutils -- -D warnings 2>&1) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*$/::${fault_type} file=\2,line=\3,col=\4::${fault_prefix}: \`cargo clippy\`: \1 (file:'\2', line:\3)/p;" -e '}' ; fault=true ; } - else - if [[ "${{ matrix.job.features }}" == "all" ]]; then - UTILITY_LIST="$(./util/show-utils.sh --all-features)" - else - UTILITY_LIST="$(./util/show-utils.sh --features ${{ matrix.job.features }})" - fi - CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)" - S=$(cargo clippy --all-targets $extra --tests --benches -pcoreutils ${CARGO_UTILITY_LIST_OPTIONS} -- -D warnings 2>&1) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*$/::${fault_type} file=\2,line=\3,col=\4::${fault_prefix}: \`cargo clippy\`: \1 (file:'\2', line:\3)/p;" -e '}' ; fault=true ; } + if [[ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ]]; then + ARGS="${ARGS} --fail-on-fault" fi - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi + python3 util/run-clippy.py ${ARGS} - name: "cargo clippy on fuzz dir" if: runner.os != 'Windows' && !matrix.job.target shell: bash diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index f963cf3ef..f31af77ad 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -41,7 +41,7 @@ jobs: sync: rsync copyback: false # We need jq and GNU coreutils to run show-utils.sh and bash to use inline shell string replacement - prepare: pkg install -y curl sudo jq coreutils bash + prepare: pkg install -y curl sudo jq coreutils bash python3 run: | ## Prepare, build, and test # implementation modelled after ref: @@ -73,7 +73,6 @@ jobs: FAULT_PREFIX=\$(echo "\${FAULT_TYPE}" | tr '[:lower:]' '[:upper:]') # * determine sub-crate utility list UTILITY_LIST="\$(./util/show-utils.sh --features ${{ matrix.job.features }})" - CARGO_UTILITY_LIST_OPTIONS="\$(for u in \${UTILITY_LIST}; do echo -n "-puu_\${u} "; done;)" ## Info # environment echo "## environment" @@ -101,8 +100,9 @@ jobs: ## cargo clippy lint testing if [ -z "\${FAULT}" ]; then echo "## cargo clippy lint testing" - # * convert any warnings to GHA UI annotations; ref: - S=\$(cargo clippy --all-targets \${CARGO_UTILITY_LIST_OPTIONS} -- -D warnings 2>&1) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s" "\$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*\$/::\${FAULT_TYPE} file=\2,line=\3,col=\4::\${FAULT_PREFIX}: \\\`cargo clippy\\\`: \1 (file:'\2', line:\3)/p;" -e '}' ; FAULT=true ; } + CLIPPY_ARGS="--features ${{ matrix.job.features }} --fault-type \${FAULT_TYPE}" + if [ -n "\${FAIL_ON_FAULT}" ]; then CLIPPY_ARGS="\${CLIPPY_ARGS} --fail-on-fault"; fi + python3 util/run-clippy.py \${CLIPPY_ARGS} || FAULT=true fi # Clean to avoid to rsync back the files cargo clean diff --git a/.github/workflows/openbsd.yml b/.github/workflows/openbsd.yml index dea831b10..56b10c1d7 100644 --- a/.github/workflows/openbsd.yml +++ b/.github/workflows/openbsd.yml @@ -47,7 +47,7 @@ jobs: prepare: | # Clean up disk space before installing packages df -h - pkg_add curl sudo-- jq coreutils bash rust rust-clippy rust-rustfmt llvm-- + pkg_add curl sudo-- jq coreutils bash rust rust-clippy rust-rustfmt llvm-- python3 rm -rf /usr/share/relink/* /usr/X11R6/* /usr/share/doc/* /usr/share/man/* & # Clean up package cache after installation pkg_delete -a & @@ -84,7 +84,6 @@ jobs: FAULT_PREFIX=\$(echo "\${FAULT_TYPE}" | tr '[:lower:]' '[:upper:]') # * determine sub-crate utility list UTILITY_LIST="\$(./util/show-utils.sh --features ${{ matrix.job.features }})" - CARGO_UTILITY_LIST_OPTIONS="\$(for u in \${UTILITY_LIST}; do echo -n "-puu_\${u} "; done;)" ## Info # environment echo "## environment" @@ -111,8 +110,9 @@ jobs: ## cargo clippy lint testing if [ -z "\${FAULT}" ]; then echo "## cargo clippy lint testing" - # * convert any warnings to GHA UI annotations; ref: - S=\$(cargo clippy --all-targets \${CARGO_UTILITY_LIST_OPTIONS} -- -D warnings 2>&1) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s" "\$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*\$/::\${FAULT_TYPE} file=\2,line=\3,col=\4::\${FAULT_PREFIX}: \\\`cargo clippy\\\`: \1 (file:'\2', line:\3)/p;" -e '}' ; FAULT=true ; } + CLIPPY_ARGS="--features ${{ matrix.job.features }} --fault-type \${FAULT_TYPE}" + if [ -n "\${FAIL_ON_FAULT}" ]; then CLIPPY_ARGS="\${CLIPPY_ARGS} --fail-on-fault"; fi + python3 util/run-clippy.py \${CLIPPY_ARGS} || FAULT=true fi # Clean to avoid to rsync back the files and free up disk space cargo clean diff --git a/util/run-clippy.py b/util/run-clippy.py new file mode 100755 index 000000000..5dcf805ba --- /dev/null +++ b/util/run-clippy.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# spell-checker:ignore pcoreutils +"""Run cargo clippy with appropriate flags and emit GitHub Actions annotations.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys + + +def run_cmd( + cmd: list[str], + *, + check: bool = False, +) -> subprocess.CompletedProcess[str]: + """Run a command with UTF-8 encoding (avoids cp1252 issues on Windows).""" + env = {**os.environ, "PYTHONUTF8": "1"} + return subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=check, + env=env, + ) + + +def get_utility_list(features: str) -> list[str]: + """Get list of utilities from cargo metadata.""" + if features == "all": + cmd = ["cargo", "metadata", "--all-features", "--format-version", "1"] + else: + cmd = ["cargo", "metadata", "--features", features, "--format-version", "1"] + result = run_cmd(cmd, check=True) + metadata = json.loads(result.stdout) + # Find the coreutils root node and collect uu_ dependencies + utilities = [] + for node in metadata["resolve"]["nodes"]: + if re.search(r"coreutils[ @#]\d+\.\d+\.\d+", node["id"]): + for dep in node["deps"]: + # The pkg field contains the crate name (uu_), + # while name is the renamed dependency alias + pkg = dep["pkg"] + match = re.search(r"uu_(\w+)[@#]", pkg) + if match: + utilities.append(match.group(1)) + break + return sorted(utilities) + + +def build_clippy_command( + features: str, + *, + workspace: bool, + target: str | None, +) -> list[str]: + """Build the cargo clippy command line.""" + cmd = ["cargo", "clippy"] + + extra = [] + if features == "all": + extra.append("--all-features") + else: + extra.extend(["--features", features]) + + if workspace: + extra.append("--workspace") + + if target: + extra.extend(["--no-default-features", "--target", target]) + # For cross-compilation targets, just check -pcoreutils + # (show-utils.sh over-resolves due to default features) + extra.append("-pcoreutils") + else: + extra.extend(["--all-targets", "--tests", "--benches", "-pcoreutils"]) + utilities = get_utility_list(features) + extra.extend(f"-puu_{u}" for u in utilities) + + cmd.extend(extra) + cmd.extend(["--", "-D", "warnings"]) + return cmd + + +# Pattern to match clippy/rustc errors for GHA annotations +ERROR_PATTERN = re.compile( + r"^error:\s+(.*)\n\s+-->\s+(.*):(\d+):(\d+)", + re.MULTILINE, +) + + +def emit_annotations(output: str, fault_type: str) -> None: + """Emit GitHub Actions annotations from cargo clippy errors.""" + fault_prefix = fault_type.upper() + for m in ERROR_PATTERN.finditer(output): + message, file, line, col = m.groups() + print( + f"::{fault_type} file={file},line={line},col={col}" + f"::{fault_prefix}: `cargo clippy`: {message} (file:'{file}', line:{line})", + ) + + +def main() -> int: + """Run cargo clippy and emit GHA annotations on failure.""" + parser = argparse.ArgumentParser(description="Run cargo clippy for CI") + parser.add_argument("--features", required=True, help="Feature set to use") + parser.add_argument( + "--workspace", + action="store_true", + help="Include --workspace flag", + ) + parser.add_argument("--target", default=None, help="Cross-compilation target") + parser.add_argument( + "--fault-type", + default="warning", + choices=["warning", "error"], + help="GHA annotation type", + ) + parser.add_argument( + "--fail-on-fault", + action="store_true", + help="Exit with error code on clippy failures", + ) + args = parser.parse_args() + + cmd = build_clippy_command( + args.features, + workspace=args.workspace, + target=args.target, + ) + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + + result = run_cmd(cmd) + output = result.stdout + result.stderr + + # Always print the full output + print(output) + + if result.returncode != 0: + emit_annotations(output, args.fault_type) + if args.fail_on_fault: + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main())