From 9aaf05ef73cdc4ea892f286be0555a95da7d36db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20CORTIER?= Date: Fri, 19 May 2023 19:08:18 -0400 Subject: [PATCH] ci: coverage reporting in PRs Adds a new workflow reporting code coverage when opening a pull request. The idea is to help find blind spots in our test suites and inform where testing efforts should go, but there is no hard requirement related to the numbers. We are allowed to merge pull requests regardless of the coverage percentage, and even if the coverage workflow fails. Past coverage data is stored in a separate branch (cov-data). It gets updated when the workflow run against master. --- .github/workflows/coverage.yml | 44 ++++++ Cargo.lock | 7 + xtask/Cargo.toml | 1 + xtask/src/cli.rs | 32 ++++- xtask/src/cov.rs | 252 ++++++++++++++++++++++++++++++--- xtask/src/main.rs | 5 +- xtask/src/tasks.rs | 1 - 7 files changed, 317 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/coverage.yml delete mode 100644 xtask/src/tasks.rs diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..fdbf40ef --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,44 @@ +name: Coverage + +on: + push: + branches: + - master + pull_request: + types: [ opened, synchronize, reopened ] + workflow_dispatch: + +env: + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + +jobs: + coverage: + name: Coverage Report + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + + - name: Rust cache + uses: Swatinem/rust-cache@v2.3.0 + + - name: Prepare runner + run: cargo xtask cov install + + - name: Generate PR report + if: github.event.number != '' + env: + GH_TOKEN: ${{ secrets.DEVOLUTIONSBOT_TOKEN }} + run: cargo xtask cov report-gh --repo "${{ github.repository }}" --pr "${{ github.event.number }}" + + - name: Configure Git Identity + if: github.ref == 'refs/heads/master' + run: | + git config --local user.name "github-actions[bot]" + git config --local user.email "github-actions[bot]@users.noreply.github.com" + + - name: Update coverage data + if: github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ secrets.DEVOLUTIONSBOT_TOKEN }} + run: cargo xtask cov update diff --git a/Cargo.lock b/Cargo.lock index a4a52c27..db6a3915 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3760,6 +3760,12 @@ dependencies = [ "strict-num", ] +[[package]] +name = "tinyjson" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab95735ea2c8fd51154d01e39cf13912a78071c2d89abc49a7ef102a7dd725a" + [[package]] name = "tinyvec" version = "1.6.0" @@ -4669,6 +4675,7 @@ version = "0.0.0" dependencies = [ "anyhow", "pico-args", + "tinyjson", "xshell", ] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 0088e629..462c77a0 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -12,3 +12,4 @@ test = false anyhow = "1" pico-args = "0.5.0" xshell = "0.2.3" +tinyjson = "2.5.1" diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs index 2d54bc98..aeb88312 100644 --- a/xtask/src/cli.rs +++ b/xtask/src/cli.rs @@ -11,11 +11,14 @@ TASKS: check fmt Check formatting check lints Check lints check tests [--no-run] Compile tests and, unless specified otherwise, run them - check wasm Ensure WASM module is compatible for the web ci Run all checks required on CI clean Clean workspace - coverage install Install dependencies required to generate the coverage report - coverage report Generate code-coverage data using tests and fuzz targets + cov grcov Generate a nice HTML report using code-coverage data from tests and fuzz targets + cov install Install cargo-llvm-cov in cargo local root + cov report-gh --repo --pr + Generate a coverage report, posting a comment in GitHub PR + cov report [--html] Generate a coverage report (optionally, a HTML report) + cov update Update coverage data in the cov-data branch fuzz corpus-fetch Fetch fuzzing corpus from Azure storage fuzz corpus-min Minify fuzzing corpus fuzz corpus-push Push fuzzing corpus to Azure storage @@ -42,8 +45,16 @@ pub enum Action { }, Ci, Clean, + CovGrcov, CovInstall, - CovReport, + CovReportGitHub { + repo: String, + pr: u32, + }, + CovReport { + html_report: bool, + }, + CovUpdate, FuzzCorpusFetch, FuzzCorpusMin, FuzzCorpusPush, @@ -78,10 +89,17 @@ pub fn parse_args() -> anyhow::Result { Some("ci") => Action::Ci, Some("clean") => Action::Clean, Some("cov") => match args.subcommand()?.as_deref() { + Some("grcov") => Action::CovGrcov, Some("install") => Action::CovInstall, - Some("report") => Action::CovReport, - Some(unknown) => anyhow::bail!("unknown coverage action: {unknown}"), - None => Action::ShowHelp, + Some("report-gh") => Action::CovReportGitHub { + repo: args.value_from_str("--repo")?, + pr: args.value_from_str("--pr")?, + }, + Some("report") => Action::CovReport { + html_report: args.contains("--html"), + }, + Some("update") => Action::CovUpdate, + None | Some(_) => anyhow::bail!("Unknown cov action"), }, Some("fuzz") => match args.subcommand()?.as_deref() { Some("corpus-fetch") => Action::FuzzCorpusFetch, diff --git a/xtask/src/cov.rs b/xtask/src/cov.rs index 38d696eb..28e22764 100644 --- a/xtask/src/cov.rs +++ b/xtask/src/cov.rs @@ -1,29 +1,185 @@ +use core::fmt; + use crate::prelude::*; +const COV_IGNORE_REGEX: &str = + r#"(crates/ironrdp-(session|.+generators|.+glutin.+|replay|client|fuzzing|tokio|web|futures|tls)|xtask|testsuite)"#; + pub fn install(sh: &Shell) -> anyhow::Result<()> { - let _s = Section::new("COVERAGE-INSTALL"); + let _s = Section::new("COV-INSTALL"); + + let llvm_cov_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "cargo-llvm-cov"].iter().collect(); + + if !sh.path_exists(llvm_cov_path) { + cmd!( + sh, + "{CARGO} install --locked --root {LOCAL_CARGO_ROOT} cargo-llvm-cov@{CARGO_LLVM_COV_VERSION}" + ) + .run()?; + } + + Ok(()) +} + +pub fn update(sh: &Shell) -> anyhow::Result<()> { + let _s = Section::new("COV-UPDATE"); + + let report = CoverageReport::generate(sh)?; + println!("New:\n{report}"); + + let initial_branch = cmd!(sh, "git rev-parse --abbrev-ref HEAD").read()?; + + println!("Switch branch"); + let _ = cmd!(sh, "git branch -D cov-data").run(); + cmd!(sh, "git checkout --orphan cov-data").run()?; + + let result = || -> anyhow::Result<()> { + cmd!(sh, "git rm --cached -r .").run()?; + + sh.write_file("./report.json", report.original_json_data)?; + + cmd!(sh, "git add ./report.json").run()?; + cmd!(sh, "git commit -m 'cov: update report data'").run()?; + cmd!(sh, "git push --force --set-upstream origin cov-data").run()?; + + Ok(()) + }(); + + println!("Clean working tree"); + cmd!(sh, "git clean -df").run()?; + + println!("Switch back to initial branch"); + cmd!(sh, "git checkout {initial_branch}").run()?; + + result?; + + Ok(()) +} + +pub fn report(sh: &Shell, html_report: bool) -> anyhow::Result<()> { + let _s = Section::new("COV-REPORT"); + + if html_report { + cmd!(sh, "{LOCAL_CARGO_ROOT}/bin/cargo-llvm-cov llvm-cov --html") + .arg("--ignore-filename-regex") + .arg(COV_IGNORE_REGEX) + .run()?; + } else { + let report = CoverageReport::generate(sh)?; + let past_report = CoverageReport::past_report(sh)?; + + println!("Past:\n{past_report}"); + println!("New:\n{report}"); + println!( + "Diff: {:+.2}%", + report.covered_lines_percent - past_report.covered_lines_percent + ); + } + + Ok(()) +} + +pub fn report_github(sh: &Shell, repo: &str, pr_id: u32) -> anyhow::Result<()> { + use std::fmt::Write as _; + + const COMMENT_HEADER: &str = "## Coverage Report :robot: :gear:"; + + let _s = Section::new("COV-REPORT"); + + let report = CoverageReport::generate(sh)?; + let past_report = CoverageReport::past_report(sh)?; + + let diff = report.covered_lines_percent - past_report.covered_lines_percent; + + let comments = cmd!(sh, "gh api") + .arg("-H") + .arg("Accept: application/vnd.github.v3+json") + .arg(format!("/repos/{repo}/issues/{pr_id}/comments")) + .read()?; + + let comments: tinyjson::JsonValue = comments.parse().context("GitHub comments")?; + let comments = comments.get::>().context("comments list")?; + + let mut prev_comment_id = None; + + for comment in comments { + let body = comment["body"].get::().context("comment body")?; + + if body.starts_with(COMMENT_HEADER) { + let comment_id = *comment["id"].get::().context("id")? as u64; + prev_comment_id = Some(comment_id); + break; + } + } + + let mut body = String::new(); + + writeln!(body, "{COMMENT_HEADER}")?; + writeln!(body, "**Past**:\n{past_report}")?; + writeln!(body, "**New**:\n{report}")?; + writeln!( + body, + "**Diff**: {:+.2}%", + report.covered_lines_percent - past_report.covered_lines_percent + )?; + writeln!(body, "\n[this comment will be updated automatically]")?; + + let command = cmd!(sh, "gh api") + .arg("-H") + .arg("Accept: application/vnd.github.v3+json") + .arg("-f") + .arg(format!("body={body}")); + + if let Some(comment_id) = prev_comment_id { + println!("Update existing comment"); + + command + .arg("--method") + .arg("PATCH") + .arg(format!("/repos/{repo}/issues/comments/{comment_id}")) + .ignore_stdout() + .run()?; + } else if diff.abs() < 0.001 { + println!("Create new comment"); + + command + .arg("--method") + .arg("POST") + .arg(format!("/repos/{repo}/issues/{pr_id}/comments")) + .ignore_stdout() + .run()?; + } else { + println!("Coverage didn’t change, skip GitHub comment"); + } + + Ok(()) +} + +pub fn grcov(sh: &Shell) -> anyhow::Result<()> { + let _s = Section::new("COV-GRCOV"); cmd!(sh, "rustup install nightly --profile=minimal").run()?; cmd!(sh, "rustup component add --toolchain nightly llvm-tools-preview").run()?; cmd!(sh, "rustup component add llvm-tools-preview").run()?; - cmd!( - sh, - "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} cargo-fuzz@{CARGO_FUZZ_VERSION}" - ) - .run()?; + let cargo_fuzz_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "cargo-fuzz"].iter().collect(); + let grcov_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "grcov"].iter().collect(); - cmd!( - sh, - "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} grcov@{GRCOV_VERSION}" - ) - .run()?; + if !sh.path_exists(cargo_fuzz_path) { + cmd!( + sh, + "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} cargo-fuzz@{CARGO_FUZZ_VERSION}" + ) + .run()?; + } - Ok(()) -} - -pub fn report(sh: &Shell) -> anyhow::Result<()> { - let _s = Section::new("COVERAGE-REPORT"); + if !sh.path_exists(grcov_path) { + cmd!( + sh, + "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} grcov@{GRCOV_VERSION}" + ) + .run()?; + } println!("Remove leftovers"); sh.remove_path("./fuzz/coverage/")?; @@ -111,3 +267,67 @@ pub fn report(sh: &Shell) -> anyhow::Result<()> { Ok(()) } + +struct CoverageReport { + total_lines: u64, + covered_lines: u64, + covered_lines_percent: f64, + original_json_data: String, +} + +impl CoverageReport { + fn from_json_value(lines: &tinyjson::JsonValue) -> anyhow::Result { + let total_lines = *lines["count"].get::().context("invalid value for `count`")? as u64; + let covered_lines = *lines["covered"].get::().context("invalid value for `covered`")? as u64; + let covered_lines_percent = *lines["percent"].get::().context("invalid value for `covered`")?; + + let original_json_data = lines.stringify().context("original json data")?; + + Ok(Self { + total_lines, + covered_lines, + covered_lines_percent, + original_json_data, + }) + } + + fn generate(sh: &Shell) -> anyhow::Result { + let output = cmd!( + sh, + "{LOCAL_CARGO_ROOT}/bin/cargo-llvm-cov llvm-cov + --ignore-filename-regex {COV_IGNORE_REGEX} + --json" + ) + .read()?; + + let report: tinyjson::JsonValue = output.parse().context("invalid JSON from cargo-llvm-cov")?; + + let lines = &report["data"][0]["totals"]["lines"]; + + Self::from_json_value(lines) + } + + fn past_report(sh: &Shell) -> anyhow::Result { + cmd!(sh, "git fetch origin cov-data").run()?; + + let output = cmd!(sh, "git show origin/cov-data:report.json").read()?; + + let lines: tinyjson::JsonValue = output + .parse() + .context("invalid JSON from origin/cov-data:report.json")?; + + Self::from_json_value(&lines) + } +} + +impl fmt::Display for CoverageReport { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Total lines: {}", self.total_lines)?; + writeln!( + f, + "Covered lines: {} ({:.2}%)", + self.covered_lines, self.covered_lines_percent + )?; + Ok(()) + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 988b1ba9..ee1da6bf 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -50,8 +50,11 @@ fn main() -> anyhow::Result<()> { web::check(&sh)?; } Action::Clean => clean::workspace(&sh)?, + Action::CovGrcov => cov::grcov(&sh)?, Action::CovInstall => cov::install(&sh)?, - Action::CovReport => cov::report(&sh)?, + Action::CovReportGitHub { repo, pr } => cov::report_github(&sh, &repo, pr)?, + Action::CovReport { html_report } => cov::report(&sh, html_report)?, + Action::CovUpdate => cov::update(&sh)?, Action::FuzzCorpusFetch => fuzz::corpus_fetch(&sh)?, Action::FuzzCorpusMin => fuzz::corpus_minify(&sh)?, Action::FuzzCorpusPush => fuzz::corpus_push(&sh)?, diff --git a/xtask/src/tasks.rs b/xtask/src/tasks.rs deleted file mode 100644 index 8b137891..00000000 --- a/xtask/src/tasks.rs +++ /dev/null @@ -1 +0,0 @@ -