diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..fdbf40ef --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,44 @@ +name: Coverage + +on: + push: + branches: + - master + pull_request: + types: [ opened, synchronize, reopened ] + workflow_dispatch: + +env: + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + +jobs: + coverage: + name: Coverage Report + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + + - name: Rust cache + uses: Swatinem/rust-cache@v2.3.0 + + - name: Prepare runner + run: cargo xtask cov install + + - name: Generate PR report + if: github.event.number != '' + env: + GH_TOKEN: ${{ secrets.DEVOLUTIONSBOT_TOKEN }} + run: cargo xtask cov report-gh --repo "${{ github.repository }}" --pr "${{ github.event.number }}" + + - name: Configure Git Identity + if: github.ref == 'refs/heads/master' + run: | + git config --local user.name "github-actions[bot]" + git config --local user.email "github-actions[bot]@users.noreply.github.com" + + - name: Update coverage data + if: github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ secrets.DEVOLUTIONSBOT_TOKEN }} + run: cargo xtask cov update diff --git a/Cargo.lock b/Cargo.lock index a4a52c27..db6a3915 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3760,6 +3760,12 @@ dependencies = [ "strict-num", ] +[[package]] +name = "tinyjson" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab95735ea2c8fd51154d01e39cf13912a78071c2d89abc49a7ef102a7dd725a" + [[package]] name = "tinyvec" version = "1.6.0" @@ -4669,6 +4675,7 @@ version = "0.0.0" dependencies = [ "anyhow", "pico-args", + "tinyjson", "xshell", ] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 0088e629..462c77a0 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -12,3 +12,4 @@ test = false anyhow = "1" pico-args = "0.5.0" xshell = "0.2.3" +tinyjson = "2.5.1" diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs index 2d54bc98..aeb88312 100644 --- a/xtask/src/cli.rs +++ b/xtask/src/cli.rs @@ -11,11 +11,14 @@ TASKS: check fmt Check formatting check lints Check lints check tests [--no-run] Compile tests and, unless specified otherwise, run them - check wasm Ensure WASM module is compatible for the web ci Run all checks required on CI clean Clean workspace - coverage install Install dependencies required to generate the coverage report - coverage report Generate code-coverage data using tests and fuzz targets + cov grcov Generate a nice HTML report using code-coverage data from tests and fuzz targets + cov install Install cargo-llvm-cov in cargo local root + cov report-gh --repo --pr + Generate a coverage report, posting a comment in GitHub PR + cov report [--html] Generate a coverage report (optionally, a HTML report) + cov update Update coverage data in the cov-data branch fuzz corpus-fetch Fetch fuzzing corpus from Azure storage fuzz corpus-min Minify fuzzing corpus fuzz corpus-push Push fuzzing corpus to Azure storage @@ -42,8 +45,16 @@ pub enum Action { }, Ci, Clean, + CovGrcov, CovInstall, - CovReport, + CovReportGitHub { + repo: String, + pr: u32, + }, + CovReport { + html_report: bool, + }, + CovUpdate, FuzzCorpusFetch, FuzzCorpusMin, FuzzCorpusPush, @@ -78,10 +89,17 @@ pub fn parse_args() -> anyhow::Result { Some("ci") => Action::Ci, Some("clean") => Action::Clean, Some("cov") => match args.subcommand()?.as_deref() { + Some("grcov") => Action::CovGrcov, Some("install") => Action::CovInstall, - Some("report") => Action::CovReport, - Some(unknown) => anyhow::bail!("unknown coverage action: {unknown}"), - None => Action::ShowHelp, + Some("report-gh") => Action::CovReportGitHub { + repo: args.value_from_str("--repo")?, + pr: args.value_from_str("--pr")?, + }, + Some("report") => Action::CovReport { + html_report: args.contains("--html"), + }, + Some("update") => Action::CovUpdate, + None | Some(_) => anyhow::bail!("Unknown cov action"), }, Some("fuzz") => match args.subcommand()?.as_deref() { Some("corpus-fetch") => Action::FuzzCorpusFetch, diff --git a/xtask/src/cov.rs b/xtask/src/cov.rs index 38d696eb..28e22764 100644 --- a/xtask/src/cov.rs +++ b/xtask/src/cov.rs @@ -1,29 +1,185 @@ +use core::fmt; + use crate::prelude::*; +const COV_IGNORE_REGEX: &str = + r#"(crates/ironrdp-(session|.+generators|.+glutin.+|replay|client|fuzzing|tokio|web|futures|tls)|xtask|testsuite)"#; + pub fn install(sh: &Shell) -> anyhow::Result<()> { - let _s = Section::new("COVERAGE-INSTALL"); + let _s = Section::new("COV-INSTALL"); + + let llvm_cov_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "cargo-llvm-cov"].iter().collect(); + + if !sh.path_exists(llvm_cov_path) { + cmd!( + sh, + "{CARGO} install --locked --root {LOCAL_CARGO_ROOT} cargo-llvm-cov@{CARGO_LLVM_COV_VERSION}" + ) + .run()?; + } + + Ok(()) +} + +pub fn update(sh: &Shell) -> anyhow::Result<()> { + let _s = Section::new("COV-UPDATE"); + + let report = CoverageReport::generate(sh)?; + println!("New:\n{report}"); + + let initial_branch = cmd!(sh, "git rev-parse --abbrev-ref HEAD").read()?; + + println!("Switch branch"); + let _ = cmd!(sh, "git branch -D cov-data").run(); + cmd!(sh, "git checkout --orphan cov-data").run()?; + + let result = || -> anyhow::Result<()> { + cmd!(sh, "git rm --cached -r .").run()?; + + sh.write_file("./report.json", report.original_json_data)?; + + cmd!(sh, "git add ./report.json").run()?; + cmd!(sh, "git commit -m 'cov: update report data'").run()?; + cmd!(sh, "git push --force --set-upstream origin cov-data").run()?; + + Ok(()) + }(); + + println!("Clean working tree"); + cmd!(sh, "git clean -df").run()?; + + println!("Switch back to initial branch"); + cmd!(sh, "git checkout {initial_branch}").run()?; + + result?; + + Ok(()) +} + +pub fn report(sh: &Shell, html_report: bool) -> anyhow::Result<()> { + let _s = Section::new("COV-REPORT"); + + if html_report { + cmd!(sh, "{LOCAL_CARGO_ROOT}/bin/cargo-llvm-cov llvm-cov --html") + .arg("--ignore-filename-regex") + .arg(COV_IGNORE_REGEX) + .run()?; + } else { + let report = CoverageReport::generate(sh)?; + let past_report = CoverageReport::past_report(sh)?; + + println!("Past:\n{past_report}"); + println!("New:\n{report}"); + println!( + "Diff: {:+.2}%", + report.covered_lines_percent - past_report.covered_lines_percent + ); + } + + Ok(()) +} + +pub fn report_github(sh: &Shell, repo: &str, pr_id: u32) -> anyhow::Result<()> { + use std::fmt::Write as _; + + const COMMENT_HEADER: &str = "## Coverage Report :robot: :gear:"; + + let _s = Section::new("COV-REPORT"); + + let report = CoverageReport::generate(sh)?; + let past_report = CoverageReport::past_report(sh)?; + + let diff = report.covered_lines_percent - past_report.covered_lines_percent; + + let comments = cmd!(sh, "gh api") + .arg("-H") + .arg("Accept: application/vnd.github.v3+json") + .arg(format!("/repos/{repo}/issues/{pr_id}/comments")) + .read()?; + + let comments: tinyjson::JsonValue = comments.parse().context("GitHub comments")?; + let comments = comments.get::>().context("comments list")?; + + let mut prev_comment_id = None; + + for comment in comments { + let body = comment["body"].get::().context("comment body")?; + + if body.starts_with(COMMENT_HEADER) { + let comment_id = *comment["id"].get::().context("id")? as u64; + prev_comment_id = Some(comment_id); + break; + } + } + + let mut body = String::new(); + + writeln!(body, "{COMMENT_HEADER}")?; + writeln!(body, "**Past**:\n{past_report}")?; + writeln!(body, "**New**:\n{report}")?; + writeln!( + body, + "**Diff**: {:+.2}%", + report.covered_lines_percent - past_report.covered_lines_percent + )?; + writeln!(body, "\n[this comment will be updated automatically]")?; + + let command = cmd!(sh, "gh api") + .arg("-H") + .arg("Accept: application/vnd.github.v3+json") + .arg("-f") + .arg(format!("body={body}")); + + if let Some(comment_id) = prev_comment_id { + println!("Update existing comment"); + + command + .arg("--method") + .arg("PATCH") + .arg(format!("/repos/{repo}/issues/comments/{comment_id}")) + .ignore_stdout() + .run()?; + } else if diff.abs() < 0.001 { + println!("Create new comment"); + + command + .arg("--method") + .arg("POST") + .arg(format!("/repos/{repo}/issues/{pr_id}/comments")) + .ignore_stdout() + .run()?; + } else { + println!("Coverage didn’t change, skip GitHub comment"); + } + + Ok(()) +} + +pub fn grcov(sh: &Shell) -> anyhow::Result<()> { + let _s = Section::new("COV-GRCOV"); cmd!(sh, "rustup install nightly --profile=minimal").run()?; cmd!(sh, "rustup component add --toolchain nightly llvm-tools-preview").run()?; cmd!(sh, "rustup component add llvm-tools-preview").run()?; - cmd!( - sh, - "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} cargo-fuzz@{CARGO_FUZZ_VERSION}" - ) - .run()?; + let cargo_fuzz_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "cargo-fuzz"].iter().collect(); + let grcov_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "grcov"].iter().collect(); - cmd!( - sh, - "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} grcov@{GRCOV_VERSION}" - ) - .run()?; + if !sh.path_exists(cargo_fuzz_path) { + cmd!( + sh, + "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} cargo-fuzz@{CARGO_FUZZ_VERSION}" + ) + .run()?; + } - Ok(()) -} - -pub fn report(sh: &Shell) -> anyhow::Result<()> { - let _s = Section::new("COVERAGE-REPORT"); + if !sh.path_exists(grcov_path) { + cmd!( + sh, + "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} grcov@{GRCOV_VERSION}" + ) + .run()?; + } println!("Remove leftovers"); sh.remove_path("./fuzz/coverage/")?; @@ -111,3 +267,67 @@ pub fn report(sh: &Shell) -> anyhow::Result<()> { Ok(()) } + +struct CoverageReport { + total_lines: u64, + covered_lines: u64, + covered_lines_percent: f64, + original_json_data: String, +} + +impl CoverageReport { + fn from_json_value(lines: &tinyjson::JsonValue) -> anyhow::Result { + let total_lines = *lines["count"].get::().context("invalid value for `count`")? as u64; + let covered_lines = *lines["covered"].get::().context("invalid value for `covered`")? as u64; + let covered_lines_percent = *lines["percent"].get::().context("invalid value for `covered`")?; + + let original_json_data = lines.stringify().context("original json data")?; + + Ok(Self { + total_lines, + covered_lines, + covered_lines_percent, + original_json_data, + }) + } + + fn generate(sh: &Shell) -> anyhow::Result { + let output = cmd!( + sh, + "{LOCAL_CARGO_ROOT}/bin/cargo-llvm-cov llvm-cov + --ignore-filename-regex {COV_IGNORE_REGEX} + --json" + ) + .read()?; + + let report: tinyjson::JsonValue = output.parse().context("invalid JSON from cargo-llvm-cov")?; + + let lines = &report["data"][0]["totals"]["lines"]; + + Self::from_json_value(lines) + } + + fn past_report(sh: &Shell) -> anyhow::Result { + cmd!(sh, "git fetch origin cov-data").run()?; + + let output = cmd!(sh, "git show origin/cov-data:report.json").read()?; + + let lines: tinyjson::JsonValue = output + .parse() + .context("invalid JSON from origin/cov-data:report.json")?; + + Self::from_json_value(&lines) + } +} + +impl fmt::Display for CoverageReport { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Total lines: {}", self.total_lines)?; + writeln!( + f, + "Covered lines: {} ({:.2}%)", + self.covered_lines, self.covered_lines_percent + )?; + Ok(()) + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 988b1ba9..ee1da6bf 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -50,8 +50,11 @@ fn main() -> anyhow::Result<()> { web::check(&sh)?; } Action::Clean => clean::workspace(&sh)?, + Action::CovGrcov => cov::grcov(&sh)?, Action::CovInstall => cov::install(&sh)?, - Action::CovReport => cov::report(&sh)?, + Action::CovReportGitHub { repo, pr } => cov::report_github(&sh, &repo, pr)?, + Action::CovReport { html_report } => cov::report(&sh, html_report)?, + Action::CovUpdate => cov::update(&sh)?, Action::FuzzCorpusFetch => fuzz::corpus_fetch(&sh)?, Action::FuzzCorpusMin => fuzz::corpus_minify(&sh)?, Action::FuzzCorpusPush => fuzz::corpus_push(&sh)?, diff --git a/xtask/src/tasks.rs b/xtask/src/tasks.rs deleted file mode 100644 index 8b137891..00000000 --- a/xtask/src/tasks.rs +++ /dev/null @@ -1 +0,0 @@ -