ci: coverage reporting in PRs

Adds a new workflow reporting code coverage when opening a pull request.

The idea is to help find blind spots in our test suites and inform where testing
efforts should go, but there is no hard requirement related to the numbers.
We are allowed to merge pull requests regardless of the coverage percentage,
and even if the coverage workflow fails.

Past coverage data is stored in a separate branch (cov-data).
It gets updated when the workflow run against master.

.github/workflows/coverage.yml

@@ -0,0 +1,44 @@
+name: Coverage
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+  workflow_dispatch:
+
+env:
+  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+
+jobs:
+  coverage:
+    name: Coverage Report
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2.3.0
+
+      - name: Prepare runner
+        run: cargo xtask cov install
+
+      - name: Generate PR report
+        if: github.event.number != ''
+        env:
+          GH_TOKEN: ${{ secrets.DEVOLUTIONSBOT_TOKEN }}
+        run: cargo xtask cov report-gh --repo "${{ github.repository }}" --pr "${{ github.event.number }}"
+
+      - name: Configure Git Identity
+        if: github.ref == 'refs/heads/master'
+        run: |
+          git config --local user.name "github-actions[bot]"
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Update coverage data
+        if: github.ref == 'refs/heads/master'
+        env:
+          GH_TOKEN: ${{ secrets.DEVOLUTIONSBOT_TOKEN }}
+        run: cargo xtask cov update

Cargo.lock

@@ -3760,6 +3760,12 @@ dependencies = [
  "strict-num",
 ]
 
+[[package]]
+name = "tinyjson"
+version = "2.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab95735ea2c8fd51154d01e39cf13912a78071c2d89abc49a7ef102a7dd725a"
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
@@ -4669,6 +4675,7 @@ version = "0.0.0"
 dependencies = [
  "anyhow",
  "pico-args",
+ "tinyjson",
  "xshell",
 ]
 

xtask/Cargo.toml

@@ -12,3 +12,4 @@ test = false
 anyhow = "1"
 pico-args = "0.5.0"
 xshell = "0.2.3"
+tinyjson = "2.5.1"

xtask/src/cli.rs

@@ -11,11 +11,14 @@ TASKS:
   check fmt               Check formatting
   check lints             Check lints
   check tests [--no-run]  Compile tests and, unless specified otherwise, run them
-  check wasm              Ensure WASM module is compatible for the web
   ci                      Run all checks required on CI
   clean                   Clean workspace
-  coverage install        Install dependencies required to generate the coverage report
-  coverage report         Generate code-coverage data using tests and fuzz targets
+  cov grcov               Generate a nice HTML report using code-coverage data from tests and fuzz targets
+  cov install             Install cargo-llvm-cov in cargo local root
+  cov report-gh --repo <REPO_NAME> --pr <PR_ID>
+                          Generate a coverage report, posting a comment in GitHub PR
+  cov report [--html]     Generate a coverage report (optionally, a HTML report)
+  cov update              Update coverage data in the cov-data branch
   fuzz corpus-fetch       Fetch fuzzing corpus from Azure storage
   fuzz corpus-min         Minify fuzzing corpus
   fuzz corpus-push        Push fuzzing corpus to Azure storage
@@ -42,8 +45,16 @@ pub enum Action {
     },
     Ci,
     Clean,
+    CovGrcov,
     CovInstall,
-    CovReport,
+    CovReportGitHub {
+        repo: String,
+        pr: u32,
+    },
+    CovReport {
+        html_report: bool,
+    },
+    CovUpdate,
     FuzzCorpusFetch,
     FuzzCorpusMin,
     FuzzCorpusPush,
@@ -78,10 +89,17 @@ pub fn parse_args() -> anyhow::Result<Action> {
             Some("ci") => Action::Ci,
             Some("clean") => Action::Clean,
             Some("cov") => match args.subcommand()?.as_deref() {
+                Some("grcov") => Action::CovGrcov,
                 Some("install") => Action::CovInstall,
-                Some("report") => Action::CovReport,
-                Some(unknown) => anyhow::bail!("unknown coverage action: {unknown}"),
-                None => Action::ShowHelp,
+                Some("report-gh") => Action::CovReportGitHub {
+                    repo: args.value_from_str("--repo")?,
+                    pr: args.value_from_str("--pr")?,
+                },
+                Some("report") => Action::CovReport {
+                    html_report: args.contains("--html"),
+                },
+                Some("update") => Action::CovUpdate,
+                None | Some(_) => anyhow::bail!("Unknown cov action"),
             },
             Some("fuzz") => match args.subcommand()?.as_deref() {
                 Some("corpus-fetch") => Action::FuzzCorpusFetch,

xtask/src/cov.rs

@@ -1,29 +1,185 @@
+use core::fmt;
+
 use crate::prelude::*;
 
+const COV_IGNORE_REGEX: &str =
+    r#"(crates/ironrdp-(session|.+generators|.+glutin.+|replay|client|fuzzing|tokio|web|futures|tls)|xtask|testsuite)"#;
+
 pub fn install(sh: &Shell) -> anyhow::Result<()> {
-    let _s = Section::new("COVERAGE-INSTALL");
+    let _s = Section::new("COV-INSTALL");
+
+    let llvm_cov_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "cargo-llvm-cov"].iter().collect();
+
+    if !sh.path_exists(llvm_cov_path) {
+        cmd!(
+            sh,
+            "{CARGO} install --locked --root {LOCAL_CARGO_ROOT} cargo-llvm-cov@{CARGO_LLVM_COV_VERSION}"
+        )
+        .run()?;
+    }
+
+    Ok(())
+}
+
+pub fn update(sh: &Shell) -> anyhow::Result<()> {
+    let _s = Section::new("COV-UPDATE");
+
+    let report = CoverageReport::generate(sh)?;
+    println!("New:\n{report}");
+
+    let initial_branch = cmd!(sh, "git rev-parse --abbrev-ref HEAD").read()?;
+
+    println!("Switch branch");
+    let _ = cmd!(sh, "git branch -D cov-data").run();
+    cmd!(sh, "git checkout --orphan cov-data").run()?;
+
+    let result = || -> anyhow::Result<()> {
+        cmd!(sh, "git rm --cached -r .").run()?;
+
+        sh.write_file("./report.json", report.original_json_data)?;
+
+        cmd!(sh, "git add ./report.json").run()?;
+        cmd!(sh, "git commit -m 'cov: update report data'").run()?;
+        cmd!(sh, "git push --force --set-upstream origin cov-data").run()?;
+
+        Ok(())
+    }();
+
+    println!("Clean working tree");
+    cmd!(sh, "git clean -df").run()?;
+
+    println!("Switch back to initial branch");
+    cmd!(sh, "git checkout {initial_branch}").run()?;
+
+    result?;
+
+    Ok(())
+}
+
+pub fn report(sh: &Shell, html_report: bool) -> anyhow::Result<()> {
+    let _s = Section::new("COV-REPORT");
+
+    if html_report {
+        cmd!(sh, "{LOCAL_CARGO_ROOT}/bin/cargo-llvm-cov llvm-cov --html")
+            .arg("--ignore-filename-regex")
+            .arg(COV_IGNORE_REGEX)
+            .run()?;
+    } else {
+        let report = CoverageReport::generate(sh)?;
+        let past_report = CoverageReport::past_report(sh)?;
+
+        println!("Past:\n{past_report}");
+        println!("New:\n{report}");
+        println!(
+            "Diff: {:+.2}%",
+            report.covered_lines_percent - past_report.covered_lines_percent
+        );
+    }
+
+    Ok(())
+}
+
+pub fn report_github(sh: &Shell, repo: &str, pr_id: u32) -> anyhow::Result<()> {
+    use std::fmt::Write as _;
+
+    const COMMENT_HEADER: &str = "## Coverage Report :robot: :gear:";
+
+    let _s = Section::new("COV-REPORT");
+
+    let report = CoverageReport::generate(sh)?;
+    let past_report = CoverageReport::past_report(sh)?;
+
+    let diff = report.covered_lines_percent - past_report.covered_lines_percent;
+
+    let comments = cmd!(sh, "gh api")
+        .arg("-H")
+        .arg("Accept: application/vnd.github.v3+json")
+        .arg(format!("/repos/{repo}/issues/{pr_id}/comments"))
+        .read()?;
+
+    let comments: tinyjson::JsonValue = comments.parse().context("GitHub comments")?;
+    let comments = comments.get::<Vec<_>>().context("comments list")?;
+
+    let mut prev_comment_id = None;
+
+    for comment in comments {
+        let body = comment["body"].get::<String>().context("comment body")?;
+
+        if body.starts_with(COMMENT_HEADER) {
+            let comment_id = *comment["id"].get::<f64>().context("id")? as u64;
+            prev_comment_id = Some(comment_id);
+            break;
+        }
+    }
+
+    let mut body = String::new();
+
+    writeln!(body, "{COMMENT_HEADER}")?;
+    writeln!(body, "**Past**:\n{past_report}")?;
+    writeln!(body, "**New**:\n{report}")?;
+    writeln!(
+        body,
+        "**Diff**: {:+.2}%",
+        report.covered_lines_percent - past_report.covered_lines_percent
+    )?;
+    writeln!(body, "\n[this comment will be updated automatically]")?;
+
+    let command = cmd!(sh, "gh api")
+        .arg("-H")
+        .arg("Accept: application/vnd.github.v3+json")
+        .arg("-f")
+        .arg(format!("body={body}"));
+
+    if let Some(comment_id) = prev_comment_id {
+        println!("Update existing comment");
+
+        command
+            .arg("--method")
+            .arg("PATCH")
+            .arg(format!("/repos/{repo}/issues/comments/{comment_id}"))
+            .ignore_stdout()
+            .run()?;
+    } else if diff.abs() < 0.001 {
+        println!("Create new comment");
+
+        command
+            .arg("--method")
+            .arg("POST")
+            .arg(format!("/repos/{repo}/issues/{pr_id}/comments"))
+            .ignore_stdout()
+            .run()?;
+    } else {
+        println!("Coverage didn’t change, skip GitHub comment");
+    }
+
+    Ok(())
+}
+
+pub fn grcov(sh: &Shell) -> anyhow::Result<()> {
+    let _s = Section::new("COV-GRCOV");
 
     cmd!(sh, "rustup install nightly --profile=minimal").run()?;
     cmd!(sh, "rustup component add --toolchain nightly llvm-tools-preview").run()?;
     cmd!(sh, "rustup component add llvm-tools-preview").run()?;
 
-    cmd!(
-        sh,
-        "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} cargo-fuzz@{CARGO_FUZZ_VERSION}"
-    )
-    .run()?;
+    let cargo_fuzz_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "cargo-fuzz"].iter().collect();
+    let grcov_path: std::path::PathBuf = [LOCAL_CARGO_ROOT, "bin", "grcov"].iter().collect();
 
-    cmd!(
-        sh,
-        "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} grcov@{GRCOV_VERSION}"
-    )
-    .run()?;
+    if !sh.path_exists(cargo_fuzz_path) {
+        cmd!(
+            sh,
+            "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} cargo-fuzz@{CARGO_FUZZ_VERSION}"
+        )
+        .run()?;
+    }
 
-    Ok(())
-}
-
-pub fn report(sh: &Shell) -> anyhow::Result<()> {
-    let _s = Section::new("COVERAGE-REPORT");
+    if !sh.path_exists(grcov_path) {
+        cmd!(
+            sh,
+            "{CARGO} install --debug --locked --root {LOCAL_CARGO_ROOT} grcov@{GRCOV_VERSION}"
+        )
+        .run()?;
+    }
 
     println!("Remove leftovers");
     sh.remove_path("./fuzz/coverage/")?;
@@ -111,3 +267,67 @@ pub fn report(sh: &Shell) -> anyhow::Result<()> {
 
     Ok(())
 }
+
+struct CoverageReport {
+    total_lines: u64,
+    covered_lines: u64,
+    covered_lines_percent: f64,
+    original_json_data: String,
+}
+
+impl CoverageReport {
+    fn from_json_value(lines: &tinyjson::JsonValue) -> anyhow::Result<Self> {
+        let total_lines = *lines["count"].get::<f64>().context("invalid value for `count`")? as u64;
+        let covered_lines = *lines["covered"].get::<f64>().context("invalid value for `covered`")? as u64;
+        let covered_lines_percent = *lines["percent"].get::<f64>().context("invalid value for `covered`")?;
+
+        let original_json_data = lines.stringify().context("original json data")?;
+
+        Ok(Self {
+            total_lines,
+            covered_lines,
+            covered_lines_percent,
+            original_json_data,
+        })
+    }
+
+    fn generate(sh: &Shell) -> anyhow::Result<Self> {
+        let output = cmd!(
+            sh,
+            "{LOCAL_CARGO_ROOT}/bin/cargo-llvm-cov llvm-cov
+            --ignore-filename-regex {COV_IGNORE_REGEX}
+            --json"
+        )
+        .read()?;
+
+        let report: tinyjson::JsonValue = output.parse().context("invalid JSON from cargo-llvm-cov")?;
+
+        let lines = &report["data"][0]["totals"]["lines"];
+
+        Self::from_json_value(lines)
+    }
+
+    fn past_report(sh: &Shell) -> anyhow::Result<Self> {
+        cmd!(sh, "git fetch origin cov-data").run()?;
+
+        let output = cmd!(sh, "git show origin/cov-data:report.json").read()?;
+
+        let lines: tinyjson::JsonValue = output
+            .parse()
+            .context("invalid JSON from origin/cov-data:report.json")?;
+
+        Self::from_json_value(&lines)
+    }
+}
+
+impl fmt::Display for CoverageReport {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "Total lines: {}", self.total_lines)?;
+        writeln!(
+            f,
+            "Covered lines: {} ({:.2}%)",
+            self.covered_lines, self.covered_lines_percent
+        )?;
+        Ok(())
+    }
+}

xtask/src/main.rs

@@ -50,8 +50,11 @@ fn main() -> anyhow::Result<()> {
             web::check(&sh)?;
         }
         Action::Clean => clean::workspace(&sh)?,
+        Action::CovGrcov => cov::grcov(&sh)?,
         Action::CovInstall => cov::install(&sh)?,
-        Action::CovReport => cov::report(&sh)?,
+        Action::CovReportGitHub { repo, pr } => cov::report_github(&sh, &repo, pr)?,
+        Action::CovReport { html_report } => cov::report(&sh, html_report)?,
+        Action::CovUpdate => cov::update(&sh)?,
         Action::FuzzCorpusFetch => fuzz::corpus_fetch(&sh)?,
         Action::FuzzCorpusMin => fuzz::corpus_minify(&sh)?,
         Action::FuzzCorpusPush => fuzz::corpus_push(&sh)?,

xtask/src/tasks.rs

@@ -1 +0,0 @@
-