diff --git a/.gitignore b/.gitignore index 203e0c82f6..7f6d4bf636 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,11 @@ svnrev.h /obj-* *.obj +# Coverage-instrumented binaries drop these wherever they are run from when +# LLVM_PROFILE_FILE is unset (tools/coverage.sh always sets it). +*.profraw +*.profdata + .DS_Store Thumbs.db diff --git a/CMakePresets.json b/CMakePresets.json index 5fe7def002..84c4d1625f 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -67,6 +67,19 @@ "USE_PERF_JITDUMP": "ON" } }, + { + "name": "clang-coverage", + "displayName": "Clang Devel + source-based coverage", + "description": "Devel build instrumented with -fprofile-instr-generate -fcoverage-mapping, into build-coverage/. Drive it with tools/coverage.sh, which runs the gtest binaries and reports line/region coverage scoped to pcsx2/arm64/.", + "inherits": "clang-devel", + "binaryDir": "${sourceDir}/build-coverage", + "cacheVariables": { + "USE_COVERAGE": "ON", + "ENABLE_TESTS": "ON", + "ENABLE_RECOMPILER_TEST_HOOKS": "ON", + "ENABLE_QT_UI": "OFF" + } + }, { "name": "clang-handheld", "displayName": "Clang Handheld (SDL3 / kmsdrm)", diff --git a/cmake/BuildParameters.cmake b/cmake/BuildParameters.cmake index fa014e29f9..617fa51210 100644 --- a/cmake/BuildParameters.cmake +++ b/cmake/BuildParameters.cmake @@ -55,6 +55,7 @@ endif() # Compiler extra #------------------------------------------------------------------------------- option(USE_ASAN "Enable address sanitizer") +option(USE_COVERAGE "Instrument the build with clang source-based coverage (-fprofile-instr-generate -fcoverage-mapping). Use with tools/coverage.sh; requires clang." OFF) #------------------------------------------------------------------------------- # if no build type is set, use Devel as default @@ -359,6 +360,18 @@ if (USE_ASAN) list(APPEND PCSX2_DEFS ASAN_WORKAROUND) endif() +if (USE_COVERAGE) + if(NOT USE_CLANG) + message(FATAL_ERROR "USE_COVERAGE requires clang (source-based coverage). Configure with the clang-coverage preset.") + endif() + # Instrument everything rather than just pcsx2/arm64: the emitters are reached + # through core and common call paths, and scoping the *report* (tools/coverage.sh + # passes -sources) is exact where scoping the *build* would silently drop + # counters for inline code that lives in headers outside the filter. + add_compile_options(-fprofile-instr-generate -fcoverage-mapping) + add_link_options(-fprofile-instr-generate) +endif() + if(USE_CLANG AND TIMETRACE) add_compile_options(-ftime-trace) endif() diff --git a/tools/coverage.sh b/tools/coverage.sh new file mode 100755 index 0000000000..b42dfce7c9 --- /dev/null +++ b/tools/coverage.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# +# Source-based coverage for the ARM64 recompilers. +# +# Configures/builds the instrumented tree (build-coverage/), runs the gtest +# binaries, and reports line/region coverage scoped to the ARM64 sources -- +# legacy x86 and shared PCSX2 code are filtered out at report time, so the +# percentage answers "how much of *our* JIT do the tests reach". +# +# Usage: +# tools/coverage.sh # build, run, print the per-file summary +# tools/coverage.sh --html # ...and write a browsable HTML report +# tools/coverage.sh --no-build # reuse the existing build-coverage/ tree +# tools/coverage.sh --uncovered # list the fully-unreached functions +# tools/coverage.sh --scope all # widen from pcsx2/arm64/ to every ARM64 file +# +# Any trailing arguments are passed to llvm-cov as extra -sources filters. + +set -euo pipefail + +# pwd -P, not pwd: /home/bmd/dev and /home/bmd/ARMSX2 are symlinks into the real +# tree, and configuring through a symlinked source dir breaks Qt AUTOMOC. Always +# resolve to the physical path before handing anything to cmake. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SRC_DIR="$(dirname "${SCRIPT_DIR}")" +BUILD_DIR="${SRC_DIR}/build-coverage" +PROF_DIR="${BUILD_DIR}/profraw" +PROFDATA="${BUILD_DIR}/coverage.profdata" +HTML_DIR="${BUILD_DIR}/coverage-html" + +LLVM_PROFDATA="${LLVM_PROFDATA:-llvm-profdata}" +LLVM_COV="${LLVM_COV:-llvm-cov}" + +TESTS=(recompiler_tests core_test common_test gs_vertex_tests mvu_progcache_versioning_tests) + +do_build=1 +do_html=0 +do_uncovered=0 +scope=arm64 +extra_sources=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-build) do_build=0 ;; + --html) do_html=1 ;; + --uncovered) do_uncovered=1 ;; + --scope) scope="$2"; shift ;; + -h|--help) sed -n '2,20p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) extra_sources+=("$1") ;; + esac + shift +done + +command -v "${LLVM_PROFDATA}" >/dev/null || { echo "error: ${LLVM_PROFDATA} not found" >&2; exit 1; } +command -v "${LLVM_COV}" >/dev/null || { echo "error: ${LLVM_COV} not found" >&2; exit 1; } + +# ---------------------------------------------------------------- build + +if [[ ${do_build} -eq 1 ]]; then + if [[ ! -f "${BUILD_DIR}/CMakeCache.txt" ]]; then + echo "==> configuring ${BUILD_DIR}" + # cd first: --preset takes the source dir from the working directory and + # ignores -S, so invoking this from the /home/bmd/ARMSX2 symlink would bake + # the symlinked path into CMAKE_HOME_DIRECTORY (and into every coverage + # mapping), which also trips the Qt AUTOMOC symlink bug. + (cd "${SRC_DIR}" && cmake --preset clang-coverage) + fi + echo "==> building ${#TESTS[@]} test targets" + cmake --build "${BUILD_DIR}" --target "${TESTS[@]}" +fi + +# The prefix the compiler actually recorded in the coverage mappings. Deriving it +# from the cache rather than assuming ${SRC_DIR} is what makes --sources reliable: +# llvm-cov matches these by string, and a prefix that matches nothing is not an +# error -- it silently reports the whole tree, which reads as a plausible number. +[[ -f "${BUILD_DIR}/CMakeCache.txt" ]] || { echo "error: no ${BUILD_DIR}/CMakeCache.txt -- run without --no-build" >&2; exit 1; } +SRC_PREFIX="$(sed -n 's/^CMAKE_HOME_DIRECTORY:INTERNAL=//p' "${BUILD_DIR}/CMakeCache.txt")" +[[ -n "${SRC_PREFIX}" ]] || { echo "error: could not read CMAKE_HOME_DIRECTORY from ${BUILD_DIR}/CMakeCache.txt" >&2; exit 1; } + +# The report scope. pcsx2/arm64/ is the JIT proper; --scope all adds the other +# files that only ever compile on ARM64. Everything else in the tree is either +# shared with upstream PCSX2 or x86-only, and is not what we gate on. +sources=("${SRC_PREFIX}/pcsx2/arm64") +if [[ "${scope}" == "all" ]]; then + sources+=( + "${SRC_PREFIX}/pcsx2/GS/GSVector4_arm64.h" + "${SRC_PREFIX}/pcsx2/GS/GSVector4i_arm64.h" + "${SRC_PREFIX}/pcsx2/GS/Renderers/SW/GSDrawScanlineCodeGenerator.arm64.cpp" + "${SRC_PREFIX}/pcsx2/GS/Renderers/SW/GSSetupPrimCodeGenerator.arm64.cpp" + "${SRC_PREFIX}/pcsx2/SPU2/spu2_neon.cpp" + ) +fi +sources+=("${extra_sources[@]+"${extra_sources[@]}"}") + +for s in "${sources[@]}"; do + [[ -e "${s}" ]] || { echo "error: coverage scope path does not exist: ${s}" >&2; exit 1; } +done + +for t in "${TESTS[@]}"; do + [[ -x "${BUILD_DIR}/bin/${t}" ]] || { echo "error: ${BUILD_DIR}/bin/${t} missing -- run without --no-build" >&2; exit 1; } +done + +# ---------------------------------------------------------------- run + +rm -rf "${PROF_DIR}"; mkdir -p "${PROF_DIR}" + +failed=() +for t in "${TESTS[@]}"; do + echo "==> running ${t}" + # %p keeps forked gtest death-test children from clobbering the parent's file. + if ! LLVM_PROFILE_FILE="${PROF_DIR}/${t}-%p.profraw" "${BUILD_DIR}/bin/${t}"; then + failed+=("${t}") + fi +done + +if [[ ${#failed[@]} -gt 0 ]]; then + echo + echo "WARNING: these suites failed: ${failed[*]}" + echo " Coverage below still reflects the code they reached before failing." +fi + +# ---------------------------------------------------------------- report + +echo "==> merging profiles" +# shellcheck disable=SC2046 +"${LLVM_PROFDATA}" merge -sparse -o "${PROFDATA}" $(find "${PROF_DIR}" -name '*.profraw') + +objects=() +for t in "${TESTS[@]}"; do + objects+=(-object "${BUILD_DIR}/bin/${t}") +done + +src_args=() +for s in "${sources[@]}"; do + src_args+=(-sources "${s}") +done + +REPORT="${BUILD_DIR}/coverage-report.txt" +"${LLVM_COV}" report "${objects[@]}" -instr-profile="${PROFDATA}" \ + -show-region-summary=true -show-branch-summary=false \ + "${src_args[@]}" > "${REPORT}" + +# Tripwire for the failure mode above: if --sources matched nothing, llvm-cov +# reports every file it has data for instead of erroring. The scoped report is a +# few dozen files; a few hundred means the filter fell off and the TOTAL is the +# whole tree, not the JIT. +rows="$(grep -cE '^[^ ].*[0-9]+\.[0-9]+%' "${REPORT}" || true)" +if [[ "${rows}" -gt 200 ]]; then + echo "error: report lists ${rows} files -- the --sources filter did not apply." >&2 + echo " Coverage data is keyed to '${SRC_PREFIX}'; check that path is right." >&2 + exit 1 +fi + +echo +cat "${REPORT}" + +if [[ ${do_uncovered} -eq 1 ]]; then + echo + echo "==> functions with zero coverage" + "${LLVM_COV}" report "${objects[@]}" -instr-profile="${PROFDATA}" \ + -show-functions "${src_args[@]}" 2>/dev/null | + awk '$0 !~ /^(Filename|---|TOTAL)/ && NF >= 4 && $4 == "0" { print }' | head -80 +fi + +if [[ ${do_html} -eq 1 ]]; then + rm -rf "${HTML_DIR}" + "${LLVM_COV}" show "${objects[@]}" -instr-profile="${PROFDATA}" \ + -format=html -output-dir="${HTML_DIR}" \ + -show-line-counts-or-regions -show-expansions \ + "${src_args[@]}" + echo + echo "HTML report: ${HTML_DIR}/index.html" +fi