From a6ca6ea1a8d1e68bebd0e8eafa826bdeadad60f5 Mon Sep 17 00:00:00 2001 From: Bevan Kay Date: Tue, 28 Nov 2023 19:53:37 +1100 Subject: [PATCH] ci: setup workflow for manual cask auditing --- .github/workflows/manual-audit.yml | 283 +++++++++++++++++++++++++++++ cmd/lib/ci_matrix.rb | 18 +- cmd/lib/generate-matrix.rb | 119 ++++++++---- 3 files changed, 374 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/manual-audit.yml diff --git a/.github/workflows/manual-audit.yml b/.github/workflows/manual-audit.yml new file mode 100644 index 0000000000..723ee28702 --- /dev/null +++ b/.github/workflows/manual-audit.yml @@ -0,0 +1,283 @@ +name: Manual Audit + +on: + workflow_dispatch: + inputs: + casks: + description: List of casks to audit (comma-separated) + required: true + skip-install: + description: Skip installation of casks + required: false + default: true + type: boolean + new-cask: + description: Apply new cask audit + required: false + default: false + type: boolean + +env: + HOMEBREW_DEVELOPER: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_FROM_API: 1 + HOMEBREW_GITHUB_API_TOKEN: ${{ github.token }} + +concurrency: + group: "${{ github.ref }}" + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + generate-matrix: + outputs: + matrix: ${{ steps.generate-matrix.outputs.matrix }} + runs-on: macos-latest + steps: + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + with: + core: false + cask: true + test-bot: false + + - name: Check out Pull Request + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Generate CI matrix + id: generate-matrix + run: | + brew ruby -- "$(brew --repository homebrew/cask)/cmd/lib/generate-matrix.rb" ${{ github.event.inputs.skip_install && '--skip-install' }} ${{ github.event.inputs.new_cask && '--new-cask' }} --casks=${{ github.event.inputs.casks }} + + test: + name: ${{ matrix.name }} + needs: generate-matrix + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + steps: + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + with: + core: false + cask: true + test-bot: false + + - name: Enable debug mode + run: | + echo 'HOMEBREW_DEBUG=1' >> "${GITHUB_ENV}" + echo 'HOMEBREW_VERBOSE=1' >> "${GITHUB_ENV}" + if: runner.debug + + - name: Check out Pull Request + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Clean up CI machine + run: | + if ! brew uninstall --cask julia && ! rm -r /Applications/Julia-*.app; then + echo '::warning::Removing Julia is no longer necessary.' + fi + + if ! rm /usr/local/bin/dotnet; then + echo '::warning::Removing `dotnet` symlink is no longer necessary.' + fi + + if ! rm /usr/local/bin/pod; then + echo '::warning::Removing `cocoapods` symlink is no longer necessary.' + fi + + if ! rm /usr/local/bin/chromedriver; then + echo '::warning::Removing `chromedriver` symlink is no longer necessary.' + fi + + brew unlink python && brew link --overwrite python + if: runner.os == 'macOS' + + - name: Cache Homebrew Gems + id: cache + uses: actions/cache@v3 + with: + path: ${{ steps.set-up-homebrew.outputs.gems-path }} + key: ${{ matrix.runner }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }} + restore-keys: ${{ matrix.runner }}-rubygems- + + - name: Install Homebrew Gems + id: gems + run: brew install-bundler-gems + if: steps.cache.outputs.cache-hit != 'true' + + - name: Run brew readall ${{ matrix.tap }} + id: readall + run: brew readall --os=all --arch=all '${{ matrix.tap }}' + if: > + always() && + contains(fromJSON('["success", "skipped"]'), steps.gems.outcome) && + !matrix.skip_readall + + - name: Run brew style ${{ matrix.tap }} + run: brew style '${{ matrix.tap }}' + if: > + always() && + contains(fromJSON('["success", "skipped"]'), steps.readall.outcome) && + !matrix.cask + + - name: Run brew fetch --cask ${{ matrix.cask.token }} + id: fetch + run: | + brew fetch --cask --retry --force ${{ join(matrix.fetch_args, ' ') }} '${{ matrix.cask.path }}' + timeout-minutes: 30 + if: > + always() && + contains(fromJSON('["success", "skipped"]'), steps.readall.outcome) && + matrix.cask + + - name: Run brew audit --cask${{ (matrix.cask && ' ') || ' --tap ' }}${{ matrix.cask.token || matrix.tap }} + id: audit + run: | + brew audit --cask ${{ join(matrix.audit_args, ' ') }}${{ (matrix.cask && ' ') || ' --tap ' }}'${{ matrix.cask.token || matrix.tap }}' + timeout-minutes: 30 + if: > + always() && + contains(fromJSON('["success", "skipped"]'), steps.readall.outcome) && + (!matrix.cask || steps.fetch.outcome == 'success') && + !matrix.skip_audit + + - name: Gather cask information + id: info + run: | + brew tap homebrew/cask-versions + brew ruby <<'EOF' + require 'cask/cask_loader' + require 'cask/installer' + + cask = Cask::CaskLoader.load('${{ matrix.cask.path }}') + + was_installed = cask.installed? + manual_installer = cask.artifacts.any? { |artifact| + artifact.is_a?(Cask::Artifact::Installer::ManualInstaller) + } + + macos_requirement_satisfied = if macos_requirement = cask.depends_on.macos + macos_requirement.satisfied? + else + true + end + + cask_conflicts = cask.conflicts_with&.dig(:cask).to_a.select { |c| Cask::CaskLoader.load(c).installed? } + formula_conflicts = cask.conflicts_with&.dig(:formula).to_a.select { |f| Formula[f].any_version_installed? } + + installer = Cask::Installer.new(cask) + cask_and_formula_dependencies = installer.missing_cask_and_formula_dependencies + + cask_dependencies = cask_and_formula_dependencies.select { |d| d.is_a?(Cask::Cask) }.map(&:full_name) + formula_dependencies = cask_and_formula_dependencies.select { |d| d.is_a?(Formula) }.map(&:full_name) + + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |f| + f.puts "was_installed=#{JSON.generate(was_installed)}" + f.puts "manual_installer=#{JSON.generate(manual_installer)}" + f.puts "macos_requirement_satisfied=#{JSON.generate(macos_requirement_satisfied)}" + f.puts "cask_conflicts=#{JSON.generate(cask_conflicts)}" + f.puts "cask_dependencies=#{JSON.generate(cask_dependencies)}" + f.puts "formula_conflicts=#{JSON.generate(formula_conflicts)}" + f.puts "formula_dependencies=#{JSON.generate(formula_dependencies)}" + end + EOF + if: always() && steps.fetch.outcome == 'success' && matrix.cask + + - name: Uninstall conflicting formulae + run: | + brew uninstall --formula ${{ join(fromJSON(steps.info.outputs.formula_conflicts), ' ') }} + if: always() && steps.info.outcome == 'success' && join(fromJSON(steps.info.outputs.formula_conflicts)) != '' + timeout-minutes: 30 + + - name: Uninstall conflicting casks + run: | + brew uninstall --cask ${{ join(fromJSON(steps.info.outputs.cask_conflicts), ' ') }} + if: always() && steps.info.outcome == 'success' && join(fromJSON(steps.info.outputs.cask_conflicts)) != '' + timeout-minutes: 30 + + - name: Run brew uninstall --cask --zap ${{ matrix.cask.token }} + run: | + brew uninstall --cask --zap '${{ matrix.cask.path }}' + if: always() && steps.info.outcome == 'success' && fromJSON(steps.info.outputs.was_installed) + timeout-minutes: 30 + + - name: Take snapshot of installed and running apps and services + id: snapshot + run: | + brew ruby -r "$(brew --repository homebrew/cask)/cmd/lib/check.rb" <<'EOF' + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |f| + f.puts "before=#{JSON.generate(Check.all)}" + end + EOF + if: always() && steps.info.outcome == 'success' + + - name: Run brew install --cask ${{ matrix.cask.token }} + id: install + run: brew install --cask '${{ matrix.cask.path }}' + if: > + always() && steps.info.outcome == 'success' && + fromJSON(steps.info.outputs.macos_requirement_satisfied) && + !matrix.skip_install + timeout-minutes: 30 + + - name: Run brew uninstall --cask ${{ matrix.cask.token }} + run: brew uninstall --cask '${{ matrix.cask.path }}' + if: always() && steps.install.outcome == 'success' && !fromJSON(steps.info.outputs.manual_installer) + timeout-minutes: 30 + + - name: Uninstall formula dependencies + run: | + brew uninstall --formula ${{ join(fromJSON(steps.info.outputs.formula_dependencies), ' ') }} + if: always() && steps.install.outcome == 'success' && join(fromJSON(steps.info.outputs.formula_dependencies)) != '' + timeout-minutes: 30 + + - name: Uninstall cask dependencies + run: | + brew uninstall --cask ${{ join(fromJSON(steps.info.outputs.cask_dependencies), ' ') }} + if: always() && steps.install.outcome == 'success' && join(fromJSON(steps.info.outputs.cask_dependencies)) != '' + timeout-minutes: 30 + + - name: Compare installed and running apps and services with snapshot + run: | + brew ruby -r "$(brew --repository homebrew/cask)/cmd/lib/check.rb" <<'EOF' + require "cask/cask_loader" + require "utils/github/actions" + + before = JSON.parse(<<~'EOS').transform_keys(&:to_sym) + ${{ steps.snapshot.outputs.before }} + EOS + after = Check.all + + cask = Cask::CaskLoader.load('${{ matrix.cask.path }}') + errors = Check.errors(before, after, cask: cask) + + errors.each do |error| + onoe error + puts GitHub::Actions::Annotation.new(:error, error, file: '${{ matrix.cask.path }}') + end + + exit 1 if errors.any? + EOF + if: always() && steps.snapshot.outcome == 'success' + + conclusion: + name: conclusion + needs: test + runs-on: ubuntu-latest + if: always() + steps: + - name: Result + run: ${{ needs.test.result == 'success' }} diff --git a/cmd/lib/ci_matrix.rb b/cmd/lib/ci_matrix.rb index 60e51ca548..6acd2d397f 100644 --- a/cmd/lib/ci_matrix.rb +++ b/cmd/lib/ci_matrix.rb @@ -101,7 +101,7 @@ module CiMatrix end end - def self.generate(tap, labels: []) + def self.generate(tap, labels: [], cask_names: [], skip_install: false, new_cask: false) odie "This command must be run from inside a tap directory." unless tap tap.extend(ChangedFiles) @@ -123,16 +123,22 @@ module CiMatrix odie "Found Ruby files in wrong directory:\n#{ruby_files_in_wrong_directory.join("\n")}" end - modified_cask_files = changed_files[:modified_cask_files] + cask_files_to_check = if cask_names.any? + cask_names.map do |cask_name| + Cask::CaskLoader.load(cask_name).sourcefile_path + end + else + changed_files[:modified_cask_files] + end - jobs = modified_cask_files.count + jobs = cask_files_to_check.count odie "Maximum job matrix size exceeded: #{jobs}/#{MAX_JOBS}" if jobs > MAX_JOBS - modified_cask_files.flat_map do |path| + cask_files_to_check.flat_map do |path| cask_token = path.basename(".rb") audit_args = ["--online"] - audit_args << "--new-cask" if changed_files[:added_files].include?(path) + audit_args << "--new-cask" if changed_files[:added_files].include?(path) || new_cask audit_args << "--signing" @@ -167,7 +173,7 @@ module CiMatrix }, audit_args: audit_args + arch_args, fetch_args: arch_args, - skip_install: labels.include?("ci-skip-install") || !native_runner_arch, + skip_install: labels.include?("ci-skip-install") || !native_runner_arch || skip_install, skip_readall: !native_runner_arch, runner: runner.fetch(:name), } diff --git a/cmd/lib/generate-matrix.rb b/cmd/lib/generate-matrix.rb index 326f42dd18..3df5f0c337 100755 --- a/cmd/lib/generate-matrix.rb +++ b/cmd/lib/generate-matrix.rb @@ -2,52 +2,91 @@ require "tap" require "utils/github/api" - +require "cli/parser" require_relative "ci_matrix" -pr_url, = ARGV +module Homebrew + module_function -labels = if pr_url - pr = GitHub::API.open_rest(pr_url) - pr.fetch("labels").map { |l| l.fetch("name") } -else - [] -end + def generate_matrix_args + Homebrew::CLI::Parser.new do + description <<~EOS + Generate a GitHub Actions matrix for a given pull request URL or list of cask names. + EOS -tap = Tap.fetch(ENV.fetch("GITHUB_REPOSITORY")) + flag "--url=", + description: "URL of a pull request to generate a matrix for." + comma_array "--casks", + description: "Comma-separated list of casks to test." + switch "--skip-install", + description: "Skip installing casks" + switch "--new-cask", + description: "Run new cask checks" -runner = CiMatrix.random_runner[:name] -syntax_job = { - name: "syntax", - tap: tap.name, - runner: runner, - skip_readall: false, -} - -matrix = [syntax_job] - -unless labels.include?("ci-syntax-only") - cask_jobs = CiMatrix.generate(tap, labels: labels) - - if cask_jobs.any? - # If casks were changed, skip `audit` for whole tap. - syntax_job[:skip_audit] = true - - # If casks were cahnged, skip `readall` in the syntax job. - syntax_job[:skip_readall] = true - - # The syntax job only runs `style` at this point, which should work on Linux. - # Running on macOS is currently faster though, since `homebrew/cask` and - # `homebrew/core` are already tapped on macOS CI machines. - # syntax_job[:runner] = "ubuntu-latest" + conflicts "--url", "--casks" + end end - matrix += cask_jobs + def generate_matrix + args = generate_matrix_args.parse + + skip_install = args.skip_install? + new_cask = args.new_cask? + casks = args.casks if args.casks&.any? + + if args.url.present? + pr_url = args.url + + labels = if pr_url + pr = GitHub::API.open_rest(pr_url) + pr.fetch("labels").map { |l| l.fetch("name") } + else + [] + end + end + + tap = Tap.fetch(ENV.fetch("GITHUB_REPOSITORY")) + + runner = CiMatrix.random_runner[:name] + syntax_job = { + name: "syntax", + tap: tap.name, + runner: runner, + skip_readall: false, + } + + matrix = [syntax_job] + + unless labels&.include?("ci-syntax-only") + cask_jobs = if args.casks&.any? + CiMatrix.generate(tap, labels: labels, cask_names: casks, skip_install: skip_install, new_cask: new_cask) + else + CiMatrix.generate(tap, labels: labels, skip_install: skip_install, new_cask: new_cask) + end + + if cask_jobs.any? + # If casks were changed, skip `audit` for whole tap. + syntax_job[:skip_audit] = true + + # If casks were cahnged, skip `readall` in the syntax job. + syntax_job[:skip_readall] = true + + # The syntax job only runs `style` at this point, which should work on Linux. + # Running on macOS is currently faster though, since `homebrew/cask` and + # `homebrew/core` are already tapped on macOS CI machines. + # syntax_job[:runner] = "ubuntu-latest" + end + + matrix += cask_jobs + end + + syntax_job[:name] += " (#{syntax_job[:runner]})" + + puts JSON.pretty_generate(matrix) + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |f| + f.puts "matrix=#{JSON.generate(matrix)}" + end + end end -syntax_job[:name] += " (#{syntax_job[:runner]})" - -puts JSON.pretty_generate(matrix) -File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |f| - f.puts "matrix=#{JSON.generate(matrix)}" -end +Homebrew.generate_matrix