From 3cc78d44712ce7e25657178b5439a0f1a2104e5b Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Wed, 1 Aug 2018 18:39:26 +0200 Subject: [PATCH] `brew cask ci` --- .travis.yml | 37 +++++- ci/travis/README.md | 5 - ci/travis/before_script.sh | 36 ------ ci/travis/helpers.sh | 49 -------- ci/travis/script.sh | 68 ---------- cmd/brewcask-ci.rb | 252 +++++++++++++++++++++++++++++++++++++ cmd/lib/capture.rb | 38 ++++++ cmd/lib/diffable.rb | 51 ++++++++ cmd/lib/github.rb | 25 ++++ cmd/lib/travis.rb | 44 +++++++ 10 files changed, 441 insertions(+), 164 deletions(-) delete mode 100644 ci/travis/README.md delete mode 100755 ci/travis/before_script.sh delete mode 100644 ci/travis/helpers.sh delete mode 100755 ci/travis/script.sh create mode 100755 cmd/brewcask-ci.rb create mode 100644 cmd/lib/capture.rb create mode 100644 cmd/lib/diffable.rb create mode 100644 cmd/lib/github.rb create mode 100644 cmd/lib/travis.rb diff --git a/.travis.yml b/.travis.yml index d9f342b996..c29ba64c71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,14 +16,39 @@ branches: cache: directories: - /usr/local/Homebrew/Library/Homebrew/vendor/bundle + - /usr/local/Homebrew/Library/Homebrew/vendor/portable-ruby -install: true # skip install step +install: + - | + # Force strict error checking. + set -o errexit + set -o pipefail + - | + # Update Travis commit range. + # This is not normally required but does prevent problems with outdated forks and + # deleted casks (see https://github.com/Homebrew/homebrew-cask/pull/43164). + BRANCH_COMMIT="${TRAVIS_COMMIT_RANGE##*.}" + TARGET_COMMIT="${TRAVIS_COMMIT_RANGE%%.*}" + if ! MERGE_BASE="$(git merge-base "${BRANCH_COMMIT}" "${TARGET_COMMIT}" 2>/dev/null)"; then + git fetch --unshallow + MERGE_BASE="$(git merge-base "${BRANCH_COMMIT}" "${TARGET_COMMIT}")" + fi + export TRAVIS_COMMIT_RANGE="${MERGE_BASE}...${BRANCH_COMMIT}" + - | + # Switch to master branch. + export HOMEBREW_COLOR=1 + export HOMEBREW_DEVELOPER=1 + export HOMEBREW_NO_AUTO_UPDATE=1 + brew update + - | + # Mirror the repo as a tap. + TAP_DIR="$(brew --repository)/Library/Taps/${TRAVIS_REPO_SLUG}" + mkdir -p "${TAP_DIR}" + rsync -az --delete "${TRAVIS_BUILD_DIR}/" "${TAP_DIR}/" + export TRAVIS_BUILD_DIR="${TAP_DIR}" + builtin cd "${TRAVIS_BUILD_DIR}" -before_script: - - . ci/travis/before_script.sh - -script: - - . ci/travis/script.sh +script: brew cask ci notifications: email: false diff --git a/ci/travis/README.md b/ci/travis/README.md deleted file mode 100644 index 9c1050bc69..0000000000 --- a/ci/travis/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Travis-CI Scripts - -The bash scripts in this directory are run only in Travis-CI, and are placed here to help simplify the [`.travis.yml`](../../.travis.yml) configuration file. - -These scripts are not meant to be run locally by users or developers of Homebrew-Cask. diff --git a/ci/travis/before_script.sh b/ci/travis/before_script.sh deleted file mode 100755 index fc79030468..0000000000 --- a/ci/travis/before_script.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# before_script.sh -# -# This file is meant to be sourced during the `before_script` phase of the -# Travis build. Do not attempt to source or run it locally. -# -# shellcheck disable=SC1090 -. "${TRAVIS_BUILD_DIR}/ci/travis/helpers.sh" - -header 'Running before_script.sh...' - -# this is not normally required but does prevent problems with outdated forks and/or deleted casks -# see https://github.com/Homebrew/homebrew-cask/pull/43164 -run export BRANCH_COMMIT="${TRAVIS_COMMIT_RANGE##*.}" -run export TARGET_COMMIT="${TRAVIS_COMMIT_RANGE%%.*}" -# shellcheck disable=SC2016 -if ! run 'MERGE_BASE="$(git merge-base "${BRANCH_COMMIT}" "${TARGET_COMMIT}")"'; then - run git fetch --unshallow - run 'MERGE_BASE="$(git merge-base "${BRANCH_COMMIT}" "${TARGET_COMMIT}")"' -fi -run export MERGE_BASE="${MERGE_BASE}" -run export TRAVIS_COMMIT_RANGE="${MERGE_BASE}...${BRANCH_COMMIT}" - -# make sure brew is on master branch -run export HOMEBREW_DEVELOPER=1 - -# update homebrew -run brew update - -# mirror the repo as a tap, then run the build from there -run export CASK_TAP_DIR="$(brew --repository)/Library/Taps/${TRAVIS_REPO_SLUG}" -run mkdir -p "${CASK_TAP_DIR}" -run rsync -az --delete "${TRAVIS_BUILD_DIR}/" "${CASK_TAP_DIR}/" -run export TRAVIS_BUILD_DIR="${CASK_TAP_DIR}" -run cd "${CASK_TAP_DIR}" || exit 1 diff --git a/ci/travis/helpers.sh b/ci/travis/helpers.sh deleted file mode 100644 index 75d7dca5d8..0000000000 --- a/ci/travis/helpers.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# -# helpers.sh -# -# Helper functions for Travis build scripts. -# - -# force strict error checking -set -o errexit -set -o pipefail - -# enable extended globbing syntax -shopt -s extglob - -CYAN='\033[0;36m' -MAGENTA='\033[1;35m' -RED='\033[1;31m' -YELLOW='\033[0;33m' -NC='\033[0m' # no color - -# log command before running and add a blank line -run () { - ohai "$*" - eval "$*" - local retval=$? - echo - return $retval -} - -ohai () { - echo -e "${MAGENTA}>>>${NC} $*" -} - -onoe () { - echo -e "${YELLOW}>>> $* ${NC}" - exit 0 -} - -odie () { - echo -e "${RED}!!! $* !!!${NC}" - exit 1 -} - -# print args as a cyan header -header () { - echo - echo -e "${CYAN}$*${NC}" - echo -} diff --git a/ci/travis/script.sh b/ci/travis/script.sh deleted file mode 100755 index c796c9a110..0000000000 --- a/ci/travis/script.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# script.sh -# -# This file is meant to be sourced during the `script` phase of the Travis -# build. Do not attempt to source or run it locally. -# -# shellcheck disable=SC1090 -. "${TRAVIS_BUILD_DIR}/ci/travis/helpers.sh" - -header 'Running script.sh...' - -apps () { /usr/bin/find /Applications -type d -name '*.app' -maxdepth 2 ; } -launchjob_install () { "$(brew --repository)/Library/Taps/Homebrew/homebrew-cask/developer/bin/list_installed_launchjob_ids" ; } -launchjob_load () { "$(brew --repository)/Library/Taps/Homebrew/homebrew-cask/developer/bin/list_loaded_launchjob_ids" ; } -pkgs () { "$(brew --repository)/Library/Taps/Homebrew/homebrew-cask/developer/bin/list_recent_pkg_ids" ; } - -checks=('pkgs' 'apps' 'launchjob_install' 'launchjob_load') - -/bin/mkdir -p "${HOME}/cask-checks/"{before,after} - -for check in "${checks[@]}"; do - "${check}" > "${HOME}/cask-checks/before/${check}" -done - -modified_casks=($(git diff --name-only --diff-filter=AMR "${TRAVIS_COMMIT_RANGE}" -- Casks/*.rb)) - -run export HOMEBREW_NO_AUTO_UPDATE=1 - -if [[ ${#modified_casks[@]} -eq 0 ]]; then - onoe 'No Casks modified, skipping' -fi - -run brew cask style "${modified_casks[@]}" - -if [[ "${TRAVIS_REPO_SLUG}" != 'Homebrew/homebrew-cask-fonts' ]]; then - if [[ ${#modified_casks[@]} -gt 1 ]]; then - run brew cask audit "${modified_casks[@]}" - odie "More than 1 Cask modified, didn't check Cask checksums or URLs" - fi -fi - -run brew cask _audit_modified_casks "${TRAVIS_COMMIT_RANGE}" - -if /usr/bin/grep --quiet "depends_on cask:" "${modified_casks[@]}"; then - run brew tap homebrew/bundle - run brew bundle dump --file="${HOME}/Brewfile" -fi - -for cask in "${modified_casks[@]}"; do - run brew cask reinstall --verbose "${cask}" - run brew cask uninstall --verbose "${cask}" -done - -if [[ -f "${HOME}/Brewfile" ]]; then - run brew bundle cleanup --force --file="${HOME}/Brewfile" -fi - -sleep 5 # Rerunning the checks too soon can result in false positives - -for check in "${checks[@]}"; do - "${check}" > "${HOME}/cask-checks/after/${check}" - - if ! /usr/bin/diff "${HOME}/cask-checks/before/${check}" "${HOME}/cask-checks/after/${check}" > /dev/null; then - ohai "Leftover: ${check}" - /usr/bin/diff "${HOME}/cask-checks/before/${check}" "${HOME}/cask-checks/after/${check}" | /usr/bin/grep '>' - fi -done diff --git a/cmd/brewcask-ci.rb b/cmd/brewcask-ci.rb new file mode 100755 index 0000000000..cbb9762969 --- /dev/null +++ b/cmd/brewcask-ci.rb @@ -0,0 +1,252 @@ +# frozen_string_literal: false + +require "utils/github" +require "utils/formatter" + +require_relative "lib/capture" +require_relative "lib/diffable" +require_relative "lib/github" +require_relative "lib/travis" + +module Hbc + class CLI + class Ci < AbstractCommand + def run + unless ENV.key?("TRAVIS") + raise CaskError, "This command isn’t meant to be run locally." + end + + unless tap + raise CaskError, "This command must be run from inside a tap directory." + end + + ruby_files_in_wrong_directory = modified_ruby_files - (modified_cask_files + modified_command_files) + + unless ruby_files_in_wrong_directory.empty? + raise CaskError, "Casks are in the wrong directory:\n" + + ruby_files_in_wrong_directory.join("\n") + end + + if modified_cask_files.count > 1 && pr_author && !maintainers.include?(pr_author) + raise CaskError, "More than one cask modified; please submit a pull request for each cask separately." + end + + overall_success = true + + modified_cask_files.each do |path| + cask = CaskLoader.load(path) + + overall_success &= step "brew cask audit #{cask.token}", "audit" do + Auditor.audit(cask, audit_download: true, + check_token_conflicts: added_cask_files.include?(path), + commit_range: ENV["TRAVIS_COMMIT_RANGE"]) + end + + overall_success &= step "brew cask style #{cask.token}", "style" do + Style.run(path) + end + + was_installed = cask.installed? + cask_dependencies = CaskDependencies.new(cask).reject(&:installed?) + + checks = { + installed_apps: Diffable.new do + sleep(5) # Allow `mdfind` to refresh. + system_command!("/usr/bin/mdfind", args: ["-onlyin", "/", "kMDItemContentType == com.apple.application-bundle"], print_stderr: false) + .stdout + .split("\n") + end, + installed_kexts: Diffable.new do + system_command!("/usr/sbin/kextstat", args: ["-kl"], print_stderr: false) + .stdout + .lines + .map { |l| l.match(/^.{52}([^\s]+)/)[1] } + .grep_v(/^com\.apple\./) + end, + installed_launchjobs: Diffable.new do + format_launchjob = lambda { |file| + name = file.basename(".plist").to_s + label = Plist.parse_xml(File.read(file))["Label"] + (name == label) ? name : "#{name} (#{label})" + } + + [ + "~/Library/LaunchAgents", + "~/Library/LaunchDaemons", + "/Library/LaunchAgents", + "/Library/LaunchDaemons", + ].map { |p| Pathname(p).expand_path } + .select(&:directory?) + .flat_map(&:children) + .select { |child| child.extname == ".plist" } + .map(&format_launchjob) + end, + loaded_launchjobs: Diffable.new do + launchctl = lambda do |sudo| + system_command!("/bin/launchctl", args: ["list"], print_stderr: false, sudo: sudo) + .stdout + .lines.drop(1) + end + + [false, true] + .flat_map(&launchctl) + .map { |l| l.split(/\s+/)[2] } + .grep_v(/^com\.apple\./) + end, + installed_pkgs: Diffable.new do + Pathname("/var/db/receipts") + .children + .grep(/\.plist$/) + .map(&:basename) + end, + } + + overall_success &= step "brew cask install #{cask.token}", "install" do + Installer.new(cask, force: true).uninstall if was_installed + + checks.values.each(&:before) + + Installer.new(cask, verbose: true).install + end + + overall_success &= step "brew cask uninstall #{cask.token}", "uninstall" do + success = begin + Installer.new(cask, verbose: true).uninstall + true + rescue => e + $stderr.puts e.message + $stderr.puts e.backtrace + false + ensure + cask_dependencies.each do |c| + Installer.new(c, verbose: true).uninstall if c.installed? + end + end + + checks.each do |name, check| + check.after + next unless check.changed? + + success = false + + message = case name + when :installed_pkgs + "Some packages were not uninstalled." + when :loaded_launchjobs + "Some launch jobs were not unloaded." + when :installed_launchjobs + "Some launch jobs were not uninstalled." + when :installed_kexts + "Some kernel extensions were not uninstalled." + when :installed_apps + "Some applications were not uninstalled." + end + + $stderr.puts Formatter.error(message, label: "Error") + $stderr.puts check.diff_lines.join("\n") + end + + success + end + end + + if overall_success + puts Formatter.success("Build finished successfully.", label: "Success") + return + end + + raise CaskError, "Build failed." + end + + private + + def step(name, travis_id) + success = false + output = nil + + Travis.fold travis_id do + print "#{Tty.bold}#{Tty.yellow}#{name}#{Tty.reset} " + + success, output = capture do + begin + yield != false + rescue => e + $stderr.puts e.message + false + end + end + + if success + puts Formatter.success("✔") + puts output + else + puts Formatter.error("✘") + end + end + + puts output unless success + + success + ensure + $stdout.flush + $stderr.flush + end + + def tap + @tap ||= if ENV.key?("TRAVIS_REPO_SLUG") + Tap.fetch(ENV["TRAVIS_REPO_SLUG"]) + else + Tap.from_path(Dir.pwd) + end + end + + def pr_author + return unless ENV.key?("TRAVIS_PULL_REQUEST") + return unless ENV.key?("TRAVIS_REPO_SLUG") + + @pr_author ||= begin + owner, repo = ENV["TRAVIS_REPO_SLUG"].split("/", 2) + + pr = GitHub.pull_request(owner, repo, ENV["TRAVIS_PULL_REQUEST"]) + pr.dig("user", "login") + end + end + + def maintainers + @maintainers ||= begin + GitHub.members("Homebrew", team: "cask").map { |member| member.fetch("login") } + rescue GitHub::AuthenticationFailedError + [] + end + end + + def modified_files + @modified_files ||= system_command!( + "git", args: ["diff", "--name-only", "--diff-filter=AMR", ENV["TRAVIS_COMMIT_RANGE"]] + ).stdout.split("\n").map { |path| Pathname(path) } + end + + def added_files + @added_files ||= system_command!( + "git", args: ["diff", "--name-only", "--diff-filter=A", ENV["TRAVIS_COMMIT_RANGE"]] + ).stdout.split("\n").map { |path| Pathname(path) } + end + + def modified_ruby_files + @modified_ruby_files ||= modified_files.select { |path| path.extname == ".rb" } + end + + def modified_command_files + @modified_command_files ||= modified_files.select { |path| tap.command_file?(path) || path.ascend.to_a.last.to_s == "cmd" } + end + + def modified_cask_files + @modified_cask_files ||= modified_files.select { |path| tap.cask_file?(path) } + end + + def added_cask_files + @added_cask_files ||= added_files.select { |path| tap.cask_file?(path) } + end + end + end +end diff --git a/cmd/lib/capture.rb b/cmd/lib/capture.rb new file mode 100644 index 0000000000..2510756b79 --- /dev/null +++ b/cmd/lib/capture.rb @@ -0,0 +1,38 @@ +require "pty" + +def capture + old_stdout = $stdout.dup + old_stderr = $stderr.dup + + PTY.open do |r, w| + $stdout.reopen(w) + $stderr.reopen(w) + + thread = Thread.new do + begin + yield + ensure + $stdout.flush + $stderr.flush + end + end + thread.abort_on_exception = true + + output = "" + + loop do + begin + output << r.readline_nonblock || "" + rescue IO::WaitReadable + break unless thread.alive? + end + end + + result = thread.value + + [result, output] + end +ensure + $stdout.reopen(old_stdout) + $stderr.reopen(old_stderr) +end diff --git a/cmd/lib/diffable.rb b/cmd/lib/diffable.rb new file mode 100644 index 0000000000..66a082119a --- /dev/null +++ b/cmd/lib/diffable.rb @@ -0,0 +1,51 @@ +class Diffable + def initialize(&block) + @before = nil + @after = nil + @gather = block + end + + def gather + @gather.call.sort.uniq + end + + def before + @before ||= gather + end + + def after + @after ||= gather + end + + def diff + removed = before.reject { |e| after.include?(e) } + added = after - before + + [removed, added] + end + + def changed? + removed, added = diff + removed.any? || added.any? + end + + def combined + (before + after).sort.uniq + end + + def diff_lines(skip_unchanged: true) + removed, added = diff + + lines = combined.flat_map do |e| + if removed.include?(e) + "#{Tty.red}- #{e}#{Tty.reset}" + elsif added.include?(e) + "#{Tty.green}+ #{e}#{Tty.reset}" + else + skip_unchanged ? [] : " #{e}" + end + end + + lines + end +end diff --git a/cmd/lib/github.rb b/cmd/lib/github.rb new file mode 100644 index 0000000000..e2ebc74928 --- /dev/null +++ b/cmd/lib/github.rb @@ -0,0 +1,25 @@ +module GitHub + module_function + + ORG_READ_ACCESS_SCOPES = ["read:org"].freeze + + def members(org, team: nil) + if team + url = "#{API_URL}/orgs/#{org}/teams" + teams = open_api(url, scopes: ORG_READ_ACCESS_SCOPES) + + team = teams.detect { |t| t["name"] == team } + + return [] unless team + open_api("#{team["url"]}/members", scopes: ORG_READ_ACCESS_SCOPES) + else + url = "#{API_URL}/orgs/#{org}/members" + open_api(url, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES) + end + end + + def pull_request(owner, repo, number) + url = "#{API_URL}/repos/#{owner}/#{repo}/pulls/#{number}" + open_api(url) + end +end diff --git a/cmd/lib/travis.rb b/cmd/lib/travis.rb new file mode 100644 index 0000000000..0a7bec1819 --- /dev/null +++ b/cmd/lib/travis.rb @@ -0,0 +1,44 @@ +module Travis + module_function + + @start = {} + + def fold(id, &block) + begin + puts fold_start(id) + time(rand(2**32).to_s(16), &block) + ensure + puts fold_end(id) + $stdout.flush + $stderr.flush + end + end + + def fold_start(id) + "travis_fold:start:#{id}" + end + + def fold_end(id) + "travis_fold:end:#{id}" + end + + def time(id) + puts time_start(id) + yield + ensure + puts time_end(id) + end + + def time_start(id) + @start[id] = Time.now + "travis_time:start:#{id}" + end + + def time_end(id) + start = (@start[id].to_f * 1_000_000_000).to_i + finish = (Time.now.to_f * 1_000_000_000).to_i + duration = finish - start + + "travis_time:end:#{id},start=#{start},finish=#{finish},duration=#{duration}" + end +end