From 4c4cc7ae327b5e4a3740ccb3f94956995fb98454 Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Mon, 12 Sep 2016 19:14:24 +0000 Subject: [PATCH] mp-buildbot: Don't re-run previously failed builds If a port's build has previously failed, we do not want to waste cycles to re-attempt it (e.g. as dependencies of other ports), unless the failed port's definition has changed. To achieve that, create a file with a reference to the last failed build for every failing port, and fail builds that depend on failed ports early. git-svn-id: https://svn.macports.org/repository/macports/contrib/mp-buildbot@152581 d073be05-634f-4543-b044-5fe20cf6d1d6 --- README.md | 3 + functions | 134 +++++++++++++++++++++++++++++++++++ mpbb | 11 ++- mpbb-install-dependencies | 52 ++++++++++++-- mpbb-install-port | 15 +++- tools/canonical_variants.tcl | 97 +++++++++++++++++++++++++ 6 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 functions create mode 100755 tools/canonical_variants.tcl diff --git a/README.md b/README.md index 8e80722..edb3358 100644 --- a/README.md +++ b/README.md @@ -91,3 +91,6 @@ Subcommand scripts may use but not modify these global shell parameters: persist for the duration of an `mpbb` run, so it may be used to share ancillary files (e.g., a Subversion checkout of the ports tree) between builds of different ports. +- `$option_failcache_dir`: + A directory for storing information about previously failed builds which + saves time because builds that are known to fail will not be attempted. diff --git a/functions b/functions new file mode 100644 index 0000000..a070a70 --- /dev/null +++ b/functions @@ -0,0 +1,134 @@ +#!/bin/bash +# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4 + +# Helper functions for mp-buildbot + +# Print $0 and arguments to standard error. +# Unset IFS to ensure "$*" uses spaces as separators. +msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*") +err() { msg error: "$@"; } +warn() { msg warning: "$@"; } + +## Compute a failcache hash for the given port +# +# Computes and prints a hash uniquely identifying a specific state of a port's +# definition files, including the Portfile's hash as well as the port's +# patchfiles. To build the hash, this function executes the following +# algorithm: +# - For the Portfile, and each file in files/ (if any), calculate a SHA256 +# hash +# - Sort the hash values alphabetically +# - Hash the result using SHA256 +# This means a failcache entry will not match if a patchfile changes. A common +# case where this is the desired behavior is a port committed without +# a required patchfile. +# +# Valid arguments are all arguments accepted by "port dir". +compute_failcache_hash() { + local portdir + local -a filelist + + portdir=$("${option_prefix}/bin/port" dir "$@") + if [ $? -ne 0 ] || [ -z "$portdir" ]; then + err "Could not compute failcache hash: port dir" "$@" "failed" + return 1 + fi + + if [ ! -d "$portdir" ]; then + err "Port directory $portdir does not exist" + return 2 + fi + + filelist=("$portdir/Portfile") + if [ -d "$portdir/files" ]; then + filelist+=("$portdir/files") + fi + + find "${filelist[@]}" -type f -exec openssl dgst -sha256 {} \; |\ + cut -d' ' -f2 |\ + sort |\ + openssl dgst -sha256 |\ + cut -d' ' -f2 +} + +## Compute a key that uniquely identifies a (port, variants, portfile-hash) tuple +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Invokes "port dir" to find the Portfile and patchfiles and +# computes a checksum of these files that will become part of the hash. +failcache_key() { + local port=$1 + if [ -z "$port" ]; then + err "failcache_key expects a port argument, but none was given." + return 1 + fi + + local checksum + checksum=$(compute_failcache_hash "$port") + if [ $? -ne 0 ]; then + err "compute_failcache_hash $port failed" + return 2 + fi + + local canonical_variants + canonical_variants=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/canonical_variants.tcl" "$@") + if [ $? -ne 0 ]; then + err "tools/canonical_variants.tcl" "$@" "failed" + return 4 + fi + + echo "$port $canonical_variants $checksum" +} + +## Test whether a given port with variants has previously failed. +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Succeeds if the port did not previsouly fail to build, +# fails if the port is known to fail. +failcache_test() { + local key + key=$(failcache_key "$@") + if [ $? -ne 0 ]; then + err "Could not determine failcache key for" "$@" + return 1 + fi + + if [ -f "${option_failcache_dir}/${key}" ]; then + printf "port %s previously failed in build %s\n" "${key}" "$(<"${option_failcache_dir}/${key}")" + return 1 + else + return 0 + fi +} + +## Mark a build of a given port with variants as successful. +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Removes any database entries that marked a port as failed. +failcache_success() { + local key + key=$(failcache_key "$@") + if [ $? -ne 0 ]; then + err "Could not determine failcache key for" "$@" + return 1 + fi + + rm -f "${option_failcache_dir}/${key}" +} + +## Mark a build of a given port with variants as failed. +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Creates or updates the timestamp of a database entry that +# marks a port as failed. +failcache_failure() { + local key + key=$(failcache_key "$@") + if [ $? -ne 0 ]; then + err "Could not determine failcache key for" "$@" + return 1 + fi + + mkdir -p "${option_failcache_dir}" + echo "${BUILDBOT_BUILDURL:-unknown}" > "${option_failcache_dir}/${key}" +} diff --git a/mpbb b/mpbb index 17fbd3a..1c5d197 100755 --- a/mpbb +++ b/mpbb @@ -8,11 +8,10 @@ set -u # Don't inherit any option variables from the calling environment. unset "${!option_@}" -# Print $0 and arguments to standard error. -# Unset IFS to ensure "$*" uses spaces as separators. -msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*") -err() { msg error: "$@"; } -warn() { msg warning: "$@"; } +# Load function library +thisdir=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=functions +. "$thisdir/functions" mpbb-usage() { # "prog" is defined in mpbb-help. @@ -126,6 +125,7 @@ parseopt prefix:,work-dir: "$@" || exit option_jobs_dir=${option_work_dir}/infrastructure/jobs option_log_dir=${option_work_dir}/logs } +option_failcache_dir=${option_work_dir}/failcache # Inform the user if old repositories are still present. if [[ -d ${option_work_dir}/tools/.svn ]]; then @@ -146,7 +146,6 @@ set -- ${args+"${args[@]}"} # must define functions "FOO" and "FOO-usage". cmds=() usages=(mpbb-usage) -thisdir=$(cd "$(dirname "$0")" && pwd) for cmdfile in "$thisdir/mpbb-"*; do # Unfortunately ShellCheck does not currently support following multiple # files, so we'll just disable the warning. diff --git a/mpbb-install-dependencies b/mpbb-install-dependencies index 150b15d..725ac2c 100644 --- a/mpbb-install-dependencies +++ b/mpbb-install-dependencies @@ -41,6 +41,9 @@ install-dependencies() { local dependencies local dependencies_count local dependencies_counter + local depname + local depvariants + local failcachecounter # $option_log_dir is set in mpbb # shellcheck disable=SC2154 local log_status_dependencies="${option_log_dir}/dependencies-progress.txt" @@ -53,7 +56,7 @@ install-dependencies() { # calculate list of dependencies in-order # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 - dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port") + dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@") if [ $? -ne 0 ]; then echo "Calculating dependencies for '$port' failed, aborting." >&2 echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" @@ -72,7 +75,31 @@ install-dependencies() { echo "$dependencies" | sed -E 's/^/ - /' | tee -a "$log_status_dependencies" echo >> "$log_status_dependencies" - echo "$dependencies" | while read -r dependency; do + # Check whether any of the dependencies have previously failed + failcachecounter=0 + while read -r dependency; do + # Split portname +variant1+variant2 into portname and variants, where + # the variants are optional. + depname=${dependency%% *} + depvariants=${dependency:${#depname}+1} + + # $depvariants isn't quoted on purpose + # shellcheck disable=SC2086 + if ! failcache_test "$depname" $depvariants; then + text="Dependency '${depname}' with variants '${depvariants}' has previously failed and is required." + echo "$text" >&2 + echo "$text" >> "$log_status_dependencies" + echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress" + failcachecounter=$((failcachecounter + 1)) + fi + done <<<"$dependencies" + + if [ $failcachecounter -gt 0 ]; then + echo "Aborting build because $failcachecounter dependencies are known to fail." >&2 + return 1 + fi + + while read -r dependency; do # Split portname +variant1+variant2 into portname and variants, where # the variants are optional. depname=${dependency%% *} @@ -84,14 +111,31 @@ install-dependencies() { # $depvariants isn't quoted on purpose # shellcheck disable=SC2086 if ! "${option_prefix}/bin/port" -d install --unrequested "$depname" $depvariants; then - echo "Build of dependency '${depname}' failed, aborting." >&2 + echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2 echo "[FAIL]" >> "$log_status_dependencies" echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress" + + # Update failcache + # $depvariants isn't quoted on purpose + # shellcheck disable=SC2086 + failcache_failure "$depname" $depvariants + if [ $? -ne 0 ]; then + err "failcache_failure $depname $depvariants failed." + return 1 + fi return 1 else echo "[OK]" >> "$log_status_dependencies" + # Remove failcache if it exists + # $depvariants isn't quoted on purpose + # shellcheck disable=SC2086 + failcache_success "$depname" $depvariants + if [ $? -ne 0 ]; then + err "failcache_success $depname $depvariants failed." + return 1 + fi dependencies_counter=$((dependencies_counter + 1)) fi - done + done <<<"$dependencies" } diff --git a/mpbb-install-port b/mpbb-install-port index adc0d22..3fcb970 100644 --- a/mpbb-install-port +++ b/mpbb-install-port @@ -55,10 +55,23 @@ install-port() { time_start=$(date +%s) # $option_prefix is set in mpbb # shellcheck disable=SC2154 - if ! "${option_prefix}/bin/port" -dk install "$port"; then + if "${option_prefix}/bin/port" -dk install "$@"; then + # Remove failcache if it exists + failcache_success "$@" + if [ $? -ne 0 ]; then + err "failcache_success" "$@" "failed." + return 1 + fi + else echo "Build of '$port' failed." # log: summary for the portwatcher echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" + # update failcache + failcache_failure "$@" + if [ $? -ne 0 ]; then + err "failcache_failure" "$@" "failed." + return 1 + fi return 1 fi time_stop=$(date +%s) diff --git a/tools/canonical_variants.tcl b/tools/canonical_variants.tcl new file mode 100755 index 0000000..0fd4897 --- /dev/null +++ b/tools/canonical_variants.tcl @@ -0,0 +1,97 @@ +#!/usr/bin/env port-tclsh +# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4 +# Writes the canonical list of variants as it appears in binary archive names +# to stdout. +# +# Copyright (c) 2016 The MacPorts Project. +# Copyright (c) 2016 Clemens Lang +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the MacPorts project, nor the names of any contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package require macports + +if {[llength $::argv] == 0} { + puts stderr "Usage: $argv0 \[(+|-)variant...\]" + exit 1 +} + +# initialize macports +if {[catch {mportinit "" "" ""} result]} { + ui_error "$errorInfo" + ui_error "Failed to initialize ports sytem: $result" + exit 1 +} + +# look up the path of the Portfile for the given port +set portname [lindex $::argv 0] +#try -pass_signal {...} +try { + set result [mportlookup $portname] + if {[llength $result] < 2} { + ui_error "No such port: $portname" + exit 1 + } +} catch {{*} eCode eMessage} { + ui_error "mportlookup $portname failed: $eMessage" + exit 1 +} + +# parse the given variants from the command line +array set variants {} +foreach item [lrange $::argv 1 end] { + foreach {_ sign variant} [regexp -all -inline -- {([-+])([[:alpha:]_]+[\w\.]*)} $item] { + set variants($variant) $sign + } +} + +# open the port to get its active variants +array set portinfo [lindex $result 1] +#try -pass_signal {...} +try { + set mport [mportopen $portinfo(porturl) [list subport $portname] [array get variants]] +} catch {{*} eCode eMessage} { + ui_error "mportopen ${portinfo(porturl)} failed: $eMessage" + exit 1 +} + +array set info [mportinfo $mport] +puts $info(canonical_active_variants) + +#try -pass_signal {...} +try { + mportclose $mport +} catch {{*} eCode eMessage} { + ui_warn "mportclose $portname failed: $eMessage" +} + +# shut down MacPorts +#try -pass_signal {...} +try { + mportshutdown +} catch {{*} eCode eMessage} { + ui_error "mportshutdown failed: $eMessage" + exit 1 +}