mirror of
https://github.com/macports/mpbb.git
synced 2026-07-12 18:18:44 -07:00
mp-buildbot: Don't re-run previously failed builds
If a port's build has previously failed, we do not want to waste cycles to re-attempt it (e.g. as dependencies of other ports), unless the failed port's definition has changed. To achieve that, create a file with a reference to the last failed build for every failing port, and fail builds that depend on failed ports early. git-svn-id: https://svn.macports.org/repository/macports/contrib/mp-buildbot@152581 d073be05-634f-4543-b044-5fe20cf6d1d6
This commit is contained in:
Clemens Lang
@@ -91,3 +91,6 @@ Subcommand scripts may use but not modify these global shell parameters:
|
||||
persist for the duration of an `mpbb` run, so it may be used to
|
||||
share ancillary files (e.g., a Subversion checkout of the ports
|
||||
tree) between builds of different ports.
|
||||
- `$option_failcache_dir`:
|
||||
A directory for storing information about previously failed builds which
|
||||
saves time because builds that are known to fail will not be attempted.
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/bin/bash
|
||||
# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4
|
||||
|
||||
# Helper functions for mp-buildbot
|
||||
|
||||
# Print $0 and arguments to standard error.
|
||||
# Unset IFS to ensure "$*" uses spaces as separators.
|
||||
msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
|
||||
err() { msg error: "$@"; }
|
||||
warn() { msg warning: "$@"; }
|
||||
|
||||
## Compute a failcache hash for the given port
|
||||
#
|
||||
# Computes and prints a hash uniquely identifying a specific state of a port's
|
||||
# definition files, including the Portfile's hash as well as the port's
|
||||
# patchfiles. To build the hash, this function executes the following
|
||||
# algorithm:
|
||||
# - For the Portfile, and each file in files/ (if any), calculate a SHA256
|
||||
# hash
|
||||
# - Sort the hash values alphabetically
|
||||
# - Hash the result using SHA256
|
||||
# This means a failcache entry will not match if a patchfile changes. A common
|
||||
# case where this is the desired behavior is a port committed without
|
||||
# a required patchfile.
|
||||
#
|
||||
# Valid arguments are all arguments accepted by "port dir".
|
||||
compute_failcache_hash() {
|
||||
local portdir
|
||||
local -a filelist
|
||||
|
||||
portdir=$("${option_prefix}/bin/port" dir "$@")
|
||||
if [ $? -ne 0 ] || [ -z "$portdir" ]; then
|
||||
err "Could not compute failcache hash: port dir" "$@" "failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$portdir" ]; then
|
||||
err "Port directory $portdir does not exist"
|
||||
return 2
|
||||
fi
|
||||
|
||||
filelist=("$portdir/Portfile")
|
||||
if [ -d "$portdir/files" ]; then
|
||||
filelist+=("$portdir/files")
|
||||
fi
|
||||
|
||||
find "${filelist[@]}" -type f -exec openssl dgst -sha256 {} \; |\
|
||||
cut -d' ' -f2 |\
|
||||
sort |\
|
||||
openssl dgst -sha256 |\
|
||||
cut -d' ' -f2
|
||||
}
|
||||
|
||||
## Compute a key that uniquely identifies a (port, variants, portfile-hash) tuple
|
||||
#
|
||||
# Valid arguments are a port name, optionally followed by a variant
|
||||
# specification. Invokes "port dir" to find the Portfile and patchfiles and
|
||||
# computes a checksum of these files that will become part of the hash.
|
||||
failcache_key() {
|
||||
local port=$1
|
||||
if [ -z "$port" ]; then
|
||||
err "failcache_key expects a port argument, but none was given."
|
||||
return 1
|
||||
fi
|
||||
|
||||
local checksum
|
||||
checksum=$(compute_failcache_hash "$port")
|
||||
if [ $? -ne 0 ]; then
|
||||
err "compute_failcache_hash $port failed"
|
||||
return 2
|
||||
fi
|
||||
|
||||
local canonical_variants
|
||||
canonical_variants=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/canonical_variants.tcl" "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
err "tools/canonical_variants.tcl" "$@" "failed"
|
||||
return 4
|
||||
fi
|
||||
|
||||
echo "$port $canonical_variants $checksum"
|
||||
}
|
||||
|
||||
## Test whether a given port with variants has previously failed.
|
||||
#
|
||||
# Valid arguments are a port name, optionally followed by a variant
|
||||
# specification. Succeeds if the port did not previsouly fail to build,
|
||||
# fails if the port is known to fail.
|
||||
failcache_test() {
|
||||
local key
|
||||
key=$(failcache_key "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
err "Could not determine failcache key for" "$@"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -f "${option_failcache_dir}/${key}" ]; then
|
||||
printf "port %s previously failed in build %s\n" "${key}" "$(<"${option_failcache_dir}/${key}")"
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
## Mark a build of a given port with variants as successful.
|
||||
#
|
||||
# Valid arguments are a port name, optionally followed by a variant
|
||||
# specification. Removes any database entries that marked a port as failed.
|
||||
failcache_success() {
|
||||
local key
|
||||
key=$(failcache_key "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
err "Could not determine failcache key for" "$@"
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -f "${option_failcache_dir}/${key}"
|
||||
}
|
||||
|
||||
## Mark a build of a given port with variants as failed.
|
||||
#
|
||||
# Valid arguments are a port name, optionally followed by a variant
|
||||
# specification. Creates or updates the timestamp of a database entry that
|
||||
# marks a port as failed.
|
||||
failcache_failure() {
|
||||
local key
|
||||
key=$(failcache_key "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
err "Could not determine failcache key for" "$@"
|
||||
return 1
|
||||
fi
|
||||
|
||||
mkdir -p "${option_failcache_dir}"
|
||||
echo "${BUILDBOT_BUILDURL:-unknown}" > "${option_failcache_dir}/${key}"
|
||||
}
|
||||
@@ -8,11 +8,10 @@ set -u
|
||||
# Don't inherit any option variables from the calling environment.
|
||||
unset "${!option_@}"
|
||||
|
||||
# Print $0 and arguments to standard error.
|
||||
# Unset IFS to ensure "$*" uses spaces as separators.
|
||||
msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
|
||||
err() { msg error: "$@"; }
|
||||
warn() { msg warning: "$@"; }
|
||||
# Load function library
|
||||
thisdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=functions
|
||||
. "$thisdir/functions"
|
||||
|
||||
mpbb-usage() {
|
||||
# "prog" is defined in mpbb-help.
|
||||
@@ -126,6 +125,7 @@ parseopt prefix:,work-dir: "$@" || exit
|
||||
option_jobs_dir=${option_work_dir}/infrastructure/jobs
|
||||
option_log_dir=${option_work_dir}/logs
|
||||
}
|
||||
option_failcache_dir=${option_work_dir}/failcache
|
||||
|
||||
# Inform the user if old repositories are still present.
|
||||
if [[ -d ${option_work_dir}/tools/.svn ]]; then
|
||||
@@ -146,7 +146,6 @@ set -- ${args+"${args[@]}"}
|
||||
# must define functions "FOO" and "FOO-usage".
|
||||
cmds=()
|
||||
usages=(mpbb-usage)
|
||||
thisdir=$(cd "$(dirname "$0")" && pwd)
|
||||
for cmdfile in "$thisdir/mpbb-"*; do
|
||||
# Unfortunately ShellCheck does not currently support following multiple
|
||||
# files, so we'll just disable the warning.
|
||||
|
||||
@@ -41,6 +41,9 @@ install-dependencies() {
|
||||
local dependencies
|
||||
local dependencies_count
|
||||
local dependencies_counter
|
||||
local depname
|
||||
local depvariants
|
||||
local failcachecounter
|
||||
# $option_log_dir is set in mpbb
|
||||
# shellcheck disable=SC2154
|
||||
local log_status_dependencies="${option_log_dir}/dependencies-progress.txt"
|
||||
@@ -53,7 +56,7 @@ install-dependencies() {
|
||||
# calculate list of dependencies in-order
|
||||
# $option_prefix and $thisdir are set in mpbb
|
||||
# shellcheck disable=SC2154
|
||||
dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port")
|
||||
dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Calculating dependencies for '$port' failed, aborting." >&2
|
||||
echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
|
||||
@@ -72,7 +75,31 @@ install-dependencies() {
|
||||
echo "$dependencies" | sed -E 's/^/ - /' | tee -a "$log_status_dependencies"
|
||||
echo >> "$log_status_dependencies"
|
||||
|
||||
echo "$dependencies" | while read -r dependency; do
|
||||
# Check whether any of the dependencies have previously failed
|
||||
failcachecounter=0
|
||||
while read -r dependency; do
|
||||
# Split portname +variant1+variant2 into portname and variants, where
|
||||
# the variants are optional.
|
||||
depname=${dependency%% *}
|
||||
depvariants=${dependency:${#depname}+1}
|
||||
|
||||
# $depvariants isn't quoted on purpose
|
||||
# shellcheck disable=SC2086
|
||||
if ! failcache_test "$depname" $depvariants; then
|
||||
text="Dependency '${depname}' with variants '${depvariants}' has previously failed and is required."
|
||||
echo "$text" >&2
|
||||
echo "$text" >> "$log_status_dependencies"
|
||||
echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
|
||||
failcachecounter=$((failcachecounter + 1))
|
||||
fi
|
||||
done <<<"$dependencies"
|
||||
|
||||
if [ $failcachecounter -gt 0 ]; then
|
||||
echo "Aborting build because $failcachecounter dependencies are known to fail." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
while read -r dependency; do
|
||||
# Split portname +variant1+variant2 into portname and variants, where
|
||||
# the variants are optional.
|
||||
depname=${dependency%% *}
|
||||
@@ -84,14 +111,31 @@ install-dependencies() {
|
||||
# $depvariants isn't quoted on purpose
|
||||
# shellcheck disable=SC2086
|
||||
if ! "${option_prefix}/bin/port" -d install --unrequested "$depname" $depvariants; then
|
||||
echo "Build of dependency '${depname}' failed, aborting." >&2
|
||||
echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2
|
||||
echo "[FAIL]" >> "$log_status_dependencies"
|
||||
echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
|
||||
|
||||
# Update failcache
|
||||
# $depvariants isn't quoted on purpose
|
||||
# shellcheck disable=SC2086
|
||||
failcache_failure "$depname" $depvariants
|
||||
if [ $? -ne 0 ]; then
|
||||
err "failcache_failure $depname $depvariants failed."
|
||||
return 1
|
||||
fi
|
||||
return 1
|
||||
else
|
||||
echo "[OK]" >> "$log_status_dependencies"
|
||||
# Remove failcache if it exists
|
||||
# $depvariants isn't quoted on purpose
|
||||
# shellcheck disable=SC2086
|
||||
failcache_success "$depname" $depvariants
|
||||
if [ $? -ne 0 ]; then
|
||||
err "failcache_success $depname $depvariants failed."
|
||||
return 1
|
||||
fi
|
||||
dependencies_counter=$((dependencies_counter + 1))
|
||||
fi
|
||||
done
|
||||
done <<<"$dependencies"
|
||||
}
|
||||
|
||||
|
||||
+14
-1
@@ -55,10 +55,23 @@ install-port() {
|
||||
time_start=$(date +%s)
|
||||
# $option_prefix is set in mpbb
|
||||
# shellcheck disable=SC2154
|
||||
if ! "${option_prefix}/bin/port" -dk install "$port"; then
|
||||
if "${option_prefix}/bin/port" -dk install "$@"; then
|
||||
# Remove failcache if it exists
|
||||
failcache_success "$@"
|
||||
if [ $? -ne 0 ]; then
|
||||
err "failcache_success" "$@" "failed."
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
echo "Build of '$port' failed."
|
||||
# log: summary for the portwatcher
|
||||
echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
|
||||
# update failcache
|
||||
failcache_failure "$@"
|
||||
if [ $? -ne 0 ]; then
|
||||
err "failcache_failure" "$@" "failed."
|
||||
return 1
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
time_stop=$(date +%s)
|
||||
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env port-tclsh
|
||||
# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4
|
||||
# Writes the canonical list of variants as it appears in binary archive names
|
||||
# to stdout.
|
||||
#
|
||||
# Copyright (c) 2016 The MacPorts Project.
|
||||
# Copyright (c) 2016 Clemens Lang <cal@macports.org>
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in
|
||||
# the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
# 3. Neither the name of the MacPorts project, nor the names of any contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
|
||||
# AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
|
||||
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
||||
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
|
||||
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package require macports
|
||||
|
||||
if {[llength $::argv] == 0} {
|
||||
puts stderr "Usage: $argv0 <portname> \[(+|-)variant...\]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# initialize macports
|
||||
if {[catch {mportinit "" "" ""} result]} {
|
||||
ui_error "$errorInfo"
|
||||
ui_error "Failed to initialize ports sytem: $result"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# look up the path of the Portfile for the given port
|
||||
set portname [lindex $::argv 0]
|
||||
#try -pass_signal {...}
|
||||
try {
|
||||
set result [mportlookup $portname]
|
||||
if {[llength $result] < 2} {
|
||||
ui_error "No such port: $portname"
|
||||
exit 1
|
||||
}
|
||||
} catch {{*} eCode eMessage} {
|
||||
ui_error "mportlookup $portname failed: $eMessage"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# parse the given variants from the command line
|
||||
array set variants {}
|
||||
foreach item [lrange $::argv 1 end] {
|
||||
foreach {_ sign variant} [regexp -all -inline -- {([-+])([[:alpha:]_]+[\w\.]*)} $item] {
|
||||
set variants($variant) $sign
|
||||
}
|
||||
}
|
||||
|
||||
# open the port to get its active variants
|
||||
array set portinfo [lindex $result 1]
|
||||
#try -pass_signal {...}
|
||||
try {
|
||||
set mport [mportopen $portinfo(porturl) [list subport $portname] [array get variants]]
|
||||
} catch {{*} eCode eMessage} {
|
||||
ui_error "mportopen ${portinfo(porturl)} failed: $eMessage"
|
||||
exit 1
|
||||
}
|
||||
|
||||
array set info [mportinfo $mport]
|
||||
puts $info(canonical_active_variants)
|
||||
|
||||
#try -pass_signal {...}
|
||||
try {
|
||||
mportclose $mport
|
||||
} catch {{*} eCode eMessage} {
|
||||
ui_warn "mportclose $portname failed: $eMessage"
|
||||
}
|
||||
|
||||
# shut down MacPorts
|
||||
#try -pass_signal {...}
|
||||
try {
|
||||
mportshutdown
|
||||
} catch {{*} eCode eMessage} {
|
||||
ui_error "mportshutdown failed: $eMessage"
|
||||
exit 1
|
||||
}
|
||||
Reference in New Issue
Block a user