mirror of
https://github.com/netbirdio/gvisor.git
synced 2026-05-22 17:12:49 -07:00
Andrei VaginandgVisor bot
c191ceb4b8
When buildkite cancels jobs, it doesn't run the post-coomand hook, so some docker containers can continue running. PiperOrigin-RevId: 721001481
98 lines
3.7 KiB
Plaintext
98 lines
3.7 KiB
Plaintext
set -x
|
|
|
|
source .buildkite/hooks/libhook
|
|
|
|
# Clear any downloaded credentials.
|
|
rm -f repo.key
|
|
|
|
# Upload all relevant test failures.
|
|
make -s testlogs 2>/dev/null | grep // | sort | uniq | (
|
|
declare log_count=0
|
|
while read target log; do
|
|
if test -z "${target}"; then
|
|
continue
|
|
fi
|
|
|
|
# N.B. If *all* tests fail due to some common cause, then we will
|
|
# end up spending way too much time uploading logs. Instead, we just
|
|
# upload the first 10 and stop. That is hopefully enough to debug.
|
|
#
|
|
# We include this test in the metadata, but note that we cannot
|
|
# upload the actual test logs. The user should rerun locally.
|
|
log_count=$((${log_count}+1))
|
|
if test "${log_count}" -ge 10; then
|
|
echo " * ${target} (no upload)" | \
|
|
buildkite-agent annotate --style error --context failures --append
|
|
else
|
|
buildkite-agent artifact upload "${log}"
|
|
echo " * [${target}](artifact://${log#/}) (${BUILDKITE_LABEL})" | \
|
|
buildkite-agent annotate --style error --context failures --append
|
|
fi
|
|
done
|
|
)
|
|
|
|
# Upload all profiles, and include in an annotation.
|
|
declare profile_output=$(mktemp --tmpdir)
|
|
for file in $(find /tmp/profile -name \*.pprof -print 2>/dev/null | sort); do
|
|
# Generate a link to the profile parsing function in gvisor.dev, which
|
|
# implicitly uses a prefix of https://storage.googleapis.com. Note that
|
|
# this relies on the specific BuildKite bucket location, and will break if
|
|
# this changes (although the artifacts will still exist and be just fine).
|
|
profile_name="${file#/tmp/profile/}"
|
|
profile_url="https://gvisor.dev/profile/gvisor-buildkite/${BUILDKITE_BUILD_ID}/${BUILDKITE_JOB_ID}/${file#/}/"
|
|
buildkite-agent artifact upload "${file}"
|
|
echo "<li><a href='${profile_url}'>${profile_name}</a></li>" >> "${profile_output}"
|
|
done
|
|
|
|
# Upload if we had outputs.
|
|
if test -s "${profile_output}"; then
|
|
# Make the list a collapsible section in markdown.
|
|
sed -i "1s|^|<details><summary>${BUILDKITE_LABEL}</summary><ul>\n|" "${profile_output}"
|
|
echo "</ul></details>" >> "${profile_output}"
|
|
cat "${profile_output}" | buildkite-agent annotate --style info --context profiles --append
|
|
fi
|
|
rm -rf "${profile_output}"
|
|
|
|
# Clean the bazel cache, if there's failure.
|
|
if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then
|
|
set -x
|
|
dmesg -kT | tail -n 50
|
|
# LINT.IfChange
|
|
runtime="buildkite_runtime"
|
|
# LINT.ThenChange(pre-command)
|
|
if [ -d "/tmp/${runtime}/" ]; then
|
|
tar -czf "/tmp/${BUILDKITE_JOB_ID}.tar.gz" -C /tmp/ "${runtime}"
|
|
buildkite-agent artifact upload "/tmp/${BUILDKITE_JOB_ID}.tar.gz"
|
|
fi
|
|
# Attempt to clear the cache and shut down.
|
|
make clean || echo "make clean failed with code $?"
|
|
make bazel-shutdown || echo "make bazel-shutdown failed with code $?"
|
|
# Attempt to clear any Go cache.
|
|
sudo rm -rf "${HOME}/.cache/go-build"
|
|
sudo rm -rf "${HOME}/go"
|
|
fi
|
|
|
|
clear_docker_containers
|
|
|
|
set -euo pipefail
|
|
|
|
if [[ "${BUILDKITE_PIPELINE_INSTALL_RUNTIME:-}" == "true" ]]; then
|
|
# Remove all Docker runtimes that may be installed.
|
|
num_docker_runtimes="$(jq \
|
|
'(if has("runtimes") then .runtimes else {} end) | length' \
|
|
< /etc/docker/daemon.json)"
|
|
if [[ "$num_docker_runtimes" -gt 0 ]]; then
|
|
cat /etc/docker/daemon.json | jq '.runtimes = {}' \
|
|
| sudo tee /etc/docker/daemon.json.tmp
|
|
sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json
|
|
echo 'Removed all Docker runtimes; reloading Docker daemon with this new configuration:' >&2
|
|
cat /etc/docker/daemon.json >&2
|
|
bash -c "$DOCKER_RELOAD_COMMAND"
|
|
fi
|
|
fi
|
|
|
|
# Cleanup temporary directory where STAGED_BINARIES may have been extracted.
|
|
if [[ -n "${BUILDKITE_STAGED_BINARIES_DIRECTORY:-}" ]]; then
|
|
rm -rf "$BUILDKITE_STAGED_BINARIES_DIRECTORY" || true
|
|
fi
|