From 9ba9d78ca9cba3347515f9ecef3fb8301e22dfad Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Tue, 10 Jan 2023 17:14:28 -0800 Subject: [PATCH] BuildKite: Make `post-command` more robust against stuck containers. Some BuildKite builds have been getting stuck on this step. Interestingly, this was not on the first container that they try to stop, implying that the problem is container-specific, rather than the Docker daemon being unresponsive as a whole. This change removes the dependency on `docker inspect` and adds a timeout to the container kill step. Will it be enough? Not sure. #codehealth PiperOrigin-RevId: 501132910 --- .buildkite/hooks/post-command | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index 77b229281..3d1a93fd8 100644 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -71,11 +71,12 @@ if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then fi # Kill any running containers (clear state), except for "bootstrap". -for container in $(docker ps -q); do - maybe_kill="$(docker inspect -f '{{if ne "/bootstrap" .Name}}true{{ end }}' "${container}")" - if test -n "${maybe_kill}"; then - docker container kill "${container}" +for id_and_name in $(docker ps --format='{{.ID}} {{.Names}}'); do + if [[ "$(echo "$id_and_name" | cut -d' ' -f2-)" == 'bootstrap' ]]; then + continue fi + timeout --kill-after=10s --preserve-status 8s \ + docker container kill "$(echo "$id_and_name" | cut -d' ' -f1)" done set -euo pipefail