From f59f942d4ff6efa22e6c7ffa7fea07d6ec43ab4c Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Thu, 1 Dec 2022 12:53:29 -0800 Subject: [PATCH] buildkite: install docker runtimes only when it is required PiperOrigin-RevId: 492275948 --- .buildkite/hooks/post-command | 20 ++++++++------- .buildkite/hooks/pre-command | 46 ++++++++++++++++++----------------- .buildkite/pipeline.yaml | 26 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index accc389c6..77b229281 100644 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -80,15 +80,17 @@ done set -euo pipefail -# Remove all Docker runtimes that may be installed. -num_docker_runtimes="$(jq \ - '(if has("runtimes") then .runtimes else {} end) | length' \ - < /etc/docker/daemon.json)" -if [[ "$num_docker_runtimes" -gt 0 ]]; then - cat /etc/docker/daemon.json | jq 'del(.runtimes)' \ - | sudo tee /etc/docker/daemon.json.tmp - sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json - bash -c "$DOCKER_RELOAD_COMMAND" +if [[ "${BUILDKITE_PIPELINE_INSTALL_RUNTIME:-}" == "true" ]]; then + # Remove all Docker runtimes that may be installed. + num_docker_runtimes="$(jq \ + '(if has("runtimes") then .runtimes else {} end) | length' \ + < /etc/docker/daemon.json)" + if [[ "$num_docker_runtimes" -gt 0 ]]; then + cat /etc/docker/daemon.json | jq 'del(.runtimes)' \ + | sudo tee /etc/docker/daemon.json.tmp + sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json + bash -c "$DOCKER_RELOAD_COMMAND" + fi fi # Cleanup temporary directory where STAGED_BINARIES may have been extracted. diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b26361652..0d2fc101e 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -47,30 +47,32 @@ else export DOCKER_RELOAD_COMMAND='sudo systemctl reload docker' fi -# Ensure Docker has experimental enabled, install runtimes. -HAD_EXPERIMENTAL="$(docker version --format='{{.Server.Experimental}}')" -if [[ -n "${STAGED_BINARIES:-}" ]]; then - # Used `runsc` from STAGED_BINARIES instead of building it from scratch. - export BUILDKITE_STAGED_BINARIES_DIRECTORY="$(mktemp -d)" - gsutil cat "$STAGED_BINARIES" \ - | tar -C "$BUILDKITE_STAGED_BINARIES_DIRECTORY" -zxvf - runsc - chmod +x "$BUILDKITE_STAGED_BINARIES_DIRECTORY/runsc" - sudo "$BUILDKITE_STAGED_BINARIES_DIRECTORY/runsc" install \ +if [[ "${BUILDKITE_PIPELINE_INSTALL_RUNTIME:-}" == "true" ]]; then + # Ensure Docker has experimental enabled, install runtimes. + HAD_EXPERIMENTAL="$(docker version --format='{{.Server.Experimental}}')" + if [[ -n "${STAGED_BINARIES:-}" ]]; then + # Used `runsc` from STAGED_BINARIES instead of building it from scratch. + export BUILDKITE_STAGED_BINARIES_DIRECTORY="$(mktemp -d)" + gsutil cat "$STAGED_BINARIES" \ + | tar -C "$BUILDKITE_STAGED_BINARIES_DIRECTORY" -zxvf - runsc + chmod +x "$BUILDKITE_STAGED_BINARIES_DIRECTORY/runsc" + sudo "$BUILDKITE_STAGED_BINARIES_DIRECTORY/runsc" install \ --experimental=true --runtime="${RUNTIME}" \ -- "${RUNTIME_ARGS:-}" -else - make sudo TARGETS=//runsc:runsc \ - ARGS="install --experimental=true --runtime=${RUNTIME} -- ${RUNTIME_ARGS:-}" -fi -if [[ "$HAD_EXPERIMENTAL" == false ]]; then - # WARNING: We may be running in a container when this command executes. - # This only makes sense if Docker's `live-restore` feature is enabled. - sudo systemctl restart docker -else - # If experimental-ness was already enabled, we don't need to restart, as the - # only thing we modified is the list of runtimes, which can be reloaded with - # just a SIGHUP. - bash -c "$DOCKER_RELOAD_COMMAND" + else + make sudo TARGETS=//runsc:runsc \ + ARGS="install --experimental=true --runtime=${RUNTIME} -- ${RUNTIME_ARGS:-}" + fi + if [[ "$HAD_EXPERIMENTAL" == false ]]; then + # WARNING: We may be running in a container when this command executes. + # This only makes sense if Docker's `live-restore` feature is enabled. + sudo systemctl restart docker + else + # If experimental-ness was already enabled, we don't need to restart, as the + # only thing we modified is the list of runtimes, which can be reloaded with + # just a SIGHUP. + bash -c "$DOCKER_RELOAD_COMMAND" + fi fi # Helper for benchmarks, based on the branch. diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 924b787e9..af4273dd9 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -12,6 +12,9 @@ _templates: platform_specific_agents: &platform_specific_agents {} kvm_agents: &kvm_agents {kvm: "true"} ubuntu_agents: &ubuntu_agents {os: "ubuntu"} + docker: &docker + env: + BUILDKITE_PIPELINE_INSTALL_RUNTIME: true benchmarks: &benchmarks timeout_in_minutes: 120 retry: @@ -25,6 +28,7 @@ _templates: BENCHMARKS_PROJECT: gvisor-benchmarks BENCHMARKS_TABLE: benchmarks BENCHMARKS_UPLOAD: true + BUILDKITE_PIPELINE_INSTALL_RUNTIME: true agents: <<: *kvm_agents <<: *platform_specific_agents @@ -194,6 +198,7 @@ steps: arch: "arm64" - <<: *common <<: *source_test + <<: *docker label: ":test_tube: Container tests (cgroupv1)" command: make container-tests agents: @@ -201,6 +206,7 @@ steps: cgroup: "v1" arch: "amd64" - <<: *common + <<: *docker # This variant is not really a source test, but we annotate it as such to # avoid running binary-only tests for all variants of cgroups. It is # sufficient to run cgroupv2 variants only for source changes. @@ -231,6 +237,7 @@ steps: # Integration tests. - <<: *common + <<: *docker <<: *source_test label: ":docker: Docker tests (cgroupv1)" command: make docker-tests @@ -239,6 +246,7 @@ steps: arch: "amd64" cgroup: "v1" - <<: *common + <<: *docker # See above: not truly a source test. <<: *source_test label: ":docker: Docker tests (cgroupv2)" @@ -248,6 +256,7 @@ steps: arch: "amd64" cgroup: "v2" - <<: *common + <<: *docker label: ":goggles: Overlay tests" command: make overlay-tests agents: @@ -255,6 +264,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":safety_pin: Host network tests" command: make hostnet-tests agents: @@ -262,6 +272,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":satellite: gVisor GSO tests" command: make swgso-tests agents: @@ -275,6 +286,7 @@ steps: agents: arch: "amd64" - <<: *common + <<: *docker <<: *source_test label: ":person_in_lotus_position: KVM tests" command: make kvm-tests @@ -282,6 +294,7 @@ steps: <<: *kvm_agents arch: "amd64" - <<: *common + <<: *docker label: ":weight_lifter: Fsstress test" command: make fsstress-test agents: @@ -289,6 +302,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":docker: Containerd 1.3.9 tests (cgroupv1)" command: make containerd-test-1.3.9 agents: @@ -297,6 +311,7 @@ steps: cgroup: "v1" arch: "amd64" - <<: *common + <<: *docker label: ":docker: Containerd 1.4.3 tests (cgroupv1)" command: make containerd-test-1.4.3 agents: @@ -304,6 +319,7 @@ steps: <<: *ubuntu_agents cgroup: "v1" - <<: *common + <<: *docker # See above: not truly a source test. <<: *source_test label: ":docker: Containerd 1.4.3 tests (cgroupv2)" @@ -313,6 +329,7 @@ steps: <<: *ubuntu_agents cgroup: "v2" - <<: *common + <<: *docker label: ":docker: Containerd 1.5.11 tests (cgroupv1)" command: make containerd-test-1.5.11 agents: @@ -320,6 +337,7 @@ steps: <<: *ubuntu_agents cgroup: "v1" - <<: *common + <<: *docker # See above: not truly a source test. <<: *source_test label: ":docker: Containerd 1.5.11 tests (cgroupv2)" @@ -328,6 +346,7 @@ steps: <<: *ubuntu_agents cgroup: "v2" - <<: *common + <<: *docker label: ":docker: Containerd 1.6.2 tests (cgroupv1)" command: make containerd-test-1.6.2 agents: @@ -335,6 +354,7 @@ steps: <<: *ubuntu_agents cgroup: "v1" - <<: *common + <<: *docker <<: *source_test label: ":docker: Containerd 1.6.2 tests (cgroupv2)" command: make containerd-test-1.6.2 @@ -379,6 +399,7 @@ steps: # Runtime tests. - <<: *common + <<: *docker label: ":php: PHP runtime tests" command: make RUNTIME_LOG_DIR=/tmp/$${BUILDKITE_JOB_ID} php8.1.1-runtime-tests parallelism: 10 @@ -387,6 +408,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":java: Java runtime tests" command: make RUNTIME_LOG_DIR=/tmp/$${BUILDKITE_JOB_ID} java17-runtime-tests parallelism: 40 @@ -395,6 +417,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":golang: Go runtime tests" command: make RUNTIME_LOG_DIR=/tmp/$${BUILDKITE_JOB_ID} go1.16-runtime-tests parallelism: 10 @@ -403,6 +426,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":node: NodeJS runtime tests" command: make RUNTIME_LOG_DIR=/tmp/$${BUILDKITE_JOB_ID} nodejs16.13.2-runtime-tests parallelism: 10 @@ -411,6 +435,7 @@ steps: <<: *ubuntu_agents arch: "amd64" - <<: *common + <<: *docker label: ":python: Python runtime tests" command: make RUNTIME_LOG_DIR=/tmp/$${BUILDKITE_JOB_ID} python3.10.2-runtime-tests parallelism: 10 @@ -421,6 +446,7 @@ steps: # Run basic benchmarks smoke tests (no upload). - <<: *common + <<: *docker label: ":fire: Benchmarks smoke test" command: make benchmark-platforms BENCHMARKS_TARGETS=test/benchmarks/base:startup_test BENCHMARKS_FILTER=BenchmarkStartupEmpty BENCHMARKS_OPTIONS=-test.benchtime=1ns # Use the opposite of the benchmarks filter.