diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 8d6218950..163c8d38c 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -95,7 +95,7 @@ steps: label: ":world_map: Build runsc and pkg (AMD64)" commands: - "make build TARGETS=//pkg/..." - - "make build TARGETS='--build_tag_filters=-network_plugins //runsc/...'" + - "make build OPTIONS='--build_tag_filters=-network_plugins' TARGETS='//runsc/...'" agents: arch: "amd64" @@ -104,7 +104,7 @@ steps: label: ":world_map: Build runsc and pkg (ARM64)" commands: - "make build TARGETS=//pkg/..." - - "make build TARGETS='--build_tag_filters=-network_plugins //runsc/...'" + - "make build OPTIONS='--build_tag_filters=-network_plugins' TARGETS='//runsc/...'" agents: arch: "arm64" @@ -113,7 +113,7 @@ steps: <<: *source_test_continuous label: ":world_map: Build everything" commands: - - "make build TARGETS='--build_tag_filters=-network_plugins //...'" + - "make build OPTIONS='--build_tag_filters=-network_plugins' TARGETS='//...'" # Check that the Go branch builds. This is not technically required, as this build is maintained # as a GitHub action in order to preserve this maintaince across forks. However, providing the diff --git a/Makefile b/Makefile index 9a458da97..fe55bd2e5 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,11 @@ help: ## Shows all targets and help from the Makefile (this message). }' build: ## Builds the given $(TARGETS) with the given $(OPTIONS). E.g. make build TARGETS=runsc - @$(call build,$(OPTIONS) $(TARGETS)) + @$(call build,$(OPTIONS) -- $(TARGETS)) .PHONY: build test: ## Tests the given $(TARGETS) with the given $(OPTIONS). E.g. make test TARGETS=pkg/buffer:buffer_test - @$(call test,$(OPTIONS) $(TARGETS)) + @$(call test,$(OPTIONS) -- $(TARGETS)) .PHONY: test copy: ## Copies the given $(TARGETS) to the given $(DESTINATION). E.g. make copy TARGETS=runsc DESTINATION=/tmp @@ -116,6 +116,7 @@ RUNTIME_BIN ?= $(RUNTIME_DIR)/runsc RUNTIME_LOG_DIR ?= $(RUNTIME_DIR)/logs RUNTIME_LOGS ?= $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND% RUNTIME_ARGS ?= +DOCKER_DAEMON_CONFIG_PATH ?= /etc/docker/daemon.json DOCKER_RELOAD_COMMAND ?= sudo systemctl reload docker SYSFS_GROUP_PATH := /sys/fs/cgroup @@ -139,7 +140,7 @@ endif # Configure helpers for below. configure_noreload = \ $(call header,CONFIGURE $(1) → $(RUNTIME_BIN) $(RUNTIME_ARGS) $(2)); \ - sudo $(RUNTIME_BIN) install --experimental=true --runtime="$(1)" -- $(RUNTIME_ARGS) --debug-log "$(RUNTIME_LOGS)" $(2) && \ + sudo $(RUNTIME_BIN) install --config_file="$(DOCKER_DAEMON_CONFIG_PATH)" --experimental=true --runtime="$(1)" -- $(RUNTIME_ARGS) --debug-log "$(RUNTIME_LOGS)" $(2) && \ sudo rm -rf "$(RUNTIME_LOG_DIR)" && mkdir -p "$(RUNTIME_LOG_DIR)" reload_docker = \ diff --git a/WORKSPACE b/WORKSPACE index 6c2b8230b..4d47d5164 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -100,6 +100,178 @@ go_download_sdk( patch = "//tools:go_types_memoize.patch", patch_strip = 1, version = "1.23.2", + # To update this dictionary, run (after updating VERSION): + # jq --arg VERSION "1.22.0" 'map(select(.version == "go" + $VERSION)).[0].files | map({(.os + "_" + .arch): [.filename, .sha256]}) | add' < <(curl 'https://go.dev/dl/?mode=json&include=all') + sdks = { + "_": [ + "go1.23.2.src.tar.gz", + "36930162a93df417d90bd22c6e14daff4705baac2b02418edda671cdfa9cd07f" + ], + "aix_ppc64": [ + "go1.23.2.aix-ppc64.tar.gz", + "560aff7fe1eeadc32248db35ed5c0a81e190d171b6ecec404cf46d808c13e92f" + ], + "darwin_amd64": [ + "go1.23.2.darwin-amd64.pkg", + "40c0b61971a1a74fd4566c536f682c9d4976fa71d40d9daabc875c06113d0fee" + ], + "darwin_arm64": [ + "go1.23.2.darwin-arm64.pkg", + "5611cd648f5100b73a7d6fd85589a481af18fdbaf9c153a92de9a8e39a6e061f" + ], + "dragonfly_amd64": [ + "go1.23.2.dragonfly-amd64.tar.gz", + "f163b99b03e4bbc64cd30363f1694a08fcd44094415db1f092f13f9d1bb7c28e" + ], + "freebsd_386": [ + "go1.23.2.freebsd-386.tar.gz", + "e4d9a1319dfdaa827407855e406c43e85c878a1f93f4f3984c85dce969c8bf70" + ], + "freebsd_amd64": [ + "go1.23.2.freebsd-amd64.tar.gz", + "025d77f1780906142023a364c31a572afd7d56d3a3be1e4e562e367ca88d3267" + ], + "freebsd_arm": [ + "go1.23.2.freebsd-arm.tar.gz", + "28af3c40687afdda6b33b300833b6d662716cc2d624fb9fd61a49bdad44cd869" + ], + "freebsd_arm64": [ + "go1.23.2.freebsd-arm64.tar.gz", + "f45af3e1434175ff85620a74c07fb41d6844655f1f2cd2389c5fca6de000f58c" + ], + "freebsd_riscv64": [ + "go1.23.2.freebsd-riscv64.tar.gz", + "f11b9b4d4a0679909202fc5e88093d6ff720a8a417bfe6a34d502c3862367039" + ], + "illumos_amd64": [ + "go1.23.2.illumos-amd64.tar.gz", + "d47e40366cd6c6b6ee14b811554cd7dde0351309f4a8a4569ec5ba2bd7689437" + ], + "linux_386": [ + "go1.23.2.linux-386.tar.gz", + "cb1ed4410f68d8be1156cee0a74fcfbdcd9bca377c83db3a9e1b07eebc6d71ef" + ], + "linux_amd64": [ + "go1.23.2.linux-amd64.tar.gz", + "542d3c1705f1c6a1c5a80d5dc62e2e45171af291e755d591c5e6531ef63b454e" + ], + "linux_arm64": [ + "go1.23.2.linux-arm64.tar.gz", + "f626cdd92fc21a88b31c1251f419c17782933a42903db87a174ce74eeecc66a9" + ], + "linux_armv6l": [ + "go1.23.2.linux-armv6l.tar.gz", + "e3286bdde186077e65e961cbe18874d42a461e5b9c472c26572b8d4a98d15c40" + ], + "linux_loong64": [ + "go1.23.2.linux-loong64.tar.gz", + "2293c5c3ffc595418308b4059ce214b99f0383cba83232e47a1a8c3b710c24e8" + ], + "linux_mips": [ + "go1.23.2.linux-mips.tar.gz", + "3bd1130a08195d23960b154d2e6eaa80ac7325ebd9d01d74c58b6d12580e6b12" + ], + "linux_mips64": [ + "go1.23.2.linux-mips64.tar.gz", + "a0cf25f236a0fa0a465816fe7f5c930f3b0b90c5c247b09c43a6adeff654e6ae" + ], + "linux_mips64le": [ + "go1.23.2.linux-mips64le.tar.gz", + "2283d12dfe7c8c8a46a41bbf7d11fe007434e7590cd1b89e221e478640b7ee3a" + ], + "linux_mipsle": [ + "go1.23.2.linux-mipsle.tar.gz", + "36b7228bae235eee6c8193f5a956e1a9a17874955affb86b3564709b0fab5874" + ], + "linux_ppc64": [ + "go1.23.2.linux-ppc64.tar.gz", + "791ca685ee5ca0f6fe849dc078145cb1323d0ea9dd308e9cca9ba2e7186dbb3d" + ], + "linux_ppc64le": [ + "go1.23.2.linux-ppc64le.tar.gz", + "c164ce7d894b10fd861d7d7b96f1dbea3f993663d9f0c30bc4f8ae3915db8b0c" + ], + "linux_riscv64": [ + "go1.23.2.linux-riscv64.tar.gz", + "ea8ab49c5c04c9f94a3f4894d1b030fbce8d10413905fa399f6c39c0a44d5556" + ], + "linux_s390x": [ + "go1.23.2.linux-s390x.tar.gz", + "de1f94d7dd3548ba3036de1ea97eb8243881c22a88fcc04cc08c704ded769e02" + ], + "netbsd_386": [ + "go1.23.2.netbsd-386.tar.gz", + "c4ae1087dce4daf45a837f5fca36ac0e29a02ada9addf857f1c426e60bce6f21" + ], + "netbsd_amd64": [ + "go1.23.2.netbsd-amd64.tar.gz", + "2734a5b54905cea45f136c28249e626d0241b865b0637fa1db64bf533d9d843e" + ], + "netbsd_arm": [ + "go1.23.2.netbsd-arm.tar.gz", + "69b31edcd3d4f7d8bbf9aee2b25cafba30b444ef19bc7a033e15026f7d0cc5c2" + ], + "netbsd_arm64": [ + "go1.23.2.netbsd-arm64.tar.gz", + "3e80b943d70c7e1633822b42c1aa7234e61da14f13ff8efff7ee6e1347f37648" + ], + "openbsd_386": [ + "go1.23.2.openbsd-386.tar.gz", + "d1fde255843fec1f7f0611d468effd98e1f4309f589ac13037db07b032f9da35" + ], + "openbsd_amd64": [ + "go1.23.2.openbsd-amd64.tar.gz", + "8734c7cd464a0620f6605bd3f9256bed062f262d0d58e4f45099c329a08ed966" + ], + "openbsd_arm": [ + "go1.23.2.openbsd-arm.tar.gz", + "980ceb889915695d94b166ca1300250dba76fa37a2d41eca2c5e7727dcb4fb7f" + ], + "openbsd_arm64": [ + "go1.23.2.openbsd-arm64.tar.gz", + "23b93144e754bbcf5eda700e9decbdbd44d29ceedb1bf1de75f95e8a6ea986bb" + ], + "openbsd_ppc64": [ + "go1.23.2.openbsd-ppc64.tar.gz", + "a13cc0d621af4f35afd90b886c60b1bf66f771939d226dc36fa61a337d90eb30" + ], + "openbsd_riscv64": [ + "go1.23.2.openbsd-riscv64.tar.gz", + "3bf66879b38a233c5cbb5d2eb982004117f05d6bf06279e886e087d7c504427d" + ], + "plan9_386": [ + "go1.23.2.plan9-386.tar.gz", + "695aac64532da8d9a243601ffa0411cd763be891fcf7fd2e857eea4ab10b8bcc" + ], + "plan9_amd64": [ + "go1.23.2.plan9-amd64.tar.gz", + "86b5de91fdf7bd9b52c77c62f8762518cf3fc256fe912af9bbff1d073054aa5b" + ], + "plan9_arm": [ + "go1.23.2.plan9-arm.tar.gz", + "6ffa4ac1f4368a3121a032917577a4e0a3feaf696c3e98f213b74ac04c318bc4" + ], + "solaris_amd64": [ + "go1.23.2.solaris-amd64.tar.gz", + "fa70d39ddeb6b55241a30b48d7af4e681c6a7d7104e8326c3bc1b12a75e091cc" + ], + "windows_386": [ + "go1.23.2.windows-386.msi", + "72a6def70300cc804c70073d8b579603d9b39b39b02b3b5d340968d9e7e0e9d4" + ], + "windows_amd64": [ + "go1.23.2.windows-amd64.msi", + "c80cbc5e66d6fb8b0c3300b0dda1fe925c429e199954d3327da2933d9870b041" + ], + "windows_arm": [ + "go1.23.2.windows-arm.msi", + "367d522b47c7ce7761a671efcb8b12c8af8f509db1cd6160c91f410ef3201987" + ], + "windows_arm64": [ + "go1.23.2.windows-arm64.msi", + "0edd985dbd6de64d9c88dbc8835bae21203c58444bf26fce0739cbec4eb1b610" + ] + }, ) gazelle_dependencies() diff --git a/pkg/test/dockerutil/dockerutil.go b/pkg/test/dockerutil/dockerutil.go index b0dddb514..8f8e1d610 100644 --- a/pkg/test/dockerutil/dockerutil.go +++ b/pkg/test/dockerutil/dockerutil.go @@ -40,6 +40,9 @@ var ( // default used by the installations. runtime = flag.String("runtime", os.Getenv("RUNTIME"), "specify which runtime to use") + // dockerCLI is the path to the docker CLI binary. + dockerCLI = flag.String("docker_cli", os.Getenv("DOCKER_CLI_PATH"), "path to the docker client command-line binary") + // config is the default Docker daemon configuration path. config = flag.String("config_path", "/etc/docker/daemon.json", "configuration file for reading paths") @@ -65,6 +68,14 @@ var ( useSystemdRgx = regexp.MustCompile("\\s*(native\\.cgroupdriver)\\s*=\\s*(systemd)\\s*") ) +// dockerCLIPath returns the path to the docker CLI binary. +func dockerCLIPath() string { + if *dockerCLI != "" { + return *dockerCLI + } + return "docker" +} + // PrintDockerConfig prints the whole Docker configuration file to the log. func PrintDockerConfig() { configBytes, err := os.ReadFile(*config) @@ -78,10 +89,10 @@ func PrintDockerConfig() { // // This logs directly to stderr, as it is typically called from a Main wrapper. func EnsureSupportedDockerVersion() { - cmd := exec.Command("docker", "version") + cmd := exec.Command(dockerCLIPath(), "version") out, err := cmd.CombinedOutput() if err != nil { - log.Fatalf("error running %q: %v", "docker version", err) + log.Fatalf("error running %q: %v", dockerCLIPath()+" version", err) } re := regexp.MustCompile(`Version:\s+(\d+)\.(\d+)\.\d.*`) matches := re.FindStringSubmatch(string(out)) @@ -97,10 +108,10 @@ func EnsureSupportedDockerVersion() { // EnsureDockerExperimentalEnabled ensures that Docker has experimental features enabled. func EnsureDockerExperimentalEnabled() { - cmd := exec.Command("docker", "version", "--format={{.Server.Experimental}}") + cmd := exec.Command(dockerCLIPath(), "version", "--format={{.Server.Experimental}}") out, err := cmd.CombinedOutput() if err != nil { - log.Fatalf("error running %s: %v", "docker version --format='{{.Server.Experimental}}'", err) + log.Fatalf("error running %s: %v", dockerCLIPath()+" version --format='{{.Server.Experimental}}'", err) } if strings.TrimSpace(string(out)) != "true" { PrintDockerConfig() @@ -239,7 +250,7 @@ func runtimeMap() (map[string]any, error) { // // This is called by criutil in order to import imports. func Save(logger testutil.Logger, image string, w io.Writer) error { - cmd := testutil.Command(logger, "docker", "save", testutil.ImageByName(image)) + cmd := testutil.Command(logger, dockerCLIPath(), "save", testutil.ImageByName(image)) cmd.Stdout = w // Send directly to the writer. return cmd.Run() } diff --git a/tools/bazel.mk b/tools/bazel.mk index 41816cdb2..877377f51 100644 --- a/tools/bazel.mk +++ b/tools/bazel.mk @@ -26,6 +26,7 @@ ## DOCKER_NAME - The container name (default: gvisor-bazel-HASH). ## DOCKER_HOSTNAME - The container name (default: same as DOCKER_NAME). ## DOCKER_PRIVILEGED - Docker privileged flags (default: --privileged). +## DOCKER_CLI_PATH - The path to the docker CLI binary. ## UNSANDBOXED_RUNTIME - Name of the Docker runtime to use for the ## unsandboxed build container. Defaults to runc. ## PRE_BAZEL_INIT - If set, run this command with bash outside the Bazel @@ -59,13 +60,24 @@ DOCKER_NAME := gvisor-bazel-$(HASH)-$(ARCH) DOCKER_HOSTNAME := $(DOCKER_NAME) DOCKER_PRIVILEGED := --privileged UNSANDBOXED_RUNTIME ?= runc -BAZEL_CACHE := $(HOME)/.cache/bazel/ +BAZEL_CACHE ?= $(HOME)/.cache/bazel/ GCLOUD_CONFIG := $(HOME)/.config/gcloud/ -DOCKER_SOCKET := /var/run/docker.sock -DOCKER_CONFIG := /etc/docker +DOCKER_HOST ?= unix:///var/run/docker.sock +DOCKER_SOCKET ?= $(patsubst unix://%,%,$(DOCKER_HOST)) +DOCKER_CONFIG ?= /etc/docker +DOCKER_CLI_PATH ?= docker DEVICE_FILE ?= PRE_BAZEL_INIT ?= +# If `GO_REPOSITORY_USE_HOST_CACHE` is set to `1`, Go environment variables +# are passed through to the build container and work as they do with +# regular Bazel. See documentation on this feature here: +# https://github.com/bazelbuild/bazel-gazelle/blob/089096315dcaa0aea52e87ecc2bd6b89b531da1e/repository.md?plain=1#L117 +GO_REPOSITORY_USE_HOST_CACHE ?= +GOPATH ?= +GOCACHE ?= +GOMODCACHE ?= + ## ## Bazel helpers. ## @@ -94,23 +106,44 @@ TEST_OPTIONS += $(BASE_OPTIONS) \ UID := $(shell id -u ${USER}) GID := $(shell id -g ${USER}) USERADD_OPTIONS := -DOCKER_RUN_OPTIONS := -DOCKER_RUN_OPTIONS += --rm -DOCKER_RUN_OPTIONS += --user $(UID):$(GID) -DOCKER_RUN_OPTIONS += --entrypoint "" -DOCKER_RUN_OPTIONS += --init +DOCKER_RUN_OPTIONS := +DOCKER_RUN_OPTIONS += --rm +DOCKER_RUN_OPTIONS += --user $(UID):$(GID) +DOCKER_RUN_OPTIONS += --entrypoint "" +DOCKER_RUN_OPTIONS += --init +DOCKER_EXEC_OPTIONS := --user $(UID):$(GID) ifneq (,$(UNSANDBOXED_RUNTIME)) DOCKER_RUN_OPTIONS += --runtime=$(UNSANDBOXED_RUNTIME) endif DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(BAZEL_CACHE)):$(BAZEL_CACHE)" +ifneq ($(patsubst %/,%,$(BAZEL_CACHE)),$(HOME)/.cache/bazel) +DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(BAZEL_CACHE)):$(HOME)/.cache/bazel" +endif +ifneq ($(GO_REPOSITORY_USE_HOST_CACHE),) +DOCKER_RUN_OPTIONS += -e GO_REPOSITORY_USE_HOST_CACHE=$(GO_REPOSITORY_USE_HOST_CACHE) +DOCKER_EXEC_OPTIONS += -e GO_REPOSITORY_USE_HOST_CACHE=$(GO_REPOSITORY_USE_HOST_CACHE) +ifneq ($(GOPATH),) +DOCKER_RUN_OPTIONS += -e GOPATH=$(GOPATH) +DOCKER_EXEC_OPTIONS += -e GOPATH=$(GOPATH) +DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(GOPATH)):$(GOPATH)" +endif +ifneq ($(GOCACHE),) +DOCKER_RUN_OPTIONS += -e GOCACHE=$(GOCACHE) +DOCKER_EXEC_OPTIONS += -e GOCACHE=$(GOCACHE) +DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(GOCACHE)):$(GOCACHE)" +endif +ifneq ($(GOMODCACHE),) +DOCKER_RUN_OPTIONS += -e GOMODCACHE=$(GOMODCACHE) +DOCKER_EXEC_OPTIONS += -e GOMODCACHE=$(GOMODCACHE) +DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(GOMODCACHE)):$(GOMODCACHE)" +endif +endif DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(GCLOUD_CONFIG)):$(GCLOUD_CONFIG)" DOCKER_RUN_OPTIONS += -v "/tmp:/tmp" -DOCKER_EXEC_OPTIONS := --user $(UID):$(GID) DOCKER_EXEC_OPTIONS += --interactive ifeq (true,$(shell test -t 1 && echo true)) DOCKER_EXEC_OPTIONS += --tty endif - # If kernel headers are available, mount them too. ifneq (,$(wildcard /lib/modules)) DOCKER_RUN_OPTIONS += -v "/lib/modules:/lib/modules" @@ -177,7 +210,7 @@ endif # Check if Docker API version supports cgroupns (supported in >=1.41). # If not, don't include it in options. ifeq ($(DOCKER_BUILD),true) -DOCKER_API_VERSION := $(shell docker version --format='{{.Server.APIVersion}}') +DOCKER_API_VERSION := $(shell $(DOCKER_CLI_PATH) version --format='{{.Server.APIVersion}}') ifeq ($(shell echo $(DOCKER_API_VERSION) | tr '.' '\n' | wc -l),2) ifeq ($(shell test $(shell echo $(DOCKER_API_VERSION) | cut -d. -f1) -gt 1 && echo true),true) DOCKER_RUN_OPTIONS += --cgroupns=host @@ -211,10 +244,10 @@ ifeq ($(DOCKER_BUILD),true) # already having been terminated. So this uses multiple ways to try to get the # container to exit, and ignores which ones work and which ones don't. # Instead, it just checks that the container no longer exists by the end of it. - @timeout --signal=KILL 10s docker wait $(DOCKER_NAME) 2>/dev/null || true - @docker stop --time=10 $(DOCKER_NAME) 2>/dev/null || true + @timeout --signal=KILL 10s $(DOCKER_CLI_PATH) wait $(DOCKER_NAME) 2>/dev/null || true + @$(DOCKER_CLI_PATH) stop --time=10 $(DOCKER_NAME) 2>/dev/null || true # Double check that the container isn't running. - @bash -c "! docker inspect $(DOCKER_NAME) &>/dev/null" + @bash -c "! $(DOCKER_CLI_PATH) inspect $(DOCKER_NAME) &>/dev/null" endif .PHONY: bazel-shutdown @@ -224,13 +257,13 @@ bazel-alias: ## Emits an alias that can be used within the shell. bazel-image: load-default ## Ensures that the local builder exists. @$(call header,DOCKER BUILD) - @docker rm -f $(BUILDER_NAME) 2>/dev/null || true - @docker run --user 0:0 --entrypoint "" \ + @$(DOCKER_CLI_PATH) rm -f $(BUILDER_NAME) 2>/dev/null || true + @$(DOCKER_CLI_PATH) run --user 0:0 --entrypoint "" \ --name $(BUILDER_NAME) --hostname $(BUILDER_HOSTNAME) \ $(shell test -n "$(UNSANDBOXED_RUNTIME)" && echo "--runtime=$(UNSANDBOXED_RUNTIME)") \ gvisor.dev/images/default \ bash -c "$(GROUPADD_DOCKER) $(USERADD_DOCKER) if test -e /dev/kvm; then chmod a+rw /dev/kvm; fi" >&2 - @docker commit $(BUILDER_NAME) gvisor.dev/images/builder >&2 + @$(DOCKER_CLI_PATH) commit $(BUILDER_NAME) gvisor.dev/images/builder >&2 .PHONY: bazel-image ifneq (true,$(shell $(wrapper echo true))) @@ -240,10 +273,12 @@ ifneq (,$(PRE_BAZEL_INIT)) @bash -euxo pipefail -c "$(PRE_BAZEL_INIT)" endif @$(call header,DOCKER RUN) - @docker rm -f $(DOCKER_NAME) 2>/dev/null || true - @mkdir -p $(BAZEL_CACHE) - @mkdir -p $(GCLOUD_CONFIG) - @docker run -d --name $(DOCKER_NAME) --hostname $(DOCKER_HOSTNAME) \ + @set -x + @$(DOCKER_CLI_PATH) rm -f $(DOCKER_NAME) 2>/dev/null || true + @mkdir -p "$(BAZEL_CACHE)" + @mkdir -p "$(GCLOUD_CONFIG)" + @$(DOCKER_CLI_PATH) run -d \ + --name $(DOCKER_NAME) --hostname $(DOCKER_HOSTNAME) \ -v "$(CURDIR):$(CURDIR)" \ --workdir "$(CURDIR)" \ --pid=host \ @@ -266,16 +301,18 @@ bazel-server-inc: bazel-server build_paths = \ (set -euo pipefail; \ $(call wrapper,$(BAZEL) build $(BASE_OPTIONS) $(BAZEL_OPTIONS) $(1)) && \ - $(call wrapper,$(BAZEL) cquery $(BASE_OPTIONS) $(BAZEL_OPTIONS) $(1) --output=starlark --starlark:file=tools/show_paths.bzl) \ + $(call wrapper,$(BAZEL) cquery $(BASE_OPTIONS) $(BAZEL_OPTIONS) --output=starlark --starlark:file=tools/show_paths.bzl $(1)) \ + | $(call wrapper,xargs -r -I {} bash -c 'test -e "{}" || exit 0; realpath -m "{}"') \ + | sed 's~^$(HOME)/\.cache/bazel/~$(patsubst %/,%,$(BAZEL_CACHE))/~' \ | xargs -r -I {} bash -c 'test -e "{}" || exit 0; realpath -m "{}"' \ | xargs -r -I {} bash -c 'set -euo pipefail; $(2)') -clean = $(call header,CLEAN) && $(call wrapper,$(BAZEL) clean) -build = $(call header,BUILD $(1)) && $(call build_paths,$(1),echo {}) -copy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),cp -fa {} $(2)) -run = $(call header,RUN $(1) $(2)) && $(call build_paths,$(1),{} $(2)) -sudo = $(call header,SUDO $(1) $(2)) && $(call build_paths,$(1),sudo -E {} $(2)) -test = $(call header,TEST $(1)) && $(call wrapper,$(BAZEL) test --strip=never $(BAZEL_OPTIONS) $(TEST_OPTIONS) $(1)) +clean = $(call header,CLEAN) && $(call wrapper,$(BAZEL) clean) +build = $(call header,BUILD $(1)) && $(call build_paths,$(1),echo {}) +copy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),cp -fa {} $(2)) +run = $(call header,RUN $(1) $(2)) && $(call build_paths,$(1),{} $(2)) +sudo = $(call header,SUDO $(1) $(2)) && $(call build_paths,$(1),sudo -E {} $(2)) +test = $(call header,TEST $(1)) && $(call wrapper,$(BAZEL) test --strip=never $(BAZEL_OPTIONS) $(TEST_OPTIONS) $(1)) clean: ## Cleans the bazel cache. @$(call clean) diff --git a/tools/images.mk b/tools/images.mk index 6607b4ab4..4b51a2870 100644 --- a/tools/images.mk +++ b/tools/images.mk @@ -48,6 +48,10 @@ TEST_IMAGES := $(subst /,_,$(subst images/,,$(shell find images/ -name D SUB_IMAGES := $(foreach image,$(ALL_IMAGES),$(if $(findstring _,$(image)),$(image),)) IMAGE_GROUPS := $(sort $(foreach image,$(SUB_IMAGES),$(firstword $(subst _, ,$(image))))) +# If set to 'true', will skip loading any image from remote. +# This will only work if local images already exist in Docker. +SKIP_IMAGE_LOAD ?= + define expand_group = load-$(1): $$(patsubst $(1)_%, load-$(1)_%, $$(filter $(1)_%,$$(ALL_IMAGES))) @ @@ -129,6 +133,8 @@ local_tag = \ docker tag $(call remote_image,$(1)):$(call tag,$(1)) $(call local_image,$(1)):$(call tag,$(1)) >&2 latest_tag = \ docker tag $(call local_image,$(1)):$(call tag,$(1)) $(call local_image,$(1)):latest >&2 +tag_exists = \ + docker image inspect $(call local_image,$(1)):$(call tag,$(1)) &>/dev/null tag-%: ## Tag a local image. @$(call header,TAG $*) @$(call local_tag,$*) && $(call latest_tag,$*) @@ -170,7 +176,14 @@ rebuild-%: register-cross ## Force rebuild an image locally. # If the image is not available for the current architecture, it is not loaded. load-%: register-cross ## Pull or build an image locally. @if [ -f "$(call path,$*)/$(call dockerfile,$*)" ]; then \ - ($(call pull,$*)) || ($(call rebuild,$*)); \ + if [ "$(SKIP_IMAGE_LOAD)" == true ]; then \ + if ! $(call tag_exists,$*); then \ + echo "Image $* does not exist locally and SKIP_IMAGE_LOAD is set so cannot pull it. Failing." >&2; \ + exit 1; \ + fi; \ + else \ + ($(call pull,$*)) || ($(call rebuild,$*)); \ + fi; \ else \ echo "Image $* is not available on $$(uname -m), ignoring it." >&2; \ fi @@ -178,6 +191,9 @@ load-%: register-cross ## Pull or build an image locally. test-%: register-cross ## Build an image locally if the remote doesn't exist. @($(call image_manifest,$*)) >&2 || ($(call rebuild,$*)) +local-image-%: register-cross ## Print current 'image:tag' for a local image. + echo "$(call local_image,$*):$(call tag,$*)" + # push pushes the remote image, after validating that the tag doesn't exist # yet. Note that this generic rule will match the fully-expanded remote image # tag.