From 37d64d348f5d81aedf6fdfcd2931f64a9a83cece Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Thu, 13 Oct 2022 14:32:49 -0700 Subject: [PATCH] Do not mount /tmp into bazel container with shared mount propagation. This is the same issue as described in moby/moby#36179. If any of the other volumes mounted in the bazel container happen to be inside /tmp, then they will be propagated to the host mount namespace, and hence never cleaned up by docker on container destroy. What's worse is that the number of leaked mounts increases exponentially. See this reproducer: ```shell $ mkdir /tmp/gvisor $ docker run --rm -d --name=repro -v /tmp/gvisor:/tmp/gvisor -v "/tmp:/tmp:shared" ubuntu:focal /bin/sleep infinity 5690703c0f6128eebb38180c1bbb5b9ba83018affa284bc82d78a52860321f97 $ docker kill repro repro $ sudo cat /proc/self/mountinfo | grep gvisor 1914 29 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version $ docker run --rm -d --name=repro -v /tmp/gvisor:/tmp/gvisor -v "/tmp:/tmp:shared" ubuntu:focal /bin/sleep infinity 11a7cfba501277e2b61fafa46c782fbd5829629dab91ff8ca0c589bdb1d63b24 $ docker kill repro repro $ sudo cat /proc/self/mountinfo | grep gvisor 1914 1946 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 1950 1914 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 1946 29 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version $ docker run --rm -d --name=repro -v /tmp/gvisor:/tmp/gvisor -v "/tmp:/tmp:shared" ubuntu:focal /bin/sleep infinity 11a7cfba501277e2b61fafa46c782fbd5829629dab91ff8ca0c589bdb1d63b24 $ docker kill repro repro $ sudo cat /proc/self/mountinfo | grep gvisor 1914 2018 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 1950 2014 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 1946 2010 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 2022 1950 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 2018 1946 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 2014 1914 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version 2010 29 254:1 /tmp/gvisor /tmp/gvisor rw,relatime shared:1 - ext4 /dev/mapper/glinux_20210409-root rw,errors=remount-ro,i_version ``` Eventually, docker becomes prohibitively slow and starts throwing ENOSPC errors randomly. PiperOrigin-RevId: 480986037 --- tools/bazel.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bazel.mk b/tools/bazel.mk index f02100e40..67da52e4b 100644 --- a/tools/bazel.mk +++ b/tools/bazel.mk @@ -89,9 +89,9 @@ DOCKER_RUN_OPTIONS += --rm DOCKER_RUN_OPTIONS += --user $(UID):$(GID) DOCKER_RUN_OPTIONS += --entrypoint "" DOCKER_RUN_OPTIONS += --init -DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(BAZEL_CACHE)):$(BAZEL_CACHE):shared" +DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(BAZEL_CACHE)):$(BAZEL_CACHE)" DOCKER_RUN_OPTIONS += -v "$(shell realpath -m $(GCLOUD_CONFIG)):$(GCLOUD_CONFIG)" -DOCKER_RUN_OPTIONS += -v "/tmp:/tmp:shared" +DOCKER_RUN_OPTIONS += -v "/tmp:/tmp" DOCKER_EXEC_OPTIONS := --user $(UID):$(GID) DOCKER_EXEC_OPTIONS += --interactive ifeq (true,$(shell test -t 1 && echo true))