From 2b97d38657de0e04a4a50473c4b704f9b5765a41 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Tue, 28 Jun 2022 18:55:27 -0700 Subject: [PATCH] make gofer directory cache size configurable Passing --dcache=N causes all gofer mounts to use a global dirent cache of size N. PiperOrigin-RevId: 457867503 --- Makefile | 6 +- pkg/sentry/fsimpl/gofer/gofer.go | 21 ++++- runsc/boot/vfs.go | 3 + runsc/config/config.go | 4 + runsc/config/flags.go | 1 + test/e2e/BUILD | 20 +++++ test/e2e/integration_runtime_test.go | 117 +++++++++++++++++++++++++++ test/e2e/integration_test.go | 33 -------- 8 files changed, 169 insertions(+), 36 deletions(-) create mode 100644 test/e2e/integration_runtime_test.go diff --git a/Makefile b/Makefile index 7d3ac31bb..2c3a397ce 100644 --- a/Makefile +++ b/Makefile @@ -268,10 +268,12 @@ docker-tests: load-basic $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME),) # Clear flags. # Used by TestRlimitNoFile. @$(call install_runtime,$(RUNTIME)-fdlimit,--fdlimit=2000) - @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS)) + @$(call install_runtime,$(RUNTIME)-dcache,--fdlimit=2000 --dcache=100) + @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test) @$(call install_runtime,$(RUNTIME), --lisafs) # Run again with lisafs. @$(call install_runtime,$(RUNTIME)-fdlimit,--lisafs --fdlimit=2000) - @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS)) + @$(call install_runtime,$(RUNTIME)-dcache,--lisafs --fdlimit=2000 --dcache=100) + @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test) .PHONY: docker-tests overlay-tests: load-basic $(RUNTIME_BIN) diff --git a/pkg/sentry/fsimpl/gofer/gofer.go b/pkg/sentry/fsimpl/gofer/gofer.go index 8eb7f1374..b08d759dc 100644 --- a/pkg/sentry/fsimpl/gofer/gofer.go +++ b/pkg/sentry/fsimpl/gofer/gofer.go @@ -111,6 +111,16 @@ type dentryCache struct { maxCachedDentries uint64 } +// SetDentryCacheSize sets the size of the global gofer dentry cache. +func SetDentryCacheSize(size int) { + globalDentryCache.mu.Lock() + defer globalDentryCache.mu.Unlock() + globalDentryCache.maxCachedDentries = uint64(size) +} + +// globalDentryCache is a global cache of dentries across all gofers. +var globalDentryCache dentryCache + // Valid values for "trans" mount option. const transportModeFD = "fd" @@ -481,8 +491,17 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt specialFileFDs: make(map[*specialFileFD]struct{}), inoByQIDPath: make(map[uint64]uint64), inoByKey: make(map[inoKey]uint64), - dentryCache: &dentryCache{maxCachedDentries: defaultMaxCachedDentries}, } + + // Did the user configure a global dentry cache? + globalDentryCache.mu.Lock() + if globalDentryCache.maxCachedDentries >= 1 { + fs.dentryCache = &globalDentryCache + } else { + fs.dentryCache = &dentryCache{maxCachedDentries: defaultMaxCachedDentries} + } + globalDentryCache.mu.Unlock() + fs.vfsfs.Init(vfsObj, &fstype, fs) if err := fs.initClientAndRoot(ctx); err != nil { diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 9ec78c3ed..69f8519eb 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -409,6 +409,9 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi // Options field). So assume root is always on top of overlayfs. data = append(data, "overlayfs_stale_read") + // Configure the gofer dentry cache size. + gofer.SetDentryCacheSize(conf.DCache) + log.Infof("Mounting root over 9P, ioFD: %d", fd) opts := &vfs.MountOptions{ ReadOnly: c.root.Readonly, diff --git a/runsc/config/config.go b/runsc/config/config.go index 3dc55ba7d..d99b56ab2 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -234,6 +234,10 @@ type Config struct { // each. FDLimit int `flag:"fdlimit"` + // DCache sets the global dirent cache size. If zero, per-mount caches are + // used. + DCache int `flag:"dcache"` + // TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in // tests. It allows runsc to start the sandbox process as the current // user, and without chrooting the sandbox process. This can be diff --git a/runsc/config/flags.go b/runsc/config/flags.go index d9a25da29..794e89cbf 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -85,6 +85,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("cgroupfs", false, "Automatically mount cgroupfs.") flagSet.Bool("ignore-cgroups", false, "don't configure cgroups.") flagSet.Int("fdlimit", -1, "Specifies a limit on the number of host file descriptors that can be open. Applies separately to the sentry and gofer. Note: each file in the sandbox holds more than one host FD open.") + flagSet.Int("dcache", 0, "Set the global gofer direct cache size. This acts as a coarse-grained control on the number of host FDs simultaneously open by the sentry. If zero, per-mount caches are used.") // Flags that control sandbox runtime behavior: network related. flagSet.Var(networkTypePtr(NetworkSandbox), "network", "specifies which network to use: sandbox (default), host, none. Using network inside the sandbox is more secure because it's isolated from the host network.") diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 1e9792b4f..d9f71f838 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -26,6 +26,26 @@ go_test( ], ) +go_test( + name = "integration_runtime_test", + size = "large", + srcs = [ + "integration_runtime_test.go", + ], + library = ":integration", + tags = [ + # Requires docker and runsc to be configured before the test runs. + "local", + "manual", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//pkg/test/testutil", + "@com_github_docker_docker//api/types/mount:go_default_library", + ], +) + go_library( name = "integration", srcs = ["integration.go"], diff --git a/test/e2e/integration_runtime_test.go b/test/e2e/integration_runtime_test.go new file mode 100644 index 000000000..e00eeea3e --- /dev/null +++ b/test/e2e/integration_runtime_test.go @@ -0,0 +1,117 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package integration provides end-to-end integration tests for runsc. +// +// Each test calls docker commands to start up a container, and tests that it is +// behaving properly, with various runsc commands. The container is killed and +// deleted at the end. +// +// Setup instruction in test/README.md. +package integration + +import ( + "context" + "flag" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types/mount" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +const ( + // defaultWait is the default wait time used for tests. + defaultWait = time.Minute + + memInfoCmd = "cat /proc/meminfo | grep MemTotal: | awk '{print $2}'" +) + +func TestMain(m *testing.M) { + dockerutil.EnsureSupportedDockerVersion() + flag.Parse() + os.Exit(m.Run()) +} + +func TestRlimitNoFile(t *testing.T) { + ctx := context.Background() + d := dockerutil.MakeContainerWithRuntime(ctx, t, "-fdlimit") + defer d.CleanUp(ctx) + + // Create a directory with a bunch of files. + const nfiles = 5000 + tmpDir := testutil.TmpDir() + for i := 0; i < nfiles; i++ { + if _, err := ioutil.TempFile(tmpDir, "tmp"); err != nil { + t.Fatalf("TempFile(): %v", err) + } + } + + // Run the container. Open a bunch of files simutaneously and sleep a bit + // to give time for everything to start. We should hit the FD limit and + // fail rather than waiting the full sleep duration. + cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 60` + got, err := d.Run(ctx, dockerutil.RunOpts{ + Image: "basic/ubuntu", + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: tmpDir, + Target: "/tmp/foo", + }, + }, + }, "bash", "-c", cmd) + if err == nil { + t.Fatalf("docker run didn't fail: %s", got) + } else if strings.Contains(err.Error(), "Unknown runtime specified") { + t.Fatalf("docker failed because -fdlimit runtime was not installed") + } +} + +func TestDentryCacheLimit(t *testing.T) { + ctx := context.Background() + d := dockerutil.MakeContainerWithRuntime(ctx, t, "-dcache") + defer d.CleanUp(ctx) + + // Create a directory with a bunch of files. + const nfiles = 5000 + tmpDir := testutil.TmpDir() + for i := 0; i < nfiles; i++ { + if _, err := ioutil.TempFile(tmpDir, "tmp"); err != nil { + t.Fatalf("TempFile(): %v", err) + } + } + + // Run the container. Open a bunch of files simutaneously and sleep a bit + // to give time for everything to start. We shouldn't hit the FD limit + // because the dentry cache is small. + cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 10` + got, err := d.Run(ctx, dockerutil.RunOpts{ + Image: "basic/ubuntu", + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: tmpDir, + Target: "/tmp/foo", + }, + }, + }, "bash", "-c", cmd) + if err != nil { + t.Fatalf("docker failed: %v, %s", err, got) + } +} diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go index 81f520a9f..8caf32eb8 100644 --- a/test/e2e/integration_test.go +++ b/test/e2e/integration_test.go @@ -997,36 +997,3 @@ func TestNonSearchableWorkingDirectory(t *testing.T) { t.Errorf("ls error message not found, want: %q, got: %q", wantErrorMsg, got) } } - -func TestRlimitNoFile(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainerWithRuntime(ctx, t, "-fdlimit") - defer d.CleanUp(ctx) - - // Create a directory with a bunch of files. - const nfiles = 5000 - tmpDir := testutil.TmpDir() - for i := 0; i < nfiles; i++ { - if _, err := ioutil.TempFile(tmpDir, "tmp"); err != nil { - t.Fatalf("TempFile(): %v", err) - } - } - - // Run the container. Open a bunch of files simutaneously and sleep a bit - // to give time for everything to start. We should hit the FD limit and - // fail rather than waiting the full sleep duration. - cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 60` - got, err := d.Run(ctx, dockerutil.RunOpts{ - Image: "basic/ubuntu", - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: tmpDir, - Target: "/tmp/foo", - }, - }, - }, "bash", "-c", cmd) - if err == nil { - t.Fatalf("docker run didn't fail: %s", got) - } -}