make gofer directory cache size configurable

Passing --dcache=N causes all gofer mounts to use a global dirent cache of size
N.

PiperOrigin-RevId: 457867503

Makefile

@@ -268,10 +268,12 @@ docker-tests: load-basic $(RUNTIME_BIN)
 	@$(call install_runtime,$(RUNTIME),) # Clear flags.
 	# Used by TestRlimitNoFile.
 	@$(call install_runtime,$(RUNTIME)-fdlimit,--fdlimit=2000)
-	@$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS))
+	@$(call install_runtime,$(RUNTIME)-dcache,--fdlimit=2000 --dcache=100)
+	@$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test)
 	@$(call install_runtime,$(RUNTIME), --lisafs) # Run again with lisafs.
 	@$(call install_runtime,$(RUNTIME)-fdlimit,--lisafs --fdlimit=2000)
-	@$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS))
+	@$(call install_runtime,$(RUNTIME)-dcache,--lisafs --fdlimit=2000 --dcache=100)
+	@$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test)
 .PHONY: docker-tests
 
 overlay-tests: load-basic $(RUNTIME_BIN)

pkg/sentry/fsimpl/gofer/gofer.go

@@ -111,6 +111,16 @@ type dentryCache struct {
 	maxCachedDentries uint64
 }
 
+// SetDentryCacheSize sets the size of the global gofer dentry cache.
+func SetDentryCacheSize(size int) {
+	globalDentryCache.mu.Lock()
+	defer globalDentryCache.mu.Unlock()
+	globalDentryCache.maxCachedDentries = uint64(size)
+}
+
+// globalDentryCache is a global cache of dentries across all gofers.
+var globalDentryCache dentryCache
+
 // Valid values for "trans" mount option.
 const transportModeFD = "fd"
 
@@ -481,8 +491,17 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 		specialFileFDs:   make(map[*specialFileFD]struct{}),
 		inoByQIDPath:     make(map[uint64]uint64),
 		inoByKey:         make(map[inoKey]uint64),
-		dentryCache:      &dentryCache{maxCachedDentries: defaultMaxCachedDentries},
 	}
+
+	// Did the user configure a global dentry cache?
+	globalDentryCache.mu.Lock()
+	if globalDentryCache.maxCachedDentries >= 1 {
+		fs.dentryCache = &globalDentryCache
+	} else {
+		fs.dentryCache = &dentryCache{maxCachedDentries: defaultMaxCachedDentries}
+	}
+	globalDentryCache.mu.Unlock()
+
 	fs.vfsfs.Init(vfsObj, &fstype, fs)
 
 	if err := fs.initClientAndRoot(ctx); err != nil {

runsc/boot/vfs.go

@@ -409,6 +409,9 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	// Options field). So assume root is always on top of overlayfs.
 	data = append(data, "overlayfs_stale_read")
 
+	// Configure the gofer dentry cache size.
+	gofer.SetDentryCacheSize(conf.DCache)
+
 	log.Infof("Mounting root over 9P, ioFD: %d", fd)
 	opts := &vfs.MountOptions{
 		ReadOnly: c.root.Readonly,

runsc/config/config.go

@@ -234,6 +234,10 @@ type Config struct {
 	// each.
 	FDLimit int `flag:"fdlimit"`
 
+	// DCache sets the global dirent cache size. If zero, per-mount caches are
+	// used.
+	DCache int `flag:"dcache"`
+
 	// TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in
 	// tests. It allows runsc to start the sandbox process as the current
 	// user, and without chrooting the sandbox process. This can be

runsc/config/flags.go

@@ -85,6 +85,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.Bool("cgroupfs", false, "Automatically mount cgroupfs.")
 	flagSet.Bool("ignore-cgroups", false, "don't configure cgroups.")
 	flagSet.Int("fdlimit", -1, "Specifies a limit on the number of host file descriptors that can be open. Applies separately to the sentry and gofer. Note: each file in the sandbox holds more than one host FD open.")
+	flagSet.Int("dcache", 0, "Set the global gofer direct cache size. This acts as a coarse-grained control on the number of host FDs simultaneously open by the sentry. If zero, per-mount caches are used.")
 
 	// Flags that control sandbox runtime behavior: network related.
 	flagSet.Var(networkTypePtr(NetworkSandbox), "network", "specifies which network to use: sandbox (default), host, none. Using network inside the sandbox is more secure because it's isolated from the host network.")

test/e2e/BUILD

@@ -26,6 +26,26 @@ go_test(
     ],
 )
 
+go_test(
+    name = "integration_runtime_test",
+    size = "large",
+    srcs = [
+        "integration_runtime_test.go",
+    ],
+    library = ":integration",
+    tags = [
+        # Requires docker and runsc to be configured before the test runs.
+        "local",
+        "manual",
+    ],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//pkg/test/testutil",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
+    ],
+)
+
 go_library(
     name = "integration",
     srcs = ["integration.go"],

test/e2e/integration_runtime_test.go

@@ -0,0 +1,117 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package integration provides end-to-end integration tests for runsc.
+//
+// Each test calls docker commands to start up a container, and tests that it is
+// behaving properly, with various runsc commands. The container is killed and
+// deleted at the end.
+//
+// Setup instruction in test/README.md.
+package integration
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types/mount"
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/pkg/test/testutil"
+)
+
+const (
+	// defaultWait is the default wait time used for tests.
+	defaultWait = time.Minute
+
+	memInfoCmd = "cat /proc/meminfo | grep MemTotal: | awk '{print $2}'"
+)
+
+func TestMain(m *testing.M) {
+	dockerutil.EnsureSupportedDockerVersion()
+	flag.Parse()
+	os.Exit(m.Run())
+}
+
+func TestRlimitNoFile(t *testing.T) {
+	ctx := context.Background()
+	d := dockerutil.MakeContainerWithRuntime(ctx, t, "-fdlimit")
+	defer d.CleanUp(ctx)
+
+	// Create a directory with a bunch of files.
+	const nfiles = 5000
+	tmpDir := testutil.TmpDir()
+	for i := 0; i < nfiles; i++ {
+		if _, err := ioutil.TempFile(tmpDir, "tmp"); err != nil {
+			t.Fatalf("TempFile(): %v", err)
+		}
+	}
+
+	// Run the container. Open a bunch of files simutaneously and sleep a bit
+	// to give time for everything to start. We should hit the FD limit and
+	// fail rather than waiting the full sleep duration.
+	cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 60`
+	got, err := d.Run(ctx, dockerutil.RunOpts{
+		Image: "basic/ubuntu",
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeBind,
+				Source: tmpDir,
+				Target: "/tmp/foo",
+			},
+		},
+	}, "bash", "-c", cmd)
+	if err == nil {
+		t.Fatalf("docker run didn't fail: %s", got)
+	} else if strings.Contains(err.Error(), "Unknown runtime specified") {
+		t.Fatalf("docker failed because -fdlimit runtime was not installed")
+	}
+}
+
+func TestDentryCacheLimit(t *testing.T) {
+	ctx := context.Background()
+	d := dockerutil.MakeContainerWithRuntime(ctx, t, "-dcache")
+	defer d.CleanUp(ctx)
+
+	// Create a directory with a bunch of files.
+	const nfiles = 5000
+	tmpDir := testutil.TmpDir()
+	for i := 0; i < nfiles; i++ {
+		if _, err := ioutil.TempFile(tmpDir, "tmp"); err != nil {
+			t.Fatalf("TempFile(): %v", err)
+		}
+	}
+
+	// Run the container. Open a bunch of files simutaneously and sleep a bit
+	// to give time for everything to start. We shouldn't hit the FD limit
+	// because the dentry cache is small.
+	cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 10`
+	got, err := d.Run(ctx, dockerutil.RunOpts{
+		Image: "basic/ubuntu",
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeBind,
+				Source: tmpDir,
+				Target: "/tmp/foo",
+			},
+		},
+	}, "bash", "-c", cmd)
+	if err != nil {
+		t.Fatalf("docker failed: %v, %s", err, got)
+	}
+}

test/e2e/integration_test.go

@@ -997,36 +997,3 @@ func TestNonSearchableWorkingDirectory(t *testing.T) {
 		t.Errorf("ls error message not found, want: %q, got: %q", wantErrorMsg, got)
 	}
 }
-
-func TestRlimitNoFile(t *testing.T) {
-	ctx := context.Background()
-	d := dockerutil.MakeContainerWithRuntime(ctx, t, "-fdlimit")
-	defer d.CleanUp(ctx)
-
-	// Create a directory with a bunch of files.
-	const nfiles = 5000
-	tmpDir := testutil.TmpDir()
-	for i := 0; i < nfiles; i++ {
-		if _, err := ioutil.TempFile(tmpDir, "tmp"); err != nil {
-			t.Fatalf("TempFile(): %v", err)
-		}
-	}
-
-	// Run the container. Open a bunch of files simutaneously and sleep a bit
-	// to give time for everything to start. We should hit the FD limit and
-	// fail rather than waiting the full sleep duration.
-	cmd := `for file in /tmp/foo/*; do (cat > "${file}") & done && sleep 60`
-	got, err := d.Run(ctx, dockerutil.RunOpts{
-		Image: "basic/ubuntu",
-		Mounts: []mount.Mount{
-			{
-				Type:   mount.TypeBind,
-				Source: tmpDir,
-				Target: "/tmp/foo",
-			},
-		},
-	}, "bash", "-c", cmd)
-	if err == nil {
-		t.Fatalf("docker run didn't fail: %s", got)
-	}
-}