fuse: add benchmarking support for FUSE

This change adds the following:
-  Add support for containerizing syscall tests for FUSE
-  Mount tmpfs in the container so we can run benchmarks against it
-  Run the server in a background process
-  benchmarks for fuse syscall

Co-authored-by: Ridwan Sharif <ridwanmsharif@google.com>

Makefile

@@ -341,6 +341,10 @@ RUNTIME_BIN     := $(RUNTIME_DIR)/runsc
 RUNTIME_LOG_DIR := $(RUNTIME_DIR)/logs
 RUNTIME_LOGS    := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%
 
+ifeq (,$(RUNTIME_NAME))
+RUNTIME_NAME := $(RUNTIME)
+endif
+
 dev: ## Installs a set of local runtimes. Requires sudo.
 	@$(call submake,refresh ARGS="--net-raw")
 	@$(call submake,configure RUNTIME_NAME="$(RUNTIME)" ARGS="--net-raw")

images/basic/fuse/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+RUN apt-get install -y build-essential git pkg-config fuse3 libfuse3-3 libfuse3-dev strace
+
+WORKDIR /fus
+
+RUN mkdir -pv mountpoint
+RUN git clone https://github.com/libfuse/libfuse
+
+RUN gcc -Wall ./libfuse/example/passthrough.c `pkg-config fuse3 --cflags --libs` -o server-bin
+
+CMD ["bash"]

pkg/sentry/fs/g3doc/fuse.md

@@ -254,6 +254,53 @@ I/O syscalls like `read(2)`, `write(2)` and `mmap(2)`.
 -   `FUSE_BMAP`: Old address space API for block defrag. Probably not needed.
 -   `FUSE_NOTIFY_REPLY`: [TODO: what does this do?]
 
+## Benchmark FUSE
+
+FUSE benchmark makes FUSE syscall inside docker container to make sure required
+environment conditions are met - such as having the right libraries to start a
+FUSE server.
+
+### Setup
+
+To run benchmark:
+
+1. Make sure you have `Docker` installed.
+2. Download all docker images `make load-all-images`.
+3. Config `runsc` docker runtime to have VFS2 and FUSE supported.
+(e.g.  `make configure RUNTIME=runsc ARGS="--vfs2 --fuse ..." ...`)
+
+You should now have a runtime with the following options configured in
+`/etc/docker/daemon.json`
+```
+"runsc": {
+            "path": "path/to/your/runsc",
+            "runtimeArgs": [
+                "--vfs2",
+                "--fuse"
+                ... 
+            ]
+        }
+```
+
+### Running benchmarks
+With above setup, benchmark can be run with following command
+```
+bazel test --test_output=all --cache_test_results=no --test_arg=-test.bench= //path/to:target
+```
+For example: if you want to run stat test
+```
+bazel test --test_output=all --cache_test_results=no --test_arg=-test.bench= //test/fuse:open_benchmark_runsc_ptrace_vfs2_fuse_container
+```
+
+Note:
+- test target need to have `vfs2_fuse_container` to run in container with `vfs2` and `fuse` enabled
+- `test_output` set to `all` to view the result in terminal
+- `--cache_test_results` set to `no` to avoid cached benchmark
+
+### Use your fuse server
+
+To use your own FUSE server, change the `images/basic/fuse/Dockerfile` to compile your FUSE server into the container and name it `server-bin`.
+
 # References
 
 -   [fuse(4) Linux manual page](https://www.man7.org/linux/man-pages/man4/fuse.4.html)

pkg/test/dockerutil/container.go

@@ -136,6 +136,11 @@ func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container
 	}
 }
 
+// Runtime returns the runtime of the container.
+func (c *Container) Runtime() string {
+	return c.runtime
+}
+
 // AddProfile adds a profile to this container.
 func (c *Container) AddProfile(p Profile) {
 	c.profiles = append(c.profiles, p)
@@ -541,3 +546,8 @@ func (c *Container) CleanUp(ctx context.Context) {
 	// Forget all mounts.
 	c.mounts = nil
 }
+
+// CopyErr returns the error that happened during copy.
+func (c *Container) CopyErr() error {
+	return c.copyErr
+}

pkg/test/dockerutil/dockerutil.go

@@ -121,6 +121,26 @@ func UsingVFS2() (bool, error) {
 	return false, nil
 }
 
+// UsingFUSE returns true if the 'runtime' has the fuse flag set.
+func UsingFUSE() (bool, error) {
+	rMap, err := runtimeMap()
+	if err != nil {
+		return false, err
+	}
+
+	list, ok := rMap["runtimeArgs"].([]interface{})
+	if !ok {
+		return false, fmt.Errorf("unexpected format: %v", rMap)
+	}
+
+	for _, element := range list {
+		if element == "--fuse" {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func runtimeMap() (map[string]interface{}, error) {
 	// Read the configuration data; the file must exist.
 	configBytes, err := ioutil.ReadFile(*config)

scripts/common_build.sh

@@ -109,8 +109,9 @@ function collect_logs() {
 }
 
 function find_branch_name() {
-  git branch --show-current \
+  (git branch --show-current \
     || git rev-parse HEAD \
     || bazel info workspace \
-    | xargs basename
+    | xargs basename) \
+    | tr '/' '-'
 }

test/e2e/integration_test.go

@@ -168,10 +168,10 @@ func TestCheckpointRestore(t *testing.T) {
 	}
 
 	// TODO(gvisor.dev/issue/3373): Remove after implementing.
-	if usingVFS2, err := dockerutil.UsingVFS2(); usingVFS2 {
-		t.Skip("CheckpointRestore not implemented in VFS2.")
-	} else if err != nil {
+	if usingVFS2, err := dockerutil.UsingVFS2(); err != nil {
 		t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err)
+	} else if usingVFS2 {
+		t.Skip("CheckpointRestore not implemented in VFS2.")
 	}
 
 	ctx := context.Background()

test/fuse/BUILD

@@ -51,3 +51,49 @@ syscall_test(
     fuse = "True",
     test = "//test/fuse/linux:readdir_test",
 )
+
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:stat_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:open_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:read_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:symlink_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:mkdir_benchmark",
+    use_image = "basic/fuse",
+)

test/fuse/benchmark/BUILD

@@ -0,0 +1,91 @@
+load("//tools:defs.bzl", "cc_binary", "gbenchmark", "gtest")
+
+package(
+    default_visibility = ["//:sandbox"],
+    licenses = ["notice"],
+)
+
+cc_binary(
+    name = "stat_benchmark",
+    testonly = 1,
+    srcs = [
+        "stat_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "open_benchmark",
+    testonly = 1,
+    srcs = [
+        "open_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "read_benchmark",
+    testonly = 1,
+    srcs = [
+        "read_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "symlink_benchmark",
+    testonly = 1,
+    srcs = [
+        "symlink_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "mkdir_benchmark",
+    testonly = 1,
+    srcs = [
+        "mkdir_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)

test/fuse/benchmark/mkdir_benchmark.cc

@@ -0,0 +1,51 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "absl/strings/str_cat.h"
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Mkdir(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::string dir_path = top_dir.path();
+
+  int index = 0;
+  for (auto t : state) {
+    const std::string new_dir_path = absl::StrCat(dir_path, index);
+    ASSERT_THAT(mkdir(new_dir_path.c_str(), 0777), SyscallSucceeds());
+    index++;
+  }
+}
+
+BENCHMARK(BM_Mkdir)->Range(1, 128)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor

test/fuse/benchmark/open_benchmark.cc

@@ -0,0 +1,60 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/logging.h"
+#include "test/util/temp_path.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Open(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  const int size = state.range(0);
+  std::vector<TempPath> cache;
+  for (int i = 0; i < size; i++) {
+    auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile());
+    cache.emplace_back(std::move(path));
+  }
+
+  unsigned int seed = 1;
+  for (auto _ : state) {
+    const int chosen = rand_r(&seed) % size;
+    const std::string file_path = JoinPath(fuse_prefix, cache[chosen].path());
+    int fd = open(file_path.c_str(), O_RDONLY);
+    TEST_CHECK(fd != -1);
+    close(fd);
+  }
+}
+
+BENCHMARK(BM_Open)->Range(1, 128)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor

test/fuse/benchmark/read_benchmark.cc

@@ -0,0 +1,57 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/logging.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Read(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  const int size = state.range(0);
+  const std::string contents(size, 0);
+  auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileWith(
+      GetAbsoluteTestTmpdir(), contents, TempPath::kDefaultFileMode));
+  FileDescriptor fd = ASSERT_NO_ERRNO_AND_VALUE(
+      Open(JoinPath(fuse_prefix, path.path()), O_RDONLY));
+
+  std::vector<char> buf(size);
+  for (auto _ : state) {
+    TEST_CHECK(PreadFd(fd.get(), buf.data(), buf.size(), 0) == size);
+  }
+
+  state.SetBytesProcessed(static_cast<int64_t>(size) *
+                          static_cast<int64_t>(state.iterations()));
+}
+
+BENCHMARK(BM_Read)->Range(1, 1 << 26)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor

test/fuse/benchmark/stat_benchmark.cc

@@ -0,0 +1,65 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "gtest/gtest.h"
+#include "absl/strings/str_cat.h"
+#include "benchmark/benchmark.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+// Creates a file in a nested directory hierarchy at least `depth` directories
+// deep, and stats that file multiple times.
+void BM_Stat(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  // Create nested directories with given depth.
+  int depth = state.range(0);
+  const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::string dir_path = top_dir.path();
+
+  while (depth-- > 0) {
+    // Don't use TempPath because it will make paths too long to use.
+    //
+    // The top_dir destructor will clean up this whole tree.
+    dir_path = JoinPath(dir_path, absl::StrCat(depth));
+    ASSERT_NO_ERRNO(Mkdir(dir_path, 0755));
+  }
+
+  // Create the file that will be stat'd.
+  const TempPath file =
+      ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileIn(dir_path));
+  std::string file_path = JoinPath(fuse_prefix, file.path());
+  struct stat st;
+  for (auto _ : state) {
+    ASSERT_THAT(stat(file_path.c_str(), &st), SyscallSucceeds());
+  }
+}
+
+BENCHMARK(BM_Stat)->Range(1, 100)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor

test/fuse/benchmark/symlink_benchmark.cc

@@ -0,0 +1,60 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "absl/strings/str_cat.h"
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Symlink(benchmark::State& state) {
+  char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+  const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::string dir_path = top_dir.path();
+
+  const int size = state.range(0);
+  std::vector<TempPath> cache;
+  for (int i = 0; i < size; i++) {
+    auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile());
+    cache.emplace_back(std::move(path));
+  }
+
+  int index = 0;
+  unsigned int seed = 1;
+  for (auto t : state) {
+    const int chosen = rand_r(&seed) % size;
+    const std::string symlink_path = absl::StrCat(fuse_prefix, dir_path, index);
+    ASSERT_THAT(symlink(cache[chosen].path().c_str(), symlink_path.c_str()),
+                SyscallSucceeds());
+    index++;
+  }
+}
+
+BENCHMARK(BM_Symlink)->Range(1, 128)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor

test/image/BUILD

@@ -24,6 +24,7 @@ go_test(
     deps = [
         "//pkg/test/dockerutil",
         "//pkg/test/testutil",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
     ],
 )
 

test/image/image_test.go

@@ -33,6 +33,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/api/types/mount"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 )
@@ -63,6 +64,57 @@ func TestHelloWorld(t *testing.T) {
 	}
 }
 
+// Test that the FUSE container is set up and being used properly.
+func TestFUSEInContainer(t *testing.T) {
+	if usingFUSE, err := dockerutil.UsingFUSE(); err != nil {
+		t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err)
+	} else if !usingFUSE {
+		t.Skip("FUSE not being used.")
+	}
+
+	ctx := context.Background()
+	d := dockerutil.MakeContainer(ctx, t)
+	defer d.CleanUp(ctx)
+
+	tmpDir := "/tmpDir/"
+	// Run the basic container.
+	err := d.Spawn(ctx, dockerutil.RunOpts{
+		Image:      "basic/fuse",
+		Privileged: true,
+		CapAdd:     []string{"CAP_SYS_ADMIN"},
+
+		// Mount a tmpfs directory for benchmark.
+		Mounts: []mount.Mount{
+			{
+				Type:     mount.TypeTmpfs,
+				Target:   tmpDir,
+				ReadOnly: false,
+			},
+		},
+	}, "sleep", "1000")
+	if err != nil {
+		t.Fatalf("docker spawn failed: %v", err)
+	}
+
+	out, err := d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", "ls")
+	if err != nil {
+		t.Fatalf("docker exec failed: %v, message %s", err, out)
+	}
+	if !strings.Contains(out, "server-bin") {
+		t.Fatalf("docker didn't find server binary: got %s", out)
+	}
+
+	// Run the server.
+	out, err = d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", "./server-bin mountpoint")
+	if err != nil {
+		t.Fatalf("docker exec failed: %v, message %s", err, out)
+	}
+}
+
 func runHTTPRequest(port int) error {
 	url := fmt.Sprintf("http://localhost:%d/not-found", port)
 	resp, err := http.Get(url)

test/runner/BUILD

@@ -11,11 +11,14 @@ go_binary(
     ],
     visibility = ["//:sandbox"],
     deps = [
+        "//pkg/context",
         "//pkg/log",
+        "//pkg/test/dockerutil",
         "//pkg/test/testutil",
         "//runsc/specutils",
         "//test/runner/gtest",
         "//test/uds",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
         "@com_github_opencontainers_runtime_spec//specs-go:go_default_library",
         "@com_github_syndtr_gocapability//capability:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",

test/runner/defs.bzl

@@ -57,6 +57,8 @@ def _syscall_test(
         platform,
         use_tmpfs,
         tags,
+        use_image = "",
+        setup_command = "",
         network = "none",
         file_access = "exclusive",
         overlay = False,
@@ -79,6 +81,8 @@ def _syscall_test(
             name += "_fuse"
     if network != "none":
         name += "_" + network + "net"
+    if use_image != "":
+        name += "_container"
 
     # Apply all tags.
     if tags == None:
@@ -107,6 +111,8 @@ def _syscall_test(
         "--platform=" + platform,
         "--network=" + network,
         "--use-tmpfs=" + str(use_tmpfs),
+        "--use-image=" + use_image,
+        "--setup-command=" + setup_command,
         "--file-access=" + file_access,
         "--overlay=" + str(overlay),
         "--add-uds-tree=" + str(add_uds_tree),
@@ -132,6 +138,8 @@ def syscall_test(
         shard_count = 5,
         size = "small",
         use_tmpfs = False,
+        use_image = "",
+        setup_command = "",
         add_overlay = False,
         add_uds_tree = False,
         add_hostinet = False,
@@ -146,6 +154,8 @@ def syscall_test(
       shard_count: shards for defined tests.
       size: the defined test size.
       use_tmpfs: use tmpfs in the defined tests.
+      use_image: use specified docker image in the defined tests.
+      setup_command: command to set up the docker container. Should be used when ise_image is.
       add_overlay: add an overlay test.
       add_uds_tree: add a UDS test.
       add_hostinet: add a hostinet test.
@@ -178,8 +188,26 @@ def syscall_test(
         vfs2 = True,
         fuse = fuse,
     )
+
+    if use_image != "":
+        # Run the test in the container specified.
+        _syscall_test(
+            test = test,
+            shard_count = shard_count,
+            size = size,
+            platform = default_platform,
+            use_tmpfs = use_tmpfs,
+            use_image = use_image,
+            setup_command = setup_command,
+            add_uds_tree = add_uds_tree,
+            tags = platforms[default_platform] + vfs2_tags,
+            vfs2 = True,
+            fuse = True,
+        )
+
     if fuse:
         # Only generate *_vfs2_fuse target if fuse parameter is enabled.
+        # The rest of the targets don't support FUSE as of yet.
         return
 
     _syscall_test(

test/runner/runner.go

@@ -23,16 +23,20 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/docker/docker/api/types/mount"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/syndtr/gocapability/capability"
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 	"gvisor.dev/gvisor/runsc/specutils"
 	"gvisor.dev/gvisor/test/runner/gtest"
@@ -40,17 +44,19 @@ import (
 )
 
 var (
-	debug      = flag.Bool("debug", false, "enable debug logs")
-	strace     = flag.Bool("strace", false, "enable strace logs")
-	platform   = flag.String("platform", "ptrace", "platform to run on")
-	network    = flag.String("network", "none", "network stack to run on (sandbox, host, none)")
-	useTmpfs   = flag.Bool("use-tmpfs", false, "mounts tmpfs for /tmp")
-	fileAccess = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode")
-	overlay    = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay")
-	vfs2       = flag.Bool("vfs2", false, "enable VFS2")
-	fuse       = flag.Bool("fuse", false, "enable FUSE")
-	parallel   = flag.Bool("parallel", false, "run tests in parallel")
-	runscPath  = flag.String("runsc", "", "path to runsc binary")
+	debug        = flag.Bool("debug", false, "enable debug logs")
+	strace       = flag.Bool("strace", false, "enable strace logs")
+	platform     = flag.String("platform", "ptrace", "platform to run on")
+	network      = flag.String("network", "none", "network stack to run on (sandbox, host, none)")
+	useTmpfs     = flag.Bool("use-tmpfs", false, "mounts tmpfs for /tmp")
+	useImage     = flag.String("use-image", "", "container image to use for test. Path relative to //images")
+	setupCommand = flag.String("setup-command", "", "command to run before running the test to set up container environment")
+	fileAccess   = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode")
+	overlay      = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay")
+	vfs2         = flag.Bool("vfs2", false, "enable VFS2")
+	fuse         = flag.Bool("fuse", false, "enable FUSE")
+	parallel     = flag.Bool("parallel", false, "run tests in parallel")
+	runscPath    = flag.String("runsc", "", "path to runsc binary")
 
 	addUDSTree = flag.Bool("add-uds-tree", false, "expose a tree of UDS utilities for use in tests")
 )
@@ -313,8 +319,92 @@ func setupUDSTree(spec *specs.Spec) (cleanup func(), err error) {
 	return cleanup, nil
 }
 
+func runTestCaseInContainer(testBin string, tc gtest.TestCase, image string, t *testing.T) {
+	if usingFUSE, err := dockerutil.UsingFUSE(); err != nil {
+		t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err)
+	} else if !usingFUSE {
+		t.Skip("FUSE not being used.")
+	}
+
+	ctx := context.Background()
+	d := dockerutil.MakeContainer(ctx, t)
+	defer d.CleanUp(ctx)
+
+	// Run the basic container.
+	tmpDir := "/tmpDir"
+	testBinDir := "/testDir"
+	opts := dockerutil.RunOpts{
+		Image:      image,
+		Privileged: true,
+		CapAdd:     []string{"CAP_SYS_ADMIN"},
+
+		// Mount a tmpfs directory to use when benchmarking.
+		Mounts: []mount.Mount{
+			{
+				Type:     mount.TypeTmpfs,
+				Target:   tmpDir,
+				ReadOnly: false,
+			},
+		},
+		Env: []string{
+			fmt.Sprintf("TEST_TMPDIR=%s", tmpDir),
+			fmt.Sprintf("TEST_FUSEPRE=%s", "/fus/mountpoint"),
+		},
+	}
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Getwd run failed: %v", err)
+	}
+
+	wdPathToTestBin := strings.TrimPrefix(testBin, wd)
+	containerTestBin := path.Join(testBinDir, path.Base(wdPathToTestBin))
+	d.CopyFiles(&opts, testBinDir, wdPathToTestBin)
+	if err := d.CopyErr(); err != nil {
+		t.Fatalf("Copy failed %v", err)
+	}
+
+	err = d.Spawn(ctx, opts, "sleep", "1000")
+	if err != nil {
+		t.Fatalf("docker run failed: %v", err)
+	}
+
+	// Run the server setup command.
+	if *setupCommand != "" {
+		out, err := d.Exec(ctx, dockerutil.ExecOpts{
+			Privileged: true,
+		}, "/bin/sh", "-c", *setupCommand)
+		if err != nil {
+			t.Fatalf("docker exec failed: %v with output %v", err, out)
+		}
+	}
+
+	cmd := "chmod +x " + containerTestBin
+	out, err := d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", cmd)
+	if err != nil {
+		t.Fatalf("docker exec failed: %v with output %v", err, out)
+	}
+
+	cmd = containerTestBin + " " + strings.Join(tc.Args(), " ")
+	out, err = d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", cmd)
+	if err != nil {
+		t.Fatalf("docker exec failed: %v with output %v", err, out)
+	}
+
+	fmt.Print(out)
+	return
+}
+
 // runsTestCaseRunsc runs the test case in runsc.
 func runTestCaseRunsc(testBin string, tc gtest.TestCase, t *testing.T) {
+	if *useImage != "" {
+		runTestCaseInContainer(testBin, tc, *useImage, t)
+		return
+	}
+
 	// Run a new container with the test executable and filter for the
 	// given test suite and name.
 	spec := testutil.NewSpecWithArgs(append([]string{testBin}, tc.Args()...)...)