Add a rootfs filesystem to fs benchmarks. Also, fix fio benchmark.

The previous "bind" filesystem, already included in go/runsc-benchmarks
is a remote re-validate mount. However, the non-re-validate mount
was not present, and it has been added in the form of rootfs.

Also, fix the fio runs to reads/writes of 10GB as running
with --test.benchtime=Xs may scale beyond the memory available
to tmpfs mounts on buildkite VMs. Currently, our buildkite
pipelines are run on e2-standard-8 machines with 32GB of memory,
allowing tmpfs mounts to safely be at least 10GB.

PiperOrigin-RevId: 362143620

.buildkite/pipeline.yaml

@@ -183,9 +183,13 @@ steps:
   - <<: *benchmarks
     label: ":metal: FFMPEG benchmarks"
     command: make benchmark-platforms BENCHMARKS_SUITE=ffmpeg BENCHMARKS_TARGETS=test/benchmarks/media:ffmpeg_test
+  # For fio, running with --test.benchtime=Xs scales the written/read
+  # bytes to several GB. This is not a problem for root/bind/volume mounts,
+  # but for tmpfs mounts, the size can grow to more memory than the machine
+  # has availabe. Fix the runs to 10GB written/read for the benchmark.
   - <<: *benchmarks
     label: ":floppy_disk: FIO benchmarks"
-    command: make benchmark-platforms BENCHMARKS_SUITE=fio BENCHMARKS_TARGETS=test/benchmarks/fs:fio_test
+    command: make benchmark-platforms BENCHMARKS_SUITE=fio BENCHMARKS_TARGETS=test/benchmarks/fs:fio_test BENCHMARKS_OPTIONS=--test.benchtime=10000x
   - <<: *benchmarks
     label: ":globe_with_meridians: HTTPD benchmarks"
     command: make benchmark-platforms BENCHMARKS_FILTER="Continuous" BENCHMARKS_SUITE=httpd BENCHMARKS_TARGETS=test/benchmarks/network:httpd_test

test/benchmarks/fs/BUILD

@@ -11,6 +11,7 @@ benchmark_test(
         "//pkg/test/dockerutil",
         "//test/benchmarks/harness",
         "//test/benchmarks/tools",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
     ],
 )
 

test/benchmarks/fs/bazel_test.go

@@ -25,6 +25,13 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+// Dimensions here are clean/dirty cache (do or don't drop caches)
+// and if the mount on which we are compiling is a tmpfs/bind mount.
+type benchmark struct {
+	clearCache bool   // clearCache drops caches before running.
+	fstype     string // type of filesystem to use.
+}
+
 // Note: CleanCache versions of this test require running with root permissions.
 func BenchmarkBuildABSL(b *testing.B) {
 	runBuildBenchmark(b, "benchmarks/absl", "/abseil-cpp", "absl/base/...")
@@ -45,17 +52,18 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) {
 	}
 	defer machine.CleanUp()
 
-	// Dimensions here are clean/dirty cache (do or don't drop caches)
-	// and if the mount on which we are compiling is a tmpfs/bind mount.
-	benchmarks := []struct {
-		clearCache bool // clearCache drops caches before running.
-		tmpfs      bool // tmpfs will run compilation on a tmpfs.
-	}{
-		{clearCache: true, tmpfs: false},
-		{clearCache: false, tmpfs: false},
-		{clearCache: true, tmpfs: true},
-		{clearCache: false, tmpfs: true},
+	benchmarks := make([]benchmark, 0, 6)
+	for _, filesys := range []string{harness.BindFS, harness.TmpFS, harness.RootFS} {
+		benchmarks = append(benchmarks, benchmark{
+			clearCache: true,
+			fstype:     filesys,
+		})
+		benchmarks = append(benchmarks, benchmark{
+			clearCache: false,
+			fstype:     filesys,
+		})
 	}
+
 	for _, bm := range benchmarks {
 		pageCache := tools.Parameter{
 			Name:  "page_cache",
@@ -67,10 +75,7 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) {
 
 		filesystem := tools.Parameter{
 			Name:  "filesystem",
-			Value: "bind",
-		}
-		if bm.tmpfs {
-			filesystem.Value = "tmpfs"
+			Value: bm.fstype,
 		}
 		name, err := tools.ParametersToName(pageCache, filesystem)
 		if err != nil {
@@ -83,21 +88,25 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) {
 			container := machine.GetContainer(ctx, b)
 			defer container.CleanUp(ctx)
 
+			mts, prefix, cleanup, err := harness.MakeMount(machine, bm.fstype)
+			if err != nil {
+				b.Fatalf("Failed to make mount: %v", err)
+			}
+			defer cleanup()
+
+			runOpts := dockerutil.RunOpts{
+				Image:  image,
+				Mounts: mts,
+			}
+
 			// Start a container and sleep.
-			if err := container.Spawn(ctx, dockerutil.RunOpts{
-				Image: image,
-			}, "sleep", fmt.Sprintf("%d", 1000000)); err != nil {
+			if err := container.Spawn(ctx, runOpts, "sleep", fmt.Sprintf("%d", 1000000)); err != nil {
 				b.Fatalf("run failed with: %v", err)
 			}
 
-			// If we are running on a tmpfs, copy to /tmp which is a tmpfs.
-			prefix := ""
-			if bm.tmpfs {
-				if out, err := container.Exec(ctx, dockerutil.ExecOpts{},
-					"cp", "-r", workdir, "/tmp/."); err != nil {
-					b.Fatalf("failed to copy directory: %v (%s)", err, out)
-				}
-				prefix = "/tmp"
+			if out, err := container.Exec(ctx, dockerutil.ExecOpts{},
+				"cp", "-rf", workdir, prefix+"/."); err != nil {
+				b.Fatalf("failed to copy directory: %v (%s)", err, out)
 			}
 
 			b.ResetTimer()
@@ -118,7 +127,7 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) {
 					WorkDir: prefix + workdir,
 				}, "bazel", "build", "-c", "opt", target)
 				if err != nil {
-					b.Fatalf("build failed with: %v", err)
+					b.Fatalf("build failed with: %v logs: %s", err, got)
 				}
 				b.StopTimer()
 

test/benchmarks/fs/fio_test.go

@@ -21,7 +21,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/docker/docker/api/types/mount"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/test/benchmarks/harness"
 	"gvisor.dev/gvisor/test/benchmarks/tools"
@@ -70,7 +69,7 @@ func BenchmarkFio(b *testing.B) {
 	}
 	defer machine.CleanUp()
 
-	for _, fsType := range []mount.Type{mount.TypeBind, mount.TypeTmpfs} {
+	for _, fsType := range []string{harness.BindFS, harness.TmpFS, harness.RootFS} {
 		for _, tc := range testCases {
 			operation := tools.Parameter{
 				Name:  "operation",
@@ -82,7 +81,7 @@ func BenchmarkFio(b *testing.B) {
 			}
 			filesystem := tools.Parameter{
 				Name:  "filesystem",
-				Value: string(fsType),
+				Value: fsType,
 			}
 			name, err := tools.ParametersToName(operation, blockSize, filesystem)
 			if err != nil {
@@ -95,13 +94,7 @@ func BenchmarkFio(b *testing.B) {
 				container := machine.GetContainer(ctx, b)
 				defer container.CleanUp(ctx)
 
-				// Directory and filename inside container where fio will read/write.
-				outdir := "/data"
-				outfile := filepath.Join(outdir, "test.txt")
-
-				// Make the required mount and grab a cleanup for bind mounts
-				// as they are backed by a temp directory (mktemp).
-				mnt, mountCleanup, err := makeMount(machine, fsType, outdir)
+				mnts, outdir, mountCleanup, err := harness.MakeMount(machine, fsType)
 				if err != nil {
 					b.Fatalf("failed to make mount: %v", err)
 				}
@@ -109,12 +102,9 @@ func BenchmarkFio(b *testing.B) {
 
 				// Start the container with the mount.
 				if err := container.Spawn(
-					ctx,
-					dockerutil.RunOpts{
-						Image: "benchmarks/fio",
-						Mounts: []mount.Mount{
-							mnt,
-						},
+					ctx, dockerutil.RunOpts{
+						Image:  "benchmarks/fio",
+						Mounts: mnts,
 					},
 					// Sleep on the order of b.N.
 					"sleep", fmt.Sprintf("%d", 1000*b.N),
@@ -122,6 +112,9 @@ func BenchmarkFio(b *testing.B) {
 					b.Fatalf("failed to start fio container with: %v", err)
 				}
 
+				// Directory and filename inside container where fio will read/write.
+				outfile := filepath.Join(outdir, "test.txt")
+
 				// For reads, we need a file to read so make one inside the container.
 				if strings.Contains(tc.Test, "read") {
 					fallocateCmd := fmt.Sprintf("fallocate -l %dK %s", tc.Size, outfile)
@@ -135,6 +128,7 @@ func BenchmarkFio(b *testing.B) {
 				if err := harness.DropCaches(machine); err != nil {
 					b.Skipf("failed to drop caches with %v. You probably need root.", err)
 				}
+
 				cmd := tc.MakeCmd(outfile)
 
 				if err := harness.DropCaches(machine); err != nil {
@@ -154,39 +148,6 @@ func BenchmarkFio(b *testing.B) {
 	}
 }
 
-// makeMount makes a mount and cleanup based on the requested type. Bind
-// and volume mounts are backed by a temp directory made with mktemp.
-// tmpfs mounts require no such backing and are just made.
-// It is up to the caller to call the returned cleanup.
-func makeMount(machine harness.Machine, mountType mount.Type, target string) (mount.Mount, func(), error) {
-	switch mountType {
-	case mount.TypeVolume, mount.TypeBind:
-		dir, err := machine.RunCommand("mktemp", "-d")
-		if err != nil {
-			return mount.Mount{}, func() {}, fmt.Errorf("failed to create tempdir: %v", err)
-		}
-		dir = strings.TrimSuffix(dir, "\n")
-
-		out, err := machine.RunCommand("chmod", "777", dir)
-		if err != nil {
-			machine.RunCommand("rm", "-rf", dir)
-			return mount.Mount{}, func() {}, fmt.Errorf("failed modify directory: %v %s", err, out)
-		}
-		return mount.Mount{
-			Target: target,
-			Source: dir,
-			Type:   mount.TypeBind,
-		}, func() { machine.RunCommand("rm", "-rf", dir) }, nil
-	case mount.TypeTmpfs:
-		return mount.Mount{
-			Target: target,
-			Type:   mount.TypeTmpfs,
-		}, func() {}, nil
-	default:
-		return mount.Mount{}, func() {}, fmt.Errorf("illegal mount time not supported: %v", mountType)
-	}
-}
-
 // TestMain is the main method for package fs.
 func TestMain(m *testing.M) {
 	harness.Init()

test/benchmarks/harness/BUILD

@@ -14,5 +14,6 @@ go_library(
     deps = [
         "//pkg/test/dockerutil",
         "//pkg/test/testutil",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
     ],
 )

test/benchmarks/harness/util.go

@@ -18,8 +18,10 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 	"testing"
 
+	"github.com/docker/docker/api/types/mount"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 )
@@ -55,3 +57,53 @@ func DebugLog(b *testing.B, msg string, args ...interface{}) {
 		b.Logf(msg, args...)
 	}
 }
+
+const (
+	// BindFS indicates a bind mount should be created.
+	BindFS = "bindfs"
+	// TmpFS indicates a tmpfs mount should be created.
+	TmpFS = "tmpfs"
+	// RootFS indicates no mount should be created and the root mount should be used.
+	RootFS = "rootfs"
+)
+
+// MakeMount makes a mount and cleanup based on the requested type. Bind
+// and volume mounts are backed by a temp directory made with mktemp.
+// tmpfs mounts require no such backing and are just made.
+// rootfs mounts do not make a mount, but instead return a target direectory at root.
+// It is up to the caller to call the returned cleanup.
+func MakeMount(machine Machine, fsType string) ([]mount.Mount, string, func(), error) {
+	mounts := make([]mount.Mount, 0, 1)
+	switch fsType {
+	case BindFS:
+		dir, err := machine.RunCommand("mktemp", "-d")
+		if err != nil {
+			return mounts, "", func() {}, fmt.Errorf("failed to create tempdir: %v", err)
+		}
+		dir = strings.TrimSuffix(dir, "\n")
+
+		out, err := machine.RunCommand("chmod", "777", dir)
+		if err != nil {
+			machine.RunCommand("rm", "-rf", dir)
+			return mounts, "", func() {}, fmt.Errorf("failed modify directory: %v %s", err, out)
+		}
+		target := "/data"
+		mounts = append(mounts, mount.Mount{
+			Target: target,
+			Source: dir,
+			Type:   mount.TypeBind,
+		})
+		return mounts, target, func() { machine.RunCommand("rm", "-rf", dir) }, nil
+	case RootFS:
+		return mounts, "/", func() {}, nil
+	case TmpFS:
+		target := "/data"
+		mounts = append(mounts, mount.Mount{
+			Target: target,
+			Type:   mount.TypeTmpfs,
+		})
+		return mounts, target, func() {}, nil
+	default:
+		return mounts, "", func() {}, fmt.Errorf("illegal mount type not supported: %v", fsType)
+	}
+}