Merge pull request #7989 from cedriccchen:benchmarks

PiperOrigin-RevId: 477586197

images/benchmarks/hackbench/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:18.04
+
+RUN set -x \
+        && apt-get update \
+        && apt-get install -y rt-tests \
+        && rm -rf /var/lib/apt/lists/*

images/benchmarks/syscallbench/Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:18.04
+
+RUN set -x \
+        && apt-get update \
+        && apt-get install -y \
+            gcc \
+        && rm -rf /var/lib/apt/lists/*
+
+COPY ./syscallbench.c /
+RUN gcc /syscallbench.c -o /usr/bin/syscallbench

images/benchmarks/syscallbench/syscallbench.c

@@ -0,0 +1,57 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+static int loops = 10000000;
+
+static void show_usage(const char *cmd) {
+  fprintf(stderr,
+          "Usage: %s [options]\n"
+          "-l, --loops <num>\t\t Number of syscall loops, default 10000000\n",
+          cmd);
+}
+
+int main(int argc, char *argv[]) {
+  int i;
+  int c;
+  struct option long_options[] = {{"loops", required_argument, 0, 'l'},
+                                  {0, 0, 0, 0}};
+  int option_index = 0;
+
+  while ((c = getopt_long(argc, argv, "l:", long_options, &option_index)) !=
+         -1) {
+    switch (c) {
+      case 'l':
+        loops = atoi(optarg);
+        if (loops <= 0) {
+          show_usage(argv[0]);
+          exit(1);
+        }
+        break;
+      default:
+        show_usage(argv[0]);
+        exit(1);
+    }
+  }
+
+  for (i = 0; i < loops; i++) syscall(SYS_getpid);
+
+  printf("# Executed %'d getpid() calls\n", loops);
+  return 0;
+}

test/benchmarks/base/BUILD

@@ -48,3 +48,25 @@ benchmark_test(
         "//test/benchmarks/tools",
     ],
 )
+
+benchmark_test(
+    name = "syscallbench_test",
+    srcs = ["syscallbench_test.go"],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
+)
+
+benchmark_test(
+    name = "hackbench_test",
+    srcs = ["hackbench_test.go"],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
+)

test/benchmarks/base/hackbench_test.go

@@ -0,0 +1,89 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hackbench_test
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
+	"gvisor.dev/gvisor/test/benchmarks/tools"
+)
+
+// BenchmarHackbench runs hackbench on the runtime.
+func BenchmarkHackbench(b *testing.B) {
+	testCases := []tools.Hackbench{
+		{
+			IpcMode:     "pipe",
+			ProcessMode: "thread",
+		},
+		{
+			IpcMode:     "socket",
+			ProcessMode: "process",
+		},
+	}
+
+	machine, err := harness.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer machine.CleanUp()
+
+	for _, tc := range testCases {
+		ipcMode := tools.Parameter{
+			Name:  "ipcMode",
+			Value: tc.IpcMode,
+		}
+		processMode := tools.Parameter{
+			Name:  "processMode",
+			Value: tc.ProcessMode,
+		}
+		name, err := tools.ParametersToName(ipcMode, processMode)
+		if err != nil {
+			b.Fatalf("Failed to parse params: %v", err)
+		}
+		b.Run(name, func(b *testing.B) {
+			ctx := context.Background()
+			container := machine.GetContainer(ctx, b)
+			defer container.CleanUp(ctx)
+
+			if err := container.Spawn(
+				ctx, dockerutil.RunOpts{
+					Image: "benchmarks/hackbench",
+				},
+				"sleep", "24h",
+			); err != nil {
+				b.Fatalf("run failed with: %v", err)
+			}
+
+			cmd := tc.MakeCmd(b)
+			b.ResetTimer()
+			out, err := container.Exec(ctx, dockerutil.ExecOpts{}, cmd...)
+			if err != nil {
+				b.Fatalf("failed to run hackbench: %v, logs:%s", err, out)
+			}
+			tc.Report(b, out)
+		})
+	}
+}
+
+// TestMain is the main method for this package.
+func TestMain(m *testing.M) {
+	harness.Init()
+	harness.SetFixedBenchmarks()
+	os.Exit(m.Run())
+}

test/benchmarks/base/sysbench_test.go

@@ -16,6 +16,7 @@ package sysbench_test
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
@@ -24,8 +25,9 @@ import (
 )
 
 type testCase struct {
-	name string
-	test tools.Sysbench
+	name    string
+	threads int
+	test    tools.Sysbench
 }
 
 // BenchmarSysbench runs sysbench on the runtime.
@@ -39,6 +41,15 @@ func BenchmarkSysbench(b *testing.B) {
 				},
 			},
 		},
+		{
+			name:    "CPU",
+			threads: 16,
+			test: &tools.SysbenchCPU{
+				SysbenchBase: tools.SysbenchBase{
+					Threads: 16,
+				},
+			},
+		},
 		{
 			name: "Memory",
 			test: &tools.SysbenchMemory{
@@ -47,6 +58,15 @@ func BenchmarkSysbench(b *testing.B) {
 				},
 			},
 		},
+		{
+			name:    "Memory",
+			threads: 16,
+			test: &tools.SysbenchMemory{
+				SysbenchBase: tools.SysbenchBase{
+					Threads: 16,
+				},
+			},
+		},
 		{
 			name: "Mutex",
 			test: &tools.SysbenchMutex{
@@ -68,9 +88,21 @@ func BenchmarkSysbench(b *testing.B) {
 			Name:  "testname",
 			Value: tc.name,
 		}
-		name, err := tools.ParametersToName(param)
-		if err != nil {
-			b.Fatalf("Failed to parse params: %v", err)
+		var name string
+		if tc.threads != 0 {
+			threads := tools.Parameter{
+				Name:  "threads",
+				Value: fmt.Sprintf("%d", tc.threads),
+			}
+			name, err = tools.ParametersToName(param, threads)
+			if err != nil {
+				b.Fatalf("Failed to parse params: %v", err)
+			}
+		} else {
+			name, err = tools.ParametersToName(param)
+			if err != nil {
+				b.Fatalf("Failed to parse params: %v", err)
+			}
 		}
 		b.Run(name, func(b *testing.B) {
 			ctx := context.Background()

test/benchmarks/base/syscallbench_test.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package syscallbench_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
+	"gvisor.dev/gvisor/test/benchmarks/tools"
+)
+
+// BenchmarSyscallbench runs syscallbench on the runtime.
+func BenchmarkSyscallbench(b *testing.B) {
+	machine, err := harness.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer machine.CleanUp()
+
+	param := tools.Parameter{
+		Name:  "syscall",
+		Value: "getpid",
+	}
+	name, err := tools.ParametersToName(param)
+	if err != nil {
+		b.Fatalf("Failed to parse params: %v", err)
+	}
+	b.Run(name, func(b *testing.B) {
+		ctx := context.Background()
+		container := machine.GetContainer(ctx, b)
+		defer container.CleanUp(ctx)
+
+		if err := container.Spawn(
+			ctx, dockerutil.RunOpts{
+				Image: "benchmarks/syscallbench",
+			},
+			"sleep", "24h",
+		); err != nil {
+			b.Fatalf("run failed with: %v", err)
+		}
+
+		cmd := []string{"syscallbench", fmt.Sprintf("--loops=%d", b.N)}
+		b.ResetTimer()
+		out, err := container.Exec(ctx, dockerutil.ExecOpts{}, cmd...)
+		if err != nil {
+			b.Fatalf("failed to run syscallbench: %v, logs:%s", err, out)
+		}
+	})
+}

test/benchmarks/fs/fio_test.go

@@ -37,6 +37,11 @@ func BenchmarkFio(b *testing.B) {
 			BlockSize: 4,
 			IODepth:   4,
 		},
+		{
+			Test:      "write",
+			BlockSize: 64,
+			IODepth:   4,
+		},
 		{
 			Test:      "write",
 			BlockSize: 1024,
@@ -47,6 +52,11 @@ func BenchmarkFio(b *testing.B) {
 			BlockSize: 4,
 			IODepth:   4,
 		},
+		{
+			Test:      "read",
+			BlockSize: 64,
+			IODepth:   4,
+		},
 		{
 			Test:      "read",
 			BlockSize: 1024,
@@ -126,7 +136,7 @@ func BenchmarkFio(b *testing.B) {
 
 				// For reads, we need a file to read so make one inside the container.
 				if strings.Contains(tc.Test, "read") {
-					fallocateCmd := fmt.Sprintf("fallocate -l %dK %s", tc.Size, outfile)
+					fallocateCmd := fmt.Sprintf("fallocate -l %dM %s", tc.Size, outfile)
 					if out, err := container.Exec(ctx, dockerutil.ExecOpts{},
 						strings.Split(fallocateCmd, " ")...); err != nil {
 						b.Fatalf("failed to create readable file on mount: %v, %s", err, out)

test/benchmarks/network/iperf_test.go

@@ -106,6 +106,170 @@ func BenchmarkIperf(b *testing.B) {
 	}
 }
 
+func BenchmarkIperfParameterized(b *testing.B) {
+	clientMachine, err := harness.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer clientMachine.CleanUp()
+
+	serverMachine, err := harness.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer serverMachine.CleanUp()
+	ctx := context.Background()
+	for _, bm := range []struct {
+		name       string
+		length     int
+		parallel   int
+		clientFunc func(context.Context, testutil.Logger) *dockerutil.Container
+		serverFunc func(context.Context, testutil.Logger) *dockerutil.Container
+	}{
+		// We are either measuring the server or the client. The other should be
+		// runc. e.g. Upload sees how fast the runtime under test uploads to a native
+		// server.
+		{
+			name:       "Upload",
+			length:     4,
+			parallel:   1,
+			clientFunc: clientMachine.GetContainer,
+			serverFunc: serverMachine.GetNativeContainer,
+		},
+		{
+			name:       "Upload",
+			length:     64,
+			parallel:   1,
+			clientFunc: clientMachine.GetContainer,
+			serverFunc: serverMachine.GetNativeContainer,
+		},
+		{
+			name:       "Upload",
+			length:     1024,
+			parallel:   1,
+			clientFunc: clientMachine.GetContainer,
+			serverFunc: serverMachine.GetNativeContainer,
+		},
+		{
+			name:       "Upload",
+			length:     4,
+			parallel:   16,
+			clientFunc: clientMachine.GetContainer,
+			serverFunc: serverMachine.GetNativeContainer,
+		},
+		{
+			name:       "Upload",
+			length:     64,
+			parallel:   16,
+			clientFunc: clientMachine.GetContainer,
+			serverFunc: serverMachine.GetNativeContainer,
+		},
+		{
+			name:       "Upload",
+			length:     1024,
+			parallel:   16,
+			clientFunc: clientMachine.GetContainer,
+			serverFunc: serverMachine.GetNativeContainer,
+		},
+		{
+			name:       "Download",
+			length:     4,
+			parallel:   1,
+			clientFunc: clientMachine.GetNativeContainer,
+			serverFunc: serverMachine.GetContainer,
+		},
+		{
+			name:       "Download",
+			length:     64,
+			parallel:   1,
+			clientFunc: clientMachine.GetNativeContainer,
+			serverFunc: serverMachine.GetContainer,
+		},
+		{
+			name:       "Download",
+			length:     1024,
+			parallel:   1,
+			clientFunc: clientMachine.GetNativeContainer,
+			serverFunc: serverMachine.GetContainer,
+		},
+		{
+			name:       "Download",
+			length:     4,
+			parallel:   16,
+			clientFunc: clientMachine.GetNativeContainer,
+			serverFunc: serverMachine.GetContainer,
+		},
+		{
+			name:       "Download",
+			length:     64,
+			parallel:   16,
+			clientFunc: clientMachine.GetNativeContainer,
+			serverFunc: serverMachine.GetContainer,
+		},
+		{
+			name:       "Download",
+			length:     1024,
+			parallel:   16,
+			clientFunc: clientMachine.GetNativeContainer,
+			serverFunc: serverMachine.GetContainer,
+		},
+	} {
+		name, err := tools.ParametersToName(tools.Parameter{
+			Name:  "operation",
+			Value: bm.name,
+		}, tools.Parameter{
+			Name:  "length",
+			Value: fmt.Sprintf("%dK", bm.length),
+		}, tools.Parameter{
+			Name:  "parallel",
+			Value: fmt.Sprintf("%d", bm.parallel),
+		})
+		if err != nil {
+			b.Fatalf("Failed to parse parameters: %v", err)
+		}
+		b.Run(name, func(b *testing.B) {
+			// Set up the containers.
+			server := bm.serverFunc(ctx, b)
+			defer server.CleanUp(ctx)
+			client := bm.clientFunc(ctx, b)
+			defer client.CleanUp(ctx)
+
+			// iperf serves on port 5001 by default.
+			port := 5001
+
+			// Start the server.
+			if err := server.Spawn(ctx, dockerutil.RunOpts{
+				Image: "benchmarks/iperf",
+				Ports: []int{port},
+			}, "iperf", "-s"); err != nil {
+				b.Fatalf("failed to start server with: %v", err)
+			}
+			if out, err := server.WaitForOutput(ctx, fmt.Sprintf("Server listening on TCP port %d", port), 10*time.Second); err != nil {
+				b.Fatalf("failed to wait for iperf server: %v %s", err, out)
+			}
+
+			iperf := tools.Iperf{
+				Num:      b.N,       // KB for the client to send.
+				Length:   bm.length, // KB for length.
+				Parallel: bm.parallel,
+			}
+
+			// Run the client.
+			b.ResetTimer()
+			out, err := client.Run(ctx, dockerutil.RunOpts{
+				Image: "benchmarks/iperf",
+				Links: []string{server.MakeLink("iperfsrv")},
+			}, iperf.MakeCmd("iperfsrv", port)...)
+			if err != nil {
+				b.Fatalf("failed to run client: %v", err)
+			}
+			b.StopTimer()
+			iperf.Report(b, out)
+			b.StartTimer()
+		})
+	}
+}
+
 func TestMain(m *testing.M) {
 	harness.Init()
 	os.Exit(m.Run())

test/benchmarks/tools/BUILD

@@ -8,6 +8,7 @@ go_library(
     srcs = [
         "ab.go",
         "fio.go",
+        "hackbench.go",
         "hey.go",
         "iperf.go",
         "meminfo.go",

test/benchmarks/tools/hackbench.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tools
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"testing"
+)
+
+// Hackbench makes 'hackbench' commands and parses their output.
+type Hackbench struct {
+	IpcMode     string // ipc mode: pipe, socket(default)
+	ProcessMode string // process mode: thread, process(default)
+}
+
+// MakeCmd makes commands for Hackbench.
+func (s *Hackbench) MakeCmd(b *testing.B) []string {
+	cmd := []string{"hackbench"}
+	// ipc mode
+	if s.IpcMode == "pipe" {
+		cmd = append(cmd, "--pipe")
+	}
+	// group num
+	cmd = append(cmd, "--groups=10")
+	// process mode
+	if s.ProcessMode == "thread" {
+		cmd = append(cmd, "--threads")
+	} else {
+		cmd = append(cmd, "--process")
+	}
+	// loops
+	cmd = append(cmd, "--loops=1000")
+	return cmd
+}
+
+// Report reports the relevant metrics for Hackbench.
+func (s *Hackbench) Report(b *testing.B, output string) {
+	b.Helper()
+	result, err := s.parseResult(output)
+	if err != nil {
+		b.Fatalf("parsing result from %s failed: %v", output, err)
+	}
+	ReportCustomMetric(b, result, "execution_time" /*metric name*/, "s" /*unit*/)
+}
+
+var hackbenchRegexp = regexp.MustCompile(`Time:\s*(\d*.?\d*)\n`)
+
+func (s *Hackbench) parseResult(data string) (float64, error) {
+	match := hackbenchRegexp.FindStringSubmatch(data)
+	if len(match) < 2 {
+		return 0.0, fmt.Errorf("could not find Time: %s", data)
+	}
+	return strconv.ParseFloat(match[1], 64)
+}

test/benchmarks/tools/iperf.go

@@ -21,24 +21,30 @@ import (
 	"testing"
 )
 
-const length = 64 * 1024
-
 // Iperf is for the client side of `iperf`.
 type Iperf struct {
-	Num int // Number of bytes to send in KB.
+	Num      int // Number of bytes to send in KB.
+	Length   int // Length in KB.
+	Parallel int // Number of parallel threads.
 }
 
 // MakeCmd returns a iperf client command.
 func (i *Iperf) MakeCmd(host string, port int) []string {
-	return []string{
-		"iperf",
-		"--format", "K", // Output in KBytes.
-		"--realtime",                       // Measured in realtime.
-		"--num", fmt.Sprintf("%dK", i.Num), // Number of bytes to send in KB.
-		"--length", fmt.Sprintf("%d", length),
-		"--client", host,
-		"--port", fmt.Sprintf("%d", port),
+	cmd := []string{"iperf"}
+	cmd = append(cmd, "--format", "K")                    // Output in KBytes.
+	cmd = append(cmd, "--realtime")                       // Measured in realtime.
+	cmd = append(cmd, "--num", fmt.Sprintf("%dK", i.Num)) // Number of bytes to send in KB.
+	len := 128
+	if i.Length > 0 {
+		len = i.Length
 	}
+	cmd = append(cmd, "--len", fmt.Sprintf("%dK", len)) // Length in KB.
+	cmd = append(cmd, "--client", host)
+	cmd = append(cmd, "--port", fmt.Sprintf("%d", port))
+	if i.Parallel > 0 {
+		cmd = append(cmd, "--parallel", fmt.Sprintf("%d", i.Parallel))
+	}
+	return cmd
 }
 
 // Report parses output from iperf client and reports metrics.
@@ -49,7 +55,7 @@ func (i *Iperf) Report(b *testing.B, output string) {
 	if err != nil {
 		b.Fatalf("failed to parse bandwitdth from %s: %v", output, err)
 	}
-	b.SetBytes(length) // Measure Bytes/sec for b.N, although below is iperf output.
+	b.SetBytes(int64(i.Length) * 1024) // Measure Bytes/sec for b.N, although below is iperf output.
 	ReportCustomMetric(b, bW*1024, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/)
 }