Add Docs to nginx benchmark.

Adds docs to nginx and refactors both Httpd and Nginx benchmarks.

Key changes:
- Add docs and make nginx tests the same as httpd (reverse, all docs, etc.).
- Make requests scale on c * b.N -> a request per thread. This works well
with both --test.benchtime=10m (do a run that lasts at least 10m) and
--test.benchtime=10x (do b.N = 10).
-- Remove a doc from both tests (1000Kb) as 1024Kb exists.

PiperOrigin-RevId: 329751091

images/benchmarks/httpd/Dockerfile

@@ -8,7 +8,7 @@ RUN set -x \
 
 # Generate a bunch of relevant files.
 RUN mkdir -p /local && \
-        for size in 1 10 100 1000 1024 10240; do \
+        for size in 1 10 100 1024 10240; do \
                 dd if=/dev/zero of=/local/latin${size}k.txt count=${size} bs=1024; \
         done
 

images/benchmarks/nginx/Dockerfile

@@ -1 +1,11 @@
 FROM nginx:1.15.10
+
+# Generate a bunch of relevant files.
+RUN mkdir -p /local && \
+        for size in 1 10 100 1024 10240; do \
+                dd if=/dev/zero of=/local/latin${size}k.txt count=${size} bs=1024; \
+        done
+
+RUN touch /local/index.html
+
+COPY ./nginx.conf /etc/nginx/nginx.conf

images/benchmarks/nginx/nginx.conf

@@ -0,0 +1,19 @@
+user  nginx;
+worker_processes  1;
+daemon off;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    server {
+      location / {
+        root /tmp/html;
+      }
+    }
+}

test/benchmarks/network/BUILD

@@ -5,8 +5,15 @@ package(licenses = ["notice"])
 go_library(
     name = "network",
     testonly = 1,
-    srcs = ["network.go"],
-    deps = ["//test/benchmarks/harness"],
+    srcs = [
+        "network.go",
+        "static_server.go",
+    ],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
 )
 
 go_test(

test/benchmarks/network/httpd_test.go

@@ -14,22 +14,19 @@
 package network
 
 import (
-	"context"
 	"fmt"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
-	"gvisor.dev/gvisor/test/benchmarks/harness"
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
 // see Dockerfile '//images/benchmarks/httpd'.
-var docs = map[string]string{
+var httpdDocs = map[string]string{
 	"notfound": "notfound",
 	"1Kb":      "latin1k.txt",
 	"10Kb":     "latin10k.txt",
 	"100Kb":    "latin100k.txt",
-	"1000Kb":   "latin1000k.txt",
 	"1Mb":      "latin1024k.txt",
 	"10Mb":     "latin10240k.txt",
 }
@@ -37,30 +34,17 @@ var docs = map[string]string{
 // BenchmarkHttpdConcurrency iterates the concurrency argument and tests
 // how well the runtime under test handles requests in parallel.
 func BenchmarkHttpdConcurrency(b *testing.B) {
-	// Grab a machine for the client and server.
-	clientMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get client: %v", err)
-	}
-	defer clientMachine.CleanUp()
-
-	serverMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get server: %v", err)
-	}
-	defer serverMachine.CleanUp()
-
 	// The test iterates over client concurrency, so set other parameters.
 	concurrency := []int{1, 25, 50, 100, 1000}
 
 	for _, c := range concurrency {
 		b.Run(fmt.Sprintf("%d", c), func(b *testing.B) {
 			hey := &tools.Hey{
-				Requests:    10000,
+				Requests:    c * b.N,
 				Concurrency: c,
-				Doc:         docs["10Kb"],
+				Doc:         httpdDocs["10Kb"],
 			}
-			runHttpd(b, clientMachine, serverMachine, hey, false /* reverse */)
+			runHttpd(b, hey, false /* reverse */)
 		})
 	}
 }
@@ -77,57 +61,30 @@ func BenchmarkReverseHttpdDocSize(b *testing.B) {
 	benchmarkHttpdDocSize(b, true /* reverse */)
 }
 
+// benchmarkHttpdDocSize iterates through all doc sizes, running subbenchmarks
+// for each size.
 func benchmarkHttpdDocSize(b *testing.B, reverse bool) {
 	b.Helper()
-
-	clientMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get machine: %v", err)
-	}
-	defer clientMachine.CleanUp()
-
-	serverMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get machine: %v", err)
-	}
-	defer serverMachine.CleanUp()
-
-	for name, filename := range docs {
+	for name, filename := range httpdDocs {
 		concurrency := []int{1, 25, 50, 100, 1000}
 		for _, c := range concurrency {
 			b.Run(fmt.Sprintf("%s_%d", name, c), func(b *testing.B) {
 				hey := &tools.Hey{
-					Requests:    10000,
+					Requests:    c * b.N,
 					Concurrency: c,
 					Doc:         filename,
 				}
-				runHttpd(b, clientMachine, serverMachine, hey, reverse)
+				runHttpd(b, hey, reverse)
 			})
 		}
 	}
 }
 
-// runHttpd runs a single test run.
-func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, hey *tools.Hey, reverse bool) {
-	b.Helper()
-
-	// Grab a container from the server.
-	ctx := context.Background()
-	var server *dockerutil.Container
-	if reverse {
-		server = serverMachine.GetNativeContainer(ctx, b)
-	} else {
-		server = serverMachine.GetContainer(ctx, b)
-	}
-
-	defer server.CleanUp(ctx)
-
-	// Copy the docs to /tmp and serve from there.
-	cmd := "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"
+// runHttpd configures the static serving methods to run httpd.
+func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) {
+	// httpd runs on port 80.
 	port := 80
-
-	// Start the server.
-	if err := server.Spawn(ctx, dockerutil.RunOpts{
+	httpdRunOpts := dockerutil.RunOpts{
 		Image: "benchmarks/httpd",
 		Ports: []int{port},
 		Env: []string{
@@ -138,44 +95,7 @@ func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, hey *t
 			"APACHE_LOG_DIR=/tmp",
 			"APACHE_PID_FILE=/tmp/apache.pid",
 		},
-	}, "sh", "-c", cmd); err != nil {
-		b.Fatalf("failed to start server: %v", err)
-	}
-
-	ip, err := serverMachine.IPAddress()
-	if err != nil {
-		b.Fatalf("failed to find server ip: %v", err)
-	}
-
-	servingPort, err := server.FindPort(ctx, port)
-	if err != nil {
-		b.Fatalf("failed to find server port %d: %v", port, err)
-	}
-
-	// Check the server is serving.
-	harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
-
-	var client *dockerutil.Container
-	// Grab a client.
-	if reverse {
-		client = clientMachine.GetContainer(ctx, b)
-	} else {
-		client = clientMachine.GetNativeContainer(ctx, b)
-	}
-	defer client.CleanUp(ctx)
-
-	b.ResetTimer()
-	server.RestartProfiles()
-	for i := 0; i < b.N; i++ {
-		out, err := client.Run(ctx, dockerutil.RunOpts{
-			Image: "benchmarks/hey",
-		}, hey.MakeCmd(ip, servingPort)...)
-		if err != nil {
-			b.Fatalf("run failed with: %v", err)
-		}
-
-		b.StopTimer()
-		hey.Report(b, out)
-		b.StartTimer()
 	}
+	httpdCmd := []string{"sh", "-c", "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"}
+	runStaticServer(b, httpdRunOpts, httpdCmd, port, hey, reverse)
 }

test/benchmarks/network/nginx_test.go

@@ -14,91 +14,80 @@
 package network
 
 import (
-	"context"
 	"fmt"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
-	"gvisor.dev/gvisor/test/benchmarks/harness"
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+// see Dockerfile '//images/benchmarks/nginx'.
+var nginxDocs = map[string]string{
+	"notfound": "notfound",
+	"1Kb":      "latin1k.txt",
+	"10Kb":     "latin10k.txt",
+	"100Kb":    "latin100k.txt",
+	"1Mb":      "latin1024k.txt",
+	"10Mb":     "latin10240k.txt",
+}
+
 // BenchmarkNginxConcurrency iterates the concurrency argument and tests
 // how well the runtime under test handles requests in parallel.
-// TODO(gvisor.dev/issue/3536): Update with different doc sizes like Httpd.
 func BenchmarkNginxConcurrency(b *testing.B) {
-	// Grab a machine for the client and server.
-	clientMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get client: %v", err)
-	}
-	defer clientMachine.CleanUp()
-
-	serverMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get server: %v", err)
-	}
-	defer serverMachine.CleanUp()
-
-	concurrency := []int{1, 5, 10, 25}
+	concurrency := []int{1, 25, 100, 1000}
 	for _, c := range concurrency {
 		b.Run(fmt.Sprintf("%d", c), func(b *testing.B) {
 			hey := &tools.Hey{
-				Requests:    10000,
+				Requests:    c * b.N,
 				Concurrency: c,
+				Doc:         nginxDocs["10kb"], // see Dockerfile '//images/benchmarks/nginx' and httpd_test.
 			}
-			runNginx(b, clientMachine, serverMachine, hey)
+			runNginx(b, hey, false /* reverse */)
 		})
 	}
 }
 
-// runHttpd runs a single test run.
-func runNginx(b *testing.B, clientMachine, serverMachine harness.Machine, hey *tools.Hey) {
+// BenchmarkNginxDocSize iterates over different sized payloads, testing how
+// well the runtime handles sending different payload sizes.
+func BenchmarkNginxDocSize(b *testing.B) {
+	benchmarkHttpdDocSize(b, false /* reverse */)
+}
+
+// BenchmarkReverseNginxDocSize iterates over different sized payloads, testing
+// how well the runtime handles receiving different payload sizes.
+func BenchmarkReverseNginxDocSize(b *testing.B) {
+	benchmarkHttpdDocSize(b, true /* reverse */)
+}
+
+// benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks
+// for each size.
+func benchmarkNginxDocSize(b *testing.B, reverse bool) {
 	b.Helper()
-
-	// Grab a container from the server.
-	ctx := context.Background()
-	server := serverMachine.GetContainer(ctx, b)
-	defer server.CleanUp(ctx)
-
-	port := 80
-	// Start the server.
-	if err := server.Spawn(ctx,
-		dockerutil.RunOpts{
-			Image: "benchmarks/nginx",
-			Ports: []int{port},
-		}); err != nil {
-		b.Fatalf("server failed to start: %v", err)
-	}
-
-	ip, err := serverMachine.IPAddress()
-	if err != nil {
-		b.Fatalf("failed to find server ip: %v", err)
-	}
-
-	servingPort, err := server.FindPort(ctx, port)
-	if err != nil {
-		b.Fatalf("failed to find server port %d: %v", port, err)
-	}
-
-	// Check the server is serving.
-	harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
-
-	// Grab a client.
-	client := clientMachine.GetNativeContainer(ctx, b)
-	defer client.CleanUp(ctx)
-
-	b.ResetTimer()
-	server.RestartProfiles()
-	for i := 0; i < b.N; i++ {
-		out, err := client.Run(ctx, dockerutil.RunOpts{
-			Image: "benchmarks/hey",
-		}, hey.MakeCmd(ip, servingPort)...)
-		if err != nil {
-			b.Fatalf("run failed with: %v", err)
+	for name, filename := range nginxDocs {
+		concurrency := []int{1, 25, 50, 100, 1000}
+		for _, c := range concurrency {
+			b.Run(fmt.Sprintf("%s_%d", name, c), func(b *testing.B) {
+				hey := &tools.Hey{
+					Requests:    c * b.N,
+					Concurrency: c,
+					Doc:         filename,
+				}
+				runNginx(b, hey, reverse)
+			})
 		}
-		b.StopTimer()
-		hey.Report(b, out)
-		b.StartTimer()
 	}
 }
+
+// runNginx configures the static serving methods to run httpd.
+func runNginx(b *testing.B, hey *tools.Hey, reverse bool) {
+	// nginx runs on port 80.
+	port := 80
+	nginxRunOpts := dockerutil.RunOpts{
+		Image: "benchmarks/nginx",
+		Ports: []int{port},
+	}
+
+	// Command copies nginxDocs to tmpfs serving directory and runs nginx.
+	nginxCmd := []string{"sh", "-c", "mkdir -p /tmp/html && cp -a /local/* /tmp/html && nginx"}
+	runStaticServer(b, nginxRunOpts, nginxCmd, port, hey, reverse)
+}

test/benchmarks/network/static_server.go

@@ -0,0 +1,88 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package network
+
+import (
+	"context"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
+	"gvisor.dev/gvisor/test/benchmarks/tools"
+)
+
+// runStaticServer runs static serving workloads (httpd, nginx).
+func runStaticServer(b *testing.B, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) {
+	b.Helper()
+	ctx := context.Background()
+
+	// Get two machines: a client and server.
+	clientMachine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer clientMachine.CleanUp()
+
+	serverMachine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer serverMachine.CleanUp()
+
+	// Make the containers. 'reverse=true' specifies that the client should use the
+	// runtime under test.
+	var client, server *dockerutil.Container
+	if reverse {
+		client = clientMachine.GetContainer(ctx, b)
+		server = serverMachine.GetNativeContainer(ctx, b)
+	} else {
+		client = clientMachine.GetNativeContainer(ctx, b)
+		server = serverMachine.GetContainer(ctx, b)
+	}
+	defer client.CleanUp(ctx)
+	defer server.CleanUp(ctx)
+
+	// Start the server.
+	if err := server.Spawn(ctx, serverOpts, serverCmd...); err != nil {
+		b.Fatalf("failed to start server: %v", err)
+	}
+
+	// Get its IP.
+	ip, err := serverMachine.IPAddress()
+	if err != nil {
+		b.Fatalf("failed to find server ip: %v", err)
+	}
+
+	// Get the published port.
+	servingPort, err := server.FindPort(ctx, port)
+	if err != nil {
+		b.Fatalf("failed to find server port %d: %v", port, err)
+	}
+
+	// Make sure the server is serving.
+	harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
+	b.ResetTimer()
+	server.RestartProfiles()
+	out, err := client.Run(ctx, dockerutil.RunOpts{
+		Image: "benchmarks/hey",
+	}, hey.MakeCmd(ip, servingPort)...)
+	if err != nil {
+		b.Fatalf("run failed with: %v", err)
+	}
+
+	b.StopTimer()
+	hey.Report(b, out)
+	b.StartTimer()
+}