diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 05a75514b..1e22d5a52 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -501,6 +501,9 @@ steps: - <<: *benchmarks label: ":weight_lifter: Size benchmarks" command: make -i benchmark-platforms BENCHMARKS_SUITE=size BENCHMARKS_TARGETS=test/benchmarks/base:size_test + - <<: *benchmarks + label: ":gorilla: Usage benchmarks" + command: make -i benchmark-platforms BENCHMARKS_SUITE=usage BENCHMARKS_TARGETS=test/benchmarks/base:usage_test - <<: *benchmarks label: ":speedboat: Startup benchmarks" command: make -i benchmark-platforms BENCHMARKS_SUITE=startup BENCHMARKS_TARGETS=test/benchmarks/base:startup_test diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 41f7aa63d..07feecee9 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -17,6 +17,7 @@ package dockerutil import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -428,6 +429,20 @@ func (c *Container) CopyFiles(opts *RunOpts, target string, sources ...string) { }) } +// Stats returns a snapshot of container stats similar to `docker stats`. +func (c *Container) Stats(ctx context.Context) (*types.StatsJSON, error) { + responseBody, err := c.client.ContainerStats(ctx, c.id, false /*stream*/) + if err != nil { + return nil, fmt.Errorf("ContainerStats failed: %v", err) + } + defer responseBody.Body.Close() + var v types.StatsJSON + if err := json.NewDecoder(responseBody.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("failed to decode container stats: %v", err) + } + return &v, nil +} + // Status inspects the container returns its status. func (c *Container) Status(ctx context.Context) (types.ContainerState, error) { resp, err := c.client.ContainerInspect(ctx, c.id) diff --git a/test/benchmarks/base/BUILD b/test/benchmarks/base/BUILD index 2572fefcb..e7b73d15f 100644 --- a/test/benchmarks/base/BUILD +++ b/test/benchmarks/base/BUILD @@ -70,3 +70,15 @@ benchmark_test( "//test/benchmarks/tools", ], ) + +benchmark_test( + name = "usage_test", + srcs = ["usage_test.go"], + deps = [ + ":base", + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + "@com_github_docker_docker//api/types:go_default_library", + ], +) diff --git a/test/benchmarks/base/base.go b/test/benchmarks/base/base.go index c3cd20b09..dc833b883 100644 --- a/test/benchmarks/base/base.go +++ b/test/benchmarks/base/base.go @@ -17,6 +17,7 @@ package base import ( "context" + "fmt" "testing" "time" @@ -40,22 +41,31 @@ func StartServers(ctx context.Context, b *testing.B, args ServerArgs) []*dockeru // Create N servers and wait until each of them is serving. for i := 0; i < b.N; i++ { - server := args.Machine.GetContainer(ctx, b) + server, err := StartServer(ctx, b, args) + if err != nil { + CleanUpContainers(ctx, servers) + b.Fatalf("failed to start server: %v", err) + } servers = append(servers, server) - if err := server.Spawn(ctx, args.RunOpts, args.Cmd...); err != nil { - CleanUpContainers(ctx, servers) - b.Fatalf("failed to spawn node instance: %v", err) - } - - // Wait until the server is up. - if err := harness.WaitUntilContainerServing(ctx, args.Machine, server, args.Port); err != nil { - CleanUpContainers(ctx, servers) - b.Fatalf("failed to wait for serving") - } } return servers } +// StartServer starts a single server and cleans it up if it fails. +func StartServer(ctx context.Context, b *testing.B, args ServerArgs) (*dockerutil.Container, error) { + server := args.Machine.GetContainer(ctx, b) + if err := server.Spawn(ctx, args.RunOpts, args.Cmd...); err != nil { + server.CleanUp(ctx) + return server, fmt.Errorf("failed to spawn server: %v", err) + } + // Wait until server is running. + if err := harness.WaitUntilContainerServing(ctx, args.Machine, server, args.Port); err != nil { + server.CleanUp(ctx) + return server, fmt.Errorf("failed to wait for serving server %q: %v", server.Name, err) + } + return server, nil +} + // CleanUpContainers cleans up a slice of containers. func CleanUpContainers(ctx context.Context, containers []*dockerutil.Container) { for _, c := range containers { diff --git a/test/benchmarks/base/usage_test.go b/test/benchmarks/base/usage_test.go new file mode 100644 index 000000000..aea491b09 --- /dev/null +++ b/test/benchmarks/base/usage_test.go @@ -0,0 +1,173 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package usage_test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/docker/docker/api/types" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/base" + "gvisor.dev/gvisor/test/benchmarks/harness" + "gvisor.dev/gvisor/test/benchmarks/tools" +) + +// BenchmarkSizeEmpty creates N alpine containers and reads memory usage using `docker stats`. +func BenchmarkSizeEmpty(b *testing.B) { + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer machine.CleanUp() + + var sumMemoryUsage uint64 + + // Make N containers. + for i := 0; i < b.N; i++ { + container := machine.GetContainer(ctx, b) + defer container.CleanUp(ctx) + if err := container.Spawn(ctx, dockerutil.RunOpts{ + Image: "benchmarks/alpine", + }, "sh", "-c", "echo Hello && sleep 1000"); err != nil { + b.Fatalf("failed to run container: %v", err) + } + if _, err := container.WaitForOutputSubmatch(ctx, "Hello", 5*time.Second); err != nil { + b.Fatalf("failed to read container output: %v", err) + } + + stats, err := container.Stats(ctx) + if err != nil { + b.Fatalf("failed to get container stats: %v", err) + } + if err := validateStats(stats); err != nil { + b.Fatalf("failed to validate container stats: %v", err) + } + sumMemoryUsage += stats.Stats.MemoryStats.Usage + } + reportMemoryUsage(b, sumMemoryUsage) +} + +// BenchmarkSizeNginx starts N containers running Nginx, checks that they're +// serving, and checks memory usage from `docker stats`. +func BenchmarkSizeNginx(b *testing.B) { + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine with: %v", err) + } + defer machine.CleanUp() + + // Make N Nginx containers. + runOpts := dockerutil.RunOpts{ + Image: "benchmarks/nginx", + } + const port = 80 + var sumMemoryUsage uint64 + for i := 0; i < b.N; i++ { + server, err := base.StartServer(ctx, b, + base.ServerArgs{ + Machine: machine, + Port: port, + RunOpts: runOpts, + Cmd: []string{"nginx", "-c", "/etc/nginx/nginx_gofer.conf"}, + }) + if err != nil { + b.Fatalf("failed to start server: %v", err) + } + defer server.CleanUp(ctx) + stats, err := server.Stats(ctx) + if err != nil { + b.Fatalf("failed to get container stats: %v", err) + } + if err := validateStats(stats); err != nil { + b.Fatalf("failed to validate container stats: %v", err) + } + sumMemoryUsage += stats.Stats.MemoryStats.Usage + } + reportMemoryUsage(b, sumMemoryUsage) +} + +// BenchmarkSizeNode starts N containers running a Node app, checks that +// they're serving, and checks memory used based on /proc/meminfo. +func BenchmarkSizeNode(b *testing.B) { + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine with: %v", err) + } + defer machine.CleanUp() + + // Make a redis instance for Node to connect. + redis := base.RedisInstance(ctx, b, machine) + defer redis.CleanUp(ctx) + + // Create N Node servers. + runOpts := dockerutil.RunOpts{ + Image: "benchmarks/node", + WorkDir: "/usr/src/app", + Links: []string{redis.MakeLink("redis")}, + } + nodeCmd := []string{"node", "index.js", "redis"} + const port = 8080 + var sumMemoryUsage uint64 + for i := 0; i < b.N; i++ { + server, err := base.StartServer(ctx, b, + base.ServerArgs{ + Machine: machine, + Port: port, + RunOpts: runOpts, + Cmd: nodeCmd, + }) + if err != nil { + b.Fatalf("failed to start server: %v", err) + } + defer server.CleanUp(ctx) + stats, err := server.Stats(ctx) + if err != nil { + b.Fatalf("failed to get container stats: %v", err) + } + if err := validateStats(stats); err != nil { + b.Fatalf("failed to validate container stats: %v", err) + } + sumMemoryUsage += stats.Stats.MemoryStats.Usage + } + reportMemoryUsage(b, sumMemoryUsage) +} + +func validateStats(stats *types.StatsJSON) error { + // The runc empty container is on the order of multiple kB, so if this is smaller than that, + // there is probably something wrong. + var memoryAtLeast uint64 = 1000 + if stats.MemoryStats.Usage < memoryAtLeast { + return fmt.Errorf("reported memory usage below sanity check minimum: %d: reported: %d", memoryAtLeast, stats.MemoryStats.Usage) + } + return nil +} + +func reportMemoryUsage(b *testing.B, sumMemoryUsage uint64) { + averageUsage := float64(sumMemoryUsage) / float64(b.N) + tools.ReportCustomMetric(b, averageUsage, "average_container_memory_usage", "bytes") +} + +// TestMain is the main method for this package. +func TestMain(m *testing.M) { + harness.Init() + os.Exit(m.Run()) +}