From cc0dc87fa27d14506404b2953d89d093b9c2019c Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Thu, 19 Jan 2023 20:58:26 -0800 Subject: [PATCH] Add usage benchmarks Add the usage benchmark which queries memory usage from the container. This is meant to replace the size benchmark which has a ton of noise. Flakes are due to the complicated measurement technique of dropping caches -> parse /proc/meminfo -> start containers -> drops caches again -> parse /proc/meminfo -> take the diference. This was done because not all runtimes support docker/containerd stats. Buuuut...gVisor does, so we'll just take the stats figures to reduce noise for the memory usage benchmark. In addition, the returned types.StatsJSON object contains several useful stats like page faults and CPU usage, so adding these will be trivial and will not slow down benchmark performance. [types.StatsJSON linky link](https://github.com/moby/moby/blob/42c8b314993e5eb3cc2776da0bbe41d5eb4b707b/api/types/stats.go#LL153) PiperOrigin-RevId: 503342363 --- .buildkite/pipeline.yaml | 3 + pkg/test/dockerutil/container.go | 15 +++ test/benchmarks/base/BUILD | 12 ++ test/benchmarks/base/base.go | 32 ++++-- test/benchmarks/base/usage_test.go | 173 +++++++++++++++++++++++++++++ 5 files changed, 224 insertions(+), 11 deletions(-) create mode 100644 test/benchmarks/base/usage_test.go diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 05a75514b..1e22d5a52 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -501,6 +501,9 @@ steps: - <<: *benchmarks label: ":weight_lifter: Size benchmarks" command: make -i benchmark-platforms BENCHMARKS_SUITE=size BENCHMARKS_TARGETS=test/benchmarks/base:size_test + - <<: *benchmarks + label: ":gorilla: Usage benchmarks" + command: make -i benchmark-platforms BENCHMARKS_SUITE=usage BENCHMARKS_TARGETS=test/benchmarks/base:usage_test - <<: *benchmarks label: ":speedboat: Startup benchmarks" command: make -i benchmark-platforms BENCHMARKS_SUITE=startup BENCHMARKS_TARGETS=test/benchmarks/base:startup_test diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 41f7aa63d..07feecee9 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -17,6 +17,7 @@ package dockerutil import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -428,6 +429,20 @@ func (c *Container) CopyFiles(opts *RunOpts, target string, sources ...string) { }) } +// Stats returns a snapshot of container stats similar to `docker stats`. +func (c *Container) Stats(ctx context.Context) (*types.StatsJSON, error) { + responseBody, err := c.client.ContainerStats(ctx, c.id, false /*stream*/) + if err != nil { + return nil, fmt.Errorf("ContainerStats failed: %v", err) + } + defer responseBody.Body.Close() + var v types.StatsJSON + if err := json.NewDecoder(responseBody.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("failed to decode container stats: %v", err) + } + return &v, nil +} + // Status inspects the container returns its status. func (c *Container) Status(ctx context.Context) (types.ContainerState, error) { resp, err := c.client.ContainerInspect(ctx, c.id) diff --git a/test/benchmarks/base/BUILD b/test/benchmarks/base/BUILD index 2572fefcb..e7b73d15f 100644 --- a/test/benchmarks/base/BUILD +++ b/test/benchmarks/base/BUILD @@ -70,3 +70,15 @@ benchmark_test( "//test/benchmarks/tools", ], ) + +benchmark_test( + name = "usage_test", + srcs = ["usage_test.go"], + deps = [ + ":base", + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + "@com_github_docker_docker//api/types:go_default_library", + ], +) diff --git a/test/benchmarks/base/base.go b/test/benchmarks/base/base.go index c3cd20b09..dc833b883 100644 --- a/test/benchmarks/base/base.go +++ b/test/benchmarks/base/base.go @@ -17,6 +17,7 @@ package base import ( "context" + "fmt" "testing" "time" @@ -40,22 +41,31 @@ func StartServers(ctx context.Context, b *testing.B, args ServerArgs) []*dockeru // Create N servers and wait until each of them is serving. for i := 0; i < b.N; i++ { - server := args.Machine.GetContainer(ctx, b) + server, err := StartServer(ctx, b, args) + if err != nil { + CleanUpContainers(ctx, servers) + b.Fatalf("failed to start server: %v", err) + } servers = append(servers, server) - if err := server.Spawn(ctx, args.RunOpts, args.Cmd...); err != nil { - CleanUpContainers(ctx, servers) - b.Fatalf("failed to spawn node instance: %v", err) - } - - // Wait until the server is up. - if err := harness.WaitUntilContainerServing(ctx, args.Machine, server, args.Port); err != nil { - CleanUpContainers(ctx, servers) - b.Fatalf("failed to wait for serving") - } } return servers } +// StartServer starts a single server and cleans it up if it fails. +func StartServer(ctx context.Context, b *testing.B, args ServerArgs) (*dockerutil.Container, error) { + server := args.Machine.GetContainer(ctx, b) + if err := server.Spawn(ctx, args.RunOpts, args.Cmd...); err != nil { + server.CleanUp(ctx) + return server, fmt.Errorf("failed to spawn server: %v", err) + } + // Wait until server is running. + if err := harness.WaitUntilContainerServing(ctx, args.Machine, server, args.Port); err != nil { + server.CleanUp(ctx) + return server, fmt.Errorf("failed to wait for serving server %q: %v", server.Name, err) + } + return server, nil +} + // CleanUpContainers cleans up a slice of containers. func CleanUpContainers(ctx context.Context, containers []*dockerutil.Container) { for _, c := range containers { diff --git a/test/benchmarks/base/usage_test.go b/test/benchmarks/base/usage_test.go new file mode 100644 index 000000000..aea491b09 --- /dev/null +++ b/test/benchmarks/base/usage_test.go @@ -0,0 +1,173 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package usage_test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/docker/docker/api/types" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/base" + "gvisor.dev/gvisor/test/benchmarks/harness" + "gvisor.dev/gvisor/test/benchmarks/tools" +) + +// BenchmarkSizeEmpty creates N alpine containers and reads memory usage using `docker stats`. +func BenchmarkSizeEmpty(b *testing.B) { + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer machine.CleanUp() + + var sumMemoryUsage uint64 + + // Make N containers. + for i := 0; i < b.N; i++ { + container := machine.GetContainer(ctx, b) + defer container.CleanUp(ctx) + if err := container.Spawn(ctx, dockerutil.RunOpts{ + Image: "benchmarks/alpine", + }, "sh", "-c", "echo Hello && sleep 1000"); err != nil { + b.Fatalf("failed to run container: %v", err) + } + if _, err := container.WaitForOutputSubmatch(ctx, "Hello", 5*time.Second); err != nil { + b.Fatalf("failed to read container output: %v", err) + } + + stats, err := container.Stats(ctx) + if err != nil { + b.Fatalf("failed to get container stats: %v", err) + } + if err := validateStats(stats); err != nil { + b.Fatalf("failed to validate container stats: %v", err) + } + sumMemoryUsage += stats.Stats.MemoryStats.Usage + } + reportMemoryUsage(b, sumMemoryUsage) +} + +// BenchmarkSizeNginx starts N containers running Nginx, checks that they're +// serving, and checks memory usage from `docker stats`. +func BenchmarkSizeNginx(b *testing.B) { + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine with: %v", err) + } + defer machine.CleanUp() + + // Make N Nginx containers. + runOpts := dockerutil.RunOpts{ + Image: "benchmarks/nginx", + } + const port = 80 + var sumMemoryUsage uint64 + for i := 0; i < b.N; i++ { + server, err := base.StartServer(ctx, b, + base.ServerArgs{ + Machine: machine, + Port: port, + RunOpts: runOpts, + Cmd: []string{"nginx", "-c", "/etc/nginx/nginx_gofer.conf"}, + }) + if err != nil { + b.Fatalf("failed to start server: %v", err) + } + defer server.CleanUp(ctx) + stats, err := server.Stats(ctx) + if err != nil { + b.Fatalf("failed to get container stats: %v", err) + } + if err := validateStats(stats); err != nil { + b.Fatalf("failed to validate container stats: %v", err) + } + sumMemoryUsage += stats.Stats.MemoryStats.Usage + } + reportMemoryUsage(b, sumMemoryUsage) +} + +// BenchmarkSizeNode starts N containers running a Node app, checks that +// they're serving, and checks memory used based on /proc/meminfo. +func BenchmarkSizeNode(b *testing.B) { + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine with: %v", err) + } + defer machine.CleanUp() + + // Make a redis instance for Node to connect. + redis := base.RedisInstance(ctx, b, machine) + defer redis.CleanUp(ctx) + + // Create N Node servers. + runOpts := dockerutil.RunOpts{ + Image: "benchmarks/node", + WorkDir: "/usr/src/app", + Links: []string{redis.MakeLink("redis")}, + } + nodeCmd := []string{"node", "index.js", "redis"} + const port = 8080 + var sumMemoryUsage uint64 + for i := 0; i < b.N; i++ { + server, err := base.StartServer(ctx, b, + base.ServerArgs{ + Machine: machine, + Port: port, + RunOpts: runOpts, + Cmd: nodeCmd, + }) + if err != nil { + b.Fatalf("failed to start server: %v", err) + } + defer server.CleanUp(ctx) + stats, err := server.Stats(ctx) + if err != nil { + b.Fatalf("failed to get container stats: %v", err) + } + if err := validateStats(stats); err != nil { + b.Fatalf("failed to validate container stats: %v", err) + } + sumMemoryUsage += stats.Stats.MemoryStats.Usage + } + reportMemoryUsage(b, sumMemoryUsage) +} + +func validateStats(stats *types.StatsJSON) error { + // The runc empty container is on the order of multiple kB, so if this is smaller than that, + // there is probably something wrong. + var memoryAtLeast uint64 = 1000 + if stats.MemoryStats.Usage < memoryAtLeast { + return fmt.Errorf("reported memory usage below sanity check minimum: %d: reported: %d", memoryAtLeast, stats.MemoryStats.Usage) + } + return nil +} + +func reportMemoryUsage(b *testing.B, sumMemoryUsage uint64) { + averageUsage := float64(sumMemoryUsage) / float64(b.N) + tools.ReportCustomMetric(b, averageUsage, "average_container_memory_usage", "bytes") +} + +// TestMain is the main method for this package. +func TestMain(m *testing.M) { + harness.Init() + os.Exit(m.Run()) +}