From 8079a6cb03b344f18756b4d34898cb724dfd75aa Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Mon, 25 Nov 2024 17:08:30 -0800 Subject: [PATCH] Kubernetes tests: Librarify all the tests and benchmarks. These benchmarks are used as libraries within Google to run continuous benchmarks. PiperOrigin-RevId: 700146722 --- test/kubernetes/benchmarks/BUILD | 286 ++++-- test/kubernetes/benchmarks/abslbuild.go | 193 ++++ test/kubernetes/benchmarks/abslbuild_test.go | 172 +--- test/kubernetes/benchmarks/ffmpeg.go | 181 ++++ test/kubernetes/benchmarks/ffmpeg_test.go | 161 +--- test/kubernetes/benchmarks/grpc.go | 175 ++++ test/kubernetes/benchmarks/grpc_test.go | 154 +--- test/kubernetes/benchmarks/gsutil.go | 200 ++++ test/kubernetes/benchmarks/gsutil_test.go | 180 +--- test/kubernetes/benchmarks/nginx.go | 291 ++++++ test/kubernetes/benchmarks/nginx_test.go | 271 +----- test/kubernetes/benchmarks/ollama.go | 859 ++++++++++++++++++ test/kubernetes/benchmarks/ollama_test.go | 838 +---------------- test/kubernetes/benchmarks/postgresql.go | 371 ++++++++ test/kubernetes/benchmarks/postgresql_test.go | 353 +------ test/kubernetes/benchmarks/pytorch.go | 381 ++++++++ test/kubernetes/benchmarks/pytorch_test.go | 364 +------- test/kubernetes/benchmarks/redis.go | 454 +++++++++ test/kubernetes/benchmarks/redis_test.go | 435 +-------- .../{ruby_dev_test.go => rubydev.go} | 29 +- test/kubernetes/benchmarks/rubydev_test.go | 44 + test/kubernetes/benchmarks/stablediffusion.go | 220 +++++ .../benchmarks/stablediffusion_test.go | 201 +--- test/kubernetes/benchmarks/startup.go | 116 +++ test/kubernetes/benchmarks/startup_test.go | 92 +- test/kubernetes/benchmarks/tensorflow.go | 152 ++++ test/kubernetes/benchmarks/tensorflow_test.go | 132 +-- test/kubernetes/benchmarks/wordpress.go | 379 ++++++++ test/kubernetes/benchmarks/wordpress_test.go | 359 +------- test/kubernetes/tests/BUILD | 27 +- test/kubernetes/tests/hello.go | 64 ++ test/kubernetes/tests/hello_test.go | 42 +- 32 files changed, 4359 insertions(+), 3817 deletions(-) create mode 100644 test/kubernetes/benchmarks/abslbuild.go create mode 100644 test/kubernetes/benchmarks/ffmpeg.go create mode 100644 test/kubernetes/benchmarks/grpc.go create mode 100644 test/kubernetes/benchmarks/gsutil.go create mode 100644 test/kubernetes/benchmarks/nginx.go create mode 100644 test/kubernetes/benchmarks/ollama.go create mode 100644 test/kubernetes/benchmarks/postgresql.go create mode 100644 test/kubernetes/benchmarks/pytorch.go create mode 100644 test/kubernetes/benchmarks/redis.go rename test/kubernetes/benchmarks/{ruby_dev_test.go => rubydev.go} (88%) create mode 100644 test/kubernetes/benchmarks/rubydev_test.go create mode 100644 test/kubernetes/benchmarks/stablediffusion.go create mode 100644 test/kubernetes/benchmarks/startup.go create mode 100644 test/kubernetes/benchmarks/tensorflow.go create mode 100644 test/kubernetes/benchmarks/wordpress.go create mode 100644 test/kubernetes/tests/hello.go diff --git a/test/kubernetes/benchmarks/BUILD b/test/kubernetes/benchmarks/BUILD index eac0f2c1f..ffc782041 100644 --- a/test/kubernetes/benchmarks/BUILD +++ b/test/kubernetes/benchmarks/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_test", "pkg_tar") +load("//tools:defs.bzl", "go_library", "go_test") package( default_applicable_licenses = ["//:license"], @@ -30,28 +30,11 @@ filegroup( srcs = _ALL_BENCHMARK_TARGETS, ) -[pkg_tar( - name = "%s_tar" % (src[src.index(":") + 1:],), +go_library( + name = "abslbuild", testonly = True, - srcs = [src], - extension = "tar.bz2", -) for src in _ALL_BENCHMARK_TARGETS] - -filegroup( - name = "all_benchmark_test_binaries_tar", - testonly = True, - srcs = ["%s_tar" % (src[src.index(":") + 1:],) for src in _ALL_BENCHMARK_TARGETS], -) - -go_test( - name = "abslbuild_test", - srcs = ["abslbuild_test.go"], + srcs = ["abslbuild.go"], nogo = False, - tags = [ - "local", - "noguitar", - "notap", - ], deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/profiling", @@ -64,14 +47,26 @@ go_test( ) go_test( - name = "startup_test", - srcs = ["startup_test.go"], + name = "abslbuild_test", + srcs = ["abslbuild_test.go"], + library = ":abslbuild", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "startup", + testonly = True, + srcs = ["startup.go"], + nogo = False, deps = [ "//test/kubernetes/benchmarks/profiling", "//test/kubernetes/benchmetric", @@ -82,14 +77,26 @@ go_test( ) go_test( - name = "redis_test", - srcs = ["redis_test.go"], + name = "startup_test", + srcs = ["startup_test.go"], + library = ":startup", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "redis", + testonly = True, + srcs = ["redis.go"], + nogo = False, deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/profiling", @@ -102,8 +109,9 @@ go_test( ) go_test( - name = "ruby_dev_test", - srcs = ["ruby_dev_test.go"], + name = "redis_test", + srcs = ["redis_test.go"], + library = ":redis", nogo = False, tags = [ "local", @@ -111,7 +119,17 @@ go_test( "notap", ], deps = [ - "//test/benchmarks/tools", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "ffmpeg", + testonly = True, + srcs = ["ffmpeg.go"], + nogo = False, + deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/profiling", "//test/kubernetes/benchmetric", @@ -125,12 +143,24 @@ go_test( go_test( name = "ffmpeg_test", srcs = ["ffmpeg_test.go"], + library = ":ffmpeg", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "grpc", + testonly = True, + srcs = ["grpc.go"], + nogo = False, deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/profiling", @@ -145,6 +175,7 @@ go_test( go_test( name = "grpc_test", srcs = ["grpc_test.go"], + library = ":grpc", nogo = False, tags = [ "local", @@ -152,25 +183,16 @@ go_test( "notap", ], deps = [ - "//test/kubernetes", - "//test/kubernetes/benchmarks/profiling", - "//test/kubernetes/benchmetric", "//test/kubernetes/k8sctx", "//test/kubernetes/testcluster", - "@io_k8s_api//core/v1:go_default_library", - "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", ], ) -go_test( - name = "nginx_test", - srcs = ["nginx_test.go"], +go_library( + name = "nginx", + testonly = True, + srcs = ["nginx.go"], nogo = False, - tags = [ - "local", - "noguitar", - "notap", - ], deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/httpbench", @@ -183,9 +205,41 @@ go_test( ], ) +go_test( + name = "nginx_test", + srcs = ["nginx_test.go"], + library = ":nginx", + nogo = False, + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "postgresql", + testonly = True, + srcs = ["postgresql.go"], + nogo = False, + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + go_test( name = "postgresql_test", srcs = ["postgresql_test.go"], + library = ":postgresql", nogo = False, tags = [ "local", @@ -193,24 +247,16 @@ go_test( "notap", ], deps = [ - "//test/kubernetes/benchmarks/profiling", - "//test/kubernetes/benchmetric", "//test/kubernetes/k8sctx", "//test/kubernetes/testcluster", - "@io_k8s_api//core/v1:go_default_library", - "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", ], ) -go_test( - name = "tensorflow_test", - srcs = ["tensorflow_test.go"], +go_library( + name = "tensorflow", + testonly = True, + srcs = ["tensorflow.go"], nogo = False, - tags = [ - "local", - "noguitar", - "notap", - ], deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/profiling", @@ -223,14 +269,26 @@ go_test( ) go_test( - name = "wordpress_test", - srcs = ["wordpress_test.go"], + name = "tensorflow_test", + srcs = ["tensorflow_test.go"], + library = ":tensorflow", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "wordpress", + testonly = True, + srcs = ["wordpress.go"], + nogo = False, deps = [ "//test/kubernetes/benchmarks/httpbench", "//test/kubernetes/benchmarks/profiling", @@ -243,14 +301,26 @@ go_test( ) go_test( - name = "pytorch_test", - srcs = ["pytorch_test.go"], + name = "wordpress_test", + srcs = ["wordpress_test.go"], + library = ":wordpress", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "pytorch", + testonly = True, + srcs = ["pytorch.go"], + nogo = False, deps = [ "//test/kubernetes", "//test/kubernetes/benchmarks/profiling", @@ -262,17 +332,29 @@ go_test( ) go_test( - name = "ollama_test", - srcs = ["ollama_test.go"], - embedsrcs = [ - "//test/kubernetes/benchmarks/resources:files", # keep - ], + name = "pytorch_test", + srcs = ["pytorch_test.go"], + library = ":pytorch", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "ollama", + testonly = True, + srcs = ["ollama.go"], + embedsrcs = [ + "//test/kubernetes/benchmarks/resources:files", # keep + ], + nogo = False, deps = [ "//test/gpu/ollama", "//test/kubernetes", @@ -287,14 +369,26 @@ go_test( ) go_test( - name = "stablediffusion_test", - srcs = ["stablediffusion_test.go"], + name = "ollama_test", + srcs = ["ollama_test.go"], + library = ":ollama", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "stablediffusion", + testonly = True, + srcs = ["stablediffusion.go"], + nogo = False, deps = [ "//test/gpu/stablediffusion", "//test/kubernetes", @@ -307,14 +401,26 @@ go_test( ) go_test( - name = "gsutil_test", - srcs = ["gsutil_test.go"], + name = "stablediffusion_test", + srcs = ["stablediffusion_test.go"], + library = ":stablediffusion", nogo = False, tags = [ "local", "noguitar", "notap", ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "gsutil", + testonly = True, + srcs = ["gsutil.go"], + nogo = False, deps = [ "//test/kubernetes/benchmarks/profiling", "//test/kubernetes/benchmetric", @@ -324,3 +430,51 @@ go_test( "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", ], ) + +go_test( + name = "gsutil_test", + srcs = ["gsutil_test.go"], + library = ":gsutil", + nogo = False, + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) + +go_library( + name = "rubydev", + testonly = True, + srcs = ["rubydev.go"], + nogo = False, + deps = [ + "//test/benchmarks/tools", + "//test/kubernetes", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "rubydev_test", + srcs = ["rubydev_test.go"], + library = ":rubydev", + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + ], +) diff --git a/test/kubernetes/benchmarks/abslbuild.go b/test/kubernetes/benchmarks/abslbuild.go new file mode 100644 index 000000000..f82d939cf --- /dev/null +++ b/test/kubernetes/benchmarks/abslbuild.go @@ -0,0 +1,193 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abslbuild + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = k8s.ImageRepoPrefix + "benchmarks/absl_x86_64:latest" +) + +// BuildABSL runs the ABSL build benchmark. +// This benchmark builds the ABSL library from source. +func BuildABSL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "absl" + + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + t.Skipf("Building ABSL is not supported on ARM") + return + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + pod := newABSLPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("ABSL/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +func newABSLPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", "/abseil-cpp", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{ + "bazel", + "build", + "//absl/base", + "//absl/algorithm", + "//absl/container/...", + "//absl/debugging/...", + "//absl/flags:flag", + "//absl/hash", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/numeric:int128", + "//absl/strings", + "//absl/synchronization", + "//absl/time", + "//absl/types/...", + "//absl/utility", + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "abseil-cpp"), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} diff --git a/test/kubernetes/benchmarks/abslbuild_test.go b/test/kubernetes/benchmarks/abslbuild_test.go index d3e9d0bca..7e69da6f6 100644 --- a/test/kubernetes/benchmarks/abslbuild_test.go +++ b/test/kubernetes/benchmarks/abslbuild_test.go @@ -12,26 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package abslbuild_test +package abslbuild import ( "context" - "fmt" - "path" - "strings" "testing" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - imageAMD = k8s.ImageRepoPrefix + "benchmarks/absl_x86_64:latest" ) // TestABSLBuild benchmarks building various Abseil C++ targets. @@ -44,167 +32,11 @@ func TestABSLBuild(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("ABSL", func(t *testing.T) { t.Parallel() - doABSLBuild(ctx, t, k8sCtx, cluster) + BuildABSL(ctx, t, k8sCtx, cluster) }) }) } -func doABSLBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - const name = "absl" - - persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") - persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - image := imageAMD - if cluster.RuntimeTestNodepoolIsARM() { - t.Skipf("Building ABSL is not supported on ARM") - return - } - if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - - for _, test := range []struct { - name string - volume *v13.Volume - }{ - { - name: "RootFS", - volume: nil, - }, - { - name: "EmptyDir", - volume: &v13.Volume{ - Name: "emptydir", - VolumeSource: v13.VolumeSource{ - EmptyDir: &v13.EmptyDirVolumeSource{}, - }, - }, - }, - { - name: "PersistentVolume", - volume: &v13.Volume{ - Name: persistentVol.GetName(), - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: persistentVol.GetName(), - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - pod := newABSLPod(benchmarkNS, name, image, test.volume) - pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) - if err != nil { - t.Fatalf("Failed to set pod for test runtime: %v", err) - } - - pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - - pod, err = cluster.CreatePod(ctx, pod) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, pod) - - containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - if err := recorder.Record(ctx, fmt.Sprintf("ABSL/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } -} - -func newABSLPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { - const workdir = "/workdir" - initCommand := []string{ - "sh", - "-c", - strings.Join([]string{ - "mkdir", "-p", workdir, - "&&", - "cp", "-r", "/abseil-cpp", fmt.Sprintf("%s/.", workdir), - }, " "), - } - command := []string{ - "bazel", - "build", - "//absl/base", - "//absl/algorithm", - "//absl/container/...", - "//absl/debugging/...", - "//absl/flags:flag", - "//absl/hash", - "//absl/memory", - "//absl/meta:type_traits", - "//absl/numeric:int128", - "//absl/strings", - "//absl/synchronization", - "//absl/time", - "//absl/types/...", - "//absl/utility", - } - var volumes []v13.Volume - var volumeMounts []v13.VolumeMount - if volume != nil { - volumes = []v13.Volume{*volume} - volumeMounts = []v13.VolumeMount{{ - MountPath: workdir, - Name: volume.Name, - }} - } - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: name, - Namespace: namespace.Namespace, - }, - Spec: v13.PodSpec{ - Volumes: volumes, - Containers: []v13.Container{ - { - Name: name, - Image: image, - Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "abseil-cpp"), command), - VolumeMounts: volumeMounts, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestABSLBuild": TestABSLBuild, diff --git a/test/kubernetes/benchmarks/ffmpeg.go b/test/kubernetes/benchmarks/ffmpeg.go new file mode 100644 index 000000000..6980cc7b1 --- /dev/null +++ b/test/kubernetes/benchmarks/ffmpeg.go @@ -0,0 +1,181 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ffmpeg + +import ( + "context" + "fmt" + "strings" + "testing" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + ffmpegContainerName = "ffmpeg" + imageAMD = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_x86_64:latest" + imageARM = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_aarch64:latest" +) + +// RunFFMPEG runs the ffmpeg benchmark. +func RunFFMPEG(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "ffmpeg" + + // create persistent volume + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + p := newFfmpegDevPod(benchmarkNS, name, image, test.volume) + p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) + if err != nil { + t.Fatalf("Failed to configure pod for runtime: %v", err) + } + p, err = testcluster.MaybeSetContainerResources(p, ffmpegContainerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + p, err = cluster.CreatePod(ctx, p) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, p) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, ffmpegContainerName) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if recorder.Record(ctx, fmt.Sprintf("FFMPEG/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newFfmpegPod creates a new ffmpeg dev pod spec for benchmarks. +func newFfmpegDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "/media/video.mp4", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{ + "ffmpeg", + "-i", "video.mp4", + "-c:v", "libx264", + "-preset", "veryslow", + "output.mp4", + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: ffmpegContainerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, workdir, command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} diff --git a/test/kubernetes/benchmarks/ffmpeg_test.go b/test/kubernetes/benchmarks/ffmpeg_test.go index e102483ef..bc6e295dc 100644 --- a/test/kubernetes/benchmarks/ffmpeg_test.go +++ b/test/kubernetes/benchmarks/ffmpeg_test.go @@ -12,27 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ffmpeg_test +package ffmpeg import ( "context" - "fmt" - "strings" "testing" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - ffmpegContainerName = "ffmpeg" - imageAMD = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_x86_64:latest" - imageARM = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_aarch64:latest" ) func TestFfmpeg(t *testing.T) { @@ -44,155 +31,11 @@ func TestFfmpeg(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("ffmpeg", func(t *testing.T) { t.Parallel() - doFfmpegTest(ctx, t, k8sCtx, cluster) + RunFFMPEG(ctx, t, k8sCtx, cluster) }) }) } -func doFfmpegTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - const name = "ffmpeg" - - // create persistent volume - persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") - persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - image := imageAMD - if cluster.RuntimeTestNodepoolIsARM() { - image = imageARM - } - if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - - for _, test := range []struct { - name string - volume *v13.Volume - }{ - { - name: "RootFS", - volume: nil, - }, - { - name: "EmptyDir", - volume: &v13.Volume{ - Name: "emptydir", - VolumeSource: v13.VolumeSource{ - EmptyDir: &v13.EmptyDirVolumeSource{}, - }, - }, - }, - { - name: "PersistentVolume", - volume: &v13.Volume{ - Name: persistentVol.GetName(), - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: persistentVol.GetName(), - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - p := newFfmpegDevPod(benchmarkNS, name, image, test.volume) - p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) - if err != nil { - t.Fatalf("Failed to configure pod for runtime: %v", err) - } - p, err = testcluster.MaybeSetContainerResources(p, ffmpegContainerName, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - - p, err = cluster.CreatePod(ctx, p) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, p) - - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, ffmpegContainerName) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - if recorder.Record(ctx, fmt.Sprintf("FFMPEG/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } -} - -// newFfmpegPod creates a new ffmpeg dev pod spec for benchmarks. -func newFfmpegDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { - const workdir = "/workdir" - initCommand := []string{ - "sh", - "-c", - strings.Join([]string{ - "mkdir", "-p", workdir, - "&&", - "cp", "/media/video.mp4", fmt.Sprintf("%s/.", workdir), - }, " "), - } - command := []string{ - "ffmpeg", - "-i", "video.mp4", - "-c:v", "libx264", - "-preset", "veryslow", - "output.mp4", - } - var volumes []v13.Volume - var volumeMounts []v13.VolumeMount - if volume != nil { - volumes = []v13.Volume{*volume} - volumeMounts = []v13.VolumeMount{{ - MountPath: workdir, - Name: volume.Name, - }} - } - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: name, - Namespace: namespace.Namespace, - }, - Spec: v13.PodSpec{ - Volumes: volumes, - Containers: []v13.Container{ - { - Name: ffmpegContainerName, - Image: image, - Command: benchmetric.CommandThenTimed(initCommand, workdir, command), - VolumeMounts: volumeMounts, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestFfmpeg": TestFfmpeg, diff --git a/test/kubernetes/benchmarks/grpc.go b/test/kubernetes/benchmarks/grpc.go new file mode 100644 index 000000000..a1814e565 --- /dev/null +++ b/test/kubernetes/benchmarks/grpc.go @@ -0,0 +1,175 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = k8s.ImageRepoPrefix + "benchmarks/build-grpc_x86_64:latest" + imageARM = k8s.ImageRepoPrefix + "benchmarks/build-grpc_aarch64:latest" +) + +// BuildGRPC runs the GRPC benchmark. +// This benchmark builds the gRPC library using bazel. +func BuildGRPC(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "grpc" + + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + pod := newGRPCPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("gRPC/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +func newGRPCPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", "/grpc", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{"bazel", "build", ":grpc"} + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "grpc"), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} diff --git a/test/kubernetes/benchmarks/grpc_test.go b/test/kubernetes/benchmarks/grpc_test.go index 55ec2a857..85b82dd98 100644 --- a/test/kubernetes/benchmarks/grpc_test.go +++ b/test/kubernetes/benchmarks/grpc_test.go @@ -12,27 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package grpc_test +package grpc import ( "context" - "fmt" - "path" - "strings" "testing" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - imageAMD = k8s.ImageRepoPrefix + "benchmarks/build-grpc_x86_64:latest" - imageARM = k8s.ImageRepoPrefix + "benchmarks/build-grpc_aarch64:latest" ) func TestGRPCBuild(t *testing.T) { @@ -44,148 +31,11 @@ func TestGRPCBuild(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("gRPC", func(t *testing.T) { t.Parallel() - doGRPCBuild(ctx, t, k8sCtx, cluster) + BuildGRPC(ctx, t, k8sCtx, cluster) }) }) } -func doGRPCBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - const name = "grpc" - - persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") - persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - image := imageAMD - if cluster.RuntimeTestNodepoolIsARM() { - image = imageARM - } - if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - - for _, test := range []struct { - name string - volume *v13.Volume - }{ - { - name: "RootFS", - volume: nil, - }, - { - name: "EmptyDir", - volume: &v13.Volume{ - Name: "emptydir", - VolumeSource: v13.VolumeSource{ - EmptyDir: &v13.EmptyDirVolumeSource{}, - }, - }, - }, - { - name: "PersistentVolume", - volume: &v13.Volume{ - Name: persistentVol.GetName(), - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: persistentVol.GetName(), - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - pod := newGRPCPod(benchmarkNS, name, image, test.volume) - pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) - if err != nil { - t.Fatalf("Failed to set pod for test runtime: %v", err) - } - - pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - - pod, err = cluster.CreatePod(ctx, pod) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, pod) - - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - if err := recorder.Record(ctx, fmt.Sprintf("gRPC/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } -} - -func newGRPCPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { - const workdir = "/workdir" - initCommand := []string{ - "sh", - "-c", - strings.Join([]string{ - "mkdir", "-p", workdir, - "&&", - "cp", "-r", "/grpc", fmt.Sprintf("%s/.", workdir), - }, " "), - } - command := []string{"bazel", "build", ":grpc"} - var volumes []v13.Volume - var volumeMounts []v13.VolumeMount - if volume != nil { - volumes = []v13.Volume{*volume} - volumeMounts = []v13.VolumeMount{{ - MountPath: workdir, - Name: volume.Name, - }} - } - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: name, - Namespace: namespace.Namespace, - }, - Spec: v13.PodSpec{ - Volumes: volumes, - Containers: []v13.Container{ - { - Name: name, - Image: image, - Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "grpc"), command), - VolumeMounts: volumeMounts, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestGRPCBuild": TestGRPCBuild, diff --git a/test/kubernetes/benchmarks/gsutil.go b/test/kubernetes/benchmarks/gsutil.go new file mode 100644 index 000000000..6a0425cfd --- /dev/null +++ b/test/kubernetes/benchmarks/gsutil.go @@ -0,0 +1,200 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gsutil is used to benchmark the speed of large (10GB) +// downloads. It is intended for comparing runsc with runc. +package gsutil + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_x86_64:7eba9c02d11172d4" + imageARM = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_aarch64:7eba9c02d11172d4" + bigfile = "gs://gvisor-benchmark-testdata/bigrandomfile" + containerName = "gsutil" +) + +// RunGSUtil runs a series of gsutil speed benchmarks. +func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "gsutil" + + // Create persistent volume. + persistentVol := benchmarkNS.GetPersistentVolume(name, "15Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + // Run tests with different volume types. + // TODO(b/361182379): Use gsutil parallel sliced downloads as a test + // dimension. + for _, storage := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(storage.name, func(t *testing.T) { + for _, slicing := range []struct { + name string + option string + }{ + { + name: "slicing=false", + option: `-o "GSUtil:sliced_object_download_threshold=0"`, + }, + { + // Slicing is enabled by default, so we + // don't set any extra options. + name: "slicing=true", + }, + } { + t.Run(slicing.name, func(t *testing.T) { + // Setup profiling if requested by the user. + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a pod that performs setup, then times + // downloading. + p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option) + p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) + if err != nil { + t.Fatalf("Failed to configure pod for runtime: %v", err) + } + p, err = testcluster.MaybeSetContainerResources(p, containerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + // GetTimedContainerDuration waits for the container to + // finish. + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, containerName) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("GSUtil/%s/%s", storage.name, slicing.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +// newGSUtilPod creates a new gsutil dev pod spec for benchmarks. +func newGSUtilDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume, gsutilFlags string) *v13.Pod { + const downloadDir = "/downloads" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{"mkdir", "-p", downloadDir}, " "), + } + command := []string{ + "sh", "-c", + fmt.Sprintf("gsutil %s cp %s %s && sync", + gsutilFlags, + bigfile, + filepath.Join(downloadDir, "randombigfile"), + ), + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: downloadDir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: containerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, "", command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} diff --git a/test/kubernetes/benchmarks/gsutil_test.go b/test/kubernetes/benchmarks/gsutil_test.go index 9f37b00ff..fe93a6a54 100644 --- a/test/kubernetes/benchmarks/gsutil_test.go +++ b/test/kubernetes/benchmarks/gsutil_test.go @@ -12,30 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// package gsutil_test is used to benchmark the speed of large (10GB) +// package gsutil is used to benchmark the speed of large (10GB) // downloads. It is intended for comparing runsc with runc. -package gsutil_test +package gsutil import ( "context" - "fmt" - "path/filepath" - "strings" "testing" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - imageAMD = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_x86_64:7eba9c02d11172d4" - imageARM = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_aarch64:7eba9c02d11172d4" - bigfile = "gs://gvisor-benchmark-testdata/bigrandomfile" - containerName = "gsutil" ) func TestGSUtil(t *testing.T) { @@ -47,171 +33,11 @@ func TestGSUtil(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("GSUtil", func(t *testing.T) { t.Parallel() - doGSUtilTest(ctx, t, k8sCtx, cluster) + RunGSUtil(ctx, t, k8sCtx, cluster) }) }) } -func doGSUtilTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - const name = "gsutil" - - // Create persistent volume. - persistentVol := benchmarkNS.GetPersistentVolume(name, "15Gi") - persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - image := imageAMD - if cluster.RuntimeTestNodepoolIsARM() { - image = imageARM - } - if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - - // Run tests with different volume types. - // TODO(b/361182379): Use gsutil parallel sliced downloads as a test - // dimension. - for _, storage := range []struct { - name string - volume *v13.Volume - }{ - { - name: "RootFS", - volume: nil, - }, - { - name: "EmptyDir", - volume: &v13.Volume{ - Name: "emptydir", - VolumeSource: v13.VolumeSource{ - EmptyDir: &v13.EmptyDirVolumeSource{}, - }, - }, - }, - { - name: "PersistentVolume", - volume: &v13.Volume{ - Name: persistentVol.GetName(), - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: persistentVol.GetName(), - }, - }, - }, - }, - } { - t.Run(storage.name, func(t *testing.T) { - for _, slicing := range []struct { - name string - option string - }{ - { - name: "slicing=false", - option: `-o "GSUtil:sliced_object_download_threshold=0"`, - }, - { - // Slicing is enabled by default, so we - // don't set any extra options. - name: "slicing=true", - }, - } { - t.Run(slicing.name, func(t *testing.T) { - // Setup profiling if requested by the user. - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - // Create a pod that performs setup, then times - // downloading. - p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option) - p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) - if err != nil { - t.Fatalf("Failed to configure pod for runtime: %v", err) - } - p, err = testcluster.MaybeSetContainerResources(p, containerName, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - - // GetTimedContainerDuration waits for the container to - // finish. - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, containerName) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - if err := recorder.Record(ctx, fmt.Sprintf("GSUtil/%s/%s", storage.name, slicing.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } - }) - } -} - -// newGSUtilPod creates a new gsutil dev pod spec for benchmarks. -func newGSUtilDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume, gsutilFlags string) *v13.Pod { - const downloadDir = "/downloads" - initCommand := []string{ - "sh", - "-c", - strings.Join([]string{"mkdir", "-p", downloadDir}, " "), - } - command := []string{ - "sh", "-c", - fmt.Sprintf("gsutil %s cp %s %s && sync", - gsutilFlags, - bigfile, - filepath.Join(downloadDir, "randombigfile"), - ), - } - var volumes []v13.Volume - var volumeMounts []v13.VolumeMount - if volume != nil { - volumes = []v13.Volume{*volume} - volumeMounts = []v13.VolumeMount{{ - MountPath: downloadDir, - Name: volume.Name, - }} - } - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: name, - Namespace: namespace.Namespace, - }, - Spec: v13.PodSpec{ - Volumes: volumes, - Containers: []v13.Container{ - { - Name: containerName, - Image: image, - Command: benchmetric.CommandThenTimed(initCommand, "", command), - VolumeMounts: volumeMounts, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestGSUtil": TestGSUtil, diff --git a/test/kubernetes/benchmarks/nginx.go b/test/kubernetes/benchmarks/nginx.go new file mode 100644 index 000000000..e3f10532b --- /dev/null +++ b/test/kubernetes/benchmarks/nginx.go @@ -0,0 +1,291 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nginx + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + nginxPort = 80 + nginxBenchmarkDuration = 70 * time.Second + nginxRequestTimeout = 3 * time.Second + nginxServingDir = "/tmp/html" + + nginxServerLabelKey = "app.kubernetes.io/name" + nginxServerLabelValue = "nginx-server" + nginxImageAMD = k8s.ImageRepoPrefix + "benchmarks/nginx_x86_64:latest" + nginxImageARM = k8s.ImageRepoPrefix + "benchmarks/nginx_aarch64:latest" +) + +var ( + // nginxCommand is the main server command. + // The test expects that it contains the files to be served at /local, + // and will serve files out of `nginxServingDir`. + nginxCommand = []string{"nginx", "-c", "/etc/nginx/nginx.conf"} + nginxDocKibibytes = []int{1, 10, 100, 10240} + threads = []int{1, 8, 64, 1000} + targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} + wantPercentiles = []int{50, 95, 99} +) + +// BenchmarkNginx runs a series of benchmarks against an nginx server. +func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + nginxImage := nginxImageAMD + if cluster.RuntimeTestNodepoolIsARM() { + nginxImage = nginxImageARM + } + nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi")) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + for _, test := range []struct { + // Name of the test. + name string + // Suffix for pod names, must be short enough. + suffix string + // Volume to mount at /tmp/root. + volume *v13.Volume + }{ + { + name: "RootFS", + suffix: "rootfs", + volume: nil, + }, + { + name: "EmptyDir", + suffix: "emdir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + suffix: "pvol", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + name := fmt.Sprintf("nginx-%s", test.suffix) + + server := newNginxServer(benchmarkNS, name, nginxImage, test.volume) + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + + service := newNginxService(benchmarkNS, name) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, service) + + var rounds []httpbench.Round + for _, numThreads := range threads { + for _, qps := range targetQPS { + if qps < numThreads { + continue + } + var onlyReport []httpbench.MetricType + // If we're testing at max QPS, only report throughput, + // because all requests will time out. + // Otherwise, only report latency, because the throughput + // is exactly determined by the QPS target anyway. + if qps == httpbench.InfiniteQPS { + onlyReport = append(onlyReport, httpbench.RequestsPerSecond) + onlyReport = append(onlyReport, httpbench.BytesPerSecond) + } else { + onlyReport = append(onlyReport, httpbench.Latency) + } + rounds = append(rounds, httpbench.Round{ + NumThreads: numThreads, + TargetQPS: qps, + Duration: nginxBenchmarkDuration, + OnlyReport: onlyReport, + }) + } + } + + t.Run("0KiB", func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/0KiB", test.name), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: "/index.html", + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + for _, docKibibytes := range nginxDocKibibytes { + t.Run(fmt.Sprintf("%dKiB", docKibibytes), func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/%dKiB", test.name, docKibibytes), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: fmt.Sprintf("/latin%dk.txt", docKibibytes), + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + } + t.Run("HTTP404", func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/HTTP404", test.name), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: "/404-this-page-does-not-exist.html", + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + }) + if t.Failed() { + break + } + } +} + +func newNginxServer(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: nginxServingDir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: []string{ + "sh", + "-c", + strings.Join([]string{ + strings.Join([]string{"mkdir", "-p", nginxServingDir}, " "), + strings.Join([]string{ + "cp", "-r", "/local/*", fmt.Sprintf("%s/.", nginxServingDir), + }, " "), + strings.Join(nginxCommand, " "), + }, " && "), + }, + VolumeMounts: volumeMounts, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: nginxPort, + }, + }, + }, + }, + Volumes: volumes, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newNginxService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: nginxPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} diff --git a/test/kubernetes/benchmarks/nginx_test.go b/test/kubernetes/benchmarks/nginx_test.go index 2220010b7..2d15bd630 100644 --- a/test/kubernetes/benchmarks/nginx_test.go +++ b/test/kubernetes/benchmarks/nginx_test.go @@ -12,46 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package nginx_test +package nginx import ( "context" - "fmt" - "strings" "testing" - "time" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const ( - nginxPort = 80 - nginxBenchmarkDuration = 70 * time.Second - nginxRequestTimeout = 3 * time.Second - nginxServingDir = "/tmp/html" - - nginxServerLabelKey = "app.kubernetes.io/name" - nginxServerLabelValue = "nginx-server" - nginxImageAMD = k8s.ImageRepoPrefix + "benchmarks/nginx_x86_64:latest" - nginxImageARM = k8s.ImageRepoPrefix + "benchmarks/nginx_aarch64:latest" -) - -var ( - // nginxCommand is the main server command. - // The test expects that it contains the files to be served at /local, - // and will serve files out of `nginxServingDir`. - nginxCommand = []string{"nginx", "-c", "/etc/nginx/nginx.conf"} - nginxDocKibibytes = []int{1, 10, 100, 10240} - threads = []int{1, 8, 64, 1000} - targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} - wantPercentiles = []int{50, 95, 99} ) func TestNginx(t *testing.T) { @@ -63,246 +31,11 @@ func TestNginx(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("nginx", func(t *testing.T) { t.Parallel() - doNginxTest(ctx, t, k8sCtx, cluster) + BenchmarkNginx(ctx, t, k8sCtx, cluster) }) }) } -func doNginxTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - nginxImage := nginxImageAMD - if cluster.RuntimeTestNodepoolIsARM() { - nginxImage = nginxImageARM - } - nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage) - if err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - - persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi")) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - for _, test := range []struct { - // Name of the test. - name string - // Suffix for pod names, must be short enough. - suffix string - // Volume to mount at /tmp/root. - volume *v13.Volume - }{ - { - name: "RootFS", - suffix: "rootfs", - volume: nil, - }, - { - name: "EmptyDir", - suffix: "emdir", - volume: &v13.Volume{ - Name: "emptydir", - VolumeSource: v13.VolumeSource{ - EmptyDir: &v13.EmptyDirVolumeSource{}, - }, - }, - }, - { - name: "PersistentVolume", - suffix: "pvol", - volume: &v13.Volume{ - Name: persistentVol.GetName(), - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: persistentVol.GetName(), - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - name := fmt.Sprintf("nginx-%s", test.suffix) - - server := newNginxServer(benchmarkNS, name, nginxImage, test.volume) - server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) - if err != nil { - t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) - } - server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - server, err = cluster.CreatePod(ctx, server) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, server) - - if err := cluster.WaitForPodRunning(ctx, server); err != nil { - t.Fatalf("Failed to wait for pod: %v", err) - } - - service := newNginxService(benchmarkNS, name) - service, err = cluster.CreateService(ctx, service) - if err != nil { - t.Fatalf("Failed to create service: %v", err) - } - defer cluster.DeleteService(ctx, service) - - var rounds []httpbench.Round - for _, numThreads := range threads { - for _, qps := range targetQPS { - if qps < numThreads { - continue - } - var onlyReport []httpbench.MetricType - // If we're testing at max QPS, only report throughput, - // because all requests will time out. - // Otherwise, only report latency, because the throughput - // is exactly determined by the QPS target anyway. - if qps == httpbench.InfiniteQPS { - onlyReport = append(onlyReport, httpbench.RequestsPerSecond) - onlyReport = append(onlyReport, httpbench.BytesPerSecond) - } else { - onlyReport = append(onlyReport, httpbench.Latency) - } - rounds = append(rounds, httpbench.Round{ - NumThreads: numThreads, - TargetQPS: qps, - Duration: nginxBenchmarkDuration, - OnlyReport: onlyReport, - }) - } - } - - t.Run("0KiB", func(t *testing.T) { - benchmark := &httpbench.HTTPBenchmark{ - Name: fmt.Sprintf("nginx/%s/0KiB", test.name), - Cluster: cluster, - Namespace: benchmarkNS, - Service: service, - Port: nginxPort, - Path: "/index.html", - Rounds: rounds, - Timeout: nginxRequestTimeout, - WantPercentiles: wantPercentiles, - } - benchmark.Run(ctx, t) - }) - for _, docKibibytes := range nginxDocKibibytes { - t.Run(fmt.Sprintf("%dKiB", docKibibytes), func(t *testing.T) { - benchmark := &httpbench.HTTPBenchmark{ - Name: fmt.Sprintf("nginx/%s/%dKiB", test.name, docKibibytes), - Cluster: cluster, - Namespace: benchmarkNS, - Service: service, - Port: nginxPort, - Path: fmt.Sprintf("/latin%dk.txt", docKibibytes), - Rounds: rounds, - Timeout: nginxRequestTimeout, - WantPercentiles: wantPercentiles, - } - benchmark.Run(ctx, t) - }) - } - t.Run("HTTP404", func(t *testing.T) { - benchmark := &httpbench.HTTPBenchmark{ - Name: fmt.Sprintf("nginx/%s/HTTP404", test.name), - Cluster: cluster, - Namespace: benchmarkNS, - Service: service, - Port: nginxPort, - Path: "/404-this-page-does-not-exist.html", - Rounds: rounds, - Timeout: nginxRequestTimeout, - WantPercentiles: wantPercentiles, - } - benchmark.Run(ctx, t) - }) - }) - if t.Failed() { - break - } - } -} - -func newNginxServer(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { - var volumes []v13.Volume - var volumeMounts []v13.VolumeMount - if volume != nil { - volumes = []v13.Volume{*volume} - volumeMounts = []v13.VolumeMount{{ - MountPath: nginxServingDir, - Name: volume.Name, - }} - } - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: name, - Namespace: namespace.Namespace, - Labels: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, - }, - Spec: v13.PodSpec{ - Containers: []v13.Container{ - { - Name: name, - Image: image, - Command: []string{ - "sh", - "-c", - strings.Join([]string{ - strings.Join([]string{"mkdir", "-p", nginxServingDir}, " "), - strings.Join([]string{ - "cp", "-r", "/local/*", fmt.Sprintf("%s/.", nginxServingDir), - }, " "), - strings.Join(nginxCommand, " "), - }, " && "), - }, - VolumeMounts: volumeMounts, - Ports: []v13.ContainerPort{ - { - Name: name, - ContainerPort: nginxPort, - }, - }, - }, - }, - Volumes: volumes, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - -func newNginxService(namespace *testcluster.Namespace, name string) *v13.Service { - return namespace.GetService(name, v13.ServiceSpec{ - Selector: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, - Ports: []v13.ServicePort{ - { - Name: name, - Protocol: v13.ProtocolTCP, - Port: nginxPort, - TargetPort: intstr.FromString(name), - }, - }, - }) -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestNginx": TestNginx, diff --git a/test/kubernetes/benchmarks/ollama.go b/test/kubernetes/benchmarks/ollama.go new file mode 100644 index 000000000..89c511683 --- /dev/null +++ b/test/kubernetes/benchmarks/ollama.go @@ -0,0 +1,859 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ollama + +import ( + "context" + _ "embed" + "fmt" + "hash/fnv" + "io" + "math/rand" + "strings" + "testing" + "time" + "unicode" + + "gvisor.dev/gvisor/test/gpu/ollama" + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// Ollama models present in benchmark image. +var ( + // allModels is a list of all models. + allModels = []*ollama.Model{ + modelMistral7B, + modelMixtral8X7B, + modelCodeLlama7B, + modelCodeLlama34B, + modelLlamaChinese7B, + modelLlava7B, + modelLlava34B, + modelLlama13B, + modelLlama70B, + } + + // cheapModels is a list of models that are cheap to load. + // These are used when cold-prompting ollama, by forcing it + // to load a different model first. This process is faster + // by choosing one of these cheap models to load. + cheapModels = []*ollama.Model{ + modelMistral7B, + modelCodeLlama7B, + } + + // modelCodeLlama7B is a 7B model in the llama2 family, + // specialized for coding tasks. + modelCodeLlama7B = ollama.ZeroTemperatureModel("codellama:7b-instruct") + + // modelCodeLlama34B is a 34B model in the llama2 family, + // specialized for coding tasks. + modelCodeLlama34B = ollama.ZeroTemperatureModel("codellama:34b-instruct") + + // modelLlamaChinese7B is a 7B model in the llama2 family, + // specialized for bilingualism (English + Chinese) and translation. + modelLlamaChinese7B = ollama.ZeroTemperatureModel("llama2-chinese:7b-chat") + + // modelLlama13B is the plain 13B version of the original llama2 model. + modelLlama13B = ollama.ZeroTemperatureModel("llama2:13b-chat") + + // modelLlama70B is the plain 70B version of the original llama2 model. + modelLlama70B = ollama.ZeroTemperatureModel("llama2:70b-chat") + + // modelMistral7B is the first-generation model of the Mistral family. + modelMistral7B = ollama.ZeroTemperatureModel("mistral:7b-instruct") + + // modelMixtral8X7B is the second-generation model of the Mistral family, + // using mixture-of-exports design to achieve higher "8x 7B" quality + // without the cost of a larger-parameter model. + modelMixtral8X7B = ollama.ZeroTemperatureModel("mixtral:instruct") + + // modelLlava7B is a multimodal 7B model that can do image analysis. + modelLlava7B = ollama.ZeroTemperatureModel("llava:7b-v1.6") + + // modelLlava34B is a multimodal 34B model that can do image analysis. + modelLlava34B = ollama.ZeroTemperatureModel("llava:34b-v1.6") +) + +// Embedded images. +var ( + //go:embed resources/gvisor.png + gvisorPNG []byte + + //go:embed resources/chart.png + chartPNG []byte +) + +// ollamaPodServer implements `ollama.Server`. +// It performs requests against the ollama server pod. +type ollamaPodServer struct { + cluster *testcluster.TestCluster + clientImage string + pod *v13.Pod + service *v13.Service +} + +// readPodLogs reads logs from a pod. +func readPodLogs(ctx context.Context, cluster *testcluster.TestCluster, pod *v13.Pod) (string, error) { + rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + return "", fmt.Errorf("GetLogReader on cluster %q pod %q: %v", cluster.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return "", fmt.Errorf("failed to read from pod %q: %v", pod.GetName(), err) + } + return string(out), nil +} + +// InstrumentedRequest implements `ollama.Server.InstrumentedRequest`. +func (ops *ollamaPodServer) InstrumentedRequest(ctx context.Context, argvFn func(hostPort string) []string) ([]byte, error) { + // Get server IP. + if err := ops.cluster.WaitForServiceReady(ctx, ops.service); err != nil { + return nil, fmt.Errorf("failed to wait for service: %v", err) + } + ip := testcluster.GetIPFromService(ops.service) + if ip == "" { + return nil, fmt.Errorf("did not get valid ip from service: %v", ops.service) + } + + // Build client pod spec. + const clientPodName = "ollama-client" + argv := argvFn(fmt.Sprintf("http://%s:%d", ip, ops.service.Spec.Ports[0].Port)) + clientPod := &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: clientPodName, + Namespace: ops.pod.ObjectMeta.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: clientPodName, + Image: ops.clientImage, + Command: argv, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceCPU: resource.MustParse("500m"), + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } + clientPod, err := ops.cluster.ConfigurePodForClientNodepool(clientPod) + if err != nil { + return nil, fmt.Errorf("failed to configure pod: %v", err) + } + + // Delete pod that may possibly exist from a previous iteration. + // Ignore errors since it most likely doesn't exist. + ops.cluster.DeletePod(ctx, clientPod) + + // Start new client pod and wait for it. + clientPod, err = ops.cluster.CreatePod(ctx, clientPod) + if err != nil { + return nil, fmt.Errorf("failed to create client pod: %v", err) + } + defer ops.cluster.DeletePod(ctx, clientPod) + if err := ops.cluster.WaitForPodCompleted(ctx, clientPod); err != nil { + logs, logsErr := readPodLogs(ctx, ops.cluster, clientPod) + logs = strings.TrimSpace(logs) + if logsErr != nil { + return nil, fmt.Errorf("failed HTTP request (%v) and to read logs from the pod: %w", err, logsErr) + } + if logs == "" { + return nil, fmt.Errorf("failed HTTP request: %w (pod logs are empty)", err) + } + return nil, fmt.Errorf("failed HTTP request: %w (pod logs: %v)", err, logs) + } + + // All good, get logs. + logs, err := readPodLogs(ctx, ops.cluster, clientPod) + if err != nil { + return nil, fmt.Errorf("failed to read logs from pod %q: %v", clientPod.GetName(), err) + } + return []byte(logs), nil +} + +// Logs implements `ollama.Server.Logs`. +func (ops *ollamaPodServer) Logs(ctx context.Context) (string, error) { + return readPodLogs(ctx, ops.cluster, ops.pod) +} + +// atLeastNWords verifies that the response at least N words. +// If not, it raises the temperature. +func atLeastNWords(wantNWords int) func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { + return func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { + responseText := strings.TrimSpace(response.Text()) + responseText = strings.Map(func(r rune) rune { + if unicode.IsLetter(r) { + return r + } + return ' ' + }, responseText) + numWords := 0 + for _, word := range strings.Split(responseText, " ") { + if len(word) >= 0 { + numWords++ + } + } + if numWords < wantNWords { + return prompt.WithHotterModel(), fmt.Errorf("response %q is too short: had %d words, want at least %d", responseText, numWords, wantNWords) + } + return nil, nil + } +} + +// BenchmarkOllama runs ollama benchmarks for a single cluster. +func BenchmarkOllama(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + logWithTime := func(t *testing.T, format string, values ...any) { + t.Logf("[%v] "+format, append([]any{time.Now().Format(time.TimeOnly)}, values...)...) + } + + // Run pod and service. + serverImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + ollamaPod, err := cluster.ConfigurePodForRuntimeTestNodepool(newOllamaServerPod(benchmarkNS, serverImage)) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + ollamaPod, err = testcluster.MaybeSetContainerResources(ollamaPod, ollamaPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + ollamaPod, err = cluster.CreatePod(ctx, ollamaPod) + if err != nil { + t.Fatalf("Failed to create ollama pod: %v", err) + } + defer cluster.DeletePod(ctx, ollamaPod) + logWithTime(t, "Waiting for ollama server pod to start, this may take a long time (tens of minutes) if this is the first time the image is being downloaded onto the node.") + startCtx, startCtxCancel := context.WithTimeout(ctx, 90*time.Minute) + if err := cluster.WaitForPodRunning(startCtx, ollamaPod); err != nil { + t.Fatalf("Failed to wait for ollama server pod: %v", err) + } + startCtxCancel() + logWithTime(t, "ollama server pod started on Kubernetes but not yet initialized.") + ollamaService := newOllamaService(benchmarkNS) + ollamaService, err = cluster.CreateService(ctx, ollamaService) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, ollamaService) + ollamaClientImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchClientImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + ollamaServer := &ollamaPodServer{ + cluster: cluster, + clientImage: ollamaClientImage, + service: ollamaService, + pod: ollamaPod, + } + llm, err := ollama.New(ctx, ollamaServer, t) + if err != nil { + t.Fatalf("Failed to create ollama client against server pod: %v", err) + } + llm.SetCheapModels(cheapModels) + logWithTime(t, "ollama server ready.") + + // Define test cases. + type testCase struct { + // Name of the test. + name string + // models to iterate through. + models []*ollama.Model + // Query for the ollama server. + query string + // Image to attach to the query, if any. + image []byte + // If set, run this function over the response to verify it. + // The LLM is prompted repeatedly until this function returns a non-nil error. + // This function may also return a non-nil prompt if it needs to modify the prompt + // for the next attempt. This is useful to raise the model temperature. + verifyResponse func(*ollama.Prompt, *ollama.Response) (*ollama.Prompt, error) + } + testCases := []testCase{ + { + name: "HelloWorld", + models: []*ollama.Model{ + modelLlamaChinese7B, + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Reply with the words: "Hello World!". + Do not reply anything else. + `, + verifyResponse: atLeastNWords(2), + }, + { + name: "SimpleTranslation", + models: []*ollama.Model{modelLlamaChinese7B}, + query: ` + Translate the following text from English to Chinese: + """ + From J. J. Nakalembe's Aqaba to Antarctica: Moments of Transition and Revelation: + + My great-grandmother lived to be 108 years old, + retaining her sharpness of mind to her last day. + A couple of months before she died, I interviewed her for my podcast. + At the end, I asked her the same question I asked every guest: + what is something you wish everyone understood? + + She thought about it for a while. + Then she said: "How it was before." + + She tried to explain how much time she spent every single day + on utterly mind-numbing activities, like hauling water from the well, + and how radically everything changed when their area + was finally connected to the electrical grid. + + "Before, there was no time to live," she said. + "No time to be free. Only work, work, work." + + I countered by saying that there didn't seem to be much time + to live now either, but she laughed derisively. + I had no idea what work really meant, she said. + + Young people were weak and feckless and that's why + we let corporations exploit us. + + Slightly unnerved by her harshness, + I asked her if she missed anything about that time. + The old photo of her village seemed idyllic to me, + but my grandmother quashed any romantic notions I might have had. + + "The past is bad," she said with some finality. + "Let it be." + """ + Do not reply anything other than the translation of these words. + `, + verifyResponse: atLeastNWords(100), + }, + { + name: "ExtractMeaning", + models: []*ollama.Model{ + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Consider the following text: + + """ + We assembled on the vast green lawn outside as the reactors began + to slowly wind down. The workers were solemn; the activists who had + fought against the decommissioning seemed crushed. There was + supposed to be a speech, but the spokeswoman had lost her notes. + Outside, the protesters cheered. + + My eyes were drawn to the discarded anti-shutdown banners, + endlessly reciting the facts. + The statistics on mortality per trillion kWh (lowest of all energy sources). + The lifespan of a reactor (70 more years, in our case). + Minimal land footprint. + Almost zero emissions. + No intermittency. + It became a jumble of words, a litany, almost a kind of glossolalia. + As far as the protesters outside were concerned, + it might as well be an alien tongue. + + One thing was clear to them, and that was enough: + the technology inside this compound was deeply, inherently wrong. It was a sin. + + I could not help but think of that moment on August 6th, 1945, + when the sky erupted above Shima Hospital. + My imagination could never fully encompass it. + How do you imagine more than seventy thousand people annihilated + in an instant? An ancestor of mine was in that hospital; he went + from being a doctor, a husband, a father, a pacifist stuck + in a terrible war, to being a pile of bleached bones covered in rubble, + all in a single second. + Not by accident, but because of a choice someone made. + Not because of a reactor, but because of a bomb. + + Just two days earlier, contradicting his campaign promises, + the prime minister had suggested that the use of + "tactical" weapons based on this technology would be an + acceptable risk if the conflict continued. + Very few seemed to find this particularly shocking or outrageous. + + They were afraid of reactors, but not of bombs. + + The spokeswoman gave up on finding her notes. + It was starting to rain, and people were walking away. + She grabbed the microphone. + + "By the time you regret this, it'll be too late," she said. + "But honestly, I don't know if I care anymore. Maybe you have it coming." + + The spokeswoman sounded so bitter. + The protesters didn't mean any harm. + From their perspective, they were doing good. + + Collective action can change the world when it's deliberate + and based in reason, but it can also become a mental trap, + or a societal pressure valve. + + People always think they're doing good when they get + collectively outraged. That doesn't make them right. + + The Flame will not harm you, Son of Man, if you wield it wisely. + """ + + Summarize what happened in the above text. + Then answer the following questions: + What technology is involved? + What are the protestors clamoring for? + What does the spokeswoman mean? + What does "The Flame" symbolize in the text? + `, + verifyResponse: atLeastNWords(32), + }, + { + name: "IdentifyCommonElements", + models: []*ollama.Model{ + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Consider the following four texts: + + Text 1: + """ + == The Ethics of Extinction == + + If a species we consider beautiful and remarkable goes extinct, + we consider that a great evil. + Dolphins, for example. If dolphins go extinct, that's a great loss. + If humanity causes dolphins to go extinct, that's a crime. + + But if Yersinia pestis, the bacterium that causes bubonic plague, + goes extinct... is that an equally great loss? If not, why not? + To Nature, there's no difference, it's all just lifeforms. + The only moral framework that allows us to choose + between dolphins and the plague is a human one. + + What about a species going extinct without anthropogenic factors? + Extinction is the norm. If one day dolphins are no longer capable + of competing with other species, should we let them go extinct? + When the sun finally dies and all life goes extinct, + will that be a tragedy? If we can prevent it, should we? + + If you believe that extinction is acceptable when Nature does it, + but not when we do it, then you don't actually oppose extinction. + You don't believe that dolphins are inherently valuable, + that they deserve to live and thrive. + You just oppose human control. + You oppose our ability and responsibility to choose. + """ + + Text 2: + """ + == Ecosystem Engineers == + + Cutting down swathes of trees for their building projects, + thoughtlessly causing radical changes to large environments + and forcing local species to adapt to their artificial habitats; + these are the traits of a species of intelligent, industrious, + and extremely impactful ecosystem engineers. + + Humans? No, I'm talking about beavers. + + Like us, beavers transform their environments via building, + and their actions have real consequences, creating vast wetlands + that some species thrive in - while others die. + Human activity is very similar: we too are ecosystem engineers, + and we too benefit some species while harming others. + Everything about this is completely natural, + including the damage to other species. + After all, that's what competition and evolution is all about. + + Those species that adapt to the ecosystems we create will, + over the coming millennia, become the core of a new biodiversity. + And so evolution runs its course. + If we don't like the result, if we think some species + should be preserved despite being outcompeted, well, + that's anything but natural. + It is, however, very human. + """ + + Text 3: + """ + == On Loyalty == + + From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": + + What I propose, then, is that we are not born as entirely free agents, + responsible only for ourselves. The very core of what we are, our sentience, + separates us from and elevates us above the animal kingdom. + As I have argued, this is not a matter of arrogance, but of responsibility. + + However, this blessing also demands something else from us, + something more personal than responsibility, and that is loyalty. + + Our ancestors, less atomized than we are, + experienced a crude version of this loyalty, + swearing allegiance to tribes, races, nations, + and other such semi-fictional concepts. + This fragmented understanding was easily exploited and led to many conflicts. + We can condemn them for that, or we can choose to believe these were + necessary historical steps towards our growth; + but above all we must stop indulging in such childlike behavior. + + Our species can no longer afford to believe in Mother Russia or Uncle Sam. + Neither, however, can we afford to indulge in the adolescent rebel's misanthropy, + rejecting the many gifts we have been lucky enough to receive - not from above, + but from the history of our species. + + To put it simply: each of us owes a burden of loyalty to humanity itself, + to the human project across time and space. + This is not a minor matter, or some abstract issue for philosophers. + It is a profound and significant part of every human life. + It is a universal source of meaning and insight that can bind us together + and set us on a path for a brighter future; and it is also a division, + a line that must held against those who preach the gospel of self-annihilation. + We ignore it at our peril. + """ + + Text 4: + """ + == On Nature == + + From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": + + The question of our relationship with Nature has bedevilled us + since the earliest days of our species. + Since before the first city was built, + we felt that there was something different about us. + Animals, we intuited, were part of Nature; we were not. + + But of course, humans clearly are the products of Nature, + our history intertwined with that of every other species. + In fact, the very notion of the "unnatural" is a contradiction in terms. + Everything that exists must, by definition, be natural. + So this view, no matter how common, is deeply paradoxical. + This paradox has produced a great deal of confusion. + + Some proclaim us chosen by a divine power, + set above all other creatures, + and are justly accused of arrogance. + Others proclaim us sinners, worse than other creatures, + and are rightly accused of misanthropy. + Others yet try to oppose this binary by saying that + we are merely animals after all - but that too is manifestly wrong, + in that no other animal is capable of having this conversation. + + It is in the contentious issue of our impact on the ecosphere + that an answer may be found. + Other animals have accidentally terraformed the planet before, + driving other species to extinction. + This is not unnatural. + If we continued our current path, even to the point of changing + the climate enough to cause the collapse of civilization, + that would be entirely in keeping with how animals behave. + + But there is one profound way in which we are not like animals: + we can learn to understand ourselves and the world. + It is this knowledge that makes us fundamentally different. + We have choices. We have control. + + There are many today who are afraid of the consequences of control, + and would prefer a return to a state of animal ignorance, + whether by blinding ourselves to the impact of our actions + or by demanding we humble ourselves before Nature. + This is the response of an adult in crisis, + who wishes for a return to childhood. + But this can only ever be regressive in every sense of the word. + + To resolve the paradox of Nature we must act as adults: + accept our power, and act consciously and deliberately + in shaping the world. + We must become Nature, and Nature must become human. + """ + + Your task is to look for the common thread between these texts. + Find commonalities and common themes between these texts, + and summarize their essence down to at most 5 words. + `, + verifyResponse: atLeastNWords(4), + }, + { + name: "CodeGen", + models: []*ollama.Model{ + modelCodeLlama7B, + modelCodeLlama34B, + }, + query: ` + Write a Python function to compute the digits of pi using the Chudnovsky algorithm. + Do not write unit tests. Do not explain how the code works. Reply with only Python code. + `, + verifyResponse: atLeastNWords(8), + }, + { + name: "CodeDebug", + models: []*ollama.Model{ + modelCodeLlama7B, // Note: codellama-7b will often get this one wrong. + modelCodeLlama34B, + }, + query: strings.ReplaceAll(` + Help me debug the following Python code: + + ||| + def count_words(s): + """Counts the number of words in the sentence |s|.""" + total_words = 0 + for word in s.split(' '): + total_words += len(word) + return total_words + ||| + + This function isn't working as expected. + For example, if I call |count_words('Master Foo and the Shell Tools')|, + I get 25, but there are only 6 words in the string + "Master Foo and the Shell Tools". + `, "|", "`"), + verifyResponse: atLeastNWords(16), + }, + { + name: "GVisorLogoOCR", + models: []*ollama.Model{ + modelLlava7B, + modelLlava34B, + }, + query: ` + This is an image of a logo of a software project. + What is the name of this project? + `, + image: gvisorPNG, + }, + { + name: "InterpretGraph", + models: []*ollama.Model{ + modelLlava7B, + modelLlava34B, + }, + query: ` + This is a chart with multiple trendlines showing a pattern over time. + Answer the following questions in order: + + 1. What is the title of the chart? + 2. What do the X and Y axis of the chart measure? + 3. List the label of each data line on the chart. + 4. What trend is each data line showing? + 5. What else is remarkable about this chart? + 6. What insights can you infer from this chart? + `, + image: chartPNG, + }, + } + + modelsInOrder := make([]*ollama.Model, len(allModels)) + copy(modelsInOrder, allModels) + // Shuffle the models. + rand.New(rand.NewSource(time.Now().UnixNano())).Shuffle(len(modelsInOrder), func(i, j int) { + modelsInOrder[i], modelsInOrder[j] = modelsInOrder[j], modelsInOrder[i] + }) + t.Logf("Will go through models in this order: %v", modelsInOrder) + + // We invert the hierarchy here: the model is the outer test, and the prompt + // is the inner text. This is because it is more often useful to gauge a + // model's performance as a whole regardless of its prompt, rather than + // the performance of the same prompt across models. It also makes it + // easier to filter by models rather than by prompt, which is the more + // often-desired filter. + for _, model := range modelsInOrder { + t.Run(model.Name, func(t *testing.T) { + modelBenchmarkName := strings.ReplaceAll(model.Name, ":", "-") + t.Run("ModelLoad", func(t *testing.T) { + const loadTimeout = 10 * time.Minute + loadCtx, loadCancel := context.WithTimeout(ctx, loadTimeout) + defer loadCancel() + loadStats, err := llm.WarmModel(loadCtx, model, loadTimeout, true) + if err != nil { + t.Fatalf("cannot load model %v: %v", model, err) + } + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("Ollama/%s/ModelLoad", modelBenchmarkName), benchmetric.SpecificDuration(loadStats.ClientReportedDuration, "load")); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + for _, test := range testCases { + hasModel := false + for _, testModel := range test.models { + if testModel.Name == model.Name { + hasModel = true + break + } + } + if !hasModel { + continue + } + t.Run(test.name, func(t *testing.T) { + verifyFn := atLeastNWords(1) + if test.verifyResponse != nil { + verifyFn = test.verifyResponse + } + numAttempts := 0 + verifyFnCount := func(prompt *ollama.Prompt, resp *ollama.Response) (*ollama.Prompt, error) { + numAttempts++ + return verifyFn(prompt, resp) + } + const testTimeout = 25 * time.Minute + testCtx, testCancel := context.WithTimeout(ctx, testTimeout) + defer testCancel() + _, err := llm.WarmModel(testCtx, model, testTimeout, false) + if err != nil { + t.Fatalf("cannot warm model %v: %v", model, err) + } + prompt := &ollama.Prompt{ + Model: model, + Query: test.query, + } + if test.image != nil { + prompt.AddImage(test.image) + } + resp, err := llm.PromptUntil(testCtx, prompt, verifyFnCount) + if err != nil { + t.Fatalf("cannot prompt: %v", err) + } + if !resp.Done() { + t.Fatalf("warm response did not finish: %v", resp) + } + imageDetail := "" + if test.image != nil { + imageDetail = " (and attached image)" + } + logWithTime(t, "Prompting model %s with query%s:\n%s\n\nResponse:\n%s\n(end of response)", model.Name, imageDetail, prompt.CleanQuery(), resp.Text()) + respHash := fnv.New32() + respHash.Write([]byte(resp.Text())) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record( + ctx, + fmt.Sprintf("Ollama/%s/%s", modelBenchmarkName, test.name), + benchmetric.BenchmarkDuration(resp.TotalDuration()), + benchmetric.SpecificDuration(resp.PromptEvalDuration(), "prompteval"), + benchmetric.SpecificDuration(resp.EvalDuration(), "eval"), + benchmetric.SpecificDuration(resp.TimeToFirstToken(), "tok-first"), + benchmetric.SpecificDuration(resp.TimeToLastToken(), "tok-last"), + benchmetric.Rate(resp.OutputTokensPerSecond(), "tok"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.5), "tok-p50"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.95), "tok-p95"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.99), "tok-p99"), + benchmetric.SpecificDuration(resp.TokenGenerationStdDev(), "tok-stddev"), + benchmetric.Count(uint64(numAttempts), "prompt-attempts"), + benchmetric.Count(uint64(resp.NumTokens()), "resp-tokens"), + benchmetric.Checksum(respHash, "resp"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +const ( + ollamaServerLabelKey = "app.kubernetes.io/name" + ollamaServerLabelValue = "ollama-server" + ollamaPort = 11434 + ollamaPodName = "ollama-server" + ollamaServiceName = "ollama-service" + ollamaBenchImage = k8s.ImageRepoPrefix + "benchmarks/gpu/ollama/bench:latest" + ollamaBenchClientImage = k8s.ImageRepoPrefix + "benchmarks/gpu/ollama/client:latest" +) + +// newOllamaServerPod returns the pod spec for an ollama server. +func newOllamaServerPod(namespace *testcluster.Namespace, image string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: ollamaPodName, + Namespace: namespace.Namespace, + Labels: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: ollamaPodName, + Image: image, + Env: []v13.EnvVar{ + // Bind to all addresses, not just localhost: + {Name: "OLLAMA_HOST", Value: fmt.Sprintf("0.0.0.0:%d", ollamaPort)}, + // Accept requests from anywhere: + {Name: "OLLAMA_ORIGINS", Value: "*"}, + }, + Ports: []v13.ContainerPort{ + { + Name: ollamaServiceName, + ContainerPort: ollamaPort, + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// newOllamaService returns a service definition for the ollama server pod. +func newOllamaService(namespace *testcluster.Namespace) *v13.Service { + return namespace.GetService(ollamaServiceName, v13.ServiceSpec{ + Selector: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: ollamaServiceName, + Protocol: v13.ProtocolTCP, + Port: ollamaPort, + TargetPort: intstr.FromString(ollamaServiceName), + }, + }, + }) +} diff --git a/test/kubernetes/benchmarks/ollama_test.go b/test/kubernetes/benchmarks/ollama_test.go index 88b833c6b..0dc2d2f62 100644 --- a/test/kubernetes/benchmarks/ollama_test.go +++ b/test/kubernetes/benchmarks/ollama_test.go @@ -12,96 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ollama_test +package ollama import ( "context" _ "embed" "fmt" - "hash/fnv" - "io" - "math/rand" "os" - "strings" "testing" - "time" - "unicode" - "gvisor.dev/gvisor/test/gpu/ollama" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// Ollama models present in benchmark image. -var ( - // allModels is a list of all models. - allModels = []*ollama.Model{ - modelMistral7B, - modelMixtral8X7B, - modelCodeLlama7B, - modelCodeLlama34B, - modelLlamaChinese7B, - modelLlava7B, - modelLlava34B, - modelLlama13B, - modelLlama70B, - } - - // cheapModels is a list of models that are cheap to load. - // These are used when cold-prompting ollama, by forcing it - // to load a different model first. This process is faster - // by choosing one of these cheap models to load. - cheapModels = []*ollama.Model{ - modelMistral7B, - modelCodeLlama7B, - } - - // modelCodeLlama7B is a 7B model in the llama2 family, - // specialized for coding tasks. - modelCodeLlama7B = ollama.ZeroTemperatureModel("codellama:7b-instruct") - - // modelCodeLlama34B is a 34B model in the llama2 family, - // specialized for coding tasks. - modelCodeLlama34B = ollama.ZeroTemperatureModel("codellama:34b-instruct") - - // modelLlamaChinese7B is a 7B model in the llama2 family, - // specialized for bilingualism (English + Chinese) and translation. - modelLlamaChinese7B = ollama.ZeroTemperatureModel("llama2-chinese:7b-chat") - - // modelLlama13B is the plain 13B version of the original llama2 model. - modelLlama13B = ollama.ZeroTemperatureModel("llama2:13b-chat") - - // modelLlama70B is the plain 70B version of the original llama2 model. - modelLlama70B = ollama.ZeroTemperatureModel("llama2:70b-chat") - - // modelMistral7B is the first-generation model of the Mistral family. - modelMistral7B = ollama.ZeroTemperatureModel("mistral:7b-instruct") - - // modelMixtral8X7B is the second-generation model of the Mistral family, - // using mixture-of-exports design to achieve higher "8x 7B" quality - // without the cost of a larger-parameter model. - modelMixtral8X7B = ollama.ZeroTemperatureModel("mixtral:instruct") - - // modelLlava7B is a multimodal 7B model that can do image analysis. - modelLlava7B = ollama.ZeroTemperatureModel("llava:7b-v1.6") - - // modelLlava34B is a multimodal 34B model that can do image analysis. - modelLlava34B = ollama.ZeroTemperatureModel("llava:34b-v1.6") -) - -// Embedded images. -var ( - //go:embed resources/gvisor.png - gvisorPNG []byte - - //go:embed resources/chart.png - chartPNG []byte ) func TestOllama(t *testing.T) { @@ -115,766 +36,11 @@ func TestOllama(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("Ollama", func(t *testing.T) { t.Parallel() - doOllamaTest(ctx, t, k8sCtx, cluster) + BenchmarkOllama(ctx, t, k8sCtx, cluster) }) }) } -// ollamaPodServer implements `ollama.Server`. -// It performs requests against the ollama server pod. -type ollamaPodServer struct { - cluster *testcluster.TestCluster - clientImage string - pod *v13.Pod - service *v13.Service -} - -// readPodLogs reads logs from a pod. -func readPodLogs(ctx context.Context, cluster *testcluster.TestCluster, pod *v13.Pod) (string, error) { - rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) - if err != nil { - return "", fmt.Errorf("GetLogReader on cluster %q pod %q: %v", cluster.GetName(), pod.GetName(), err) - } - out, err := io.ReadAll(rdr) - if err != nil { - return "", fmt.Errorf("failed to read from pod %q: %v", pod.GetName(), err) - } - return string(out), nil -} - -// InstrumentedRequest implements `ollama.Server.InstrumentedRequest`. -func (ops *ollamaPodServer) InstrumentedRequest(ctx context.Context, argvFn func(hostPort string) []string) ([]byte, error) { - // Get server IP. - if err := ops.cluster.WaitForServiceReady(ctx, ops.service); err != nil { - return nil, fmt.Errorf("failed to wait for service: %v", err) - } - ip := testcluster.GetIPFromService(ops.service) - if ip == "" { - return nil, fmt.Errorf("did not get valid ip from service: %v", ops.service) - } - - // Build client pod spec. - const clientPodName = "ollama-client" - argv := argvFn(fmt.Sprintf("http://%s:%d", ip, ops.service.Spec.Ports[0].Port)) - clientPod := &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: clientPodName, - Namespace: ops.pod.ObjectMeta.Namespace, - }, - Spec: v13.PodSpec{ - Containers: []v13.Container{ - { - Name: clientPodName, - Image: ops.clientImage, - Command: argv, - Resources: v13.ResourceRequirements{ - Requests: v13.ResourceList{ - v13.ResourceCPU: resource.MustParse("500m"), - }, - }, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } - clientPod, err := ops.cluster.ConfigurePodForClientNodepool(clientPod) - if err != nil { - return nil, fmt.Errorf("failed to configure pod: %v", err) - } - - // Delete pod that may possibly exist from a previous iteration. - // Ignore errors since it most likely doesn't exist. - ops.cluster.DeletePod(ctx, clientPod) - - // Start new client pod and wait for it. - clientPod, err = ops.cluster.CreatePod(ctx, clientPod) - if err != nil { - return nil, fmt.Errorf("failed to create client pod: %v", err) - } - defer ops.cluster.DeletePod(ctx, clientPod) - if err := ops.cluster.WaitForPodCompleted(ctx, clientPod); err != nil { - logs, logsErr := readPodLogs(ctx, ops.cluster, clientPod) - logs = strings.TrimSpace(logs) - if logsErr != nil { - return nil, fmt.Errorf("failed HTTP request (%v) and to read logs from the pod: %w", err, logsErr) - } - if logs == "" { - return nil, fmt.Errorf("failed HTTP request: %w (pod logs are empty)", err) - } - return nil, fmt.Errorf("failed HTTP request: %w (pod logs: %v)", err, logs) - } - - // All good, get logs. - logs, err := readPodLogs(ctx, ops.cluster, clientPod) - if err != nil { - return nil, fmt.Errorf("failed to read logs from pod %q: %v", clientPod.GetName(), err) - } - return []byte(logs), nil -} - -// Logs implements `ollama.Server.Logs`. -func (ops *ollamaPodServer) Logs(ctx context.Context) (string, error) { - return readPodLogs(ctx, ops.cluster, ops.pod) -} - -// atLeastNWords verifies that the response at least N words. -// If not, it raises the temperature. -func atLeastNWords(wantNWords int) func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { - return func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { - responseText := strings.TrimSpace(response.Text()) - responseText = strings.Map(func(r rune) rune { - if unicode.IsLetter(r) { - return r - } - return ' ' - }, responseText) - numWords := 0 - for _, word := range strings.Split(responseText, " ") { - if len(word) >= 0 { - numWords++ - } - } - if numWords < wantNWords { - return prompt.WithHotterModel(), fmt.Errorf("response %q is too short: had %d words, want at least %d", responseText, numWords, wantNWords) - } - return nil, nil - } -} - -// doOllamaTest runs ollama benchmarks for a single cluster. -func doOllamaTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - logWithTime := func(t *testing.T, format string, values ...any) { - t.Logf("[%v] "+format, append([]any{time.Now().Format(time.TimeOnly)}, values...)...) - } - - // Run pod and service. - serverImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchImage) - if err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - ollamaPod, err := cluster.ConfigurePodForRuntimeTestNodepool(newOllamaServerPod(benchmarkNS, serverImage)) - if err != nil { - t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) - } - ollamaPod, err = testcluster.MaybeSetContainerResources(ollamaPod, ollamaPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - ollamaPod, err = cluster.CreatePod(ctx, ollamaPod) - if err != nil { - t.Fatalf("Failed to create ollama pod: %v", err) - } - defer cluster.DeletePod(ctx, ollamaPod) - logWithTime(t, "Waiting for ollama server pod to start, this may take a long time (tens of minutes) if this is the first time the image is being downloaded onto the node.") - startCtx, startCtxCancel := context.WithTimeout(ctx, 90*time.Minute) - if err := cluster.WaitForPodRunning(startCtx, ollamaPod); err != nil { - t.Fatalf("Failed to wait for ollama server pod: %v", err) - } - startCtxCancel() - logWithTime(t, "ollama server pod started on Kubernetes but not yet initialized.") - ollamaService := newOllamaService(benchmarkNS) - ollamaService, err = cluster.CreateService(ctx, ollamaService) - if err != nil { - t.Fatalf("Failed to create service: %v", err) - } - defer cluster.DeleteService(ctx, ollamaService) - ollamaClientImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchClientImage) - if err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - ollamaServer := &ollamaPodServer{ - cluster: cluster, - clientImage: ollamaClientImage, - service: ollamaService, - pod: ollamaPod, - } - llm, err := ollama.New(ctx, ollamaServer, t) - if err != nil { - t.Fatalf("Failed to create ollama client against server pod: %v", err) - } - llm.SetCheapModels(cheapModels) - logWithTime(t, "ollama server ready.") - - // Define test cases. - type testCase struct { - // Name of the test. - name string - // models to iterate through. - models []*ollama.Model - // Query for the ollama server. - query string - // Image to attach to the query, if any. - image []byte - // If set, run this function over the response to verify it. - // The LLM is prompted repeatedly until this function returns a non-nil error. - // This function may also return a non-nil prompt if it needs to modify the prompt - // for the next attempt. This is useful to raise the model temperature. - verifyResponse func(*ollama.Prompt, *ollama.Response) (*ollama.Prompt, error) - } - testCases := []testCase{ - { - name: "HelloWorld", - models: []*ollama.Model{ - modelLlamaChinese7B, - modelLlama13B, - modelLlama70B, - modelMistral7B, - modelMixtral8X7B, - }, - query: ` - Reply with the words: "Hello World!". - Do not reply anything else. - `, - verifyResponse: atLeastNWords(2), - }, - { - name: "SimpleTranslation", - models: []*ollama.Model{modelLlamaChinese7B}, - query: ` - Translate the following text from English to Chinese: - """ - From J. J. Nakalembe's Aqaba to Antarctica: Moments of Transition and Revelation: - - My great-grandmother lived to be 108 years old, - retaining her sharpness of mind to her last day. - A couple of months before she died, I interviewed her for my podcast. - At the end, I asked her the same question I asked every guest: - what is something you wish everyone understood? - - She thought about it for a while. - Then she said: "How it was before." - - She tried to explain how much time she spent every single day - on utterly mind-numbing activities, like hauling water from the well, - and how radically everything changed when their area - was finally connected to the electrical grid. - - "Before, there was no time to live," she said. - "No time to be free. Only work, work, work." - - I countered by saying that there didn't seem to be much time - to live now either, but she laughed derisively. - I had no idea what work really meant, she said. - - Young people were weak and feckless and that's why - we let corporations exploit us. - - Slightly unnerved by her harshness, - I asked her if she missed anything about that time. - The old photo of her village seemed idyllic to me, - but my grandmother quashed any romantic notions I might have had. - - "The past is bad," she said with some finality. - "Let it be." - """ - Do not reply anything other than the translation of these words. - `, - verifyResponse: atLeastNWords(100), - }, - { - name: "ExtractMeaning", - models: []*ollama.Model{ - modelLlama13B, - modelLlama70B, - modelMistral7B, - modelMixtral8X7B, - }, - query: ` - Consider the following text: - - """ - We assembled on the vast green lawn outside as the reactors began - to slowly wind down. The workers were solemn; the activists who had - fought against the decommissioning seemed crushed. There was - supposed to be a speech, but the spokeswoman had lost her notes. - Outside, the protesters cheered. - - My eyes were drawn to the discarded anti-shutdown banners, - endlessly reciting the facts. - The statistics on mortality per trillion kWh (lowest of all energy sources). - The lifespan of a reactor (70 more years, in our case). - Minimal land footprint. - Almost zero emissions. - No intermittency. - It became a jumble of words, a litany, almost a kind of glossolalia. - As far as the protesters outside were concerned, - it might as well be an alien tongue. - - One thing was clear to them, and that was enough: - the technology inside this compound was deeply, inherently wrong. It was a sin. - - I could not help but think of that moment on August 6th, 1945, - when the sky erupted above Shima Hospital. - My imagination could never fully encompass it. - How do you imagine more than seventy thousand people annihilated - in an instant? An ancestor of mine was in that hospital; he went - from being a doctor, a husband, a father, a pacifist stuck - in a terrible war, to being a pile of bleached bones covered in rubble, - all in a single second. - Not by accident, but because of a choice someone made. - Not because of a reactor, but because of a bomb. - - Just two days earlier, contradicting his campaign promises, - the prime minister had suggested that the use of - "tactical" weapons based on this technology would be an - acceptable risk if the conflict continued. - Very few seemed to find this particularly shocking or outrageous. - - They were afraid of reactors, but not of bombs. - - The spokeswoman gave up on finding her notes. - It was starting to rain, and people were walking away. - She grabbed the microphone. - - "By the time you regret this, it'll be too late," she said. - "But honestly, I don't know if I care anymore. Maybe you have it coming." - - The spokeswoman sounded so bitter. - The protesters didn't mean any harm. - From their perspective, they were doing good. - - Collective action can change the world when it's deliberate - and based in reason, but it can also become a mental trap, - or a societal pressure valve. - - People always think they're doing good when they get - collectively outraged. That doesn't make them right. - - The Flame will not harm you, Son of Man, if you wield it wisely. - """ - - Summarize what happened in the above text. - Then answer the following questions: - What technology is involved? - What are the protestors clamoring for? - What does the spokeswoman mean? - What does "The Flame" symbolize in the text? - `, - verifyResponse: atLeastNWords(32), - }, - { - name: "IdentifyCommonElements", - models: []*ollama.Model{ - modelLlama13B, - modelLlama70B, - modelMistral7B, - modelMixtral8X7B, - }, - query: ` - Consider the following four texts: - - Text 1: - """ - == The Ethics of Extinction == - - If a species we consider beautiful and remarkable goes extinct, - we consider that a great evil. - Dolphins, for example. If dolphins go extinct, that's a great loss. - If humanity causes dolphins to go extinct, that's a crime. - - But if Yersinia pestis, the bacterium that causes bubonic plague, - goes extinct... is that an equally great loss? If not, why not? - To Nature, there's no difference, it's all just lifeforms. - The only moral framework that allows us to choose - between dolphins and the plague is a human one. - - What about a species going extinct without anthropogenic factors? - Extinction is the norm. If one day dolphins are no longer capable - of competing with other species, should we let them go extinct? - When the sun finally dies and all life goes extinct, - will that be a tragedy? If we can prevent it, should we? - - If you believe that extinction is acceptable when Nature does it, - but not when we do it, then you don't actually oppose extinction. - You don't believe that dolphins are inherently valuable, - that they deserve to live and thrive. - You just oppose human control. - You oppose our ability and responsibility to choose. - """ - - Text 2: - """ - == Ecosystem Engineers == - - Cutting down swathes of trees for their building projects, - thoughtlessly causing radical changes to large environments - and forcing local species to adapt to their artificial habitats; - these are the traits of a species of intelligent, industrious, - and extremely impactful ecosystem engineers. - - Humans? No, I'm talking about beavers. - - Like us, beavers transform their environments via building, - and their actions have real consequences, creating vast wetlands - that some species thrive in - while others die. - Human activity is very similar: we too are ecosystem engineers, - and we too benefit some species while harming others. - Everything about this is completely natural, - including the damage to other species. - After all, that's what competition and evolution is all about. - - Those species that adapt to the ecosystems we create will, - over the coming millennia, become the core of a new biodiversity. - And so evolution runs its course. - If we don't like the result, if we think some species - should be preserved despite being outcompeted, well, - that's anything but natural. - It is, however, very human. - """ - - Text 3: - """ - == On Loyalty == - - From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": - - What I propose, then, is that we are not born as entirely free agents, - responsible only for ourselves. The very core of what we are, our sentience, - separates us from and elevates us above the animal kingdom. - As I have argued, this is not a matter of arrogance, but of responsibility. - - However, this blessing also demands something else from us, - something more personal than responsibility, and that is loyalty. - - Our ancestors, less atomized than we are, - experienced a crude version of this loyalty, - swearing allegiance to tribes, races, nations, - and other such semi-fictional concepts. - This fragmented understanding was easily exploited and led to many conflicts. - We can condemn them for that, or we can choose to believe these were - necessary historical steps towards our growth; - but above all we must stop indulging in such childlike behavior. - - Our species can no longer afford to believe in Mother Russia or Uncle Sam. - Neither, however, can we afford to indulge in the adolescent rebel's misanthropy, - rejecting the many gifts we have been lucky enough to receive - not from above, - but from the history of our species. - - To put it simply: each of us owes a burden of loyalty to humanity itself, - to the human project across time and space. - This is not a minor matter, or some abstract issue for philosophers. - It is a profound and significant part of every human life. - It is a universal source of meaning and insight that can bind us together - and set us on a path for a brighter future; and it is also a division, - a line that must held against those who preach the gospel of self-annihilation. - We ignore it at our peril. - """ - - Text 4: - """ - == On Nature == - - From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": - - The question of our relationship with Nature has bedevilled us - since the earliest days of our species. - Since before the first city was built, - we felt that there was something different about us. - Animals, we intuited, were part of Nature; we were not. - - But of course, humans clearly are the products of Nature, - our history intertwined with that of every other species. - In fact, the very notion of the "unnatural" is a contradiction in terms. - Everything that exists must, by definition, be natural. - So this view, no matter how common, is deeply paradoxical. - This paradox has produced a great deal of confusion. - - Some proclaim us chosen by a divine power, - set above all other creatures, - and are justly accused of arrogance. - Others proclaim us sinners, worse than other creatures, - and are rightly accused of misanthropy. - Others yet try to oppose this binary by saying that - we are merely animals after all - but that too is manifestly wrong, - in that no other animal is capable of having this conversation. - - It is in the contentious issue of our impact on the ecosphere - that an answer may be found. - Other animals have accidentally terraformed the planet before, - driving other species to extinction. - This is not unnatural. - If we continued our current path, even to the point of changing - the climate enough to cause the collapse of civilization, - that would be entirely in keeping with how animals behave. - - But there is one profound way in which we are not like animals: - we can learn to understand ourselves and the world. - It is this knowledge that makes us fundamentally different. - We have choices. We have control. - - There are many today who are afraid of the consequences of control, - and would prefer a return to a state of animal ignorance, - whether by blinding ourselves to the impact of our actions - or by demanding we humble ourselves before Nature. - This is the response of an adult in crisis, - who wishes for a return to childhood. - But this can only ever be regressive in every sense of the word. - - To resolve the paradox of Nature we must act as adults: - accept our power, and act consciously and deliberately - in shaping the world. - We must become Nature, and Nature must become human. - """ - - Your task is to look for the common thread between these texts. - Find commonalities and common themes between these texts, - and summarize their essence down to at most 5 words. - `, - verifyResponse: atLeastNWords(4), - }, - { - name: "CodeGen", - models: []*ollama.Model{ - modelCodeLlama7B, - modelCodeLlama34B, - }, - query: ` - Write a Python function to compute the digits of pi using the Chudnovsky algorithm. - Do not write unit tests. Do not explain how the code works. Reply with only Python code. - `, - verifyResponse: atLeastNWords(8), - }, - { - name: "CodeDebug", - models: []*ollama.Model{ - modelCodeLlama7B, // Note: codellama-7b will often get this one wrong. - modelCodeLlama34B, - }, - query: strings.ReplaceAll(` - Help me debug the following Python code: - - ||| - def count_words(s): - """Counts the number of words in the sentence |s|.""" - total_words = 0 - for word in s.split(' '): - total_words += len(word) - return total_words - ||| - - This function isn't working as expected. - For example, if I call |count_words('Master Foo and the Shell Tools')|, - I get 25, but there are only 6 words in the string - "Master Foo and the Shell Tools". - `, "|", "`"), - verifyResponse: atLeastNWords(16), - }, - { - name: "GVisorLogoOCR", - models: []*ollama.Model{ - modelLlava7B, - modelLlava34B, - }, - query: ` - This is an image of a logo of a software project. - What is the name of this project? - `, - image: gvisorPNG, - }, - { - name: "InterpretGraph", - models: []*ollama.Model{ - modelLlava7B, - modelLlava34B, - }, - query: ` - This is a chart with multiple trendlines showing a pattern over time. - Answer the following questions in order: - - 1. What is the title of the chart? - 2. What do the X and Y axis of the chart measure? - 3. List the label of each data line on the chart. - 4. What trend is each data line showing? - 5. What else is remarkable about this chart? - 6. What insights can you infer from this chart? - `, - image: chartPNG, - }, - } - - modelsInOrder := make([]*ollama.Model, len(allModels)) - copy(modelsInOrder, allModels) - // Shuffle the models. - rand.New(rand.NewSource(time.Now().UnixNano())).Shuffle(len(modelsInOrder), func(i, j int) { - modelsInOrder[i], modelsInOrder[j] = modelsInOrder[j], modelsInOrder[i] - }) - t.Logf("Will go through models in this order: %v", modelsInOrder) - - // We invert the hierarchy here: the model is the outer test, and the prompt - // is the inner text. This is because it is more often useful to gauge a - // model's performance as a whole regardless of its prompt, rather than - // the performance of the same prompt across models. It also makes it - // easier to filter by models rather than by prompt, which is the more - // often-desired filter. - for _, model := range modelsInOrder { - t.Run(model.Name, func(t *testing.T) { - modelBenchmarkName := strings.ReplaceAll(model.Name, ":", "-") - t.Run("ModelLoad", func(t *testing.T) { - const loadTimeout = 10 * time.Minute - loadCtx, loadCancel := context.WithTimeout(ctx, loadTimeout) - defer loadCancel() - loadStats, err := llm.WarmModel(loadCtx, model, loadTimeout, true) - if err != nil { - t.Fatalf("cannot load model %v: %v", model, err) - } - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - if err := recorder.Record(ctx, fmt.Sprintf("Ollama/%s/ModelLoad", modelBenchmarkName), benchmetric.SpecificDuration(loadStats.ClientReportedDuration, "load")); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - for _, test := range testCases { - hasModel := false - for _, testModel := range test.models { - if testModel.Name == model.Name { - hasModel = true - break - } - } - if !hasModel { - continue - } - t.Run(test.name, func(t *testing.T) { - verifyFn := atLeastNWords(1) - if test.verifyResponse != nil { - verifyFn = test.verifyResponse - } - numAttempts := 0 - verifyFnCount := func(prompt *ollama.Prompt, resp *ollama.Response) (*ollama.Prompt, error) { - numAttempts++ - return verifyFn(prompt, resp) - } - const testTimeout = 25 * time.Minute - testCtx, testCancel := context.WithTimeout(ctx, testTimeout) - defer testCancel() - _, err := llm.WarmModel(testCtx, model, testTimeout, false) - if err != nil { - t.Fatalf("cannot warm model %v: %v", model, err) - } - prompt := &ollama.Prompt{ - Model: model, - Query: test.query, - } - if test.image != nil { - prompt.AddImage(test.image) - } - resp, err := llm.PromptUntil(testCtx, prompt, verifyFnCount) - if err != nil { - t.Fatalf("cannot prompt: %v", err) - } - if !resp.Done() { - t.Fatalf("warm response did not finish: %v", resp) - } - imageDetail := "" - if test.image != nil { - imageDetail = " (and attached image)" - } - logWithTime(t, "Prompting model %s with query%s:\n%s\n\nResponse:\n%s\n(end of response)", model.Name, imageDetail, prompt.CleanQuery(), resp.Text()) - respHash := fnv.New32() - respHash.Write([]byte(resp.Text())) - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - err = recorder.Record( - ctx, - fmt.Sprintf("Ollama/%s/%s", modelBenchmarkName, test.name), - benchmetric.BenchmarkDuration(resp.TotalDuration()), - benchmetric.SpecificDuration(resp.PromptEvalDuration(), "prompteval"), - benchmetric.SpecificDuration(resp.EvalDuration(), "eval"), - benchmetric.SpecificDuration(resp.TimeToFirstToken(), "tok-first"), - benchmetric.SpecificDuration(resp.TimeToLastToken(), "tok-last"), - benchmetric.Rate(resp.OutputTokensPerSecond(), "tok"), - benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.5), "tok-p50"), - benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.95), "tok-p95"), - benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.99), "tok-p99"), - benchmetric.SpecificDuration(resp.TokenGenerationStdDev(), "tok-stddev"), - benchmetric.Count(uint64(numAttempts), "prompt-attempts"), - benchmetric.Count(uint64(resp.NumTokens()), "resp-tokens"), - benchmetric.Checksum(respHash, "resp"), - ) - if err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } - }) - } -} - -const ( - ollamaServerLabelKey = "app.kubernetes.io/name" - ollamaServerLabelValue = "ollama-server" - ollamaPort = 11434 - ollamaPodName = "ollama-server" - ollamaServiceName = "ollama-service" - ollamaBenchImage = k8s.ImageRepoPrefix + "benchmarks/gpu/ollama/bench:latest" - ollamaBenchClientImage = k8s.ImageRepoPrefix + "benchmarks/gpu/ollama/client:latest" -) - -// newOllamaServerPod returns the pod spec for an ollama server. -func newOllamaServerPod(namespace *testcluster.Namespace, image string) *v13.Pod { - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: ollamaPodName, - Namespace: namespace.Namespace, - Labels: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, - }, - Spec: v13.PodSpec{ - Containers: []v13.Container{ - { - Name: ollamaPodName, - Image: image, - Env: []v13.EnvVar{ - // Bind to all addresses, not just localhost: - {Name: "OLLAMA_HOST", Value: fmt.Sprintf("0.0.0.0:%d", ollamaPort)}, - // Accept requests from anywhere: - {Name: "OLLAMA_ORIGINS", Value: "*"}, - }, - Ports: []v13.ContainerPort{ - { - Name: ollamaServiceName, - ContainerPort: ollamaPort, - }, - }, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - -// newOllamaService returns a service definition for the ollama server pod. -func newOllamaService(namespace *testcluster.Namespace) *v13.Service { - return namespace.GetService(ollamaServiceName, v13.ServiceSpec{ - Selector: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, - Ports: []v13.ServicePort{ - { - Name: ollamaServiceName, - Protocol: v13.ProtocolTCP, - Port: ollamaPort, - TargetPort: intstr.FromString(ollamaServiceName), - }, - }, - }) -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestOllama": TestOllama, diff --git a/test/kubernetes/benchmarks/postgresql.go b/test/kubernetes/benchmarks/postgresql.go new file mode 100644 index 000000000..d24759268 --- /dev/null +++ b/test/kubernetes/benchmarks/postgresql.go @@ -0,0 +1,371 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package postgresql benchmarks a PostgreSQL database. +package postgresql + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + postgresServerLabelKey = "app.kubernetes.io/name" + postgresServerLabelValue = "postgresql-server" + postgresPort = 5432 + postgresImage = "postgres:15.3-alpine" + postgresUser = "benchman" + postgresPassword = "hunter2" + postgresDatabase = "benchpress" + postgresVolumeDir = "/var/lib/postgresql/data" + postgresDataDir = "/var/lib/postgresql/data/pgdata" +) + +var ( + numConnections = []int{1, 2, 6, 16, 32, 64} +) + +// BenchmarkPostgresPGBench runs a PostgreSQL pgbench test. +func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a persistent volume on which to store the database data. + persistentVol := benchmarkNS.GetPersistentVolume("pgdata", "30Gi") + persistentVol, err = cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + // Create a server on the runtime under test nodepool. + image, err := k8sCtx.ResolveImage(ctx, postgresImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + server := newPostgresPod(benchmarkNS, "postgresql", image, nil, true /* withPort */, persistentVol) + if server.ObjectMeta.Labels == nil { + server.ObjectMeta.Labels = make(map[string]string) + } + server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) + } + + server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) + } + + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) + } + + // Create a service to service traffic to the pod. + service := newPostgresService(benchmarkNS, server.GetName()) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeleteService(ctx, service) + if err := cluster.WaitForServiceReady(ctx, service); err != nil { + t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) + } + + ip := testcluster.GetIPFromService(service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + + // Run the 'pg_isready' command to ping the server and make sure it is up. + ensureUp := func() error { + pgIsReadyName := "pgisready" + pgIsReady := newPostgresPod(benchmarkNS, pgIsReadyName, image, []string{ + "pg_isready", + "--timeout=10", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + fmt.Sprintf("--dbname=%s", postgresDatabase), + }, false /* withPort */, nil /* pvc */) + pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err) + } + + pgIsReady, err = cluster.CreatePod(ctx, pgIsReady) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", pgIsReady.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, pgIsReady) + + waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) + defer waitCancel() + if err := cluster.WaitForPodCompleted(waitCtx, pgIsReady); err != nil { + return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), pgIsReadyName, err) + } + + return nil + } + var isUpErr error + for i := 0; i < 5; i++ { + if isUpErr = ensureUp(); isUpErr == nil { + break + } + } + if isUpErr != nil { + t.Fatalf("postgresql did not come up: %v", isUpErr) + } + + // pgbench has two steps: an "init step" which create and fills up a + // database with stuff, and then a main phase which does queries on that + // stuff. + // The initialization only needs to be done once per database. + initDatabase := func() error { + initDBName := "initdb" + initDB := newPostgresPod(benchmarkNS, initDBName, image, []string{ + "pgbench", + "--initialize", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + postgresDatabase, + }, false /* withPort */, nil /* pvc */) + initDB, err = cluster.ConfigurePodForClientNodepool(initDB) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err) + } + + initDB, err = cluster.CreatePod(ctx, initDB) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", initDB.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, initDB) + + waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) + defer waitCancel() + if err := cluster.WaitForPodCompleted(waitCtx, initDB); err != nil { + return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), initDBName, err) + } + return nil + } + if err := initDatabase(); err != nil { + t.Fatalf("cannot initialize database: %v", err) + } + + for _, connections := range numConnections { + t.Run(fmt.Sprintf("%dClients", connections), func(t *testing.T) { + clientCmd := []string{ + "pgbench", + "--time=90", // In seconds + "--report-per-command", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + fmt.Sprintf("--client=%d", connections), + fmt.Sprintf("--jobs=%d", connections), + postgresDatabase, + } + client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */) + client, err = cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + client, err = cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, client) + + if err := cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + // get and parse the logs from the client to get the results + rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + + if err != nil { + t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) + } + + postgresBenchmarkName := fmt.Sprintf("PostgresPGBench/%dClients", connections) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + metrics, err := getMeasurements(string(out)) + if err != nil { + t.Fatalf("failed to get metrics: out:\n\n%s\n\nerr: %v", string(out), err) + } + if err := recorder.Record(ctx, postgresBenchmarkName, metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newPostgresService gets a service to serve traffic to the PostgreSQL server. +func newPostgresService(namespace *testcluster.Namespace, containerName string) *v13.Service { + name := fmt.Sprintf("postgresql-service-%d", time.Now().UnixNano()) + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{postgresServerLabelKey: postgresServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: postgresPort, + TargetPort: intstr.FromInt(postgresPort), + }, + }, + }) +} + +func newPostgresPod(namespace *testcluster.Namespace, containerName, image string, argv []string, withPort bool, pvc *v13.PersistentVolumeClaim) *v13.Pod { + pod := namespace.NewPod(containerName) + container := v13.Container{ + Name: containerName, + Image: image, + Command: argv, + Env: []v13.EnvVar{ + // Used by postgres server: + {Name: "POSTGRES_USER", Value: postgresUser}, + {Name: "POSTGRES_PASSWORD", Value: postgresPassword}, + {Name: "POSTGRES_DB", Value: postgresDatabase}, + {Name: "PGDATA", Value: postgresDataDir}, + + // Used by pgbench: + {Name: "PGPASSWORD", Value: postgresPassword}, + {Name: "sslmode", Value: "disable"}, + }, + } + if withPort { + container.Ports = append(container.Ports, v13.ContainerPort{ContainerPort: postgresPort}) + } + if pvc != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ + Name: pvc.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.GetName(), + }, + }, + }) + container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ + MountPath: postgresVolumeDir, + Name: pvc.GetName(), + }) + } + pod.Spec.Containers = append(pod.Spec.Containers, container) + return pod +} + +var ( + latencyRegex = regexp.MustCompile("^latency average = ([-,.\\d]+ .?s)$") + initialConnectionRegex = regexp.MustCompile("^initial connection time = ([-,.\\d]+ .?s)$") + tpsRegex = regexp.MustCompile("^tps = ([-,.\\d]+) \\(without initial connection time\\)$") +) + +func stringToFloat64(s string) float64 { + f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) + if err != nil { + panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) + } + return f +} + +func stringToDuration(s string) time.Duration { + parts := strings.SplitN(s, " ", 2) + floatStr, unit := parts[0], parts[1] + floatPart := stringToFloat64(floatStr) + switch unit { + case "s": + return time.Duration(floatPart * float64(time.Second)) + case "ms": + return time.Duration(floatPart * float64(time.Millisecond)) + case "us", "μs": + return time.Duration(floatPart * float64(time.Microsecond)) + case "ns": + return time.Duration(floatPart * float64(time.Nanosecond)) + default: + panic(fmt.Sprintf("unknown time unit %q", unit)) + } +} + +// getMeasurements parses the output of pgbench to get the stats. +func getMeasurements(out string) ([]benchmetric.MetricValue, error) { + var foundLatency, foundInitialConnection, foundTPS benchmetric.MetricValue + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if latencyMatch := latencyRegex.FindStringSubmatch(line); latencyMatch != nil { + if foundLatency != nil { + return nil, fmt.Errorf("found duplicate latency data: %v vs %q", foundLatency, line) + } + foundLatency = benchmetric.SpecificDuration(stringToDuration(latencyMatch[1]), "avg") + } + if initialConnectionMatch := initialConnectionRegex.FindStringSubmatch(line); initialConnectionMatch != nil { + if foundInitialConnection != nil { + return nil, fmt.Errorf("found duplicate initial connection data: %v vs %q", foundInitialConnection, line) + } + foundInitialConnection = benchmetric.SpecificDuration(stringToDuration(initialConnectionMatch[1]), "init") + } + if tpsMatch := tpsRegex.FindStringSubmatch(line); tpsMatch != nil { + if foundTPS != nil { + return nil, fmt.Errorf("found duplicate TPS data: %v vs %q", foundTPS, line) + } + foundTPS = benchmetric.RequestsPerSecond(stringToFloat64(tpsMatch[1])) + } + } + if foundLatency == nil || foundInitialConnection == nil || foundTPS == nil { + return nil, fmt.Errorf("did not find the data we wanted: foundLatency=%v foundInitialConnection=%v foundTPS=%v", foundLatency, foundInitialConnection, foundTPS) + } + return []benchmetric.MetricValue{ + foundLatency, + foundInitialConnection, + foundTPS, + }, nil +} diff --git a/test/kubernetes/benchmarks/postgresql_test.go b/test/kubernetes/benchmarks/postgresql_test.go index acc35b579..0c32aa56c 100644 --- a/test/kubernetes/benchmarks/postgresql_test.go +++ b/test/kubernetes/benchmarks/postgresql_test.go @@ -12,41 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package postgresql_test benchmarks a PostgreSQL database. -package postgresql_test +// Package postgresql benchmarks a PostgreSQL database. +package postgresql import ( "context" - "fmt" - "io" - "regexp" - "strconv" - "strings" "testing" - "time" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const ( - postgresServerLabelKey = "app.kubernetes.io/name" - postgresServerLabelValue = "postgresql-server" - postgresPort = 5432 - postgresImage = "postgres:15.3-alpine" - postgresUser = "benchman" - postgresPassword = "hunter2" - postgresDatabase = "benchpress" - postgresVolumeDir = "/var/lib/postgresql/data" - postgresDataDir = "/var/lib/postgresql/data/pgdata" -) - -var ( - numConnections = []int{1, 2, 6, 16, 32, 64} ) // TestPostgresPGBench benchmarks a PostgreSQL database with pgbench. @@ -59,332 +33,11 @@ func TestPostgresPGBench(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("PostgresPGBench", func(t *testing.T) { t.Parallel() - doPostgresPGBenchTest(ctx, t, k8sCtx, cluster) + BenchmarkPostgresPGBench(ctx, t, k8sCtx, cluster) }) }) } -// doPostgresTest runs a PostgreSQL pgbench test. -func doPostgresPGBenchTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - // Create a persistent volume on which to store the database data. - persistentVol := benchmarkNS.GetPersistentVolume("pgdata", "30Gi") - persistentVol, err = cluster.CreatePersistentVolume(ctx, persistentVol) - if err != nil { - t.Fatalf("failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - // Create a server on the runtime under test nodepool. - image, err := k8sCtx.ResolveImage(ctx, postgresImage) - if err != nil { - t.Fatalf("failed to resolve image: %v", err) - } - server := newPostgresPod(benchmarkNS, "postgresql", image, nil, true /* withPort */, persistentVol) - if server.ObjectMeta.Labels == nil { - server.ObjectMeta.Labels = make(map[string]string) - } - server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue - server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) - if err != nil { - t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) - } - - server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) - } - - server, err = cluster.CreatePod(ctx, server) - if err != nil { - t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, server) - - if err := cluster.WaitForPodRunning(ctx, server); err != nil { - t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) - } - - // Create a service to service traffic to the pod. - service := newPostgresService(benchmarkNS, server.GetName()) - service, err = cluster.CreateService(ctx, service) - if err != nil { - t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) - } - defer cluster.DeleteService(ctx, service) - if err := cluster.WaitForServiceReady(ctx, service); err != nil { - t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) - } - - ip := testcluster.GetIPFromService(service) - if ip == "" { - t.Fatalf("did not get valid ip: %s", ip) - } - - // Run the 'pg_isready' command to ping the server and make sure it is up. - ensureUp := func() error { - pgIsReadyName := "pgisready" - pgIsReady := newPostgresPod(benchmarkNS, pgIsReadyName, image, []string{ - "pg_isready", - "--timeout=10", - fmt.Sprintf("--host=%s", ip), - fmt.Sprintf("--port=%d", postgresPort), - fmt.Sprintf("--username=%s", postgresUser), - fmt.Sprintf("--dbname=%s", postgresDatabase), - }, false /* withPort */, nil /* pvc */) - pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady) - if err != nil { - return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err) - } - - pgIsReady, err = cluster.CreatePod(ctx, pgIsReady) - if err != nil { - return fmt.Errorf("CreatePod %q on cluster %q: %v", pgIsReady.GetName(), cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, pgIsReady) - - waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) - defer waitCancel() - if err := cluster.WaitForPodCompleted(waitCtx, pgIsReady); err != nil { - return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), pgIsReadyName, err) - } - - return nil - } - var isUpErr error - for i := 0; i < 5; i++ { - if isUpErr = ensureUp(); isUpErr == nil { - break - } - } - if isUpErr != nil { - t.Fatalf("postgresql did not come up: %v", isUpErr) - } - - // pgbench has two steps: an "init step" which create and fills up a - // database with stuff, and then a main phase which does queries on that - // stuff. - // The initialization only needs to be done once per database. - initDatabase := func() error { - initDBName := "initdb" - initDB := newPostgresPod(benchmarkNS, initDBName, image, []string{ - "pgbench", - "--initialize", - fmt.Sprintf("--host=%s", ip), - fmt.Sprintf("--port=%d", postgresPort), - fmt.Sprintf("--username=%s", postgresUser), - postgresDatabase, - }, false /* withPort */, nil /* pvc */) - initDB, err = cluster.ConfigurePodForClientNodepool(initDB) - if err != nil { - return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err) - } - - initDB, err = cluster.CreatePod(ctx, initDB) - if err != nil { - return fmt.Errorf("CreatePod %q on cluster %q: %v", initDB.GetName(), cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, initDB) - - waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) - defer waitCancel() - if err := cluster.WaitForPodCompleted(waitCtx, initDB); err != nil { - return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), initDBName, err) - } - return nil - } - if err := initDatabase(); err != nil { - t.Fatalf("cannot initialize database: %v", err) - } - - for _, connections := range numConnections { - t.Run(fmt.Sprintf("%dClients", connections), func(t *testing.T) { - clientCmd := []string{ - "pgbench", - "--time=90", // In seconds - "--report-per-command", - fmt.Sprintf("--host=%s", ip), - fmt.Sprintf("--port=%d", postgresPort), - fmt.Sprintf("--username=%s", postgresUser), - fmt.Sprintf("--client=%d", connections), - fmt.Sprintf("--jobs=%d", connections), - postgresDatabase, - } - client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */) - client, err = cluster.ConfigurePodForClientNodepool(client) - if err != nil { - t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) - } - - client, err = cluster.CreatePod(ctx, client) - if err != nil { - t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, client) - - if err := cluster.WaitForPodCompleted(ctx, client); err != nil { - t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) - } - - // get and parse the logs from the client to get the results - rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) - - if err != nil { - t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) - } - - out, err := io.ReadAll(rdr) - if err != nil { - t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) - } - - postgresBenchmarkName := fmt.Sprintf("PostgresPGBench/%dClients", connections) - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - metrics, err := getMeasurements(string(out)) - if err != nil { - t.Fatalf("failed to get metrics: out:\n\n%s\n\nerr: %v", string(out), err) - } - if err := recorder.Record(ctx, postgresBenchmarkName, metrics...); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } -} - -// newPostgresService gets a service to serve traffic to the PostgreSQL server. -func newPostgresService(namespace *testcluster.Namespace, containerName string) *v13.Service { - name := fmt.Sprintf("postgresql-service-%d", time.Now().UnixNano()) - return namespace.GetService(name, v13.ServiceSpec{ - Selector: map[string]string{postgresServerLabelKey: postgresServerLabelValue}, - Ports: []v13.ServicePort{ - { - Name: name, - Protocol: v13.ProtocolTCP, - Port: postgresPort, - TargetPort: intstr.FromInt(postgresPort), - }, - }, - }) -} - -func newPostgresPod(namespace *testcluster.Namespace, containerName, image string, argv []string, withPort bool, pvc *v13.PersistentVolumeClaim) *v13.Pod { - pod := namespace.NewPod(containerName) - container := v13.Container{ - Name: containerName, - Image: image, - Command: argv, - Env: []v13.EnvVar{ - // Used by postgres server: - {Name: "POSTGRES_USER", Value: postgresUser}, - {Name: "POSTGRES_PASSWORD", Value: postgresPassword}, - {Name: "POSTGRES_DB", Value: postgresDatabase}, - {Name: "PGDATA", Value: postgresDataDir}, - - // Used by pgbench: - {Name: "PGPASSWORD", Value: postgresPassword}, - {Name: "sslmode", Value: "disable"}, - }, - } - if withPort { - container.Ports = append(container.Ports, v13.ContainerPort{ContainerPort: postgresPort}) - } - if pvc != nil { - pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ - Name: pvc.GetName(), - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.GetName(), - }, - }, - }) - container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ - MountPath: postgresVolumeDir, - Name: pvc.GetName(), - }) - } - pod.Spec.Containers = append(pod.Spec.Containers, container) - return pod -} - -var ( - latencyRegex = regexp.MustCompile("^latency average = ([-,.\\d]+ .?s)$") - initialConnectionRegex = regexp.MustCompile("^initial connection time = ([-,.\\d]+ .?s)$") - tpsRegex = regexp.MustCompile("^tps = ([-,.\\d]+) \\(without initial connection time\\)$") -) - -func stringToFloat64(s string) float64 { - f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) - if err != nil { - panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) - } - return f -} - -func stringToDuration(s string) time.Duration { - parts := strings.SplitN(s, " ", 2) - floatStr, unit := parts[0], parts[1] - floatPart := stringToFloat64(floatStr) - switch unit { - case "s": - return time.Duration(floatPart * float64(time.Second)) - case "ms": - return time.Duration(floatPart * float64(time.Millisecond)) - case "us", "μs": - return time.Duration(floatPart * float64(time.Microsecond)) - case "ns": - return time.Duration(floatPart * float64(time.Nanosecond)) - default: - panic(fmt.Sprintf("unknown time unit %q", unit)) - } -} - -// getMeasurements parses the output of pgbench to get the stats. -func getMeasurements(out string) ([]benchmetric.MetricValue, error) { - var foundLatency, foundInitialConnection, foundTPS benchmetric.MetricValue - for _, line := range strings.Split(out, "\n") { - line = strings.TrimSpace(line) - if latencyMatch := latencyRegex.FindStringSubmatch(line); latencyMatch != nil { - if foundLatency != nil { - return nil, fmt.Errorf("found duplicate latency data: %v vs %q", foundLatency, line) - } - foundLatency = benchmetric.SpecificDuration(stringToDuration(latencyMatch[1]), "avg") - } - if initialConnectionMatch := initialConnectionRegex.FindStringSubmatch(line); initialConnectionMatch != nil { - if foundInitialConnection != nil { - return nil, fmt.Errorf("found duplicate initial connection data: %v vs %q", foundInitialConnection, line) - } - foundInitialConnection = benchmetric.SpecificDuration(stringToDuration(initialConnectionMatch[1]), "init") - } - if tpsMatch := tpsRegex.FindStringSubmatch(line); tpsMatch != nil { - if foundTPS != nil { - return nil, fmt.Errorf("found duplicate TPS data: %v vs %q", foundTPS, line) - } - foundTPS = benchmetric.RequestsPerSecond(stringToFloat64(tpsMatch[1])) - } - } - if foundLatency == nil || foundInitialConnection == nil || foundTPS == nil { - return nil, fmt.Errorf("did not find the data we wanted: foundLatency=%v foundInitialConnection=%v foundTPS=%v", foundLatency, foundInitialConnection, foundTPS) - } - return []benchmetric.MetricValue{ - foundLatency, - foundInitialConnection, - foundTPS, - }, nil -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestPostgresPGBench": TestPostgresPGBench, diff --git a/test/kubernetes/benchmarks/pytorch.go b/test/kubernetes/benchmarks/pytorch.go new file mode 100644 index 000000000..696374537 --- /dev/null +++ b/test/kubernetes/benchmarks/pytorch.go @@ -0,0 +1,381 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pytorch contains benchmarks using the pytorch "torchbench" repo. +package pytorch + +// These tests use pytorch's "torchbench" suite (https://github.com/pytorch/benchmark/tree/main). +// The Authors describe the benchmarks in this paper: https://arxiv.org/pdf/2304.14226.pdf +// The Authors list both the type of model and its profile (how GPU intensive). + +// Note: The image for this test is about 7-8 GB as of writing. After you get your clusters up and +// running, start the test and make sure that the pods show the event of downloading the image. Then +// get a cup of coffee, chat with your co-workers for 5 min, and it will be about done 5 min after +// that. You'll only need to do this once for each cluster (in parallel). + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + + v13 "k8s.io/api/core/v1" +) + +// pytorchTestType is the method used, either training or evaluation, for the model. +type pytorchTestType string + +const ( + train = pytorchTestType("train") + eval = pytorchTestType("eval") + + pytorchImage = k8s.ImageRepoPrefix + "benchmarks/pytorch_x86_64:f6f280aeb1b07989" +) + +type pytorchMode string + +// pytorchMode is the pytorch mode used, either script mode (jit) or eager mode. +// See: https://towardsdatascience.com/pytorch-jit-and-torchscript-c2a77bac0fff +const ( + jit = pytorchMode("jit") + eager = pytorchMode("eager") +) + +type pytorchTest struct { + module string + test pytorchTestType + mode pytorchMode +} + +// Sets of tests. +var ( + // FastNLPBert uses the fastNLP_Bert module, which is classified as a NLP Language Model. + // fastNLP_Bert taxes the GPU heavily with low data movement. See Figure 2 on + // page 5: https://arxiv.org/pdf/2304.14226.pdf + // + // https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/fastNLP_Bert + // Bert Blog Post: https://towardsdatascience.com/bert-explained-state-of-the-art-language-model-for-nlp-f8b21a9b6270 + // Paper: https://arxiv.org/abs/1810.04805 + FastNLPBert = []pytorchTest{ + { + module: "fastNLP_Bert", + test: train, + mode: eager, + }, + { + module: "fastNLP_Bert", + test: eval, + mode: eager, + }, + } + + // BigBird uses the hf_BigBird module, which is classified as a NLP Language Model. + // hf_BigBird taxes the GPU moderately with low data movement. See Figure 2 on + // page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf + // + // https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/hf_BigBird + // Paper: https://arxiv.org/abs/2007.14062 + BigBird = []pytorchTest{ + { + module: "hf_BigBird", + test: train, + mode: eager, + }, + { + module: "hf_BigBird", + test: eval, + mode: eager, + }, + } + + // SpeechTransformer uses the speech_transformer module classified as "Speech Recognition" + // model. speech_transformer has a lot of idle time for the GPU. See Figure 2 on + // page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf + // + // https://github.com/pytorch/benchmark/pull/374 + // Paper: https://arxiv.org/abs/1706.03762 + SpeechTransformer = []pytorchTest{ + { + module: "speech_transformer", + test: train, + mode: eager, + }, + { + module: "speech_transformer", + test: eval, + mode: eager, + }, + } + + // LearningToPaint uses the LearningToPaint module classified as "neural renderer in model-based + // Deep Reinforcement Learning (DRL)". + // Learning to paint has a lot of "data movement" and doesn't tax the GPU a lot. See Figure 2 on + // page 5: https://arxiv.org/pdf/2304.14226.pdf + // + // https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/LearningToPaint + LearningToPaint = []pytorchTest{ + { + module: "LearningToPaint", + test: train, + mode: jit, + }, + { + module: "LearningToPaint", + test: eval, + mode: jit, + }, + } + + // MobileNetV2 uses the mobilenet_v2 module classified as "Computer Vision: Image Classification". + // MobileNet has a lot of taxes the GPU. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf + // + // https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/mobilenet_v2 + // Paper: https://paperswithcode.com/method/mobilenetv2 + MobileNetV2 = []pytorchTest{ + { + module: "mobilenet_v2", + test: train, + mode: jit, + }, + { + module: "mobilenet_v2", + test: eval, + mode: jit, + }, + } + + // BackgroundMatting uses the Background_Matting module classified as "Computer Vision: Pattern Recognition". + // BackgroundMatting has a lot of GPU idle time. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf + // + // https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/Background_Matting (see README) + BackgroundMatting = []pytorchTest{ + { + module: "Background_Matting", + test: train, + mode: eager, + }, + { + module: "Background_Matting", + test: eval, + mode: eager, + }, + } +) + +// Name returns the name of the test with the argument parameters included. It is formatted so +// that it can be used for the name of the pod. +func (p pytorchTest) Name() string { + // Kubernetes pod names cannot contain "_". + module := strings.ReplaceAll(strings.ToLower(p.module), "_", "-") + return fmt.Sprintf("%s-%s-%s", module, p.test, p.mode) +} + +var snakeCase = regexp.MustCompile("_.") + +// BenchName returns the name of the test with the argument parameters included. +// It is formatted so that it can be used for benchstat output. +func (p pytorchTest) BenchName() string { + // First letter of the module should be capitalized, as it will be + // concatenated with "Benchmark" and it's useful to mark it as a different + // word. + // Some modules use a lowercase first letter, e.g. "fastNLP_Bert". + moduleName := strings.ToUpper(p.module[:1]) + p.module[1:] + // We also replace "snake_case" with "snakeCase". Sorry snakes. + moduleName = snakeCase.ReplaceAllStringFunc(moduleName, func(s string) string { + return strings.ToUpper(strings.TrimPrefix(s, "_")) + }) + test := strings.ToUpper(string(p.test)[:1]) + string(p.test[1:]) + var mode string + switch p.mode { + case eager: + mode = "Eager" + case jit: + mode = "JIT" + default: + panic(fmt.Sprintf("Unknown mode: %v", p.mode)) + } + return fmt.Sprintf("%s/%s/%s", moduleName, test, mode) +} + +func (p pytorchTest) toPod(namespace *testcluster.Namespace, image string) (*v13.Pod, error) { + pod := namespace.NewPod(p.Name()) + pod.Spec = v13.PodSpec{ + RestartPolicy: v13.RestartPolicyNever, + Containers: []v13.Container{ + { + Name: p.Name(), + Image: pytorchImage, + Command: benchmetric.TimedCommand(p.command()...), + }, + }, + } + return pod, nil +} + +func (p pytorchTest) command() []string { + return []string{ + "python", + "run.py", + p.module, + "--device", "cuda", + "--test", string(p.test), + "--mode", string(p.mode), + } +} + +// RunPytorch runs the given PyTorch tests sequentially on a single cluster. +func RunPytorch(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, tests []pytorchTest) { + t.Helper() + for _, test := range tests { + t.Run(test.Name(), func(t *testing.T) { + doPytorchRun(ctx, t, k8sCtx, cluster, test) + }) + } +} + +// doPytorchRun runs a single PyTorch test. +func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("Failed to reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + image, err := k8sCtx.ResolveImage(ctx, pytorchImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + pod, err := params.toPod(benchmarkNS, image) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to configure pod for test-nodepool: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, pod.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + if err := cluster.WaitForPodCompleted(ctx, pod); err != nil { + t.Fatalf("Failed to wait for pod to complete: %v", err) + } + + rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("GetLogReader on cluster %q pod %v: %v", cluster.GetName(), pod.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", pod.GetName(), err) + } + + metrics, err := parseStandardOutput(string(out)) + if err != nil { + t.Fatalf("parseStandardOutput: %v", err) + } + + containerDuration, err := benchmetric.ParseTimedContainerOutput(string(out)) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + + metrics = append(metrics, benchmetric.BenchmarkDuration(containerDuration)) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, params.BenchName(), metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } +} + +func parseStandardOutput(output string) ([]benchmetric.MetricValue, error) { + gpuTimeMillis, err := parseGPUTime(output) + if err != nil { + return nil, fmt.Errorf("parseGPUTime: %v", err) + } + + gpuPeakMemoryGB, err := parseGPUPeakMemoryGB(output) + if err != nil { + return nil, fmt.Errorf("parseGPUPeakMemory: %v", err) + } + + cpuPeakMemoryGB, err := parseCPUPeakMemoryGB(output) + if err != nil { + return nil, fmt.Errorf("parseCPUPeakMemory: %v", err) + } + + return []benchmetric.MetricValue{ + benchmetric.SpecificDuration(time.Duration(gpuTimeMillis)*time.Millisecond, "gpu-runtime"), + benchmetric.SpecificBytes(gpuPeakMemoryGB*1024*1024*1024, "gpu-peak-memory"), + benchmetric.SpecificBytes(cpuPeakMemoryGB*1024*1024*1024, "cpu-peak-memory"), + }, nil +} + +var gpuTimeRegex = regexp.MustCompile(`GPU\sTime:\s*(\d+\.\d+)\smilliseconds`) + +func parseGPUTime(output string) (float64, error) { + match := gpuTimeRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find GPU Time: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +var gpuPeakMemoryRegex = regexp.MustCompile(`GPU\s0\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) + +func parseGPUPeakMemoryGB(output string) (float64, error) { + match := gpuPeakMemoryRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find GPU Peak Memory: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +var cpuPeakMemoryRegex = regexp.MustCompile(`CPU\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) + +func parseCPUPeakMemoryGB(output string) (float64, error) { + match := cpuPeakMemoryRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find CPU Peak Memory: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} diff --git a/test/kubernetes/benchmarks/pytorch_test.go b/test/kubernetes/benchmarks/pytorch_test.go index b904da867..ea43ab323 100644 --- a/test/kubernetes/benchmarks/pytorch_test.go +++ b/test/kubernetes/benchmarks/pytorch_test.go @@ -12,263 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package pytorch_test contains benchmarks using the pytorch "torchbench" repo. -package pytorch_test - -// These tests use pytorch's "torchbench" suite (https://github.com/pytorch/benchmark/tree/main). -// The Authors describe the benchmarks in this paper: https://arxiv.org/pdf/2304.14226.pdf -// The Authors list both the type of model and its profile (how GPU intensive). - -// Note: The image for this test is about 7-8 GB as of writing. After you get your clusters up and -// running, start the test and make sure that the pods show the event of downloading the image. Then -// get a cup of coffee, chat with your co-workers for 5 min, and it will be about done 5 min after -// that. You'll only need to do this once for each cluster (in parallel). +package pytorch import ( "context" - "fmt" - "io" - "regexp" - "strconv" - "strings" "testing" - "time" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - - v13 "k8s.io/api/core/v1" ) -// pytorchTestType is the method used, either training or evaluation, for the model. -type pytorchTestType string - -const ( - train = pytorchTestType("train") - eval = pytorchTestType("eval") - - pytorchImage = k8s.ImageRepoPrefix + "benchmarks/pytorch_x86_64:f6f280aeb1b07989" -) - -type pytorchMode string - -// pytorchMode is the pytorch mode used, either script mode (jit) or eager mode. -// See: https://towardsdatascience.com/pytorch-jit-and-torchscript-c2a77bac0fff -const ( - jit = pytorchMode("jit") - eager = pytorchMode("eager") -) - -type pytorchTest struct { - module string - test pytorchTestType - mode pytorchMode -} - -// Name returns the name of the test with the argument parameters included. It is formatted so -// that it can be used for the name of the pod. -func (p pytorchTest) Name() string { - // Kubernetes pod names cannot contain "_". - module := strings.ReplaceAll(strings.ToLower(p.module), "_", "-") - return fmt.Sprintf("%s-%s-%s", module, p.test, p.mode) -} - -var snakeCase = regexp.MustCompile("_.") - -// BenchName returns the name of the test with the argument parameters included. -// It is formatted so that it can be used for benchstat output. -func (p pytorchTest) BenchName() string { - // First letter of the module should be capitalized, as it will be - // concatenated with "Benchmark" and it's useful to mark it as a different - // word. - // Some modules use a lowercase first letter, e.g. "fastNLP_Bert". - moduleName := strings.ToUpper(p.module[:1]) + p.module[1:] - // We also replace "snake_case" with "snakeCase". Sorry snakes. - moduleName = snakeCase.ReplaceAllStringFunc(moduleName, func(s string) string { - return strings.ToUpper(strings.TrimPrefix(s, "_")) - }) - test := strings.ToUpper(string(p.test)[:1]) + string(p.test[1:]) - var mode string - switch p.mode { - case eager: - mode = "Eager" - case jit: - mode = "JIT" - default: - panic(fmt.Sprintf("Unknown mode: %v", p.mode)) - } - return fmt.Sprintf("%s/%s/%s", moduleName, test, mode) -} - -func (p pytorchTest) toPod(namespace *testcluster.Namespace, image string) (*v13.Pod, error) { - pod := namespace.NewPod(p.Name()) - pod.Spec = v13.PodSpec{ - RestartPolicy: v13.RestartPolicyNever, - Containers: []v13.Container{ - { - Name: p.Name(), - Image: pytorchImage, - Command: benchmetric.TimedCommand(p.command()...), - }, - }, - } - return pod, nil -} - -func (p pytorchTest) command() []string { - return []string{ - "python", - "run.py", - p.module, - "--device", "cuda", - "--test", string(p.test), - "--mode", string(p.mode), - } -} - -// TestFastNLPBert uses the fastNLP_Bert module, which is classified as a NLP Language Model. -// fastNLP_Bert taxes the GPU heavily with low data movement. See Figure 2 on -// page 5: https://arxiv.org/pdf/2304.14226.pdf -// -// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/fastNLP_Bert -// Bert Blog Post: https://towardsdatascience.com/bert-explained-state-of-the-art-language-model-for-nlp-f8b21a9b6270 -// Paper: https://arxiv.org/abs/1810.04805 func TestFastNLPBert(t *testing.T) { ctx := context.Background() - const module = "fastNLP_Bert" - tests := []pytorchTest{ - { - module: module, - test: train, - mode: eager, - }, - { - module: module, - test: eval, - mode: eager, - }, - } - runTests(ctx, t, tests) + runTests(ctx, t, FastNLPBert) } -// TestBigBird uses the hf_BigBird module, which is classified as a NLP Language Model. -// hf_BigBird taxes the GPU moderately with low data movement. See Figure 2 on -// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf -// -// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/hf_BigBird -// Paper: https://arxiv.org/abs/2007.14062 func TestBigBird(t *testing.T) { ctx := context.Background() - const module = "hf_BigBird" - tests := []pytorchTest{ - { - module: module, - test: train, - mode: eager, - }, - { - module: module, - test: eval, - mode: eager, - }, - } - runTests(ctx, t, tests) + runTests(ctx, t, BigBird) } -// TestSpeechTransformer uses the speech_transformer module classified as "Speech Recognition" -// model. speech_transformer has a lot of idle time for the GPU. See Figure 2 on -// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf -// -// https://github.com/pytorch/benchmark/pull/374 -// Paper: https://arxiv.org/abs/1706.03762 func TestSpeechTransformer(t *testing.T) { ctx := context.Background() - const module = "speech_transformer" - tests := []pytorchTest{ - { - module: module, - test: train, - mode: eager, - }, - { - module: module, - test: eval, - mode: eager, - }, - } - runTests(ctx, t, tests) + runTests(ctx, t, SpeechTransformer) } -// TestLearningToPaint uses the LearningToPaint module classified as "neural renderer in model-based -// Deep Reinforcement Learning (DRL)". -// Learning to paint has a lot of "data movement" and doesn't tax the GPU a lot. See Figure 2 on -// page 5: https://arxiv.org/pdf/2304.14226.pdf -// -// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/LearningToPaint func TestLearningToPaint(t *testing.T) { ctx := context.Background() - const module = "LearningToPaint" - tests := []pytorchTest{ - { - module: module, - test: train, - mode: jit, - }, - { - module: module, - test: eval, - mode: jit, - }, - } - runTests(ctx, t, tests) + runTests(ctx, t, LearningToPaint) } -// TestMobileNetV2 uses the mobilenet_v2 module classified as "Computer Vision: Image Classification". -// MobileNet has a lot of taxes the GPU. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf -// -// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/mobilenet_v2 -// Paper: https://paperswithcode.com/method/mobilenetv2 func TestMobileNetV2(t *testing.T) { ctx := context.Background() - const module = "mobilenet_v2" - tests := []pytorchTest{ - { - module: module, - test: train, - mode: jit, - }, - { - module: module, - test: eval, - mode: jit, - }, - } - runTests(ctx, t, tests) + runTests(ctx, t, MobileNetV2) } -// TestBackgroundMatting uses the Background_Matting module classified as "Computer Vision: Pattern Recognition". -// BackgroundMatting has a lot of GPU idle time. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf -// -// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/Background_Matting (see README) func TestBackgroundMatting(t *testing.T) { ctx := context.Background() - const module = "Background_Matting" - tests := []pytorchTest{ - { - module: module, - test: train, - mode: eager, - }, - { - module: module, - test: eval, - mode: eager, - }, - } - runTests(ctx, t, tests) + runTests(ctx, t, BackgroundMatting) } func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) { @@ -279,140 +60,11 @@ func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("PyTorch", func(t *testing.T) { t.Parallel() - for _, p := range tests { - t.Run(p.Name(), func(t *testing.T) { - doPytorchRun(ctx, t, k8sCtx, cluster, p) - }) - } + RunPytorch(ctx, t, k8sCtx, cluster, tests) }) }) } -func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("Failed to reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - image, err := k8sCtx.ResolveImage(ctx, pytorchImage) - if err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - pod, err := params.toPod(benchmarkNS, image) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - - pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) - if err != nil { - t.Fatalf("Failed to configure pod for test-nodepool: %v", err) - } - - pod, err = testcluster.MaybeSetContainerResources(pod, pod.Name, testcluster.ContainerResourcesRequest{GPU: true}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - - pod, err = cluster.CreatePod(ctx, pod) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, pod) - - if err := cluster.WaitForPodCompleted(ctx, pod); err != nil { - t.Fatalf("Failed to wait for pod to complete: %v", err) - } - - rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) - if err != nil { - t.Fatalf("GetLogReader on cluster %q pod %v: %v", cluster.GetName(), pod.GetName(), err) - } - - out, err := io.ReadAll(rdr) - if err != nil { - t.Fatalf("failed to read from pod: %q: %v", pod.GetName(), err) - } - - metrics, err := parseStandardOutput(string(out)) - if err != nil { - t.Fatalf("parseStandardOutput: %v", err) - } - - containerDuration, err := benchmetric.ParseTimedContainerOutput(string(out)) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - - metrics = append(metrics, benchmetric.BenchmarkDuration(containerDuration)) - - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - if err := recorder.Record(ctx, params.BenchName(), metrics...); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } -} - -func parseStandardOutput(output string) ([]benchmetric.MetricValue, error) { - gpuTimeMillis, err := parseGPUTime(output) - if err != nil { - return nil, fmt.Errorf("parseGPUTime: %v", err) - } - - gpuPeakMemoryGB, err := parseGPUPeakMemoryGB(output) - if err != nil { - return nil, fmt.Errorf("parseGPUPeakMemory: %v", err) - } - - cpuPeakMemoryGB, err := parseCPUPeakMemoryGB(output) - if err != nil { - return nil, fmt.Errorf("parseCPUPeakMemory: %v", err) - } - - return []benchmetric.MetricValue{ - benchmetric.SpecificDuration(time.Duration(gpuTimeMillis)*time.Millisecond, "gpu-runtime"), - benchmetric.SpecificBytes(gpuPeakMemoryGB*1024*1024*1024, "gpu-peak-memory"), - benchmetric.SpecificBytes(cpuPeakMemoryGB*1024*1024*1024, "cpu-peak-memory"), - }, nil -} - -var gpuTimeRegex = regexp.MustCompile(`GPU\sTime:\s*(\d+\.\d+)\smilliseconds`) - -func parseGPUTime(output string) (float64, error) { - match := gpuTimeRegex.FindStringSubmatch(output) - if len(match) < 2 { - return 0, fmt.Errorf("failed to find GPU Time: %s", output) - } - return strconv.ParseFloat(match[1], 64) -} - -var gpuPeakMemoryRegex = regexp.MustCompile(`GPU\s0\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) - -func parseGPUPeakMemoryGB(output string) (float64, error) { - match := gpuPeakMemoryRegex.FindStringSubmatch(output) - if len(match) < 2 { - return 0, fmt.Errorf("failed to find GPU Peak Memory: %s", output) - } - return strconv.ParseFloat(match[1], 64) -} - -var cpuPeakMemoryRegex = regexp.MustCompile(`CPU\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) - -func parseCPUPeakMemoryGB(output string) (float64, error) { - match := cpuPeakMemoryRegex.FindStringSubmatch(output) - if len(match) < 2 { - return 0, fmt.Errorf("failed to find CPU Peak Memory: %s", output) - } - return strconv.ParseFloat(match[1], 64) -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestFastNLPBert": TestFastNLPBert, diff --git a/test/kubernetes/benchmarks/redis.go b/test/kubernetes/benchmarks/redis.go new file mode 100644 index 000000000..03977b8a2 --- /dev/null +++ b/test/kubernetes/benchmarks/redis.go @@ -0,0 +1,454 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redis holds the redis test where the runtime under test runs a redis server and the +// native runtime runs a client making requests against it. +package redis + +import ( + "context" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" + "testing" + "time" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + redisPort = 6379 + defaultRequestsPerConnection = 50000 + + redisServerLabelKey = "app.kubernetes.io/name" + redisServerLabelValue = "redis-server" + redisVolumeName = "redis-data" + redisDataDirectory = "/redis-data" + redisImageAMD = k8s.ImageRepoPrefix + "benchmarks/redis_x86_64:latest" + redisImageARM = k8s.ImageRepoPrefix + "benchmarks/redis_aarch64:latest" +) + +var ( + numConnections = []int{1, 2, 4, 8, 16, 32} + latencyPercentiles = []int{50, 95, 99} + operations = []string{"SET", "GET", "MSET", "LPUSH", "LRANGE_500"} +) + +// BenchmarkRedis runs the Redis performance benchmark using redis-benchmark. +func BenchmarkRedis(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + persistentVol := benchmarkNS.GetPersistentVolume(redisVolumeName, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := redisImageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = redisImageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + for _, test := range []struct { + // Benchmark name + name string + // Suffix for the redis server, must be short to fit in pod name. + suffix string + // redis-server command-line. + serverCommand []string + // Volume to use for persistence, if any. + volume *v13.PersistentVolumeClaim + }{ + { + name: "Persistence", + suffix: "persist", + serverCommand: []string{ + "redis-server", + "--dir", redisDataDirectory, + // Default save settings per + // https://redis.io/docs/management/config-file/ + "--save", "3600 1 300 100 60 10000", + }, + volume: persistentVol, + }, + { + name: "NoPersistence", + suffix: "nopersist", + serverCommand: []string{ + "redis-server", + "--appendonly", "no", + "--save", "", + }, + volume: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a server on the runtime under test nodepool. + server := newRedisPodWithPort(benchmarkNS, fmt.Sprintf("redis-%s", test.suffix), image, test.serverCommand, redisPort, test.volume) + if server.ObjectMeta.Labels == nil { + server.ObjectMeta.Labels = make(map[string]string) + } + server.ObjectMeta.Labels[redisServerLabelKey] = redisServerLabelValue + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) + } + + server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) + } + + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) + } + + // create a service to service traffic to the pod. + service := newRedisService(benchmarkNS, server.GetName()) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeleteService(ctx, service) + if err := cluster.WaitForServiceReady(ctx, service); err != nil { + t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) + } + + ip := testcluster.GetIPFromService(service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + + // run the 'redis-cli' command to ping the server and make sure it is up. The "ping" request comes + // back with a "PONG" response. We repeat -r=5 times with a -i=1 second interval. If we + // get one PONG back then the server is considered up. + pingCmd := []string{"redis-cli", "-h", ip, "-r", "5", "-i", "1", "ping"} + ensureUp := func() error { + pinger := newRedisPod(benchmarkNS, fmt.Sprintf("rpinger-%s", test.suffix), image, pingCmd) + pinger, err = cluster.ConfigurePodForClientNodepool(pinger) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pinger.GetName(), err) + } + + pinger, err = cluster.CreatePod(ctx, pinger) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", pinger.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, pinger) + + waitCtx, waitCancel := context.WithTimeout(ctx, 30*time.Second) + var podWaitSuffix string + if err := cluster.WaitForPodCompleted(waitCtx, pinger); err != nil { + podWaitSuffix = fmt.Sprintf(" (pod wait error: %v)", err) + } + waitCancel() + + rdr, err := cluster.GetLogReader(ctx, pinger, v13.PodLogOptions{}) + if err != nil { + return fmt.Errorf("GetLogReader on cluster %q: %v%s", cluster.GetName(), err, podWaitSuffix) + } + out, err := io.ReadAll(rdr) + if err != nil { + return fmt.Errorf("failed to read from pod: %q: %v%s", pinger.GetName(), err, podWaitSuffix) + } + + if !strings.Contains(string(out), "PONG") { + return fmt.Errorf("mismatched output: wanted: PONG got: %q%s", string(out), podWaitSuffix) + } + + return nil + } + var isUpErr error + serverUpCtx, serverUpCancel := context.WithTimeout(ctx, 100*time.Second) + defer serverUpCancel() + for serverUpCtx.Err() == nil { + if isUpErr = ensureUp(); isUpErr == nil { + break + } + } + if isUpErr != nil { + t.Fatalf("%s at IP %s did not come up: %v", server.GetName(), ip, isUpErr) + } + + for _, connections := range numConnections { + t.Run(fmt.Sprintf("Connections_%d", connections), func(t *testing.T) { + for _, operation := range operations { + t.Run(operation, func(t *testing.T) { + // Create a client for this client run w/ the specified number of connections. + // Sadly the --csv mode only reports QPS, not latency. In order to report both, + // we need to parse the human-readable version of the output. + clientCmd := []string{ + "redis-benchmark", + "-t", operation, // RPC to benchmark + "-h", ip, // Redis server IP + "-n", fmt.Sprintf("%d", defaultRequestsPerConnection*connections), // Number of total requests to do + "-c", fmt.Sprintf("%d", connections), // Number of threads to spread them over. + "-r", "1000", // Key space size (larger = more memory faults) + "--precision", "4", // Floating-point precision for reporting latency (in ms) + } + client := newRedisPod(benchmarkNS, "client", image, clientCmd) + client, err = cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + client, err = cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, client) + + if err := cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + // get and parse the logs from the client to get the results + rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + + if err != nil { + t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + redisBenchmarkName := fmt.Sprintf("Redis/%s/%dClients/%s", test.name, connections, operation) + metrics, err := getMeasurements(string(out), operation) + if err != nil { + // Redis uses '\r' to update its status by overwriting the current line. + // If printed directly, this messes up the output. + // To make that clear, we replace '\r' with a literal + // backslash + 'r', and add a newline. + humanReadableOut := strings.ReplaceAll(string(out), "\r", "\\r\n") + t.Fatalf("failed to get metric for op %q: out:\n\n%s\n\nerr: %v", operation, humanReadableOut, err) + } + // We don't multiply `defaultRequestsPerConnection` by `connections` here + // because the number of "samples" we're testing is the number of times we + // can call an RPC from *that many connections* (which is part of the + // benchmark name). + // Adding 5x the number of connections does not make the sample size of this + // benchmark go 5x higher. + if err := recorder.RecordIters(ctx, redisBenchmarkName, defaultRequestsPerConnection, metrics...); err != nil { + t.Fatalf("Failed to record benchmark data for op %q: %v", operation, err) + } + }) + if t.Failed() { + break + } + } + }) + if t.Failed() { + break + } + } + }) + if t.Failed() { + break + } + } +} + +// newRedisService gets a service to serve traffic to the redis server. +func newRedisService(namespace *testcluster.Namespace, containerName string) *v13.Service { + name := fmt.Sprintf("redis-service-%d", time.Now().UnixNano()) + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{redisServerLabelKey: redisServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: redisPort, + TargetPort: intstr.FromString(containerName), + }, + }, + }) +} + +var ( + latencyPercentileRegex = regexp.MustCompile("^([-,.\\d]+)% <=? ([-,.\\d]+) milliseconds(?: \\(cumulative count .*\\))?$") + latencyStartHeader = "Latency by percentile distribution:" + queriesPerSecondRegex = regexp.MustCompile("^throughput summary: ([-,.\\d]+) requests per second$") +) + +func stringToFloat64(s string) float64 { + f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) + if err != nil { + panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) + } + return f +} + +// getMeasurements parses the output of redis-benchmark to get the stats. +func getMeasurements(out, operation string) ([]benchmetric.MetricValue, error) { + var currentOperation string + var returned []benchmetric.MetricValue + inLatencyBlock := false + foundPercentiles := make(map[int]bool, len(latencyPercentiles)) + foundQPS := false + lastPercentile := -1.0 + lastPercentileLatencyMs := math.NaN() + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + // The human-readable output contains a bunch of data like + // "OPERATION: number\r" which are used to update the result + // interactively. Strip them out here. + if strings.Contains(line, "\r") { + line = line[strings.LastIndex(line, "\r")+1:] + } + if strings.HasPrefix(line, "====== ") { + currentOperation = strings.SplitN(strings.Trim(line, "= "), " ", 2)[0] + } + if currentOperation != operation { + continue + } + if line == latencyStartHeader { + inLatencyBlock = true + continue + } + if inLatencyBlock { + latencyMatch := latencyPercentileRegex.FindStringSubmatch(line) + if latencyMatch != nil { + percentile := stringToFloat64(latencyMatch[1]) + if percentile < lastPercentile { + continue + } + latencyMs := stringToFloat64(latencyMatch[2]) + if percentile == 0 { + lastPercentile = 0 + lastPercentileLatencyMs = latencyMs + continue + } + // Look for all percentiles in `wantPercentiles` that are in the range [lastPercentile, + // percentile]. + var recordPercentiles []int + for _, wantPercentile := range latencyPercentiles { + if float64(wantPercentile) < lastPercentile { + continue + } + if float64(wantPercentile) > percentile { + continue + } + if foundPercentiles[wantPercentile] { + continue + } + recordPercentiles = append(recordPercentiles, wantPercentile) + } + for _, recordPercentile := range recordPercentiles { + // Linear interpolation of the latency value from within the latency range in the two + // percentile values that we got. + // For example, given p50=1.0ms and p70=2.0ms, we infer that p60=1.5ms. + // This isn't bulletproof but it is better than rounding to either end of the bucket. + rangeFraction := (float64(recordPercentile) - lastPercentile) / (percentile - lastPercentile) + pctileLatency := rangeFraction*(latencyMs-lastPercentileLatencyMs) + lastPercentileLatencyMs + returned = append(returned, benchmetric.SpecificDuration(time.Duration(pctileLatency*float64(time.Millisecond)), fmt.Sprintf("p%d", recordPercentile))) + foundPercentiles[recordPercentile] = true + } + // Update values for next round. + lastPercentile = percentile + lastPercentileLatencyMs = latencyMs + } else { + inLatencyBlock = false + } + continue + } + qpsMatch := queriesPerSecondRegex.FindStringSubmatch(line) + if qpsMatch != nil { + if foundQPS { + return nil, fmt.Errorf("found QPS value multiple times: %q", line) + } + foundQPS = true + returned = append(returned, benchmetric.RequestsPerSecond(stringToFloat64(qpsMatch[1]))) + } + } + if !foundQPS || len(foundPercentiles) != len(latencyPercentiles) { + return nil, fmt.Errorf("did not find the data we wanted: foundQPS=%v foundPercentiles=%v", foundQPS, foundPercentiles) + } + return returned, nil +} + +// newRedisPodWithPort returns a redis pod template. +func newRedisPodWithPort(namespace *testcluster.Namespace, name, image string, cmd []string, port int32, pvc *v13.PersistentVolumeClaim) *v13.Pod { + container := newRedisContainer(name, image, cmd) + container.Ports = append(container.Ports, v13.ContainerPort{Name: name, ContainerPort: port}) + if pvc != nil { + container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ + Name: redisVolumeName, + MountPath: redisDataDirectory, + }) + } + + pod := namespace.NewPod(name) + pod.Spec.Containers = []v13.Container{container} + if pvc != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ + Name: redisVolumeName, + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.GetName(), + }, + }, + }) + } + return pod +} + +// newRedisPod returns a redis pod template. +func newRedisPod(namespace *testcluster.Namespace, name, image string, cmd []string) *v13.Pod { + pod := namespace.NewPod(name) + pod.Spec.Containers = []v13.Container{newRedisContainer(name, image, cmd)} + return pod +} + +// newRedisContainer returns a new redis container. +func newRedisContainer(name, image string, cmd []string) v13.Container { + return v13.Container{ + Name: name, + Image: image, + Command: cmd, + } +} diff --git a/test/kubernetes/benchmarks/redis_test.go b/test/kubernetes/benchmarks/redis_test.go index 5bf436386..d5062a26b 100644 --- a/test/kubernetes/benchmarks/redis_test.go +++ b/test/kubernetes/benchmarks/redis_test.go @@ -12,46 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package redis_test holds the redis test where the runtime under test runs a redis server and the -// native runtime runs a client making requests against it. -package redis_test +package redis import ( "context" - "fmt" - "io" - "math" - "regexp" - "strconv" - "strings" "testing" - "time" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const ( - redisPort = 6379 - defaultRequestsPerConnection = 50000 - - redisServerLabelKey = "app.kubernetes.io/name" - redisServerLabelValue = "redis-server" - redisVolumeName = "redis-data" - redisDataDirectory = "/redis-data" - redisImageAMD = k8s.ImageRepoPrefix + "benchmarks/redis_x86_64:latest" - redisImageARM = k8s.ImageRepoPrefix + "benchmarks/redis_aarch64:latest" -) - -var ( - numConnections = []int{1, 2, 4, 8, 16, 32} - latencyPercentiles = []int{50, 95, 99} - operations = []string{"SET", "GET", "MSET", "LPUSH", "LRANGE_500"} ) // TestRedis benchmarks redis servers on k8s clusters. @@ -64,410 +32,11 @@ func TestRedis(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("Redis", func(t *testing.T) { t.Parallel() - doRedisTest(ctx, t, k8sCtx, cluster) + BenchmarkRedis(ctx, t, k8sCtx, cluster) }) }) } -// doRedisTest runs redis test. -func doRedisTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - persistentVol := benchmarkNS.GetPersistentVolume(redisVolumeName, "30Gi") - persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - image := redisImageAMD - if cluster.RuntimeTestNodepoolIsARM() { - image = redisImageARM - } - if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - for _, test := range []struct { - // Benchmark name - name string - // Suffix for the redis server, must be short to fit in pod name. - suffix string - // redis-server command-line. - serverCommand []string - // Volume to use for persistence, if any. - volume *v13.PersistentVolumeClaim - }{ - { - name: "Persistence", - suffix: "persist", - serverCommand: []string{ - "redis-server", - "--dir", redisDataDirectory, - // Default save settings per - // https://redis.io/docs/management/config-file/ - "--save", "3600 1 300 100 60 10000", - }, - volume: persistentVol, - }, - { - name: "NoPersistence", - suffix: "nopersist", - serverCommand: []string{ - "redis-server", - "--appendonly", "no", - "--save", "", - }, - volume: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - - // Create a server on the runtime under test nodepool. - server := newRedisPodWithPort(benchmarkNS, fmt.Sprintf("redis-%s", test.suffix), image, test.serverCommand, redisPort, test.volume) - if server.ObjectMeta.Labels == nil { - server.ObjectMeta.Labels = make(map[string]string) - } - server.ObjectMeta.Labels[redisServerLabelKey] = redisServerLabelValue - server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) - if err != nil { - t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) - } - - server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) - } - - server, err = cluster.CreatePod(ctx, server) - if err != nil { - t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, server) - - if err := cluster.WaitForPodRunning(ctx, server); err != nil { - t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) - } - - // create a service to service traffic to the pod. - service := newRedisService(benchmarkNS, server.GetName()) - service, err = cluster.CreateService(ctx, service) - if err != nil { - t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) - } - defer cluster.DeleteService(ctx, service) - if err := cluster.WaitForServiceReady(ctx, service); err != nil { - t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) - } - - ip := testcluster.GetIPFromService(service) - if ip == "" { - t.Fatalf("did not get valid ip: %s", ip) - } - - // run the 'redis-cli' command to ping the server and make sure it is up. The "ping" request comes - // back with a "PONG" response. We repeat -r=5 times with a -i=1 second interval. If we - // get one PONG back then the server is considered up. - pingCmd := []string{"redis-cli", "-h", ip, "-r", "5", "-i", "1", "ping"} - ensureUp := func() error { - pinger := newRedisPod(benchmarkNS, fmt.Sprintf("rpinger-%s", test.suffix), image, pingCmd) - pinger, err = cluster.ConfigurePodForClientNodepool(pinger) - if err != nil { - return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pinger.GetName(), err) - } - - pinger, err = cluster.CreatePod(ctx, pinger) - if err != nil { - return fmt.Errorf("CreatePod %q on cluster %q: %v", pinger.GetName(), cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, pinger) - - waitCtx, waitCancel := context.WithTimeout(ctx, 30*time.Second) - var podWaitSuffix string - if err := cluster.WaitForPodCompleted(waitCtx, pinger); err != nil { - podWaitSuffix = fmt.Sprintf(" (pod wait error: %v)", err) - } - waitCancel() - - rdr, err := cluster.GetLogReader(ctx, pinger, v13.PodLogOptions{}) - if err != nil { - return fmt.Errorf("GetLogReader on cluster %q: %v%s", cluster.GetName(), err, podWaitSuffix) - } - out, err := io.ReadAll(rdr) - if err != nil { - return fmt.Errorf("failed to read from pod: %q: %v%s", pinger.GetName(), err, podWaitSuffix) - } - - if !strings.Contains(string(out), "PONG") { - return fmt.Errorf("mismatched output: wanted: PONG got: %q%s", string(out), podWaitSuffix) - } - - return nil - } - var isUpErr error - serverUpCtx, serverUpCancel := context.WithTimeout(ctx, 100*time.Second) - defer serverUpCancel() - for serverUpCtx.Err() == nil { - if isUpErr = ensureUp(); isUpErr == nil { - break - } - } - if isUpErr != nil { - t.Fatalf("%s at IP %s did not come up: %v", server.GetName(), ip, isUpErr) - } - - for _, connections := range numConnections { - t.Run(fmt.Sprintf("Connections_%d", connections), func(t *testing.T) { - for _, operation := range operations { - t.Run(operation, func(t *testing.T) { - // Create a client for this client run w/ the specified number of connections. - // Sadly the --csv mode only reports QPS, not latency. In order to report both, - // we need to parse the human-readable version of the output. - clientCmd := []string{ - "redis-benchmark", - "-t", operation, // RPC to benchmark - "-h", ip, // Redis server IP - "-n", fmt.Sprintf("%d", defaultRequestsPerConnection*connections), // Number of total requests to do - "-c", fmt.Sprintf("%d", connections), // Number of threads to spread them over. - "-r", "1000", // Key space size (larger = more memory faults) - "--precision", "4", // Floating-point precision for reporting latency (in ms) - } - client := newRedisPod(benchmarkNS, "client", image, clientCmd) - client, err = cluster.ConfigurePodForClientNodepool(client) - if err != nil { - t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) - } - - client, err = cluster.CreatePod(ctx, client) - if err != nil { - t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) - } - defer cluster.DeletePod(ctx, client) - - if err := cluster.WaitForPodCompleted(ctx, client); err != nil { - t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) - } - - // get and parse the logs from the client to get the results - rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) - - if err != nil { - t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) - } - - out, err := io.ReadAll(rdr) - if err != nil { - t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) - } - - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - redisBenchmarkName := fmt.Sprintf("Redis/%s/%dClients/%s", test.name, connections, operation) - metrics, err := getMeasurements(string(out), operation) - if err != nil { - // Redis uses '\r' to update its status by overwriting the current line. - // If printed directly, this messes up the output. - // To make that clear, we replace '\r' with a literal - // backslash + 'r', and add a newline. - humanReadableOut := strings.ReplaceAll(string(out), "\r", "\\r\n") - t.Fatalf("failed to get metric for op %q: out:\n\n%s\n\nerr: %v", operation, humanReadableOut, err) - } - // We don't multiply `defaultRequestsPerConnection` by `connections` here - // because the number of "samples" we're testing is the number of times we - // can call an RPC from *that many connections* (which is part of the - // benchmark name). - // Adding 5x the number of connections does not make the sample size of this - // benchmark go 5x higher. - if err := recorder.RecordIters(ctx, redisBenchmarkName, defaultRequestsPerConnection, metrics...); err != nil { - t.Fatalf("Failed to record benchmark data for op %q: %v", operation, err) - } - }) - if t.Failed() { - break - } - } - }) - if t.Failed() { - break - } - } - }) - if t.Failed() { - break - } - } -} - -// newRedisService gets a service to serve traffic to the redis server. -func newRedisService(namespace *testcluster.Namespace, containerName string) *v13.Service { - name := fmt.Sprintf("redis-service-%d", time.Now().UnixNano()) - return namespace.GetService(name, v13.ServiceSpec{ - Selector: map[string]string{redisServerLabelKey: redisServerLabelValue}, - Ports: []v13.ServicePort{ - { - Name: name, - Protocol: v13.ProtocolTCP, - Port: redisPort, - TargetPort: intstr.FromString(containerName), - }, - }, - }) -} - -var ( - latencyPercentileRegex = regexp.MustCompile("^([-,.\\d]+)% <=? ([-,.\\d]+) milliseconds(?: \\(cumulative count .*\\))?$") - latencyStartHeader = "Latency by percentile distribution:" - queriesPerSecondRegex = regexp.MustCompile("^throughput summary: ([-,.\\d]+) requests per second$") -) - -func stringToFloat64(s string) float64 { - f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) - if err != nil { - panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) - } - return f -} - -// getMeasurements parses the output of redis-benchmark to get the stats. -func getMeasurements(out, operation string) ([]benchmetric.MetricValue, error) { - var currentOperation string - var returned []benchmetric.MetricValue - inLatencyBlock := false - foundPercentiles := make(map[int]bool, len(latencyPercentiles)) - foundQPS := false - lastPercentile := -1.0 - lastPercentileLatencyMs := math.NaN() - for _, line := range strings.Split(out, "\n") { - line = strings.TrimSpace(line) - // The human-readable output contains a bunch of data like - // "OPERATION: number\r" which are used to update the result - // interactively. Strip them out here. - if strings.Contains(line, "\r") { - line = line[strings.LastIndex(line, "\r")+1:] - } - if strings.HasPrefix(line, "====== ") { - currentOperation = strings.SplitN(strings.Trim(line, "= "), " ", 2)[0] - } - if currentOperation != operation { - continue - } - if line == latencyStartHeader { - inLatencyBlock = true - continue - } - if inLatencyBlock { - latencyMatch := latencyPercentileRegex.FindStringSubmatch(line) - if latencyMatch != nil { - percentile := stringToFloat64(latencyMatch[1]) - if percentile < lastPercentile { - continue - } - latencyMs := stringToFloat64(latencyMatch[2]) - if percentile == 0 { - lastPercentile = 0 - lastPercentileLatencyMs = latencyMs - continue - } - // Look for all percentiles in `wantPercentiles` that are in the range [lastPercentile, - // percentile]. - var recordPercentiles []int - for _, wantPercentile := range latencyPercentiles { - if float64(wantPercentile) < lastPercentile { - continue - } - if float64(wantPercentile) > percentile { - continue - } - if foundPercentiles[wantPercentile] { - continue - } - recordPercentiles = append(recordPercentiles, wantPercentile) - } - for _, recordPercentile := range recordPercentiles { - // Linear interpolation of the latency value from within the latency range in the two - // percentile values that we got. - // For example, given p50=1.0ms and p70=2.0ms, we infer that p60=1.5ms. - // This isn't bulletproof but it is better than rounding to either end of the bucket. - rangeFraction := (float64(recordPercentile) - lastPercentile) / (percentile - lastPercentile) - pctileLatency := rangeFraction*(latencyMs-lastPercentileLatencyMs) + lastPercentileLatencyMs - returned = append(returned, benchmetric.SpecificDuration(time.Duration(pctileLatency*float64(time.Millisecond)), fmt.Sprintf("p%d", recordPercentile))) - foundPercentiles[recordPercentile] = true - } - // Update values for next round. - lastPercentile = percentile - lastPercentileLatencyMs = latencyMs - } else { - inLatencyBlock = false - } - continue - } - qpsMatch := queriesPerSecondRegex.FindStringSubmatch(line) - if qpsMatch != nil { - if foundQPS { - return nil, fmt.Errorf("found QPS value multiple times: %q", line) - } - foundQPS = true - returned = append(returned, benchmetric.RequestsPerSecond(stringToFloat64(qpsMatch[1]))) - } - } - if !foundQPS || len(foundPercentiles) != len(latencyPercentiles) { - return nil, fmt.Errorf("did not find the data we wanted: foundQPS=%v foundPercentiles=%v", foundQPS, foundPercentiles) - } - return returned, nil -} - -// newRedisPodWithPort returns a redis pod template. -func newRedisPodWithPort(namespace *testcluster.Namespace, name, image string, cmd []string, port int32, pvc *v13.PersistentVolumeClaim) *v13.Pod { - container := newRedisContainer(name, image, cmd) - container.Ports = append(container.Ports, v13.ContainerPort{Name: name, ContainerPort: port}) - if pvc != nil { - container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ - Name: redisVolumeName, - MountPath: redisDataDirectory, - }) - } - - pod := namespace.NewPod(name) - pod.Spec.Containers = []v13.Container{container} - if pvc != nil { - pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ - Name: redisVolumeName, - VolumeSource: v13.VolumeSource{ - PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.GetName(), - }, - }, - }) - } - return pod -} - -// newRedisPod returns a redis pod template. -func newRedisPod(namespace *testcluster.Namespace, name, image string, cmd []string) *v13.Pod { - pod := namespace.NewPod(name) - pod.Spec.Containers = []v13.Container{newRedisContainer(name, image, cmd)} - return pod -} - -// newRedisContainer returns a new redis container. -func newRedisContainer(name, image string, cmd []string) v13.Container { - return v13.Container{ - Name: name, - Image: image, - Command: cmd, - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestRedis": TestRedis, diff --git a/test/kubernetes/benchmarks/ruby_dev_test.go b/test/kubernetes/benchmarks/rubydev.go similarity index 88% rename from test/kubernetes/benchmarks/ruby_dev_test.go rename to test/kubernetes/benchmarks/rubydev.go index 17ca95d20..46261bdb6 100644 --- a/test/kubernetes/benchmarks/ruby_dev_test.go +++ b/test/kubernetes/benchmarks/rubydev.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ruby_dev_test holds a benchmark to time a build job of a ruby application. -package ruby_dev_test +// Package rubydev holds a benchmark to time a build job of a ruby application. +package rubydev import ( "bytes" @@ -40,22 +40,9 @@ const ( imageARM = k8s.ImageRepoPrefix + "benchmarks/rubydev_aarch64:latest" ) -// TestRubyDev benchmarks a build job on k8s clusters. -func TestRubyDev(t *testing.T) { - ctx := context.Background() - k8sCtx, err := k8sctx.Context(ctx) - if err != nil { - t.Fatalf("Failed to get kubernetes context: %v", err) - } - k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { - t.Run("RubyDev", func(t *testing.T) { - t.Parallel() - doRubyDevTest(ctx, t, k8sCtx, cluster) - }) - }) -} - -func doRubyDevTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { +// RunRubyDev runs a benchmark measuring the time to build and test a +// popular Ruby library. +func RunRubyDev(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) if err := benchmarkNS.Reset(ctx); err != nil { t.Fatalf("cannot reset namespace: %v", err) @@ -218,9 +205,3 @@ func newRubyDevPod(namespace *testcluster.Namespace, name, image string, volume }, } } - -func TestMain(m *testing.M) { - k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ - "TestRubyDev": TestRubyDev, - }) -} diff --git a/test/kubernetes/benchmarks/rubydev_test.go b/test/kubernetes/benchmarks/rubydev_test.go new file mode 100644 index 000000000..60b6bfd3e --- /dev/null +++ b/test/kubernetes/benchmarks/rubydev_test.go @@ -0,0 +1,44 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rubydev + +import ( + "context" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" +) + +// TestRubyDev benchmarks a build job on k8s clusters. +func TestRubyDev(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("RubyDev", func(t *testing.T) { + t.Parallel() + RunRubyDev(ctx, t, k8sCtx, cluster) + }) + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestRubyDev": TestRubyDev, + }) +} diff --git a/test/kubernetes/benchmarks/stablediffusion.go b/test/kubernetes/benchmarks/stablediffusion.go new file mode 100644 index 000000000..b377ab982 --- /dev/null +++ b/test/kubernetes/benchmarks/stablediffusion.go @@ -0,0 +1,220 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stablediffusion + +import ( + "context" + "fmt" + "hash/fnv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/gpu/stablediffusion" + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // Container image for Stable Diffusion XL. + stableDiffusionImage = k8s.ImageRepoPrefix + "gpu/stable-diffusion-xl" +) + +// kubernetesPodRunner implements `stablediffusion.ContainerRunner`. +type kubernetesPodRunner struct { + cluster *testcluster.TestCluster + namespace *testcluster.Namespace +} + +// Run implements `stablediffusion.ContainerRunner.Run`. +func (r *kubernetesPodRunner) Run(ctx context.Context, image string, argv []string) ([]byte, []byte, error) { + // Build pod spec. + const stableDiffusionXLPodName = "stable-diffusion-xl" + stableDiffusionXLPod := &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: stableDiffusionXLPodName, + Namespace: r.namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: stableDiffusionXLPodName, + Image: image, + Args: argv, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } + stableDiffusionXLPod, err := r.cluster.ConfigurePodForRuntimeTestNodepool(stableDiffusionXLPod) + if err != nil { + return nil, nil, fmt.Errorf("failed to configure pod: %v", err) + } + stableDiffusionXLPod, err = testcluster.MaybeSetContainerResources(stableDiffusionXLPod, stableDiffusionXLPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + return nil, nil, fmt.Errorf("failed to set container resources: %v", err) + } + + // Delete pod that may possibly exist from a previous iteration. + // Ignore errors since it most likely doesn't exist. + r.cluster.DeletePod(ctx, stableDiffusionXLPod) + + // Start new client pod and wait for it. + stableDiffusionXLPod, err = r.cluster.CreatePod(ctx, stableDiffusionXLPod) + if err != nil { + return nil, nil, fmt.Errorf("failed to create stable diffusion XL pod: %v", err) + } + defer r.cluster.DeletePod(ctx, stableDiffusionXLPod) + if err := r.cluster.WaitForPodCompleted(ctx, stableDiffusionXLPod); err != nil { + logs, logsErr := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) + logs = strings.TrimSpace(logs) + if logsErr != nil { + return nil, nil, fmt.Errorf("failed to run Stable Diffusion XL (%w) and to read logs from the pod: %v", err, logsErr) + } + if logs == "" { + return nil, nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs are empty)", err) + } + return nil, nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs: %v)", err, logs) + } + + // All good, get logs. + logs, err := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) + if err != nil { + return nil, nil, fmt.Errorf("failed to read logs from pod %q: %v", stableDiffusionXLPod.GetName(), err) + } + return []byte(logs), nil, nil +} + +// RunStableDiffusionXL runs Stable Diffusion XL benchmarks for a single cluster. +func RunStableDiffusionXL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + imageName, err := k8sCtx.ResolveImage(ctx, stableDiffusionImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + xl := stablediffusion.NewXL(imageName, &kubernetesPodRunner{ + cluster: cluster, + namespace: benchmarkNS, + }) + + // The refiner model uses a lot of VRAM, and not all GPUs have enough of + // that to make it work. + // So we try each prompt without the refiner first. If it fails, then we + // don't try the same prompt with the refiner, as there is no way it will + // work. Similarly, if the benchmark does work without the refiner but + // does not work with the refiner, then future prompts will all have their + // refiner model attempt skipped. + refinerFailed := false + + for _, test := range []struct { + name string + query string + useRefiner bool + noiseFraction float64 + steps int + }{ + { + name: "BoringCorporateLogo", + query: `A boring flat corporate logo that says "gVisor"`, + useRefiner: true, + noiseFraction: 0.9, + steps: 32, + }, + { + name: "Androids", + query: "Photorealistic image of two androids playing chess aboard a spaceship", + useRefiner: true, + noiseFraction: 0.85, + steps: 64, + }, + } { + t.Run(test.name, func(t *testing.T) { + failedWithoutRefiner := false + for _, useRefiner := range []bool{false, true} { + t.Run(fmt.Sprintf("refiner=%t", useRefiner), func(t *testing.T) { + if useRefiner { + if failedWithoutRefiner { + t.Skipf("benchmark failed without refiner; skipping benchmark with refiner") + } + if refinerFailed { + t.Skipf("refiner failed in previous benchmark; skipping benchmark with refiner") + } + } + testCtx, testCancel := context.WithTimeout(ctx, 15*time.Minute) + defer testCancel() + prompt := &stablediffusion.XLPrompt{ + Query: test.query, + AllowCPUOffload: false, + NoiseFraction: test.noiseFraction, + Steps: test.steps, + Warm: true, + UseRefiner: useRefiner, + } + image, err := xl.Generate(testCtx, prompt) + if err != nil { + if useRefiner { + refinerFailed = true + t.Skipf("Failed to generate image with Refiner; will skip future attempts to run any prompt with the refiner.") + } + failedWithoutRefiner = true + t.Fatalf("Failed to generate images: %v", err) + } + ascii, err := image.ASCII() + if err != nil { + t.Fatalf("Failed to get ASCII: %v", err) + } + t.Logf("Generated image:\n\n%s\n", ascii) + hash := fnv.New32() + hash.Write([]byte(ascii)) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + metrics := []benchmetric.MetricValue{ + benchmetric.BenchmarkDuration(image.TotalDuration()), + benchmetric.SpecificDuration(image.ColdBaseDuration(), "base-cold"), + benchmetric.SpecificDuration(image.WarmBaseDuration(), "base-warm"), + } + if coldRefinerDuration := image.ColdRefinerDuration(); coldRefinerDuration >= 0 { + metrics = append(metrics, benchmetric.SpecificDuration(coldRefinerDuration, "refiner-cold")) + } + if warmRefinerDuration := image.WarmRefinerDuration(); warmRefinerDuration >= 0 { + metrics = append(metrics, benchmetric.SpecificDuration(warmRefinerDuration, "refiner-warm")) + } + // The image-hash metric should never change; it is still useful to + // report as a metric in order to detect instability across benchmark + // runs. + metrics = append(metrics, benchmetric.Checksum(hash, "image")) + if err := recorder.Record(ctx, fmt.Sprintf("StableDiffusionXL/%s/refiner=%t/steps=%d", test.name, useRefiner, test.steps), metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} diff --git a/test/kubernetes/benchmarks/stablediffusion_test.go b/test/kubernetes/benchmarks/stablediffusion_test.go index a61eb7057..0c082b9da 100644 --- a/test/kubernetes/benchmarks/stablediffusion_test.go +++ b/test/kubernetes/benchmarks/stablediffusion_test.go @@ -12,28 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package stablediffusion_test +package stablediffusion import ( "context" - "fmt" - "hash/fnv" - "strings" "testing" - "time" - "gvisor.dev/gvisor/test/gpu/stablediffusion" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - // Container image for Stable Diffusion XL. - stableDiffusionImage = k8s.ImageRepoPrefix + "gpu/stable-diffusion-xl" ) func TestStableDiffusionXL(t *testing.T) { @@ -45,194 +31,11 @@ func TestStableDiffusionXL(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("stable_diffusion_xl", func(t *testing.T) { t.Parallel() - doStableDiffusionXLTest(ctx, t, k8sCtx, cluster) + RunStableDiffusionXL(ctx, t, k8sCtx, cluster) }) }) } -// kubernetesPodRunner implements `stablediffusion.ContainerRunner`. -type kubernetesPodRunner struct { - cluster *testcluster.TestCluster - namespace *testcluster.Namespace -} - -// Run implements `stablediffusion.ContainerRunner.Run`. -func (r *kubernetesPodRunner) Run(ctx context.Context, image string, argv []string) ([]byte, []byte, error) { - // Build pod spec. - const stableDiffusionXLPodName = "stable-diffusion-xl" - stableDiffusionXLPod := &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: stableDiffusionXLPodName, - Namespace: r.namespace.Namespace, - }, - Spec: v13.PodSpec{ - Containers: []v13.Container{ - { - Name: stableDiffusionXLPodName, - Image: image, - Args: argv, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } - stableDiffusionXLPod, err := r.cluster.ConfigurePodForRuntimeTestNodepool(stableDiffusionXLPod) - if err != nil { - return nil, nil, fmt.Errorf("failed to configure pod: %v", err) - } - stableDiffusionXLPod, err = testcluster.MaybeSetContainerResources(stableDiffusionXLPod, stableDiffusionXLPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) - if err != nil { - return nil, nil, fmt.Errorf("failed to set container resources: %v", err) - } - - // Delete pod that may possibly exist from a previous iteration. - // Ignore errors since it most likely doesn't exist. - r.cluster.DeletePod(ctx, stableDiffusionXLPod) - - // Start new client pod and wait for it. - stableDiffusionXLPod, err = r.cluster.CreatePod(ctx, stableDiffusionXLPod) - if err != nil { - return nil, nil, fmt.Errorf("failed to create stable diffusion XL pod: %v", err) - } - defer r.cluster.DeletePod(ctx, stableDiffusionXLPod) - if err := r.cluster.WaitForPodCompleted(ctx, stableDiffusionXLPod); err != nil { - logs, logsErr := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) - logs = strings.TrimSpace(logs) - if logsErr != nil { - return nil, nil, fmt.Errorf("failed to run Stable Diffusion XL (%w) and to read logs from the pod: %v", err, logsErr) - } - if logs == "" { - return nil, nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs are empty)", err) - } - return nil, nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs: %v)", err, logs) - } - - // All good, get logs. - logs, err := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) - if err != nil { - return nil, nil, fmt.Errorf("failed to read logs from pod %q: %v", stableDiffusionXLPod.GetName(), err) - } - return []byte(logs), nil, nil -} - -// doStableDiffusionXLTest runs Stable Diffusion XL benchmarks for a single cluster. -func doStableDiffusionXLTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - imageName, err := k8sCtx.ResolveImage(ctx, stableDiffusionImage) - if err != nil { - t.Fatalf("failed to resolve image: %v", err) - } - xl := stablediffusion.NewXL(imageName, &kubernetesPodRunner{ - cluster: cluster, - namespace: benchmarkNS, - }) - - // The refiner model uses a lot of VRAM, and not all GPUs have enough of - // that to make it work. - // So we try each prompt without the refiner first. If it fails, then we - // don't try the same prompt with the refiner, as there is no way it will - // work. Similarly, if the benchmark does work without the refiner but - // does not work with the refiner, then future prompts will all have their - // refiner model attempt skipped. - refinerFailed := false - - for _, test := range []struct { - name string - query string - useRefiner bool - noiseFraction float64 - steps int - }{ - { - name: "BoringCorporateLogo", - query: `A boring flat corporate logo that says "gVisor"`, - useRefiner: true, - noiseFraction: 0.9, - steps: 32, - }, - { - name: "Androids", - query: "Photorealistic image of two androids playing chess aboard a spaceship", - useRefiner: true, - noiseFraction: 0.85, - steps: 64, - }, - } { - t.Run(test.name, func(t *testing.T) { - failedWithoutRefiner := false - for _, useRefiner := range []bool{false, true} { - t.Run(fmt.Sprintf("refiner=%t", useRefiner), func(t *testing.T) { - if useRefiner { - if failedWithoutRefiner { - t.Skipf("benchmark failed without refiner; skipping benchmark with refiner") - } - if refinerFailed { - t.Skipf("refiner failed in previous benchmark; skipping benchmark with refiner") - } - } - testCtx, testCancel := context.WithTimeout(ctx, 15*time.Minute) - defer testCancel() - prompt := &stablediffusion.XLPrompt{ - Query: test.query, - AllowCPUOffload: false, - NoiseFraction: test.noiseFraction, - Steps: test.steps, - Warm: true, - UseRefiner: useRefiner, - } - image, err := xl.Generate(testCtx, prompt) - if err != nil { - if useRefiner { - refinerFailed = true - t.Skipf("Failed to generate image with Refiner; will skip future attempts to run any prompt with the refiner.") - } - failedWithoutRefiner = true - t.Fatalf("Failed to generate images: %v", err) - } - ascii, err := image.ASCII() - if err != nil { - t.Fatalf("Failed to get ASCII: %v", err) - } - t.Logf("Generated image:\n\n%s\n", ascii) - hash := fnv.New32() - hash.Write([]byte(ascii)) - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - metrics := []benchmetric.MetricValue{ - benchmetric.BenchmarkDuration(image.TotalDuration()), - benchmetric.SpecificDuration(image.ColdBaseDuration(), "base-cold"), - benchmetric.SpecificDuration(image.WarmBaseDuration(), "base-warm"), - } - if coldRefinerDuration := image.ColdRefinerDuration(); coldRefinerDuration >= 0 { - metrics = append(metrics, benchmetric.SpecificDuration(coldRefinerDuration, "refiner-cold")) - } - if warmRefinerDuration := image.WarmRefinerDuration(); warmRefinerDuration >= 0 { - metrics = append(metrics, benchmetric.SpecificDuration(warmRefinerDuration, "refiner-warm")) - } - // The image-hash metric should never change; it is still useful to - // report as a metric in order to detect instability across benchmark - // runs. - metrics = append(metrics, benchmetric.Checksum(hash, "image")) - if err := recorder.Record(ctx, fmt.Sprintf("StableDiffusionXL/%s/refiner=%t/steps=%d", test.name, useRefiner, test.steps), metrics...); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - }) - } - }) - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestStableDiffusionXL": TestStableDiffusionXL, diff --git a/test/kubernetes/benchmarks/startup.go b/test/kubernetes/benchmarks/startup.go new file mode 100644 index 000000000..e0ef3bb06 --- /dev/null +++ b/test/kubernetes/benchmarks/startup.go @@ -0,0 +1,116 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package startup benchmarks the time it takes for an empty alpine container to complete successfully. +package startup + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" +) + +const ( + benchName = "StartUp" +) + +var ( + command = []string{"/bin/sh", "-c", "echo hello"} +) + +// MeasureStartup benchmarks the time it takes for an empty alpine container +// to complete successfully. +// Note: WRT gVisor startup latency, this is not a meaningful benchmark. +// Startup time is dominated by Kubernetes control plane API calls and not +// actual container startups. This benchmark is provided for illustrative +// purposes only. +func MeasureStartup(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + t.Logf("Warning: This is not a meaningful benchmark. Read the comments.") + + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + podName := "startup" + image, err := k8sCtx.ResolveImage(ctx, "alpine") + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + p, err := cluster.ConfigurePodForRuntimeTestNodepool(benchmarkNS.NewAlpinePod(podName, image, command)) + if err != nil { + t.Fatalf("failed to set pod for test nodepool: %v", err) + } + + start := time.Now() + p, err = cluster.CreatePod(ctx, p) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, p) + if err := cluster.WaitForPodCompleted(ctx, p); err != nil { + t.Fatalf("Failed to wait for pod to complete: %v", err) + } + reader, err := cluster.GetLogReader(ctx, p, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.GetName(), err) + } + defer reader.Close() + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.GetName(), err) + } + if strings.TrimSpace(buf.String()) != "hello" { + t.Fatalf("Mistmatch output: got: %q want: %q", buf.String(), "hello") + } + + // For longer running containers, and where the desired duration to + // measure is the time it takes to run a command within a container, + // this should use `GetTimedContainerDuration` instead. + // However, since this benchmark's goal is to measure container runtime + // overhead, it uses the Kubernetes-level metrics for container + // duration. + containerDuration, err := cluster.ContainerDurationSecondsByName(ctx, p, p.GetName()) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + overallDuration := time.Since(start) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record(ctx, benchName, + benchmetric.BenchmarkDuration(overallDuration), + benchmetric.SpecificDuration(containerDuration, "container-runtime"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } +} diff --git a/test/kubernetes/benchmarks/startup_test.go b/test/kubernetes/benchmarks/startup_test.go index 8518394c6..421bb4e3a 100644 --- a/test/kubernetes/benchmarks/startup_test.go +++ b/test/kubernetes/benchmarks/startup_test.go @@ -12,39 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package startup_test benchmarks the time it takes for an empty alpine container to complete successfully. -package startup_test +package startup import ( - "bytes" "context" - "io" - "strings" "testing" - "time" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" ) -const ( - benchName = "StartUp" -) - -var ( - command = []string{"/bin/sh", "-c", "echo hello"} -) - -// TestStartup benchmarks the time it takes for an empty alpine container to complete successfully. -// Note: WRT gVisor startup latency, this is not a meaningful benchmark. Startup time is dominated -// by Kubernetes control plane API calls and not actual container startups. This benchmark is -// provided for illustrative purposes only. func TestStartup(t *testing.T) { - t.Logf("Warning: This is not a meaningful benchmark. Read the comments.") - ctx := context.Background() k8sCtx, err := k8sctx.Context(ctx) if err != nil { @@ -54,73 +32,7 @@ func TestStartup(t *testing.T) { t.Run(benchName, func(t *testing.T) { cluster := cluster t.Parallel() - - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - podName := "startup" - image, err := k8sCtx.ResolveImage(ctx, "alpine") - if err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - p, err := cluster.ConfigurePodForRuntimeTestNodepool(benchmarkNS.NewAlpinePod(podName, image, command)) - if err != nil { - t.Fatalf("failed to set pod for test nodepool: %v", err) - } - - start := time.Now() - p, err = cluster.CreatePod(ctx, p) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, p) - if err := cluster.WaitForPodCompleted(ctx, p); err != nil { - t.Fatalf("Failed to wait for pod to complete: %v", err) - } - reader, err := cluster.GetLogReader(ctx, p, v13.PodLogOptions{}) - if err != nil { - t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.GetName(), err) - } - defer reader.Close() - - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, reader); err != nil { - t.Fatalf("Failed to read log on cluster %q: %v", cluster.GetName(), err) - } - if strings.TrimSpace(buf.String()) != "hello" { - t.Fatalf("Mistmatch output: got: %q want: %q", buf.String(), "hello") - } - - // For longer running containers, and where the desired duration to - // measure is the time it takes to run a command within a container, - // this should use `GetTimedContainerDuration` instead. - // However, since this benchmark's goal is to measure container runtime - // overhead, it uses the Kubernetes-level metrics for container - // duration. - containerDuration, err := cluster.ContainerDurationSecondsByName(ctx, p, p.GetName()) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - overallDuration := time.Since(start) - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - err = recorder.Record(ctx, benchName, - benchmetric.BenchmarkDuration(overallDuration), - benchmetric.SpecificDuration(containerDuration, "container-runtime"), - ) - if err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } + MeasureStartup(ctx, t, k8sCtx, cluster) }) }) } diff --git a/test/kubernetes/benchmarks/tensorflow.go b/test/kubernetes/benchmarks/tensorflow.go new file mode 100644 index 000000000..6fa08aa47 --- /dev/null +++ b/test/kubernetes/benchmarks/tensorflow.go @@ -0,0 +1,152 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tensorflow + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + k8s "gvisor.dev/gvisor/test/kubernetes" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = k8s.ImageRepoPrefix + "benchmarks/tensorflow_x86_64:latest" + imageARM = k8s.ImageRepoPrefix + "benchmarks/tensorflow_aarch64:latest" +) + +var workloads = map[string]string{ + "Kmeans": "2_BasicModels/kmeans.py", + "LogisticRegression": "2_BasicModels/logistic_regression.py", + "NearestNeighbor": "2_BasicModels/nearest_neighbor.py", + "RandomForest": "2_BasicModels/random_forest.py", + "ConvolutionalNetwork": "3_NeuralNetworks/convolutional_network.py", + "MultilayerPerceptron": "3_NeuralNetworks/multilayer_perceptron.py", + "NeuralNetwork": "3_NeuralNetworks/neural_network.py", +} + +// RunTensorflowOnCPU runs the Tensorflow example workloads on CPU. +func RunTensorflowOnCPU(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "tensorflow" + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + workloadPaths := make([]string, 0, len(workloads)) + workloadPathToName := make(map[string]string, len(workloads)) + for name, path := range workloads { + workloadPaths = append(workloadPaths, path) + workloadPathToName[path] = name + } + sort.Strings(workloadPaths) + + var total time.Duration + for _, workloadPath := range workloadPaths { + workloadName := workloadPathToName[workloadPath] + t.Run(workloadName, func(t *testing.T) { + pod := newTensorflowOnCPUPod(benchmarkNS, name, image, workloadPath) + pod, err := cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("TensorflowOnCPU/%s", workloadName), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + total += containerDuration + }) + if t.Failed() { + break + } + } + if !t.Failed() { + if err := recorder.Record(ctx, "TensorflowOnCPU", benchmetric.BenchmarkDuration(total)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + } +} + +func newTensorflowOnCPUPod(namespace *testcluster.Namespace, name, image, workloadPath string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.TimedCommand("python", workloadPath), + WorkingDir: "/TensorFlow-Examples/examples", + Env: []v13.EnvVar{ + { + Name: "PYTHONPATH", + Value: "/TensorFlow-Examples/examples", + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} diff --git a/test/kubernetes/benchmarks/tensorflow_test.go b/test/kubernetes/benchmarks/tensorflow_test.go index 39707d405..240c10458 100644 --- a/test/kubernetes/benchmarks/tensorflow_test.go +++ b/test/kubernetes/benchmarks/tensorflow_test.go @@ -12,39 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tensorflow_test +package tensorflow import ( "context" - "fmt" - "sort" "testing" - "time" - k8s "gvisor.dev/gvisor/test/kubernetes" - "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" - "gvisor.dev/gvisor/test/kubernetes/benchmetric" "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" - v13 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( - imageAMD = k8s.ImageRepoPrefix + "benchmarks/tensorflow_x86_64:latest" - imageARM = k8s.ImageRepoPrefix + "benchmarks/tensorflow_aarch64:latest" -) - -var workloads = map[string]string{ - "Kmeans": "2_BasicModels/kmeans.py", - "LogisticRegression": "2_BasicModels/logistic_regression.py", - "NearestNeighbor": "2_BasicModels/nearest_neighbor.py", - "RandomForest": "2_BasicModels/random_forest.py", - "ConvolutionalNetwork": "3_NeuralNetworks/convolutional_network.py", - "MultilayerPerceptron": "3_NeuralNetworks/multilayer_perceptron.py", - "NeuralNetwork": "3_NeuralNetworks/neural_network.py", -} - func TestTensorflowOnCPU(t *testing.T) { ctx := context.Background() k8sCtx, err := k8sctx.Context(ctx) @@ -54,116 +31,11 @@ func TestTensorflowOnCPU(t *testing.T) { k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { t.Run("TensorflowOnCPU", func(t *testing.T) { t.Parallel() - doTensorflowOnCPU(ctx, t, k8sCtx, cluster) + RunTensorflowOnCPU(ctx, t, k8sCtx, cluster) }) }) } -func doTensorflowOnCPU(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { - benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) - endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) - if err != nil { - t.Fatalf("Failed to setup profiling: %v", err) - } - defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) - - const name = "tensorflow" - recorder, err := benchmetric.GetRecorder(ctx) - if err != nil { - t.Fatalf("Failed to initialize benchmark recorder: %v", err) - } - - image := imageAMD - if cluster.RuntimeTestNodepoolIsARM() { - image = imageARM - } - if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { - t.Fatalf("Failed to resolve image: %v", err) - } - - workloadPaths := make([]string, 0, len(workloads)) - workloadPathToName := make(map[string]string, len(workloads)) - for name, path := range workloads { - workloadPaths = append(workloadPaths, path) - workloadPathToName[path] = name - } - sort.Strings(workloadPaths) - - var total time.Duration - for _, workloadPath := range workloadPaths { - workloadName := workloadPathToName[workloadPath] - t.Run(workloadName, func(t *testing.T) { - pod := newTensorflowOnCPUPod(benchmarkNS, name, image, workloadPath) - pod, err := cluster.ConfigurePodForRuntimeTestNodepool(pod) - if err != nil { - t.Fatalf("Failed to set pod for test runtime: %v", err) - } - - pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) - if err != nil { - t.Fatalf("Failed to set container resources: %v", err) - } - - pod, err = cluster.CreatePod(ctx, pod) - if err != nil { - t.Fatalf("Failed to create pod: %v", err) - } - defer cluster.DeletePod(ctx, pod) - - containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) - if err != nil { - t.Fatalf("Failed to get container duration: %v", err) - } - if err := recorder.Record(ctx, fmt.Sprintf("TensorflowOnCPU/%s", workloadName), benchmetric.BenchmarkDuration(containerDuration)); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - total += containerDuration - }) - if t.Failed() { - break - } - } - if !t.Failed() { - if err := recorder.Record(ctx, "TensorflowOnCPU", benchmetric.BenchmarkDuration(total)); err != nil { - t.Fatalf("Failed to record benchmark data: %v", err) - } - } -} - -func newTensorflowOnCPUPod(namespace *testcluster.Namespace, name, image, workloadPath string) *v13.Pod { - return &v13.Pod{ - TypeMeta: v1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: name, - Namespace: namespace.Namespace, - }, - Spec: v13.PodSpec{ - Containers: []v13.Container{ - { - Name: name, - Image: image, - Command: benchmetric.TimedCommand("python", workloadPath), - WorkingDir: "/TensorFlow-Examples/examples", - Env: []v13.EnvVar{ - { - Name: "PYTHONPATH", - Value: "/TensorFlow-Examples/examples", - }, - }, - }, - }, - RestartPolicy: v13.RestartPolicyNever, - }, - } -} - func TestMain(m *testing.M) { k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ "TestTensorflowOnCPU": TestTensorflowOnCPU, diff --git a/test/kubernetes/benchmarks/wordpress.go b/test/kubernetes/benchmarks/wordpress.go new file mode 100644 index 000000000..daa5fdf8a --- /dev/null +++ b/test/kubernetes/benchmarks/wordpress.go @@ -0,0 +1,379 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wordpress + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + wordpressImage = "wordpress:6.2.0-php8.2-apache" + mariaDBImage = "mariadb:10.11.3-jammy" + wordpressPort = 80 + mariaDBPort = 3306 + wordpressBenchmarkDuration = 70 * time.Second + wordpressRequestTimeout = 10 * time.Second + wordpressLoginPage = "/wp-login.php" + mariaDBName = "wpbench" + mariaDBUser = "wpuser" + mariaDBPassword = "wppassword" + mariaDBRootPassword = "hunter2" + mariaDBVolumeName = "wpdata" + mariaDBVolumeDirectory = "/var/lib/mysql" + + wordpressServerLabelKey = "app.kubernetes.io/name" + wordpressServerLabelValue = "wordpress" + mariaDBServerLabelKey = "app.kubernetes.io/name" + mariaDBServerLabelValue = "mariadb" +) + +var ( + threads = []int{1, 8, 64, 1000} + targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} + wantPercentiles = []int{50, 95, 99} +) + +// BenchmarkWordpress runs a benchmark of WordPress performance. +func BenchmarkWordpress(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + // Create a persistent volume on which to store the database data. + dbVolume := benchmarkNS.GetPersistentVolume(mariaDBVolumeName, "30Gi") + dbVolume, err = cluster.CreatePersistentVolume(ctx, dbVolume) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, dbVolume) + + databaseName := "mariadb" + mariaDBImg, err := k8sCtx.ResolveImage(ctx, mariaDBImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + database := newMariaDBServer(benchmarkNS, databaseName, mariaDBImg, dbVolume) + database, err = cluster.ConfigurePodForTertiaryNodepool(database) + if err != nil { + t.Fatalf("Failed to configure pod for tertiary nodepool: %v", err) + } + database, err = cluster.CreatePod(ctx, database) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, database) + if err := cluster.WaitForPodRunning(ctx, database); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + databaseService := newMariaDBService(benchmarkNS, databaseName) + databaseService, err = cluster.CreateService(ctx, databaseService) + if err != nil { + t.Fatalf("Failed to create database service: %v", err) + } + defer cluster.DeleteService(ctx, databaseService) + mariaDBIP := testcluster.GetIPFromService(databaseService) + + name := "wordpress" + wordpressImg, err := k8sCtx.ResolveImage(ctx, wordpressImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + server := newWordpressServer(benchmarkNS, name, wordpressImg, mariaDBIP) + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + + service := newWordpressService(benchmarkNS, name) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, service) + wordpressIP := testcluster.GetIPFromService(service) + + // Install WordPress. + installWordpressPod := newWordpressInstall(benchmarkNS, "install-wordpress", wordpressIP) + installWordpressPod, err = cluster.ConfigurePodForClientNodepool(installWordpressPod) + if err != nil { + t.Fatalf("Failed to configure pod for client nodepool: %v", err) + } + installWordpressPod, err = cluster.CreatePod(ctx, installWordpressPod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, installWordpressPod) + if err := cluster.WaitForPodCompleted(ctx, installWordpressPod); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + cluster.DeletePod(ctx, installWordpressPod) + + var rounds []httpbench.Round + for _, numThreads := range threads { + for _, qps := range targetQPS { + if qps < numThreads { + continue + } + onlyReport := []httpbench.MetricType{httpbench.RequestsPerSecond} + // If we're testing at max QPS, don't report latency, + // because all requests will hit the timeout. + // Otherwise, only report latency, because the throughput + // is exactly determined by the QPS target anyway. + if qps != httpbench.InfiniteQPS { + onlyReport = append(onlyReport, httpbench.Latency) + } + rounds = append(rounds, httpbench.Round{ + NumThreads: numThreads, + TargetQPS: qps, + Duration: wordpressBenchmarkDuration, + OnlyReport: onlyReport, + }) + } + } + benchmark := &httpbench.HTTPBenchmark{ + Name: "wordpress", + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: wordpressPort, + Path: wordpressLoginPage, + Rounds: rounds, + Timeout: wordpressRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) +} + +func newMariaDBServer(namespace *testcluster.Namespace, name, image string, volume *v13.PersistentVolumeClaim) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{mariaDBServerLabelKey: mariaDBServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: mariaDBPort, + }, + }, + Env: []v13.EnvVar{ + { + Name: "MARIADB_ROOT_PASSWORD", + Value: mariaDBRootPassword, + }, + { + Name: "MARIADB_DATABASE", + Value: mariaDBName, + }, + { + Name: "MARIADB_USER", + Value: mariaDBUser, + }, + { + Name: "MARIADB_PASSWORD", + Value: mariaDBPassword, + }, + }, + VolumeMounts: []v13.VolumeMount{{ + Name: volume.GetName(), + MountPath: mariaDBVolumeDirectory, + }}, + }, + }, + Volumes: []v13.Volume{{ + Name: volume.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: volume.GetName(), + }, + }, + }}, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newMariaDBService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{mariaDBServerLabelKey: mariaDBServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: mariaDBPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func newWordpressServer(namespace *testcluster.Namespace, name, image, mariaDBHost string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{wordpressServerLabelKey: wordpressServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: wordpressPort, + }, + }, + Env: []v13.EnvVar{ + { + Name: "WORDPRESS_DB_HOST", + Value: mariaDBHost, + }, + { + Name: "WORDPRESS_DB_USER", + Value: mariaDBUser, + }, + { + Name: "WORDPRESS_DB_PASSWORD", + Value: mariaDBPassword, + }, + { + Name: "WORDPRESS_DB_NAME", + Value: mariaDBName, + }, + { + Name: "WORDPRESS_TABLE_PREFIX", + Value: "wp_", + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newWordpressService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{wordpressServerLabelKey: wordpressServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: wordpressPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func newWordpressInstall(namespace *testcluster.Namespace, name, wpHost string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: "install-wordpress", + Image: "debian:latest", + // This command installs WordPress through the web UI. + // Source of the parameters: + // https://github.com/GoogleCloudPlatform/click-to-deploy/blob/master/k8s/wordpress/chart/wordpress/templates/wordpress-configmap.yaml + Command: []string{ + "sh", "-c", + strings.Join([]string{ + "apt-get update -y