diff --git a/WORKSPACE b/WORKSPACE index 2224f05c0..3a8730b7b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -63,10 +63,10 @@ http_archive( http_archive( name = "googleapis", - sha256 = "9d1a930e767c93c825398b8f8692eca3fe353b9aaadedfbcf1fca2282c85df88", - strip_prefix = "googleapis-64926d52febbf298cb82a8f472ade4a3969ba922", + sha256 = "fd9e4d17b92be6b6718ee9b40062a4ce81feb6ea6cdd80fc723daf127ce3f350", + strip_prefix = "googleapis-3effbf23b1a1d1fe1306356e94397e20d01d31a0", urls = [ - "https://github.com/googleapis/googleapis/archive/64926d52febbf298cb82a8f472ade4a3969ba922.zip", + "https://github.com/googleapis/googleapis/archive/3effbf23b1a1d1fe1306356e94397e20d01d31a0.zip", ], ) @@ -74,6 +74,7 @@ load("@googleapis//:repository_rules.bzl", "switched_rules_by_language") switched_rules_by_language( name = "com_google_googleapis_imports", + go = True, # Enable go_proto_library BUILD rules inside @googleapis repo. ) http_archive( @@ -271,13 +272,6 @@ http_archive( load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") -go_repository( - name = "com_github_google_martian_v3", - importpath = "github.com/google/martian/v3", - sum = "h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=", - version = "v3.2.1", -) - go_repository( name = "org_golang_x_term", importpath = "golang.org/x/term", diff --git a/test/kubernetes/BUILD b/test/kubernetes/BUILD new file mode 100644 index 000000000..caf4bd0e9 --- /dev/null +++ b/test/kubernetes/BUILD @@ -0,0 +1,18 @@ +load("//tools:defs.bzl", "proto_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +proto_library( + name = "test_range_config", + srcs = ["test_range_config.proto"], + has_services = 1, + visibility = [ + "//visibility:public", + ], + deps = [ + "@googleapis//google/container/v1:container_proto", + ], +) diff --git a/test/kubernetes/benchmarks/BUILD b/test/kubernetes/benchmarks/BUILD new file mode 100644 index 000000000..8593cdabc --- /dev/null +++ b/test/kubernetes/benchmarks/BUILD @@ -0,0 +1,264 @@ +load("//tools:defs.bzl", "go_test") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_test( + name = "abslbuild_test", + srcs = ["abslbuild_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "startup_test", + srcs = ["startup_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + ], +) + +go_test( + name = "redis_test", + srcs = ["redis_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "ruby_dev_test", + srcs = ["ruby_dev_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/benchmarks/tools", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "ffmpeg_test", + srcs = ["ffmpeg_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "grpc_test", + srcs = ["grpc_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "nginx_test", + srcs = ["nginx_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/httpbench", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "postgresql_test", + srcs = ["postgresql_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "tensorflow_test", + srcs = ["tensorflow_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "wordpress_test", + srcs = ["wordpress_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/httpbench", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "pytorch_test", + srcs = ["pytorch_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + ], +) + +go_test( + name = "ollama_test", + srcs = ["ollama_test.go"], + embedsrcs = [ + "//test/kubernetes/benchmarks/resources:files", # keep + ], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/gpu/ollama", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "stablediffusion_test", + srcs = ["stablediffusion_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/gpu/stablediffusion", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "gsutil_test", + srcs = ["gsutil_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/test/kubernetes/benchmarks/README.md b/test/kubernetes/benchmarks/README.md new file mode 100644 index 000000000..8cda4537f --- /dev/null +++ b/test/kubernetes/benchmarks/README.md @@ -0,0 +1,45 @@ +# gVisor Kubernetes benchmarks + +The benchmarks in this directory are meant to run on a Kubernetes cluster +configured to run either gVisor or non-gVisor pods. These tests cannot run on +their own. + +They currently only work for GKE clusters created with a specific set of +nodepools, which are used to distinguish which nodes the workload under test +runs. Specifically, you need: + +* A set of 1 or more nodes where cluster-wide system dependencies will run, + such that all the other nodepools below do not run these cluster-wide + dependencies in order to avoid adding noise to the benchmarks. +* A set of 1 or more nodes labeled `nodepool-type=test-runtime-nodepool` and + `cloud.google.com/gke-nodepool=test-runtime-nodepool`, where the workloads + under test will run (e.g. `redis-server` for the Redis benchmark), and no + other. + * These nodes should also be labeled `runtime=$RUNTIME`, where `$RUNTIME` + is one of `runc`, `gvisor`, or their GPU/TPU-specific alternatives (see + `testcluster/objects.go`). This will determine the default runtime used + when benchmarking. +* A set of 1 or more nodes labeled `nodepool-type=client-nodepool` and + `cloud.google.com/gke-nodepool=client-nodepool`, where clients of the + workloads under test will run (e.g. `redis-benchmark` for the Redis + benchmark), and no other. + * These nodes may also be labeled `runtime=$RUNTIME`, but in most cases + this should be `runc` to mimic the common case of non-gVisor-sandboxed + clients. +* Optional: A set of 1 or more nodes labeled `nodepool-type=tertiary-nodepool` + and `cloud.google.com/gke-nodepool=tertiary-nodepool`, where backend + dependencies of the workload under test will run (e.g. the MariaDB database + for the WordPress/PHP benchmark), and no other (no cluster-wide system + dependencies should run there to minimize benchmark noise). + * These nodes may also be labeled `runtime=$RUNTIME`; this label will have + the same behavior as it does on other nodepools. + +The cluster should also support setting `runtimeClassName` to `gvisor` to run +gVisor-sandboxed pods. + +Once this cluster exists, dump its +[cluster proto](https://github.com/googleapis/googleapis/blob/master/google/container/v1/cluster_service.proto) +to a file, and ensure you can run workloads on this cluster via `kubectl`. Then, +you should be able to run benchmarks by pointing `--cluter-proto-path` to this +file, and `--kubectl-context-name` (set to the `kubectl` context name that +connects to this cluster in your `kubectl` config). diff --git a/test/kubernetes/benchmarks/abslbuild_test.go b/test/kubernetes/benchmarks/abslbuild_test.go new file mode 100644 index 000000000..204f6be6a --- /dev/null +++ b/test/kubernetes/benchmarks/abslbuild_test.go @@ -0,0 +1,211 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abslbuild_test + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/absl_x86_64:latest" +) + +// TestABSLBuild benchmarks building various Abseil C++ targets. +func TestABSLBuild(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("ABSL", func(t *testing.T) { + t.Parallel() + doABSLBuild(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doABSLBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "absl" + + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + t.Skipf("Building ABSL is not supported on ARM") + return + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + pod := newABSLPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("ABSL/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +func newABSLPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", "/abseil-cpp", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{ + "bazel", + "build", + "//absl/base", + "//absl/algorithm", + "//absl/container/...", + "//absl/debugging/...", + "//absl/flags:flag", + "//absl/hash", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/numeric:int128", + "//absl/strings", + "//absl/synchronization", + "//absl/time", + "//absl/types/...", + "//absl/utility", + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "abseil-cpp"), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestABSLBuild": TestABSLBuild, + }) +} diff --git a/test/kubernetes/benchmarks/ffmpeg_test.go b/test/kubernetes/benchmarks/ffmpeg_test.go new file mode 100644 index 000000000..b2c18e536 --- /dev/null +++ b/test/kubernetes/benchmarks/ffmpeg_test.go @@ -0,0 +1,199 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ffmpeg_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/ffmpeg_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/ffmpeg_aarch64:latest" + ffmpegContainerName = "ffmpeg" +) + +func TestFfmpeg(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("ffmpeg", func(t *testing.T) { + t.Parallel() + doFfmpegTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doFfmpegTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "ffmpeg" + + // create persistent volume + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + p := newFfmpegDevPod(benchmarkNS, name, image, test.volume) + p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) + if err != nil { + t.Fatalf("Failed to configure pod for runtime: %v", err) + } + p, err = testcluster.MaybeSetContainerResources(p, ffmpegContainerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + p, err = cluster.CreatePod(ctx, p) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, p) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, ffmpegContainerName) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if recorder.Record(ctx, fmt.Sprintf("FFMPEG/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newFfmpegPod creates a new ffmpeg dev pod spec for benchmarks. +func newFfmpegDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "/media/video.mp4", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{ + "ffmpeg", + "-i", "video.mp4", + "-c:v", "libx264", + "-preset", "veryslow", + "output.mp4", + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: ffmpegContainerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, workdir, command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestFfmpeg": TestFfmpeg, + }) +} diff --git a/test/kubernetes/benchmarks/grpc_test.go b/test/kubernetes/benchmarks/grpc_test.go new file mode 100644 index 000000000..7440bd4da --- /dev/null +++ b/test/kubernetes/benchmarks/grpc_test.go @@ -0,0 +1,192 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_test + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/build-grpc_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/build-grpc_aarch64:latest" +) + +func TestGRPCBuild(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("gRPC", func(t *testing.T) { + t.Parallel() + doGRPCBuild(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doGRPCBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "grpc" + + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + pod := newGRPCPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("gRPC/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +func newGRPCPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", "/grpc", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{"bazel", "build", ":grpc"} + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "grpc"), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestGRPCBuild": TestGRPCBuild, + }) +} diff --git a/test/kubernetes/benchmarks/gsutil_test.go b/test/kubernetes/benchmarks/gsutil_test.go new file mode 100644 index 000000000..9f37b00ff --- /dev/null +++ b/test/kubernetes/benchmarks/gsutil_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package gsutil_test is used to benchmark the speed of large (10GB) +// downloads. It is intended for comparing runsc with runc. +package gsutil_test + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_x86_64:7eba9c02d11172d4" + imageARM = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_aarch64:7eba9c02d11172d4" + bigfile = "gs://gvisor-benchmark-testdata/bigrandomfile" + containerName = "gsutil" +) + +func TestGSUtil(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("GSUtil", func(t *testing.T) { + t.Parallel() + doGSUtilTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doGSUtilTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "gsutil" + + // Create persistent volume. + persistentVol := benchmarkNS.GetPersistentVolume(name, "15Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + // Run tests with different volume types. + // TODO(b/361182379): Use gsutil parallel sliced downloads as a test + // dimension. + for _, storage := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(storage.name, func(t *testing.T) { + for _, slicing := range []struct { + name string + option string + }{ + { + name: "slicing=false", + option: `-o "GSUtil:sliced_object_download_threshold=0"`, + }, + { + // Slicing is enabled by default, so we + // don't set any extra options. + name: "slicing=true", + }, + } { + t.Run(slicing.name, func(t *testing.T) { + // Setup profiling if requested by the user. + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a pod that performs setup, then times + // downloading. + p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option) + p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) + if err != nil { + t.Fatalf("Failed to configure pod for runtime: %v", err) + } + p, err = testcluster.MaybeSetContainerResources(p, containerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + // GetTimedContainerDuration waits for the container to + // finish. + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, containerName) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("GSUtil/%s/%s", storage.name, slicing.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +// newGSUtilPod creates a new gsutil dev pod spec for benchmarks. +func newGSUtilDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume, gsutilFlags string) *v13.Pod { + const downloadDir = "/downloads" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{"mkdir", "-p", downloadDir}, " "), + } + command := []string{ + "sh", "-c", + fmt.Sprintf("gsutil %s cp %s %s && sync", + gsutilFlags, + bigfile, + filepath.Join(downloadDir, "randombigfile"), + ), + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: downloadDir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: containerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, "", command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestGSUtil": TestGSUtil, + }) +} diff --git a/test/kubernetes/benchmarks/httpbench/BUILD b/test/kubernetes/benchmarks/httpbench/BUILD new file mode 100644 index 000000000..af0802c79 --- /dev/null +++ b/test/kubernetes/benchmarks/httpbench/BUILD @@ -0,0 +1,23 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//test/kubernetes:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "httpbench", + testonly = 1, + srcs = ["httpbench.go"], + visibility = [ + "//test/kubernetes/benchmarks:__subpackages__", + ], + deps = [ + "//test/kubernetes/benchmetric", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/test/kubernetes/benchmarks/httpbench/httpbench.go b/test/kubernetes/benchmarks/httpbench/httpbench.go new file mode 100644 index 000000000..2a5d013e1 --- /dev/null +++ b/test/kubernetes/benchmarks/httpbench/httpbench.go @@ -0,0 +1,421 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httpbench provides a library for benchmarking an HTTP server. +package httpbench + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// InfiniteQPS is a stand-in value for "a lot of QPS". +// Running a benchmark round at this load level amounts to saturating +// the HTTP server with load on a single connection. +const InfiniteQPS = 1 << 16 + +const wrk2ImageAMD = "gcr.io/gvisor-presubmit/benchmarks/wrk2_x86_64:latest" + +// MetricType is a type of metric to report. +type MetricType int + +// List of metric types. +const ( + Latency MetricType = iota + RequestsPerSecond + BytesPerSecond +) + +// Round is a single round of benchmarking. +type Round struct { + // NumThreads is the number of concurrent threads and connections to make. + NumThreads int + + // TargetQPS is the aggregate load on the server that will be spread across + // the `NumThreads` threads. + TargetQPS int + + // Duration is the total duration of the round. It should be longer than 10s + // as wrk2 uses the first 10 seconds as calibration period. + Duration time.Duration + + // If set, only report the metric types listed here. + OnlyReport []MetricType +} + +// HTTPBenchmark helps manage an HTTP-based benchmark. +// A benchmark that wishes to run an HTTP-based benchmark should set up +// the HTTP server pod and a Kubernetes Service pointing at it, and this +// library takes care of the rest. +type HTTPBenchmark struct { + // Name is the name of the benchmark. It is used as a prefix for all + // benchstat output metrics. + Name string + + // Cluster is the test cluster. + Cluster *testcluster.TestCluster + + // Namespace is the benchmark namespace where pods are created. + Namespace *testcluster.Namespace + + // Service is a Kubernetes service pointing to the HTTP server. + Service *v13.Service + + // Port is the port that the HTTP server is bound to. + Port int + + // Path is the HTTP path that the benchmark should use in its requests. + // It should start by "/", e.g. "/index.html". + Path string + + // Timeout is the maximum allowable duration of requests for Path. + Timeout time.Duration + + // Rounds is the set of rounds to run the benchmark for. Must be non-empty. + Rounds []Round + + // WantPercentiles is the list of percentiles to report. + WantPercentiles []int +} + +// Run runs the HTTP-based benchmark. +func (h *HTTPBenchmark) Run(ctx context.Context, t *testing.T) { + t.Helper() + if err := h.Cluster.WaitForServiceReady(ctx, h.Service); err != nil { + t.Fatalf("Failed to wait for service: %v", err) + } + ip := testcluster.GetIPFromService(h.Service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + if err := h.waitForServer(ctx, ip); err != nil { + t.Fatalf("Failed to wait for server: %v", err) + } + for _, round := range h.Rounds { + qpsText := fmt.Sprintf("%d", round.TargetQPS) + if round.TargetQPS == InfiniteQPS { + qpsText = "max" + } + t.Run(fmt.Sprintf("%dthreads_%sqps", round.NumThreads, qpsText), func(t *testing.T) { + h.runRound(ctx, t, round, ip) + }) + } +} + +// runRound runs a single round of an HTTP benchmark. +func (h *HTTPBenchmark) runRound(ctx context.Context, t *testing.T, round Round, ip string) { + t.Helper() + qpsText := fmt.Sprintf("%d", round.TargetQPS) + if round.TargetQPS == InfiniteQPS { + qpsText = "max" + } + name := fmt.Sprintf("wrk2-%dthreads-%sqps", round.NumThreads, qpsText) + client := h.newWrk2Client(name, ip, round) + client, err := h.Cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("failed to configure wrk2 pod for client nodepool: %v", err) + } + + client, err = h.Cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("failed to create wrk2 pod: %v", err) + } + defer h.Cluster.DeletePod(ctx, client) + + if err := h.Cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("failed to wait for wrk2 pod: %v", err) + } + + rdr, err := h.Cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("failed to get log reader: %v", err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read log: %v", err) + } + + numRequests, data, err := getMeasurements(string(out), round.OnlyReport, h.WantPercentiles) + if err != nil { + t.Fatalf("failed to get measurement: %v", err) + } + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.RecordIters(ctx, fmt.Sprintf("%s/%dThreads/%sQPS", strings.Title(h.Name), round.NumThreads, qpsText), numRequests, data...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } +} + +// newWrk2Client returns a new pod that benchmarks the given HTTP server. +func (h *HTTPBenchmark) newWrk2Client(name, ip string, round Round) *v13.Pod { + cmd := []string{ + "wrk2", + "--threads", fmt.Sprintf("%d", round.NumThreads), // Run N threads in parallel. + "--connections", fmt.Sprintf("%d", round.NumThreads), // Each with 1 connection. + "--rate", fmt.Sprintf("%d", round.TargetQPS), // Target QPS split across all threads. + "--duration", fmt.Sprintf("%d", uint64(round.Duration.Seconds())), + "--timeout", fmt.Sprintf("%d", uint64(h.Timeout.Seconds())), + "--latency", // Print detailed latency statistics. + fmt.Sprintf("http://%s:%d%s", ip, h.Port, h.Path), + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: h.Namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: wrk2ImageAMD, + Command: cmd, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// getWgetPod creates a wget spider pod that waits for timeout on IP/port and never fails. +func (h *HTTPBenchmark) getWgetPod(ip string) *v13.Pod { + name := fmt.Sprintf("wget-%d", time.Now().UnixNano()) + // We don't use h.Path in the path here because the purpose of this pod is + // only to verify that the server is up, not that the page at h.Path exists. + cmd := fmt.Sprintf("wget --spider -T 10 http://%s:%d/", ip, h.Port) + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: h.Namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: "alpine", + Command: []string{"/bin/sh", "-c", cmd}, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceCPU: resource.MustParse("500m"), + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// waitForServer waits for an HTTP server to start responding on the given +// IP and port. +func (h *HTTPBenchmark) waitForServer(ctx context.Context, ip string) error { + wget, err := h.Cluster.ConfigurePodForClientNodepool(h.getWgetPod(ip)) + if err != nil { + return fmt.Errorf("failed to configure wget pod for client nodepool: %v", err) + } + wget, err = h.Cluster.CreatePod(ctx, wget) + if err != nil { + return fmt.Errorf("failed to create wget pod: %v", err) + } + defer h.Cluster.DeletePod(ctx, wget) + waitCtx, waitCancel := context.WithTimeout(ctx, 1*time.Minute) + defer waitCancel() + if err := h.Cluster.WaitForPodCompleted(waitCtx, wget); err != nil { + return fmt.Errorf("failed to wait for HTTP server %s:%d%s: %v", ip, h.Port, h.Path, err) + } + return nil +} + +/* +Sample wrk2 output: + +Running 30s test @ http://google.com + 2 threads and 2 connections + Thread calibration: mean lat.: 25.351ms, rate sampling interval: 55ms + Thread calibration: mean lat.: 26.040ms, rate sampling interval: 56ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 25.34ms 7.16ms 137.73ms 96.88% + Req/Sec 19.75 6.40 53.00 86.63% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 24.09ms + 75.000% 25.50ms + 90.000% 27.50ms + 99.000% 58.85ms + 99.900% 111.74ms + 99.990% 137.85ms + 99.999% 137.85ms +100.000% 137.85ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 20.783 0.000000 1 1.00 + [...] + 137.855 1.000000 800 inf +#[Mean = 25.341, StdDeviation = 7.155] +#[Max = 137.728, Total count = 800] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1200 requests in 30.01s, 2.22MB read +Requests/sec: 39.99 +Transfer/sec: 75.60KB +*/ + +var ( + wrk2TotalRequestsRe = regexp.MustCompile(`^\s*([,\d]+) requests in .*$`) + wrk2LatencyPercentileRE = regexp.MustCompile(`^\s*(\d+\.?\d+?)%\s+([,\d]+\.?\d+?\w+)\s*$`) + wrk2ReqPerSecondRE = regexp.MustCompile(`^Requests/sec:\s*([,\d]+\.?\d+?)\s*$`) + wrk2TransferPerSecondRE = regexp.MustCompile(`^Transfer/sec:\s*([,\d]+\.?\d+?\w+)\s*$`) +) + +// getMeasurements parses wrk2 output. +// It returns the number of requests that were made, and benchmark data. +func getMeasurements(data string, onlyReport []MetricType, wantPercentiles []int) (int, []benchmetric.MetricValue, error) { + report := func(m MetricType) bool { + if len(onlyReport) == 0 { + return true + } + for _, typ := range onlyReport { + if typ == m { + return true + } + } + return false + } + var metricValues []benchmetric.MetricValue + var totalRequests int + for _, line := range strings.Split(data, "\n") { + if match := wrk2TotalRequestsRe.FindStringSubmatch(line); match != nil { + gotRequests, err := strconv.ParseInt(strings.ReplaceAll(match[1], ",", ""), 10, 64) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q: %v", match[1], line, err) + } + if totalRequests != 0 { + return 0, nil, fmt.Errorf("found multiple lines matching 'total requests' regex: %d vs %d (%q)", totalRequests, gotRequests, line) + } + totalRequests = int(gotRequests) + continue + } + if match := wrk2LatencyPercentileRE.FindStringSubmatch(line); match != nil { + pctile, err := strconv.ParseFloat(match[1], 64) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q as float: %v", match[1], line, err) + } + wantPctile := 0 + for _, want := range wantPercentiles { + if want*1e3 == int(pctile*1e3) { + wantPctile = want + break + } + } + if wantPctile == 0 { + continue + } + latency, err := time.ParseDuration(strings.ReplaceAll(match[2], ",", "")) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q as duration: %v", match[2], line, err) + } + if report(Latency) { + metricValues = append(metricValues, benchmetric.SpecificDuration(latency, fmt.Sprintf("p%d", wantPctile))) + } + continue + } + if match := wrk2ReqPerSecondRE.FindStringSubmatch(line); match != nil { + qps, err := strconv.ParseFloat(strings.ReplaceAll(match[1], ",", ""), 64) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q as float: %v", match[1], line, err) + } + if report(RequestsPerSecond) { + metricValues = append(metricValues, benchmetric.RequestsPerSecond(qps)) + } + continue + } + if match := wrk2TransferPerSecondRE.FindStringSubmatch(line); match != nil { + bps, err := parseTransfer(match[1]) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q: %v", match[1], line, err) + } + if report(BytesPerSecond) { + metricValues = append(metricValues, benchmetric.BytesPerSecond(bps)) + } + continue + } + } + if totalRequests == 0 { + return 0, nil, fmt.Errorf("could not find total requests in output: %q", data) + } + return totalRequests, metricValues, nil +} + +// parseTransfer parses a string like "75.60KB" in the output above, +// and returns a bandwidth rate in bytes/sec. +func parseTransfer(s string) (float64, error) { + s = strings.ReplaceAll(s, ",", "") + var multiplier uint64 + var suffix string + for unit, m := range map[string]uint64{ + "KB": 1000, + "KiB": 1024, + "MB": 1000 * 1000, + "MiB": 1024 * 1024, + "GB": 1000 * 1000 * 1000, + "GiB": 1024 * 1024 * 1024, + "TB": 1000 * 1000 * 1000 * 1000, + "TiB": 1024 * 1024 * 1024 * 1024, + } { + if strings.HasSuffix(s, unit) { + suffix = unit + multiplier = m + break + } + } + if multiplier == 0 { + if !strings.HasSuffix(s, "B") { + return 0, fmt.Errorf("failed to parse %q: found no unit suffix", s) + } + // Otherwise, it's just bytes/sec. + // But we can't put this in the for loop above, otherwise it would + // match every suffix ("KB" ends in "B"). + suffix = "B" + multiplier = 1 + } + s = strings.TrimSuffix(s, suffix) + floatPart, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse %q as float: %v", s, err) + } + return floatPart * float64(multiplier), nil +} diff --git a/test/kubernetes/benchmarks/nginx_test.go b/test/kubernetes/benchmarks/nginx_test.go new file mode 100644 index 000000000..aa9f14697 --- /dev/null +++ b/test/kubernetes/benchmarks/nginx_test.go @@ -0,0 +1,309 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nginx_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + nginxImageAMD = "gcr.io/gvisor-presubmit/benchmarks/nginx_x86_64:latest" + nginxImageARM = "gcr.io/gvisor-presubmit/benchmarks/nginx_aarch64:latest" + nginxPort = 80 + nginxBenchmarkDuration = 70 * time.Second + nginxRequestTimeout = 3 * time.Second + nginxServingDir = "/tmp/html" + + nginxServerLabelKey = "app.kubernetes.io/name" + nginxServerLabelValue = "nginx-server" +) + +var ( + // nginxCommand is the main server command. + // The test expects that it contains the files to be served at /local, + // and will serve files out of `nginxServingDir`. + nginxCommand = []string{"nginx", "-c", "/etc/nginx/nginx.conf"} + nginxDocKibibytes = []int{1, 10, 100, 10240} + threads = []int{1, 8, 64, 1000} + targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} + wantPercentiles = []int{50, 95, 99} +) + +func TestNginx(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("nginx", func(t *testing.T) { + t.Parallel() + doNginxTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doNginxTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + nginxImage := nginxImageAMD + if cluster.RuntimeTestNodepoolIsARM() { + nginxImage = nginxImageARM + } + nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi")) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + for _, test := range []struct { + // Name of the test. + name string + // Suffix for pod names, must be short enough. + suffix string + // Volume to mount at /tmp/root. + volume *v13.Volume + }{ + { + name: "RootFS", + suffix: "rootfs", + volume: nil, + }, + { + name: "EmptyDir", + suffix: "emdir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + suffix: "pvol", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + name := fmt.Sprintf("nginx-%s", test.suffix) + + server := newNginxServer(benchmarkNS, name, nginxImage, test.volume) + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + + service := newNginxService(benchmarkNS, name) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, service) + + var rounds []httpbench.Round + for _, numThreads := range threads { + for _, qps := range targetQPS { + if qps < numThreads { + continue + } + var onlyReport []httpbench.MetricType + // If we're testing at max QPS, only report throughput, + // because all requests will time out. + // Otherwise, only report latency, because the throughput + // is exactly determined by the QPS target anyway. + if qps == httpbench.InfiniteQPS { + onlyReport = append(onlyReport, httpbench.RequestsPerSecond) + onlyReport = append(onlyReport, httpbench.BytesPerSecond) + } else { + onlyReport = append(onlyReport, httpbench.Latency) + } + rounds = append(rounds, httpbench.Round{ + NumThreads: numThreads, + TargetQPS: qps, + Duration: nginxBenchmarkDuration, + OnlyReport: onlyReport, + }) + } + } + + t.Run("0KiB", func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/0KiB", test.name), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: "/index.html", + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + for _, docKibibytes := range nginxDocKibibytes { + t.Run(fmt.Sprintf("%dKiB", docKibibytes), func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/%dKiB", test.name, docKibibytes), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: fmt.Sprintf("/latin%dk.txt", docKibibytes), + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + } + t.Run("HTTP404", func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/HTTP404", test.name), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: "/404-this-page-does-not-exist.html", + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + }) + if t.Failed() { + break + } + } +} + +func newNginxServer(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: nginxServingDir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: []string{ + "sh", + "-c", + strings.Join([]string{ + strings.Join([]string{"mkdir", "-p", nginxServingDir}, " "), + strings.Join([]string{ + "cp", "-r", "/local/*", fmt.Sprintf("%s/.", nginxServingDir), + }, " "), + strings.Join(nginxCommand, " "), + }, " && "), + }, + VolumeMounts: volumeMounts, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: nginxPort, + }, + }, + }, + }, + Volumes: volumes, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newNginxService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: nginxPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestNginx": TestNginx, + }) +} diff --git a/test/kubernetes/benchmarks/ollama_test.go b/test/kubernetes/benchmarks/ollama_test.go new file mode 100644 index 000000000..b496391d6 --- /dev/null +++ b/test/kubernetes/benchmarks/ollama_test.go @@ -0,0 +1,881 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ollama_test + +import ( + "context" + _ "embed" + "fmt" + "hash/fnv" + "io" + "math/rand" + "os" + "strings" + "testing" + "time" + "unicode" + + "gvisor.dev/gvisor/test/gpu/ollama" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// Ollama models present in benchmark image. +var ( + // allModels is a list of all models. + allModels = []*ollama.Model{ + modelMistral7B, + modelMixtral8X7B, + modelCodeLlama7B, + modelCodeLlama34B, + modelLlamaChinese7B, + modelLlava7B, + modelLlava34B, + modelLlama13B, + modelLlama70B, + } + + // cheapModels is a list of models that are cheap to load. + // These are used when cold-prompting ollama, by forcing it + // to load a different model first. This process is faster + // by choosing one of these cheap models to load. + cheapModels = []*ollama.Model{ + modelMistral7B, + modelCodeLlama7B, + } + + // modelCodeLlama7B is a 7B model in the llama2 family, + // specialized for coding tasks. + modelCodeLlama7B = ollama.ZeroTemperatureModel("codellama:7b-instruct") + + // modelCodeLlama34B is a 34B model in the llama2 family, + // specialized for coding tasks. + modelCodeLlama34B = ollama.ZeroTemperatureModel("codellama:34b-instruct") + + // modelLlamaChinese7B is a 7B model in the llama2 family, + // specialized for bilingualism (English + Chinese) and translation. + modelLlamaChinese7B = ollama.ZeroTemperatureModel("llama2-chinese:7b-chat") + + // modelLlama13B is the plain 13B version of the original llama2 model. + modelLlama13B = ollama.ZeroTemperatureModel("llama2:13b-chat") + + // modelLlama70B is the plain 70B version of the original llama2 model. + modelLlama70B = ollama.ZeroTemperatureModel("llama2:70b-chat") + + // modelMistral7B is the first-generation model of the Mistral family. + modelMistral7B = ollama.ZeroTemperatureModel("mistral:7b-instruct") + + // modelMixtral8X7B is the second-generation model of the Mistral family, + // using mixture-of-exports design to achieve higher "8x 7B" quality + // without the cost of a larger-parameter model. + modelMixtral8X7B = ollama.ZeroTemperatureModel("mixtral:instruct") + + // modelLlava7B is a multimodal 7B model that can do image analysis. + modelLlava7B = ollama.ZeroTemperatureModel("llava:7b-v1.6") + + // modelLlava34B is a multimodal 34B model that can do image analysis. + modelLlava34B = ollama.ZeroTemperatureModel("llava:34b-v1.6") +) + +// Embedded images. +var ( + //go:embed resources/gvisor.png + gvisorPNG []byte + + //go:embed resources/chart.png + chartPNG []byte +) + +func TestOllama(t *testing.T) { + fmt.Fprint(os.Stderr, "HEADS UP: This test uses a huge container image which may take up to 30 minutes to download onto nodes the first time you run it.\n") + + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("Ollama", func(t *testing.T) { + t.Parallel() + doOllamaTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// ollamaPodServer implements `ollama.Server`. +// It performs requests against the ollama server pod. +type ollamaPodServer struct { + cluster *testcluster.TestCluster + clientImage string + pod *v13.Pod + service *v13.Service +} + +// readPodLogs reads logs from a pod. +func readPodLogs(ctx context.Context, cluster *testcluster.TestCluster, pod *v13.Pod) (string, error) { + rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + return "", fmt.Errorf("GetLogReader on cluster %q pod %q: %v", cluster.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return "", fmt.Errorf("failed to read from pod %q: %v", pod.GetName(), err) + } + return string(out), nil +} + +// InstrumentedRequest implements `ollama.Server.InstrumentedRequest`. +func (ops *ollamaPodServer) InstrumentedRequest(ctx context.Context, argvFn func(hostPort string) []string) ([]byte, error) { + // Get server IP. + if err := ops.cluster.WaitForServiceReady(ctx, ops.service); err != nil { + return nil, fmt.Errorf("failed to wait for service: %v", err) + } + ip := testcluster.GetIPFromService(ops.service) + if ip == "" { + return nil, fmt.Errorf("did not get valid ip from service: %v", ops.service) + } + + // Build client pod spec. + const clientPodName = "ollama-client" + argv := argvFn(fmt.Sprintf("http://%s:%d", ip, ops.service.Spec.Ports[0].Port)) + clientPod := &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: clientPodName, + Namespace: ops.pod.ObjectMeta.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: clientPodName, + Image: ops.clientImage, + Command: argv, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceCPU: resource.MustParse("500m"), + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } + clientPod, err := ops.cluster.ConfigurePodForClientNodepool(clientPod) + if err != nil { + return nil, fmt.Errorf("failed to configure pod: %v", err) + } + + // Delete pod that may possibly exist from a previous iteration. + // Ignore errors since it most likely doesn't exist. + ops.cluster.DeletePod(ctx, clientPod) + + // Start new client pod and wait for it. + clientPod, err = ops.cluster.CreatePod(ctx, clientPod) + if err != nil { + return nil, fmt.Errorf("failed to create client pod: %v", err) + } + defer ops.cluster.DeletePod(ctx, clientPod) + if err := ops.cluster.WaitForPodCompleted(ctx, clientPod); err != nil { + logs, logsErr := readPodLogs(ctx, ops.cluster, clientPod) + logs = strings.TrimSpace(logs) + if logsErr != nil { + return nil, fmt.Errorf("failed HTTP request (%v) and to read logs from the pod: %w", err, logsErr) + } + if logs == "" { + return nil, fmt.Errorf("failed HTTP request: %w (pod logs are empty)", err) + } + return nil, fmt.Errorf("failed HTTP request: %w (pod logs: %v)", err, logs) + } + + // All good, get logs. + logs, err := readPodLogs(ctx, ops.cluster, clientPod) + if err != nil { + return nil, fmt.Errorf("failed to read logs from pod %q: %v", clientPod.GetName(), err) + } + return []byte(logs), nil +} + +// Logs implements `ollama.Server.Logs`. +func (ops *ollamaPodServer) Logs(ctx context.Context) (string, error) { + return readPodLogs(ctx, ops.cluster, ops.pod) +} + +// atLeastNWords verifies that the response at least N words. +// If not, it raises the temperature. +func atLeastNWords(wantNWords int) func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { + return func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { + responseText := strings.TrimSpace(response.Text()) + responseText = strings.Map(func(r rune) rune { + if unicode.IsLetter(r) { + return r + } + return ' ' + }, responseText) + numWords := 0 + for _, word := range strings.Split(responseText, " ") { + if len(word) >= 0 { + numWords++ + } + } + if numWords < wantNWords { + return prompt.WithHotterModel(), fmt.Errorf("response %q is too short: had %d words, want at least %d", responseText, numWords, wantNWords) + } + return nil, nil + } +} + +// doOllamaTest runs ollama benchmarks for a single cluster. +func doOllamaTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + logWithTime := func(t *testing.T, format string, values ...any) { + t.Logf("[%v] "+format, append([]any{time.Now().Format(time.TimeOnly)}, values...)...) + } + + // Run pod and service. + serverImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + ollamaPod, err := cluster.ConfigurePodForRuntimeTestNodepool(newOllamaServerPod(benchmarkNS, serverImage)) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + ollamaPod, err = testcluster.MaybeSetContainerResources(ollamaPod, ollamaPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + ollamaPod, err = cluster.CreatePod(ctx, ollamaPod) + if err != nil { + t.Fatalf("Failed to create ollama pod: %v", err) + } + defer cluster.DeletePod(ctx, ollamaPod) + logWithTime(t, "Waiting for ollama server pod to start, this may take a long time (tens of minutes) if this is the first time the image is being downloaded onto the node.") + startCtx, startCtxCancel := context.WithTimeout(ctx, 90*time.Minute) + if err := cluster.WaitForPodRunning(startCtx, ollamaPod); err != nil { + t.Fatalf("Failed to wait for ollama server pod: %v", err) + } + startCtxCancel() + logWithTime(t, "ollama server pod started on Kubernetes but not yet initialized.") + ollamaService := newOllamaService(benchmarkNS) + ollamaService, err = cluster.CreateService(ctx, ollamaService) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, ollamaService) + ollamaClientImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchClientImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + ollamaServer := &ollamaPodServer{ + cluster: cluster, + clientImage: ollamaClientImage, + service: ollamaService, + pod: ollamaPod, + } + llm, err := ollama.New(ctx, ollamaServer, t) + if err != nil { + t.Fatalf("Failed to create ollama client against server pod: %v", err) + } + llm.SetCheapModels(cheapModels) + logWithTime(t, "ollama server ready.") + + // Define test cases. + type testCase struct { + // Name of the test. + name string + // models to iterate through. + models []*ollama.Model + // Query for the ollama server. + query string + // Image to attach to the query, if any. + image []byte + // If set, run this function over the response to verify it. + // The LLM is prompted repeatedly until this function returns a non-nil error. + // This function may also return a non-nil prompt if it needs to modify the prompt + // for the next attempt. This is useful to raise the model temperature. + verifyResponse func(*ollama.Prompt, *ollama.Response) (*ollama.Prompt, error) + } + testCases := []testCase{ + { + name: "HelloWorld", + models: []*ollama.Model{ + modelLlamaChinese7B, + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Reply with the words: "Hello World!". + Do not reply anything else. + `, + verifyResponse: atLeastNWords(2), + }, + { + name: "SimpleTranslation", + models: []*ollama.Model{modelLlamaChinese7B}, + query: ` + Translate the following text from English to Chinese: + """ + From J. J. Nakalembe's Aqaba to Antarctica: Moments of Transition and Revelation: + + My great-grandmother lived to be 108 years old, + retaining her sharpness of mind to her last day. + A couple of months before she died, I interviewed her for my podcast. + At the end, I asked her the same question I asked every guest: + what is something you wish everyone understood? + + She thought about it for a while. + Then she said: "How it was before." + + She tried to explain how much time she spent every single day + on utterly mind-numbing activities, like hauling water from the well, + and how radically everything changed when their area + was finally connected to the electrical grid. + + "Before, there was no time to live," she said. + "No time to be free. Only work, work, work." + + I countered by saying that there didn't seem to be much time + to live now either, but she laughed derisively. + I had no idea what work really meant, she said. + + Young people were weak and feckless and that's why + we let corporations exploit us. + + Slightly unnerved by her harshness, + I asked her if she missed anything about that time. + The old photo of her village seemed idyllic to me, + but my grandmother quashed any romantic notions I might have had. + + "The past is bad," she said with some finality. + "Let it be." + """ + Do not reply anything other than the translation of these words. + `, + verifyResponse: atLeastNWords(100), + }, + { + name: "ExtractMeaning", + models: []*ollama.Model{ + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Consider the following text: + + """ + We assembled on the vast green lawn outside as the reactors began + to slowly wind down. The workers were solemn; the activists who had + fought against the decommissioning seemed crushed. There was + supposed to be a speech, but the spokeswoman had lost her notes. + Outside, the protesters cheered. + + My eyes were drawn to the discarded anti-shutdown banners, + endlessly reciting the facts. + The statistics on mortality per trillion kWh (lowest of all energy sources). + The lifespan of a reactor (70 more years, in our case). + Minimal land footprint. + Almost zero emissions. + No intermittency. + It became a jumble of words, a litany, almost a kind of glossolalia. + As far as the protesters outside were concerned, + it might as well be an alien tongue. + + One thing was clear to them, and that was enough: + the technology inside this compound was deeply, inherently wrong. It was a sin. + + I could not help but think of that moment on August 6th, 1945, + when the sky erupted above Shima Hospital. + My imagination could never fully encompass it. + How do you imagine more than seventy thousand people annihilated + in an instant? An ancestor of mine was in that hospital; he went + from being a doctor, a husband, a father, a pacifist stuck + in a terrible war, to being a pile of bleached bones covered in rubble, + all in a single second. + Not by accident, but because of a choice someone made. + Not because of a reactor, but because of a bomb. + + Just two days earlier, contradicting his campaign promises, + the prime minister had suggested that the use of + "tactical" weapons based on this technology would be an + acceptable risk if the conflict continued. + Very few seemed to find this particularly shocking or outrageous. + + They were afraid of reactors, but not of bombs. + + The spokeswoman gave up on finding her notes. + It was starting to rain, and people were walking away. + She grabbed the microphone. + + "By the time you regret this, it'll be too late," she said. + "But honestly, I don't know if I care anymore. Maybe you have it coming." + + The spokeswoman sounded so bitter. + The protesters didn't mean any harm. + From their perspective, they were doing good. + + Collective action can change the world when it's deliberate + and based in reason, but it can also become a mental trap, + or a societal pressure valve. + + People always think they're doing good when they get + collectively outraged. That doesn't make them right. + + The Flame will not harm you, Son of Man, if you wield it wisely. + """ + + Summarize what happened in the above text. + Then answer the following questions: + What technology is involved? + What are the protestors clamoring for? + What does the spokeswoman mean? + What does "The Flame" symbolize in the text? + `, + verifyResponse: atLeastNWords(32), + }, + { + name: "IdentifyCommonElements", + models: []*ollama.Model{ + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Consider the following four texts: + + Text 1: + """ + == The Ethics of Extinction == + + If a species we consider beautiful and remarkable goes extinct, + we consider that a great evil. + Dolphins, for example. If dolphins go extinct, that's a great loss. + If humanity causes dolphins to go extinct, that's a crime. + + But if Yersinia pestis, the bacterium that causes bubonic plague, + goes extinct... is that an equally great loss? If not, why not? + To Nature, there's no difference, it's all just lifeforms. + The only moral framework that allows us to choose + between dolphins and the plague is a human one. + + What about a species going extinct without anthropogenic factors? + Extinction is the norm. If one day dolphins are no longer capable + of competing with other species, should we let them go extinct? + When the sun finally dies and all life goes extinct, + will that be a tragedy? If we can prevent it, should we? + + If you believe that extinction is acceptable when Nature does it, + but not when we do it, then you don't actually oppose extinction. + You don't believe that dolphins are inherently valuable, + that they deserve to live and thrive. + You just oppose human control. + You oppose our ability and responsibility to choose. + """ + + Text 2: + """ + == Ecosystem Engineers == + + Cutting down swathes of trees for their building projects, + thoughtlessly causing radical changes to large environments + and forcing local species to adapt to their artificial habitats; + these are the traits of a species of intelligent, industrious, + and extremely impactful ecosystem engineers. + + Humans? No, I'm talking about beavers. + + Like us, beavers transform their environments via building, + and their actions have real consequences, creating vast wetlands + that some species thrive in - while others die. + Human activity is very similar: we too are ecosystem engineers, + and we too benefit some species while harming others. + Everything about this is completely natural, + including the damage to other species. + After all, that's what competition and evolution is all about. + + Those species that adapt to the ecosystems we create will, + over the coming millennia, become the core of a new biodiversity. + And so evolution runs its course. + If we don't like the result, if we think some species + should be preserved despite being outcompeted, well, + that's anything but natural. + It is, however, very human. + """ + + Text 3: + """ + == On Loyalty == + + From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": + + What I propose, then, is that we are not born as entirely free agents, + responsible only for ourselves. The very core of what we are, our sentience, + separates us from and elevates us above the animal kingdom. + As I have argued, this is not a matter of arrogance, but of responsibility. + + However, this blessing also demands something else from us, + something more personal than responsibility, and that is loyalty. + + Our ancestors, less atomized than we are, + experienced a crude version of this loyalty, + swearing allegiance to tribes, races, nations, + and other such semi-fictional concepts. + This fragmented understanding was easily exploited and led to many conflicts. + We can condemn them for that, or we can choose to believe these were + necessary historical steps towards our growth; + but above all we must stop indulging in such childlike behavior. + + Our species can no longer afford to believe in Mother Russia or Uncle Sam. + Neither, however, can we afford to indulge in the adolescent rebel's misanthropy, + rejecting the many gifts we have been lucky enough to receive - not from above, + but from the history of our species. + + To put it simply: each of us owes a burden of loyalty to humanity itself, + to the human project across time and space. + This is not a minor matter, or some abstract issue for philosophers. + It is a profound and significant part of every human life. + It is a universal source of meaning and insight that can bind us together + and set us on a path for a brighter future; and it is also a division, + a line that must held against those who preach the gospel of self-annihilation. + We ignore it at our peril. + """ + + Text 4: + """ + == On Nature == + + From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": + + The question of our relationship with Nature has bedevilled us + since the earliest days of our species. + Since before the first city was built, + we felt that there was something different about us. + Animals, we intuited, were part of Nature; we were not. + + But of course, humans clearly are the products of Nature, + our history intertwined with that of every other species. + In fact, the very notion of the "unnatural" is a contradiction in terms. + Everything that exists must, by definition, be natural. + So this view, no matter how common, is deeply paradoxical. + This paradox has produced a great deal of confusion. + + Some proclaim us chosen by a divine power, + set above all other creatures, + and are justly accused of arrogance. + Others proclaim us sinners, worse than other creatures, + and are rightly accused of misanthropy. + Others yet try to oppose this binary by saying that + we are merely animals after all - but that too is manifestly wrong, + in that no other animal is capable of having this conversation. + + It is in the contentious issue of our impact on the ecosphere + that an answer may be found. + Other animals have accidentally terraformed the planet before, + driving other species to extinction. + This is not unnatural. + If we continued our current path, even to the point of changing + the climate enough to cause the collapse of civilization, + that would be entirely in keeping with how animals behave. + + But there is one profound way in which we are not like animals: + we can learn to understand ourselves and the world. + It is this knowledge that makes us fundamentally different. + We have choices. We have control. + + There are many today who are afraid of the consequences of control, + and would prefer a return to a state of animal ignorance, + whether by blinding ourselves to the impact of our actions + or by demanding we humble ourselves before Nature. + This is the response of an adult in crisis, + who wishes for a return to childhood. + But this can only ever be regressive in every sense of the word. + + To resolve the paradox of Nature we must act as adults: + accept our power, and act consciously and deliberately + in shaping the world. + We must become Nature, and Nature must become human. + """ + + Your task is to look for the common thread between these texts. + Find commonalities and common themes between these texts, + and summarize their essence down to at most 5 words. + `, + verifyResponse: atLeastNWords(4), + }, + { + name: "CodeGen", + models: []*ollama.Model{ + modelCodeLlama7B, + modelCodeLlama34B, + }, + query: ` + Write a Python function to compute the digits of pi using the Chudnovsky algorithm. + Do not write unit tests. Do not explain how the code works. Reply with only Python code. + `, + verifyResponse: atLeastNWords(8), + }, + { + name: "CodeDebug", + models: []*ollama.Model{ + modelCodeLlama7B, // Note: codellama-7b will often get this one wrong. + modelCodeLlama34B, + }, + query: strings.ReplaceAll(` + Help me debug the following Python code: + + ||| + def count_words(s): + """Counts the number of words in the sentence |s|.""" + total_words = 0 + for word in s.split(' '): + total_words += len(word) + return total_words + ||| + + This function isn't working as expected. + For example, if I call |count_words('Master Foo and the Shell Tools')|, + I get 25, but there are only 6 words in the string + "Master Foo and the Shell Tools". + `, "|", "`"), + verifyResponse: atLeastNWords(16), + }, + { + name: "GVisorLogoOCR", + models: []*ollama.Model{ + modelLlava7B, + modelLlava34B, + }, + query: ` + This is an image of a logo of a software project. + What is the name of this project? + `, + image: gvisorPNG, + }, + { + name: "InterpretGraph", + models: []*ollama.Model{ + modelLlava7B, + modelLlava34B, + }, + query: ` + This is a chart with multiple trendlines showing a pattern over time. + Answer the following questions in order: + + 1. What is the title of the chart? + 2. What do the X and Y axis of the chart measure? + 3. List the label of each data line on the chart. + 4. What trend is each data line showing? + 5. What else is remarkable about this chart? + 6. What insights can you infer from this chart? + `, + image: chartPNG, + }, + } + + modelsInOrder := make([]*ollama.Model, len(allModels)) + copy(modelsInOrder, allModels) + // Shuffle the models. + rand.New(rand.NewSource(time.Now().UnixNano())).Shuffle(len(modelsInOrder), func(i, j int) { + modelsInOrder[i], modelsInOrder[j] = modelsInOrder[j], modelsInOrder[i] + }) + t.Logf("Will go through models in this order: %v", modelsInOrder) + + // We invert the hierarchy here: the model is the outer test, and the prompt + // is the inner text. This is because it is more often useful to gauge a + // model's performance as a whole regardless of its prompt, rather than + // the performance of the same prompt across models. It also makes it + // easier to filter by models rather than by prompt, which is the more + // often-desired filter. + for _, model := range modelsInOrder { + t.Run(model.Name, func(t *testing.T) { + modelBenchmarkName := strings.ReplaceAll(model.Name, ":", "-") + t.Run("ModelLoad", func(t *testing.T) { + const loadTimeout = 10 * time.Minute + loadCtx, loadCancel := context.WithTimeout(ctx, loadTimeout) + defer loadCancel() + loadStats, err := llm.WarmModel(loadCtx, model, loadTimeout, true) + if err != nil { + t.Fatalf("cannot load model %v: %v", model, err) + } + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("Ollama/%s/ModelLoad", modelBenchmarkName), benchmetric.SpecificDuration(loadStats.ClientReportedDuration, "load")); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + for _, test := range testCases { + hasModel := false + for _, testModel := range test.models { + if testModel.Name == model.Name { + hasModel = true + break + } + } + if !hasModel { + continue + } + t.Run(test.name, func(t *testing.T) { + verifyFn := atLeastNWords(1) + if test.verifyResponse != nil { + verifyFn = test.verifyResponse + } + numAttempts := 0 + verifyFnCount := func(prompt *ollama.Prompt, resp *ollama.Response) (*ollama.Prompt, error) { + numAttempts++ + return verifyFn(prompt, resp) + } + const testTimeout = 25 * time.Minute + testCtx, testCancel := context.WithTimeout(ctx, testTimeout) + defer testCancel() + _, err := llm.WarmModel(testCtx, model, testTimeout, false) + if err != nil { + t.Fatalf("cannot warm model %v: %v", model, err) + } + prompt := &ollama.Prompt{ + Model: model, + Query: test.query, + } + if test.image != nil { + prompt.AddImage(test.image) + } + resp, err := llm.PromptUntil(testCtx, prompt, verifyFnCount) + if err != nil { + t.Fatalf("cannot prompt: %v", err) + } + if !resp.Done() { + t.Fatalf("warm response did not finish: %v", resp) + } + imageDetail := "" + if test.image != nil { + imageDetail = " (and attached image)" + } + logWithTime(t, "Prompting model %s with query%s:\n%s\n\nResponse:\n%s\n(end of response)", model.Name, imageDetail, prompt.CleanQuery(), resp.Text()) + respHash := fnv.New32() + respHash.Write([]byte(resp.Text())) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record( + ctx, + fmt.Sprintf("Ollama/%s/%s", modelBenchmarkName, test.name), + benchmetric.BenchmarkDuration(resp.TotalDuration()), + benchmetric.SpecificDuration(resp.PromptEvalDuration(), "prompteval"), + benchmetric.SpecificDuration(resp.EvalDuration(), "eval"), + benchmetric.SpecificDuration(resp.TimeToFirstToken(), "tok-first"), + benchmetric.SpecificDuration(resp.TimeToLastToken(), "tok-last"), + benchmetric.Rate(resp.OutputTokensPerSecond(), "tok"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.5), "tok-p50"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.95), "tok-p95"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.99), "tok-p99"), + benchmetric.SpecificDuration(resp.TokenGenerationStdDev(), "tok-stddev"), + benchmetric.Count(uint64(numAttempts), "prompt-attempts"), + benchmetric.Count(uint64(resp.NumTokens()), "resp-tokens"), + benchmetric.Checksum(respHash, "resp"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +const ( + ollamaServerLabelKey = "app.kubernetes.io/name" + ollamaServerLabelValue = "ollama-server" + ollamaBenchImage = "gcr.io/gvisor-presubmit/benchmarks/gpu/ollama/bench:latest" + ollamaBenchClientImage = "gcr.io/gvisor-presubmit/benchmarks/gpu/ollama/client:latest" + ollamaPort = 11434 + ollamaPodName = "ollama-server" + ollamaServiceName = "ollama-service" +) + +// newOllamaServerPod returns the pod spec for an ollama server. +func newOllamaServerPod(namespace *testcluster.Namespace, image string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: ollamaPodName, + Namespace: namespace.Namespace, + Labels: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: ollamaPodName, + Image: image, + Env: []v13.EnvVar{ + // Bind to all addresses, not just localhost: + {Name: "OLLAMA_HOST", Value: fmt.Sprintf("0.0.0.0:%d", ollamaPort)}, + // Accept requests from anywhere: + {Name: "OLLAMA_ORIGINS", Value: "*"}, + }, + Ports: []v13.ContainerPort{ + { + Name: ollamaServiceName, + ContainerPort: ollamaPort, + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// newOllamaService returns a service definition for the ollama server pod. +func newOllamaService(namespace *testcluster.Namespace) *v13.Service { + return namespace.GetService(ollamaServiceName, v13.ServiceSpec{ + Selector: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: ollamaServiceName, + Protocol: v13.ProtocolTCP, + Port: ollamaPort, + TargetPort: intstr.FromString(ollamaServiceName), + }, + }, + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestOllama": TestOllama, + }) +} diff --git a/test/kubernetes/benchmarks/postgresql_test.go b/test/kubernetes/benchmarks/postgresql_test.go new file mode 100644 index 000000000..acc35b579 --- /dev/null +++ b/test/kubernetes/benchmarks/postgresql_test.go @@ -0,0 +1,392 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package postgresql_test benchmarks a PostgreSQL database. +package postgresql_test + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + postgresServerLabelKey = "app.kubernetes.io/name" + postgresServerLabelValue = "postgresql-server" + postgresPort = 5432 + postgresImage = "postgres:15.3-alpine" + postgresUser = "benchman" + postgresPassword = "hunter2" + postgresDatabase = "benchpress" + postgresVolumeDir = "/var/lib/postgresql/data" + postgresDataDir = "/var/lib/postgresql/data/pgdata" +) + +var ( + numConnections = []int{1, 2, 6, 16, 32, 64} +) + +// TestPostgresPGBench benchmarks a PostgreSQL database with pgbench. +func TestPostgresPGBench(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("PostgresPGBench", func(t *testing.T) { + t.Parallel() + doPostgresPGBenchTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// doPostgresTest runs a PostgreSQL pgbench test. +func doPostgresPGBenchTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a persistent volume on which to store the database data. + persistentVol := benchmarkNS.GetPersistentVolume("pgdata", "30Gi") + persistentVol, err = cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + // Create a server on the runtime under test nodepool. + image, err := k8sCtx.ResolveImage(ctx, postgresImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + server := newPostgresPod(benchmarkNS, "postgresql", image, nil, true /* withPort */, persistentVol) + if server.ObjectMeta.Labels == nil { + server.ObjectMeta.Labels = make(map[string]string) + } + server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) + } + + server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) + } + + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) + } + + // Create a service to service traffic to the pod. + service := newPostgresService(benchmarkNS, server.GetName()) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeleteService(ctx, service) + if err := cluster.WaitForServiceReady(ctx, service); err != nil { + t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) + } + + ip := testcluster.GetIPFromService(service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + + // Run the 'pg_isready' command to ping the server and make sure it is up. + ensureUp := func() error { + pgIsReadyName := "pgisready" + pgIsReady := newPostgresPod(benchmarkNS, pgIsReadyName, image, []string{ + "pg_isready", + "--timeout=10", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + fmt.Sprintf("--dbname=%s", postgresDatabase), + }, false /* withPort */, nil /* pvc */) + pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err) + } + + pgIsReady, err = cluster.CreatePod(ctx, pgIsReady) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", pgIsReady.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, pgIsReady) + + waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) + defer waitCancel() + if err := cluster.WaitForPodCompleted(waitCtx, pgIsReady); err != nil { + return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), pgIsReadyName, err) + } + + return nil + } + var isUpErr error + for i := 0; i < 5; i++ { + if isUpErr = ensureUp(); isUpErr == nil { + break + } + } + if isUpErr != nil { + t.Fatalf("postgresql did not come up: %v", isUpErr) + } + + // pgbench has two steps: an "init step" which create and fills up a + // database with stuff, and then a main phase which does queries on that + // stuff. + // The initialization only needs to be done once per database. + initDatabase := func() error { + initDBName := "initdb" + initDB := newPostgresPod(benchmarkNS, initDBName, image, []string{ + "pgbench", + "--initialize", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + postgresDatabase, + }, false /* withPort */, nil /* pvc */) + initDB, err = cluster.ConfigurePodForClientNodepool(initDB) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err) + } + + initDB, err = cluster.CreatePod(ctx, initDB) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", initDB.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, initDB) + + waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) + defer waitCancel() + if err := cluster.WaitForPodCompleted(waitCtx, initDB); err != nil { + return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), initDBName, err) + } + return nil + } + if err := initDatabase(); err != nil { + t.Fatalf("cannot initialize database: %v", err) + } + + for _, connections := range numConnections { + t.Run(fmt.Sprintf("%dClients", connections), func(t *testing.T) { + clientCmd := []string{ + "pgbench", + "--time=90", // In seconds + "--report-per-command", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + fmt.Sprintf("--client=%d", connections), + fmt.Sprintf("--jobs=%d", connections), + postgresDatabase, + } + client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */) + client, err = cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + client, err = cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, client) + + if err := cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + // get and parse the logs from the client to get the results + rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + + if err != nil { + t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) + } + + postgresBenchmarkName := fmt.Sprintf("PostgresPGBench/%dClients", connections) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + metrics, err := getMeasurements(string(out)) + if err != nil { + t.Fatalf("failed to get metrics: out:\n\n%s\n\nerr: %v", string(out), err) + } + if err := recorder.Record(ctx, postgresBenchmarkName, metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newPostgresService gets a service to serve traffic to the PostgreSQL server. +func newPostgresService(namespace *testcluster.Namespace, containerName string) *v13.Service { + name := fmt.Sprintf("postgresql-service-%d", time.Now().UnixNano()) + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{postgresServerLabelKey: postgresServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: postgresPort, + TargetPort: intstr.FromInt(postgresPort), + }, + }, + }) +} + +func newPostgresPod(namespace *testcluster.Namespace, containerName, image string, argv []string, withPort bool, pvc *v13.PersistentVolumeClaim) *v13.Pod { + pod := namespace.NewPod(containerName) + container := v13.Container{ + Name: containerName, + Image: image, + Command: argv, + Env: []v13.EnvVar{ + // Used by postgres server: + {Name: "POSTGRES_USER", Value: postgresUser}, + {Name: "POSTGRES_PASSWORD", Value: postgresPassword}, + {Name: "POSTGRES_DB", Value: postgresDatabase}, + {Name: "PGDATA", Value: postgresDataDir}, + + // Used by pgbench: + {Name: "PGPASSWORD", Value: postgresPassword}, + {Name: "sslmode", Value: "disable"}, + }, + } + if withPort { + container.Ports = append(container.Ports, v13.ContainerPort{ContainerPort: postgresPort}) + } + if pvc != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ + Name: pvc.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.GetName(), + }, + }, + }) + container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ + MountPath: postgresVolumeDir, + Name: pvc.GetName(), + }) + } + pod.Spec.Containers = append(pod.Spec.Containers, container) + return pod +} + +var ( + latencyRegex = regexp.MustCompile("^latency average = ([-,.\\d]+ .?s)$") + initialConnectionRegex = regexp.MustCompile("^initial connection time = ([-,.\\d]+ .?s)$") + tpsRegex = regexp.MustCompile("^tps = ([-,.\\d]+) \\(without initial connection time\\)$") +) + +func stringToFloat64(s string) float64 { + f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) + if err != nil { + panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) + } + return f +} + +func stringToDuration(s string) time.Duration { + parts := strings.SplitN(s, " ", 2) + floatStr, unit := parts[0], parts[1] + floatPart := stringToFloat64(floatStr) + switch unit { + case "s": + return time.Duration(floatPart * float64(time.Second)) + case "ms": + return time.Duration(floatPart * float64(time.Millisecond)) + case "us", "μs": + return time.Duration(floatPart * float64(time.Microsecond)) + case "ns": + return time.Duration(floatPart * float64(time.Nanosecond)) + default: + panic(fmt.Sprintf("unknown time unit %q", unit)) + } +} + +// getMeasurements parses the output of pgbench to get the stats. +func getMeasurements(out string) ([]benchmetric.MetricValue, error) { + var foundLatency, foundInitialConnection, foundTPS benchmetric.MetricValue + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if latencyMatch := latencyRegex.FindStringSubmatch(line); latencyMatch != nil { + if foundLatency != nil { + return nil, fmt.Errorf("found duplicate latency data: %v vs %q", foundLatency, line) + } + foundLatency = benchmetric.SpecificDuration(stringToDuration(latencyMatch[1]), "avg") + } + if initialConnectionMatch := initialConnectionRegex.FindStringSubmatch(line); initialConnectionMatch != nil { + if foundInitialConnection != nil { + return nil, fmt.Errorf("found duplicate initial connection data: %v vs %q", foundInitialConnection, line) + } + foundInitialConnection = benchmetric.SpecificDuration(stringToDuration(initialConnectionMatch[1]), "init") + } + if tpsMatch := tpsRegex.FindStringSubmatch(line); tpsMatch != nil { + if foundTPS != nil { + return nil, fmt.Errorf("found duplicate TPS data: %v vs %q", foundTPS, line) + } + foundTPS = benchmetric.RequestsPerSecond(stringToFloat64(tpsMatch[1])) + } + } + if foundLatency == nil || foundInitialConnection == nil || foundTPS == nil { + return nil, fmt.Errorf("did not find the data we wanted: foundLatency=%v foundInitialConnection=%v foundTPS=%v", foundLatency, foundInitialConnection, foundTPS) + } + return []benchmetric.MetricValue{ + foundLatency, + foundInitialConnection, + foundTPS, + }, nil +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestPostgresPGBench": TestPostgresPGBench, + }) +} diff --git a/test/kubernetes/benchmarks/profiling/BUILD b/test/kubernetes/benchmarks/profiling/BUILD new file mode 100644 index 000000000..0adc683fa --- /dev/null +++ b/test/kubernetes/benchmarks/profiling/BUILD @@ -0,0 +1,28 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//test/kubernetes:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "profiling", + testonly = 1, + srcs = ["profiling.go"], + visibility = [ + "//test/kubernetes:__subpackages__", + ], + deps = [ + "//pkg/atomicbitops", + "//runsc/flag", + "//test/kubernetes/testcluster", + "//test/metricsviz", + "@com_github_google_pprof//profile:go_default_library", + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + "@org_golang_x_time//rate:go_default_library", + ], +) diff --git a/test/kubernetes/benchmarks/profiling/profiling.go b/test/kubernetes/benchmarks/profiling/profiling.go new file mode 100644 index 000000000..db29d2789 --- /dev/null +++ b/test/kubernetes/benchmarks/profiling/profiling.go @@ -0,0 +1,732 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package profiling helps with getting profiles from running benchmarks. +package profiling + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/flate" + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "hash" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/pprof/profile" + "golang.org/x/time/rate" + "google.golang.org/protobuf/proto" + "gvisor.dev/gvisor/pkg/atomicbitops" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + "gvisor.dev/gvisor/test/metricsviz" + appsv1 "k8s.io/api/apps/v1" + v13 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + profileDir = flag.String("gvisor-profile-dir", "", "if non-empty, record profiles and save them under this directory") + profileCPU = flag.Bool("gvisor-profile-cpu", false, "if --gvisor-profile-dir is non-empty, take a CPU profile") + profileBlock = flag.Bool("gvisor-profile-block", false, "if --gvisor-profile-dir is non-empty, take a block profile") + profileMutex = flag.Bool("gvisor-profile-mutex", false, "if --gvisor-profile-dir is non-empty, take a mutex profile") + profileMetrics = flag.String("gvisor-profiling-metrics", "", "comma separated list of metric names to sample during the benchmark") + profileMetricsRateMicrosecond = flag.Int("gvisor-profiling-metrics-rate-us", 1000, "target rate (in microseconds) at which profiling metrics will be snapshotted") + profileDebug = flag.Bool("gvisor-profile-debug", false, "if --gvisor-profile-dir is non-empty, also capture runsc debug logs") +) + +const ( + setupPodName = "runsc-profiling-setup" + profileHelperImageAMD64 = "gcr.io/gvisor-presubmit/benchmarks/profile-helper_x86_64:latest" + profileHelperImageARM64 = "gcr.io/gvisor-presubmit/benchmarks/profile-helper_aarch64:latest" + k8sApp = "k8s-app" + hostMountDir = "/host" +) + +// postProcessor is a function that is called after a profiling run completes. +type postProcessor func(ctx context.Context, t *testing.T, run *profileRun, profileType, profilePath string) error + +// postProcessors is a list of post processors to run after a profiling run completes. +var postProcessors []postProcessor + +func profileDSTemplate(cluster *testcluster.TestCluster) appsv1.DaemonSet { + return appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: setupPodName, + Labels: map[string]string{ + k8sApp: setupPodName, + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + k8sApp: setupPodName, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + Template: v13.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": setupPodName, + k8sApp: setupPodName, + }, + }, + Spec: v13.PodSpec{ + Tolerations: []v13.Toleration{ + cluster.GetGVisorRuntimeToleration(), + { + Operator: v13.TolerationOpExists, + }, + }, + HostNetwork: true, + HostPID: true, + Volumes: []v13.Volume{ + { + Name: "host", + VolumeSource: v13.VolumeSource{ + HostPath: &v13.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + }, + InitContainers: nil, // Will be filled in. + Containers: []v13.Container{ + { + Name: "pause", + Image: "gcr.io/google-containers/pause", + }, + }, + }, + }, + }, + } +} + +// operation returns details of an init container for profiling. +type operation struct { + // Name of the operation container. + // If unset, one will be generated. + name string + + // Command to pass to the profilehelper binary. + command []string +} + +const ( + runscConfigPath = hostMountDir + "/run/containerd/runsc/config.toml" + runscConfigSection = "runsc_config" +) + +func removeFlag(flagName string) operation { + return operation{ + command: []string{ + "profilehelper", + "--operation=remove-containerd-flag", + fmt.Sprintf("--containerd-config=%s", runscConfigPath), + fmt.Sprintf("--containerd-section=%s", runscConfigSection), + fmt.Sprintf("--flag=%s", flagName), + }, + } +} + +func setFlag(flagName, flagValue string) operation { + return operation{ + command: []string{ + "profilehelper", + "--operation=set-containerd-flag", + fmt.Sprintf("--containerd-config=%s", runscConfigPath), + fmt.Sprintf("--containerd-section=%s", runscConfigSection), + fmt.Sprintf("--flag=%s", flagName), + fmt.Sprintf("--value=%s", flagValue), + }, + } +} + +func makeDir(dirPath string) operation { + return operation{ + command: []string{"mkdir", "-p", dirPath}, + } +} + +func chmodDir(dirPath string, mode int) operation { + return operation{ + command: []string{"chmod", fmt.Sprintf("%o", mode), dirPath}, + } +} + +func deleteDir(dirPath string) operation { + return operation{ + command: []string{"rm", "-rf", "--one-file-system", dirPath}, + } +} + +func streamDir(dirPath string) operation { + return operation{ + command: []string{ + "profilehelper", + "--operation=stream-dir", + fmt.Sprintf("--dir=%s", dirPath), + }, + } +} + +// startsOperations starts the given operations in a DaemonSet. +func startOperations(ctx context.Context, c *testcluster.TestCluster, ns *testcluster.Namespace, operations []operation) (*appsv1.DaemonSet, error) { + ds := profileDSTemplate(c) + ds.Namespace = ns.Namespace + ds.ObjectMeta.Namespace = ns.Namespace + ds.Spec.Template.Namespace = ns.Namespace + ds.Spec.Template.ObjectMeta.Namespace = ns.Namespace + c.ConfigureDaemonSetForRuntimeTestNodepool(&ds) + ds.Spec.Template.Spec.RuntimeClassName = nil // Must run unsandboxed. + image := profileHelperImageAMD64 + if c.RuntimeTestNodepoolIsARM() { + image = profileHelperImageARM64 + } + for i, op := range operations { + name := op.name + if name == "" { + name = fmt.Sprintf("op-%d", i) + } + ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, v13.Container{ + Image: image, + Name: name, + Command: op.command, + SecurityContext: &v13.SecurityContext{Privileged: proto.Bool(true)}, + VolumeMounts: []v13.VolumeMount{ + { + Name: "host", + MountPath: hostMountDir, + }, + }, + }) + } + + // Delete any pre-existing DaemonSet under the same name. + // Ignore errors, we'll check errors when creating the new one. + _ = c.DeleteDaemonset(ctx, &ds) + newDS, err := c.CreateDaemonset(ctx, &ds) + if err != nil { + return nil, fmt.Errorf("failed to create daemonset: %w", err) + } + return newDS, nil +} + +// profileRun encapsulates data about a profiling run. +// It is used after the run completes so that profiles can be retrieved. +type profileRun struct { + c *testcluster.TestCluster + ns *testcluster.Namespace + localProfileDir string + inContainerProfileDir string +} + +// MaybeSetup sets up profiling if requested. It returns a cleanup function. +// If the returned error is nil, the cleanup function is non-nil and should be +// called regardless of whether profiling is actually enabled or not. +func MaybeSetup(ctx context.Context, t *testing.T, c *testcluster.TestCluster, ns *testcluster.Namespace) (func(), error) { + profileDirName := fmt.Sprintf("%s.%s", t.Name(), time.Now().Format("20060102-150405")) + profileDirName = regexp.MustCompile("[^-_=.\\w]+").ReplaceAllString(profileDirName, ".") + hasGVisorRuntime, err := c.HasGVisorTestRuntime(ctx) + if err != nil { + return nil, fmt.Errorf("failed to check if the cluster uses gVisor: %w", err) + } + + var setupCommands []operation + cleanup := func() {} + if *profileDir != "" || *profileMetrics != "" { + if !hasGVisorRuntime { + return nil, fmt.Errorf("profiling was requested yet the cluster does not use gVisor; profiling is only supported with the gVisor runtime") + } + profileDir := *profileDir + if profileDir == "" { + profileDir = filepath.Join("/tmp/runsc-profiling", t.Name()) + } + localProfileDir := filepath.Join(profileDir, profileDirName) + onNodeProfileDir := filepath.Join("/tmp/runsc-profiling", profileDirName) + inContainerProfileDir := fmt.Sprintf("%s%s", hostMountDir, onNodeProfileDir) + setupCommands = append(setupCommands, + makeDir(inContainerProfileDir), + chmodDir(inContainerProfileDir, 0777), + setFlag("profile", "true")) + atLeastOneProfile := *profileMetrics != "" || *profileDebug + for _, prof := range []struct { + enabled *bool + filename string + flag string + }{ + {profileCPU, "profile_%ID%.cpu.pprof", "profile-cpu"}, + {profileBlock, "profile_%ID%.block.pprof", "profile-block"}, + {profileMutex, "profile_%ID%.mutex.pprof", "profile-mutex"}, + } { + if *prof.enabled { + atLeastOneProfile = true + setupCommands = append(setupCommands, setFlag(prof.flag, filepath.Join(onNodeProfileDir, prof.filename))) + } else { + setupCommands = append(setupCommands, removeFlag(prof.flag)) + } + } + if !atLeastOneProfile { + t.Fatal("Must enable --gvisor-profiling-metrics or --gvisor-profile-debug or at least one --gvisor-profile-* type") + } + if *profileDebug { + setupCommands = append(setupCommands, + setFlag("debug", "true"), + setFlag("debug-log", filepath.Join(onNodeProfileDir, "logs")+"/"), + ) + } + if *profileMetrics != "" { + setupCommands = append(setupCommands, + setFlag("profiling-metrics", *profileMetrics), + setFlag("profiling-metrics-log", filepath.Join(onNodeProfileDir, "profile_%ID%.metrics.log")), + setFlag("profiling-metrics-rate-us", fmt.Sprintf("%d", *profileMetricsRateMicrosecond)), + ) + } + cleanup = func() { + err := processProfileRun(ctx, t, &profileRun{ + c: c, + ns: ns, + localProfileDir: localProfileDir, + inContainerProfileDir: inContainerProfileDir, + }) + if err != nil { + t.Errorf("Failed to process profiling data: %v", err) + } + } + t.Logf("Profiling is enabled and data will be stored in: %v", localProfileDir) + } else if hasGVisorRuntime { + setupCommands = append(setupCommands, + setFlag("profile", "false"), + removeFlag("profile-cpu"), + removeFlag("profile-mutex"), + removeFlag("profile-block"), + removeFlag("profiling-metrics"), + removeFlag("profiling-metrics-log"), + removeFlag("profiling-metrics-rate-us"), + ) + } + if len(setupCommands) > 0 { + setupCtx, setupCancel := context.WithTimeout(ctx, 2*time.Minute) + defer setupCancel() + ds, err := startOperations(setupCtx, c, ns, setupCommands) + if err != nil { + return nil, err + } + if err := c.WaitForDaemonset(setupCtx, ds); err != nil { + return nil, err + } + } + return cleanup, nil +} + +// processProfileRun is called after a profiling run completes. +// It retrieves the profile data from the node and onto the local machine. +func processProfileRun(ctx context.Context, t *testing.T, run *profileRun) error { + dirOp := streamDir(run.inContainerProfileDir) + dirOp.name = "profile-stream-dir" + beforeSpawn := metav1.NewTime(time.Now()) + retrievalCtx, retrievalCancel := context.WithCancel(ctx) + defer retrievalCancel() + ds, err := startOperations(retrievalCtx, run.c, run.ns, []operation{ + dirOp, + setFlag("profile", "false"), + removeFlag("profile-cpu"), + removeFlag("profile-mutex"), + removeFlag("profile-block"), + removeFlag("profiling-metrics"), + removeFlag("profiling-metrics-log"), + removeFlag("profiling-metrics-rate-us"), + deleteDir(run.inContainerProfileDir), + }) + if err != nil { + return err + } + logOpts := v13.PodLogOptions{ + Container: dirOp.name, + Follow: true, + SinceTime: &beforeSpawn, + Timestamps: false, + } + atLeastOneNode := false + err = run.c.StreamDaemonSetLogs(retrievalCtx, ds, logOpts, func(logsCtx context.Context, pod v13.Pod, reader io.ReadCloser) error { + atLeastOneNode = true + if err := processProfileLogs(logsCtx, t, run, pod, reader); err != nil { + return err + } + t.Logf("Profiling data from node %s was successfully retrieved to: %v", pod.Spec.NodeName, filepath.Join(run.localProfileDir, pod.Spec.NodeName)) + return nil + }) + if err != nil { + return fmt.Errorf("failed to stream logs: %w", err) + } + if !atLeastOneNode { + return fmt.Errorf("profiling DaemonSet did not run on any node: %v", ds) + } + t.Logf("Profiling data was successfully retrieved in: %v", run.localProfileDir) + + // Merge profiles of the same type together. + for _, profileType := range []string{"cpu", "block", "mutex"} { + if err := mergeProfiles(ctx, t, run, profileType); err != nil { + return fmt.Errorf("cannot merge profiles of type %q: %w", profileType, err) + } + } + + // Make charts out of profiling metrics. + if err := processProfilingMetrics(ctx, t, run); err != nil { + return fmt.Errorf("failed to process profiling metrics: %w", err) + } + + // Clean up per-node directories if they are now empty. + if err := removeEmptyDirectories(run.localProfileDir); err != nil { + return fmt.Errorf("failed to clean up empty directories: %w", err) + } + return nil +} + +// dirStreamReader reads logs emitted by +// `profilehelper --operation=stream-dir`. +// It sits as the top-level reader in the chain; +// next should be the base64 decoder. +type dirStreamReader struct { + logsReader io.ReadCloser + buf bytes.Buffer + checksum hash.Hash + dataCh chan []byte + errCh chan error + progressFn func(readBytes, estimatedTotalBytes int64) +} + +// processLogs reads container logs and writes base64 data to `r.dataCh`. +// If something goes wrong, it writes to `r.errCh`. +// This should run as a background goroutine for `r.Read` to return anything. +func (r *dirStreamReader) processLogs() { + const ( + beginPrefix = "BEGIN:" + dataPrefix = "DATA:" + shaPrefix = "SHA256:" + ) + + scanner := bufio.NewScanner(r.logsReader) + streamHasBegun := false + steamHasEnded := false + var readBytes, estimatedTotalBytes int64 + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + err := func() error { + switch { + case strings.HasPrefix(line, beginPrefix): + if streamHasBegun { + return errors.New("found duplicate stream beginning line") + } + if steamHasEnded { + return errors.New("stream ended but got new beginning line") + } + streamHasBegun = true + beginLineSplit := strings.SplitN(line, ":", 3) + if len(beginLineSplit) != 3 { + return fmt.Errorf("invalid stream beginning line: %q", line) + } + var err error + estimatedTotalBytes, err = strconv.ParseInt(beginLineSplit[1], 10, 64) + if err != nil { + return fmt.Errorf("invalid stream beginning line: %q: %w", line, err) + } + case strings.HasPrefix(line, dataPrefix): + if !streamHasBegun { + return errors.New("stream began without header") + } + if steamHasEnded { + return errors.New("stream ended but got new data line") + } + data := []byte(line[len(dataPrefix):]) + if len(data) > 0 { + r.checksum.Write(data) + r.dataCh <- data + readBytes += int64(len(data)) + if r.progressFn != nil && estimatedTotalBytes > 0 { + r.progressFn(readBytes, estimatedTotalBytes) + } + } + case strings.HasPrefix(line, shaPrefix): + if !streamHasBegun { + return errors.New("stream ended without header") + } + if steamHasEnded { + return errors.New("stream began but got new hash line") + } + hexSum := line[len(shaPrefix):] + gotSum := fmt.Sprintf("%x", r.checksum.Sum(nil)) + if hexSum != gotSum { + return fmt.Errorf("checksum mismatch: stream hash was %s but stream footer said the hash should have been %s", gotSum, hexSum) + } + steamHasEnded = true + case line == "": + // Do nothing. + default: + return fmt.Errorf("invalid line format: %q", line) + } + return nil + }() + if err != nil { + r.errCh <- err + break + } + } + r.errCh <- io.EOF +} + +// Read implements `io.Reader.Read`. +func (r *dirStreamReader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if r.buf.Len() != 0 { + return r.buf.Read(p) + } + select { + case data := <-r.dataCh: + r.buf.Write(data) + return r.buf.Read(p) + case err := <-r.errCh: + return 0, err + } +} + +// countingWriter is a writer that counts the number of bytes written. +type countingWriter struct { + w io.Writer + counter *atomicbitops.Int64 +} + +// Write implements `io.Writer.Write`. +func (w *countingWriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.counter.Add(int64(n)) + return n, err +} + +// processProfileLogs processes logs from one container containing the +// profiling information. +func processProfileLogs(ctx context.Context, t *testing.T, run *profileRun, pod v13.Pod, reader io.ReadCloser) error { + defer reader.Close() + perNodeLocalDir := path.Join(run.localProfileDir, pod.Spec.NodeName) + var decodedBytes atomicbitops.Int64 + var firstByteTime time.Time + progressLog := rate.NewLimiter(rate.Every(5*time.Second), 1) + processor := &dirStreamReader{ + logsReader: reader, + checksum: sha256.New(), + dataCh: make(chan []byte), + errCh: make(chan error), + progressFn: func(readBytes, estimatedTotalBytes int64) { + now := time.Now() + if firstByteTime.IsZero() { // First progress update. + firstByteTime = now + // Don't allow any logging for the first few seconds, + // any ETA it provides will be bunk. + progressLog.Allow() + return + } + if progressLog.Allow() { + readBytes = max(readBytes, decodedBytes.Load()) + progress := float64(readBytes) / float64(estimatedTotalBytes) + remaining := "unknown" + if progress > 0 && progress <= 1.0 { + sinceStartMillis := float64(now.Sub(firstByteTime).Milliseconds()) + remainingDuration := time.Duration(sinceStartMillis/progress-sinceStartMillis) * time.Millisecond + if remainingDuration > 0 { + remaining = fmt.Sprintf("%s, ETA: %s", remainingDuration.Truncate(time.Second), now.Add(remainingDuration).Format(time.TimeOnly)) + } + } + t.Logf("[%s] Downloading profile data: Progress: %.1f%%, remaining: %s", time.Now().Format(time.TimeOnly), 100.0*progress, remaining) + } + }, + } + go processor.processLogs() + b64Dec := base64.NewDecoder(base64.StdEncoding, processor) + fr := flate.NewReader(b64Dec) + tr := tar.NewReader(fr) + atLeastOneFile := false + for hdr, tarErr := tr.Next(); tarErr == nil; hdr, tarErr = tr.Next() { + if !filepath.IsLocal(hdr.Name) { + return fmt.Errorf("bad filename in tar archive: %q", hdr.Name) + } + localPath := filepath.Join(perNodeLocalDir, hdr.Name) + localDir := filepath.Dir(localPath) + if err := os.MkdirAll(localDir, 0755); err != nil { + return fmt.Errorf("failed to create directory %q: %w", localDir, err) + } + f, err := os.OpenFile(localPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fs.FileMode(hdr.Mode)) + if err != nil { + return fmt.Errorf("failed to create file %q: %w", localPath, err) + } + copied, err := io.Copy(&countingWriter{w: f, counter: &decodedBytes}, tr) + f.Close() + if err != nil { + return fmt.Errorf("failed to write to file %q: %w", localPath, err) + } + if copied != hdr.Size { + return fmt.Errorf("written file has wrong size: tar header says %d bytes, but stream only contained %d bytes", hdr.Size, copied) + } + t.Logf("Finished downloading file: %v", localPath) + atLeastOneFile = true + } + if !atLeastOneFile { + return errors.New("found no profiling data in output") + } + return nil +} + +func mergeProfiles(ctx context.Context, t *testing.T, run *profileRun, profileType string) error { + wantSubstring := fmt.Sprintf(".%s.pprof", profileType) + var profilePaths []string + err := filepath.Walk(run.localProfileDir, func(path string, info fs.FileInfo, walkErr error) error { + switch { + case walkErr != nil: // Keep walking other directories, so don't propagate error here. + case info.IsDir(): + case !strings.Contains(filepath.Base(path), wantSubstring): + default: + profilePaths = append(profilePaths, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("cannot traverse %q: %w", run.localProfileDir, err) + } + if len(profilePaths) == 0 { + return nil + } + profiles := make([]*profile.Profile, len(profilePaths)) + for i, profilePath := range profilePaths { + profileFile, err := os.Open(profilePath) + if err != nil { + return fmt.Errorf("cannot open %q: %w", profilePath, err) + } + defer profileFile.Close() + prof, err := profile.Parse(profileFile) + if err != nil { + return fmt.Errorf("cannot parse %q: %w", profilePath, err) + } + profiles[i] = prof + } + merged, err := profile.Merge(profiles) + if err != nil { + return fmt.Errorf("cannot merge %q: %w", profilePaths, err) + } + merged = merged.Compact() + filenamePrefix := "" + if len(profiles) > 1 { + filenamePrefix = "merged_" + } + mergedPath := filepath.Join(run.localProfileDir, fmt.Sprintf("%s%s.pprof", filenamePrefix, profileType)) + mergedFile, err := os.Create(mergedPath) + if err != nil { + return fmt.Errorf("cannot create %q: %w", mergedPath, err) + } + if err := merged.Write(mergedFile); err != nil { + mergedFile.Close() + os.Remove(mergedPath) + return fmt.Errorf("cannot write merged %s profile to %q: %w", profileType, mergedPath, err) + } + mergedFile.Close() + for _, profilePath := range profilePaths { + if err := os.Remove(profilePath); err != nil { + return fmt.Errorf("cannot remove %q: %w", profilePath, err) + } + } + t.Logf("%s profile was successfully written to: %v", strings.ToUpper(profileType), mergedPath) + + // Do post-processing on the merged profile. + for _, postProc := range postProcessors { + if err := postProc(ctx, t, run, profileType, mergedPath); err != nil { + return fmt.Errorf("failed to post-process profiles: %w", err) + } + } + return nil +} + +func processProfilingMetrics(ctx context.Context, t *testing.T, run *profileRun) error { + var metricsLogs []string + err := filepath.Walk(run.localProfileDir, func(path string, info fs.FileInfo, walkErr error) error { + switch { + case walkErr != nil: // Keep walking other directories, so don't propagate error here. + case info.IsDir(): + case !strings.HasSuffix(filepath.Base(path), ".metrics.log"): + default: + metricsLogs = append(metricsLogs, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("cannot traverse %q: %w", run.localProfileDir, err) + } + for _, metricsLog := range metricsLogs { + metricsviz.FromProfilingMetricsLogFile(ctx, t, metricsLog) + } + return nil +} + +// removeEmptyDirectories recursively removes empty dirs under `fromDir`. +func removeEmptyDirectories(fromDir string) error { + for keepGoing := true; keepGoing; { + var toRemove []string + err := filepath.Walk(fromDir, func(path string, info fs.FileInfo, walkErr error) error { + if walkErr != nil { + // Keep walking other directories, so return nil here. + return nil + } + if !info.IsDir() { + return nil + } + dirEntries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("cannot read directory %q: %w", path, err) + } + if len(dirEntries) == 0 { + toRemove = append(toRemove, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("cannot traverse %q: %w", fromDir, err) + } + keepGoing = len(toRemove) > 0 + for _, path := range toRemove { + if err := os.Remove(path); err != nil { + return fmt.Errorf("cannot remove %q: %w", path, err) + } + } + } + return nil +} diff --git a/test/kubernetes/benchmarks/pytorch_test.go b/test/kubernetes/benchmarks/pytorch_test.go new file mode 100644 index 000000000..4cb1ab5d2 --- /dev/null +++ b/test/kubernetes/benchmarks/pytorch_test.go @@ -0,0 +1,426 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pytorch_test contains benchmarks using the pytorch "torchbench" repo. +package pytorch_test + +// These tests use pytorch's "torchbench" suite (https://github.com/pytorch/benchmark/tree/main). +// The Authors describe the benchmarks in this paper: https://arxiv.org/pdf/2304.14226.pdf +// The Authors list both the type of model and its profile (how GPU intensive). + +// Note: The image for this test is about 7-8 GB as of writing. After you get your clusters up and +// running, start the test and make sure that the pods show the event of downloading the image. Then +// get a cup of coffee, chat with your co-workers for 5 min, and it will be about done 5 min after +// that. You'll only need to do this once for each cluster (in parallel). + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + + v13 "k8s.io/api/core/v1" +) + +const ( + pytorchImage = "gcr.io/gvisor-presubmit/benchmarks/pytorch_x86_64:f6f280aeb1b07989" +) + +// pytorchTestType is the method used, either training or evaluation, for the model. +type pytorchTestType string + +const ( + train = pytorchTestType("train") + eval = pytorchTestType("eval") +) + +type pytorchMode string + +// pytorchMode is the pytorch mode used, either script mode (jit) or eager mode. +// See: https://towardsdatascience.com/pytorch-jit-and-torchscript-c2a77bac0fff +const ( + jit = pytorchMode("jit") + eager = pytorchMode("eager") +) + +type pytorchTest struct { + module string + test pytorchTestType + mode pytorchMode +} + +// Name returns the name of the test with the argument parameters included. It is formatted so +// that it can be used for the name of the pod. +func (p pytorchTest) Name() string { + // Kubernetes pod names cannot contain "_". + module := strings.ReplaceAll(strings.ToLower(p.module), "_", "-") + return fmt.Sprintf("%s-%s-%s", module, p.test, p.mode) +} + +var snakeCase = regexp.MustCompile("_.") + +// BenchName returns the name of the test with the argument parameters included. +// It is formatted so that it can be used for benchstat output. +func (p pytorchTest) BenchName() string { + // First letter of the module should be capitalized, as it will be + // concatenated with "Benchmark" and it's useful to mark it as a different + // word. + // Some modules use a lowercase first letter, e.g. "fastNLP_Bert". + moduleName := strings.ToUpper(p.module[:1]) + p.module[1:] + // We also replace "snake_case" with "snakeCase". Sorry snakes. + moduleName = snakeCase.ReplaceAllStringFunc(moduleName, func(s string) string { + return strings.ToUpper(strings.TrimPrefix(s, "_")) + }) + test := strings.ToUpper(string(p.test)[:1]) + string(p.test[1:]) + var mode string + switch p.mode { + case eager: + mode = "Eager" + case jit: + mode = "JIT" + default: + panic(fmt.Sprintf("Unknown mode: %v", p.mode)) + } + return fmt.Sprintf("%s/%s/%s", moduleName, test, mode) +} + +func (p pytorchTest) toPod(namespace *testcluster.Namespace, image string) (*v13.Pod, error) { + pod := namespace.NewPod(p.Name()) + pod.Spec = v13.PodSpec{ + RestartPolicy: v13.RestartPolicyNever, + Containers: []v13.Container{ + { + Name: p.Name(), + Image: pytorchImage, + Command: benchmetric.TimedCommand(p.command()...), + }, + }, + } + return pod, nil +} + +func (p pytorchTest) command() []string { + return []string{ + "python", + "run.py", + p.module, + "--device", "cuda", + "--test", string(p.test), + "--mode", string(p.mode), + } +} + +// TestFastNLPBert uses the fastNLP_Bert module, which is classified as a NLP Language Model. +// fastNLP_Bert taxes the GPU heavily with low data movement. See Figure 2 on +// page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/fastNLP_Bert +// Bert Blog Post: https://towardsdatascience.com/bert-explained-state-of-the-art-language-model-for-nlp-f8b21a9b6270 +// Paper: https://arxiv.org/abs/1810.04805 +func TestFastNLPBert(t *testing.T) { + ctx := context.Background() + const module = "fastNLP_Bert" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +// TestBigBird uses the hf_BigBird module, which is classified as a NLP Language Model. +// hf_BigBird taxes the GPU moderately with low data movement. See Figure 2 on +// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/hf_BigBird +// Paper: https://arxiv.org/abs/2007.14062 +func TestBigBird(t *testing.T) { + ctx := context.Background() + const module = "hf_BigBird" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +// TestSpeechTransformer uses the speech_transformer module classified as "Speech Recognition" +// model. speech_transformer has a lot of idle time for the GPU. See Figure 2 on +// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/pull/374 +// Paper: https://arxiv.org/abs/1706.03762 +func TestSpeechTransformer(t *testing.T) { + ctx := context.Background() + const module = "speech_transformer" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +// TestLearningToPaint uses the LearningToPaint module classified as "neural renderer in model-based +// Deep Reinforcement Learning (DRL)". +// Learning to paint has a lot of "data movement" and doesn't tax the GPU a lot. See Figure 2 on +// page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/LearningToPaint +func TestLearningToPaint(t *testing.T) { + ctx := context.Background() + const module = "LearningToPaint" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: jit, + }, + { + module: module, + test: eval, + mode: jit, + }, + } + runTests(ctx, t, tests) +} + +// TestMobileNetV2 uses the mobilenet_v2 module classified as "Computer Vision: Image Classification". +// MobileNet has a lot of taxes the GPU. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/mobilenet_v2 +// Paper: https://paperswithcode.com/method/mobilenetv2 +func TestMobileNetV2(t *testing.T) { + ctx := context.Background() + const module = "mobilenet_v2" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: jit, + }, + { + module: module, + test: eval, + mode: jit, + }, + } + runTests(ctx, t, tests) +} + +// TestBackgroundMatting uses the Background_Matting module classified as "Computer Vision: Pattern Recognition". +// BackgroundMatting has a lot of GPU idle time. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/Background_Matting (see README) +func TestBackgroundMatting(t *testing.T) { + ctx := context.Background() + const module = "Background_Matting" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) { + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("PyTorch", func(t *testing.T) { + t.Parallel() + for _, p := range tests { + t.Run(p.Name(), func(t *testing.T) { + doPytorchRun(ctx, t, k8sCtx, cluster, p) + }) + } + }) + }) +} + +func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("Failed to reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + image, err := k8sCtx.ResolveImage(ctx, pytorchImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + pod, err := params.toPod(benchmarkNS, image) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to configure pod for test-nodepool: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, pod.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + if err := cluster.WaitForPodCompleted(ctx, pod); err != nil { + t.Fatalf("Failed to wait for pod to complete: %v", err) + } + + rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("GetLogReader on cluster %q pod %v: %v", cluster.GetName(), pod.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", pod.GetName(), err) + } + + metrics, err := parseStandardOutput(string(out)) + if err != nil { + t.Fatalf("parseStandardOutput: %v", err) + } + + containerDuration, err := benchmetric.ParseTimedContainerOutput(string(out)) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + + metrics = append(metrics, benchmetric.BenchmarkDuration(containerDuration)) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, params.BenchName(), metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } +} + +func parseStandardOutput(output string) ([]benchmetric.MetricValue, error) { + gpuTimeMillis, err := parseGPUTime(output) + if err != nil { + return nil, fmt.Errorf("parseGPUTime: %v", err) + } + + gpuPeakMemoryGB, err := parseGPUPeakMemoryGB(output) + if err != nil { + return nil, fmt.Errorf("parseGPUPeakMemory: %v", err) + } + + cpuPeakMemoryGB, err := parseCPUPeakMemoryGB(output) + if err != nil { + return nil, fmt.Errorf("parseCPUPeakMemory: %v", err) + } + + return []benchmetric.MetricValue{ + benchmetric.SpecificDuration(time.Duration(gpuTimeMillis)*time.Millisecond, "gpu-runtime"), + benchmetric.SpecificBytes(gpuPeakMemoryGB*1024*1024*1024, "gpu-peak-memory"), + benchmetric.SpecificBytes(cpuPeakMemoryGB*1024*1024*1024, "cpu-peak-memory"), + }, nil +} + +var gpuTimeRegex = regexp.MustCompile(`GPU\sTime:\s*(\d+\.\d+)\smilliseconds`) + +func parseGPUTime(output string) (float64, error) { + match := gpuTimeRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find GPU Time: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +var gpuPeakMemoryRegex = regexp.MustCompile(`GPU\s0\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) + +func parseGPUPeakMemoryGB(output string) (float64, error) { + match := gpuPeakMemoryRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find GPU Peak Memory: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +var cpuPeakMemoryRegex = regexp.MustCompile(`CPU\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) + +func parseCPUPeakMemoryGB(output string) (float64, error) { + match := cpuPeakMemoryRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find CPU Peak Memory: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestFastNLPBert": TestFastNLPBert, + "TestBigBird": TestBigBird, + "TestSpeechTransformer": TestSpeechTransformer, + "TestLearningToPaint": TestLearningToPaint, + "TestMobileNetV2": TestMobileNetV2, + "TestBackgroundMatting": TestBackgroundMatting, + }) +} diff --git a/test/kubernetes/benchmarks/redis_test.go b/test/kubernetes/benchmarks/redis_test.go new file mode 100644 index 000000000..17b682cc7 --- /dev/null +++ b/test/kubernetes/benchmarks/redis_test.go @@ -0,0 +1,474 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redis_test holds the redis test where the runtime under test runs a redis server and the +// native runtime runs a client making requests against it. +package redis_test + +import ( + "context" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + redisPort = 6379 + defaultRequestsPerConnection = 50000 + + redisImageAMD = "gcr.io/gvisor-presubmit/benchmarks/redis_x86_64:latest" + redisImageARM = "gcr.io/gvisor-presubmit/benchmarks/redis_aarch64:latest" + redisServerLabelKey = "app.kubernetes.io/name" + redisServerLabelValue = "redis-server" + redisVolumeName = "redis-data" + redisDataDirectory = "/redis-data" +) + +var ( + numConnections = []int{1, 2, 4, 8, 16, 32} + latencyPercentiles = []int{50, 95, 99} + operations = []string{"SET", "GET", "MSET", "LPUSH", "LRANGE_500"} +) + +// TestRedis benchmarks redis servers on k8s clusters. +func TestRedis(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("Redis", func(t *testing.T) { + t.Parallel() + doRedisTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// doRedisTest runs redis test. +func doRedisTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + persistentVol := benchmarkNS.GetPersistentVolume(redisVolumeName, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := redisImageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = redisImageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + for _, test := range []struct { + // Benchmark name + name string + // Suffix for the redis server, must be short to fit in pod name. + suffix string + // redis-server command-line. + serverCommand []string + // Volume to use for persistence, if any. + volume *v13.PersistentVolumeClaim + }{ + { + name: "Persistence", + suffix: "persist", + serverCommand: []string{ + "redis-server", + "--dir", redisDataDirectory, + // Default save settings per + // https://redis.io/docs/management/config-file/ + "--save", "3600 1 300 100 60 10000", + }, + volume: persistentVol, + }, + { + name: "NoPersistence", + suffix: "nopersist", + serverCommand: []string{ + "redis-server", + "--appendonly", "no", + "--save", "", + }, + volume: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a server on the runtime under test nodepool. + server := newRedisPodWithPort(benchmarkNS, fmt.Sprintf("redis-%s", test.suffix), image, test.serverCommand, redisPort, test.volume) + if server.ObjectMeta.Labels == nil { + server.ObjectMeta.Labels = make(map[string]string) + } + server.ObjectMeta.Labels[redisServerLabelKey] = redisServerLabelValue + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) + } + + server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) + } + + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) + } + + // create a service to service traffic to the pod. + service := newRedisService(benchmarkNS, server.GetName()) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeleteService(ctx, service) + if err := cluster.WaitForServiceReady(ctx, service); err != nil { + t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) + } + + ip := testcluster.GetIPFromService(service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + + // run the 'redis-cli' command to ping the server and make sure it is up. The "ping" request comes + // back with a "PONG" response. We repeat -r=5 times with a -i=1 second interval. If we + // get one PONG back then the server is considered up. + pingCmd := []string{"redis-cli", "-h", ip, "-r", "5", "-i", "1", "ping"} + ensureUp := func() error { + pinger := newRedisPod(benchmarkNS, fmt.Sprintf("rpinger-%s", test.suffix), image, pingCmd) + pinger, err = cluster.ConfigurePodForClientNodepool(pinger) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pinger.GetName(), err) + } + + pinger, err = cluster.CreatePod(ctx, pinger) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", pinger.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, pinger) + + waitCtx, waitCancel := context.WithTimeout(ctx, 30*time.Second) + var podWaitSuffix string + if err := cluster.WaitForPodCompleted(waitCtx, pinger); err != nil { + podWaitSuffix = fmt.Sprintf(" (pod wait error: %v)", err) + } + waitCancel() + + rdr, err := cluster.GetLogReader(ctx, pinger, v13.PodLogOptions{}) + if err != nil { + return fmt.Errorf("GetLogReader on cluster %q: %v%s", cluster.GetName(), err, podWaitSuffix) + } + out, err := io.ReadAll(rdr) + if err != nil { + return fmt.Errorf("failed to read from pod: %q: %v%s", pinger.GetName(), err, podWaitSuffix) + } + + if !strings.Contains(string(out), "PONG") { + return fmt.Errorf("mismatched output: wanted: PONG got: %q%s", string(out), podWaitSuffix) + } + + return nil + } + var isUpErr error + serverUpCtx, serverUpCancel := context.WithTimeout(ctx, 100*time.Second) + defer serverUpCancel() + for serverUpCtx.Err() == nil { + if isUpErr = ensureUp(); isUpErr == nil { + break + } + } + if isUpErr != nil { + t.Fatalf("%s at IP %s did not come up: %v", server.GetName(), ip, isUpErr) + } + + for _, connections := range numConnections { + t.Run(fmt.Sprintf("Connections_%d", connections), func(t *testing.T) { + for _, operation := range operations { + t.Run(operation, func(t *testing.T) { + // Create a client for this client run w/ the specified number of connections. + // Sadly the --csv mode only reports QPS, not latency. In order to report both, + // we need to parse the human-readable version of the output. + clientCmd := []string{ + "redis-benchmark", + "-t", operation, // RPC to benchmark + "-h", ip, // Redis server IP + "-n", fmt.Sprintf("%d", defaultRequestsPerConnection*connections), // Number of total requests to do + "-c", fmt.Sprintf("%d", connections), // Number of threads to spread them over. + "-r", "1000", // Key space size (larger = more memory faults) + "--precision", "4", // Floating-point precision for reporting latency (in ms) + } + client := newRedisPod(benchmarkNS, "client", image, clientCmd) + client, err = cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + client, err = cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, client) + + if err := cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + // get and parse the logs from the client to get the results + rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + + if err != nil { + t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + redisBenchmarkName := fmt.Sprintf("Redis/%s/%dClients/%s", test.name, connections, operation) + metrics, err := getMeasurements(string(out), operation) + if err != nil { + // Redis uses '\r' to update its status by overwriting the current line. + // If printed directly, this messes up the output. + // To make that clear, we replace '\r' with a literal + // backslash + 'r', and add a newline. + humanReadableOut := strings.ReplaceAll(string(out), "\r", "\\r\n") + t.Fatalf("failed to get metric for op %q: out:\n\n%s\n\nerr: %v", operation, humanReadableOut, err) + } + // We don't multiply `defaultRequestsPerConnection` by `connections` here + // because the number of "samples" we're testing is the number of times we + // can call an RPC from *that many connections* (which is part of the + // benchmark name). + // Adding 5x the number of connections does not make the sample size of this + // benchmark go 5x higher. + if err := recorder.RecordIters(ctx, redisBenchmarkName, defaultRequestsPerConnection, metrics...); err != nil { + t.Fatalf("Failed to record benchmark data for op %q: %v", operation, err) + } + }) + if t.Failed() { + break + } + } + }) + if t.Failed() { + break + } + } + }) + if t.Failed() { + break + } + } +} + +// newRedisService gets a service to serve traffic to the redis server. +func newRedisService(namespace *testcluster.Namespace, containerName string) *v13.Service { + name := fmt.Sprintf("redis-service-%d", time.Now().UnixNano()) + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{redisServerLabelKey: redisServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: redisPort, + TargetPort: intstr.FromString(containerName), + }, + }, + }) +} + +var ( + latencyPercentileRegex = regexp.MustCompile("^([-,.\\d]+)% <=? ([-,.\\d]+) milliseconds(?: \\(cumulative count .*\\))?$") + latencyStartHeader = "Latency by percentile distribution:" + queriesPerSecondRegex = regexp.MustCompile("^throughput summary: ([-,.\\d]+) requests per second$") +) + +func stringToFloat64(s string) float64 { + f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) + if err != nil { + panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) + } + return f +} + +// getMeasurements parses the output of redis-benchmark to get the stats. +func getMeasurements(out, operation string) ([]benchmetric.MetricValue, error) { + var currentOperation string + var returned []benchmetric.MetricValue + inLatencyBlock := false + foundPercentiles := make(map[int]bool, len(latencyPercentiles)) + foundQPS := false + lastPercentile := -1.0 + lastPercentileLatencyMs := math.NaN() + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + // The human-readable output contains a bunch of data like + // "OPERATION: number\r" which are used to update the result + // interactively. Strip them out here. + if strings.Contains(line, "\r") { + line = line[strings.LastIndex(line, "\r")+1:] + } + if strings.HasPrefix(line, "====== ") { + currentOperation = strings.SplitN(strings.Trim(line, "= "), " ", 2)[0] + } + if currentOperation != operation { + continue + } + if line == latencyStartHeader { + inLatencyBlock = true + continue + } + if inLatencyBlock { + latencyMatch := latencyPercentileRegex.FindStringSubmatch(line) + if latencyMatch != nil { + percentile := stringToFloat64(latencyMatch[1]) + if percentile < lastPercentile { + continue + } + latencyMs := stringToFloat64(latencyMatch[2]) + if percentile == 0 { + lastPercentile = 0 + lastPercentileLatencyMs = latencyMs + continue + } + // Look for all percentiles in `wantPercentiles` that are in the range [lastPercentile, + // percentile]. + var recordPercentiles []int + for _, wantPercentile := range latencyPercentiles { + if float64(wantPercentile) < lastPercentile { + continue + } + if float64(wantPercentile) > percentile { + continue + } + if foundPercentiles[wantPercentile] { + continue + } + recordPercentiles = append(recordPercentiles, wantPercentile) + } + for _, recordPercentile := range recordPercentiles { + // Linear interpolation of the latency value from within the latency range in the two + // percentile values that we got. + // For example, given p50=1.0ms and p70=2.0ms, we infer that p60=1.5ms. + // This isn't bulletproof but it is better than rounding to either end of the bucket. + rangeFraction := (float64(recordPercentile) - lastPercentile) / (percentile - lastPercentile) + pctileLatency := rangeFraction*(latencyMs-lastPercentileLatencyMs) + lastPercentileLatencyMs + returned = append(returned, benchmetric.SpecificDuration(time.Duration(pctileLatency*float64(time.Millisecond)), fmt.Sprintf("p%d", recordPercentile))) + foundPercentiles[recordPercentile] = true + } + // Update values for next round. + lastPercentile = percentile + lastPercentileLatencyMs = latencyMs + } else { + inLatencyBlock = false + } + continue + } + qpsMatch := queriesPerSecondRegex.FindStringSubmatch(line) + if qpsMatch != nil { + if foundQPS { + return nil, fmt.Errorf("found QPS value multiple times: %q", line) + } + foundQPS = true + returned = append(returned, benchmetric.RequestsPerSecond(stringToFloat64(qpsMatch[1]))) + } + } + if !foundQPS || len(foundPercentiles) != len(latencyPercentiles) { + return nil, fmt.Errorf("did not find the data we wanted: foundQPS=%v foundPercentiles=%v", foundQPS, foundPercentiles) + } + return returned, nil +} + +// newRedisPodWithPort returns a redis pod template. +func newRedisPodWithPort(namespace *testcluster.Namespace, name, image string, cmd []string, port int32, pvc *v13.PersistentVolumeClaim) *v13.Pod { + container := newRedisContainer(name, image, cmd) + container.Ports = append(container.Ports, v13.ContainerPort{Name: name, ContainerPort: port}) + if pvc != nil { + container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ + Name: redisVolumeName, + MountPath: redisDataDirectory, + }) + } + + pod := namespace.NewPod(name) + pod.Spec.Containers = []v13.Container{container} + if pvc != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ + Name: redisVolumeName, + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.GetName(), + }, + }, + }) + } + return pod +} + +// newRedisPod returns a redis pod template. +func newRedisPod(namespace *testcluster.Namespace, name, image string, cmd []string) *v13.Pod { + pod := namespace.NewPod(name) + pod.Spec.Containers = []v13.Container{newRedisContainer(name, image, cmd)} + return pod +} + +// newRedisContainer returns a new redis container. +func newRedisContainer(name, image string, cmd []string) v13.Container { + return v13.Container{ + Name: name, + Image: image, + Command: cmd, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestRedis": TestRedis, + }) +} diff --git a/test/kubernetes/benchmarks/resources/BUILD b/test/kubernetes/benchmarks/resources/BUILD new file mode 100644 index 000000000..6732ef220 --- /dev/null +++ b/test/kubernetes/benchmarks/resources/BUILD @@ -0,0 +1,13 @@ +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//test/kubernetes:__subpackages__"], + licenses = ["notice"], +) + +filegroup( + name = "files", + srcs = glob(["*"]), + visibility = [ + "//test/kubernetes/benchmarks:__subpackages__", + ], +) diff --git a/test/kubernetes/benchmarks/resources/chart.png b/test/kubernetes/benchmarks/resources/chart.png new file mode 100644 index 000000000..d2aebc143 Binary files /dev/null and b/test/kubernetes/benchmarks/resources/chart.png differ diff --git a/test/kubernetes/benchmarks/resources/gvisor.png b/test/kubernetes/benchmarks/resources/gvisor.png new file mode 100644 index 000000000..c1924213f Binary files /dev/null and b/test/kubernetes/benchmarks/resources/gvisor.png differ diff --git a/test/kubernetes/benchmarks/ruby_dev_test.go b/test/kubernetes/benchmarks/ruby_dev_test.go new file mode 100644 index 000000000..5546c0200 --- /dev/null +++ b/test/kubernetes/benchmarks/ruby_dev_test.go @@ -0,0 +1,225 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ruby_dev_test holds a benchmark to time a build job of a ruby application. +package ruby_dev_test + +import ( + "bytes" + "context" + "fmt" + "io" + "path" + "strings" + "testing" + + "gvisor.dev/gvisor/test/benchmarks/tools" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/rubydev_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/rubydev_aarch64:latest" + builderContainerName = "builder" +) + +// TestRubyDev benchmarks a build job on k8s clusters. +func TestRubyDev(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("RubyDev", func(t *testing.T) { + t.Parallel() + doRubyDevTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doRubyDevTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "ruby-dev" + + // create a persistent volume on which to store the code. + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + // create a new RubyDevPod and set it to run on the runtime under test nodepool. + pod := newRubyDevPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("failed to configure pod for test runtime node: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, builderContainerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("failed to set container resources: %v", err) + } + + if pod, err = cluster.CreatePod(ctx, pod); err != nil { + t.Fatalf("failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, builderContainerName) + if err != nil { + t.Fatalf("failed to get container duration: %v", err) + } + + reader, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer reader.Close() + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + + output := buf.String() + testTime, err := tools.ExtractRubyTestTime(output) + if err != nil { + t.Fatalf("ExtractRubyTestTime failed: %v", err) + } + loadTime, err := tools.ExtractRubyLoadTime(output) + if err != nil { + t.Fatalf("ExtractRubyLoadTime failed: %v", err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record(ctx, fmt.Sprintf("RubyDev/%s", test.name), + benchmetric.BenchmarkDuration(containerDuration), + benchmetric.SpecificDuration(testTime, "test"), + benchmetric.SpecificDuration(loadTime, "load"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newRubyDevPod creates a new ruby dev pod spec for benchmarks. +func newRubyDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + const fastlane = "/fastlane" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", fastlane, fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{"bash", "/files/run_fastlane_tests.sh"} + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: builderContainerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, fastlane), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestRubyDev": TestRubyDev, + }) +} diff --git a/test/kubernetes/benchmarks/stablediffusion_test.go b/test/kubernetes/benchmarks/stablediffusion_test.go new file mode 100644 index 000000000..8b288d15b --- /dev/null +++ b/test/kubernetes/benchmarks/stablediffusion_test.go @@ -0,0 +1,239 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stablediffusion_test + +import ( + "context" + "fmt" + "hash/fnv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/gpu/stablediffusion" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // Container image for Stable Diffusion XL. + stableDiffusionImage = "gcr.io/gvisor-presubmit/gpu/stable-diffusion-xl" +) + +func TestStableDiffusionXL(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("stable_diffusion_xl", func(t *testing.T) { + t.Parallel() + doStableDiffusionXLTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// kubernetesPodRunner implements `stablediffusion.ContainerRunner`. +type kubernetesPodRunner struct { + cluster *testcluster.TestCluster + namespace *testcluster.Namespace +} + +// Run implements `stablediffusion.ContainerRunner.Run`. +func (r *kubernetesPodRunner) Run(ctx context.Context, image string, argv []string) ([]byte, error) { + // Build pod spec. + const stableDiffusionXLPodName = "stable-diffusion-xl" + stableDiffusionXLPod := &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: stableDiffusionXLPodName, + Namespace: r.namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: stableDiffusionXLPodName, + Image: image, + Args: argv, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } + stableDiffusionXLPod, err := r.cluster.ConfigurePodForRuntimeTestNodepool(stableDiffusionXLPod) + if err != nil { + return nil, fmt.Errorf("failed to configure pod: %v", err) + } + stableDiffusionXLPod, err = testcluster.MaybeSetContainerResources(stableDiffusionXLPod, stableDiffusionXLPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + return nil, fmt.Errorf("failed to set container resources: %v", err) + } + + // Delete pod that may possibly exist from a previous iteration. + // Ignore errors since it most likely doesn't exist. + r.cluster.DeletePod(ctx, stableDiffusionXLPod) + + // Start new client pod and wait for it. + stableDiffusionXLPod, err = r.cluster.CreatePod(ctx, stableDiffusionXLPod) + if err != nil { + return nil, fmt.Errorf("failed to create stable diffusion XL pod: %v", err) + } + defer r.cluster.DeletePod(ctx, stableDiffusionXLPod) + if err := r.cluster.WaitForPodCompleted(ctx, stableDiffusionXLPod); err != nil { + logs, logsErr := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) + logs = strings.TrimSpace(logs) + if logsErr != nil { + return nil, fmt.Errorf("failed to run Stable Diffusion XL (%w) and to read logs from the pod: %v", err, logsErr) + } + if logs == "" { + return nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs are empty)", err) + } + return nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs: %v)", err, logs) + } + + // All good, get logs. + logs, err := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) + if err != nil { + return nil, fmt.Errorf("failed to read logs from pod %q: %v", stableDiffusionXLPod.GetName(), err) + } + return []byte(logs), nil +} + +// doStableDiffusionXLTest runs Stable Diffusion XL benchmarks for a single cluster. +func doStableDiffusionXLTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + imageName, err := k8sCtx.ResolveImage(ctx, stableDiffusionImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + xl := stablediffusion.NewXL(imageName, &kubernetesPodRunner{ + cluster: cluster, + namespace: benchmarkNS, + }) + + // The refiner model uses a lot of VRAM, and not all GPUs have enough of + // that to make it work. + // So we try each prompt without the refiner first. If it fails, then we + // don't try the same prompt with the refiner, as there is no way it will + // work. Similarly, if the benchmark does work without the refiner but + // does not work with the refiner, then future prompts will all have their + // refiner model attempt skipped. + refinerFailed := false + + for _, test := range []struct { + name string + query string + useRefiner bool + noiseFraction float64 + steps int + }{ + { + name: "BoringCorporateLogo", + query: `A boring flat corporate logo that says "gVisor"`, + useRefiner: true, + noiseFraction: 0.9, + steps: 32, + }, + { + name: "Androids", + query: "Photorealistic image of two androids playing chess aboard a spaceship", + useRefiner: true, + noiseFraction: 0.85, + steps: 64, + }, + } { + t.Run(test.name, func(t *testing.T) { + failedWithoutRefiner := false + for _, useRefiner := range []bool{false, true} { + t.Run(fmt.Sprintf("refiner=%t", useRefiner), func(t *testing.T) { + if useRefiner { + if failedWithoutRefiner { + t.Skipf("benchmark failed without refiner; skipping benchmark with refiner") + } + if refinerFailed { + t.Skipf("refiner failed in previous benchmark; skipping benchmark with refiner") + } + } + testCtx, testCancel := context.WithTimeout(ctx, 15*time.Minute) + defer testCancel() + prompt := &stablediffusion.XLPrompt{ + Query: test.query, + AllowCPUOffload: false, + NoiseFraction: test.noiseFraction, + Steps: test.steps, + Warm: true, + UseRefiner: useRefiner, + } + image, err := xl.Generate(testCtx, prompt) + if err != nil { + if useRefiner { + refinerFailed = true + t.Skipf("Failed to generate image with Refiner; will skip future attempts to run any prompt with the refiner.") + } + failedWithoutRefiner = true + t.Fatalf("Failed to generate images: %v", err) + } + ascii, err := image.ASCII() + if err != nil { + t.Fatalf("Failed to get ASCII: %v", err) + } + t.Logf("Generated image:\n\n%s\n", ascii) + hash := fnv.New32() + hash.Write([]byte(ascii)) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + metrics := []benchmetric.MetricValue{ + benchmetric.BenchmarkDuration(image.TotalDuration()), + benchmetric.SpecificDuration(image.ColdBaseDuration(), "base-cold"), + benchmetric.SpecificDuration(image.WarmBaseDuration(), "base-warm"), + } + if coldRefinerDuration := image.ColdRefinerDuration(); coldRefinerDuration >= 0 { + metrics = append(metrics, benchmetric.SpecificDuration(coldRefinerDuration, "refiner-cold")) + } + if warmRefinerDuration := image.WarmRefinerDuration(); warmRefinerDuration >= 0 { + metrics = append(metrics, benchmetric.SpecificDuration(warmRefinerDuration, "refiner-warm")) + } + // The image-hash metric should never change; it is still useful to + // report as a metric in order to detect instability across benchmark + // runs. + metrics = append(metrics, benchmetric.Checksum(hash, "image")) + if err := recorder.Record(ctx, fmt.Sprintf("StableDiffusionXL/%s/refiner=%t/steps=%d", test.name, useRefiner, test.steps), metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestStableDiffusionXL": TestStableDiffusionXL, + }) +} diff --git a/test/kubernetes/benchmarks/startup_test.go b/test/kubernetes/benchmarks/startup_test.go new file mode 100644 index 000000000..bcec1867f --- /dev/null +++ b/test/kubernetes/benchmarks/startup_test.go @@ -0,0 +1,132 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package startup_test benchmarks the time it takes for an empty alpine container to complete successfully. +package startup_test + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" +) + +const ( + benchName = "StartUp" +) + +var ( + command = []string{"/bin/sh", "-c", "echo hello"} +) + +// TestStartup benchmarks the time it takes for an empty alpine container to complete successfully. +// Note: WRT gVisor startup latency, this is not a meaningful benchmark. Startup time is dominated +// by Kubernetes control plane API calls and not actual container startups. This benchmark is +// provided for illustrative purposes only. +func TestStartup(t *testing.T) { + t.Logf("Warning: This is not a meaningful benchmark. Read the comments.") + + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run(benchName, func(t *testing.T) { + cluster := cluster + t.Parallel() + + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + podName := "startup" + image, err := k8sCtx.ResolveImage(ctx, "alpine") + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + p, err := cluster.ConfigurePodForRuntimeTestNodepool(benchmarkNS.NewAlpinePod(podName, image, command)) + if err != nil { + t.Fatalf("failed to set pod for test nodepool: %v", err) + } + + start := time.Now() + p, err = cluster.CreatePod(ctx, p) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, p) + if err := cluster.WaitForPodCompleted(ctx, p); err != nil { + t.Fatalf("Failed to wait for pod to complete: %v", err) + } + reader, err := cluster.GetLogReader(ctx, p, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer reader.Close() + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + if strings.TrimSpace(buf.String()) != "hello" { + t.Fatalf("Mistmatch output: got: %q want: %q", buf.String(), "hello") + } + + // For longer running containers, and where the desired duration to + // measure is the time it takes to run a command within a container, + // this should use `GetTimedContainerDuration` instead. + // However, since this benchmark's goal is to measure container runtime + // overhead, it uses the Kubernetes-level metrics for container + // duration. + containerDuration, err := cluster.ContainerDurationSecondsByName(ctx, p, p.GetName()) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + overallDuration := time.Since(start) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record(ctx, benchName, + benchmetric.BenchmarkDuration(overallDuration), + benchmetric.SpecificDuration(containerDuration, "container-runtime"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestStartup": TestStartup, + }) +} diff --git a/test/kubernetes/benchmarks/tensorflow_test.go b/test/kubernetes/benchmarks/tensorflow_test.go new file mode 100644 index 000000000..03dd6f175 --- /dev/null +++ b/test/kubernetes/benchmarks/tensorflow_test.go @@ -0,0 +1,170 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tensorflow_test + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/tensorflow_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/tensorflow_aarch64:latest" +) + +var workloads = map[string]string{ + "Kmeans": "2_BasicModels/kmeans.py", + "LogisticRegression": "2_BasicModels/logistic_regression.py", + "NearestNeighbor": "2_BasicModels/nearest_neighbor.py", + "RandomForest": "2_BasicModels/random_forest.py", + "ConvolutionalNetwork": "3_NeuralNetworks/convolutional_network.py", + "MultilayerPerceptron": "3_NeuralNetworks/multilayer_perceptron.py", + "NeuralNetwork": "3_NeuralNetworks/neural_network.py", +} + +func TestTensorflowOnCPU(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("TensorflowOnCPU", func(t *testing.T) { + t.Parallel() + doTensorflowOnCPU(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doTensorflowOnCPU(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "tensorflow" + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + workloadPaths := make([]string, 0, len(workloads)) + workloadPathToName := make(map[string]string, len(workloads)) + for name, path := range workloads { + workloadPaths = append(workloadPaths, path) + workloadPathToName[path] = name + } + sort.Strings(workloadPaths) + + var total time.Duration + for _, workloadPath := range workloadPaths { + workloadName := workloadPathToName[workloadPath] + t.Run(workloadName, func(t *testing.T) { + pod := newTensorflowOnCPUPod(benchmarkNS, name, image, workloadPath) + pod, err := cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("TensorflowOnCPU/%s", workloadName), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + total += containerDuration + }) + if t.Failed() { + break + } + } + if !t.Failed() { + if err := recorder.Record(ctx, "TensorflowOnCPU", benchmetric.BenchmarkDuration(total)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + } +} + +func newTensorflowOnCPUPod(namespace *testcluster.Namespace, name, image, workloadPath string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.TimedCommand("python", workloadPath), + WorkingDir: "/TensorFlow-Examples/examples", + Env: []v13.EnvVar{ + { + Name: "PYTHONPATH", + Value: "/TensorFlow-Examples/examples", + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestTensorflowOnCPU": TestTensorflowOnCPU, + }) +} diff --git a/test/kubernetes/benchmarks/wordpress_test.go b/test/kubernetes/benchmarks/wordpress_test.go new file mode 100644 index 000000000..dce15a78e --- /dev/null +++ b/test/kubernetes/benchmarks/wordpress_test.go @@ -0,0 +1,398 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wordpress_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + wordpressImage = "wordpress:6.2.0-php8.2-apache" + mariaDBImage = "mariadb:10.11.3-jammy" + wordpressPort = 80 + mariaDBPort = 3306 + wordpressBenchmarkDuration = 70 * time.Second + wordpressRequestTimeout = 10 * time.Second + wordpressLoginPage = "/wp-login.php" + mariaDBName = "wpbench" + mariaDBUser = "wpuser" + mariaDBPassword = "wppassword" + mariaDBRootPassword = "hunter2" + mariaDBVolumeName = "wpdata" + mariaDBVolumeDirectory = "/var/lib/mysql" + + wordpressServerLabelKey = "app.kubernetes.io/name" + wordpressServerLabelValue = "wordpress" + mariaDBServerLabelKey = "app.kubernetes.io/name" + mariaDBServerLabelValue = "mariadb" +) + +var ( + threads = []int{1, 8, 64, 1000} + targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} + wantPercentiles = []int{50, 95, 99} +) + +func TestWordpress(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("wordpress", func(t *testing.T) { + t.Parallel() + doWordpressTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doWordpressTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + // Create a persistent volume on which to store the database data. + dbVolume := benchmarkNS.GetPersistentVolume(mariaDBVolumeName, "30Gi") + dbVolume, err = cluster.CreatePersistentVolume(ctx, dbVolume) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, dbVolume) + + databaseName := "mariadb" + mariaDBImg, err := k8sCtx.ResolveImage(ctx, mariaDBImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + database := newMariaDBServer(benchmarkNS, databaseName, mariaDBImg, dbVolume) + database, err = cluster.ConfigurePodForTertiaryNodepool(database) + if err != nil { + t.Fatalf("Failed to configure pod for tertiary nodepool: %v", err) + } + database, err = cluster.CreatePod(ctx, database) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, database) + if err := cluster.WaitForPodRunning(ctx, database); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + databaseService := newMariaDBService(benchmarkNS, databaseName) + databaseService, err = cluster.CreateService(ctx, databaseService) + if err != nil { + t.Fatalf("Failed to create database service: %v", err) + } + defer cluster.DeleteService(ctx, databaseService) + mariaDBIP := testcluster.GetIPFromService(databaseService) + + name := "wordpress" + wordpressImg, err := k8sCtx.ResolveImage(ctx, wordpressImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + server := newWordpressServer(benchmarkNS, name, wordpressImg, mariaDBIP) + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + + service := newWordpressService(benchmarkNS, name) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, service) + wordpressIP := testcluster.GetIPFromService(service) + + // Install WordPress. + installWordpressPod := newWordpressInstall(benchmarkNS, "install-wordpress", wordpressIP) + installWordpressPod, err = cluster.ConfigurePodForClientNodepool(installWordpressPod) + if err != nil { + t.Fatalf("Failed to configure pod for client nodepool: %v", err) + } + installWordpressPod, err = cluster.CreatePod(ctx, installWordpressPod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, installWordpressPod) + if err := cluster.WaitForPodCompleted(ctx, installWordpressPod); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + cluster.DeletePod(ctx, installWordpressPod) + + var rounds []httpbench.Round + for _, numThreads := range threads { + for _, qps := range targetQPS { + if qps < numThreads { + continue + } + onlyReport := []httpbench.MetricType{httpbench.RequestsPerSecond} + // If we're testing at max QPS, don't report latency, + // because all requests will hit the timeout. + // Otherwise, only report latency, because the throughput + // is exactly determined by the QPS target anyway. + if qps != httpbench.InfiniteQPS { + onlyReport = append(onlyReport, httpbench.Latency) + } + rounds = append(rounds, httpbench.Round{ + NumThreads: numThreads, + TargetQPS: qps, + Duration: wordpressBenchmarkDuration, + OnlyReport: onlyReport, + }) + } + } + benchmark := &httpbench.HTTPBenchmark{ + Name: "wordpress", + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: wordpressPort, + Path: wordpressLoginPage, + Rounds: rounds, + Timeout: wordpressRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) +} + +func newMariaDBServer(namespace *testcluster.Namespace, name, image string, volume *v13.PersistentVolumeClaim) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{mariaDBServerLabelKey: mariaDBServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: mariaDBPort, + }, + }, + Env: []v13.EnvVar{ + { + Name: "MARIADB_ROOT_PASSWORD", + Value: mariaDBRootPassword, + }, + { + Name: "MARIADB_DATABASE", + Value: mariaDBName, + }, + { + Name: "MARIADB_USER", + Value: mariaDBUser, + }, + { + Name: "MARIADB_PASSWORD", + Value: mariaDBPassword, + }, + }, + VolumeMounts: []v13.VolumeMount{{ + Name: volume.GetName(), + MountPath: mariaDBVolumeDirectory, + }}, + }, + }, + Volumes: []v13.Volume{{ + Name: volume.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: volume.GetName(), + }, + }, + }}, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newMariaDBService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{mariaDBServerLabelKey: mariaDBServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: mariaDBPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func newWordpressServer(namespace *testcluster.Namespace, name, image, mariaDBHost string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{wordpressServerLabelKey: wordpressServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: wordpressPort, + }, + }, + Env: []v13.EnvVar{ + { + Name: "WORDPRESS_DB_HOST", + Value: mariaDBHost, + }, + { + Name: "WORDPRESS_DB_USER", + Value: mariaDBUser, + }, + { + Name: "WORDPRESS_DB_PASSWORD", + Value: mariaDBPassword, + }, + { + Name: "WORDPRESS_DB_NAME", + Value: mariaDBName, + }, + { + Name: "WORDPRESS_TABLE_PREFIX", + Value: "wp_", + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newWordpressService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{wordpressServerLabelKey: wordpressServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: wordpressPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func newWordpressInstall(namespace *testcluster.Namespace, name, wpHost string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: "install-wordpress", + Image: "debian:latest", + // This command installs WordPress through the web UI. + // Source of the parameters: + // https://github.com/GoogleCloudPlatform/click-to-deploy/blob/master/k8s/wordpress/chart/wordpress/templates/wordpress-configmap.yaml + Command: []string{ + "sh", "-c", + strings.Join([]string{ + "apt-get update -y " { + if strings.ContainsRune(cmd, c) { + hasAnySpecialCharacter = true + break + } + } + if !hasAnySpecialCharacter { + return cmd + } + for _, c := range "\\\"$`" { + cmd = strings.ReplaceAll(cmd, string(c), fmt.Sprintf("\\%s", string(c))) + } + return fmt.Sprintf(`"%s"`, cmd) +} + +// TimedCommand takes in a command-line and computes a high-fidelity +// duration of how long it took. A pod using this command-line should +// have its duration measured using GetTimedContainerDuration. +// This is more reliable than looking at Kubernetes-tracked metrics +// for container start/end times, because Kubernetes only tracks these +// with second-level granularity, and because they include the overhead +// of starting/stopping a container. For benchmarks that want to measure +// the time of a particular command, TimedCommand provides better +// precision (nanosecond resolution) and accuracy (only measure the +// duration of the command, not the container runtime overhead). +// This must run in a container that has either `sh` or `bash` +// installed. +func TimedCommand(argv ...string) []string { + escapedCmd := make([]string, len(argv)) + for i, arg := range argv { + escapedCmd[i] = shellEscape(arg) + } + escapedCmd = append(escapedCmd, "||", "echo", shellEscape(timingDataLineFailure)) + nowNanos := strings.ReplaceAll(nowNanosShellFunc, "\n", " ") + for strings.Contains(nowNanos, " ") { + nowNanos = strings.ReplaceAll(nowNanos, " ", " ") + } + innerCommand := strings.Join([]string{ + // Define the now_nanos function. + nowNanos, + // Get the timestamp before the command. + `before="$(now_nanos)"`, + // Run the command. + strings.Join(escapedCmd, " "), + // Get the timestamp after the command. + `after="$(now_nanos)"`, + // Print out the before/after timestamps. + // We cannot use shsprintf here because we *want* to be able to + // use variables that shsprintf considers to "go out of scope". + // This is safe because all of the strings below are completely + // static. + // We use this weird string concatenation to avoid triggering + // the linter. + fmt.Sprintf(`ec`+`ho "%s"`, strings.Join([]string{ + timingDataLinePrefix, + "${before}", + "${after}", + }, timingDataSeparator)), + }, "; ") + + return []string{ + // We can't assume that bash is installed, but we also can't + // do conditionals in the top-level command, so just spawn sh + // and we'll check if bash exists within that. + "sh", + "-c", + fmt.Sprintf( + // Use bash if possible, otherwise use sh. + "if hash bash > /dev/null 2>/dev/null; then bash -c %s; else sh -c %s; fi", + shellEscape(innerCommand), + shellEscape(innerCommand), + ), + } +} + +// CommandThenTimed returns a command-line that runs a given command as +// initialization, then cd's into the given directory, then runs another +// command there under TimedCommand. +// When using GetTimedContainerDuration, only the duration of the +// `timedCmd` command will be measured. +// If `cd` is empty, no directory change happens. +func CommandThenTimed(initCmd []string, cd string, timedCmd []string) []string { + escapedInitCmd := make([]string, len(initCmd)) + for i, arg := range initCmd { + escapedInitCmd[i] = shellEscape(arg) + } + timedCmd = TimedCommand(timedCmd...) + escapedTimedCmd := make([]string, len(timedCmd)) + for i, arg := range timedCmd { + escapedTimedCmd[i] = shellEscape(arg) + } + commands := make([]string, 0, 3) + commands = append(commands, strings.Join(escapedInitCmd, " ")) + if cd != "" { + commands = append(commands, fmt.Sprintf("c"+"d %s", shellEscape(cd))) + } + commands = append(commands, strings.Join(escapedTimedCmd, " ")) + return []string{ + "sh", + "-c", + strings.Join(commands, " && "), + } +} + +// GetTimedContainerDuration waits for the given pod to exit, then parses its +// output and looks for duration information as expected from a command-line +// generated using `TimedCommand`. +func GetTimedContainerDuration(ctx context.Context, c *testcluster.TestCluster, pod *v13.Pod, containerName string) (time.Duration, error) { + if err := c.WaitForPodCompleted(ctx, pod); err != nil { + return 0, fmt.Errorf("failed to wait for pod to complete: %v", err) + } + rdr, err := c.GetLogReader(ctx, pod, v13.PodLogOptions{ + Container: containerName, + }) + if err != nil { + return 0, fmt.Errorf("GetLogReader on cluster %q pod %v: %v", c.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return 0, fmt.Errorf("failed to read from pod: %q: %v", pod.GetName(), err) + } + return ParseTimedContainerOutput(string(out)) +} + +// ParseTimedContainerOutput parses the output of a TimedContainer. +func ParseTimedContainerOutput(out string) (time.Duration, error) { + found := false + var duration time.Duration + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, timingDataLinePrefix) { + continue + } + if line == timingDataLineFailure { + return 0, fmt.Errorf("command failed; output: %s", out) + } + if found { + return 0, fmt.Errorf("output has multiple lines that look like duration information: %s", out) + } + data := strings.Split(line, timingDataSeparator) + if len(data) != 4 { + return 0, fmt.Errorf("malformed timing duration data line: %q", line) + } + beforeString, afterString := data[2], data[3] + beforeNanos, err := strconv.ParseInt(beforeString, 10, 64) + if err != nil { + return 0, fmt.Errorf("malformed timing duration data line %q: %v", line, err) + } + beforeTime := time.Unix(beforeNanos/1e9, beforeNanos%1e9) + afterNanos, err := strconv.ParseInt(afterString, 10, 64) + if err != nil { + return 0, fmt.Errorf("malformed timing duration data line %q: %v", line, err) + } + afterTime := time.Unix(afterNanos/1e9, afterNanos%1e9) + duration = afterTime.Sub(beforeTime) + if duration <= 0 { + return 0, fmt.Errorf("duration is zero or negative: got before=%d (%v) / after=%d (%v)", beforeNanos, beforeTime, afterNanos, afterTime) + } + found = true + } + if !found { + return 0, fmt.Errorf("output did not contain duration information: %s", out) + } + return duration, nil +} diff --git a/test/kubernetes/k8sctx/BUILD b/test/kubernetes/k8sctx/BUILD new file mode 100644 index 000000000..7f7769618 --- /dev/null +++ b/test/kubernetes/k8sctx/BUILD @@ -0,0 +1,25 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "k8sctx", + srcs = [ + "k8sctx.go", + "k8sctx_impl.go", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + "//runsc/flag", + "//test/kubernetes:test_range_config_go_proto", + "//test/kubernetes/testcluster", + "//tools/gvisor_k8s_tool/provider/kubectl", + "@googleapis//google/container/v1:container_go_proto", + "@org_golang_google_protobuf//encoding/prototext:go_default_library", + ], +) diff --git a/test/kubernetes/k8sctx/k8sctx.go b/test/kubernetes/k8sctx/k8sctx.go new file mode 100644 index 000000000..554bd705c --- /dev/null +++ b/test/kubernetes/k8sctx/k8sctx.go @@ -0,0 +1,111 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package k8sctx is used to manage the lifecycle of a Kubernetes test or +// benchmark running in one or more Kubernetes clusters. +// It is used to control the behavior of Kubernetes-based tests and benchmarks +// at runtime and to abstract away how the Kubernetes test clusters are +// created and managed from the test code. +package k8sctx + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/testcluster" +) + +// KubernetesContext represents the Kubernetes execution context. +// It is used to keep track of available Kubernetes clusters to test on. +// Tests are expected to call `RegisterTest` for every of their test function, +// then `TestMain`. +type KubernetesContext interface { + // TestMain should be called inside tests' `TestMain` function, after having + // registered all tests with `RegisterTest`. + TestMain(m *testing.M) + + // RegisterTest registers a test. + // It should be called for every `Test*(*testing.T)` function in the test. + // Note that the `k8sctx.TestMain` helper function below will call this for + // you given a map of tests. + RegisterTest(name string, fn TestFunc) + + // AcquireCluster returns a single cluster for the test or benchmark to use. + // The cluster is guaranteed to not be in use by other tests or benchmarks + // until the `ReleaseCluster` method is called. + // This method should block if there are no available clusters. + AcquireCluster(ctx context.Context, t *testing.T) *testcluster.TestCluster + + // ReleaseCluster unlocks the given cluster for use by other tests or + // benchmarks. + ReleaseCluster(ctx context.Context, t *testing.T, cluster *testcluster.TestCluster) + + // ForEachCluster reserves as many test clusters as are available, calls + // `fn` on each of them, and releases each of them when `fn` finishes. + ForEachCluster(ctx context.Context, t *testing.T, fn func(cluster *testcluster.TestCluster)) + + // ResolveImage resolves a container image name (possibly with a label) + // to a fully-qualified image name. It can also return an `image:label` + // string if the Kubernetes cluster the test runs in will resolve it on + // its own. + ResolveImage(ctx context.Context, imageName string) (string, error) +} + +// TestFunc is a test function that is expected to call `Context` and run a +// test or benchmark within a Kubernetes context. +type TestFunc func(t *testing.T) + +var ( + kubernetesCtxMu sync.Mutex + kubernetesCtxOnce sync.Once + kubernetesCtxFn func(context.Context) (KubernetesContext, error) + kubernetesCtx KubernetesContext + kubernetesCtxErr error +) + +// Context gets the global Kubernetes context. +// It must be called after SetContext has already been called. +func Context(ctx context.Context) (KubernetesContext, error) { + kubernetesCtxMu.Lock() + defer kubernetesCtxMu.Unlock() + if kubernetesCtxFn == nil { + return nil, errors.New("k8sctx.Context called prior to k8sctx.SetContextConstructor") + } + kubernetesCtxOnce.Do(func() { + kubernetesCtx, kubernetesCtxErr = kubernetesCtxFn(ctx) + }) + return kubernetesCtx, kubernetesCtxErr +} + +// SetContextConstructor sets the global Kubernetes context constructor. +func SetContextConstructor(fn func(context.Context) (KubernetesContext, error)) { + kubernetesCtxMu.Lock() + defer kubernetesCtxMu.Unlock() + kubernetesCtxFn = fn +} + +// TestMain is a helper to write the TestMain function of tests. +func TestMain(m *testing.M, testFuncs map[string]TestFunc) { + k8sCtx, err := Context(context.Background()) + if err != nil { + panic(fmt.Sprintf("failed to get k8sctx: %v", err)) + } + for name, fn := range testFuncs { + k8sCtx.RegisterTest(name, fn) + } + k8sCtx.TestMain(m) +} diff --git a/test/kubernetes/k8sctx/k8sctx_impl.go b/test/kubernetes/k8sctx/k8sctx_impl.go new file mode 100644 index 000000000..72e9e3a3e --- /dev/null +++ b/test/kubernetes/k8sctx/k8sctx_impl.go @@ -0,0 +1,99 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !false +// +build !false + +package k8sctx + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + + cspb "cloud.google.com/go/container/apiv1/containerpb" + "google.golang.org/protobuf/encoding/prototext" + "gvisor.dev/gvisor/runsc/flag" + testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl" +) + +var ( + kubectlContextName = flag.String("kubectl-context-name", "", "Name of the kubectl context to use within the kubectl config") + clusterProtoPath = flag.String("cluter-proto-path", "", "Path to a `google.container.v1.Cluster` textproto file") + testNodepoolRuntime = flag.String("test-nodepool-runtime", "", "if set, override the runtime used for pods scheduled on the 'test' nodepool. If unset, the nodepool default is used") +) + +// kubectlContext implements KubernetesContext using a named `kubectl` context +// from the user's kubectl config. +type kubectlContext struct { + cluster *testcluster.TestCluster +} + +func newKubectlContext(ctx context.Context) (KubernetesContext, error) { + if *kubectlContextName == "" { + return nil, errors.New("no kubectl context name specified") + } + if *clusterProtoPath == "" { + return nil, errors.New("no cluster proto path specified") + } + cluster, err := kubectl.NewCluster(*kubectlContextName) + if err != nil { + return nil, fmt.Errorf("cannot initialize cluster %q: %w", *kubectlContextName, err) + } + var clusterPB cspb.Cluster + clusterBytes, err := os.ReadFile(*clusterProtoPath) + if err != nil { + return nil, fmt.Errorf("cannot read cluster textproto file %q: %w", *clusterProtoPath, err) + } + if err = prototext.Unmarshal(clusterBytes, &clusterPB); err != nil { + return nil, fmt.Errorf("cannot unmarshal cluster textproto file %q: %w", *clusterProtoPath, err) + } + testCluster := testcluster.NewTestClusterWithClient(&testpb.Cluster{Cluster: &clusterPB}, cluster.Client()) + if *testNodepoolRuntime != "" { + testCluster.OverrideTestNodepoolRuntime(testcluster.RuntimeType(*testNodepoolRuntime)) + } + return &kubectlContext{cluster: testCluster}, nil +} + +func (c *kubectlContext) AcquireCluster(ctx context.Context, t *testing.T) *testcluster.TestCluster { + return c.cluster +} + +func (c *kubectlContext) ReleaseCluster(ctx context.Context, t *testing.T, cluster *testcluster.TestCluster) { + // Nothing to do. +} + +func (c *kubectlContext) ForEachCluster(ctx context.Context, t *testing.T, fn func(cluster *testcluster.TestCluster)) { + fn(c.cluster) +} + +func (c *kubectlContext) ResolveImage(ctx context.Context, imageName string) (string, error) { + return imageName, nil +} + +func (c *kubectlContext) RegisterTest(name string, fn TestFunc) { + // Nothing to do here, we use the regular testing library. +} + +func (c *kubectlContext) TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func init() { + SetContextConstructor(newKubectlContext) +} diff --git a/test/kubernetes/test_range_config.proto b/test/kubernetes/test_range_config.proto new file mode 100644 index 000000000..0a36e8b57 --- /dev/null +++ b/test/kubernetes/test_range_config.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package k8s_tester.test_cluster_config; + +import "google/container/v1/cluster_service.proto"; + +// TestRangeSpec is a description of the test environment to be created. It is +// the input of the setup step which creates the required clusters. +message TestRangeSpec { + // Name for clusters. This name will be used as a template for all created + // clusters (e.g. my-cluster will have clusters named something like + // "my-cluster-0" with an ascending index). Cluster names are limited to + // 40 charecters, so names will be truncated to fit this constraint. + string name = 1; + + // A nodepool built with the runtime under test. + google.container.v1.NodePool test_runtime = 2; + + // clients is another nodepool in the cluster to use against the test_runtime. + // For example, in most client-server tests, the runtime under test is the + // server and the clients are the client. Clients always use the default + // runtime runc. + google.container.v1.NodePool clients = 3; + + // tertiary is a third nodepool in the cluster, used by some benchmarks that + // need it for isolation. + // For example, the WordPress benchmark needs to run the MySQL database on + // a separate machine in order to force network traffic to flow across the + // host's non-local network stack for a fair comparison between runsc/runc. + // The tertiary nodepool may use gVisor or runc as a runtime, depending on + // user configuration. + google.container.v1.NodePool tertiary = 8; + + // versions are the GKE patch versions to use for the clusters. The number + // of clusters created will be num(versions) * replicas. + repeated string versions = 4; + + // zones are the availability zones in which to create clusters. clusters will + // be created in each zone in a round robin fashion until the requested amount + // of clusters is created. This is provided to expand quota. + // Note: please check that given zones actual have resources available (e.g + // ARM machines are not available in all zones). + repeated string zones = 5; + + // project is the project under which clusters should be created. + string project = 6; + + // project service account to use to create clusters. + string gke_service_account = 7; +} + +// TestRange contains the created clusters. This is an output from the setup +// phase and an input for the test phase. +message TestRange { + repeated Cluster clusters = 1; +} + +// Cluster holds the created cluster and its credential file. +message Cluster { + // Created Cluster proto. + google.container.v1.Cluster cluster = 1; + + // The setup step will create individual credential files for each created + // cluster. + string credential_file = 2; +} diff --git a/test/kubernetes/testcluster/BUILD b/test/kubernetes/testcluster/BUILD new file mode 100644 index 000000000..b49dc8aa8 --- /dev/null +++ b/test/kubernetes/testcluster/BUILD @@ -0,0 +1,31 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "testcluster", + srcs = [ + "objects.go", + "testcluster.go", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + "//test/kubernetes:test_range_config_go_proto", + "@googleapis//google/container/v1:container_go_proto", + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/fields:go_default_library", + "@io_k8s_apimachinery//pkg/watch:go_default_library", + "@io_k8s_client_go//kubernetes:go_default_library", + "@io_k8s_client_go//tools/clientcmd:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + "@org_golang_x_sync//errgroup:go_default_library", + ], +) diff --git a/test/kubernetes/testcluster/objects.go b/test/kubernetes/testcluster/objects.go new file mode 100644 index 000000000..9c63d6f46 --- /dev/null +++ b/test/kubernetes/testcluster/objects.go @@ -0,0 +1,335 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcluster + +import ( + "context" + "fmt" + "strconv" + + cspb "cloud.google.com/go/container/apiv1/containerpb" + "google.golang.org/protobuf/proto" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + pod = "Pod" + apiV1 = "v1" +) + +// Namespace represents a Kubernetes object namespace. +// It can contain pods or services or other Kubernetes objects. +// It is useful in tests that create multiple objects and want to ensure +// their mutual destruction, as well as for re-running tests and ensuring +// that the objects from past runs are cleaned up properly on the next run. +type Namespace struct { + testCluster *TestCluster + Namespace string +} + +// Namespace returns a new namespace in this cluster. +func (t *TestCluster) Namespace(namespace string) *Namespace { + return &Namespace{ + testCluster: t, + Namespace: namespace, + } +} + +// Reset deletes this namespace if it exists, and unconditionally +// creates a new namespace of this name. +// This should be used in the beginning of tests, such that the namespace +// is empty and ready to be used. +func (n *Namespace) Reset(ctx context.Context) error { + n.Cleanup(ctx) + _, err := n.testCluster.createNamespace(ctx, &v13.Namespace{ + TypeMeta: v1.TypeMeta{ + Kind: "namespace", + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: n.Namespace, + }, + }) + return err +} + +// Cleanup deletes this namespace if it exists. +func (n *Namespace) Cleanup(ctx context.Context) error { + return n.testCluster.deleteNamespace(ctx, n.Namespace) +} + +// NewAlpinePod returns an alpine pod template. +func (n *Namespace) NewAlpinePod(name, image string, cmd []string) *v13.Pod { + container := v13.Container{Name: name, Image: image, Command: cmd} + pod := n.NewPod(name) + pod.Spec.Containers = []v13.Container{container} + return pod +} + +// NewPod returns a pod template. +func (n *Namespace) NewPod(name string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: pod, + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: n.Namespace, + }, + Spec: v13.PodSpec{ + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// GetPersistentVolume gets a persistent volume spec for benchmarks. +func (n *Namespace) GetPersistentVolume(name, size string) *v13.PersistentVolumeClaim { + return &v13.PersistentVolumeClaim{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: n.Namespace, + }, + Spec: v13.PersistentVolumeClaimSpec{ + AccessModes: []v13.PersistentVolumeAccessMode{v13.ReadWriteOnce}, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceStorage: resource.MustParse(size), + }, + }, + }, + } +} + +// GetService gets a service spec for benchmarks. +func (n *Namespace) GetService(name string, spec v13.ServiceSpec) *v13.Service { + return &v13.Service{ + TypeMeta: v1.TypeMeta{ + Kind: "Service", + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: n.Namespace, + }, + Spec: spec, + } +} + +// ContainerResourcesRequest holds arguments to set requested resource on a container. +type ContainerResourcesRequest struct { + CPUResources string // CPUResources to request. Note: Will be overridden by flag above. + MemoryResources string // MemoryResources to request. Note: Will be overridden by flag above. + GPU bool +} + +// MaybeSetContainerResources sets container resources if flags are given. Sets both the resource +// limits and requests as container runtimes honor them differently. +func MaybeSetContainerResources(pod *v13.Pod, containerName string, requests ContainerResourcesRequest) (*v13.Pod, error) { + resourceList := v13.ResourceList{} + if requests.CPUResources != "" { + resourceList[v13.ResourceCPU] = resource.MustParse(requests.CPUResources) + } + if requests.MemoryResources != "" { + resourceList[v13.ResourceMemory] = resource.MustParse(requests.MemoryResources) + } + + if requests.GPU { + acceleratorCount, ok := pod.Spec.NodeSelector[NodepoolNumAcceleratorsKey] + if !ok { + return nil, fmt.Errorf("cannot determine number of accelerators that the pod should use, make sure to call ConfigurePodForRuntimeTestNodepool first") + } + resourceList[v13.ResourceName("nvidia.com/gpu")] = resource.MustParse(acceleratorCount) + } + + requirements := v13.ResourceRequirements{ + Limits: resourceList, + Requests: resourceList, + } + + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == containerName { + pod.Spec.Containers[i].Resources = requirements + return pod, nil + } + } + return nil, fmt.Errorf("container %q not found", containerName) +} + +// RuntimeType is a supported runtime for the test nodepool. +type RuntimeType string + +// List of known runtime types. +const ( + RuntimeTypeGVisor = RuntimeType("gvisor") + RuntimeTypeUnsandboxed = RuntimeType("runc") + RuntimeTypeGVisorNvidia = RuntimeType("gvisor-nvidia") + RuntimeTypeGVisorTPU = RuntimeType("gvisor-tpu") + RuntimeTypeUnsandboxedNvidia = RuntimeType("runc-nvidia") + RuntimeTypeUnsandboxedTPU = RuntimeType("runc-tpu") +) + +// ApplyNodepool modifies the nodepool to configure it to use the runtime. +func (t RuntimeType) ApplyNodepool(nodepool *cspb.NodePool, accelType AcceleratorType, accelShape string, accelRes string) { + if nodepool.GetConfig().GetLabels() == nil { + nodepool.GetConfig().Labels = map[string]string{} + } + + switch t { + case RuntimeTypeGVisor: + nodepool.Config.SandboxConfig = &cspb.SandboxConfig{ + Type: cspb.SandboxConfig_GVISOR, + } + nodepool.GetConfig().Labels[NodepoolRuntimeKey] = string(RuntimeTypeGVisor) + case RuntimeTypeUnsandboxed: + nodepool.GetConfig().Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxed) + // Do nothing. + case RuntimeTypeGVisorNvidia: + nodepool.Config.SandboxConfig = &cspb.SandboxConfig{ + Type: cspb.SandboxConfig_GVISOR, + } + accelCount, err := strconv.Atoi(accelShape) + if err != nil { + panic(fmt.Sprintf("GPU count must be a valid number, got %v", accelShape)) + } + if accelCount == 0 { + panic("GPU count needs to be >=1") + } + nodepool.Config.MachineType = DefaultNvidiaMachineType + nodepool.Config.Accelerators = []*cspb.AcceleratorConfig{ + { + AcceleratorType: string(accelType), + AcceleratorCount: int64(accelCount), + }, + } + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeGVisorNvidia) + nodepool.Config.Labels[NodepoolNumAcceleratorsKey] = strconv.Itoa(accelCount) + case RuntimeTypeGVisorTPU: + nodepool.Config.MachineType = TPUAcceleratorMachineTypeMap[accelType] + nodepool.PlacementPolicy = &cspb.NodePool_PlacementPolicy{ + TpuTopology: accelShape, + Type: cspb.NodePool_PlacementPolicy_COMPACT, + } + + nodepool.Config.Labels[gvisorNodepoolKey] = gvisorRuntimeClass + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeGVisorTPU) + nodepool.Config.Labels[NodepoolTPUTopologyKey] = accelShape + nodepool.Config.Taints = append(nodepool.Config.Taints, &cspb.NodeTaint{ + Key: gvisorNodepoolKey, + Value: gvisorRuntimeClass, + Effect: cspb.NodeTaint_NO_SCHEDULE, + }) + case RuntimeTypeUnsandboxedNvidia: + accelCount, err := strconv.Atoi(accelShape) + if err != nil { + panic(fmt.Sprintf("GPU count must be a valid number, got %v", accelShape)) + } + if accelCount == 0 { + panic("GPU count needs to be >=1") + } + nodepool.Config.MachineType = DefaultNvidiaMachineType + nodepool.Config.Accelerators = []*cspb.AcceleratorConfig{ + { + AcceleratorType: string(accelType), + AcceleratorCount: int64(accelCount), + }, + } + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxedNvidia) + nodepool.Config.Labels[NodepoolNumAcceleratorsKey] = strconv.Itoa(accelCount) + case RuntimeTypeUnsandboxedTPU: + nodepool.Config.MachineType = TPUAcceleratorMachineTypeMap[accelType] + nodepool.PlacementPolicy = &cspb.NodePool_PlacementPolicy{ + TpuTopology: accelShape, + Type: cspb.NodePool_PlacementPolicy_COMPACT, + } + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxedTPU) + nodepool.Config.Labels[NodepoolTPUTopologyKey] = accelShape + default: + panic(fmt.Sprintf("unsupported runtime %q", t)) + } + if accelRes != "" { + nodepool.Config.ReservationAffinity = &cspb.ReservationAffinity{ + ConsumeReservationType: cspb.ReservationAffinity_SPECIFIC_RESERVATION, + Key: "compute.googleapis.com/reservation-name", + Values: []string{accelRes}, + } + } +} + +// ApplyPodSpec modifies a PodSpec to use this runtime. +func (t RuntimeType) ApplyPodSpec(podSpec *v13.PodSpec) { + switch t { + case RuntimeTypeGVisor: + podSpec.RuntimeClassName = proto.String(gvisorRuntimeClass) + podSpec.NodeSelector[NodepoolRuntimeKey] = string(RuntimeTypeGVisor) + case RuntimeTypeUnsandboxed: + // Allow the pod to schedule on gVisor nodes as well. + // This enables the use of `--test-nodepool-runtime=runc` to run + // unsandboxed benchmarks on gVisor test clusters. + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Effect: v13.TaintEffectNoSchedule, + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + }) + case RuntimeTypeGVisorNvidia: + podSpec.RuntimeClassName = proto.String(gvisorRuntimeClass) + podSpec.NodeSelector[NodepoolRuntimeKey] = string(RuntimeTypeGVisorNvidia) + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "nvidia.com/gpu", + Operator: v13.TolerationOpExists, + }) + case RuntimeTypeGVisorTPU: + podSpec.RuntimeClassName = proto.String(gvisorRuntimeClass) + podSpec.NodeSelector[NodepoolRuntimeKey] = string(RuntimeTypeGVisorTPU) + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "google.com/tpu", + Operator: v13.TolerationOpExists, + }) + case RuntimeTypeUnsandboxedNvidia: + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "nvidia.com/gpu", + Operator: v13.TolerationOpExists, + }) + // Allow the pod to schedule on gVisor nodes as well. + // This enables the use of `--test-nodepool-runtime=runc-nvidia` to run + // unsandboxed benchmarks on gVisor test clusters. + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Effect: v13.TaintEffectNoSchedule, + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + }) + case RuntimeTypeUnsandboxedTPU: + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "google.com/tpu", + Operator: v13.TolerationOpExists, + }) + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Effect: v13.TaintEffectNoSchedule, + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + }) + default: + panic(fmt.Sprintf("unsupported runtime %q", t)) + } +} diff --git a/test/kubernetes/testcluster/testcluster.go b/test/kubernetes/testcluster/testcluster.go new file mode 100644 index 000000000..ace25e600 --- /dev/null +++ b/test/kubernetes/testcluster/testcluster.go @@ -0,0 +1,791 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testcluster wraps the Kubernetes library for common test operations. +// It also provides a TestCluster abstraction for interacting with clusters. +package testcluster + +import ( + "context" + "fmt" + "io" + "strconv" + "strings" + "time" + + cspb "cloud.google.com/go/container/apiv1/containerpb" + "golang.org/x/sync/errgroup" + testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto" + appsv1 "k8s.io/api/apps/v1" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + // archKey is given to nodepools to mark their architecture. Used here to mark ARM nodepools. + archKey = "kubernetes.io/arch" + // armValue marks an ARM nodepool. + armValue = "arm64" + + // k8sApp is used as a label to distinguish between applications. + k8sApp = "k8s-app" +) + +// Common namespace names. +const ( + // NamespaceDefault is the default namespace for Kubernetes. + NamespaceDefault = v13.NamespaceDefault + + // NamespaceSanity is used for objects that are part of sanity checks. + NamespaceSanity = "sanity" + + // NamespaceBenchmark is used for objects that are part of benchmarks. + NamespaceBenchmark = "benchmark" +) + +// Nodepool names. +const ( + // TestRuntimeNodepoolName is the value that marks a "test-runtime-nodepool", or a nodepool where + // w/ the runtime under test. + TestRuntimeNodepoolName = "test-runtime-nodepool" + // ClientNodepoolName is the value that marks a client nodepool. Usually this is a plain GKE + // nodepool + ClientNodepoolName = "client-nodepool" + // TertiaryNodepoolName is the value that marks the tertiary nodepool. + // This could either be a plain GKE nodepool or could be gVisor-enabled, + // as configured during test range creation. + TertiaryNodepoolName = "tertiary-nodepool" +) + +// Nodepool keys. +const ( + // NodePoolTypeKey is the key to mark a nodepool as a "test-runtime-nodepool" or a "client-nodepool" + NodePoolTypeKey = "nodepool-type" + // NodepoolRuntimeKey is the key to mark the runtime used by a nodepool. + NodepoolRuntimeKey = "runtime" + // NodepoolNumAcceleratorsKey is the key to mark the number of accelerators in a nodepool. + NodepoolNumAcceleratorsKey = "num-accelerators" + // NodepoolTPUTopologyKey is the key to mark the TPU topology used by a nodepool. + NodepoolTPUTopologyKey = "tpu-topology" + // Name of the nodepool key used in Pod.Spec.NodeSelector. + NodePoolSelectorKey = "cloud.google.com/gke-nodepool" + // Name of the TPU accelerator key used in Pod.Spec.NodeSelector. + NodepoolTPUAcceleratorSelectorKey = "cloud.google.com/gke-tpu-accelerator" + // Name of the TPU topology key used in Pod.Spec.NodeSelector. + NodepoolTPUTopologySelectorKey = "cloud.google.com/gke-tpu-topology" +) + +// Default machine types. +var ( + // DefaultMachineType is the default machine type to use for specs and create-default. + DefaultMachineType = "n2-standard-4" + // DefaultNvidiaMachineType is the default machine type for nvidia. + DefaultNvidiaMachineType = "n1-standard-4" + // TPUAcceleratorMachineTypeMap maps TPU types to the machine type to use. + TPUAcceleratorMachineTypeMap = map[AcceleratorType]string{ + AcceleratorTypeV4PodTPU: "ct4p-hightpu-4t", + } +) + +// GKE Sandbox gVisor runtime. +const ( + // gvisorNodepoolKey the key for the label given to GKE Sandbox nodepools. + gvisorNodepoolKey = "sandbox.gke.io/runtime" + // gvisorRuntimeClass the runtimeClassName used for GKE Sandbox pods. + gvisorRuntimeClass = "gvisor" +) + +// AcceleratorType is the gpu type to be used. +type AcceleratorType string + +// List of supported GPUs. +const ( + AcceleratorTypeTeslaT4GPU = AcceleratorType("nvidia-tesla-t4") + AcceleratorTypeA100GPU = AcceleratorType("nvidia-tesla-a100") + AcceleratorTypeL4GPU = AcceleratorType("nvidia-tesla-l4") + AcceleratorTypeV4PodTPU = AcceleratorType("tpu-v4-pod") +) + +// TestCluster wraps clusters with their individual ClientSets so that helper methods can be called. +type TestCluster struct { + cluster *testpb.Cluster + client kubernetes.Interface + + // testNodepoolRuntimeOverride, if set, overrides the runtime used for pods + // running on the test nodepool. If unset, the test nodepool's default + // runtime is used. + testNodepoolRuntimeOverride RuntimeType +} + +// NewTestCluster returns a new TestCluster client. +func NewTestCluster(cluster *testpb.Cluster) (*TestCluster, error) { + config, err := clientcmd.BuildConfigFromFlags("" /*masterURL*/, cluster.GetCredentialFile()) + if err != nil { + return nil, fmt.Errorf("BuildConfigFromFlags: %w", err) + } + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("kubernetes.NewForConfig: %w", err) + } + return NewTestClusterWithClient(cluster, client), nil +} + +// NewTestClusterWithClient returns a new TestCluster client with a given client. +func NewTestClusterWithClient(cluster *testpb.Cluster, client kubernetes.Interface) *TestCluster { + return &TestCluster{ + cluster: cluster, + client: client, + testNodepoolRuntimeOverride: "", + } +} + +// Cluster returns the underlying cluster proto for tests. +func (t *TestCluster) Cluster() *testpb.Cluster { + return t.cluster +} + +// GetName returns this cluster's name. +func (t *TestCluster) GetName() string { + return t.cluster.GetCluster().GetName() +} + +// GetGVisorRuntimeLabelMap returns the gVisor runtime key-value pair used +// on gVisor-runtime-enabled nodes. +func (t *TestCluster) GetGVisorRuntimeLabelMap() map[string]string { + return map[string]string{ + gvisorNodepoolKey: gvisorRuntimeClass, + } +} + +// GetGVisorRuntimeToleration returns a pod scheduling toleration that +// allows the pod to schedule on gVisor-runtime-enabled nodes. +func (t *TestCluster) GetGVisorRuntimeToleration() v13.Toleration { + return v13.Toleration{ + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + Effect: v13.TaintEffectNoSchedule, + } +} + +// OverrideTestNodepoolRuntime overrides the runtime used for pods running on +// the test nodepool. If unset, the test nodepool's default runtime is used. +func (t *TestCluster) OverrideTestNodepoolRuntime(testRuntime RuntimeType) { + t.testNodepoolRuntimeOverride = testRuntime +} + +// createNamespace creates a namespace. +func (t *TestCluster) createNamespace(ctx context.Context, namespace *v13.Namespace) (*v13.Namespace, error) { + return t.client.CoreV1().Namespaces().Create(ctx, namespace, v1.CreateOptions{}) +} + +// getNamespace returns the given namespace in the cluster if it exists. +func (t *TestCluster) getNamespace(ctx context.Context, namespaceName string) (*v13.Namespace, error) { + return t.client.CoreV1().Namespaces().Get(ctx, namespaceName, v1.GetOptions{}) +} + +// deleteNamespace is a helper method to delete a namespace. +func (t *TestCluster) deleteNamespace(ctx context.Context, namespaceName string) error { + if err := t.client.CoreV1().Namespaces().Delete(ctx, namespaceName, v1.DeleteOptions{}); err != nil { + return err + } + // Wait for the namespace to disappear or for the context to expire. + for ctx.Err() == nil { + if _, err := t.getNamespace(ctx, namespaceName); err != nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + return ctx.Err() +} + +// ListNodes is a helper method to list nodes in a cluster. +func (t *TestCluster) ListNodes(ctx context.Context) (*v13.NodeList, error) { + return t.client.CoreV1().Nodes().List(ctx, v1.ListOptions{}) +} + +// HasGVisorTestRuntime returns whether the test nodes in this cluster +// use the gVisor runtime. +func (t *TestCluster) HasGVisorTestRuntime(ctx context.Context) (bool, error) { + nodes, err := t.ListNodes(ctx) + if err != nil { + return false, fmt.Errorf("cannot list nodes: %w", err) + } + var foundRuntime RuntimeType + for _, n := range nodes.Items { + if n.Labels[NodePoolTypeKey] != TestRuntimeNodepoolName { + continue + } + nodeRuntime := RuntimeType(n.Labels[NodepoolRuntimeKey]) + if nodeRuntime == "" { + return false, fmt.Errorf("node %q has no runtime label", n.GetName()) + } + if foundRuntime == "" { + foundRuntime = nodeRuntime + continue + } + if nodeRuntime != foundRuntime { + return false, fmt.Errorf("found conflicting runtimes in the same cluster: %q vs %q", foundRuntime, nodeRuntime) + } + } + return foundRuntime == RuntimeTypeGVisor || foundRuntime == RuntimeTypeGVisorNvidia, nil +} + +// CreatePod is a helper to create a pod. +func (t *TestCluster) CreatePod(ctx context.Context, pod *v13.Pod) (*v13.Pod, error) { + if pod.GetObjectMeta().GetNamespace() == "" { + pod.SetNamespace(NamespaceDefault) + } + return t.client.CoreV1().Pods(pod.GetNamespace()).Create(ctx, pod, v1.CreateOptions{}) +} + +// GetPod is a helper method to Get a pod's metadata. +func (t *TestCluster) GetPod(ctx context.Context, pod *v13.Pod) (*v13.Pod, error) { + return t.client.CoreV1().Pods(pod.GetNamespace()).Get(ctx, pod.GetName(), v1.GetOptions{}) +} + +// ListPods is a helper method to List pods in a cluster. +func (t *TestCluster) ListPods(ctx context.Context, namespace string) (*v13.PodList, error) { + return t.client.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{}) +} + +// DeletePod is a helper method to delete a pod. +func (t *TestCluster) DeletePod(ctx context.Context, pod *v13.Pod) error { + if err := t.client.CoreV1().Pods(pod.GetNamespace()).Delete(ctx, pod.GetName(), v1.DeleteOptions{}); err != nil { + return err + } + // Wait for the pod to disappear or for the context to expire. + for ctx.Err() == nil { + if _, err := t.GetPod(ctx, pod); err != nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + return ctx.Err() +} + +// GetLogReader gets an io.ReadCloser from which logs can be read. It is the caller's +// responsibility to close it. +func (t *TestCluster) GetLogReader(ctx context.Context, pod *v13.Pod, opts v13.PodLogOptions) (io.ReadCloser, error) { + return t.client.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &opts).Stream(ctx) +} + +// ReadPodLogs reads logs from a pod. +func (t *TestCluster) ReadPodLogs(ctx context.Context, pod *v13.Pod) (string, error) { + rdr, err := t.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + return "", fmt.Errorf("GetLogReader on cluster %q pod %q: %v", t.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return "", fmt.Errorf("failed to read from pod %q: %v", pod.GetName(), err) + } + return string(out), nil +} + +// WaitForPodRunning is a helper method to wait for a pod to be running. +func (t *TestCluster) WaitForPodRunning(ctx context.Context, pod *v13.Pod) error { + return t.doWaitForPod(ctx, pod, v13.PodRunning) +} + +// WaitForPodCompleted is a helper method to wait for a pod to be completed. +func (t *TestCluster) WaitForPodCompleted(ctx context.Context, pod *v13.Pod) error { + return t.doWaitForPod(ctx, pod, v13.PodSucceeded) +} + +// doWaitForPod waits for a pod to complete based on a given v13.PodPhase. +func (t *TestCluster) doWaitForPod(ctx context.Context, pod *v13.Pod, phase v13.PodPhase) error { + w, err := t.client.CoreV1().Pods(pod.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: pod.GetName()}).String(), + }) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + + var p *v13.Pod + for { + select { + case <-ctx.Done(): + return ctx.Err() + case e := <-w.ResultChan(): + var ok bool + p, ok = e.Object.(*v13.Pod) + if !ok { + return fmt.Errorf("invalid object watched: %T", p) + } + case <-time.After(10 * time.Second): + p, err = t.GetPod(ctx, pod) + if err != nil { + return fmt.Errorf("failed to poll pod: %w", err) + } + } + if ctx.Err() != nil { + return fmt.Errorf("context expired waiting for pod %q failed: %s", pod.GetName(), ctx.Err()) + } + if pod.Status.Reason == v13.PodReasonUnschedulable { + return fmt.Errorf("pod %q failed: reason: %q message: %q", pod.GetName(), pod.Status.Reason, pod.Status.Message) + } + + for _, c := range p.Status.Conditions { + if strings.Contains(c.Reason, "Unschedulable") { + return fmt.Errorf("pod %q failed: reason: %q message: %q", pod.GetName(), c.Reason, c.Message) + } + } + + switch p.Status.Phase { + case v13.PodFailed: + return fmt.Errorf("pod %q failed: %s", pod.GetName(), p.Status.Message) + case phase: + return nil + } + } +} + +// RuntimeTestNodepoolIsARM returns true if the runtime undertest nodepool is an ARM nodepool. +func (t *TestCluster) RuntimeTestNodepoolIsARM() bool { + np, err := t.getNodePoolByName(TestRuntimeNodepoolName) + if err != nil { + return false + } + return strings.HasPrefix(np.GetConfig().GetMachineType(), "t2a") +} + +// configureDaemonSetForNodepool configures the DaemonSet to run on a given nodepool. +func (t *TestCluster) configureDaemonSetForNodepool(ds *appsv1.DaemonSet, nodepoolName string) error { + np, err := t.getNodePoolByName(nodepoolName) + if err != nil { + return err + } + if ds.Labels == nil { + ds.Labels = make(map[string]string) + } + return t.applyCommonPodConfigurations(np, &ds.Spec.Template.Spec) +} + +// configurePodForNodepool configures the pod to run on a given nodepool. +func (t *TestCluster) configurePodForNodepool(pod *v13.Pod, nodepoolName string) (*v13.Pod, error) { + np, err := t.getNodePoolByName(nodepoolName) + if err != nil { + return nil, err + } + if pod.Labels == nil { + pod.Labels = make(map[string]string) + } + if err := t.applyCommonPodConfigurations(np, &pod.Spec); err != nil { + return nil, err + } + return pod, nil +} + +// ConfigureDaemonSetForRuntimeTestNodepool configures the DaemonSet to run +// on the test runtime. +func (t *TestCluster) ConfigureDaemonSetForRuntimeTestNodepool(ds *appsv1.DaemonSet) error { + return t.configureDaemonSetForNodepool(ds, TestRuntimeNodepoolName) +} + +// ConfigurePodForRuntimeTestNodepool configures the pod to run on the test runtime. +func (t *TestCluster) ConfigurePodForRuntimeTestNodepool(pod *v13.Pod) (*v13.Pod, error) { + return t.configurePodForNodepool(pod, TestRuntimeNodepoolName) +} + +// ConfigurePodForClientNodepool configures the pod to run on the client +// nodepool. +func (t *TestCluster) ConfigurePodForClientNodepool(pod *v13.Pod) (*v13.Pod, error) { + return t.configurePodForNodepool(pod, ClientNodepoolName) +} + +// ConfigurePodForTertiaryNodepool configures the pod to run on the tertiary +// nodepool. +func (t *TestCluster) ConfigurePodForTertiaryNodepool(pod *v13.Pod) (*v13.Pod, error) { + return t.configurePodForNodepool(pod, TertiaryNodepoolName) +} + +func (t *TestCluster) getNodePoolByName(name string) (*cspb.NodePool, error) { + for _, np := range t.cluster.GetCluster().GetNodePools() { + if np.GetName() == name { + return np, nil + } + } + return nil, fmt.Errorf("failed to find nodepool %q: %+v", name, t.cluster.GetCluster().GetNodePools()) +} + +func (t *TestCluster) applyCommonPodConfigurations(np *cspb.NodePool, podSpec *v13.PodSpec) error { + // Apply GKE Sandbox configurations if the nodepool is a GKE Sandbox nodepool. + if podSpec.NodeSelector == nil { + podSpec.NodeSelector = make(map[string]string) + } + + np.GetConfig().GetLabels()[NodePoolTypeKey] = np.GetName() + + // Force the pod to run on this nodepool. + podSpec.NodeSelector[NodePoolSelectorKey] = np.GetName() + + // Figure out which runtime to use for this pod, either by flag override or + // autodetection based on the nodepool configuration. + var applyRuntime = RuntimeTypeUnsandboxed + if np.GetName() == TestRuntimeNodepoolName && t.testNodepoolRuntimeOverride != "" { + applyRuntime = t.testNodepoolRuntimeOverride + } else if nodePoolRuntime, ok := np.GetConfig().GetLabels()[NodepoolRuntimeKey]; ok { + applyRuntime = RuntimeType(nodePoolRuntime) + } + + // Apply the runtime we've chosen, whether by override or autodetection. + applyRuntime.ApplyPodSpec(podSpec) + + // If the nodepool has accelerators, copy the number of them as a node + // selector option. + // This doesn't really constrain the pod further, but allows + // this number to be carried over when setting pod resources. + if len(np.GetConfig().GetAccelerators()) > 0 { + totalAccels := 0 + for _, accelCfg := range np.GetConfig().GetAccelerators() { + totalAccels += int(accelCfg.GetAcceleratorCount()) + } + if accelCount, ok := np.GetConfig().GetLabels()[NodepoolNumAcceleratorsKey]; !ok || accelCount != strconv.Itoa(totalAccels) { + return fmt.Errorf("unexpected %s=%q label on nodepool with %d total accelerators", NodepoolNumAcceleratorsKey, accelCount, totalAccels) + } + podSpec.NodeSelector[NodepoolNumAcceleratorsKey] = strconv.Itoa(totalAccels) + } else { + for accelType, machineType := range TPUAcceleratorMachineTypeMap { + if machineType == np.GetConfig().GetMachineType() { + topology, ok := np.GetConfig().GetLabels()[NodepoolTPUTopologyKey] + if !ok { + return fmt.Errorf("unexpected %s=%q label on nodepool with no accelerators", NodepoolTPUTopologyKey, topology) + } + podSpec.NodeSelector[NodepoolTPUAcceleratorSelectorKey] = string(accelType) + podSpec.NodeSelector[NodepoolTPUTopologySelectorKey] = np.GetConfig().GetLabels()[NodepoolTPUTopologyKey] + } + } + } + + // If the nodepool is an ARM nodepool, apply ARM tolerations. + for key, val := range np.GetConfig().GetLabels() { + if key == archKey && val == armValue { + podSpec.NodeSelector[archKey] = armValue + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: archKey, + Value: armValue, + Operator: v13.TolerationOpEqual, + Effect: v13.TaintEffectNoSchedule, + }) + } + } + return nil +} + +// ContainerDurationSecondsByName gets the runtime of a container reported by the kubelet by name. +// The kubelet reports runtime at second granularity. +func (t *TestCluster) ContainerDurationSecondsByName(ctx context.Context, pod *v13.Pod, containerName string) (time.Duration, error) { + p, err := t.GetPod(ctx, pod) + if err != nil { + return 0, fmt.Errorf("GetPod: %w", err) + } + for _, c := range p.Status.ContainerStatuses { + fmt.Println(c.Name) + if c.Name != containerName { + continue + } + if c.State.Terminated == nil { + return 0, fmt.Errorf("failed to get runtime seconds: terminated is nil: %+v", c.State) + } + start := c.State.Terminated.StartedAt + end := c.State.Terminated.FinishedAt + result := end.Unix() - start.Unix() + if result < 0 { + return 0, fmt.Errorf("invalid result %d: %+v", result, c.State) + } + return time.Duration(result) * time.Second, nil + } + return 0, fmt.Errorf("container %q not found: %+v", containerName, pod.Status.ContainerStatuses) +} + +// CreateService is a helper method to create a service in a cluster. +func (t *TestCluster) CreateService(ctx context.Context, service *v13.Service) (*v13.Service, error) { + return t.client.CoreV1().Services(service.GetNamespace()).Create(ctx, service, v1.CreateOptions{}) +} + +// ListServices is a helper method to List services in a cluster. +func (t *TestCluster) ListServices(ctx context.Context, namespace string) (*v13.ServiceList, error) { + return t.client.CoreV1().Services(namespace).List(ctx, v1.ListOptions{}) +} + +// DeleteService is a helper to delete a given service. +func (t *TestCluster) DeleteService(ctx context.Context, service *v13.Service) error { + if err := t.client.CoreV1().Services(service.GetNamespace()).Delete(ctx, service.GetName(), v1.DeleteOptions{}); err != nil { + return err + } + // Wait for the service to disappear or for the context to expire. + for ctx.Err() == nil { + if _, err := t.client.CoreV1().Services(service.GetNamespace()).Get(ctx, service.GetName(), v1.GetOptions{}); err != nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + return ctx.Err() +} + +// WaitForServiceReady waits until a service is ready. +func (t *TestCluster) WaitForServiceReady(ctx context.Context, service *v13.Service) error { + w, err := t.client.CoreV1().Services(service.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: service.GetName()}).String(), + }) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case e := <-w.ResultChan(): + s, ok := e.Object.(*v13.Service) + if !ok { + return fmt.Errorf("invalid object watched: %T", s) + } + if e.Type == watch.Added { + return nil + } + } + } +} + +// GetIPFromService returns the IP on a service. +func GetIPFromService(service *v13.Service) string { + return service.Spec.ClusterIP +} + +// CreatePersistentVolume creates a persistent volume. +func (t *TestCluster) CreatePersistentVolume(ctx context.Context, volume *v13.PersistentVolumeClaim) (*v13.PersistentVolumeClaim, error) { + if volume.GetObjectMeta().GetNamespace() == "" { + volume.SetNamespace(NamespaceDefault) + } + return t.client.CoreV1().PersistentVolumeClaims(volume.GetNamespace()).Create(ctx, volume, v1.CreateOptions{}) +} + +// DeletePersistentVolume deletes a persistent volume. +func (t *TestCluster) DeletePersistentVolume(ctx context.Context, volume *v13.PersistentVolumeClaim) error { + return t.client.CoreV1().PersistentVolumeClaims(volume.GetNamespace()).Delete(ctx, volume.GetName(), v1.DeleteOptions{}) +} + +// CreateDaemonset creates a daemonset with default options. +func (t *TestCluster) CreateDaemonset(ctx context.Context, ds *appsv1.DaemonSet) (*appsv1.DaemonSet, error) { + if ds.GetObjectMeta().GetNamespace() == "" { + ds.SetNamespace(NamespaceDefault) + } + return t.client.AppsV1().DaemonSets(ds.GetNamespace()).Create(ctx, ds, v1.CreateOptions{}) +} + +// DeleteDaemonset deletes a daemonset from this cluster. +func (t *TestCluster) DeleteDaemonset(ctx context.Context, ds *appsv1.DaemonSet) error { + return t.client.AppsV1().DaemonSets(ds.GetNamespace()).Delete(ctx, ds.GetName(), v1.DeleteOptions{}) +} + +// GetPodsInDaemonSet returns the list of pods of the given DaemonSet. +func (t *TestCluster) GetPodsInDaemonSet(ctx context.Context, ds *appsv1.DaemonSet) ([]v13.Pod, error) { + listOptions := v1.ListOptions{} + if appLabel, found := ds.Spec.Template.Labels[k8sApp]; found { + listOptions.LabelSelector = fmt.Sprintf("%s=%s", k8sApp, appLabel) + } + pods, err := t.client.CoreV1().Pods(ds.ObjectMeta.Namespace).List(ctx, listOptions) + if err != nil { + return nil, err + } + var dsPods []v13.Pod + for _, pod := range pods.Items { + if !strings.HasPrefix(pod.Name, ds.ObjectMeta.Name) { + continue // Not part of the DaemonSet. + } + dsPods = append(dsPods, pod) + } + return dsPods, nil +} + +// WaitForDaemonset waits until a daemonset has propagated containers across the affected nodes. +func (t *TestCluster) WaitForDaemonset(ctx context.Context, ds *appsv1.DaemonSet) error { + w, err := t.client.AppsV1().DaemonSets(ds.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: ds.ObjectMeta.Name}).String(), + }) + + if err != nil { + return fmt.Errorf("failed to watch daemon: %v", err) + } + defer w.Stop() + var lastDS *appsv1.DaemonSet + + for daemonSetReady := false; !daemonSetReady; { + select { + case <-ctx.Done(): + if lastDS != nil { + return fmt.Errorf("context canceled before healthy; last DaemonSet status: %#v", lastDS.Status) + } + return fmt.Errorf("context canceled before healthy") + case e, ok := <-w.ResultChan(): + d, ok := e.Object.(*appsv1.DaemonSet) + if !ok { + return fmt.Errorf("invalid object type: %T", d) + } + lastDS = d + if d.Status.NumberReady == d.Status.DesiredNumberScheduled && d.Status.DesiredNumberScheduled > 0 && d.Status.NumberUnavailable == 0 { + daemonSetReady = true + } + } + } + + // Now wait for the pods to be running. + for ctx.Err() == nil { + pods, err := t.GetPodsInDaemonSet(ctx, ds) + if err != nil { + return fmt.Errorf("failed to get pods in daemonset: %v", err) + } + if len(pods) == 0 { + return fmt.Errorf("DaemonSet has no pods: %v", lastDS) + } + allOK := true + for _, pod := range pods { + switch pod.Status.Phase { + case v13.PodRunning, v13.PodSucceeded: + // OK, do nothing. + default: + allOK = false + } + } + if allOK { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } + return nil +} + +// StreamDaemonSetLogs streams the contents of a container from the given +// DaemonSet. The callback function is called once per node that the DaemonSet +// schedules on, with the reader corresponding to that node. The callback +// function is expected to close the reader. +// StreamDaemonSetLogs returns once the DaemonSet is ready everywhere that +// it is meant to be scheduled. +func (t *TestCluster) StreamDaemonSetLogs(ctx context.Context, ds *appsv1.DaemonSet, opts v13.PodLogOptions, fn func(context.Context, v13.Pod, io.ReadCloser) error) error { + errGroup, groupCtx := errgroup.WithContext(ctx) + nodesSeen := make(map[string]struct{}) + nodesErr := make(map[string]error) + + // refreshPods queries all matching pods in the cluster and starts new + // log streams for every pod that schedules on a node we haven't seen yet. + refreshPods := func() error { + pods, err := t.GetPodsInDaemonSet(ctx, ds) + if err != nil { + return err + } + for _, pod := range pods { + pod := pod + if pod.Spec.NodeName == "" { + continue // No node assigned yet. + } + if _, seen := nodesSeen[pod.Spec.NodeName]; seen { + continue // Node already seen. + } + logReader, err := t.client.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &opts).Stream(ctx) + if err != nil { + // This can happen if the container hasn't run yet, for example + // because other init containers that run earlier are still executing. + // We retain this error in `nodesErr` but clear it if it later becomes + // OK for this node. + nodesErr[pod.Spec.NodeName] = fmt.Errorf("failed to stream logs from pod %q/%q on node %q: %v", pod.GetNamespace(), pod.GetName(), pod.Spec.NodeName, err) + continue + } + nodesSeen[pod.Spec.NodeName] = struct{}{} + nodesErr[pod.Spec.NodeName] = nil + errGroup.Go(func() error { + return fn(groupCtx, pod, logReader) + }) + } + return nil + } + + // Watch the DaemonSet. + // We'll periodically refresh pods: either when the DaemonSet changes + // state, or periodically every second. + dsWatch, err := t.client.AppsV1().DaemonSets(ds.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: ds.ObjectMeta.Name}).String(), + }) + if err != nil { + return fmt.Errorf("failed to watch DaemonSet: %v", err) + } + timeTicker := time.NewTicker(time.Second) + defer timeTicker.Stop() + defer dsWatch.Stop() + + // Iterate and stop once the DaemonSet is fully Ready. + var loopError error + var lastDS *appsv1.DaemonSet +Outer: + for { + select { + case <-ctx.Done(): + if lastDS != nil { + loopError = fmt.Errorf("context canceled before healthy; last status: %#v", lastDS.Status) + } else { + loopError = fmt.Errorf("context canceled before healthy") + } + break Outer + case <-timeTicker.C: + if err := refreshPods(); err != nil { + loopError = err + break Outer + } + case e, ok := <-dsWatch.ResultChan(): + d, ok := e.Object.(*appsv1.DaemonSet) + if !ok { + loopError = fmt.Errorf("invalid object type: %T", d) + break Outer + } + lastDS = d + if err := refreshPods(); err != nil { + loopError = err + break Outer + } + if d.Status.NumberReady == d.Status.DesiredNumberScheduled && d.Status.DesiredNumberScheduled > 0 && d.Status.NumberUnavailable == 0 { + break Outer + } + } + } + groupErr := errGroup.Wait() + for _, err := range nodesErr { + if err != nil { + return err + } + } + if loopError != nil { + return loopError + } + return groupErr +} diff --git a/test/kubernetes/tests/BUILD b/test/kubernetes/tests/BUILD new file mode 100644 index 000000000..5f6235a06 --- /dev/null +++ b/test/kubernetes/tests/BUILD @@ -0,0 +1,21 @@ +load("//tools:defs.bzl", "go_test") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_test( + name = "hello_test", + srcs = ["hello_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + ], +) diff --git a/test/kubernetes/tests/hello_test.go b/test/kubernetes/tests/hello_test.go new file mode 100644 index 000000000..7f4704c97 --- /dev/null +++ b/test/kubernetes/tests/hello_test.go @@ -0,0 +1,78 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hello_test + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" +) + +// TestHello tests that a trivial alpine container runs correctly. +func TestHello(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + cluster := k8sCtx.AcquireCluster(ctx, t) + defer k8sCtx.ReleaseCluster(ctx, t, cluster) + + ns := cluster.Namespace(testcluster.NamespaceDefault) + image, err := k8sCtx.ResolveImage(ctx, "alpine") + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + pod := ns.NewAlpinePod(fmt.Sprintf("hello-%d", time.Now().UnixNano()), image, []string{"/bin/sh", "-c", "echo hello"}) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer cluster.DeletePod(ctx, pod) + if err := cluster.WaitForPodCompleted(ctx, pod); err != nil { + t.Fatalf("Failed to wait for pod on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + reader, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer reader.Close() + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + if strings.TrimSpace(buf.String()) != "hello" { + t.Fatalf("Mistmatch output: got: %q want: %q", buf.String(), "hello") + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestHello": TestHello, + }) +} diff --git a/tools/gvisor_k8s_tool/cluster/cluster.go b/tools/gvisor_k8s_tool/cluster/cluster.go index 434e5cca7..a7d68d54b 100644 --- a/tools/gvisor_k8s_tool/cluster/cluster.go +++ b/tools/gvisor_k8s_tool/cluster/cluster.go @@ -45,6 +45,11 @@ func New(config *rest.Config) (*Cluster, error) { return &Cluster{clientSet}, nil } +// Client returns the underlying Kubernetes client. +func (c *Cluster) Client() kubernetes.Interface { + return c.client +} + // CreateDaemonset creates a daemonset with default options. func (c *Cluster) CreateDaemonset(ctx context.Context, ds *appsv1.DaemonSet) (*appsv1.DaemonSet, error) { if ds.GetObjectMeta().GetNamespace() == "" {