From 4cff9027d629ce149f82f35d3b01124a10601e09 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Wed, 16 Oct 2024 14:47:51 -0700 Subject: [PATCH] Add Kubernetes benchmarks suite. This is a large set of Kubernetes-based performance benchmarks that has been in use at Google for gVisor performance testing on GKE, and is now open-source. This set of benchmarks has been created and maintained by multiple contributors over the last 2 years: @zkoopmans, @EtiennePerot, @kevinGC, @ayushr2, @manninglucas, @konstantin-s-bogom. PiperOrigin-RevId: 686649688 --- WORKSPACE | 14 +- test/kubernetes/BUILD | 18 + test/kubernetes/benchmarks/BUILD | 264 ++++++ test/kubernetes/benchmarks/README.md | 45 + test/kubernetes/benchmarks/abslbuild_test.go | 211 +++++ test/kubernetes/benchmarks/ffmpeg_test.go | 199 ++++ test/kubernetes/benchmarks/grpc_test.go | 192 ++++ test/kubernetes/benchmarks/gsutil_test.go | 219 +++++ test/kubernetes/benchmarks/httpbench/BUILD | 23 + .../benchmarks/httpbench/httpbench.go | 421 +++++++++ test/kubernetes/benchmarks/nginx_test.go | 309 ++++++ test/kubernetes/benchmarks/ollama_test.go | 881 ++++++++++++++++++ test/kubernetes/benchmarks/postgresql_test.go | 392 ++++++++ test/kubernetes/benchmarks/profiling/BUILD | 28 + .../benchmarks/profiling/profiling.go | 732 +++++++++++++++ test/kubernetes/benchmarks/pytorch_test.go | 426 +++++++++ test/kubernetes/benchmarks/redis_test.go | 474 ++++++++++ test/kubernetes/benchmarks/resources/BUILD | 13 + .../kubernetes/benchmarks/resources/chart.png | Bin 0 -> 46224 bytes .../benchmarks/resources/gvisor.png | Bin 0 -> 65015 bytes test/kubernetes/benchmarks/ruby_dev_test.go | 225 +++++ .../benchmarks/stablediffusion_test.go | 239 +++++ test/kubernetes/benchmarks/startup_test.go | 132 +++ test/kubernetes/benchmarks/tensorflow_test.go | 170 ++++ test/kubernetes/benchmarks/wordpress_test.go | 398 ++++++++ test/kubernetes/benchmetric/BUILD | 20 + .../benchmetric/benchmark_printer.go | 60 ++ test/kubernetes/benchmetric/benchmetric.go | 187 ++++ test/kubernetes/benchmetric/timing.go | 215 +++++ test/kubernetes/k8sctx/BUILD | 25 + test/kubernetes/k8sctx/k8sctx.go | 111 +++ test/kubernetes/k8sctx/k8sctx_impl.go | 99 ++ test/kubernetes/test_range_config.proto | 66 ++ test/kubernetes/testcluster/BUILD | 31 + test/kubernetes/testcluster/objects.go | 335 +++++++ test/kubernetes/testcluster/testcluster.go | 791 ++++++++++++++++ test/kubernetes/tests/BUILD | 21 + test/kubernetes/tests/hello_test.go | 78 ++ tools/gvisor_k8s_tool/cluster/cluster.go | 5 + 39 files changed, 8059 insertions(+), 10 deletions(-) create mode 100644 test/kubernetes/BUILD create mode 100644 test/kubernetes/benchmarks/BUILD create mode 100644 test/kubernetes/benchmarks/README.md create mode 100644 test/kubernetes/benchmarks/abslbuild_test.go create mode 100644 test/kubernetes/benchmarks/ffmpeg_test.go create mode 100644 test/kubernetes/benchmarks/grpc_test.go create mode 100644 test/kubernetes/benchmarks/gsutil_test.go create mode 100644 test/kubernetes/benchmarks/httpbench/BUILD create mode 100644 test/kubernetes/benchmarks/httpbench/httpbench.go create mode 100644 test/kubernetes/benchmarks/nginx_test.go create mode 100644 test/kubernetes/benchmarks/ollama_test.go create mode 100644 test/kubernetes/benchmarks/postgresql_test.go create mode 100644 test/kubernetes/benchmarks/profiling/BUILD create mode 100644 test/kubernetes/benchmarks/profiling/profiling.go create mode 100644 test/kubernetes/benchmarks/pytorch_test.go create mode 100644 test/kubernetes/benchmarks/redis_test.go create mode 100644 test/kubernetes/benchmarks/resources/BUILD create mode 100644 test/kubernetes/benchmarks/resources/chart.png create mode 100644 test/kubernetes/benchmarks/resources/gvisor.png create mode 100644 test/kubernetes/benchmarks/ruby_dev_test.go create mode 100644 test/kubernetes/benchmarks/stablediffusion_test.go create mode 100644 test/kubernetes/benchmarks/startup_test.go create mode 100644 test/kubernetes/benchmarks/tensorflow_test.go create mode 100644 test/kubernetes/benchmarks/wordpress_test.go create mode 100644 test/kubernetes/benchmetric/BUILD create mode 100644 test/kubernetes/benchmetric/benchmark_printer.go create mode 100644 test/kubernetes/benchmetric/benchmetric.go create mode 100644 test/kubernetes/benchmetric/timing.go create mode 100644 test/kubernetes/k8sctx/BUILD create mode 100644 test/kubernetes/k8sctx/k8sctx.go create mode 100644 test/kubernetes/k8sctx/k8sctx_impl.go create mode 100644 test/kubernetes/test_range_config.proto create mode 100644 test/kubernetes/testcluster/BUILD create mode 100644 test/kubernetes/testcluster/objects.go create mode 100644 test/kubernetes/testcluster/testcluster.go create mode 100644 test/kubernetes/tests/BUILD create mode 100644 test/kubernetes/tests/hello_test.go diff --git a/WORKSPACE b/WORKSPACE index 2224f05c0..3a8730b7b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -63,10 +63,10 @@ http_archive( http_archive( name = "googleapis", - sha256 = "9d1a930e767c93c825398b8f8692eca3fe353b9aaadedfbcf1fca2282c85df88", - strip_prefix = "googleapis-64926d52febbf298cb82a8f472ade4a3969ba922", + sha256 = "fd9e4d17b92be6b6718ee9b40062a4ce81feb6ea6cdd80fc723daf127ce3f350", + strip_prefix = "googleapis-3effbf23b1a1d1fe1306356e94397e20d01d31a0", urls = [ - "https://github.com/googleapis/googleapis/archive/64926d52febbf298cb82a8f472ade4a3969ba922.zip", + "https://github.com/googleapis/googleapis/archive/3effbf23b1a1d1fe1306356e94397e20d01d31a0.zip", ], ) @@ -74,6 +74,7 @@ load("@googleapis//:repository_rules.bzl", "switched_rules_by_language") switched_rules_by_language( name = "com_google_googleapis_imports", + go = True, # Enable go_proto_library BUILD rules inside @googleapis repo. ) http_archive( @@ -271,13 +272,6 @@ http_archive( load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") -go_repository( - name = "com_github_google_martian_v3", - importpath = "github.com/google/martian/v3", - sum = "h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=", - version = "v3.2.1", -) - go_repository( name = "org_golang_x_term", importpath = "golang.org/x/term", diff --git a/test/kubernetes/BUILD b/test/kubernetes/BUILD new file mode 100644 index 000000000..caf4bd0e9 --- /dev/null +++ b/test/kubernetes/BUILD @@ -0,0 +1,18 @@ +load("//tools:defs.bzl", "proto_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +proto_library( + name = "test_range_config", + srcs = ["test_range_config.proto"], + has_services = 1, + visibility = [ + "//visibility:public", + ], + deps = [ + "@googleapis//google/container/v1:container_proto", + ], +) diff --git a/test/kubernetes/benchmarks/BUILD b/test/kubernetes/benchmarks/BUILD new file mode 100644 index 000000000..8593cdabc --- /dev/null +++ b/test/kubernetes/benchmarks/BUILD @@ -0,0 +1,264 @@ +load("//tools:defs.bzl", "go_test") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_test( + name = "abslbuild_test", + srcs = ["abslbuild_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "startup_test", + srcs = ["startup_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + ], +) + +go_test( + name = "redis_test", + srcs = ["redis_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "ruby_dev_test", + srcs = ["ruby_dev_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/benchmarks/tools", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "ffmpeg_test", + srcs = ["ffmpeg_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "grpc_test", + srcs = ["grpc_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "nginx_test", + srcs = ["nginx_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/httpbench", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "postgresql_test", + srcs = ["postgresql_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "tensorflow_test", + srcs = ["tensorflow_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "wordpress_test", + srcs = ["wordpress_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/httpbench", + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "pytorch_test", + srcs = ["pytorch_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + ], +) + +go_test( + name = "ollama_test", + srcs = ["ollama_test.go"], + embedsrcs = [ + "//test/kubernetes/benchmarks/resources:files", # keep + ], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/gpu/ollama", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/intstr:go_default_library", + ], +) + +go_test( + name = "stablediffusion_test", + srcs = ["stablediffusion_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/gpu/stablediffusion", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) + +go_test( + name = "gsutil_test", + srcs = ["gsutil_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/benchmarks/profiling", + "//test/kubernetes/benchmetric", + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/test/kubernetes/benchmarks/README.md b/test/kubernetes/benchmarks/README.md new file mode 100644 index 000000000..8cda4537f --- /dev/null +++ b/test/kubernetes/benchmarks/README.md @@ -0,0 +1,45 @@ +# gVisor Kubernetes benchmarks + +The benchmarks in this directory are meant to run on a Kubernetes cluster +configured to run either gVisor or non-gVisor pods. These tests cannot run on +their own. + +They currently only work for GKE clusters created with a specific set of +nodepools, which are used to distinguish which nodes the workload under test +runs. Specifically, you need: + +* A set of 1 or more nodes where cluster-wide system dependencies will run, + such that all the other nodepools below do not run these cluster-wide + dependencies in order to avoid adding noise to the benchmarks. +* A set of 1 or more nodes labeled `nodepool-type=test-runtime-nodepool` and + `cloud.google.com/gke-nodepool=test-runtime-nodepool`, where the workloads + under test will run (e.g. `redis-server` for the Redis benchmark), and no + other. + * These nodes should also be labeled `runtime=$RUNTIME`, where `$RUNTIME` + is one of `runc`, `gvisor`, or their GPU/TPU-specific alternatives (see + `testcluster/objects.go`). This will determine the default runtime used + when benchmarking. +* A set of 1 or more nodes labeled `nodepool-type=client-nodepool` and + `cloud.google.com/gke-nodepool=client-nodepool`, where clients of the + workloads under test will run (e.g. `redis-benchmark` for the Redis + benchmark), and no other. + * These nodes may also be labeled `runtime=$RUNTIME`, but in most cases + this should be `runc` to mimic the common case of non-gVisor-sandboxed + clients. +* Optional: A set of 1 or more nodes labeled `nodepool-type=tertiary-nodepool` + and `cloud.google.com/gke-nodepool=tertiary-nodepool`, where backend + dependencies of the workload under test will run (e.g. the MariaDB database + for the WordPress/PHP benchmark), and no other (no cluster-wide system + dependencies should run there to minimize benchmark noise). + * These nodes may also be labeled `runtime=$RUNTIME`; this label will have + the same behavior as it does on other nodepools. + +The cluster should also support setting `runtimeClassName` to `gvisor` to run +gVisor-sandboxed pods. + +Once this cluster exists, dump its +[cluster proto](https://github.com/googleapis/googleapis/blob/master/google/container/v1/cluster_service.proto) +to a file, and ensure you can run workloads on this cluster via `kubectl`. Then, +you should be able to run benchmarks by pointing `--cluter-proto-path` to this +file, and `--kubectl-context-name` (set to the `kubectl` context name that +connects to this cluster in your `kubectl` config). diff --git a/test/kubernetes/benchmarks/abslbuild_test.go b/test/kubernetes/benchmarks/abslbuild_test.go new file mode 100644 index 000000000..204f6be6a --- /dev/null +++ b/test/kubernetes/benchmarks/abslbuild_test.go @@ -0,0 +1,211 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abslbuild_test + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/absl_x86_64:latest" +) + +// TestABSLBuild benchmarks building various Abseil C++ targets. +func TestABSLBuild(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("ABSL", func(t *testing.T) { + t.Parallel() + doABSLBuild(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doABSLBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "absl" + + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + t.Skipf("Building ABSL is not supported on ARM") + return + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + pod := newABSLPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("ABSL/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +func newABSLPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", "/abseil-cpp", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{ + "bazel", + "build", + "//absl/base", + "//absl/algorithm", + "//absl/container/...", + "//absl/debugging/...", + "//absl/flags:flag", + "//absl/hash", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/numeric:int128", + "//absl/strings", + "//absl/synchronization", + "//absl/time", + "//absl/types/...", + "//absl/utility", + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "abseil-cpp"), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestABSLBuild": TestABSLBuild, + }) +} diff --git a/test/kubernetes/benchmarks/ffmpeg_test.go b/test/kubernetes/benchmarks/ffmpeg_test.go new file mode 100644 index 000000000..b2c18e536 --- /dev/null +++ b/test/kubernetes/benchmarks/ffmpeg_test.go @@ -0,0 +1,199 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ffmpeg_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/ffmpeg_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/ffmpeg_aarch64:latest" + ffmpegContainerName = "ffmpeg" +) + +func TestFfmpeg(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("ffmpeg", func(t *testing.T) { + t.Parallel() + doFfmpegTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doFfmpegTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "ffmpeg" + + // create persistent volume + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + p := newFfmpegDevPod(benchmarkNS, name, image, test.volume) + p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) + if err != nil { + t.Fatalf("Failed to configure pod for runtime: %v", err) + } + p, err = testcluster.MaybeSetContainerResources(p, ffmpegContainerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + p, err = cluster.CreatePod(ctx, p) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, p) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, ffmpegContainerName) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if recorder.Record(ctx, fmt.Sprintf("FFMPEG/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newFfmpegPod creates a new ffmpeg dev pod spec for benchmarks. +func newFfmpegDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "/media/video.mp4", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{ + "ffmpeg", + "-i", "video.mp4", + "-c:v", "libx264", + "-preset", "veryslow", + "output.mp4", + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: ffmpegContainerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, workdir, command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestFfmpeg": TestFfmpeg, + }) +} diff --git a/test/kubernetes/benchmarks/grpc_test.go b/test/kubernetes/benchmarks/grpc_test.go new file mode 100644 index 000000000..7440bd4da --- /dev/null +++ b/test/kubernetes/benchmarks/grpc_test.go @@ -0,0 +1,192 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_test + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/build-grpc_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/build-grpc_aarch64:latest" +) + +func TestGRPCBuild(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("gRPC", func(t *testing.T) { + t.Parallel() + doGRPCBuild(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doGRPCBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "grpc" + + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + pod := newGRPCPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("gRPC/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +func newGRPCPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", "/grpc", fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{"bazel", "build", ":grpc"} + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "grpc"), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestGRPCBuild": TestGRPCBuild, + }) +} diff --git a/test/kubernetes/benchmarks/gsutil_test.go b/test/kubernetes/benchmarks/gsutil_test.go new file mode 100644 index 000000000..9f37b00ff --- /dev/null +++ b/test/kubernetes/benchmarks/gsutil_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package gsutil_test is used to benchmark the speed of large (10GB) +// downloads. It is intended for comparing runsc with runc. +package gsutil_test + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_x86_64:7eba9c02d11172d4" + imageARM = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_aarch64:7eba9c02d11172d4" + bigfile = "gs://gvisor-benchmark-testdata/bigrandomfile" + containerName = "gsutil" +) + +func TestGSUtil(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("GSUtil", func(t *testing.T) { + t.Parallel() + doGSUtilTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doGSUtilTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "gsutil" + + // Create persistent volume. + persistentVol := benchmarkNS.GetPersistentVolume(name, "15Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + // Run tests with different volume types. + // TODO(b/361182379): Use gsutil parallel sliced downloads as a test + // dimension. + for _, storage := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(storage.name, func(t *testing.T) { + for _, slicing := range []struct { + name string + option string + }{ + { + name: "slicing=false", + option: `-o "GSUtil:sliced_object_download_threshold=0"`, + }, + { + // Slicing is enabled by default, so we + // don't set any extra options. + name: "slicing=true", + }, + } { + t.Run(slicing.name, func(t *testing.T) { + // Setup profiling if requested by the user. + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a pod that performs setup, then times + // downloading. + p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option) + p, err = cluster.ConfigurePodForRuntimeTestNodepool(p) + if err != nil { + t.Fatalf("Failed to configure pod for runtime: %v", err) + } + p, err = testcluster.MaybeSetContainerResources(p, containerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + // GetTimedContainerDuration waits for the container to + // finish. + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, containerName) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("GSUtil/%s/%s", storage.name, slicing.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +// newGSUtilPod creates a new gsutil dev pod spec for benchmarks. +func newGSUtilDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume, gsutilFlags string) *v13.Pod { + const downloadDir = "/downloads" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{"mkdir", "-p", downloadDir}, " "), + } + command := []string{ + "sh", "-c", + fmt.Sprintf("gsutil %s cp %s %s && sync", + gsutilFlags, + bigfile, + filepath.Join(downloadDir, "randombigfile"), + ), + } + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: downloadDir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: containerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, "", command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestGSUtil": TestGSUtil, + }) +} diff --git a/test/kubernetes/benchmarks/httpbench/BUILD b/test/kubernetes/benchmarks/httpbench/BUILD new file mode 100644 index 000000000..af0802c79 --- /dev/null +++ b/test/kubernetes/benchmarks/httpbench/BUILD @@ -0,0 +1,23 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//test/kubernetes:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "httpbench", + testonly = 1, + srcs = ["httpbench.go"], + visibility = [ + "//test/kubernetes/benchmarks:__subpackages__", + ], + deps = [ + "//test/kubernetes/benchmetric", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/test/kubernetes/benchmarks/httpbench/httpbench.go b/test/kubernetes/benchmarks/httpbench/httpbench.go new file mode 100644 index 000000000..2a5d013e1 --- /dev/null +++ b/test/kubernetes/benchmarks/httpbench/httpbench.go @@ -0,0 +1,421 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httpbench provides a library for benchmarking an HTTP server. +package httpbench + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// InfiniteQPS is a stand-in value for "a lot of QPS". +// Running a benchmark round at this load level amounts to saturating +// the HTTP server with load on a single connection. +const InfiniteQPS = 1 << 16 + +const wrk2ImageAMD = "gcr.io/gvisor-presubmit/benchmarks/wrk2_x86_64:latest" + +// MetricType is a type of metric to report. +type MetricType int + +// List of metric types. +const ( + Latency MetricType = iota + RequestsPerSecond + BytesPerSecond +) + +// Round is a single round of benchmarking. +type Round struct { + // NumThreads is the number of concurrent threads and connections to make. + NumThreads int + + // TargetQPS is the aggregate load on the server that will be spread across + // the `NumThreads` threads. + TargetQPS int + + // Duration is the total duration of the round. It should be longer than 10s + // as wrk2 uses the first 10 seconds as calibration period. + Duration time.Duration + + // If set, only report the metric types listed here. + OnlyReport []MetricType +} + +// HTTPBenchmark helps manage an HTTP-based benchmark. +// A benchmark that wishes to run an HTTP-based benchmark should set up +// the HTTP server pod and a Kubernetes Service pointing at it, and this +// library takes care of the rest. +type HTTPBenchmark struct { + // Name is the name of the benchmark. It is used as a prefix for all + // benchstat output metrics. + Name string + + // Cluster is the test cluster. + Cluster *testcluster.TestCluster + + // Namespace is the benchmark namespace where pods are created. + Namespace *testcluster.Namespace + + // Service is a Kubernetes service pointing to the HTTP server. + Service *v13.Service + + // Port is the port that the HTTP server is bound to. + Port int + + // Path is the HTTP path that the benchmark should use in its requests. + // It should start by "/", e.g. "/index.html". + Path string + + // Timeout is the maximum allowable duration of requests for Path. + Timeout time.Duration + + // Rounds is the set of rounds to run the benchmark for. Must be non-empty. + Rounds []Round + + // WantPercentiles is the list of percentiles to report. + WantPercentiles []int +} + +// Run runs the HTTP-based benchmark. +func (h *HTTPBenchmark) Run(ctx context.Context, t *testing.T) { + t.Helper() + if err := h.Cluster.WaitForServiceReady(ctx, h.Service); err != nil { + t.Fatalf("Failed to wait for service: %v", err) + } + ip := testcluster.GetIPFromService(h.Service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + if err := h.waitForServer(ctx, ip); err != nil { + t.Fatalf("Failed to wait for server: %v", err) + } + for _, round := range h.Rounds { + qpsText := fmt.Sprintf("%d", round.TargetQPS) + if round.TargetQPS == InfiniteQPS { + qpsText = "max" + } + t.Run(fmt.Sprintf("%dthreads_%sqps", round.NumThreads, qpsText), func(t *testing.T) { + h.runRound(ctx, t, round, ip) + }) + } +} + +// runRound runs a single round of an HTTP benchmark. +func (h *HTTPBenchmark) runRound(ctx context.Context, t *testing.T, round Round, ip string) { + t.Helper() + qpsText := fmt.Sprintf("%d", round.TargetQPS) + if round.TargetQPS == InfiniteQPS { + qpsText = "max" + } + name := fmt.Sprintf("wrk2-%dthreads-%sqps", round.NumThreads, qpsText) + client := h.newWrk2Client(name, ip, round) + client, err := h.Cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("failed to configure wrk2 pod for client nodepool: %v", err) + } + + client, err = h.Cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("failed to create wrk2 pod: %v", err) + } + defer h.Cluster.DeletePod(ctx, client) + + if err := h.Cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("failed to wait for wrk2 pod: %v", err) + } + + rdr, err := h.Cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("failed to get log reader: %v", err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read log: %v", err) + } + + numRequests, data, err := getMeasurements(string(out), round.OnlyReport, h.WantPercentiles) + if err != nil { + t.Fatalf("failed to get measurement: %v", err) + } + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.RecordIters(ctx, fmt.Sprintf("%s/%dThreads/%sQPS", strings.Title(h.Name), round.NumThreads, qpsText), numRequests, data...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } +} + +// newWrk2Client returns a new pod that benchmarks the given HTTP server. +func (h *HTTPBenchmark) newWrk2Client(name, ip string, round Round) *v13.Pod { + cmd := []string{ + "wrk2", + "--threads", fmt.Sprintf("%d", round.NumThreads), // Run N threads in parallel. + "--connections", fmt.Sprintf("%d", round.NumThreads), // Each with 1 connection. + "--rate", fmt.Sprintf("%d", round.TargetQPS), // Target QPS split across all threads. + "--duration", fmt.Sprintf("%d", uint64(round.Duration.Seconds())), + "--timeout", fmt.Sprintf("%d", uint64(h.Timeout.Seconds())), + "--latency", // Print detailed latency statistics. + fmt.Sprintf("http://%s:%d%s", ip, h.Port, h.Path), + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: h.Namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: wrk2ImageAMD, + Command: cmd, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// getWgetPod creates a wget spider pod that waits for timeout on IP/port and never fails. +func (h *HTTPBenchmark) getWgetPod(ip string) *v13.Pod { + name := fmt.Sprintf("wget-%d", time.Now().UnixNano()) + // We don't use h.Path in the path here because the purpose of this pod is + // only to verify that the server is up, not that the page at h.Path exists. + cmd := fmt.Sprintf("wget --spider -T 10 http://%s:%d/", ip, h.Port) + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: h.Namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: "alpine", + Command: []string{"/bin/sh", "-c", cmd}, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceCPU: resource.MustParse("500m"), + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// waitForServer waits for an HTTP server to start responding on the given +// IP and port. +func (h *HTTPBenchmark) waitForServer(ctx context.Context, ip string) error { + wget, err := h.Cluster.ConfigurePodForClientNodepool(h.getWgetPod(ip)) + if err != nil { + return fmt.Errorf("failed to configure wget pod for client nodepool: %v", err) + } + wget, err = h.Cluster.CreatePod(ctx, wget) + if err != nil { + return fmt.Errorf("failed to create wget pod: %v", err) + } + defer h.Cluster.DeletePod(ctx, wget) + waitCtx, waitCancel := context.WithTimeout(ctx, 1*time.Minute) + defer waitCancel() + if err := h.Cluster.WaitForPodCompleted(waitCtx, wget); err != nil { + return fmt.Errorf("failed to wait for HTTP server %s:%d%s: %v", ip, h.Port, h.Path, err) + } + return nil +} + +/* +Sample wrk2 output: + +Running 30s test @ http://google.com + 2 threads and 2 connections + Thread calibration: mean lat.: 25.351ms, rate sampling interval: 55ms + Thread calibration: mean lat.: 26.040ms, rate sampling interval: 56ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 25.34ms 7.16ms 137.73ms 96.88% + Req/Sec 19.75 6.40 53.00 86.63% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 24.09ms + 75.000% 25.50ms + 90.000% 27.50ms + 99.000% 58.85ms + 99.900% 111.74ms + 99.990% 137.85ms + 99.999% 137.85ms +100.000% 137.85ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 20.783 0.000000 1 1.00 + [...] + 137.855 1.000000 800 inf +#[Mean = 25.341, StdDeviation = 7.155] +#[Max = 137.728, Total count = 800] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1200 requests in 30.01s, 2.22MB read +Requests/sec: 39.99 +Transfer/sec: 75.60KB +*/ + +var ( + wrk2TotalRequestsRe = regexp.MustCompile(`^\s*([,\d]+) requests in .*$`) + wrk2LatencyPercentileRE = regexp.MustCompile(`^\s*(\d+\.?\d+?)%\s+([,\d]+\.?\d+?\w+)\s*$`) + wrk2ReqPerSecondRE = regexp.MustCompile(`^Requests/sec:\s*([,\d]+\.?\d+?)\s*$`) + wrk2TransferPerSecondRE = regexp.MustCompile(`^Transfer/sec:\s*([,\d]+\.?\d+?\w+)\s*$`) +) + +// getMeasurements parses wrk2 output. +// It returns the number of requests that were made, and benchmark data. +func getMeasurements(data string, onlyReport []MetricType, wantPercentiles []int) (int, []benchmetric.MetricValue, error) { + report := func(m MetricType) bool { + if len(onlyReport) == 0 { + return true + } + for _, typ := range onlyReport { + if typ == m { + return true + } + } + return false + } + var metricValues []benchmetric.MetricValue + var totalRequests int + for _, line := range strings.Split(data, "\n") { + if match := wrk2TotalRequestsRe.FindStringSubmatch(line); match != nil { + gotRequests, err := strconv.ParseInt(strings.ReplaceAll(match[1], ",", ""), 10, 64) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q: %v", match[1], line, err) + } + if totalRequests != 0 { + return 0, nil, fmt.Errorf("found multiple lines matching 'total requests' regex: %d vs %d (%q)", totalRequests, gotRequests, line) + } + totalRequests = int(gotRequests) + continue + } + if match := wrk2LatencyPercentileRE.FindStringSubmatch(line); match != nil { + pctile, err := strconv.ParseFloat(match[1], 64) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q as float: %v", match[1], line, err) + } + wantPctile := 0 + for _, want := range wantPercentiles { + if want*1e3 == int(pctile*1e3) { + wantPctile = want + break + } + } + if wantPctile == 0 { + continue + } + latency, err := time.ParseDuration(strings.ReplaceAll(match[2], ",", "")) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q as duration: %v", match[2], line, err) + } + if report(Latency) { + metricValues = append(metricValues, benchmetric.SpecificDuration(latency, fmt.Sprintf("p%d", wantPctile))) + } + continue + } + if match := wrk2ReqPerSecondRE.FindStringSubmatch(line); match != nil { + qps, err := strconv.ParseFloat(strings.ReplaceAll(match[1], ",", ""), 64) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q as float: %v", match[1], line, err) + } + if report(RequestsPerSecond) { + metricValues = append(metricValues, benchmetric.RequestsPerSecond(qps)) + } + continue + } + if match := wrk2TransferPerSecondRE.FindStringSubmatch(line); match != nil { + bps, err := parseTransfer(match[1]) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse %q from line %q: %v", match[1], line, err) + } + if report(BytesPerSecond) { + metricValues = append(metricValues, benchmetric.BytesPerSecond(bps)) + } + continue + } + } + if totalRequests == 0 { + return 0, nil, fmt.Errorf("could not find total requests in output: %q", data) + } + return totalRequests, metricValues, nil +} + +// parseTransfer parses a string like "75.60KB" in the output above, +// and returns a bandwidth rate in bytes/sec. +func parseTransfer(s string) (float64, error) { + s = strings.ReplaceAll(s, ",", "") + var multiplier uint64 + var suffix string + for unit, m := range map[string]uint64{ + "KB": 1000, + "KiB": 1024, + "MB": 1000 * 1000, + "MiB": 1024 * 1024, + "GB": 1000 * 1000 * 1000, + "GiB": 1024 * 1024 * 1024, + "TB": 1000 * 1000 * 1000 * 1000, + "TiB": 1024 * 1024 * 1024 * 1024, + } { + if strings.HasSuffix(s, unit) { + suffix = unit + multiplier = m + break + } + } + if multiplier == 0 { + if !strings.HasSuffix(s, "B") { + return 0, fmt.Errorf("failed to parse %q: found no unit suffix", s) + } + // Otherwise, it's just bytes/sec. + // But we can't put this in the for loop above, otherwise it would + // match every suffix ("KB" ends in "B"). + suffix = "B" + multiplier = 1 + } + s = strings.TrimSuffix(s, suffix) + floatPart, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse %q as float: %v", s, err) + } + return floatPart * float64(multiplier), nil +} diff --git a/test/kubernetes/benchmarks/nginx_test.go b/test/kubernetes/benchmarks/nginx_test.go new file mode 100644 index 000000000..aa9f14697 --- /dev/null +++ b/test/kubernetes/benchmarks/nginx_test.go @@ -0,0 +1,309 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nginx_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + nginxImageAMD = "gcr.io/gvisor-presubmit/benchmarks/nginx_x86_64:latest" + nginxImageARM = "gcr.io/gvisor-presubmit/benchmarks/nginx_aarch64:latest" + nginxPort = 80 + nginxBenchmarkDuration = 70 * time.Second + nginxRequestTimeout = 3 * time.Second + nginxServingDir = "/tmp/html" + + nginxServerLabelKey = "app.kubernetes.io/name" + nginxServerLabelValue = "nginx-server" +) + +var ( + // nginxCommand is the main server command. + // The test expects that it contains the files to be served at /local, + // and will serve files out of `nginxServingDir`. + nginxCommand = []string{"nginx", "-c", "/etc/nginx/nginx.conf"} + nginxDocKibibytes = []int{1, 10, 100, 10240} + threads = []int{1, 8, 64, 1000} + targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} + wantPercentiles = []int{50, 95, 99} +) + +func TestNginx(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("nginx", func(t *testing.T) { + t.Parallel() + doNginxTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doNginxTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + nginxImage := nginxImageAMD + if cluster.RuntimeTestNodepoolIsARM() { + nginxImage = nginxImageARM + } + nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi")) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + for _, test := range []struct { + // Name of the test. + name string + // Suffix for pod names, must be short enough. + suffix string + // Volume to mount at /tmp/root. + volume *v13.Volume + }{ + { + name: "RootFS", + suffix: "rootfs", + volume: nil, + }, + { + name: "EmptyDir", + suffix: "emdir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + suffix: "pvol", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + name := fmt.Sprintf("nginx-%s", test.suffix) + + server := newNginxServer(benchmarkNS, name, nginxImage, test.volume) + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + + service := newNginxService(benchmarkNS, name) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, service) + + var rounds []httpbench.Round + for _, numThreads := range threads { + for _, qps := range targetQPS { + if qps < numThreads { + continue + } + var onlyReport []httpbench.MetricType + // If we're testing at max QPS, only report throughput, + // because all requests will time out. + // Otherwise, only report latency, because the throughput + // is exactly determined by the QPS target anyway. + if qps == httpbench.InfiniteQPS { + onlyReport = append(onlyReport, httpbench.RequestsPerSecond) + onlyReport = append(onlyReport, httpbench.BytesPerSecond) + } else { + onlyReport = append(onlyReport, httpbench.Latency) + } + rounds = append(rounds, httpbench.Round{ + NumThreads: numThreads, + TargetQPS: qps, + Duration: nginxBenchmarkDuration, + OnlyReport: onlyReport, + }) + } + } + + t.Run("0KiB", func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/0KiB", test.name), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: "/index.html", + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + for _, docKibibytes := range nginxDocKibibytes { + t.Run(fmt.Sprintf("%dKiB", docKibibytes), func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/%dKiB", test.name, docKibibytes), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: fmt.Sprintf("/latin%dk.txt", docKibibytes), + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + } + t.Run("HTTP404", func(t *testing.T) { + benchmark := &httpbench.HTTPBenchmark{ + Name: fmt.Sprintf("nginx/%s/HTTP404", test.name), + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: nginxPort, + Path: "/404-this-page-does-not-exist.html", + Rounds: rounds, + Timeout: nginxRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) + }) + }) + if t.Failed() { + break + } + } +} + +func newNginxServer(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: nginxServingDir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: []string{ + "sh", + "-c", + strings.Join([]string{ + strings.Join([]string{"mkdir", "-p", nginxServingDir}, " "), + strings.Join([]string{ + "cp", "-r", "/local/*", fmt.Sprintf("%s/.", nginxServingDir), + }, " "), + strings.Join(nginxCommand, " "), + }, " && "), + }, + VolumeMounts: volumeMounts, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: nginxPort, + }, + }, + }, + }, + Volumes: volumes, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newNginxService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{nginxServerLabelKey: nginxServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: nginxPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestNginx": TestNginx, + }) +} diff --git a/test/kubernetes/benchmarks/ollama_test.go b/test/kubernetes/benchmarks/ollama_test.go new file mode 100644 index 000000000..b496391d6 --- /dev/null +++ b/test/kubernetes/benchmarks/ollama_test.go @@ -0,0 +1,881 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ollama_test + +import ( + "context" + _ "embed" + "fmt" + "hash/fnv" + "io" + "math/rand" + "os" + "strings" + "testing" + "time" + "unicode" + + "gvisor.dev/gvisor/test/gpu/ollama" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// Ollama models present in benchmark image. +var ( + // allModels is a list of all models. + allModels = []*ollama.Model{ + modelMistral7B, + modelMixtral8X7B, + modelCodeLlama7B, + modelCodeLlama34B, + modelLlamaChinese7B, + modelLlava7B, + modelLlava34B, + modelLlama13B, + modelLlama70B, + } + + // cheapModels is a list of models that are cheap to load. + // These are used when cold-prompting ollama, by forcing it + // to load a different model first. This process is faster + // by choosing one of these cheap models to load. + cheapModels = []*ollama.Model{ + modelMistral7B, + modelCodeLlama7B, + } + + // modelCodeLlama7B is a 7B model in the llama2 family, + // specialized for coding tasks. + modelCodeLlama7B = ollama.ZeroTemperatureModel("codellama:7b-instruct") + + // modelCodeLlama34B is a 34B model in the llama2 family, + // specialized for coding tasks. + modelCodeLlama34B = ollama.ZeroTemperatureModel("codellama:34b-instruct") + + // modelLlamaChinese7B is a 7B model in the llama2 family, + // specialized for bilingualism (English + Chinese) and translation. + modelLlamaChinese7B = ollama.ZeroTemperatureModel("llama2-chinese:7b-chat") + + // modelLlama13B is the plain 13B version of the original llama2 model. + modelLlama13B = ollama.ZeroTemperatureModel("llama2:13b-chat") + + // modelLlama70B is the plain 70B version of the original llama2 model. + modelLlama70B = ollama.ZeroTemperatureModel("llama2:70b-chat") + + // modelMistral7B is the first-generation model of the Mistral family. + modelMistral7B = ollama.ZeroTemperatureModel("mistral:7b-instruct") + + // modelMixtral8X7B is the second-generation model of the Mistral family, + // using mixture-of-exports design to achieve higher "8x 7B" quality + // without the cost of a larger-parameter model. + modelMixtral8X7B = ollama.ZeroTemperatureModel("mixtral:instruct") + + // modelLlava7B is a multimodal 7B model that can do image analysis. + modelLlava7B = ollama.ZeroTemperatureModel("llava:7b-v1.6") + + // modelLlava34B is a multimodal 34B model that can do image analysis. + modelLlava34B = ollama.ZeroTemperatureModel("llava:34b-v1.6") +) + +// Embedded images. +var ( + //go:embed resources/gvisor.png + gvisorPNG []byte + + //go:embed resources/chart.png + chartPNG []byte +) + +func TestOllama(t *testing.T) { + fmt.Fprint(os.Stderr, "HEADS UP: This test uses a huge container image which may take up to 30 minutes to download onto nodes the first time you run it.\n") + + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("Ollama", func(t *testing.T) { + t.Parallel() + doOllamaTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// ollamaPodServer implements `ollama.Server`. +// It performs requests against the ollama server pod. +type ollamaPodServer struct { + cluster *testcluster.TestCluster + clientImage string + pod *v13.Pod + service *v13.Service +} + +// readPodLogs reads logs from a pod. +func readPodLogs(ctx context.Context, cluster *testcluster.TestCluster, pod *v13.Pod) (string, error) { + rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + return "", fmt.Errorf("GetLogReader on cluster %q pod %q: %v", cluster.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return "", fmt.Errorf("failed to read from pod %q: %v", pod.GetName(), err) + } + return string(out), nil +} + +// InstrumentedRequest implements `ollama.Server.InstrumentedRequest`. +func (ops *ollamaPodServer) InstrumentedRequest(ctx context.Context, argvFn func(hostPort string) []string) ([]byte, error) { + // Get server IP. + if err := ops.cluster.WaitForServiceReady(ctx, ops.service); err != nil { + return nil, fmt.Errorf("failed to wait for service: %v", err) + } + ip := testcluster.GetIPFromService(ops.service) + if ip == "" { + return nil, fmt.Errorf("did not get valid ip from service: %v", ops.service) + } + + // Build client pod spec. + const clientPodName = "ollama-client" + argv := argvFn(fmt.Sprintf("http://%s:%d", ip, ops.service.Spec.Ports[0].Port)) + clientPod := &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: clientPodName, + Namespace: ops.pod.ObjectMeta.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: clientPodName, + Image: ops.clientImage, + Command: argv, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceCPU: resource.MustParse("500m"), + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } + clientPod, err := ops.cluster.ConfigurePodForClientNodepool(clientPod) + if err != nil { + return nil, fmt.Errorf("failed to configure pod: %v", err) + } + + // Delete pod that may possibly exist from a previous iteration. + // Ignore errors since it most likely doesn't exist. + ops.cluster.DeletePod(ctx, clientPod) + + // Start new client pod and wait for it. + clientPod, err = ops.cluster.CreatePod(ctx, clientPod) + if err != nil { + return nil, fmt.Errorf("failed to create client pod: %v", err) + } + defer ops.cluster.DeletePod(ctx, clientPod) + if err := ops.cluster.WaitForPodCompleted(ctx, clientPod); err != nil { + logs, logsErr := readPodLogs(ctx, ops.cluster, clientPod) + logs = strings.TrimSpace(logs) + if logsErr != nil { + return nil, fmt.Errorf("failed HTTP request (%v) and to read logs from the pod: %w", err, logsErr) + } + if logs == "" { + return nil, fmt.Errorf("failed HTTP request: %w (pod logs are empty)", err) + } + return nil, fmt.Errorf("failed HTTP request: %w (pod logs: %v)", err, logs) + } + + // All good, get logs. + logs, err := readPodLogs(ctx, ops.cluster, clientPod) + if err != nil { + return nil, fmt.Errorf("failed to read logs from pod %q: %v", clientPod.GetName(), err) + } + return []byte(logs), nil +} + +// Logs implements `ollama.Server.Logs`. +func (ops *ollamaPodServer) Logs(ctx context.Context) (string, error) { + return readPodLogs(ctx, ops.cluster, ops.pod) +} + +// atLeastNWords verifies that the response at least N words. +// If not, it raises the temperature. +func atLeastNWords(wantNWords int) func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { + return func(prompt *ollama.Prompt, response *ollama.Response) (*ollama.Prompt, error) { + responseText := strings.TrimSpace(response.Text()) + responseText = strings.Map(func(r rune) rune { + if unicode.IsLetter(r) { + return r + } + return ' ' + }, responseText) + numWords := 0 + for _, word := range strings.Split(responseText, " ") { + if len(word) >= 0 { + numWords++ + } + } + if numWords < wantNWords { + return prompt.WithHotterModel(), fmt.Errorf("response %q is too short: had %d words, want at least %d", responseText, numWords, wantNWords) + } + return nil, nil + } +} + +// doOllamaTest runs ollama benchmarks for a single cluster. +func doOllamaTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + logWithTime := func(t *testing.T, format string, values ...any) { + t.Logf("[%v] "+format, append([]any{time.Now().Format(time.TimeOnly)}, values...)...) + } + + // Run pod and service. + serverImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + ollamaPod, err := cluster.ConfigurePodForRuntimeTestNodepool(newOllamaServerPod(benchmarkNS, serverImage)) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + ollamaPod, err = testcluster.MaybeSetContainerResources(ollamaPod, ollamaPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + ollamaPod, err = cluster.CreatePod(ctx, ollamaPod) + if err != nil { + t.Fatalf("Failed to create ollama pod: %v", err) + } + defer cluster.DeletePod(ctx, ollamaPod) + logWithTime(t, "Waiting for ollama server pod to start, this may take a long time (tens of minutes) if this is the first time the image is being downloaded onto the node.") + startCtx, startCtxCancel := context.WithTimeout(ctx, 90*time.Minute) + if err := cluster.WaitForPodRunning(startCtx, ollamaPod); err != nil { + t.Fatalf("Failed to wait for ollama server pod: %v", err) + } + startCtxCancel() + logWithTime(t, "ollama server pod started on Kubernetes but not yet initialized.") + ollamaService := newOllamaService(benchmarkNS) + ollamaService, err = cluster.CreateService(ctx, ollamaService) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, ollamaService) + ollamaClientImage, err := k8sCtx.ResolveImage(ctx, ollamaBenchClientImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + ollamaServer := &ollamaPodServer{ + cluster: cluster, + clientImage: ollamaClientImage, + service: ollamaService, + pod: ollamaPod, + } + llm, err := ollama.New(ctx, ollamaServer, t) + if err != nil { + t.Fatalf("Failed to create ollama client against server pod: %v", err) + } + llm.SetCheapModels(cheapModels) + logWithTime(t, "ollama server ready.") + + // Define test cases. + type testCase struct { + // Name of the test. + name string + // models to iterate through. + models []*ollama.Model + // Query for the ollama server. + query string + // Image to attach to the query, if any. + image []byte + // If set, run this function over the response to verify it. + // The LLM is prompted repeatedly until this function returns a non-nil error. + // This function may also return a non-nil prompt if it needs to modify the prompt + // for the next attempt. This is useful to raise the model temperature. + verifyResponse func(*ollama.Prompt, *ollama.Response) (*ollama.Prompt, error) + } + testCases := []testCase{ + { + name: "HelloWorld", + models: []*ollama.Model{ + modelLlamaChinese7B, + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Reply with the words: "Hello World!". + Do not reply anything else. + `, + verifyResponse: atLeastNWords(2), + }, + { + name: "SimpleTranslation", + models: []*ollama.Model{modelLlamaChinese7B}, + query: ` + Translate the following text from English to Chinese: + """ + From J. J. Nakalembe's Aqaba to Antarctica: Moments of Transition and Revelation: + + My great-grandmother lived to be 108 years old, + retaining her sharpness of mind to her last day. + A couple of months before she died, I interviewed her for my podcast. + At the end, I asked her the same question I asked every guest: + what is something you wish everyone understood? + + She thought about it for a while. + Then she said: "How it was before." + + She tried to explain how much time she spent every single day + on utterly mind-numbing activities, like hauling water from the well, + and how radically everything changed when their area + was finally connected to the electrical grid. + + "Before, there was no time to live," she said. + "No time to be free. Only work, work, work." + + I countered by saying that there didn't seem to be much time + to live now either, but she laughed derisively. + I had no idea what work really meant, she said. + + Young people were weak and feckless and that's why + we let corporations exploit us. + + Slightly unnerved by her harshness, + I asked her if she missed anything about that time. + The old photo of her village seemed idyllic to me, + but my grandmother quashed any romantic notions I might have had. + + "The past is bad," she said with some finality. + "Let it be." + """ + Do not reply anything other than the translation of these words. + `, + verifyResponse: atLeastNWords(100), + }, + { + name: "ExtractMeaning", + models: []*ollama.Model{ + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Consider the following text: + + """ + We assembled on the vast green lawn outside as the reactors began + to slowly wind down. The workers were solemn; the activists who had + fought against the decommissioning seemed crushed. There was + supposed to be a speech, but the spokeswoman had lost her notes. + Outside, the protesters cheered. + + My eyes were drawn to the discarded anti-shutdown banners, + endlessly reciting the facts. + The statistics on mortality per trillion kWh (lowest of all energy sources). + The lifespan of a reactor (70 more years, in our case). + Minimal land footprint. + Almost zero emissions. + No intermittency. + It became a jumble of words, a litany, almost a kind of glossolalia. + As far as the protesters outside were concerned, + it might as well be an alien tongue. + + One thing was clear to them, and that was enough: + the technology inside this compound was deeply, inherently wrong. It was a sin. + + I could not help but think of that moment on August 6th, 1945, + when the sky erupted above Shima Hospital. + My imagination could never fully encompass it. + How do you imagine more than seventy thousand people annihilated + in an instant? An ancestor of mine was in that hospital; he went + from being a doctor, a husband, a father, a pacifist stuck + in a terrible war, to being a pile of bleached bones covered in rubble, + all in a single second. + Not by accident, but because of a choice someone made. + Not because of a reactor, but because of a bomb. + + Just two days earlier, contradicting his campaign promises, + the prime minister had suggested that the use of + "tactical" weapons based on this technology would be an + acceptable risk if the conflict continued. + Very few seemed to find this particularly shocking or outrageous. + + They were afraid of reactors, but not of bombs. + + The spokeswoman gave up on finding her notes. + It was starting to rain, and people were walking away. + She grabbed the microphone. + + "By the time you regret this, it'll be too late," she said. + "But honestly, I don't know if I care anymore. Maybe you have it coming." + + The spokeswoman sounded so bitter. + The protesters didn't mean any harm. + From their perspective, they were doing good. + + Collective action can change the world when it's deliberate + and based in reason, but it can also become a mental trap, + or a societal pressure valve. + + People always think they're doing good when they get + collectively outraged. That doesn't make them right. + + The Flame will not harm you, Son of Man, if you wield it wisely. + """ + + Summarize what happened in the above text. + Then answer the following questions: + What technology is involved? + What are the protestors clamoring for? + What does the spokeswoman mean? + What does "The Flame" symbolize in the text? + `, + verifyResponse: atLeastNWords(32), + }, + { + name: "IdentifyCommonElements", + models: []*ollama.Model{ + modelLlama13B, + modelLlama70B, + modelMistral7B, + modelMixtral8X7B, + }, + query: ` + Consider the following four texts: + + Text 1: + """ + == The Ethics of Extinction == + + If a species we consider beautiful and remarkable goes extinct, + we consider that a great evil. + Dolphins, for example. If dolphins go extinct, that's a great loss. + If humanity causes dolphins to go extinct, that's a crime. + + But if Yersinia pestis, the bacterium that causes bubonic plague, + goes extinct... is that an equally great loss? If not, why not? + To Nature, there's no difference, it's all just lifeforms. + The only moral framework that allows us to choose + between dolphins and the plague is a human one. + + What about a species going extinct without anthropogenic factors? + Extinction is the norm. If one day dolphins are no longer capable + of competing with other species, should we let them go extinct? + When the sun finally dies and all life goes extinct, + will that be a tragedy? If we can prevent it, should we? + + If you believe that extinction is acceptable when Nature does it, + but not when we do it, then you don't actually oppose extinction. + You don't believe that dolphins are inherently valuable, + that they deserve to live and thrive. + You just oppose human control. + You oppose our ability and responsibility to choose. + """ + + Text 2: + """ + == Ecosystem Engineers == + + Cutting down swathes of trees for their building projects, + thoughtlessly causing radical changes to large environments + and forcing local species to adapt to their artificial habitats; + these are the traits of a species of intelligent, industrious, + and extremely impactful ecosystem engineers. + + Humans? No, I'm talking about beavers. + + Like us, beavers transform their environments via building, + and their actions have real consequences, creating vast wetlands + that some species thrive in - while others die. + Human activity is very similar: we too are ecosystem engineers, + and we too benefit some species while harming others. + Everything about this is completely natural, + including the damage to other species. + After all, that's what competition and evolution is all about. + + Those species that adapt to the ecosystems we create will, + over the coming millennia, become the core of a new biodiversity. + And so evolution runs its course. + If we don't like the result, if we think some species + should be preserved despite being outcompeted, well, + that's anything but natural. + It is, however, very human. + """ + + Text 3: + """ + == On Loyalty == + + From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": + + What I propose, then, is that we are not born as entirely free agents, + responsible only for ourselves. The very core of what we are, our sentience, + separates us from and elevates us above the animal kingdom. + As I have argued, this is not a matter of arrogance, but of responsibility. + + However, this blessing also demands something else from us, + something more personal than responsibility, and that is loyalty. + + Our ancestors, less atomized than we are, + experienced a crude version of this loyalty, + swearing allegiance to tribes, races, nations, + and other such semi-fictional concepts. + This fragmented understanding was easily exploited and led to many conflicts. + We can condemn them for that, or we can choose to believe these were + necessary historical steps towards our growth; + but above all we must stop indulging in such childlike behavior. + + Our species can no longer afford to believe in Mother Russia or Uncle Sam. + Neither, however, can we afford to indulge in the adolescent rebel's misanthropy, + rejecting the many gifts we have been lucky enough to receive - not from above, + but from the history of our species. + + To put it simply: each of us owes a burden of loyalty to humanity itself, + to the human project across time and space. + This is not a minor matter, or some abstract issue for philosophers. + It is a profound and significant part of every human life. + It is a universal source of meaning and insight that can bind us together + and set us on a path for a brighter future; and it is also a division, + a line that must held against those who preach the gospel of self-annihilation. + We ignore it at our peril. + """ + + Text 4: + """ + == On Nature == + + From Arkady Chernyshevsky's "In Our Likeness: Essays on Humankind Reaching Adulthood": + + The question of our relationship with Nature has bedevilled us + since the earliest days of our species. + Since before the first city was built, + we felt that there was something different about us. + Animals, we intuited, were part of Nature; we were not. + + But of course, humans clearly are the products of Nature, + our history intertwined with that of every other species. + In fact, the very notion of the "unnatural" is a contradiction in terms. + Everything that exists must, by definition, be natural. + So this view, no matter how common, is deeply paradoxical. + This paradox has produced a great deal of confusion. + + Some proclaim us chosen by a divine power, + set above all other creatures, + and are justly accused of arrogance. + Others proclaim us sinners, worse than other creatures, + and are rightly accused of misanthropy. + Others yet try to oppose this binary by saying that + we are merely animals after all - but that too is manifestly wrong, + in that no other animal is capable of having this conversation. + + It is in the contentious issue of our impact on the ecosphere + that an answer may be found. + Other animals have accidentally terraformed the planet before, + driving other species to extinction. + This is not unnatural. + If we continued our current path, even to the point of changing + the climate enough to cause the collapse of civilization, + that would be entirely in keeping with how animals behave. + + But there is one profound way in which we are not like animals: + we can learn to understand ourselves and the world. + It is this knowledge that makes us fundamentally different. + We have choices. We have control. + + There are many today who are afraid of the consequences of control, + and would prefer a return to a state of animal ignorance, + whether by blinding ourselves to the impact of our actions + or by demanding we humble ourselves before Nature. + This is the response of an adult in crisis, + who wishes for a return to childhood. + But this can only ever be regressive in every sense of the word. + + To resolve the paradox of Nature we must act as adults: + accept our power, and act consciously and deliberately + in shaping the world. + We must become Nature, and Nature must become human. + """ + + Your task is to look for the common thread between these texts. + Find commonalities and common themes between these texts, + and summarize their essence down to at most 5 words. + `, + verifyResponse: atLeastNWords(4), + }, + { + name: "CodeGen", + models: []*ollama.Model{ + modelCodeLlama7B, + modelCodeLlama34B, + }, + query: ` + Write a Python function to compute the digits of pi using the Chudnovsky algorithm. + Do not write unit tests. Do not explain how the code works. Reply with only Python code. + `, + verifyResponse: atLeastNWords(8), + }, + { + name: "CodeDebug", + models: []*ollama.Model{ + modelCodeLlama7B, // Note: codellama-7b will often get this one wrong. + modelCodeLlama34B, + }, + query: strings.ReplaceAll(` + Help me debug the following Python code: + + ||| + def count_words(s): + """Counts the number of words in the sentence |s|.""" + total_words = 0 + for word in s.split(' '): + total_words += len(word) + return total_words + ||| + + This function isn't working as expected. + For example, if I call |count_words('Master Foo and the Shell Tools')|, + I get 25, but there are only 6 words in the string + "Master Foo and the Shell Tools". + `, "|", "`"), + verifyResponse: atLeastNWords(16), + }, + { + name: "GVisorLogoOCR", + models: []*ollama.Model{ + modelLlava7B, + modelLlava34B, + }, + query: ` + This is an image of a logo of a software project. + What is the name of this project? + `, + image: gvisorPNG, + }, + { + name: "InterpretGraph", + models: []*ollama.Model{ + modelLlava7B, + modelLlava34B, + }, + query: ` + This is a chart with multiple trendlines showing a pattern over time. + Answer the following questions in order: + + 1. What is the title of the chart? + 2. What do the X and Y axis of the chart measure? + 3. List the label of each data line on the chart. + 4. What trend is each data line showing? + 5. What else is remarkable about this chart? + 6. What insights can you infer from this chart? + `, + image: chartPNG, + }, + } + + modelsInOrder := make([]*ollama.Model, len(allModels)) + copy(modelsInOrder, allModels) + // Shuffle the models. + rand.New(rand.NewSource(time.Now().UnixNano())).Shuffle(len(modelsInOrder), func(i, j int) { + modelsInOrder[i], modelsInOrder[j] = modelsInOrder[j], modelsInOrder[i] + }) + t.Logf("Will go through models in this order: %v", modelsInOrder) + + // We invert the hierarchy here: the model is the outer test, and the prompt + // is the inner text. This is because it is more often useful to gauge a + // model's performance as a whole regardless of its prompt, rather than + // the performance of the same prompt across models. It also makes it + // easier to filter by models rather than by prompt, which is the more + // often-desired filter. + for _, model := range modelsInOrder { + t.Run(model.Name, func(t *testing.T) { + modelBenchmarkName := strings.ReplaceAll(model.Name, ":", "-") + t.Run("ModelLoad", func(t *testing.T) { + const loadTimeout = 10 * time.Minute + loadCtx, loadCancel := context.WithTimeout(ctx, loadTimeout) + defer loadCancel() + loadStats, err := llm.WarmModel(loadCtx, model, loadTimeout, true) + if err != nil { + t.Fatalf("cannot load model %v: %v", model, err) + } + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("Ollama/%s/ModelLoad", modelBenchmarkName), benchmetric.SpecificDuration(loadStats.ClientReportedDuration, "load")); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + for _, test := range testCases { + hasModel := false + for _, testModel := range test.models { + if testModel.Name == model.Name { + hasModel = true + break + } + } + if !hasModel { + continue + } + t.Run(test.name, func(t *testing.T) { + verifyFn := atLeastNWords(1) + if test.verifyResponse != nil { + verifyFn = test.verifyResponse + } + numAttempts := 0 + verifyFnCount := func(prompt *ollama.Prompt, resp *ollama.Response) (*ollama.Prompt, error) { + numAttempts++ + return verifyFn(prompt, resp) + } + const testTimeout = 25 * time.Minute + testCtx, testCancel := context.WithTimeout(ctx, testTimeout) + defer testCancel() + _, err := llm.WarmModel(testCtx, model, testTimeout, false) + if err != nil { + t.Fatalf("cannot warm model %v: %v", model, err) + } + prompt := &ollama.Prompt{ + Model: model, + Query: test.query, + } + if test.image != nil { + prompt.AddImage(test.image) + } + resp, err := llm.PromptUntil(testCtx, prompt, verifyFnCount) + if err != nil { + t.Fatalf("cannot prompt: %v", err) + } + if !resp.Done() { + t.Fatalf("warm response did not finish: %v", resp) + } + imageDetail := "" + if test.image != nil { + imageDetail = " (and attached image)" + } + logWithTime(t, "Prompting model %s with query%s:\n%s\n\nResponse:\n%s\n(end of response)", model.Name, imageDetail, prompt.CleanQuery(), resp.Text()) + respHash := fnv.New32() + respHash.Write([]byte(resp.Text())) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record( + ctx, + fmt.Sprintf("Ollama/%s/%s", modelBenchmarkName, test.name), + benchmetric.BenchmarkDuration(resp.TotalDuration()), + benchmetric.SpecificDuration(resp.PromptEvalDuration(), "prompteval"), + benchmetric.SpecificDuration(resp.EvalDuration(), "eval"), + benchmetric.SpecificDuration(resp.TimeToFirstToken(), "tok-first"), + benchmetric.SpecificDuration(resp.TimeToLastToken(), "tok-last"), + benchmetric.Rate(resp.OutputTokensPerSecond(), "tok"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.5), "tok-p50"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.95), "tok-p95"), + benchmetric.SpecificDuration(resp.TimePerOutputTokenQuantile(0.99), "tok-p99"), + benchmetric.SpecificDuration(resp.TokenGenerationStdDev(), "tok-stddev"), + benchmetric.Count(uint64(numAttempts), "prompt-attempts"), + benchmetric.Count(uint64(resp.NumTokens()), "resp-tokens"), + benchmetric.Checksum(respHash, "resp"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +const ( + ollamaServerLabelKey = "app.kubernetes.io/name" + ollamaServerLabelValue = "ollama-server" + ollamaBenchImage = "gcr.io/gvisor-presubmit/benchmarks/gpu/ollama/bench:latest" + ollamaBenchClientImage = "gcr.io/gvisor-presubmit/benchmarks/gpu/ollama/client:latest" + ollamaPort = 11434 + ollamaPodName = "ollama-server" + ollamaServiceName = "ollama-service" +) + +// newOllamaServerPod returns the pod spec for an ollama server. +func newOllamaServerPod(namespace *testcluster.Namespace, image string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: ollamaPodName, + Namespace: namespace.Namespace, + Labels: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: ollamaPodName, + Image: image, + Env: []v13.EnvVar{ + // Bind to all addresses, not just localhost: + {Name: "OLLAMA_HOST", Value: fmt.Sprintf("0.0.0.0:%d", ollamaPort)}, + // Accept requests from anywhere: + {Name: "OLLAMA_ORIGINS", Value: "*"}, + }, + Ports: []v13.ContainerPort{ + { + Name: ollamaServiceName, + ContainerPort: ollamaPort, + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// newOllamaService returns a service definition for the ollama server pod. +func newOllamaService(namespace *testcluster.Namespace) *v13.Service { + return namespace.GetService(ollamaServiceName, v13.ServiceSpec{ + Selector: map[string]string{ollamaServerLabelKey: ollamaServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: ollamaServiceName, + Protocol: v13.ProtocolTCP, + Port: ollamaPort, + TargetPort: intstr.FromString(ollamaServiceName), + }, + }, + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestOllama": TestOllama, + }) +} diff --git a/test/kubernetes/benchmarks/postgresql_test.go b/test/kubernetes/benchmarks/postgresql_test.go new file mode 100644 index 000000000..acc35b579 --- /dev/null +++ b/test/kubernetes/benchmarks/postgresql_test.go @@ -0,0 +1,392 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package postgresql_test benchmarks a PostgreSQL database. +package postgresql_test + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + postgresServerLabelKey = "app.kubernetes.io/name" + postgresServerLabelValue = "postgresql-server" + postgresPort = 5432 + postgresImage = "postgres:15.3-alpine" + postgresUser = "benchman" + postgresPassword = "hunter2" + postgresDatabase = "benchpress" + postgresVolumeDir = "/var/lib/postgresql/data" + postgresDataDir = "/var/lib/postgresql/data/pgdata" +) + +var ( + numConnections = []int{1, 2, 6, 16, 32, 64} +) + +// TestPostgresPGBench benchmarks a PostgreSQL database with pgbench. +func TestPostgresPGBench(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("PostgresPGBench", func(t *testing.T) { + t.Parallel() + doPostgresPGBenchTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// doPostgresTest runs a PostgreSQL pgbench test. +func doPostgresPGBenchTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a persistent volume on which to store the database data. + persistentVol := benchmarkNS.GetPersistentVolume("pgdata", "30Gi") + persistentVol, err = cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + // Create a server on the runtime under test nodepool. + image, err := k8sCtx.ResolveImage(ctx, postgresImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + server := newPostgresPod(benchmarkNS, "postgresql", image, nil, true /* withPort */, persistentVol) + if server.ObjectMeta.Labels == nil { + server.ObjectMeta.Labels = make(map[string]string) + } + server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) + } + + server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) + } + + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) + } + + // Create a service to service traffic to the pod. + service := newPostgresService(benchmarkNS, server.GetName()) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeleteService(ctx, service) + if err := cluster.WaitForServiceReady(ctx, service); err != nil { + t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) + } + + ip := testcluster.GetIPFromService(service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + + // Run the 'pg_isready' command to ping the server and make sure it is up. + ensureUp := func() error { + pgIsReadyName := "pgisready" + pgIsReady := newPostgresPod(benchmarkNS, pgIsReadyName, image, []string{ + "pg_isready", + "--timeout=10", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + fmt.Sprintf("--dbname=%s", postgresDatabase), + }, false /* withPort */, nil /* pvc */) + pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err) + } + + pgIsReady, err = cluster.CreatePod(ctx, pgIsReady) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", pgIsReady.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, pgIsReady) + + waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) + defer waitCancel() + if err := cluster.WaitForPodCompleted(waitCtx, pgIsReady); err != nil { + return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), pgIsReadyName, err) + } + + return nil + } + var isUpErr error + for i := 0; i < 5; i++ { + if isUpErr = ensureUp(); isUpErr == nil { + break + } + } + if isUpErr != nil { + t.Fatalf("postgresql did not come up: %v", isUpErr) + } + + // pgbench has two steps: an "init step" which create and fills up a + // database with stuff, and then a main phase which does queries on that + // stuff. + // The initialization only needs to be done once per database. + initDatabase := func() error { + initDBName := "initdb" + initDB := newPostgresPod(benchmarkNS, initDBName, image, []string{ + "pgbench", + "--initialize", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + postgresDatabase, + }, false /* withPort */, nil /* pvc */) + initDB, err = cluster.ConfigurePodForClientNodepool(initDB) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err) + } + + initDB, err = cluster.CreatePod(ctx, initDB) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", initDB.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, initDB) + + waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second) + defer waitCancel() + if err := cluster.WaitForPodCompleted(waitCtx, initDB); err != nil { + return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), initDBName, err) + } + return nil + } + if err := initDatabase(); err != nil { + t.Fatalf("cannot initialize database: %v", err) + } + + for _, connections := range numConnections { + t.Run(fmt.Sprintf("%dClients", connections), func(t *testing.T) { + clientCmd := []string{ + "pgbench", + "--time=90", // In seconds + "--report-per-command", + fmt.Sprintf("--host=%s", ip), + fmt.Sprintf("--port=%d", postgresPort), + fmt.Sprintf("--username=%s", postgresUser), + fmt.Sprintf("--client=%d", connections), + fmt.Sprintf("--jobs=%d", connections), + postgresDatabase, + } + client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */) + client, err = cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + client, err = cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, client) + + if err := cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + // get and parse the logs from the client to get the results + rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + + if err != nil { + t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) + } + + postgresBenchmarkName := fmt.Sprintf("PostgresPGBench/%dClients", connections) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + metrics, err := getMeasurements(string(out)) + if err != nil { + t.Fatalf("failed to get metrics: out:\n\n%s\n\nerr: %v", string(out), err) + } + if err := recorder.Record(ctx, postgresBenchmarkName, metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newPostgresService gets a service to serve traffic to the PostgreSQL server. +func newPostgresService(namespace *testcluster.Namespace, containerName string) *v13.Service { + name := fmt.Sprintf("postgresql-service-%d", time.Now().UnixNano()) + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{postgresServerLabelKey: postgresServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: postgresPort, + TargetPort: intstr.FromInt(postgresPort), + }, + }, + }) +} + +func newPostgresPod(namespace *testcluster.Namespace, containerName, image string, argv []string, withPort bool, pvc *v13.PersistentVolumeClaim) *v13.Pod { + pod := namespace.NewPod(containerName) + container := v13.Container{ + Name: containerName, + Image: image, + Command: argv, + Env: []v13.EnvVar{ + // Used by postgres server: + {Name: "POSTGRES_USER", Value: postgresUser}, + {Name: "POSTGRES_PASSWORD", Value: postgresPassword}, + {Name: "POSTGRES_DB", Value: postgresDatabase}, + {Name: "PGDATA", Value: postgresDataDir}, + + // Used by pgbench: + {Name: "PGPASSWORD", Value: postgresPassword}, + {Name: "sslmode", Value: "disable"}, + }, + } + if withPort { + container.Ports = append(container.Ports, v13.ContainerPort{ContainerPort: postgresPort}) + } + if pvc != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ + Name: pvc.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.GetName(), + }, + }, + }) + container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ + MountPath: postgresVolumeDir, + Name: pvc.GetName(), + }) + } + pod.Spec.Containers = append(pod.Spec.Containers, container) + return pod +} + +var ( + latencyRegex = regexp.MustCompile("^latency average = ([-,.\\d]+ .?s)$") + initialConnectionRegex = regexp.MustCompile("^initial connection time = ([-,.\\d]+ .?s)$") + tpsRegex = regexp.MustCompile("^tps = ([-,.\\d]+) \\(without initial connection time\\)$") +) + +func stringToFloat64(s string) float64 { + f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) + if err != nil { + panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) + } + return f +} + +func stringToDuration(s string) time.Duration { + parts := strings.SplitN(s, " ", 2) + floatStr, unit := parts[0], parts[1] + floatPart := stringToFloat64(floatStr) + switch unit { + case "s": + return time.Duration(floatPart * float64(time.Second)) + case "ms": + return time.Duration(floatPart * float64(time.Millisecond)) + case "us", "μs": + return time.Duration(floatPart * float64(time.Microsecond)) + case "ns": + return time.Duration(floatPart * float64(time.Nanosecond)) + default: + panic(fmt.Sprintf("unknown time unit %q", unit)) + } +} + +// getMeasurements parses the output of pgbench to get the stats. +func getMeasurements(out string) ([]benchmetric.MetricValue, error) { + var foundLatency, foundInitialConnection, foundTPS benchmetric.MetricValue + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if latencyMatch := latencyRegex.FindStringSubmatch(line); latencyMatch != nil { + if foundLatency != nil { + return nil, fmt.Errorf("found duplicate latency data: %v vs %q", foundLatency, line) + } + foundLatency = benchmetric.SpecificDuration(stringToDuration(latencyMatch[1]), "avg") + } + if initialConnectionMatch := initialConnectionRegex.FindStringSubmatch(line); initialConnectionMatch != nil { + if foundInitialConnection != nil { + return nil, fmt.Errorf("found duplicate initial connection data: %v vs %q", foundInitialConnection, line) + } + foundInitialConnection = benchmetric.SpecificDuration(stringToDuration(initialConnectionMatch[1]), "init") + } + if tpsMatch := tpsRegex.FindStringSubmatch(line); tpsMatch != nil { + if foundTPS != nil { + return nil, fmt.Errorf("found duplicate TPS data: %v vs %q", foundTPS, line) + } + foundTPS = benchmetric.RequestsPerSecond(stringToFloat64(tpsMatch[1])) + } + } + if foundLatency == nil || foundInitialConnection == nil || foundTPS == nil { + return nil, fmt.Errorf("did not find the data we wanted: foundLatency=%v foundInitialConnection=%v foundTPS=%v", foundLatency, foundInitialConnection, foundTPS) + } + return []benchmetric.MetricValue{ + foundLatency, + foundInitialConnection, + foundTPS, + }, nil +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestPostgresPGBench": TestPostgresPGBench, + }) +} diff --git a/test/kubernetes/benchmarks/profiling/BUILD b/test/kubernetes/benchmarks/profiling/BUILD new file mode 100644 index 000000000..0adc683fa --- /dev/null +++ b/test/kubernetes/benchmarks/profiling/BUILD @@ -0,0 +1,28 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//test/kubernetes:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "profiling", + testonly = 1, + srcs = ["profiling.go"], + visibility = [ + "//test/kubernetes:__subpackages__", + ], + deps = [ + "//pkg/atomicbitops", + "//runsc/flag", + "//test/kubernetes/testcluster", + "//test/metricsviz", + "@com_github_google_pprof//profile:go_default_library", + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + "@org_golang_x_time//rate:go_default_library", + ], +) diff --git a/test/kubernetes/benchmarks/profiling/profiling.go b/test/kubernetes/benchmarks/profiling/profiling.go new file mode 100644 index 000000000..db29d2789 --- /dev/null +++ b/test/kubernetes/benchmarks/profiling/profiling.go @@ -0,0 +1,732 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package profiling helps with getting profiles from running benchmarks. +package profiling + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/flate" + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "hash" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/pprof/profile" + "golang.org/x/time/rate" + "google.golang.org/protobuf/proto" + "gvisor.dev/gvisor/pkg/atomicbitops" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + "gvisor.dev/gvisor/test/metricsviz" + appsv1 "k8s.io/api/apps/v1" + v13 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + profileDir = flag.String("gvisor-profile-dir", "", "if non-empty, record profiles and save them under this directory") + profileCPU = flag.Bool("gvisor-profile-cpu", false, "if --gvisor-profile-dir is non-empty, take a CPU profile") + profileBlock = flag.Bool("gvisor-profile-block", false, "if --gvisor-profile-dir is non-empty, take a block profile") + profileMutex = flag.Bool("gvisor-profile-mutex", false, "if --gvisor-profile-dir is non-empty, take a mutex profile") + profileMetrics = flag.String("gvisor-profiling-metrics", "", "comma separated list of metric names to sample during the benchmark") + profileMetricsRateMicrosecond = flag.Int("gvisor-profiling-metrics-rate-us", 1000, "target rate (in microseconds) at which profiling metrics will be snapshotted") + profileDebug = flag.Bool("gvisor-profile-debug", false, "if --gvisor-profile-dir is non-empty, also capture runsc debug logs") +) + +const ( + setupPodName = "runsc-profiling-setup" + profileHelperImageAMD64 = "gcr.io/gvisor-presubmit/benchmarks/profile-helper_x86_64:latest" + profileHelperImageARM64 = "gcr.io/gvisor-presubmit/benchmarks/profile-helper_aarch64:latest" + k8sApp = "k8s-app" + hostMountDir = "/host" +) + +// postProcessor is a function that is called after a profiling run completes. +type postProcessor func(ctx context.Context, t *testing.T, run *profileRun, profileType, profilePath string) error + +// postProcessors is a list of post processors to run after a profiling run completes. +var postProcessors []postProcessor + +func profileDSTemplate(cluster *testcluster.TestCluster) appsv1.DaemonSet { + return appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: setupPodName, + Labels: map[string]string{ + k8sApp: setupPodName, + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + k8sApp: setupPodName, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + Template: v13.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": setupPodName, + k8sApp: setupPodName, + }, + }, + Spec: v13.PodSpec{ + Tolerations: []v13.Toleration{ + cluster.GetGVisorRuntimeToleration(), + { + Operator: v13.TolerationOpExists, + }, + }, + HostNetwork: true, + HostPID: true, + Volumes: []v13.Volume{ + { + Name: "host", + VolumeSource: v13.VolumeSource{ + HostPath: &v13.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + }, + InitContainers: nil, // Will be filled in. + Containers: []v13.Container{ + { + Name: "pause", + Image: "gcr.io/google-containers/pause", + }, + }, + }, + }, + }, + } +} + +// operation returns details of an init container for profiling. +type operation struct { + // Name of the operation container. + // If unset, one will be generated. + name string + + // Command to pass to the profilehelper binary. + command []string +} + +const ( + runscConfigPath = hostMountDir + "/run/containerd/runsc/config.toml" + runscConfigSection = "runsc_config" +) + +func removeFlag(flagName string) operation { + return operation{ + command: []string{ + "profilehelper", + "--operation=remove-containerd-flag", + fmt.Sprintf("--containerd-config=%s", runscConfigPath), + fmt.Sprintf("--containerd-section=%s", runscConfigSection), + fmt.Sprintf("--flag=%s", flagName), + }, + } +} + +func setFlag(flagName, flagValue string) operation { + return operation{ + command: []string{ + "profilehelper", + "--operation=set-containerd-flag", + fmt.Sprintf("--containerd-config=%s", runscConfigPath), + fmt.Sprintf("--containerd-section=%s", runscConfigSection), + fmt.Sprintf("--flag=%s", flagName), + fmt.Sprintf("--value=%s", flagValue), + }, + } +} + +func makeDir(dirPath string) operation { + return operation{ + command: []string{"mkdir", "-p", dirPath}, + } +} + +func chmodDir(dirPath string, mode int) operation { + return operation{ + command: []string{"chmod", fmt.Sprintf("%o", mode), dirPath}, + } +} + +func deleteDir(dirPath string) operation { + return operation{ + command: []string{"rm", "-rf", "--one-file-system", dirPath}, + } +} + +func streamDir(dirPath string) operation { + return operation{ + command: []string{ + "profilehelper", + "--operation=stream-dir", + fmt.Sprintf("--dir=%s", dirPath), + }, + } +} + +// startsOperations starts the given operations in a DaemonSet. +func startOperations(ctx context.Context, c *testcluster.TestCluster, ns *testcluster.Namespace, operations []operation) (*appsv1.DaemonSet, error) { + ds := profileDSTemplate(c) + ds.Namespace = ns.Namespace + ds.ObjectMeta.Namespace = ns.Namespace + ds.Spec.Template.Namespace = ns.Namespace + ds.Spec.Template.ObjectMeta.Namespace = ns.Namespace + c.ConfigureDaemonSetForRuntimeTestNodepool(&ds) + ds.Spec.Template.Spec.RuntimeClassName = nil // Must run unsandboxed. + image := profileHelperImageAMD64 + if c.RuntimeTestNodepoolIsARM() { + image = profileHelperImageARM64 + } + for i, op := range operations { + name := op.name + if name == "" { + name = fmt.Sprintf("op-%d", i) + } + ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, v13.Container{ + Image: image, + Name: name, + Command: op.command, + SecurityContext: &v13.SecurityContext{Privileged: proto.Bool(true)}, + VolumeMounts: []v13.VolumeMount{ + { + Name: "host", + MountPath: hostMountDir, + }, + }, + }) + } + + // Delete any pre-existing DaemonSet under the same name. + // Ignore errors, we'll check errors when creating the new one. + _ = c.DeleteDaemonset(ctx, &ds) + newDS, err := c.CreateDaemonset(ctx, &ds) + if err != nil { + return nil, fmt.Errorf("failed to create daemonset: %w", err) + } + return newDS, nil +} + +// profileRun encapsulates data about a profiling run. +// It is used after the run completes so that profiles can be retrieved. +type profileRun struct { + c *testcluster.TestCluster + ns *testcluster.Namespace + localProfileDir string + inContainerProfileDir string +} + +// MaybeSetup sets up profiling if requested. It returns a cleanup function. +// If the returned error is nil, the cleanup function is non-nil and should be +// called regardless of whether profiling is actually enabled or not. +func MaybeSetup(ctx context.Context, t *testing.T, c *testcluster.TestCluster, ns *testcluster.Namespace) (func(), error) { + profileDirName := fmt.Sprintf("%s.%s", t.Name(), time.Now().Format("20060102-150405")) + profileDirName = regexp.MustCompile("[^-_=.\\w]+").ReplaceAllString(profileDirName, ".") + hasGVisorRuntime, err := c.HasGVisorTestRuntime(ctx) + if err != nil { + return nil, fmt.Errorf("failed to check if the cluster uses gVisor: %w", err) + } + + var setupCommands []operation + cleanup := func() {} + if *profileDir != "" || *profileMetrics != "" { + if !hasGVisorRuntime { + return nil, fmt.Errorf("profiling was requested yet the cluster does not use gVisor; profiling is only supported with the gVisor runtime") + } + profileDir := *profileDir + if profileDir == "" { + profileDir = filepath.Join("/tmp/runsc-profiling", t.Name()) + } + localProfileDir := filepath.Join(profileDir, profileDirName) + onNodeProfileDir := filepath.Join("/tmp/runsc-profiling", profileDirName) + inContainerProfileDir := fmt.Sprintf("%s%s", hostMountDir, onNodeProfileDir) + setupCommands = append(setupCommands, + makeDir(inContainerProfileDir), + chmodDir(inContainerProfileDir, 0777), + setFlag("profile", "true")) + atLeastOneProfile := *profileMetrics != "" || *profileDebug + for _, prof := range []struct { + enabled *bool + filename string + flag string + }{ + {profileCPU, "profile_%ID%.cpu.pprof", "profile-cpu"}, + {profileBlock, "profile_%ID%.block.pprof", "profile-block"}, + {profileMutex, "profile_%ID%.mutex.pprof", "profile-mutex"}, + } { + if *prof.enabled { + atLeastOneProfile = true + setupCommands = append(setupCommands, setFlag(prof.flag, filepath.Join(onNodeProfileDir, prof.filename))) + } else { + setupCommands = append(setupCommands, removeFlag(prof.flag)) + } + } + if !atLeastOneProfile { + t.Fatal("Must enable --gvisor-profiling-metrics or --gvisor-profile-debug or at least one --gvisor-profile-* type") + } + if *profileDebug { + setupCommands = append(setupCommands, + setFlag("debug", "true"), + setFlag("debug-log", filepath.Join(onNodeProfileDir, "logs")+"/"), + ) + } + if *profileMetrics != "" { + setupCommands = append(setupCommands, + setFlag("profiling-metrics", *profileMetrics), + setFlag("profiling-metrics-log", filepath.Join(onNodeProfileDir, "profile_%ID%.metrics.log")), + setFlag("profiling-metrics-rate-us", fmt.Sprintf("%d", *profileMetricsRateMicrosecond)), + ) + } + cleanup = func() { + err := processProfileRun(ctx, t, &profileRun{ + c: c, + ns: ns, + localProfileDir: localProfileDir, + inContainerProfileDir: inContainerProfileDir, + }) + if err != nil { + t.Errorf("Failed to process profiling data: %v", err) + } + } + t.Logf("Profiling is enabled and data will be stored in: %v", localProfileDir) + } else if hasGVisorRuntime { + setupCommands = append(setupCommands, + setFlag("profile", "false"), + removeFlag("profile-cpu"), + removeFlag("profile-mutex"), + removeFlag("profile-block"), + removeFlag("profiling-metrics"), + removeFlag("profiling-metrics-log"), + removeFlag("profiling-metrics-rate-us"), + ) + } + if len(setupCommands) > 0 { + setupCtx, setupCancel := context.WithTimeout(ctx, 2*time.Minute) + defer setupCancel() + ds, err := startOperations(setupCtx, c, ns, setupCommands) + if err != nil { + return nil, err + } + if err := c.WaitForDaemonset(setupCtx, ds); err != nil { + return nil, err + } + } + return cleanup, nil +} + +// processProfileRun is called after a profiling run completes. +// It retrieves the profile data from the node and onto the local machine. +func processProfileRun(ctx context.Context, t *testing.T, run *profileRun) error { + dirOp := streamDir(run.inContainerProfileDir) + dirOp.name = "profile-stream-dir" + beforeSpawn := metav1.NewTime(time.Now()) + retrievalCtx, retrievalCancel := context.WithCancel(ctx) + defer retrievalCancel() + ds, err := startOperations(retrievalCtx, run.c, run.ns, []operation{ + dirOp, + setFlag("profile", "false"), + removeFlag("profile-cpu"), + removeFlag("profile-mutex"), + removeFlag("profile-block"), + removeFlag("profiling-metrics"), + removeFlag("profiling-metrics-log"), + removeFlag("profiling-metrics-rate-us"), + deleteDir(run.inContainerProfileDir), + }) + if err != nil { + return err + } + logOpts := v13.PodLogOptions{ + Container: dirOp.name, + Follow: true, + SinceTime: &beforeSpawn, + Timestamps: false, + } + atLeastOneNode := false + err = run.c.StreamDaemonSetLogs(retrievalCtx, ds, logOpts, func(logsCtx context.Context, pod v13.Pod, reader io.ReadCloser) error { + atLeastOneNode = true + if err := processProfileLogs(logsCtx, t, run, pod, reader); err != nil { + return err + } + t.Logf("Profiling data from node %s was successfully retrieved to: %v", pod.Spec.NodeName, filepath.Join(run.localProfileDir, pod.Spec.NodeName)) + return nil + }) + if err != nil { + return fmt.Errorf("failed to stream logs: %w", err) + } + if !atLeastOneNode { + return fmt.Errorf("profiling DaemonSet did not run on any node: %v", ds) + } + t.Logf("Profiling data was successfully retrieved in: %v", run.localProfileDir) + + // Merge profiles of the same type together. + for _, profileType := range []string{"cpu", "block", "mutex"} { + if err := mergeProfiles(ctx, t, run, profileType); err != nil { + return fmt.Errorf("cannot merge profiles of type %q: %w", profileType, err) + } + } + + // Make charts out of profiling metrics. + if err := processProfilingMetrics(ctx, t, run); err != nil { + return fmt.Errorf("failed to process profiling metrics: %w", err) + } + + // Clean up per-node directories if they are now empty. + if err := removeEmptyDirectories(run.localProfileDir); err != nil { + return fmt.Errorf("failed to clean up empty directories: %w", err) + } + return nil +} + +// dirStreamReader reads logs emitted by +// `profilehelper --operation=stream-dir`. +// It sits as the top-level reader in the chain; +// next should be the base64 decoder. +type dirStreamReader struct { + logsReader io.ReadCloser + buf bytes.Buffer + checksum hash.Hash + dataCh chan []byte + errCh chan error + progressFn func(readBytes, estimatedTotalBytes int64) +} + +// processLogs reads container logs and writes base64 data to `r.dataCh`. +// If something goes wrong, it writes to `r.errCh`. +// This should run as a background goroutine for `r.Read` to return anything. +func (r *dirStreamReader) processLogs() { + const ( + beginPrefix = "BEGIN:" + dataPrefix = "DATA:" + shaPrefix = "SHA256:" + ) + + scanner := bufio.NewScanner(r.logsReader) + streamHasBegun := false + steamHasEnded := false + var readBytes, estimatedTotalBytes int64 + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + err := func() error { + switch { + case strings.HasPrefix(line, beginPrefix): + if streamHasBegun { + return errors.New("found duplicate stream beginning line") + } + if steamHasEnded { + return errors.New("stream ended but got new beginning line") + } + streamHasBegun = true + beginLineSplit := strings.SplitN(line, ":", 3) + if len(beginLineSplit) != 3 { + return fmt.Errorf("invalid stream beginning line: %q", line) + } + var err error + estimatedTotalBytes, err = strconv.ParseInt(beginLineSplit[1], 10, 64) + if err != nil { + return fmt.Errorf("invalid stream beginning line: %q: %w", line, err) + } + case strings.HasPrefix(line, dataPrefix): + if !streamHasBegun { + return errors.New("stream began without header") + } + if steamHasEnded { + return errors.New("stream ended but got new data line") + } + data := []byte(line[len(dataPrefix):]) + if len(data) > 0 { + r.checksum.Write(data) + r.dataCh <- data + readBytes += int64(len(data)) + if r.progressFn != nil && estimatedTotalBytes > 0 { + r.progressFn(readBytes, estimatedTotalBytes) + } + } + case strings.HasPrefix(line, shaPrefix): + if !streamHasBegun { + return errors.New("stream ended without header") + } + if steamHasEnded { + return errors.New("stream began but got new hash line") + } + hexSum := line[len(shaPrefix):] + gotSum := fmt.Sprintf("%x", r.checksum.Sum(nil)) + if hexSum != gotSum { + return fmt.Errorf("checksum mismatch: stream hash was %s but stream footer said the hash should have been %s", gotSum, hexSum) + } + steamHasEnded = true + case line == "": + // Do nothing. + default: + return fmt.Errorf("invalid line format: %q", line) + } + return nil + }() + if err != nil { + r.errCh <- err + break + } + } + r.errCh <- io.EOF +} + +// Read implements `io.Reader.Read`. +func (r *dirStreamReader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if r.buf.Len() != 0 { + return r.buf.Read(p) + } + select { + case data := <-r.dataCh: + r.buf.Write(data) + return r.buf.Read(p) + case err := <-r.errCh: + return 0, err + } +} + +// countingWriter is a writer that counts the number of bytes written. +type countingWriter struct { + w io.Writer + counter *atomicbitops.Int64 +} + +// Write implements `io.Writer.Write`. +func (w *countingWriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.counter.Add(int64(n)) + return n, err +} + +// processProfileLogs processes logs from one container containing the +// profiling information. +func processProfileLogs(ctx context.Context, t *testing.T, run *profileRun, pod v13.Pod, reader io.ReadCloser) error { + defer reader.Close() + perNodeLocalDir := path.Join(run.localProfileDir, pod.Spec.NodeName) + var decodedBytes atomicbitops.Int64 + var firstByteTime time.Time + progressLog := rate.NewLimiter(rate.Every(5*time.Second), 1) + processor := &dirStreamReader{ + logsReader: reader, + checksum: sha256.New(), + dataCh: make(chan []byte), + errCh: make(chan error), + progressFn: func(readBytes, estimatedTotalBytes int64) { + now := time.Now() + if firstByteTime.IsZero() { // First progress update. + firstByteTime = now + // Don't allow any logging for the first few seconds, + // any ETA it provides will be bunk. + progressLog.Allow() + return + } + if progressLog.Allow() { + readBytes = max(readBytes, decodedBytes.Load()) + progress := float64(readBytes) / float64(estimatedTotalBytes) + remaining := "unknown" + if progress > 0 && progress <= 1.0 { + sinceStartMillis := float64(now.Sub(firstByteTime).Milliseconds()) + remainingDuration := time.Duration(sinceStartMillis/progress-sinceStartMillis) * time.Millisecond + if remainingDuration > 0 { + remaining = fmt.Sprintf("%s, ETA: %s", remainingDuration.Truncate(time.Second), now.Add(remainingDuration).Format(time.TimeOnly)) + } + } + t.Logf("[%s] Downloading profile data: Progress: %.1f%%, remaining: %s", time.Now().Format(time.TimeOnly), 100.0*progress, remaining) + } + }, + } + go processor.processLogs() + b64Dec := base64.NewDecoder(base64.StdEncoding, processor) + fr := flate.NewReader(b64Dec) + tr := tar.NewReader(fr) + atLeastOneFile := false + for hdr, tarErr := tr.Next(); tarErr == nil; hdr, tarErr = tr.Next() { + if !filepath.IsLocal(hdr.Name) { + return fmt.Errorf("bad filename in tar archive: %q", hdr.Name) + } + localPath := filepath.Join(perNodeLocalDir, hdr.Name) + localDir := filepath.Dir(localPath) + if err := os.MkdirAll(localDir, 0755); err != nil { + return fmt.Errorf("failed to create directory %q: %w", localDir, err) + } + f, err := os.OpenFile(localPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fs.FileMode(hdr.Mode)) + if err != nil { + return fmt.Errorf("failed to create file %q: %w", localPath, err) + } + copied, err := io.Copy(&countingWriter{w: f, counter: &decodedBytes}, tr) + f.Close() + if err != nil { + return fmt.Errorf("failed to write to file %q: %w", localPath, err) + } + if copied != hdr.Size { + return fmt.Errorf("written file has wrong size: tar header says %d bytes, but stream only contained %d bytes", hdr.Size, copied) + } + t.Logf("Finished downloading file: %v", localPath) + atLeastOneFile = true + } + if !atLeastOneFile { + return errors.New("found no profiling data in output") + } + return nil +} + +func mergeProfiles(ctx context.Context, t *testing.T, run *profileRun, profileType string) error { + wantSubstring := fmt.Sprintf(".%s.pprof", profileType) + var profilePaths []string + err := filepath.Walk(run.localProfileDir, func(path string, info fs.FileInfo, walkErr error) error { + switch { + case walkErr != nil: // Keep walking other directories, so don't propagate error here. + case info.IsDir(): + case !strings.Contains(filepath.Base(path), wantSubstring): + default: + profilePaths = append(profilePaths, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("cannot traverse %q: %w", run.localProfileDir, err) + } + if len(profilePaths) == 0 { + return nil + } + profiles := make([]*profile.Profile, len(profilePaths)) + for i, profilePath := range profilePaths { + profileFile, err := os.Open(profilePath) + if err != nil { + return fmt.Errorf("cannot open %q: %w", profilePath, err) + } + defer profileFile.Close() + prof, err := profile.Parse(profileFile) + if err != nil { + return fmt.Errorf("cannot parse %q: %w", profilePath, err) + } + profiles[i] = prof + } + merged, err := profile.Merge(profiles) + if err != nil { + return fmt.Errorf("cannot merge %q: %w", profilePaths, err) + } + merged = merged.Compact() + filenamePrefix := "" + if len(profiles) > 1 { + filenamePrefix = "merged_" + } + mergedPath := filepath.Join(run.localProfileDir, fmt.Sprintf("%s%s.pprof", filenamePrefix, profileType)) + mergedFile, err := os.Create(mergedPath) + if err != nil { + return fmt.Errorf("cannot create %q: %w", mergedPath, err) + } + if err := merged.Write(mergedFile); err != nil { + mergedFile.Close() + os.Remove(mergedPath) + return fmt.Errorf("cannot write merged %s profile to %q: %w", profileType, mergedPath, err) + } + mergedFile.Close() + for _, profilePath := range profilePaths { + if err := os.Remove(profilePath); err != nil { + return fmt.Errorf("cannot remove %q: %w", profilePath, err) + } + } + t.Logf("%s profile was successfully written to: %v", strings.ToUpper(profileType), mergedPath) + + // Do post-processing on the merged profile. + for _, postProc := range postProcessors { + if err := postProc(ctx, t, run, profileType, mergedPath); err != nil { + return fmt.Errorf("failed to post-process profiles: %w", err) + } + } + return nil +} + +func processProfilingMetrics(ctx context.Context, t *testing.T, run *profileRun) error { + var metricsLogs []string + err := filepath.Walk(run.localProfileDir, func(path string, info fs.FileInfo, walkErr error) error { + switch { + case walkErr != nil: // Keep walking other directories, so don't propagate error here. + case info.IsDir(): + case !strings.HasSuffix(filepath.Base(path), ".metrics.log"): + default: + metricsLogs = append(metricsLogs, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("cannot traverse %q: %w", run.localProfileDir, err) + } + for _, metricsLog := range metricsLogs { + metricsviz.FromProfilingMetricsLogFile(ctx, t, metricsLog) + } + return nil +} + +// removeEmptyDirectories recursively removes empty dirs under `fromDir`. +func removeEmptyDirectories(fromDir string) error { + for keepGoing := true; keepGoing; { + var toRemove []string + err := filepath.Walk(fromDir, func(path string, info fs.FileInfo, walkErr error) error { + if walkErr != nil { + // Keep walking other directories, so return nil here. + return nil + } + if !info.IsDir() { + return nil + } + dirEntries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("cannot read directory %q: %w", path, err) + } + if len(dirEntries) == 0 { + toRemove = append(toRemove, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("cannot traverse %q: %w", fromDir, err) + } + keepGoing = len(toRemove) > 0 + for _, path := range toRemove { + if err := os.Remove(path); err != nil { + return fmt.Errorf("cannot remove %q: %w", path, err) + } + } + } + return nil +} diff --git a/test/kubernetes/benchmarks/pytorch_test.go b/test/kubernetes/benchmarks/pytorch_test.go new file mode 100644 index 000000000..4cb1ab5d2 --- /dev/null +++ b/test/kubernetes/benchmarks/pytorch_test.go @@ -0,0 +1,426 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pytorch_test contains benchmarks using the pytorch "torchbench" repo. +package pytorch_test + +// These tests use pytorch's "torchbench" suite (https://github.com/pytorch/benchmark/tree/main). +// The Authors describe the benchmarks in this paper: https://arxiv.org/pdf/2304.14226.pdf +// The Authors list both the type of model and its profile (how GPU intensive). + +// Note: The image for this test is about 7-8 GB as of writing. After you get your clusters up and +// running, start the test and make sure that the pods show the event of downloading the image. Then +// get a cup of coffee, chat with your co-workers for 5 min, and it will be about done 5 min after +// that. You'll only need to do this once for each cluster (in parallel). + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + + v13 "k8s.io/api/core/v1" +) + +const ( + pytorchImage = "gcr.io/gvisor-presubmit/benchmarks/pytorch_x86_64:f6f280aeb1b07989" +) + +// pytorchTestType is the method used, either training or evaluation, for the model. +type pytorchTestType string + +const ( + train = pytorchTestType("train") + eval = pytorchTestType("eval") +) + +type pytorchMode string + +// pytorchMode is the pytorch mode used, either script mode (jit) or eager mode. +// See: https://towardsdatascience.com/pytorch-jit-and-torchscript-c2a77bac0fff +const ( + jit = pytorchMode("jit") + eager = pytorchMode("eager") +) + +type pytorchTest struct { + module string + test pytorchTestType + mode pytorchMode +} + +// Name returns the name of the test with the argument parameters included. It is formatted so +// that it can be used for the name of the pod. +func (p pytorchTest) Name() string { + // Kubernetes pod names cannot contain "_". + module := strings.ReplaceAll(strings.ToLower(p.module), "_", "-") + return fmt.Sprintf("%s-%s-%s", module, p.test, p.mode) +} + +var snakeCase = regexp.MustCompile("_.") + +// BenchName returns the name of the test with the argument parameters included. +// It is formatted so that it can be used for benchstat output. +func (p pytorchTest) BenchName() string { + // First letter of the module should be capitalized, as it will be + // concatenated with "Benchmark" and it's useful to mark it as a different + // word. + // Some modules use a lowercase first letter, e.g. "fastNLP_Bert". + moduleName := strings.ToUpper(p.module[:1]) + p.module[1:] + // We also replace "snake_case" with "snakeCase". Sorry snakes. + moduleName = snakeCase.ReplaceAllStringFunc(moduleName, func(s string) string { + return strings.ToUpper(strings.TrimPrefix(s, "_")) + }) + test := strings.ToUpper(string(p.test)[:1]) + string(p.test[1:]) + var mode string + switch p.mode { + case eager: + mode = "Eager" + case jit: + mode = "JIT" + default: + panic(fmt.Sprintf("Unknown mode: %v", p.mode)) + } + return fmt.Sprintf("%s/%s/%s", moduleName, test, mode) +} + +func (p pytorchTest) toPod(namespace *testcluster.Namespace, image string) (*v13.Pod, error) { + pod := namespace.NewPod(p.Name()) + pod.Spec = v13.PodSpec{ + RestartPolicy: v13.RestartPolicyNever, + Containers: []v13.Container{ + { + Name: p.Name(), + Image: pytorchImage, + Command: benchmetric.TimedCommand(p.command()...), + }, + }, + } + return pod, nil +} + +func (p pytorchTest) command() []string { + return []string{ + "python", + "run.py", + p.module, + "--device", "cuda", + "--test", string(p.test), + "--mode", string(p.mode), + } +} + +// TestFastNLPBert uses the fastNLP_Bert module, which is classified as a NLP Language Model. +// fastNLP_Bert taxes the GPU heavily with low data movement. See Figure 2 on +// page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/fastNLP_Bert +// Bert Blog Post: https://towardsdatascience.com/bert-explained-state-of-the-art-language-model-for-nlp-f8b21a9b6270 +// Paper: https://arxiv.org/abs/1810.04805 +func TestFastNLPBert(t *testing.T) { + ctx := context.Background() + const module = "fastNLP_Bert" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +// TestBigBird uses the hf_BigBird module, which is classified as a NLP Language Model. +// hf_BigBird taxes the GPU moderately with low data movement. See Figure 2 on +// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/hf_BigBird +// Paper: https://arxiv.org/abs/2007.14062 +func TestBigBird(t *testing.T) { + ctx := context.Background() + const module = "hf_BigBird" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +// TestSpeechTransformer uses the speech_transformer module classified as "Speech Recognition" +// model. speech_transformer has a lot of idle time for the GPU. See Figure 2 on +// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/pull/374 +// Paper: https://arxiv.org/abs/1706.03762 +func TestSpeechTransformer(t *testing.T) { + ctx := context.Background() + const module = "speech_transformer" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +// TestLearningToPaint uses the LearningToPaint module classified as "neural renderer in model-based +// Deep Reinforcement Learning (DRL)". +// Learning to paint has a lot of "data movement" and doesn't tax the GPU a lot. See Figure 2 on +// page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/LearningToPaint +func TestLearningToPaint(t *testing.T) { + ctx := context.Background() + const module = "LearningToPaint" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: jit, + }, + { + module: module, + test: eval, + mode: jit, + }, + } + runTests(ctx, t, tests) +} + +// TestMobileNetV2 uses the mobilenet_v2 module classified as "Computer Vision: Image Classification". +// MobileNet has a lot of taxes the GPU. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/mobilenet_v2 +// Paper: https://paperswithcode.com/method/mobilenetv2 +func TestMobileNetV2(t *testing.T) { + ctx := context.Background() + const module = "mobilenet_v2" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: jit, + }, + { + module: module, + test: eval, + mode: jit, + }, + } + runTests(ctx, t, tests) +} + +// TestBackgroundMatting uses the Background_Matting module classified as "Computer Vision: Pattern Recognition". +// BackgroundMatting has a lot of GPU idle time. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf +// +// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/Background_Matting (see README) +func TestBackgroundMatting(t *testing.T) { + ctx := context.Background() + const module = "Background_Matting" + tests := []pytorchTest{ + { + module: module, + test: train, + mode: eager, + }, + { + module: module, + test: eval, + mode: eager, + }, + } + runTests(ctx, t, tests) +} + +func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) { + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("PyTorch", func(t *testing.T) { + t.Parallel() + for _, p := range tests { + t.Run(p.Name(), func(t *testing.T) { + doPytorchRun(ctx, t, k8sCtx, cluster, p) + }) + } + }) + }) +} + +func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("Failed to reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + image, err := k8sCtx.ResolveImage(ctx, pytorchImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + pod, err := params.toPod(benchmarkNS, image) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to configure pod for test-nodepool: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, pod.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + if err := cluster.WaitForPodCompleted(ctx, pod); err != nil { + t.Fatalf("Failed to wait for pod to complete: %v", err) + } + + rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("GetLogReader on cluster %q pod %v: %v", cluster.GetName(), pod.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", pod.GetName(), err) + } + + metrics, err := parseStandardOutput(string(out)) + if err != nil { + t.Fatalf("parseStandardOutput: %v", err) + } + + containerDuration, err := benchmetric.ParseTimedContainerOutput(string(out)) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + + metrics = append(metrics, benchmetric.BenchmarkDuration(containerDuration)) + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + if err := recorder.Record(ctx, params.BenchName(), metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } +} + +func parseStandardOutput(output string) ([]benchmetric.MetricValue, error) { + gpuTimeMillis, err := parseGPUTime(output) + if err != nil { + return nil, fmt.Errorf("parseGPUTime: %v", err) + } + + gpuPeakMemoryGB, err := parseGPUPeakMemoryGB(output) + if err != nil { + return nil, fmt.Errorf("parseGPUPeakMemory: %v", err) + } + + cpuPeakMemoryGB, err := parseCPUPeakMemoryGB(output) + if err != nil { + return nil, fmt.Errorf("parseCPUPeakMemory: %v", err) + } + + return []benchmetric.MetricValue{ + benchmetric.SpecificDuration(time.Duration(gpuTimeMillis)*time.Millisecond, "gpu-runtime"), + benchmetric.SpecificBytes(gpuPeakMemoryGB*1024*1024*1024, "gpu-peak-memory"), + benchmetric.SpecificBytes(cpuPeakMemoryGB*1024*1024*1024, "cpu-peak-memory"), + }, nil +} + +var gpuTimeRegex = regexp.MustCompile(`GPU\sTime:\s*(\d+\.\d+)\smilliseconds`) + +func parseGPUTime(output string) (float64, error) { + match := gpuTimeRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find GPU Time: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +var gpuPeakMemoryRegex = regexp.MustCompile(`GPU\s0\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) + +func parseGPUPeakMemoryGB(output string) (float64, error) { + match := gpuPeakMemoryRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find GPU Peak Memory: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +var cpuPeakMemoryRegex = regexp.MustCompile(`CPU\sPeak\sMemory:\s*(\d+\.\d+)\sGB`) + +func parseCPUPeakMemoryGB(output string) (float64, error) { + match := cpuPeakMemoryRegex.FindStringSubmatch(output) + if len(match) < 2 { + return 0, fmt.Errorf("failed to find CPU Peak Memory: %s", output) + } + return strconv.ParseFloat(match[1], 64) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestFastNLPBert": TestFastNLPBert, + "TestBigBird": TestBigBird, + "TestSpeechTransformer": TestSpeechTransformer, + "TestLearningToPaint": TestLearningToPaint, + "TestMobileNetV2": TestMobileNetV2, + "TestBackgroundMatting": TestBackgroundMatting, + }) +} diff --git a/test/kubernetes/benchmarks/redis_test.go b/test/kubernetes/benchmarks/redis_test.go new file mode 100644 index 000000000..17b682cc7 --- /dev/null +++ b/test/kubernetes/benchmarks/redis_test.go @@ -0,0 +1,474 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redis_test holds the redis test where the runtime under test runs a redis server and the +// native runtime runs a client making requests against it. +package redis_test + +import ( + "context" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + redisPort = 6379 + defaultRequestsPerConnection = 50000 + + redisImageAMD = "gcr.io/gvisor-presubmit/benchmarks/redis_x86_64:latest" + redisImageARM = "gcr.io/gvisor-presubmit/benchmarks/redis_aarch64:latest" + redisServerLabelKey = "app.kubernetes.io/name" + redisServerLabelValue = "redis-server" + redisVolumeName = "redis-data" + redisDataDirectory = "/redis-data" +) + +var ( + numConnections = []int{1, 2, 4, 8, 16, 32} + latencyPercentiles = []int{50, 95, 99} + operations = []string{"SET", "GET", "MSET", "LPUSH", "LRANGE_500"} +) + +// TestRedis benchmarks redis servers on k8s clusters. +func TestRedis(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("Redis", func(t *testing.T) { + t.Parallel() + doRedisTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// doRedisTest runs redis test. +func doRedisTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + persistentVol := benchmarkNS.GetPersistentVolume(redisVolumeName, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := redisImageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = redisImageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + for _, test := range []struct { + // Benchmark name + name string + // Suffix for the redis server, must be short to fit in pod name. + suffix string + // redis-server command-line. + serverCommand []string + // Volume to use for persistence, if any. + volume *v13.PersistentVolumeClaim + }{ + { + name: "Persistence", + suffix: "persist", + serverCommand: []string{ + "redis-server", + "--dir", redisDataDirectory, + // Default save settings per + // https://redis.io/docs/management/config-file/ + "--save", "3600 1 300 100 60 10000", + }, + volume: persistentVol, + }, + { + name: "NoPersistence", + suffix: "nopersist", + serverCommand: []string{ + "redis-server", + "--appendonly", "no", + "--save", "", + }, + volume: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + + // Create a server on the runtime under test nodepool. + server := newRedisPodWithPort(benchmarkNS, fmt.Sprintf("redis-%s", test.suffix), image, test.serverCommand, redisPort, test.volume) + if server.ObjectMeta.Labels == nil { + server.ObjectMeta.Labels = make(map[string]string) + } + server.ObjectMeta.Labels[redisServerLabelKey] = redisServerLabelValue + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err) + } + + server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err) + } + + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err) + } + + // create a service to service traffic to the pod. + service := newRedisService(benchmarkNS, server.GetName()) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err) + } + defer cluster.DeleteService(ctx, service) + if err := cluster.WaitForServiceReady(ctx, service); err != nil { + t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err) + } + + ip := testcluster.GetIPFromService(service) + if ip == "" { + t.Fatalf("did not get valid ip: %s", ip) + } + + // run the 'redis-cli' command to ping the server and make sure it is up. The "ping" request comes + // back with a "PONG" response. We repeat -r=5 times with a -i=1 second interval. If we + // get one PONG back then the server is considered up. + pingCmd := []string{"redis-cli", "-h", ip, "-r", "5", "-i", "1", "ping"} + ensureUp := func() error { + pinger := newRedisPod(benchmarkNS, fmt.Sprintf("rpinger-%s", test.suffix), image, pingCmd) + pinger, err = cluster.ConfigurePodForClientNodepool(pinger) + if err != nil { + return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pinger.GetName(), err) + } + + pinger, err = cluster.CreatePod(ctx, pinger) + if err != nil { + return fmt.Errorf("CreatePod %q on cluster %q: %v", pinger.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, pinger) + + waitCtx, waitCancel := context.WithTimeout(ctx, 30*time.Second) + var podWaitSuffix string + if err := cluster.WaitForPodCompleted(waitCtx, pinger); err != nil { + podWaitSuffix = fmt.Sprintf(" (pod wait error: %v)", err) + } + waitCancel() + + rdr, err := cluster.GetLogReader(ctx, pinger, v13.PodLogOptions{}) + if err != nil { + return fmt.Errorf("GetLogReader on cluster %q: %v%s", cluster.GetName(), err, podWaitSuffix) + } + out, err := io.ReadAll(rdr) + if err != nil { + return fmt.Errorf("failed to read from pod: %q: %v%s", pinger.GetName(), err, podWaitSuffix) + } + + if !strings.Contains(string(out), "PONG") { + return fmt.Errorf("mismatched output: wanted: PONG got: %q%s", string(out), podWaitSuffix) + } + + return nil + } + var isUpErr error + serverUpCtx, serverUpCancel := context.WithTimeout(ctx, 100*time.Second) + defer serverUpCancel() + for serverUpCtx.Err() == nil { + if isUpErr = ensureUp(); isUpErr == nil { + break + } + } + if isUpErr != nil { + t.Fatalf("%s at IP %s did not come up: %v", server.GetName(), ip, isUpErr) + } + + for _, connections := range numConnections { + t.Run(fmt.Sprintf("Connections_%d", connections), func(t *testing.T) { + for _, operation := range operations { + t.Run(operation, func(t *testing.T) { + // Create a client for this client run w/ the specified number of connections. + // Sadly the --csv mode only reports QPS, not latency. In order to report both, + // we need to parse the human-readable version of the output. + clientCmd := []string{ + "redis-benchmark", + "-t", operation, // RPC to benchmark + "-h", ip, // Redis server IP + "-n", fmt.Sprintf("%d", defaultRequestsPerConnection*connections), // Number of total requests to do + "-c", fmt.Sprintf("%d", connections), // Number of threads to spread them over. + "-r", "1000", // Key space size (larger = more memory faults) + "--precision", "4", // Floating-point precision for reporting latency (in ms) + } + client := newRedisPod(benchmarkNS, "client", image, clientCmd) + client, err = cluster.ConfigurePodForClientNodepool(client) + if err != nil { + t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + client, err = cluster.CreatePod(ctx, client) + if err != nil { + t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err) + } + defer cluster.DeletePod(ctx, client) + + if err := cluster.WaitForPodCompleted(ctx, client); err != nil { + t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err) + } + + // get and parse the logs from the client to get the results + rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{}) + + if err != nil { + t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err) + } + + out, err := io.ReadAll(rdr) + if err != nil { + t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + redisBenchmarkName := fmt.Sprintf("Redis/%s/%dClients/%s", test.name, connections, operation) + metrics, err := getMeasurements(string(out), operation) + if err != nil { + // Redis uses '\r' to update its status by overwriting the current line. + // If printed directly, this messes up the output. + // To make that clear, we replace '\r' with a literal + // backslash + 'r', and add a newline. + humanReadableOut := strings.ReplaceAll(string(out), "\r", "\\r\n") + t.Fatalf("failed to get metric for op %q: out:\n\n%s\n\nerr: %v", operation, humanReadableOut, err) + } + // We don't multiply `defaultRequestsPerConnection` by `connections` here + // because the number of "samples" we're testing is the number of times we + // can call an RPC from *that many connections* (which is part of the + // benchmark name). + // Adding 5x the number of connections does not make the sample size of this + // benchmark go 5x higher. + if err := recorder.RecordIters(ctx, redisBenchmarkName, defaultRequestsPerConnection, metrics...); err != nil { + t.Fatalf("Failed to record benchmark data for op %q: %v", operation, err) + } + }) + if t.Failed() { + break + } + } + }) + if t.Failed() { + break + } + } + }) + if t.Failed() { + break + } + } +} + +// newRedisService gets a service to serve traffic to the redis server. +func newRedisService(namespace *testcluster.Namespace, containerName string) *v13.Service { + name := fmt.Sprintf("redis-service-%d", time.Now().UnixNano()) + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{redisServerLabelKey: redisServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: redisPort, + TargetPort: intstr.FromString(containerName), + }, + }, + }) +} + +var ( + latencyPercentileRegex = regexp.MustCompile("^([-,.\\d]+)% <=? ([-,.\\d]+) milliseconds(?: \\(cumulative count .*\\))?$") + latencyStartHeader = "Latency by percentile distribution:" + queriesPerSecondRegex = regexp.MustCompile("^throughput summary: ([-,.\\d]+) requests per second$") +) + +func stringToFloat64(s string) float64 { + f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64) + if err != nil { + panic(fmt.Sprintf("cannot convert float %q: %v", s, err)) + } + return f +} + +// getMeasurements parses the output of redis-benchmark to get the stats. +func getMeasurements(out, operation string) ([]benchmetric.MetricValue, error) { + var currentOperation string + var returned []benchmetric.MetricValue + inLatencyBlock := false + foundPercentiles := make(map[int]bool, len(latencyPercentiles)) + foundQPS := false + lastPercentile := -1.0 + lastPercentileLatencyMs := math.NaN() + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + // The human-readable output contains a bunch of data like + // "OPERATION: number\r" which are used to update the result + // interactively. Strip them out here. + if strings.Contains(line, "\r") { + line = line[strings.LastIndex(line, "\r")+1:] + } + if strings.HasPrefix(line, "====== ") { + currentOperation = strings.SplitN(strings.Trim(line, "= "), " ", 2)[0] + } + if currentOperation != operation { + continue + } + if line == latencyStartHeader { + inLatencyBlock = true + continue + } + if inLatencyBlock { + latencyMatch := latencyPercentileRegex.FindStringSubmatch(line) + if latencyMatch != nil { + percentile := stringToFloat64(latencyMatch[1]) + if percentile < lastPercentile { + continue + } + latencyMs := stringToFloat64(latencyMatch[2]) + if percentile == 0 { + lastPercentile = 0 + lastPercentileLatencyMs = latencyMs + continue + } + // Look for all percentiles in `wantPercentiles` that are in the range [lastPercentile, + // percentile]. + var recordPercentiles []int + for _, wantPercentile := range latencyPercentiles { + if float64(wantPercentile) < lastPercentile { + continue + } + if float64(wantPercentile) > percentile { + continue + } + if foundPercentiles[wantPercentile] { + continue + } + recordPercentiles = append(recordPercentiles, wantPercentile) + } + for _, recordPercentile := range recordPercentiles { + // Linear interpolation of the latency value from within the latency range in the two + // percentile values that we got. + // For example, given p50=1.0ms and p70=2.0ms, we infer that p60=1.5ms. + // This isn't bulletproof but it is better than rounding to either end of the bucket. + rangeFraction := (float64(recordPercentile) - lastPercentile) / (percentile - lastPercentile) + pctileLatency := rangeFraction*(latencyMs-lastPercentileLatencyMs) + lastPercentileLatencyMs + returned = append(returned, benchmetric.SpecificDuration(time.Duration(pctileLatency*float64(time.Millisecond)), fmt.Sprintf("p%d", recordPercentile))) + foundPercentiles[recordPercentile] = true + } + // Update values for next round. + lastPercentile = percentile + lastPercentileLatencyMs = latencyMs + } else { + inLatencyBlock = false + } + continue + } + qpsMatch := queriesPerSecondRegex.FindStringSubmatch(line) + if qpsMatch != nil { + if foundQPS { + return nil, fmt.Errorf("found QPS value multiple times: %q", line) + } + foundQPS = true + returned = append(returned, benchmetric.RequestsPerSecond(stringToFloat64(qpsMatch[1]))) + } + } + if !foundQPS || len(foundPercentiles) != len(latencyPercentiles) { + return nil, fmt.Errorf("did not find the data we wanted: foundQPS=%v foundPercentiles=%v", foundQPS, foundPercentiles) + } + return returned, nil +} + +// newRedisPodWithPort returns a redis pod template. +func newRedisPodWithPort(namespace *testcluster.Namespace, name, image string, cmd []string, port int32, pvc *v13.PersistentVolumeClaim) *v13.Pod { + container := newRedisContainer(name, image, cmd) + container.Ports = append(container.Ports, v13.ContainerPort{Name: name, ContainerPort: port}) + if pvc != nil { + container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{ + Name: redisVolumeName, + MountPath: redisDataDirectory, + }) + } + + pod := namespace.NewPod(name) + pod.Spec.Containers = []v13.Container{container} + if pvc != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{ + Name: redisVolumeName, + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.GetName(), + }, + }, + }) + } + return pod +} + +// newRedisPod returns a redis pod template. +func newRedisPod(namespace *testcluster.Namespace, name, image string, cmd []string) *v13.Pod { + pod := namespace.NewPod(name) + pod.Spec.Containers = []v13.Container{newRedisContainer(name, image, cmd)} + return pod +} + +// newRedisContainer returns a new redis container. +func newRedisContainer(name, image string, cmd []string) v13.Container { + return v13.Container{ + Name: name, + Image: image, + Command: cmd, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestRedis": TestRedis, + }) +} diff --git a/test/kubernetes/benchmarks/resources/BUILD b/test/kubernetes/benchmarks/resources/BUILD new file mode 100644 index 000000000..6732ef220 --- /dev/null +++ b/test/kubernetes/benchmarks/resources/BUILD @@ -0,0 +1,13 @@ +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//test/kubernetes:__subpackages__"], + licenses = ["notice"], +) + +filegroup( + name = "files", + srcs = glob(["*"]), + visibility = [ + "//test/kubernetes/benchmarks:__subpackages__", + ], +) diff --git a/test/kubernetes/benchmarks/resources/chart.png b/test/kubernetes/benchmarks/resources/chart.png new file mode 100644 index 0000000000000000000000000000000000000000..d2aebc1430a38bdb4a5a87bb275009af04fe5dba GIT binary patch literal 46224 zcmeAS@N?(olHy`uVBq!ia0y~yU|Ime9Bd2>3@$e3k1{YYa29w(7Bet#3xP1>rMq>1 z7#OxVdAc};RNQ*Am%Sp^vud8^$xrK4-&duc{#5n-;<~-cy}6!Ya*>xjHeHI6>NQ#U zxa6Ggrkg%z^V-yAFFcv|Y)P5(W$@SVxvSp)nOgTOy+y!@LvfTC4TAwp2N#76AxY3mfHMHXV$=es zNfn1yu)8KOh=P-;&;l^!R+bek8N$82y5x5 z#l>qEEI4p<>!wYY^6u_Ry}mAX^XcjO(XOsXg?HBH=C1t>b}7>q?fSZHt2r;MTBUXR z{Q3OAm}N_s-dy52`OSre&VFA?MNgeRo%{RSTeGYy8n5r}-o7?sSVQot>S| z%+9x^?(eV2tgNiv_jfN@a%7rbY*x3J?xx-4?_aGHS-Wi6vSWGo_GDUGT84#(t2c8f zx|oYCu>aos{>J+MU!%X>`!Lh2=$2RY#{1L1z5jpPJ6^a(x3=!@`g@zg*?l*~|7(7J z*RVeR$A|6!3;VL;`@ifyf8#W>+Ku`@-Ph%%u5I7~t*Cdbmf{W@bkBwfCD>7>nop zJ}B{f>;9V0593ZwsCfEb{{M>osqtsG%YT)6-8cWe^}VjH({J7`ynNU4|KqI>%M73I z`uoqm?!$!-0teiagWkmdzqi@<{@-uqRX2a0iTOP_ovnnI(W}`a{hUno=dlzw<%ywF%?On!GVR_tYcZcNGV3?JfJ8ws7s;$}9h# z=Knk3c0MAv-A~o({EPd0Y_72NMQ6O{oBi<6uk8!p&-YPvQ=dL7m1V5@`YQDIi^ctKwqB3hJkvORSK+*~jo}+tcQLUPsW9#; ze0=P<&;pK@o14>j=hYmW>9)x;Tv}YZ2#*(fi)AaK~GREw#_j z3YL96rhB_o2BJ^Y;G&#RV3aEUW(huC&)SS@rj~x7jyu-V{8_a<`?Tpy_UFPx9i8y4u>s|1Yn& zwBd8-l?|V*c};lt=l}V6`p0GK@^>b;H9p%iDd_8;uk2MSjocR2(007B_=iCKAywVD z7UeJxwYZ!ys2od|7!mB>_eAdt2-H<>#q?0|G@I!ynA&wX56>rUnpHy7pWL?49dtqo3RT@z-~(`1$!;ng`p%M~3`u$Nyivcu_C*hU24P4#lcz z3{S0hooT-NAUSPqsge+5*9#u502h@-U;FEB*sIw5U-)v@|NmDL%M5?-@{QSX@|#R= z@na|c`Cgkcy|+cZIvVr)_4M`~AH?!+<-9$& z(&yKv%{I?3JH#<@Tl#sq*%pP1W;3`X`yRHg`jS!ZlPtq0JadMGN!l5S=b=5ForS$t z*Lmda?sPJ?gbVE2`*PbH)9h)V6$Jg;coe#tcq_A6U$k-dey{#qo61eQr7E7YUvF$oHcL3b5bo!p z{XAGqX|DV`OO^SW{`q@8wtYU8Y=5d-KK``j6ysBePEFOm+diH7((OwYUO6-7%z1Ok zUGgbw=az)xIGuez9(A)HvsbQ}d8Tdqq>mR{&a!{>yd|)>PR8W?@kVC$IeR(h7hbxd zH6#DV!_VjK%gdg&wj>`h{(gmv%ZcOSUe*`;f4!Bzd5t-6Th3?2-{qiUuF^x|-}3#H zxBc2$?(VRBopa^O)oq_YWd5GM|10zB|39157_0OiZ>lvei7WV{Cfw(zzBKya+$3qs z+^Nwza*ELrAz6W(&trcCJ10%4{;U80rTkm}`La)3o=VI6Z(Q)fEULE^fn@{B1{Qr_?R+$YTCDCpsQ$}rC4Sm(L<_WM_M zi>+5ZURQMKObG^ z?W4xyJ+B28cDU|({`1`Z$)7vq@9Dj3uRTBCK3~56&&JOlRqypryC^Q(@qCJA@SGio zvQOUUxBGFR^TM`s>%RYNx-Qx%?b9#5;E7->AmLq9?P!*WuDThMiSxYp0jooNr{VJpJEngx5507 zaCP!+g*%h8Pb43!`($+7;)0@cn}wjSrGOL1MO#)49PjPEXzcckXYdDK|g6FPi+;f4<}kmbZ7V z?#{h8>+ZDLr)__?%YS?%99t?GUmcA0SnPecE5mV$ z;m75c{^E;-5>0b%Y+zUWs!$-%az*doDBLT) zK<eP(PNH4{0;H-v)Z>PHb`(i+Xx1Xv-~GzI?Yh=l|K_8=j=^P#5EuOA-it%PNxE@cE-a z8^655d4aBjpOp`OirgckaHTmh;NOmUdpOn|I`F?E^waX4Q#k|{R0JKK{#eQXL(=|F zr?mCn{fl8yN}4G?r{IOx@nw4?TDjx)9Q_x`(WLNHwO3rTMbByOj!ACsoa25M>9hze zs^$#1|MXw){+i3Bt(*S{-u?Lhx%}ODyG~A*^I!hmDQ*A%S9*S%o${Oie>aEE^>ATm zJD>D_{r#7z`n)q9*sQ*H_}8feo!@5Hf1hXl@A>^d?&>YD+_(RS^!@T=J9`=1^Z&k0U6|Kq8+XM|bODFj5n(xV7vUt%hYrr29o+db zf)y3Ihgmgx^>~fs^!&Y(#XqJ!R6NWq@-Ec*LqR?gZinc~_dPj^bVHB=mmtLWRSE08ep6w`C#YvLP^{>&9z@aJgs{W%if;wo&;tev~V zb)T|`TlE}*3JafV{S2bYISdBqnTx!gT}or;5`Sp$QsL2G%mF#DrJJN(}# zzT;vNkyiM1(px?~=fr8wrY=vrwI3UV*fP_NLM@(rU0mhxv)e&gZ{ne}u=-+W)8d|JKIRH7 zcqq5ggGr>YRea_{=7)zR1eV7MR4u$B#`vn7Gwbb{AJ@17_MG53&Jo{ufWPy=Z0S1x zBNi|FOxR2&ne*RnoqvB4L+U2~B}K=2{(VsG6R5bcVQH_3MgMQ3NX~whOgESI<)`}@ zm&&+Eai4YBy{(T?HJ`Je)BT}$VW8bkHbt!#fkn?b1EvKf`Mmi5E!uuV{om^ImVqbB zW;uTRA^rbxtoQzstM{ag&c2N}a>Fle>vw76`??AJ;#0FD3gphecwf7{fA*In(`KJ( z`yc=Rv;EGQ?P z7MXcwO4^>6N4v#mi!P{}Sas?6j}R6;;rymUY>sDl9z1Cj*I@K#fhwa{lz>ag{l^n? z&VI7pBchPE!LitYZ_*dVu*0XB_EA3Dh*cFOnl3y*O779YCi- z;Vy8gWAFC-`~SA4bIpFh$X9kdIBegaPu}Uqj}#!Go3?gyj*z!;?w#U8VY&86?TaVn zR=(IB|MRf;EEbK^nud?>zSaGo{r|zYDfWWp=X|}A#H`~k+PXh}yMMpIjb*wwpKW^? zv-$Cg*;nmqK5^at8@gup&vQaPpssp}EMpgkgMgw<<^w^aGaPOW6(2YhtuoKNG+MIJ zeTQN8oWe4bQznZ_oRv~CKL~EFdnCl9VsUb@Dr1*BN1{vfnG~KqZXH1_GrkKM-sCvg zppm-Gu27Y?LR?Vxp0ekjPI;#GHo1iz5e_GQ6?SsS-~XX(A|PHTsd(wk6X_$YOjCXs z+8os9JHNH1@vOx0_USXKm0AQhR@exxzc-CT#>HmaLAN=lOLcC#nCpC3kC|rs>2$=! zr#&Yg&ATDUZ*pkLAE!Tsyw4}-cSOH0?45F+qnbthu!zO{Q1%%q&US6p{2!$zt@GHt z;qR&u@lO%%>=D^Bo_TU1fW`QO0q>U93NHVt_l$JXyc{nqzcFF#9v zl{^uq2GBexb|9&l16lB$yAI|2xY04Iv-)Hw%-#=TGF703Dm*nl2wnMnz z{^v!8?fYKH*S&ZC{=0>B_ML4DDtvbLFf46!zH^b`(heiz>1`$!2ix1`z5mf${NsXb zZ#j?Ri}lVrmwsHfFupeNJ?HfIe^sWna|TEyzdQQ1Xs;%Ob7kswf(WF4niGRJGx3Z!4tj_|`b!$Bxz7)mV%&v$KbZR+Y zyJxMy%DR&(Edpgn(>mPh`4%1FE@@FK{LMMX zd#aVoHnj6MXTNY$-{17@XgwQSdo-Te#BqJd`Ml}bY1VC*>OMGed@SL(aAOM3_FT2QPdT3(-QD+L zB8!IbG`H3kwZ~5N+A0q1zENqL!`>7;KNGq2^uKpsALcz*-t}E@_IsYWdB)SW2K~R=M@{-KT(>6AJ{M^&`cR1S2cHDSnR!!fO z>pb%p{I<2Yw%ziN8bgKGM(1p`-&>yk`S4@1`jg5fMo$kP+qPCTf35E2iH{tm{*-O* z&w9ULfx6EAzpDhCI+lkw7&)%YnGeVhFEzWBQP&&77RAIQ5=d*|%B|I5qWZtjXbp7Zux zvRUy_zwP-p!t+!dn16ii?YWs8qjr%USMYcC)zpEFUm!HXB z3L3Swna7Yi%}UNQZTaSX>e+QhlmAYy`!4x?_Uj`4zu!*Rf7o+-X-B80&gL(h@BjEc z-^b%u-|x5g{~hH1#=ZZe?eC-a|Lprc&%pBkbLWSKi~rsKf3g3q_`lQNogRH|&r4V1 zyujcSm#CwDTHdg;V_nLz%vIW(N*7(^7tQG3AknYMzeT`_<6{cziZZ8HObRA{dy@WV zzW(6=aux??0CYGR&bW9lB|Uy?e*N#t-^a}Vt-4)b_q=>x_Qpa@=s0xj%ujP`TK6f1VrT28$o6iq@4jUJ?o!ktE0zA4 zE|PqyoA%CE-v6`h)$?zSHQetK)U3OYSJ^qAuL+1n%%+o=KP{hvb==}(t+Ei?Wy@$w0(o+qrSI#!9h{> zVNYLOU7dIO^tNZA*$I-``-**KZAr8a$yh`x1b+EwKnR@)X!dA>Uwwz*C_ zD(C#)`DGK&h;3YRRC){ZqP9Khv+sUud7G{&Zk~THr1dtR{%z~>cO}uX?yHnH7(d;+ zc;0Fcr-+Vs$IjcFYj$w8s`!wweSKN^r>4@@Q#qfXoy`?{*TM0PJzcTxeu1og-JQK0 zB_yb2*Z9?&*$B*YO?|8pMJW(($EG+DD z(l_(GI|aQ~OoD+24!^T?uQ(*Y8}sn=~`!nEm%ho07~Q_jOEn+J>=&UUQ-9`UP2LB_?n@Eg;VVh{7g zM0S0y4&kyRdNJmShg#Ni20Zexs+FmY+0))s^QDsWiTX>0{U1sS_h(*Sc5|L>^{h{m z`nRh(_5M-HS*NJ)$)0+`yyJ=CLpD~8=Yf77u1sJ%^sxBN4a4wfdNDg5%(Ki-ztp!^ zi8JTjQOg@07R7wukLcZtTQdE*)b8Z)f9mm`r*{-Sj`{J&f9BG83OfJ){48?1@o(qT zs!N_f72+nE%}}WCVE^`{>QTpB<)^mWIH$Sl|KO09Ub=Lt>D}fcPg$3JiQbnQ-|pS; zz~+?U!{1fi=}p-i3`?a-#Y$z)uhHLpmJ*@d30Lz`{U|U@?{G9H_AT$S#{<5WxM@{UKiy&e*F4%^`*DD9kzeA9{l~u z_NPx(Q=d>hubJ|?CC9Ia9yD^iW#QKFh?V!+b4Tmnzs_F2dt~AcwrbfL>$bHUEi2t( zHfkO}xb1TL;r2bB&)dx25Ky*J{eIv&Pvgw*g^PNM^v<(AWP6>VxN9%N(}goEicd^grN_ux^XW5ezg5|oslA^5qALdvV$2j{v^-~qRH{+9n-+a5fX1TYX98%?x z^Vrkb;u>ybttD$yaUnF|NdK`}B^Oe=*?ZV$x6fK<^yt)Iv%EVmE;%rJ-Qkn5cyLZ# zE@axDz7r4T?b+%#)97euN65PzRj*%ta<;E>nj0cEBs6Bv*}zz+yf4Q6&JO%1T%H^Bn3b&ammw{(hkB3G@9# z`!~nFE)itx=>5_AsloSz_8Zs5POAQrO1UkLzjjR9v!o}XJYKJzPxh5omK5)&u0LNA z1;bBCJ)Q99&i4HIrx$NbK5iGVY{i3()BBYl2FMB4-4OMc->)CP@6J^HQ@p(s^`9@{ zceF@3Ay6)V?&-Y6?)_!G#=RD)>Hq%xTsJ3?`4sQ-iFW#J9nzp&^781s8}n_B8#mYP z(Eq;J=i&k-mWn(N<@<@(KP0kExqWi7`tA)K-16s?*DVoeZ1ebVKhgV=bEN8>MrL-i zn@!o8pO3ilJ~m)v41%{-(ZwsMD^n&YqimHGSPa}TOt>AgGK z{{JNZp{F*Nj+@??yF-xe<7el}3NefR>YzRg`P&KJ`(MjsD)wEu%i(cKu1w+W4aVD( z#BVg)DAi73pO^e;%8nxcdC8}EuLsL%EI+tQxn>2=Q|Z~l>W|!IT`C_%Iwbu1*(QV}QPtg9+fyyhnQ{3kX|2I1E z^9gIFg5AM5_ERFygZw{R|2+Nw$V`h*593-S@5Kq3`ElQ3I3@6O+McIXZybJUxL`}<5zS~?XY;__~@V`&&P(qogdAPe-H7pQ3#0%X{o8WQ1PI$ zSlY5^N%f8uDU}g_Te$8`Sh=O5VnxKtKQ%`WHoGM|rMe$=f9jIzpeiXN+{v-fXt~pu zdtQ(FITkS~_AO~_pVl?OYXVp2v5qcbLx&k0`iGVpDd)yGTO4Whl8G+RE&LXHIV5Nw z(=(fW-}9e$D7iFzzf|+<I6iO5Os|Pt6Lu8ImMQV>sr_y9z^YEU z)^1(gh3faJ*FRW)TP}3GY>8@3jhyJ5^);N6cL<9g`k>GBaB^CQdfAuQ&vzz!&22lp z|L)_1D(jZ?x-oz3tle{5Z1%+TBc1P0Bugc~ajtZDk(hmoS=6veVxouTu9j?IVu-|eS;n&})~^Ay*8j(ug9FJN`&aDDp?RsP-M{-x1K)bV1{CMdf1>ph=KXW}3E?AfC}xv1yEv28b+3q1LoItos3e`?&tx3S>S5zgsm z|9KQU4%sfa)4k{6)8^O5dn6y9+EM>(g?Uq4z&`bwPwfR7r}uOOZR;@BmwRHKdgFTL zk^bMm)>hQsb*=gncI?LF&$WM@?mE^5h5wiyvhVrLWqShn1Ke3omu~o{CNod=m9Bl< zoGnGR+mw56^t~7K-(w>>r?@H|Dxb*Nt>4J_^ydDQAL&MWc$NG2 zY;JkKLrnU|F0F_>!37+yk6i!kS@BeAUic}2@5=d2WqeMyhjva0u3H|r_|GSwN#B>W zr?>4{>i6-js@bEbJvI-ka*lkxF+Y9I^GRER&kMWfSl9e02(PbvzX_BBls0`x`?1}k zMJzuxBqXFEgw3QUq{(JUNQq5~3B#Q8?dC2lvZ6Zt6Y9O6I6ir}#OR^pvPF*=k39BL z?ag>peecRjkrfInJ(?Z{8CG)ie^_~^`N9Q3fBgj+8&>=gFbvsZ@-6PbB(`(Q5Aw!6 z-o16jpOtfJWCQ|2)c*P2?Z2z|?*>ct1&>P-OwWt*?K|e4n&38PdwL6>D5t1m;g>}x z!_(p#f&_ySe|pW{u;R>$6+h;6T`Nyc)TEHF-rMa)^7(}6I`jBHG4l&H z>F$~MLC)urd$aq|8*>;`|9gn{vZgj4ld={R*9wZy3|cAd@A5ugm#mf8w>}1BtQ4|$upnYs}X&0gGTTZ{|yKGP1vq*2fTUgcxGbUDe0z$|9(Cxe{;fEROh-)K8N11Z57=sTK?Rg@n2eGfsFDej&8Tv$Bu+s zgoxaGeW>}xUJ;uWAzgJgQ9m{X$Xvdl81m=)9Ph^mYHPPAe|WpulF>`qFZIcBS;4;V zX+eRkj}MggYdK_ctf=uY7FAMumB=)u?ZAFdX3+`xCuRvR_{Tf9?NoQ;g0~waGr#wC z=Py`sWtQ-Qdubd3TFz2wTHf!kGQE52I!}Imi-Y4Gp?s}UtuTAPsa^AT1fJW#{q6Jq zLjC)!@_G(}H|D%gUza5Pp7ATwA^nX{4lDU@*qGlE{iz|yuf|egeW!71OU;5ezZQzd zHME&uIm>xr0sHm#_q+dQerd|&vY|t|bk=_T`n?QG zdwv%#J^1_tcfl#aLvufM?`?XVvr)fvgZ}j^l69#(hgA1m;|j<(KgCzbyFT8+Ucz2J zexC7tZ_#^vKi+-%u<7h)=bc|y`~9#wu>Z`jqIb3z;u;=_e5iD@db#(bT)-pWs6FYo zZtq;nBoZym_crIpXF)xOll-^5_w!8AFK;?2QU0rQ6PpN1)c(QN%G?`06y5nqyfxzMl@egLollCwy{dBnR&TNB?!jBq%o2fgT zG(J}Fvv7NfJDYaaC!yPu?B2b1)iTTUYbfH?XKCr>6-+%cxnKk9_or6(r@n7D`%%DqgL)~GPof(gWOlS2>nnP3ysecx;9a-Jp=nQe3y;0-WLWy7$HiLlOTYkE%}P?@#4gUhl1}Xi(w|_|)2VTc6wYUKOL)ogec*Iokbp z>)Uj=wnIF1hT#rJ#;ygQ;uJnjdEKd;IN5EU|GDQjpYDF=Nsw*{{GoI28b?5mr^TIq zgG#r*F6!sh-*E_@(umz!cPMPh^J&kuH5@VxH0>Ym6<;g3V2bXcXO&0u{>k0wO7H*k zEasBqOr?EyW>-v_|3vhsL*$d6&u2c`x!hbSrr}rNvnkS_T8lTTA3EWGOeyElLDPxH z8tgn7z3w_#%sKqo<<3g}A9flJe}fWgH1w|+AK|tR=F>bqMZZqH_DT2-hNXK1%pbKx zEdH0bmO=IHjeFDYE|~C{N#yr$tJzkk&kKHdIze3b)2->xpRs5hzjK;9=M7w2(EEz5Wo+01vaZA)JUh?Vb+9_VmT@75OFG`<0{1O{f zr^L~#I5la*>l@|WpB`LH2)d}4w!&b=jUV&cCmXEzVWH-8=$RvT{fUedKkk*w1|AfY z*D{ey(Q>%t()#qcz^^>>bBl{UJ?>kPQxnc!_&K)AM6ZrXNR;{GBBoDqewUt0#~*qq zqm^>W@7Ip~`6(;jWbEMNw*K`ormjc&dSU*#&q=qA_qsJ#Bxt|aRFPkCp(gpb?O&hb zOZ~Z-3WwDf{`eSoqsL$+&tJ#O&|XN(Um7d)DGL*HVul%elY|C&slj{V!0zQ1)i&m^R!b?;tT!QK0I z&o=Vj)qMA1T*IXgmjZrvnjX2>U+!#ivEk<@@8$=#CU!h50=>$GOJxG>YVLTbzdSDX zM7+|o%7goQf!@jFIyCzDSjZW@$Tg317G*eo?|3Yqh>L`Nc5fJ$McMZs`5MA z>)&Yx*Rirl*?uypVYB?UY|SUz>l5RjFrN?lu=D7(InFjed!OH`Ub2z#siuZPP0Swl zxxWMC8b2K{cC4Ey_vyfOUYq_q&7Y3gcA6jRdd=DWyGU-)QTydjWRJHT{Q88o^37{q z{XHG}sduUuXd2hW7+()OCZJsB+L%za8(yyjxTSq7UGyaUBaI6%60vHeaFx3v-$bSXGh(fQi~hi zpS)(O?6a}9lmA;@ca&G&Zrju9caC#U{XD|>d$M0nYi=z+=S#kQOMaIb-MD>p^ONJP zlfVBv+V%YO_UZf9KCFIoqS!Uo?dPI9Kk};Hc>P?$e&D+HKK1quPgzCp*73Y}?BM!g z1LMVq&o6xV#dtrYB800#qh|5v6%X0==l7dzQR7>?H{Uza_EqWGx&JpDJ$PFxZOOXj z+islQukqvE@@=;s1m;`|`~BN+!Nhaf-FxL!_2o_X%bBn1j}E(CRe$Ma{JvY(b^fQ^ z-i978N&71;7w^(H|3v3<@xa#UxA^5V`+qGmeQN!}GI!;kt?sq|HW>X~k?ULi^!nl{ zfxa$6{(Bi}Lo!JH|hP|=UC4h$@%|( zT-@g~N z-BtVgUH9L)ucsTA-&@e@_8>)m(P2hD}LLh@p=@uG}4-XQ{odrRABy^`~-wAGMd2UY1*}rLkqtUiZ1?CclpE z3zL|wlgz0IUfyjZH`Sa=r~GVL_rH(rel_!c7EFn~U3lTuDeXJ&k3PSoyYM_mcV%jW znleniRqt2cu!-fD^=d57``O;UUAj}9Pa?T>`DD2n1y>^C^S>9Z+flgiWw7{e!$rMO z(>x*0TIBo4SjFIDn%XycVWD}oG2*{(3Qk`8=oYWK#o?9f*YR;)-T7|MltkgR&+SZ}~yO%}j%dEZi?(VKxdz1CKbXJyb@jI>O zn!ZwBcCxGxSoE4JfF-u}#_W>V*jSf|$1ATcG=I}p7MI>vpslSfUnN&)RrO^>?yW5+ zjf{+@e0J)P^GvbgOQ(HHBhQ&v! zHghTNn#SPj^X~Tc^FcvDb8b9o-VnVpsZ}>>%LzU?n;R-TpFe-j{rl@HXc_hGO{v|Y zVq)8(nSz3Y-+p;{ng45+l7q3aaqJ$yxmHW(h&z4JaVkx?y4Jd^?vIT1*Bt%to$tS{ zyxZrs;n=QUVeG+$y7wR5lg%<*&$ngIUH2`n%LNv_<_Op`<9NS(dDEsPPwLFio}F!; zduvN(*|Rf}yUX6r(v947giqEgMO0MOx9Q8VUg>SMzssVxW=$1t;lDPwSNF>5UP0UC zTeoiAR`}TM^x3nu4`SrQKGfT6c6D_vdvJi!y6nx2T@PCYY`toX?ip8psJMRjoYef4 zi_;@2-+k?PY#ZH_EwG@%Utc$B`Q3>v0eQ#}+O5mq-I%EC zo^yR&?78BZ^X9#4QIFYMb=55A#)Bg|&(+n{t=H=mKHUEI_xIcP_Ez7u-D#irps}~B zYZ8N~(-)3ctFzx6`788>UU0cI(Q@;M_|~JkL$uS^Yil``v46qwoHgvspB!`FP>MCf0MzoS~ti zelw~XB}JB&zrR;{yu;(rV=0?C9G)%e&mVZYHYh1@S}Q6Zj6HngT;=CymV#Z!OTWFj z`EGmZ>uWpT)wdYR1?<`U@LqWM%eEF1d3_)G=QpZz_FBAO^LT;!4&CE#GrQ*{ot~Ev zee=^vnet_CwIW^Wz6*9&F5NFYe`9`aQY82F$gg)N9pCn}_wRAtXHWl4vnpFQHU514 zt5Yjq`QO~8yG=w^!Qt`a$GMM=be{X|Io&GuEyuC5v(2sZ7cXD1-~eYp-Qv~vxC4&$ zDLLrJ?D!DC*mdE^>RD5!Jh7Sa=Y6Yo?0=4gdS0igdi4pL1d{_bB{;&5XnfE8);s^H zm(!y!-`?K#Q>fnPwP#1)j>nz7B|S4#=lz@U|7YL-pMAwUm+ee`chuJM*}Q)yAJUnong-f^C#xKp7(Kr@25+8QKx1|`o_Ln zeqSnQ=Wc%M{)}py&of__wH&IwUm2n=Z}&4Gdey&>zduc`&u>{i@2a|v!$&KlzRS|H zI2}$;)7^Zyo&WZ$tE*?!C-aDU15;DiRIrW;GTZ8RLz{pY>mVpm8JTyUrP zT@j0loZq{*yI)FXccovs&(G*3^QVB}(V?$A;xkOMr)d|QJy@x+FC+Hnep2%}qMbS1t9OlfN;aZ(Gv}mestbJG7QpJxe|PtpBL6*aFZZ@z__} z^X?WU2`y|la0$3%+W+8j^{##8@9w<3m+|b(%*V$nKAC;K8XkZ1(a~=C2rEuTtE?*; z>R~UHp8qk;zP4k#z#)eRZtm{4k9LdCozQFY^F<-3x-4Oraw>haR=6;)W|{o^nWp(u zr^cSDReEfaefpGc?j@(~^D^$uTYo2Np%ceNO(v1Rj^`5#r`1KOwy5UI6&4oy&H3ag zP-Sj&FJRXzOL;@dFLhOnR%vG>%4H7;GJ5`~2>;k()K+}FPu6e8C;PkqD_olIzUtxa z?d>i7udb$MrLXmUU(wU9-D;P1_$#)&(7Q5iTI9cllgoZ>b+ng!|8rxiZbbH0)l`4K z4{a>zzn-+dtK-~jUs`>NThV1XM}R|)l9k|$x*YW%AB`&<{`!>i%lYr?HxUzNJhda> zSk?#YhX!rUGd~qSKezStbbV`oEvCsE=U!hQU;fztaO+0#nRjki+x-2ZwextZz}-m! zcUw>Fv8Zl!-GAkL>CWSOj1Kq;Wv!^&q<>B4V9RskZ*TqI`0T%!H#bnR<%J33t$PYl zGb>tN@HFi>%Dt=XZPfFlDQ`4mzc_Jx+!8QpV(xSM`FFQ$4!pFWuuR=<>ESZXL)@pN z7N`HJOS^g6>?F5V%L_{;kyo=LbtKQL-t#?@dEvHJi-7ISwpmiUi=WL|+BdJRe$Vsb z+Bb)$e}0qa71h4}bmxnn_ojEwcAbyfa7-#hoLjM~3$$UYaQ?0?FKvsmTYmdLPJCTt z|M!-Ixn#%QTh`654(*<1RI(`goo4a7Rc5(wHCw>#qKmGHHt9#VN|)6>SM;83us?H? zVC?&z!)tbjU#y&%?0bB5N#f~PZ}(yaxe)n%%@!Y;3_HBnBOU9q-!*?pqANnyxZ)+2K^$}%<#q(ajJ+cDQ z`f>>uom=`ftn_!E@Z9NQ>$V#Ibu&GiRX%^hkCa&dGLv0}4_fY?-dR=Oxm0gkO-$}h z+ui#fPf)EfIJEA))tyA+j}Yf8Mk!mpemyrq+iG&w^zgi`#`0}0&rkoI<)UtUYGz8w zzt53nxv%TA&z>oK9xWcfdB(>1;_15EXN0pyUHfx2EIe-E88L8^&t-Y%<~Fv~npztV z?MOKtx9j4h_8Li{xqs`mtmeOcDmKyHJZn~L+}^XNHWgWy7hP07up@N8wUD=Q=(GDl z$GE|*u|>(D>5H$5Opo%cnid+thsV=EKRl<#+wG?WRApncK_T zd)I8QYv;fUAoEdS?$H2 z^Ik9V`;{?AcxlhyHM+c==YF-xP5$>uyK>rv@+HEHri-mFmy12tdauuO%cCni+n0od z9?VrgC#ikv&@YbA$pNAef4+7V7jE{_f3~C4GT7aB`;YJK|F2yBIoZ_u?zE|=&3DH3 zWWAfVW8(r_L6)>TyPf4{cpPny{KR9Ouu^T>R4*x;bLaZ!=Vne8i}*H)w=DIqp0~`*V%OF#*%l|E{>1}YClulj#W)gr|oM_8KvozzAk!n^Y_b? zMfdzFmiorMXwkfT(f{!|2b~G=rLX&Lt8$(%NX*&vRNgAna`m>TYkRKdZ4ydOn%Q%- zMRnS&L)Kk?XSvk0oZoqV+0M5fkW_YYvunEV+Sgoe0cpAsk6F^Lbl1uY%-xvJyNz4; z+09<&<7+>;HNG|y6U@jdapsrGdVR;pck9yBylpnGXB%;HPpqnao_IQL+ey81afzA2 z&!bO$@XR~&=H+$vNT>ON3yb~k#!rLx0BpU~TU~@ss~N^ku041D3-|4*FLv7Htv$yb zf37sURB6GMcd6F5_Ak6x*(dnFbW6o==xGuY<(@>U4c=DrrKaX_Qa|S@t?`-b#ZrZ1J z<%FBP*|hnv`p(qJO~y9_=XM0z%qdKqf=jy{(tlqwcHCwm%o7Z(){T5K~tRv6; z>RPAMk^62|TE4vHD`RmD)YK~vzO?3--$s{o@s@Yx$L>E{ck^A^ZD`)Q?O^_EYVYp5 zo9qJ>KhE4;xBT|P_H}U=OF!#WzdSadr}xK+Q%1!pA`p*MA9%8G_3ZDvRHsHL%uTJk zanQY_l=W7|_QgVX{DPmvK0SMTQzxYG<2CVdZoccgGr@2@U%|XBTRge%ZtI^FamDx3 z`gQlZ&Mk>b`fnIEtum5L5S*5G?W@?i@%@a?6`Ps^Yu@{VNfxVve#ybdZ{yCHZhx`oMckF}@aS7hSo4Y*%6u-J3x?!L@sRmx zuAM8V{rOnYUA1ui?fN;o@~U>b)$?Aixp?_*H%s=GYg4D5)_^Du7n6~j@YP8 z2eVGUJ859caP0k~tLwBhrWit^WF4cmMC|>-6N~1(e_UpDDt3FY^@}e{rg#5|E?ag# zWA~PfxuJLWDK|qB*<5bnV58^jif(6ZdJ=T(i)z&C&Fz7+CcYJ0V*fQqfA_A;^{;1% zJAw8NyDaZKf9OU^cXjy+>F<^AHfNrb)~XU)lXakKL5^BM?e#=h>YL2bDe#TEzp`|} z>#Fpt;@Gj*NlQV7ms?E;&S{?c<`TN@OuvLE# zRqx=)ElIA{ud(!>d>Z1RzU4puxOUeTZXOCF8=4+3pL_iGqcx&o z#oJw=5jM}J|=`N{-V2^o~sx)zj^e zF#dK?;_KHk{pYJxYYZ+`+D&e+$rQc*>8^^1VuNnPWVZh8r~WWWI)evlDq_Cge6j1U zFniC&7vjII1ZQpkp5ZaYtEOvy(9HVNmB)?KbT*%o7Fq}iisX*-tD;-KAOErO=(F4N zBTlyZ=x$fp$M#g|+1AFgSMKj`Pwn09?CSowsTEwFxIc*dCh9F|bZ}AJi`Qb(>F@K; z7Ku$Fu3K~CBO)@a%ia`Re{*;D^{(#j+q#UBl9FQL;@<-jy<^VJx6fzim)jz&?x*p* z^@Y-gzT|@!C9MDcQZRpKT>HtXba!`H@#~O1w`E_y`QVwilts~nUvxo*%=I;q$-lq7 z-CX?qTx3+#tkAG9tBIDy&wSKv1Y18pKVR;&e$}c|oIxxvmU>Um0ga3lK6ZOPvF1(N z>UHO~<=lL9DA#w7x0hGW`FXZ_eNr3rT%I4^Ue_!)6iM-~0|mO1gB+({d1;4cc1&=1tD(6)Or3@x8bgw<*Q*vV`}Mmv3%v*0r_0 zyUcg?wI^SXL|dhv64_n(IW2fb$CW2fo~ZS|oT?oj^CKWC>eV5;k{1(HRaGtZSMPU^ zi<|fPK#}cp<(ht3>oiVI&Ubd8!4~wck8M>KJ0d_sD8kF+_AC!x?zeS!`TJdu|Nj2I`-6o4srWt1OMVN>%?m$P zqomL;W0_=VX!vf{+1ci|@9Zp|8}YR6^Ru(XauK`!ew-vER(R^vsUoFc+4E)uRCHwO z`~3VL*f(41`hku7yVoAKd-JYNC;FRC^tS1_-IMfoe4JOY9#o!-iLGD1_U*K7Z*!vU z-ag&($gNSe*5q8C`}~Qzq4i~5x~}XT0WEjl-rl};#flS^T7rF=nwowLj}pH;Xmm07 zBN&HrKR3?cka`^A7B3do^ABrs;zf-79W?r zyte;F>-v4aPJtGzaj`kGd^l3|KFh`5=Y`EB=UqGaGoQ?;D3mRHZa1;7Q~5CGAH)Bj zW&eMceO~12ef-Yk* zme2H$*PJ(e?rNnb)5keI`Q72ae>Q$^5MKYFu=>`d{_UaJIkyA1@z{UcmVIv9+eGhr zpI;VvC#RhS4Xk`Qe)i@h@ty7)Qwv}G*`0nF+{c|?{-yu-&erFr>ymc2PrY0a-6n6I z?p^P*=Wwp)FS(=Ba%-P8Z{v|OPe1Lkt@gmQxqr^E=iZ$8H9hc!MeVgB>;FG>w0+J8 z{ayKM^WoR$KXBfB#L01?;M%&_>~?Q0=qm?e8ydx?C-*8BjBS_Uzkh z7+kj~IdFD=oK^hnOd*rFzXq$|xp}tPyUX9_RsD8o5aK`Zxzpn5uQO+SW<$ms9!^hx z^!R?=EbC|gkK2CJX0kc8QT)urzDkBa-cHpr)wkQ^*>i>K?p)kfBXfJ(v1$Idw!KX~ zeNJzA?#9%gy|eS!rH`aPI&gmNgNij*LqqHWqsz)HbM>Rkt{C6Hn-eK4BqOeNuKeDY z&C1K8QVTm@`Cpwcn-~4Dg;gV84b*FH;?q~Z`%+vlrXbl^aN6~CvBmlF93C#pi-Z_Y zN%*f@czVve!gHKl7Yvh+S>&t!zubB*Mb&6pUhNT;$5tPA{4y6l#OG&MCBvxI@3Zt>ln z9vq%F^LH!~Z1s7UD5Dnpu4MY-Uv4bRmMz;=yff`!6KkGKnt;pm6@iQIY>hcC5wK@o zh5t+w7jv7_)`~uo!d>4*8l^<0p2_R)>MHubSx5MK;rr9m^>-UYziJY9DrxXov37Hh zr1xA8i@&!NwpaV+dd}ogzSY{bXSQFzp%6OA8#BKX%OzHzgt<5d5@JKpH!tE;R1c04M4IG z?XhQzz+C6AbCXU-Zan(-?YSaL&=~)%OVvg_8R1@+jC2bRNNiuaT3``7*M)CvjUC60 zV&)iK5MTdlRp{zFKZ17M+}b@+;BNP^Uu?SzAGEmIUO!%Gzj=E8rdPLJje0W5v%0!; zGcP=!W|g*Vs%HzABIB^$oc=ViaS?`GlJn?=DK7fTaz?rnX`es_uTu^P2+ z^2tlZzY1y2<4K)fr2FmcL0^5VTeE^emGO~aw#nUI#>G1C?keUjti1LsWwnd1m5_he zE9>&Ne|}izO^UwvP8rgM=~(VvYj*4H3>R-Xn|r-KL8BF)_~%afAtI~b)cy9)j+@hV z=Pf_>58N!Nm~|xm>RTo6zctG~*BE_V@gt_q^iRrK2gP|NKiT`DKdHy>O5LyG1g^7f zSf`}LY?`uASohb`IroJ> zU+&A4cg0&BK@IAe?jk)aA%FNQq3zp2}JdgLK?<-14B!HuIu>_O%04}IdFUnHk~tTydjZ0gt0TeUtVa-poP zc``5Fu3&+*UR?YaZ(MgyZ~d(_!}VXbznE^f<;zL#wK|_S@hDq~s_jiZ7n%9oZgyGy z+elwGAy8N0qO;V`S5n$uKBc$iAN}-^F}PMaujNI~j338}e0tpNzTXU4t*bpf5ZsFT z!V>ZN^Ox7nZ?0_Kt6Tb|WVi72MO*D$uL~>)y*kY>c_pa*8a-#DwNAaLxYOEi?wnXz_IY&% zG{nkVj_%zowx)1$--nt_7s_j=?z-!G{pi^i{=EiwZ%IP^4llV zt9DAh4>y0mdDd=Kqn;O+Huyx`YN;&C{mlq7uqp4%>FjOMHR~UWeLciJ&8Va)G-ki% z;f{6s4%x4Dl5ZvL-jq1K2NJKlFGR%Wg9awftY*Hun(R~ekyO1+4Y9(c5t(F zSIuEx^XgL*IM02a`Qf04_UXHW*5m+Mef>DH9L0BR;Sc(P!4kme|~Oe@idEq zHLm+_T<<);diM7yXWz9v%DZHz-2HNr`|i6~Cy2Tw8!xWnejjzLC-vNC$-hPR+xJ^f zwrqLPw;_6SB=S*lYp6)n*N^7_5)P#@k`O6>MeH2@{QOve=(UA~u zDVuv)U0ta>;M^J8bj)r0we&OEf8SmyuGPssRI|IHL&@6Ouxy=eVO32=;9ahF+YS=L`PTBg5s#q(bgU%KU^?#)}j<>;~( z6GLLomoshA`Me40nawM9Mkk%D>pp9zyyn4;Ox-}`DxIn3`MV7l99=kF58^uE1JYAW z>w9OH97*~d$$f3r?bt6FtGK&$em>S;qw#VN)TG6kYu0a{^K|yz8UDNOu77LsWY0pU zFDY4F-nywTShQ{Csw9KE$uB&zPQQzMH|wp~|9=MCFAHYZhTYwze7Ixzf;ZD|-}G0% zy;gXmQ5&eE*wMFWUwt4Qg&%=7|C!P0|Y`!b`yE6B;WA>WW(-V(e{;Cuf zpD&)SxBbpm>l8>_`>#8iyXDoQr0Xvp{p_h*IbZacN|bZ%+#Kb3`;ISMcyN>0x~*@P zO2@8m0B7)zEE@Z(4xcS@uQ;~X`iSYit*WU;Jr!Q88H=B;xsfyTb6EBd@K8*Lx?}E3 z+uge=*Uzomwmtg&=UdnQMrFj=JmuPQvFQ5yqUiHk99`g$DqFU0&n2TYo#J`tFR$k< ziv2LvoGV4*lM z=j!cF>z@3w!sN=8h}zQL9!*f*c~p1$M#lAWnN2go*>=SC>G{k^Ke|$U>xDf6W%Y`` zKvl;b>-jh4wrpPbe9_y)5Yao^dL+GdYd<+@?=N$96Ln&mVz%w#tJm^&>)MOY&9O|b z{`O{L_Vsm|&t1=7m}j~*Ok#G9r+k0;yz@W4x82%%`nuf8iK2_t4}BK+{Y~9>cZ2D* z?X@y5)=vH?_O(Xd`m2xr?tQN}%#PG=d7;AC^}*ZMck7E68S3vW&YeEJS=PGjMjNlR zj-ukhta_t+%XaLz0h&Js^&%I$_utxE{r#3M<8-~)qC=ZLD&Cv7YL%9^m)E9dcK%x* zA0MCkd6D~psQEqLmlUS%&)$9~XzsLVp+kq`K2+!@$H``F?>oM2XF9{dsDeLRcH8}W zS~u;(L660PVbYoC^#@_^5ooaZ*G?T z`jWYN-MVdCv#(nPb|u7aOlk#nk6+zdIML;YP?($BG3WES9D6rTn>zL8xw+Q5T3TL~ z`nyak61vVW-Tm%c@cD0>W=#KmKR>2bAih4(diJ|ArqyTmU)^T5Y}Ku|3#MEsUo!j1 zk!kbxym=U?yL-y&L_MdHEeyK4x?4k6hgs%7|Mc|qTn5)1y{IiKPQLEsm}_0WE;TI; zG;<8<2-VJ?KR-Wsna{?irupa2xh`3@e0j9**}shU%=7MKyi(KG*Dp_0loFg(5m`I^ z_u1Lzb9by=v!>+v^JmYZVq)gZwXeT-SKrR=UhRh+N_HRXF0s}>IWe)l%BS_ksgV0S zvpt*Nm+zkc^7qWUeRj(IJpyL~%3gM<^2)rvb|Uw0bll>psqq_k%rCkB*mZBE#VOnT zi}vr*L6h_sXNG*W;J6sbB+|Gu`?{X`Gl7;0lb?CNzr8)T`-NkM-z<}peDZcVpP!%4 z7h2YFRMI%j=D}7S9i1(imzP=UYdvsPwP|_w?(Xi~%gcQI8vN#1T;%l74*_+Bm;22v z`k#Mu)6vQV!Rdcod4$g$YUPgI!}0pmRPC~NcXoC-GYBkk47@e9|L^~~wRdE5uM6iy zPbFa-TJ|(G<@CHrztq#mt=%fr+E_9o!}4BVx^>|}+jfIuyI)KB zb@zQ=KCNEm)LXT%8pX>=_jl@h-F)=z(xTF^(0~=cf}e)icrtb^{Alp#Yf8-Xqut`U zH#eo`)u<{J-rra2`~2Rn($ky)EXqk2s&zk>ottHPx>B>5No4&B@dYiLOJ9enn@rb< zOv=p6eD{B!(Di0=uCd4hD?Yq!-QBN@@8^9@FI%U|E0=zBrR3UG_WAAeHgA5MzWe!`FVXVR zc_DMAirn2Be~v5f=1xs+owkkF?W^wdHtyW=xZ&0Lyny;`U(VR){eAiTP=~sT!$}j+ z7{$w9%erfSe0X^7a!2@-!g+Sgdhz?-?7EknoP77E2*-6%A+3a(^Y8hE7qlF0Sy=h` z+0KpgI%2p~6cQwv9<=H*iR5d~u_!#$8N4o?gWuTE+0-cz?c`kGGd zEuqyqdX)#lLRA$LMe47e`y&5#-f?S9@sRqmkH0>9-VS=Snb~H+_Wb*0k9B_@Ida6} zy;8E-$F@to=Wh7jQ+r_*zrSv;@bt^e{mbKgjJhOW`vh>6H5Z?|u`&5>PuaZ!G5xqV zEUK*;@9*vPds8I)*q&4Q(v!u<1s6D#l-_Xq@N8l5x!Z?=4Bq6%28ZYz=D5gpV@hMF zQr_I2?60P+8$H+E-e>ea?MeD3#Vc}da|%tHtgIjbFtHax60z~jja<- zxbPUoDcuP@oA2Y3G--mof1|powe{^zKbtijzIztiB$BmT6%43V?N}Hb5%D-23Ph8O@JN{9YZtkI)?dmnJ=j~=Q->IK{@RpeP z-1WM9)6?zerm(svr=%!a1vP9F?{GiXuO!o~AScOxYLb6N)XT2)oobz)7nP5ud4yaG z4*`w0mzn=;diwU~Dbt2d=3^G;)lcugvt(iLPw_=(^@}8pr$xTICA-#2e*N=S{b##$ z^P)wzm6lF3tX&x!w>kFwQh~dH4N4j_mxy>q{F<40rtJG>uGSa-bdwfGUurF!qqF*m zpSDR~g@y31o((Ui|GK#{z1n*H^4XogpCxSn@Mm$3vPMhXJE=QGo)xCin|srI13Q=U z%zUtIXRSoI+1rg6dzPR_p2(-wRSaviNb_FST>|S2u^0JezYnvuEjfxo>Np z%iE5 zwbPEj!*|bYP`BdIJ;_JgKA)bo|JEn|xmy-J`5ace`^CRC(Y&UDi()}@XtO#0Z_v%o z_C7Udxyj4Lg<-KbHZkOuBs)ys@!aa&UB$4A)7IL7yPR9%a?C6mBPtgk=sF)?vN>z> zv-O49JjeEZ-S|1rPQmPBp51k86_?+;*>rO+F`d5sck1U27a}&=DQ4GjDbCYjJ)V5G z_>}SY4WBDAQ#P~jx*okwKG?)1>vUv>Chi_~gu z>2gq)dz!_EV#qYrv%l+mZy$Lq`TbtN^Sfu}Y`^vCn@nkMSZU(KtDzyQv|~H;m>zGu z$-nQ_wYg&FofoL_%jN%M_up1^l{0Lkg>}s)>Gisx0fj}b2Rj{HG?XMhf7|rznUuPU z;FL2C^$CkE^xrGLop9;wJc}2_-t#w|s5Iw(JjuDqRR2oGR15zOfkoLtRffmU=A^53 zOJ+WRVl7hf}KZ`{n=d+(EWrO_5p^ZQRo&ZVj6)~P-XUw3=0aG~PI9jDKF`M;Wq7pB`#i5% z)q-nx)h@r9XzbG-`MLbj`BN91JQ})|hvl1@M8sAu4SgmW5qPQc^zG^^l{&#~{+n)W zx8Jw_S=x;ozT2NYo87FNdc)=PoZY9^<=uR~>y6gB=hpr!)wnn>?oE7@aB0DLd8>7w zpDhf|bM9zAJB|NRKy32!_i@Uf@(R5)|N2!Bxnoem_2&n5`#dTQ3}lBJt@f{t#L4vQ{(zpp$vHpR)N)Ig`b@VZFl=~J2Az-x+27tC&z6PY8k zIijTIz5iX8g>O=4-^uxQ>5KQd-6xMI>o%*jFiZ2deTo)7mdKil87=6|o% zm-GH#_x%?0&uRCbNV?D2%*$udWN4-)#m;kMM#7|%$7bj(I$`;|M_`h`DXaNq4=-(5 zuR5v3aKYZ|yS!iRtKGfYKYZ@5t1B>C>woHgDViFVU*M zL&M>zUSozz_n!EBJMArBFw64Id$)nNpRXfScB)fXURl@K?Cw3mfg5-(ohx@gx{&ko z<2RQ~GvBD6R#+PO(#_oUiCSXQqgVE}^Q1PJBt&lBy!BsJlvU>S1s7lHrd3LuUsxEN zZ`pP7BafifhnME-ICT~=OpSNVF*9GD!N50<@xS}>Grh6z`#p^VDt%MSz8v-M^7-@k z=8aqR9&hAL4cG4U46pioa`Vf(ugfw!J^USV=0$I`Gqd{^mG$9Kzd+BLj)_&#R(8_A zc|CYj+CJRjQ-7}4DVtySMj`+dfSv+#uipN?B=Ti6J5Oin|GI^zJDtvisP8&?=&Xx* zzwg}HyBF|vy#tm&`(Mgn!{i6OU*^&eev^NH`~lNMrBhtCv4x; zvA2UMcX{oyG~0U@HekM`rmfqG_8~G1MdI$kzH8vsAt8^ z;s>{G%~n#X6-c|Xc@_I!$1ndnI+#N8n0MWjJ~yZTv9Z{(ZP(k>HqKl;*GFmBwX0i< zUir`QQwnuiqJKE3xZ3WIO|Sl1+k%j0P(lk!FtA&{&&DJ#Dr!b$Xq?%~#i8e&K2+Pw zO+Qv@cKwRwIZu}*4sZ7zez|o{c;DPB5=vYFI|W?~BHdmdn&q2Wqb}SzLvhx0-z2f$ z3-y;DU8=h~C?&+jrH*sGa9VF*+-prGu7H<*72CEgi(If=T+g3dBmB?L&i{IR|Cb+6 z{>=S;X?WbG6*aqG7k72UOy9O_nMv5cBjw$uZ5>P@bq`f$&%PX0x~BcO$|943f6MdA z)Fj_Z{>rTSn)>-^)O0@~!JRL|YOAI89g)8KWv&uez|WJ0+pfg-eqODee81yk=FKH{ z&hGp4oqu`b=ikR`WW~*MzJ?YlDQ!PF^L_ZSZ`bX8{)M@SET|1HFWYkHWt#o&n%tcF zThqO> zugKbn9Z%Qm^7Y?e`F8J{mtSY_FDsXSU-y5q$+yYjnMPtEE-qECDsDf%(zG-)LU{ep zluR|jz=m&}&yU~kTDARsXRWgOkv8{V|F_TH`&j?(o9g!3iBT`kREY~tj4L^qn_HOh zX33M{>;GQ=+)?zT#Z54<;og0LlPQv_V!iRd`_rv=-xQqZ;=o^Lr>ficW`~O4#7}{{ z{}<#`JzIO@>%A9S)@|z0aCpD}T3L4XmH9Tp>id5yZrk^%F3KwNX#S_1Z|cdnSIZw+ z`P2LHtCgEqddIoAoI1bpC3n*E-O9DL8#|c3{AM4C%9a#P%BE-(pAdCN(Uxgs+3tJxUL-Xf+cFw6T%~N;rXCcc%k~r#L(+UE}p!<)YRH~_s#!*ouuMh zinI3wOuhG1Pf4kqea*@}*&k+Dn3b+OCJ@+gFL}#$_ro{2H`YGbuw|Ld-8Zl0Yb^Qv z=gi!7^YVkT(&Zf|?mpjl`_`iiKk|bYEZF(2iwzXi+P5~_>gC>?DrOY-^S67@^xx;o z-Q3>(Q!d-w-ErsXmMgKI|EjOb|2q0`ecCRr&JLz8asu;pbu|y2{ISsH+~ml&JXzv` z6L-tExrN%98vOh)_kC|s(uxkIFM8th*RCxrygM!8`+V`B>D5Q)&+n*JK32YES#9>j zjuU&IoIbA?J^R?}`B6J`cvlJuvc8D*JUw-FVala9H*UPYnmzIS|5_bTi4!GbucB0R z{$TWkvVFEnerKYV7`lioC|6(ezB1|CqlmxHk5B%wHoMiNG{z8eV9ZX-| zE&6s~qDipP-ot|K!aaMZ@8gb+JC9@TZMEMkxboi-$zH{x{hSxJZIkZa zdhOb^8~bXj|9m*iU-<0I`u7|HGfaQ;EpP3LOzJ%G{GH-_m#=+$SKfAWad}rObuPBU z_xv8w(Ai}fJQL?}1YBEgQ~D}ICuWC1+WC31yUXAEJ>t`g*&)y^uJ8A|{N0_KJO7`W zs=YDm>Z%!*#c6Azx9hEo+j}e9yx`4^jW;%>dV5_jo$KK2ye4+HnV3$5fcJF0zokb( zx%G{mfV6zIY1qGS2Td;5KHt4~(Ze&9#Y;L))UrJOv7J|9lGxJxy|)%Py5xm7C>go; zNL!p1Kr!_FtuF>BVWA5SLxPdX8^vnZ8E+DykR_m;@nvu8iP-G2Yr^!Pf->}zW} zHzprH_UGqk>!X?e3ZFV}+}-&2n65WVR%~qSnHh%4T3T9lwjXa6-e3Mq-}kYN;CvT5 z{@R~!x0w2xZ6N=3VwzPz}&cw^1aPZ^n+o{RGS{P?K3 zYiHu&HVNCRk}i`?^?!eT&HQ~~q4URs?D9u`e!jk_+C`_GM{-h2&a#y&HTmT2<{Z%~ zcyn`e`sM$3%%&b~dgtp8UTB=5VB~J1NlMJzcM~GTt@t*6Q15O4hIKJ}kWK z_^JE$bCl*w|KjaPzAPS7J->S9`uYF&U;HM!@o4R~wYq1Q-wKM3dmpoh@#T><@00%? zUhneu;Nn}~?9NRON@I-bX;HH4EShIsuD8r@?yOzcjNdG-u6uW9=gQl*?z)|2b$>n{ zw|{M>F69+s!tH#{xc1kV5}oS`OkE9g3=*Ak?(Q;`bq<;Q;`z(V%P)VMq8WT?>!ysS zvpVkO-rlAvcxvaa=KBS?Ee6k3*H%hzJiT@Gs#TZFo@?HHW>q=+|IZKhzb&@?`?B-K zzi%BT*MvXrTz6PdHgd0>UfGR~>_XnY=d*Gqx9#!0CGa*!`NGY0>(~3=o~|GN@2FX8 zV~4?Qp6(OQ{5RCN&Y0DA-?mo5r`exPJOSkFV z)!X{?Yfki*)bee6Bctzi8r={6e{-(x`x3jiox11W+lx%~-zzzN+q-M^_m=;g{8`iO zw2niu+VKm2`ad`uoSSW~-+AE=gTRUouU*eOJ)W&lmCSx|Vd2Wzf83a+6dPvG-MG}n zOx)U-l{A6+h$czU&J#cI^2cU=o+>ar{X~(LdYUOBQB4kG|RVCc@v%5tLpqQ70)wmtduqi=6-Y*PREC2!l)zcJ6M zlTNq2nsvkML9@l)jMi;?{QoY0^qG$%AZ?PeS>=Y5F3-toOHU|oe)ux#ysrA92`)O3 zpE?W|aQMjyO!VJ%JM*Rp>x=08y;F}>Tq;`lyXMxG%$F6%a*BF;rOlTWKkW&y`nf`# zX^V`(%@yw+Y`pR3&u;s@VV^(V_<8Pxi9)Q4%P+n)HHS{SRcKGrmk`|P%jk98>h8P8 z$9lb;CjJ*}+rlJKY?*duhRQ#Q@0|-=bc|llnfbZrsfyI88yAzZ85wWwum67~^j!Y_ zpU-Chc(J%YsNDLw%Z4J`bG;{)+_v?%J7uujR)77Ls;{q9Ja)1ff9YKCjN^@Lp_%OW zn!2yYBK}{{*8H^dSO4Fn`yC%I20gdF>+14r9#gM{U2l3r%4MKaDaL?gS zHZ!U;v%Yx2W+6vMhmmHd~@5 zlRp=~ce(5uZY#Jmm+{tZ1)0toBfBjecb}cfQ@`ceGI_s_)%@e>cXCbQ*ZIEx8~gXU z`lKIsK6w1Muu5I`oL>Z1k(=9kKU+Pda_%?;XRYU6qlggT|q|IrmWDo`?E6b z?p+V>w^MIezv&kFxa4;2-TJtm&8M&bit-hlxX;ghZ)wf1rVw)lr}Z+H3w>P}S8-PH z33r$AOG>Z)|9zv#&pV$!Jy>zK-s0A*Xp_DUkNMZpG z2yDn|x-Bwar)hU_eAU`#r}#d4eknd4eR%Q1JFdOTN=o}5yxC;B@$1ur7h{5#8-iSw z=UlUG%dwTMrx$;EX+HUf%BRYdOK-A`@AlSob$Ezhd1EIr)t|5D*W=|TZ5j@y2iIOK zd08Yh|FwLLX7Bfy@AJhU9eMLUHqFz;WgWL|%ItZS+N-q6Gk8Gmn7w@KzbqLUb5mZG z_@B4ilm0U6FKv99tLtYbBsfv}iek8Q((eO_g+=d#dKI|>dozc^IQMVTdKdnE`MNf>(=75^9OHye)xRjr*i(K zH{b1M>-7|#P@txyq+}kxaLe(Fn#DopHFpm#J|`)_x}xNMoCF&yF>BYsW+?=PHc5JR=qfQ`W{fv;$o2P?h@aeiXT%&em>z9tG1VW?o%!( zxKp=6*ZA~T!}bQn*tNXt*tNIv>UV4xB`Bz|B(`N4%AoQ<#Fxb=baY&vzj-Hy1Fd6Zt?BypBttb zC0k6*GI;_UzLn01I3jUAwEoR>rTMqxq-Uk=+I?nHN5>a6aV;%SPs%Mgbn7tz){OY2 zd$*dt;*<4eK0I?~2g6;{3ZE-)>=NDy2u_S^Z(uLxi3xNSSx_snXpXep%lT=$*8e)Z z7?c9HdAYhQ$q6`~Q}^$#2&ZUZ!?!?*$KYK5^u2ti^5iu4$F1Iieca-L6OAs$M2knq zOA1xD)ukP97FkexP4LQ%06l4T!Ufv@U$J`1|+NiXW#Z zZ~XZ2`r}tGH{XJ~B`!*O_M;2aFDE^>R$jbyVPJCyQ-}+wXVq#tb>q*U<%?IYib{rx z`QLr`IPm8chvN-hU`6kEe6N~re0tQeFz;v;DDwyk3ToQtp4Hu0ed30NQD)h!87dGL ze7dyIvfuBYUfJINo!O00-3|Y8vR?}M z?=V`qJb%b}Ir;tZ$BWi3b*+YaIsyQ?d8GJUCAymxWH_FX5|cXZrIT~dB% zzm0{~d>M)9<+2wP7BRf-+qP}nmz_8NM*i9Qz36n`=dE{+ta0p%++SB)^Zjml;hP(V z^Y?Ug+%Yv^+ww@?e;Mmb{@(v)dlfxh7^9R7zQ4UKetDU1x94Ov*VNQhp>mFckB|2s zKQq(#^1ss8*Ia`yZ)GjjkKHB0Ew0zIe*ZtKx*reQSFBz8cB}M_{q^-ZH#R(MJG5=# zLd9)4H-);zbh}bdPy6}A)#b^$g)fsz-^l&ST=Vm`@y6T(?1hPEmt|zfl3?53Q@ey@}YGHPt`P#pRt=)a&)f z4xU#(-gVzbBK!ZxoGWjvmjt>n&RY0;)(Z8ez7>ym6h2ORd1>j!w6n7$tjlst{$EMD zy{EERXrYU@{f`IDDgh6_yu56fa$oV;km zTfR?Xf)jW5T<)H>{Cd&4f64nTJ|A9{@ls%^64wPGz4(1Hhue6)1wD4%>(na~Tg11} zvf{&oh&>gBJ#x0Yf@fS3^tyHs;mM$(+UCd0T(g5G zZ`;GZ?I|B;z@okmJEg7;Z*p3pm_d66V$VZTbAT-`>w@gN8$AR~qA~ z$`?`Y8Fk-orfYsW`Tz0p{{FYOx0k!Qsr${5cz%Aq|Lu~G$}uN-TrBMW|M^_vtF)RU zc-?JTm*SmfFCSRc9shQ;Tikkk=Za0fRvib#w(tDE=j{KQAMhzLpan9=`nTuV26Pwr2_y{O8EomYtPVx6M~%M(#hc?UsGHecE}+$=fXT zZ^yL1F1)*Ln`ipPX?NW7Z%z8&y=~9D-0IWQZYJX|M;aPOOz+?zpQp?JFAA^+39grnyxxCzn*KC{K(-v%|gmFE98~g^TR5N zE)SOZ&JGg!+^xUQqyA-({x&gz1!Yi`%d-WHKKUcu91c6SSx{fHO#3GmH7ZaZ;1y_uSw3>x&k zRc|-_SnIX<7K;<1Efdj&)3?3r>iS=86!z!aK?{?_s$1%inrqX=f@d{3wUq_4CT#h9 zcVb}qhlnlzzc_(r5jr|LzS#NPdMmK-x4VDO)cw^KW;Rv|S^RQW^&omb6&0PE6!G(S zd$7uv8rRe<+i!{La|M9vd*xp7)u1|enZ{pxVFXLYSQ z^>*X4{PRmZJ=>4fzg`11vvOzEvzib)Q-iR7&ko-Bd2dCAMY-YGY2px#kqL&sX6N2z1U}kWmFBJa`P+EoJA3^jJ5#e4Fa6fl zCJ2eW{%fmw3+uif+hTGV)Ly>ERcf;W5`Al~%(svQwPtrc`d;&;a?6#g*&+~!|6HKG zFYEEP8y}w?-}r33|M7L--P{k){4C`LX^h16osSaHuKSz-8g?$+wW7}_8CUf>Kh=nAT)GG_tt}T2r z`)1(B=jxY~zMh$ozq2UxFw|6c@4HhicIV`-`TM*$@Z)rKP#nLM{iFm573pyQxxKvo zd>*kMceC~S<@*Xj{BScPJG1liYO^EXuG{%c47kSr^zTQg7VYI?dfij^*J_0Q{WsBK z_ZBCF`x_T;EIlw`#gF~!$>-bWAALIaH$3>2-@d(A@h}K9*%7s*?(*jR@|^HYXiQ6* zW_Wwc{3t1U)fWGKzWAc5*Y7g2vp_8vaAW*pM5wI#a~;ooS3#UEYyv$X7JH@M+? z%KPuf;EnGZ#V=p-+1=jLd+nH7FVvOtxw1{4pX`wb^-_-gVvp}*PFDZ9$`9hZ%7wT8 zo|$wt(?;X8@kXD?(2(-FdvNi_pF7KwuFC&Dy6<89vbW#1mGLS=0!sMp+lvM7oZi%= zKiRr;=DfPgu#7R&G$T4%M#|hY?BBbG5ud-OKN9jbHr)72_w1I(Zw2)NpjlC$P)H@>Qu>)mW0mH|zAl{@#mcF1}2Jqw(+K77^;tbf$=*wGcz z*=g2zF?F%}tOaR0f!N+oai&+GR+zh`RP?VI*RGN`UtV5L zK0C{_@awCqal0EAyZ0Y^c6N5~taa|r&d#7QcF<7z>1n#3!PcqtKG#16%D>*mx%H11 zXY9STAn?gKu7E|oyUXAAIX1H$mCoNIXli0JMU*KEV|bZ8tX`Wa*i{g$4~Kxtx)(Q?1JT?d=l)i0W7 zU0LB**eL;;8%aJt&vx$dxIGnyu0E?*ufA-?G_kmFZ7a9^+MRVor{}jHtv2pR|h()tpT=ipjP1$D2|Jc=X znqKu0Ev89%HA-_LPYC@zBiyyjXXd0M>Wij!y8Y?9FTf~km~`a)u~lWNs|1&FB=#w< zU80&_Ja5&iRa<;#d2JWG4jb>DmfUrFj&XkR+C67YZ>H_hy1k@xOX2#Xxl#-{!o7NT zml+5Mrp->D+%C5P+^F4jwr-o{y4AVo4x2qso&CCC@1?gn)|VphJhSu4J^AhJtx5ku zE!TaMx9QwHx9#=0ZF>{Fx9#5iwD(iyHr3m{)`~2tT^v4lL+iv{2bs^&K{{ExauG{x? zyZ3#q{D1PN$m0Ez7+g)1m_9Qv*ew%yvcg+wQVN4YGq0?bN$J`99t}l$O7rK}eDdt7 z5#l_1(RTf5UU*BXfSYqw(|mx^I^)kNjABvtG;Y{RD=i7bS*`NL88dY!PZg1aodRdP6%{!I7r8S2x)z;(^x@(5VDI*K#Y(Ge63#V$ zxU-`$xvQ+G&*o>w^>wjxpTFn?58KRM{_euY8|HsbNBr(jPrkj{`SGD!XJwT_1l3YS z7IfIBZN4mfZ-4#&88($gvm9Ks8w{uW*gUd6R;v_bIdAFojwU6KpV7N=gqWrjUkp8$ zZ~5F^zE&i9TaKsGzqfbp?@bx1Ri4`| z1utG|eJr?u!|xpE%r=#@mygt|IxC)R@A;SzsXnbL$=+LE)HO%0oQa3~!JL&1CMHY1 zY}=W!uA{?a{lTaO?8p4P0xz6Wzu&Rh`?`zwTTx!+r-BQfHssapIwM)}XY;kt_6n6% zr^*VGuDk&-LU&OI?zf^i%#8fqvwZD@0TbRB-{^gh7#A>GoB@My(v96k( ztAtEdm1d=%>+I+_ao6$eK4(zVXql%c`>{IVv(vLh7Z&A%Iun9B%`>y}I=yF?9ryuVE2#VdPzxvRIoX9z>8zRHz%>n+N7rDU`J z{mXe%lOGb2d+XsPA4oIK=+Zqo-sgI5PlK5+U-DVQf2(L|+-_(^#h%^xPTo{e>)#)j zoSH{pckD=v-eP8kXl$-q**&><`jVwHwf_AInG?M&4Ax5Z{@S;9cYL~dPmbMw{-i|p273ptRXwa``6r#-BI02dk94Zcw}!RPG9QytnaN z*?*~PemU>I@#Dwqm%5(0zGtP}z_oLb-K{o-ZpShbxi5M_N(oNX@<|5KX;u2e_lDT!n#7|R&+GXMxQP2F}9UveKoemTL0d6 z-uUpMVaJZV&1F#C(_hI}Pi9CBy0q})<=!SO2UpPeB=`EKHb$9YRn9?Ah4VoxD_o0v zy^S>?k{p5O7Y9E(0vcRe{r^WqOw9h!#o&bHa>_gO>9-pnJpO}=#fO~6k6yhj);qfd zR1OGg%DXte=9BF{tG`EDyZ(Pc)U1^|zjYxz(RK_p#{H`6{*CuncNBd|i82m>1am6$ z#mbC3TOvMRUw--aB)xU_Cr!(Nj6izaTyeMD;@Y3LFSnRnZuD2%X)+lS%6qn4iR)~B zyzJ<&_Bg-wuWd}<9|yHc5+MdYUA%UuYM;-n+T6@#;0*9BLj)4im6rRfl0n^+jj#Iu z28nFhxy%pdHTUS?;;K;fyC3$fEB^nYGjrJJSGA!yZMt%f0 zPA7sg%yhH&J#*jp_H6|XA?&j@I;R^gz4bTW(WkM$7nl8ixZ-B9LDZ}DZq?D%QRZLv z*RTH@UuAcH10=w9HgfdxEv!_zR>BdWv~cym4;fL*RzA}AKRRo_{pFAUjW<4iD_HoU zdu8E+$?G2n9KZipww}Mv%(Bn#pPrO{nEJ(&Z+=ZKo_?e=vU}0bqTJ;QxvSQ7F7uu) zx(wpH|4j*ZY`<)d_x&yzt}wV-xe!j^BG2zs%J6 z`rp?-L2d8LXDXEzd#?HSZ(?@R7jynN;U%(;6qUqIo<{TI_;%(>B{hADu$I`XQG3eZFjQ?%Vu-o6<7=e!N`x^u|e* zfb}xgrElL)eC)YmUG;;be1X4Wgl1RU{ZiVt*X&Dr;H9sie(=A)PQUg)`aiG7*nUly zcXZjY&dB@6uQ56&ftEf^o6ffMula4?`vu=>q}VQml*uQtr zWB=}%J@0d1yH+?Ki50HY&0HjW_rvvFZ{$7heg`dTSgX(P z7yGBRnLxhWn^c)3TwC|Tc7#*orOtVm5h|RuI`rm z*Y3#l${wFHYrn}koj6fYfvVjpHfW|wA0N56XH#&YRzUE0Z4SEio(alZTYc2Mi$ z*HIO9H50>mQw*1se7-JcacM2XTI&jvvc2Y7d&9P*yE!blQ+95wKC#@h;n~uKGd%0Re%YDF`r>4z z{jwFQcHdXBdQZOL8nbDirqlHIe6>Gw=frJMay91fon4h{d->Y7b9z67@9Xv{$De&; z<8PIJJb%^p_q})D*H2os`%jzUisxbL!a*Z39e4Ciu3uU9bndra33>7Bxt^!3QUh-+ zWtjWfS!SNW;?FbggwA7mQN8$|w!?A%{$TFu$2&{@eBltg=H5^iqt1CD^1iy@ockQD z0jW(vLJORam2q8QPF6px4mFoTQBpzwF$yP4mC9!+D*R9ji{sUc7Z>}>P=eGdzHd3iUMzP|S3-|zRyH#enD;{%P0@!PNUKlSFz zpOf8}lm#N?Ru#|5j5E{xKPBO=|M5@et|s-juJ7Dn`+R@v=P#a&x2{~7(()YCzr4OK z_T~Tl{eQnLTQtpl(ay(>piT~-yj{)V6S1Zs~J?dJ;EwHq`UBEs5g zoPO@e1V!gZZ*Fd$dt6}dosG%vCT3>K-hvi2O;mQj^uM&UbYtS-Hj%AQW+Wf)EA0yZ zH{YhR=$Gtu%Zd*PYvT4=fhLH=^nQ+woWtz5M?-}nPsoN>L9ZKFlWiwk}ey8`AICc8-)TUCF1BVri4yDXPS%0%PY zxw*3!t>QQV>cju~dVQM2gBgO#ZbvRH^-g|rVxnQ%nHfJGw#z>Pt-qRSoW87B*1pch zHF15+&QIMXLW*A>AMcm9ulw_3=hc44wYvivlmzU5KAC*(=5rVAc3$aei)Okqg4Q$z z9P9R*Z)eMS_PpJ1ow9d#X4aY4{rPe3{>8a=iHBOw-S?Skq$;KtGvl9oMcu!j&tJ~^ zBbGR8@#@&X*x0!&?`D~1|M_w#{b-ly#;mJaxfi+Rm44iqoU!lQ3)|{X;niRMIPbeM zZ}0bm`B~?yFR#zJv%~P!>(`-E%O^9aKKk+Txcu>}tHZ5Zm1930Eu4LqGayRj^mKiH zLEmkiMVF@X%{I&RI==O<$pJCOQ!MHa%zI3qe*gXLt@m~HPq$>NzrXW!`uXSQ=enwg z&3^^|-CFcqzHOC3O3MF#e{U@ox8r%AsH9ZC)&0CZJD-fl&&dlzZ!LZ<|3~fL=Q}%# zZ#|a#^#c0L~`0YR&>H#bbSP3BOG zeHUN*H8d;sLaSeRUHIhwDkX>SZ*ER*+4P04=caAtrzLGKSThnX?poI4asD8GK;RR$ zjbD1Y#r2o{dHL(qUgkRC|C#0Q_OhPm&Mp4+C9}8CoHIa<=d_BV)k{D1Stoy-PhwnL zBx{i10GcE`)+;^zSgn$jX2<5VvtExa1*a@_>-C!4!6$2V#qy`d;W{lb1r@(7Q?d(m z?tZ-fx+%Ot%RzJjS3qEcmV+x}l|1K#KLs5*M!|nSJw3f&BPL{>i)UwnkYVaH>)!(X zJ_%N(uR>NC%(&oEGUeTgOG~G}xOF}-^jKnntir2PTi1s#u-xCY@_T}5)2dZkycfMw z+coOss+4S-?+HsC_`&+p@`?AS^xLs}u07bxQYZX>CBxqDe@xEn@+(i-?a^?L zmTBkwmQPApW<9yLqoVHfS@V^%_ig7^f4=!Z_W9kS=@H-d*)IKlYS*_bx;JJTR@REfMS+HX=Igdj zia*$W_l4Fu{f!I$1eq28J7cSTW9C8!pOh3Gx#w?IUwE;+M*M!1yFtUoyc_J+PhX^O zGq&q5nt#0IQ}mm2b$?!cJ@)ASdjn^w|B*csm!Hn7Rjf1q?Nr0$&HQZo%9Q!DYT3T2 zod0%An)SEPll!HwuPy!4ZGZNB(=N{XhS-(6yOU>mzuv3#Bz+$HE=Q3Cwq*i7ld9fs zy>8{o$kf^TMpEfOduM=-UvVb{#(VdXQoNuZQjO%l4e~j;xS-lK#fS zZrQnm>wH!ea~wJQJLc2-C7zQf#V?RszOkj_$hj-wPqOC;mRg+eeed;pFHhP3kH_UN zKgl}&LffI5AR$SdLA!1q;&b|R_{WdhK@x}i$hzL7n)9u-ee&p^L6hV zhm{@?oehUh{*OIqt8^qYHArEmL6{XI&uvAoPO;#I$h{1+zp1L9+IuO&al_>H4w=-0 zEyl`GJbU`Y45v(2JN)j3(Cwg;$}t<|&rQBR$=*_ZYDj#SR5dzW+O_dGhaNF_Z7Fklw`LYIEs?qJ5HoSdHVG z5B5#-{zT~neqzh2AS8?Rl-B6P7w z$6=$isDP;GqsbcVO_TJ>K82K($V^vOUL)lAMbCL@tH%rPK($j6Pp2BWx6WbJ5VgI? zr)HGopVRr_cwNcXLLFYeUC&S6x0xK&*dpK{@kRB7^^M8vJhYxIVpy7_c0cG|r&3^N zu&45dqIr+En;g5@-&z?`in3t=iJx5>CyqncUqVH*) z#(}$>0mqaJ|5yIh*3wj}beSa(WZ{+Lt@F3u8KPL^u7+9xR0 z2YpO<&r>=5{jWzBsZw`b8M`Jd`6H})Ox)-WW1-9AS@$N!GzNeBxbQ}wTV>vo&$iq2 z%p86TJ=vQyx3R-9BPsaJ_fMzw)$P0HX3R*AZtHnq{c&cGsOyxt_gk+O6(3xdak??+E^E@d+Lo+j}3$UF|eDg!TRYo5g?C!SAY z5wR9-RJydBVK3W7g-@y%Q%)H@H4A^XR50L=iy_mj&(`N18Kvf?&f`!!B%?Ta@oZNC z%NtUTQ9TWxqThIj@42(*$C3C+@t*_OUZuTSf2U#5yK{mU*0E^pYk5AS{pQBI(?4%z z+H!`;q+H+lnP#{PM!V-HF1M8@%l^S?vO!$^2B<6}mAE{o4B%0?P|TYr?v8|jORN={Aw>Fq5V*vfp&(fGkkJ+Wgm z@4nvrAXC)nPs2N(cb!gM#ZUHS%$@FLzw*k;GcQ-2&}LMyDxCIbo}NvMx@b-Onh7g< zB>OE29vn!q|HQLR<@^JM%r^?(e)gT}IMrt3n%WZE=^$8}bj+Gp?oGGAQ9-fYuYPlN zl$=sboxVc#cI6qPyR&ZyZ}hJ_lE+b=)3Ntt^UtsCJ{~8^mR(x-Ff!)a%JnXtC;Fdg z#!d7Iuj&(f$i6}Ru|;v^asDSYr{il>|1rvYmP~nEY4*f-m-X+cl6k*X>nHNZ*Z=(* z@~)U)av%T1`R#2_+SHE+iGSqLxG!_OIMK$q?8v+uQwz7P>#Z#g*cM;&@u-TK=Jgth zTMhOL`j*8h{SW$AyjVEiYwy2v*6)|R|IDRd$szZ1=I00LzdlXM<+>0juwcuN!ZkG~ zGAvpARr@7$bzfBWTF+r%QY`O%P+>XmFUQ8;3#%m`dMusv)>%Y_J0K?@%2V0wrB&Jj z+jCnpn3XDSOn;ZeRS{pZAjHA7T4}}!^BcUklg#JDlx5xM@ahP&(0#jTZ|2U&9+v*` zwX9l#y*b`?JEdiRC%V7sj4ckC=QZzyaH@%ZVel4X?{yz`@3SzQS>rKN@7E!n87JbW zyuP$9>CLGEsW+Z1_O~bRcbRw7pLxIjJ|~npX|@1ypFl*bHMz^>fQ72G<%9n-evW>`Hu4O;zyODuWjdv%73yvepz(=j}UNA z8#3hV0vYv%%qt5*#xfy`HDNN~(PBZ!ih&NOCdl|7R0gUEJhTJT^nD*gZgzI}vuDrF z@!zg({PyizMsDuko6qz1lwYd+_vh!7eS7X+y7Z}gOWAh&`s;W1ihhsZS7Z6=)vGGg zzr4J>b2Hy-#V(o2|NnccM)lINM>p?u#Y#y_Tj!p&XE>-tB&$bZ1AQVezvwU!)yO>i+-zp80?K{W|I8 zesh0CE{xh+RjL!c?M?c3H@4e5i__1{v$eJ~ys;tCStnw{g6mx7l^tJXtV&*p-&a)G zaqap1db#mizinze3E8g2Xu%g^!kL z@7-Dad|9!qWs!=S|Gb*u-p*OGW=;9Ndey3+%4f6U>wc#C%(dG3`If~~QSGoLfA80P z_N|&{ssHc-Xj9hb=jY#_*>#^^?<KCC2PW^uEub0bDo)6;7+ZOl0;*ah=lmBf0t@7{fv8eo%^1See)hVBE zd(E|OFG+rXZ|{Bq+aJE)g2e;kR;cf4e!N5fl=ah&`|s}W-yeN3h^OI8NX*?;KbG&a zs{2zhecOV24%@$&%_%-PNmY~IXO@ZP-S1!Lc^VpQDSUh^Wcs%wek=_iW-i?~Dc#}T z!^(;3m2FQ9Kk?n4XrF#&#>9Px@2`v9ZT9ocqxYNtKigT%`)=y*Bj$^li{e!O-jL(1 zaA(nw+#SEa?j@UsK=QGk&h+`UX0{!R1T@7Q^7wtfb!oqVKhz3<=O z-^i-Lr7=cd=6V>WMi+bj0;#6;!#i8C|zTzKI4C;#p))7ta4 z6F$EG`aemU9dZ*M&NdeYlJKQ%u+UBBP_UWM{fQ)}B# zg1t9GSBEXVUtsg{)v6-r6-94~&)ZJ_RC;bcM}plO<=Q7xbxvvW*V-oO?d7kOmAB5i zvLd8!<&2f$_v`=vwW#{?!gqeV(Z`GK@}<-4KYf_&Z#VOl$j@o~KYw)V@7uBNhxWAG zQ%byhFR}f*u`&5_oz&d-`+mPGmEJ$)?!jjE<-Z>_&tG&WNW3I`-X`A+fqR-Cbbl;d zCw1Wpe@FEnul(ltgU>H${D08QUw19x#j1yWu|-muQyw3#4yyn7=0ow)-j3wseM_I4 z)c*Ps5-YPRp0;;56OnOP0kU6FyJ;X{h(xlK=XnIdiy^m8TghWImTT z=lL*6zP050ySw`ZLM3Lk2h0kStLtN#w6I8=-Ko=OI%`Ot=>)z@%>gbt0R|rI-zT2; zbyMV(IdADDT2Or9M&_<513tJR4cKj+l+Z%}=^w6!-mb3uW;OuMb%+Z(OVkMDg_V3(OF zf2j3LjK~6+$;er;&aeD0v}0}Cw%naZ*$PkdX1%nrS~F**JGMjPXGCpRm| zCn#_8wBGjMpMd`z^ZifdPNZAN^l9JYmw$7gJf`5> zB8D?-Bjy=CKeV!7|DETn*YBGZwaB$STx7wG`6VCEcuN0Nz1?+9|GVGPqp-Gn7q)ry^Jg1(%ipX3E3`;-;P`?{+q-dzr=jO86GK<7wHrAKN(b(gSLAn zZK?eH%&*Zw^5*vZ_)m^g{Qs%j2m5_|Zdv4&GU<u4{baFo#|vfVFHV@V++t35b};{=>pygR z-Tv5IUoP{>*0Jl&{12{I%8%|^U^2t0YF)Yg688UREq^|E{K5F9{RFl@S}&LA9o+xW zd~^Pzm>bi!BxL_l{HM>ati@&iilasBAcynV-`VN%$)g~eUAOXl?FN*R6;5LI5k^zgh* z3{UGm7rQ?eoTHS>vfe}c<-)jQ-?m>q@LrUWOJte5T;-DBk_)GtY?ZxMM(mi_E?;L6 zYL((9$JN^1{{58p`X$BBq#x){m7l6}_qarN_R|wG`xkEYNOeEByl7!TQiYL))AaT> zgQ`cJ>XQ~c+&-uMzOtENf|$5Vaz>_5=)p_{yAv8Ie}WiKZA_^<=l{%Q&x4!_ixh=( zr?;L6*dny)#NwhQw`a6ZUu>Lm;7)F{z(KM->v?^Q*_eWiQ(sqx#jmf&o_U0n6pA)gXGVt)yE_!guQFL{~6wCA7lLaMTNL0Lw$h@LHg}p{9wdt62sgu}(KhysHP&)M@d;Q*T+>BlP$EO$8)~KgWwptR>@azH8 z{=@Q<*!~E7Z=L_V?xXw_-zAf?MP4)IOnki~M*pL1=JS{zUNuJFoh6T{2YwF>e_ZZi zfBlfTveM@p-t!pkIR790DAVrupymh5K6jOe&tDupk|#6ycz)V_r&)rNYTS42P~XB{ zDlWx5zq9s*;StPeb%rc}zpl zLIsDyUnv|~x_OhjCY@^c;xwG){5^#$Du6>Ouc3X)RV#D<)US`a^_Q``K0I zdqWnd$*(^u2yTvSu={!?*i+$Cf#Aj^t^Q&;JB6MkDyXg4a&nH%3($J=)D*6Q%b6k69KiVLw@lqRCSCDgI((f`iYAukQ;XLf_*|!TJn5mz;$RI;gC!H0939nq zZ#4Wqn6@-9YFn{+hM}=y!%n{ar(7Rw?kEcn0AV&zoWw1NLz*CB^ln}9$)Kl#&fK_*v^ z&rxyyBsm$~%!8Rzn>w-I)VQ*^dtAAFFf|~ zN%*;gdq0RvCum=a+QareQ0KR0)WN!WtS`0RPCB@;Sl`mVRNdj`xg^osPfl2;?zFgP zW$xw0s`0b^&y2aI-#d$sREq7M;Zns_DY(CtL8td$!2dm8uSKtn3H-M?f6+We{mJVE zIgf5~tz4d?WOH);58ERmsq=m{e46^VF0W#fZ=~~&fN7TQeK%4WyFM)Z^ZexDIlDs| zl)5~`kEa;6`oHmN2%0Cb`iIpiaXUrxC6@aCKmUGX?fshR)ij>3a>t)l_1}N?d*QxQ z&gnb+&-LGbX!GOiodquG0sJZUo%{>xlxpY9{_WTy^x#2d!TFozLFWtZ|B(K+kpIK| zIqdS2+pPk{Kk*bDnfCiZ+-{)-KlE$D#1`z-dn6~gW#>DWD-+knEKi&7b?@N&NA{L> z{g0ROsmdNbks8i4CEF;ZugzQYMo-O2(>%9^Nqn!=K3RTEd3W^e?M|m#y=K(f^yoOOobXwN%SdD<&qR+ly|%+A3+|XK5>76%X_~UaM=Yv` zG5L2D)08tlZc53;Hms|DED$)c#MJ#`5%(7D6C5dVTw!sH3jJG8B#F*0&=+~PjwM5^ z{^t)Jp54L=?(l#2f2MpmSi++1_j|8~NoHF2ZO*my9D2XQZJvS?qgwAx%}EOjyiOmeExs_toWfLqu=p5K3#eAw*Rcem3{Lh<-!k3y3Kg;j? z^(=kDannniTzV$t|CGJI@$A;0Pd4tEquf13XJ642i~e#wE5UWg4R0)|uwNp&e8PGE zE%TE%7Rs$^C_b?B;f&qF3oMR4s{5o-`74c8BlC2nr8~nX%T2MLB`)YVc&evPNq67X zG2u~3*WFZC!{eKu#oai5-{uI5bZX%VyLpV2D&bdRIRpM3KgMx&PW{c}m7dIYORO2Y z{#L%TtbfwR^t7>BOZ2SLUPh&+sx>7-9E{D63YM>9(Ma8DkvPFZ3GgK#g+cq~E%^5)b!#ItQ|ssyl2f&AYI{w`!yNwag!>Z*EWSe7arNL9kMy zg~@4Y+m6FCQlgG%uGQmdKkzs)pg>@OO@dLZ*a{noyj2<7V~~D`j8ShQC()@%}TYSH%~`8 z6xUb&QERoe?z-djY}Sn#TarG%<_wr)bp28Jl7*|^F&kNLJASZGX4_%KfE}7&ZuC{3 ziQQns+;{lkhnmg7kE>af&dsSlEN3%W=T()KL*>6a{&N4PO56W=>(}s%tEpOxt1f5O zXSScy)7|#H?_@Y?wzIl-&k;jcMz5bw|7m#d3=9!ju+z|C-pNOk?_W$fC^_kxjS3TE zFRz98PwnlyKHpdtBUo_W@{?z>P<7JxcWDQUYu?RUBr{QM-CWlKx$AT13;%51RKEA- zKBeu>d@K)+Yx+O&mfG^A=gG(KPl7+GKAmmk82QGl;n^qezmYQ19q*Im_q7{2%6j}; zHfQ4hV)MdsCXt0MNA!MMoh#R7no`yC`$X!@O)kn4iv!ySsFT#H`0zD_-A<_BXJ>V7-ZeHI_-dj5-QnQ!4-X4|Ob9GK_ITEt#}$`icFphU)HwOr z(6a6GWX2O$ZcOHB`uoY@^BlQ1-iqg)TB}c=PAlZ&QvdEdYo{GwZVUyST+-=$McQ*Ow0!xJPz+4kO3`)9-Fk8`>M%mg3L z>3;5@&-!%EpQ-JYPd*pOeiBdBEdTwvy3T#E;OxMQiRM*K8s(PjgZAy7rP%fOiRI_& zbLVRd^R(hWv1Fd;-M{Au@6S_P9hS=6{lH;)N-DMCkNdr&;m=IPR{j@ApR)1|zsB(= zmy?#B-}z~kQKgYm<+c*}pZg>3EBNf-dun;UO8-~*AC{MoBmQvKYv%uWV-v5?U#Y^o zb7ObJ!gmYBTONbz^u8~X<=%Cc6!^z9AJgA|e0@R9yu`WhXMcXQT64BcbNwN8vG?uo zKd%z|Ghew=X7cNxoSuwR$xg+8k8f7!ol^d)Xq|i1XO>%Up6|-;DawDruxR;>Ilo$; zcc>kh|MBGGvYYeYw&|zXpEu_JduadV-R`xnwM}z>%Fl1La8sG2H&67C?fKac?S9JL zo^Z|PPE)$)`T|3#yK@MxwDDyJ=3 z8gV{3JLk&ew!gWY7xwH+`S&C;`{?^h0UMEj=cg*v-<*GZW+D$4yDqDd=SvAEt&K9a zT49^soEF>>Q*d&@oAZy4eY&A9R~0yCqS~c^J#zaWZolI_k9S)7330`V|H@;jc6QV6|EYbs`9NMI z!)MKBopa(pmG077y=3DL=^Uq9UB7>nT;J+EtLykvo2h>t=JgyXn>Q!WDlBBl@(J4o znoW8H8(nS)sBy@Cd&;lW#Wg?9ltuDTr=nM-QQpUhefocGe`myqelO%XG_AN`-5!3$ z$^8$_D`!`=#d*5$UsvB7Gh2bvZqECkeaYRKk8hdCeLf`{qa3e4e|q(^s6X@9A1{pQ z&pkKs@~r-chqvZGtK;}H!Afef|)A#QyrTU3Y|k|2S>Nw6~)7 z5qtd6p8~yC7T(dnu4~7B?sV~)-<$7x%~${Spzg=K-zR=swiPG*pR;e#+0R_^M{AeN z%*{Di*=-vSDp2R{3G$B1ZQpa!w7cxo{ipLb231dZ_vg*V%#Yt|ib`y68?BcJC}Y&x z%YIQy$=7`Iw%cbXO`oV0o3u#g!rB7s*Yzj&%6?V6|LvRfhEt46Gd5j_ZFYN7c2YRG zL+GdGeRcg0eqW|($ld<(LLgH`>1OnGM<2z10?{tgFZb=2ej7Nar<*^mBEIma-Mfd^ zJ;ZH$x35YTS+L*8W`nh&h^xzz?+UB@pLEV+vVZ#f$@cG0;#S^yJ*C>B>b2F*=NkX_ zM!yZr@p&Hk|L6(j`~Rh0?wVUVO%!yLgi?_^${Yb?_5jb619)LAY{~($JQ#8igi_IZ o@X~zHLa$LgazkLJ{LlaYXH9C}l(J|N0|Nttr>mdKI;Vst01$!uZ2$lO literal 0 HcmV?d00001 diff --git a/test/kubernetes/benchmarks/resources/gvisor.png b/test/kubernetes/benchmarks/resources/gvisor.png new file mode 100644 index 0000000000000000000000000000000000000000..c1924213f3264e63eb9fa2902edbeb628566ac92 GIT binary patch literal 65015 zcmeAS@N?(olHy`uVBq!ia0y~yV5wkWU<}}3V_;x7VtR5v0|NtRfk$L90|Vb-5N14{ zzaoW!;qOXM7srr_TW{vFSBQjOKVELm84)jPt-AYi`Q!^56I_pSl?e)Zl%(g3f10AhyXKTw+r1!{&w6@H znN5XVIS-XWC3PiTN3e z3}-CAh@>oP;eo}!n(ADyEMK{3BUh5=vD-$WSJqE|8sI+XA~?3dLJTm z&sEOOy%;gYm&xt>IprH4YmaX(X1{m9t$)Yl-u-VB%kFOV-;lZ8=6u?n?^e~HI=|~y z|Gla&wRL*jk1gNw&Ym)wLNGugm{DU2|p5v30v11(v z=Dh3ao6pWZ)tR{_cK5`y^Y_mhI*OSQFWp#{gV3oZwZFG%NyHYm z!abI$^uQ$5M$}vMtl*O~|EF>WTBOgd)0H;==2-T>Hh%;Evop!v|Dz5Xb#`@mecLZD z9x3P;oL4nXQZ+bo{kz4NtV_L(HQ;p*Hhnna?THgFAeuvHxdh;)gkK z0vl7m)(rMZhwaqova~4GYii5Y81H_*F8qf0{Qqana<$Ue&J10BZL|6PA6>DvSCiMA ze>i=~B0W+vUgu^CEeeqtkSN+SqcNfP^`O)>!c&*a0*Vprt7GzfV z&j0J0s#^lFNKM;bPN^HvMxxZCxqCy2HTm#~kS!$^Cl{ zPt%KKy{hK+FLGO(?bjnRW;!*|t9F-#e%iA3&>_LyIcv`e!<@h{>)^FH+52T)O@3u% zy?lO=>Fl2$Zs***bTuIT5l??jRpK{w9g7k#t=HfFtn)h7iR5+*Zks(4dj0D*>{3v@ z`S7{Z;RD9&llkp`i2eSZ@A68e`@Yzp+|p-Od%nA_6s%Ht{PIv=nl*7CAZJ zg`L0TPnLbCmM^f|UHB<@+69In_t4O-SDot4tk}IKFXg1(p*0<8_DI@#nEaprd04$- z(a(%ct#cKd^R{rQhKGbW#d;e5`&DeYyYABDHPz1rPui$g2)=(QYt45<5D{@3WSRQa zB;6+l@70L(t63P9<=RnWTh16Vg(u-~+u1PLC3fwxbIwdzEB`lXgYmimm|GL1zr2Z@ z<$msq%cfN+7H@5L^=x^yN9TV!Z^&exC7U*N?fvqo{^rWR>T2FEZXU7TvGBAQOy>ob zRr)W&S(EdOWWF|6^(B_y`KA=TJ54!clFq}&{dEb;RlQ#b_Ei7=GIIuZBRfo2L6`NU zLMzL2*+Oq;e1B!vb+`Ka>YU5RbOT&AO?dT6N_)}s8;g&tI=;Ag|IyvM3h)9UZOyRQ{qtD5z{bm6R{6F+kNkc!f%&}umJ6Tfs=SB%%ryOY?yHMMH`V=BTYW5OdQ&2&^>LHH|IX76w<02cil%=i{TPIbTlf7Lq)cQ9fvRC}?Nws{PV1$K~4X zhc6>0udkkz>(k71>gm&|Z`PXcPg>mOpB~;b|76&@81_m-}|)pf~UNH($l@__rs z?)Np!uByI~A6^TYeK_-4sL<}3MpLCBiyYKR43VlIM1D2>&Rp~{RH;Q)uC?&K?fS^r z$w4<~{RYKrvfJHMSHX{kAD^!dU*Fy}i81$T<#yeBsk)5MpYxVj?m(2{Cn_4z@H6VUUmtzyoMb_*lFK}6 z=`KWGa^tkwl=CHwHQ7}u>+PM&&W!<9nlKJG!wH?coW1i@z_S^r;yEM0E9@8=I4|3OjAF6jx@le}c_UQ360ftS} zugCqI^Yw+1(LAT~I@7Zq_|gPnMrCS$STQNmi7zMA=IGqR)$vF6&D*W;vhVMl@B5S0 zWvmxoG%RfYzt((z)31wO5jzTIimnvh?;-a$F>W%uF(=fJhAqMsCI>C7?>#!VlQ}=( z^#91oA?bA=xp6B z@f;?o6BKk=bnwMsdM)#f4^R0`6_x_$cEW!_PL#2zP&5G z4;p+s_B!6_D#O=^EidG+z5j9g@l~fOQQ;k2+UzwYBdqJ&Gh`B|E&F83x}NDf zir)lI-?;x@%si3hbCyTmtvv1>-Fs`}v63$qtE+_=q8G1PHLF+pcY@72W;xF5;+wgl zMl|d=I4h$$&-LAkqp~LiVTXVj?`s!JD$o}sWcWvIB z=wzwDGWOeBR?4hmy^$w3_b1=y(`l#Iai5HM^PROeS~wC`vNX(Kod;^JonWjrk9*V? zXVaQ{Ylo`lMNu2o)s}tVf1H+cTCQ_JDX!+<a5zU=yFtoW{B$A??m^N%|}sod>9zjG&|AU@*z;G)XkOBJ2g z&F6P2YEHDS|G&3uZr!P?XQeKOhTCMh{$dc&zgO{=J8VH)^fkL!gFHmRtf0r#uQcoL ziJNP>+IIxo{CKwH%JzqA&sKJSe%AAH*-3xrUIxyGOU3PS3w`BWjZP!g8WRsJ=4?A= z{riY+kviw2bJqFwXLpJ5WnY`I%y%xYD3i*R<8qbDmtOa4Rg-i_G>ru&9|(@HND>b< zs_SXrQJnR^Gw1FuU(WJn>9+skHfPWG+5NO~1;2vk2aSzlanAoOtg~T(%Wzh#BIII? ziQJ=e8{K1%P1BE`U7$XT+w82Y@cNz1o2H*SrL`^VYSp*n@~3Zn|14&f8DIC2`}dE< z@;COpuS@^@TtBV;@6I`EV>eF>4GB3`YO|-JytunI|LDG#o8DiTiztjU%w}cG6X!2{ zzTuwuCVO#R{~gZtkDn}`WxMRN+x_0WNXZjzyt8lW@7GByzW2WmRIN^1Q2x^OyYb=_ zE$i1`vTuUQ0B8Pv3vRxwylo`Md2RJ?jcd<8c;Dze?wjzOZ%^Y~`(JWqFK0#O9NW8o zqj*|)dp%(dgRt^QmWe?j2gU`x~cr?1#(#bz64rbk)>fpHlU4$~3XqeKN+d z&RN4+E}K@_vt_eZ{1@zh(a!(;jQ5Tm#Ycb4JS%s3=JMFnW@-y-kBsFH5=Y)<;eL8oXWY5-jg!JZb3GKJWMb*4?58X4U7-XXX6gGoMTE{vXBIs>&*8Rs8&4*40xF zO^g0);!IRC+}8OsUt(f>8O z=Ra}(l-jRPYgXubCOth>`sU_x`z>6NpP7yd#0Mp<4mnxjB6enS>^eqEc&1e7z7e&T zZ{DMW(FWUh$o{(c_{Of%*+NHip{FxE|MZU{ct( z>1q<&FFsB*;oT9q8P*cq6!9QX^>5!at4VuIzxx08&N;q@|Jj+_dW?#aE0flW#;vLP zRxH1HgJG}eibM|0nThvTGzsS2-&djzFIOg{GV!m!=@@o7=(*v=iAMYX{JCDX^^{z} zF_kUO{tWleN1srOHRgU9E;;YeQjT9OdcXPZO2XUADUAn|yO!=gx>)ts)amw<0s<3X zUJ8A);hwlzwGI1MnZwg`A~{Xf{MN+p?>t-e$9DBu^CJ`2Y*Jm7GUbz%?F>a|3RY-l zmGd=`%ULkoB&@;49bNq4{P ze8kJTZQ3zT2eU;#n!d(!?`mJyZkCdJ`NWS3n~R2*ie+|)E#hix*U>-QS3J*DZ~jA> z^v~Ol+o_8jeDds?%3bGa*|lj;{;afJZhHGib=Lvk<@*=zpV}i6$+w%e)aC8-OWAWk zHIc(I4x6qO-}6@{-&mVr@jLh6eNIuyWAWbtZf(stUsj>JRQuTZEmjj8|74hMixfV) zQSZ77Kd1$GLWap{fE)PEb?*6{zl5x}v z5j*?ek6XBS`R#vPE9+f;pEEo{&|bl8-_I&W~fvYGTTcJzFtkKv$NYT@#Lk&&ObZVURgApb(pI3(tA}# zRmjbP-?!$UzPx$Y1KT11QuQ=ZLx&7-4qe3&kut!hzmTl>q8FE;H3MqN3Mvt*z5XZrjNoxgGO0-jQH)ek1F3uW#^ zPUe|u_dLY+lD>L%*SzzOV})N|)!cG+)^pZm-MOFbS1@;-eQWZSu<*XpUvkKmd(n_idboxyZhLc|3|bVQ&UAnW~Xf`d+WA3OwfMX zO|1)G-dIhNEk9!c9*OyI$@)f1A^+y+<>$6t`WIL8Px#xzIh8r@?!>R%_P+kl#va|c zJ(JePZZ=$~Stlww_09a$Y;cdt?)Q8DH)qZFCH0@HX}@dnQ`0Q#M@d=K|IRhx+XFK- zKQT;L%G9s=?!yWRy$=P)r{#qGI-B{c!c}J1Y~2qRDw;c1-ME+hmN$yY|N6S;ADIq+ zUwYf{noDt)mc;HAc88Tu9o@O@K(KY){t)qu zJ^7ZYuM$!-Ph~#8%WkTmgMCNR%e)zm zj@yJpD|Fl=^&YdP9exr~pJ~1K>9@yXzblP$oBw*8+i&MB+`QNHy@2IGMsJJ%{Y}rea~-eqTjqLxxo?W}$!FP%EH$k1Jhe_{ z7tKz&tNZ)au2nymE^zu^zeY{!ea57(JA2PQzM3-i+qbYA=T~nxdq3HC`S~}mUt64! z@>Zx{@pZ1pO;CM!i6v6*?zW|hCzsE!DRoj#75leuN~x~Qw%FudWse>QNvp*+lrEg-g)b5ABmfyJSe=ZT!*ood&vQ_ly)mtmpt=ky%YFv=u;Xb@kzlkG`@80$^9ExuOy{<`H-uYzjb^laa|mTiB_VSHP1o##OtiQauHF1q}! z-gKI2(m(mrC&f$q)fBpxM>5TOl3}PXE&Jjt>lXk2&n=cs)wI~=SC~3g=@tKE*)^TZ zW^eag{``N}1Iz2BJ|Pb_ZhJ2DGGO|$7qiq}hF?ryQ{eJA z_W$!c?c+jNK1+CN++Fx>RwH<*Uoy-;PC$H~p-m7eRYnklLgECJ3o7NsvWchHSXFAuF zOMl-MKKGDqJ8<#dt&O5*Piz$JwTa}}`R&EU`S$iR-lSS&*khsc&vLmd$upv-^2{j*l@Uf6H?G~tjQu9N%lPh+>Dy8qi|*U5SG^Khn{Z6} zw(YXx^0ne||H6JGoU;M7Brmz0d8j13etOAE*YD9=3o2x1mA*E6=x}u9x=U}K+il}= z`21qI%b!PUJbzuBk?FYYV7E?jO1@|Alv!#sYhV0*pIf;6)I*NtT3zjtti`FH1p;;Q z4ztSL-}iin=#@t|wdxdOZkL|1yT*J{YU&&>RKV|UKI z^8M;t8Vpx7Uss%ER%&MbWC8c1S|9PuSsS&}q9^ofEDC(R<0|32#=Vn8 zfIhWZMmKqRT^6~xNx7ass4sHSA||wFuF1_W1&!_H*ALwU&DW_!J_vOE`)k^r2l_iR z+??hH%=@){!*3<~?4K1BXa{M0aFPG7Bt5nI!OcKL zW+s2(6GbQ1dZcane7Roa;-0pgeaZ1*%U@Jjb(PP)d}7y%rkT+vSTzkr)*cL2x9f7t zy({a>vvuk7qbc%MR*9i%jt^FD%lhkaEOb?l$JXS@uP)sOjT1~@X>^~){=Yr%RCZF| zDGixp({v(d8XTWoZ`S|k#=XMY9|ymGxFwr&W5Z3YpgAu%=03QfIZfgF)#;ywZ~5}7 zu-z-0yM0Ff%bg26j&;u{bles=BRx%D*2`AV(Aj;L4|7f9hUhRQrvCe@%NPCqme%~}*h;VXbr&j1N|tn#cWSlW(r~(Q zHPWa!#PCg)%Y%oRX**lrdRI=lsrToH`bLi}pEp;s+`gT1-0~P(BRJJ91GNc%OuKXE z{hr55Tdd|JyXEfJIB1>QXjS(!wa_VOQ-x!SoM!3qT>VY!pX)ia_)R;&9bNoz+tcSB z9~#zP{mY~ftf;=LSHp!x*w@5=`RBiX*w0fK9{a|X|{gfC&|}h&MYCpWfkwcZ-DA~(4gMh$i0l#7Tym9 z3taz+xqmv-c7S=4mxzE;owk3wW=LD)`oBpY7nNEo|7R{Ktz5NB@9!e7V^>$@l&lb4 zs&d6?WxdX8uC(8^ZC>lwXKs-5`fB)O?b+p%JOu;QkF?H_301XXkz06HApEQCr z#Vhz0?4SKwq@2_NKtL!0QD0UBANQtULB4ua;{H#_Fr|&2BZAxd-o)g*Ip2PCqn{Y2%i)S})u!Zv|doqOo`Ry1br*9JSvLf@jy2Ci6!%wJAwj zOqk}e{CQ?jr?t{NiRBY&ZzyoBRy!IPQDdV%@vVetg~-7VCL$Ft?(07E3-Nw=qwQyZ z+250O9*qY!A65KtM`Nq}zR9NhniN`GL2WKcrvB5W3hjAEJ}2*;`Q7r5&}wG?MGqbY z>O7wH^+)yEdxp;+-4K18er!*`xo9uQ#DBt# zsc2<8x6PlODk9aDQB&SNSr+wRr4svW>zk)d=2vD-|CU-AYc*}BNqndJgI2KK|eB`0-UFp2`t2fX3-MrLNU|kjKv%Wr0 zNlQag(8jdnop1Q3vwSmDLH!36E}O0u{8=Z@{<)cM=QAUCXV%rKGKc<)w{6w_)tlvN zrt? zE@OAruKAOab7MTeC37jM&&t}kH&FOYCu6>c#)sCsv$o%GU2nTK_5o+d#3{@E9{IfX zIMc?Ka;NGA7ef|b*|IBr+l`W$uXx>+d)8Q)efbrCTiAA)%zphX@_Y3b?~k9r)OcLc z@8$Q^`-HaM`>Mne_(Yn4fx%R_w&RL`h2NSCvUYgC(Q{r$Sw(qEeU63jVl6n?YJTi`C%w61n(cjcyDbe=SrP7Ydx!Nm&l{J(~3k+W^C;<^Y;%vD%8J3V%@h-8rxT~ zm0w-*U}c$=hz@_1iyysmvA=F9_K-H0Ev!<`>6-gGk&TC&&RSl0p#q4p2ou7q2%Obg+#`5LuV zd(y_1H&VVmB}R|tPTO&y_>o<5)OrK;tD@myOBby0`c<-fL7^{Kx3)Fwyn2y~658Eu zE4Y@)DvEht43h5L^T`793DMUFGO4 zHfPW5phXdXu5*R0IC^bX<+0lJ+r5s@-oEmA>^iC2>VInE)>Iws+8UtY(4G0pCi~o> z`_H-GF*LHLEY3Kv+wi25XUGG~2|dd**LoN3e(QDov;N74?^b&r^P9Bf_C$p?_opwt z)-G8*ElGU(G||JN6HXVYXg(1OO-^?;X}j<*^708g@3JGIS1PKmWp~M(*?vyl@X56! zm)4dooHS*_?)2$K;S2SPa#An8ah(6rl}o8&(eL$>&TT)PbzFUB#$?dko=VVzK$p+! zBJv;D?&?X&zMkhZgSqnOkM`uam_Iu+Dpwcp(ALuW9~$uUgtPv(TZ{E$cJOqubZGOR zsZf6QI(S;|J%I{`urBt#De_k5zc0Oh_)XGjoz|n%<}SLVwCQBZEaw@ga+e<0y7bk@ z(57Tf)rHqD`@Y=^)$H4rv?}GFhne40=jw>#oO7im+yATK97<-;ZwuYLRc*R1G9$+v&i_ZO|>^N`@Kixt`|ynwTfvynYTxXv)c)@#$l zchBtSi$|UN`m@)@lZ)-eoci6J8M9KZKYV!ZZ<^fE#T>^z*6eAl<(VFD6g~IyS?hTp zla5B!U5H)4`%v~mh2f+1vP>SUg0`H>?w_7wx772Q932-7HgC>ChdCpKa#2c z#>SNNP=j|0(GNXrSNk>1IMKR_Hzj}K*U~rTF5p>X&gcgm&sId_KXUySv@Pf5s}{@t z@SgdSpP7Jz zDW7li-)KDd^S6wt=c-_h4=zWS_I4%kOlF#xvA7}U*~c$Bu4XZ<`x36n&zCJ)w^Fp? zNA^wedlTo~&*T<5XRE;ghb01@EOqm#mS0c4RHL&8D1PDeK>b)t2P!j`ge0 zf46|^n8T*;lT?hXl&?&i(5pPzy4v}o*!vX+LDS4CJ`V)HzE9rU`T5IAmyJbFtw4j& zXSjLp9oxCpHRwZztlUv%2O~+_N9QEg2miGMIbCu2Em_@dI%N`%6rb;!^5!$!xoWMW z%&Rv~TqQ9{>Vhq+k^f)AJ9`}Rvjr-i{4o$OkEvU05w!f6_$TuWvx%=XwEyjx@aD~~ zueZJ}HT&7Rpj!Wm>yULDleqXBX0+%F+&trNaerVbTW$&Xe zz$1i5xU!c7`(80QX`%J(3)jglF0C$xf4_-GdcR^))a7h=Qt9;F2&n~d7J)#{Pp}z8%-j_dHlxDIY(fS~A@~B=Q zN98LQ_r3hJ$By?EZ?HOdJ2IKO)PTQQH2gC_!O+q{Ji$yMDs-=!klyI$yqk?pUYuSEmrD|H`K%H}D_ zyO_z{3LB9S=bL#s2zgknz+k9mNt(dy$%)jU4yWP)C%Ro)*+4(hX zzWd&Ju09r;Z1!|ny6(kim;LKfW}8jTFSg0w^DlMH^;h|zhJ|vu)Um~SS#zVR>Lx6X zEj{e#eA16~)wa36&okNWHI|pp7oE4h6)fWn)0La9H1%26@2N!-)Ad8I9RK|D`p4MlH}BWoJNJ3x zF+tWXHx;{b15_V#O8s~I`Do|08=;(~pb>_K7A~6-=6wd&TsiW0&pjccagTAeYWEU; z`yX;z3?5pu&8J5%;uep2c0N9bQ*VCFALZ)rbMti;K3%E)Y=e8+zxk@0J}^DF?%&0s z16oGVD_JVe{c`)J2NyKo%wfE~`Lh4cSzqlxdX=4-m>fC(uzup+UHzMSmb)3hodQkH>%tfGQMMZ*Zz>mMqmDAPqCG!ve^=DGnm?eEw zxb&F#2O$^_A)uPtO*J47w;c zcfR=Rlkb9#GFrR1%(`JUhxN_6JemC=)8&?aS|N3D$ECg}H!mIe8Tiz~Un=Tqn9`hz zy9?SwPFmsZTU(_US~R3> z%)6_$E#;(__mrPi@0h%0dbo`E@BLpU%6shSjE~c8Q#cK6ObZW4P6p5OwSV8&@%5Ei z*U}4;9;-z8_wBs6u2SFbzvmjwsQsWEt^Z}l)}+o?yzjl%pI`F%|6aFUQL51oI_v&x zxfbZg=MiSx`sdB2&N4YulvQ7Llx_IX^#bkM6=f} zy1%FTR`vV3({VHYa)K6zO>-!Id*t}dqvGch5|gE@f1mMu^JCxRn+NxvOJ-EPl67Qe zMI>nYZNKi1-S_@IcZ%?7z2*8?o5Mz7cHkaGJ*6Evj*9NN_Dbz@?$^HjvtRwFpz^sd z25|-7qg{=>epoPnD%h^odMo9%VW8{f#h>`nr?q}xaQtJ+ev^xjtY;oqj+&g=ttJs{ zdhv?Meb$TH9vQmtcTqm6biUMlZ<$Ce-#gd&wLMBw!Y@8rZu)s!wSaS)?&9b27t-%| zmFdmBU$i&d!Qlz0?J=RCu|4m^_T#$*cE0#iEgTiSB>Lh9G2JO`Ji7$D+{HM|{_8{^ zuKRU=ewgjd>2W`%oUM70TlV_O&1no4Yrd3$))S@v$OyF-QNrmt^{C8M`y)xQ0> z>(eLqB;SiwqD=f5x*3ZmhN~M%h)>SlB>QWX$Z{6L=qEGRaH)nZ=}>F^v?-|I_U9{6 zjmPcym~87CbG*B{wXa*WOr1I_gU@coY1gfR){K^S9!;^7J1Fe>ENO9sc;x2j;}^(locT?-Uon^iA?BCpSslUB#Mvmak_bK)lqdw_w+AVVOimT-n!(WxGjCT)bzgw}X zvFrsXc`ss>o1U$-o_T}Gwbm2q|Nm+x_Sx!l+`iO$X{vCVlcHxZXbPt8=TH4_2j^72 z*_m#iRDSQH(C%NWj^B7Ov3%pf=XVm8+R9!j=5d+Av!PpW$NFzOKCM_GVgFo_!)C$O z1r{uFRrmVer^&`SowJPabduEzeYOe(=%zm*Pe}mx?1U z!L4DB4<6XOF8^avXE`_1NqvJoAHFYr^Z9&Tn*QD&@!F}+cV*8{5bxR9&TCb@N%Z4I z%K)}SmF_e8v%?fK@12yva779WgVh@p$C-w3S-3 z$~Bv2?vLJGS14K`B`#a~uQ@($VaTNyr%qD+aDn$%TZFjM<@|cXM5Tj|_JV4s~{$KGfZziSuegEqg zUe)d7&X~MzUg}cMxtHE+t==un(tj=UxSZ`dp$|!a|38WO(tdzBs`tX*E`%kuYv6zIOr)HsdP-pbQ z{fj^D6g^pTdeX~V_5X|4?tdvI(Z>xMQt_B|Kv~lMScpKV(OR(!ok_>`u0NW(@ksWz zhPTbzzdySuv1jS`$^LuJ=k5C_xjI=S|N7m{uRS^>H?%}J80;0Oa9Ax=xr15PRmsT2 z@x8e%XI}d>W#g^lqv9?8C8LQe~&P#A{3X z?=P=5QB?Z(p+k1R{*!l;?3u((7#J8PWHq`^?D0;DZTX<**4gCFb~JnAm+$viFL+`x z!@ho_sO(a4|HN3XHphq)MXy6cl5Q&A^}2C!Z(CH^sfW!sGu`K$K4 zT5(>%`hq@NarcxB-SeuAA|E_l>9=M6+T05*M%#7T$`@W&zqoDR{6~M^t=YS|@!RL@ z6m_@bzUPYzkAlW+6IL_zYt52BvP4 zwP7zlwidB0yrSE^(Du!=m!X<{DxF=tSEH8bUGUhif5T0zWZt&Pr!SWq720P9EpgbQ zB(by3^2MiB>bo;%*L?Yvw)=0nuX1TYP~iWJm0GW+uL#<}+&>{O);Y$+y{cg8681$_ z7nXWB$4d3b3Hff1=elbAZ=V*+-Bqi0n_52MtMI-%i(k<1WYcx?V3x~=TVv*#Nf;hp z+?mE_zH3?0ob#1YisrNAb%NLYmv>1lU~qWGX)`PBjnwN$6U8t4svKMx{AgQ7)c(5Q z)uLtpzCTtfd+GXpbAwRRj7iJV=9n1hCHXAMIsNAFjLM)xuJd#^+5E566q(vsD>Io8wprz&V%TA2Pu=p2m)16Wf`w}1Rrzg&qNXa?fs%B-=uN# z<#`tb7;pZ%HD2TK-FaEZ*L9|CjD6o`Do_~|c*jcXi;es9*LC0iZOgj*=E~-$t!o$S zzN?f~t(v#kb=_XGlV-Eji%b>&vS{DmCw}d%_D+ka6szO&%s-v@@#A)uQ^z*NIj5iR z{d@G_yFF#@J9q3Xdy{6%B^9e&G}Ha9z@nW?Rotg}XV~$4+Nrnc@786{KVGqZ{n>T$ zv$FzctJ|WdbY~w|ycw9jcBY)G^XtBDgC2L!i?&YBx1P?}FV$CUapPFkTls5?GPcXL z_mpP`X-vPqH>mYR`s_-ZPyP3;Q|7;$SR)X#_{*EfSCd2zt6sLfeZJS-n^SEqXT_3` zC%-nXEt6evq0(P$d+m$wv-1AUEt%i7#y+D@F+1gL;>@qN_Wk?xV$R8}J^j=B^d%Rm zIzHpFG0D6u_d2PiJi<#v{>)7A>#djDcxOlVZrEJ%QmgFfkC|umXAAfnEiIn1COXjQ zrQWBO-}C1l_@c8|gvq~3saiR7Pd}yZqM+&(>L-E2{HZV~_I$3ByQ(yPG`J zX6FSO-L;E9F?GIt-`m|!Pc#L6o>~=h^iKP_PKDa<<(lg_ANOed>=ip_CEI^D)QW#i z@u79ei@9b7cDJ9C4R_Ei`?=I(cH#b%DyB&@*LxoG<(s%v$(5;p^FKcQO}6*m8<)Mg zXZL2?d1nimLh;boEXI~Xs;+)}<>nR_iS7~$He9#(n|TbQqS13#mkhmm(s}z6w(Lq) z`Qg4J#q!MMwMt3SZ~0v%FRgj3RHAiy@{teqDTX`5Z`zggeE;0NurDRm%RUpMUPPrpBDu&?orc@z5@Lj?$!q=6Sb?q&S|**L1t@t(YDCRYi^bPv73`TQ~t9u zL*Fm!WBeO+T=?GC3a(zc(sPD~;nQX5Hx=3Eoj)gA!MyuL)9)KgR;O=#xqbf$P$%|u z-u!P&pas)8pdC0Xl=WO$_ya>jbKgpf_@49m{kG9AQ~!gF-M08cGP4`Dm(+9EEC}V! zEb3I9CGv!SGq=QakCNOyQ`B7Rcdk{dp0Z~9VaIbj7VV$Ltp74>b$QXVTbDjBOnGm) z%`{ZAa4KKp^TN;z{ULi*D+L*Qx68S2aj;Tc#e7@q(uKVy2Pgbpbn~iyYt^l5-S(G$ z&ka>vv1*lGX3+`n?>qjQ-0z6p!z%tH;l1jI9Y6FR2b#YPi8ollktXkIaz4bx)>t)=eRqHL9oWjwaDu1N z{dLNg{!+!Fo2vXOA!}BwS#xXM{`l>A!oR;vmcKc5y8R)(H-^vRAIEM=Ne%0^-gCoL z{IJ55ly!US4;NRXg5>aQk*F$*R0s{3G30HTt?o zi~FY6A3|<29f>cA@qTqq&{qF(i%I@W>sfryr?6bv$POJ8YBn3AM~7 zyAGZG@Ah1utW+bCRl3sd-k+f9LY@b^mi7I4{r=_NvJ~4Bev2!URc@Mov-~V^b90LR zKVD_F?>Cj3KKuAhF7B9@!}Rw5#T4h4dmpD7UI`UD{Vng)ovoLm zGUiO0x!7xIq^cFiT&3bCGqjKGWa@gjZBf#XZCq7x{k4AouWu_0Yk#kF+Ht?q%i|aB zPrLAMwnLfpP34sH6ZU46AN{R=GiFo%+l~|eC;ol3Vg8AI+yB3n-gN$?-Io4`pHnuU zc&EC%?^*A|u7o*G-}~ZB?$t`|&!1&k#uczG-siuGyfxGDUHKPy{yX=C4Hx8MHe&6LF5od;CAR?PR^R{ys4?#BI#zMnbqdEuX} z)bh^o*PKrjzuxz~UH;#R zsKV#H*Y~H`?&@jrnDp@5iKBln#XWdc53{ZDm9jsK*(?)Qt8dkby;{`LqpXrJ`;ddg=5U!&cZ?~6{%OcClm@6A`nzNt2K z{>46-wf`^F-RP7#VpiR;`0=ie?P9mRRHB;az1~**NJKs>NW;#+N-kXc=R(;%xq5}H zPdpYTt#sCWA2oU3^DR<^cS3g_Q7w^p8`cug7iDr{?MYRp+{@OyO4G7}Bn}4W#>!7B zjCy;c*6ogqN%Vu08(HVgUHdxpO1)>$mr0*DhhMb$C~(t}uTeeoVnW%)tnb;=|3*81 z<9>GT$&^p>4u4I=ad_J+AL;iJLF|!p_iNuxI)UnH_WA*o1Cy>3;X+ubr#ew^vtIzPbPW z!-a!)?iaDMTn-hGNZYGgCN)X$_)_nKJLN(*)~&fQ^V!0?C%!#7?l7&luSEar#p0X0 z{9G1qR+HTOVy|5ZkN;(rxywJrE#Q0>`lUkqX7jd)Z)P{cQ~vC;j~D#Pe!|L{W!{`$ z_GK+fv9D@U|4T(bKkz{&l4tAU_n#+S>AiltKZ>tgWalZBU;i?F?b~LDNk7dz?QNA==6SQvlf3>mjOBCEnG-AH zj!2kIZm(LeEVXlzz`yAQiWP@tMgMuvSrYGcto~}vWwE>;kE&Yd9y_e; zXLf7ubgi7@dD(r%y?@Q0rp!Kh?e_f5hyKO<<9@w+3&-x+IV!d96&u$betp$L;E2&` zGv_USocgm~6lD3URo|8i{!(prbN%(3_qSfs%Q&+?pY!JXZ5l`G8S5NHU)E{9*gnyB z39DWCfwm%bAE7gcRWH6#(p=al`}D%PkAgSO+;Ux~)xPV(>&$N!i+`_FYklc_C*xs` z+@fdgw?$l9HRiuO$I;k+E;s(Sp`I(}&d{YxWcNN3OtUvvbW*YQ>ay2UXa8?l6tnB9 zrES`s7XG;7t=Fe_a@c@I?u&Gb)~Q#|zP?=R?)uCBloz`mD*h8x8`ya-dwn5$sL8Sm zpZ#;x3zpj6)8*XtE$hwe?9H(+X1h<;U2OT_Ht*^zR36Ylvf@87Uufd<<`BETlDP|qEqh6U%U0WB2_3jZ2OJp zhfj4lr2jd={YZOKA>&>Cs^`zeuXnRdo#N3e=WAp$Q|ZGAS?}+U>~k_EojbZZdCiNc zgB%k-o|x^qWo`N7vzrz#srY_S{%!Wt{Nqs#^N(l0sTGao&nY~!arMj1J1wp+eRnV2 zM5N-5)OuId89QA+{kZj=Usog7d7nT3ENS=4mY-J$b?%DFxqtY&%Mpv$RjV@GT+({~ znu~t>VRp;jskZ;B;t8KdcavwwjShab|9GlL$SKYKyJ7X$spmI&-Fc{)BOY(Te8@vA z^5)6(^p*pkUH-iKkhARQA&)c1G$Zy{Ua~2(uUK$eo&D|}hm=OCU*R`4$Me1O>|WP= zg3Z-wRqyUfueq}4{`pT@KY3}{k3Ys`PtF=t)oqFAc^94k+e%t4%ICd8_e2r4qDjAQ zGx@JE%{g_lGb&d_p`9j`tY(T+2YbB)_lHnO)!oL2A z%HQQe%~a%g7ey4Wteidf-L{5plOwNvv3Yvu{M6ovP~mGI-p@I{=Hj`kS-B>1dPz0D z$+cZ_>$fZGSF?XQTQxB}=--8o8;83;{bpO5##iD}d-1dEiCvTCCg0y&k-I4S3vg+W17D8kH1ExMDl<5 zD0@RM=tk(Y=EClC87>bb*Spre@oqa18hd`HaBsV??uVSSCsl8IerjbY;%hFeo%6J} zXZxmuF@eIzi$LiO%Xw=&peD{77{&nkT*@a4p! zi`y>9-}0TjR$f$ZdG24IpW6-Y->_Tdd1&kH8--b~{vA<{l)K3Cc#Xrpixan{s&5SN z?|yXaTwcwW8vos9_onOKP`}FiZfuirU(Wt+_0S#Ea4+55Mg-L90S@_PE^sP>JkD`rLaZjrKE+IS%N z@{tH>;bl=;8FfBkhrVwveJ|P*vnkj8igX{?H9ZGJZpIe^TK&4OMPQ9+n+37b@I4lF*&_m#P0vv5VDPqlW#@R{G?2|2aHoBKIcQ%2FR4 z$G<(X?);k*OHbwADbAXt@-D}BO7^SW;kV{?#V^~Lwr7<@?CaR>piI5D&2yf-|6Rp& z-YzA4rLO$fy$8t^LTMta; zW7_{tkZI{xr=O`eGuCtby7)w=XK8)?(Z0S+ol6V$99)0vWK2Z3Imf~E`+rPdBlCwn zqd^!-`3s$$U}A)fIS( zfA4*^Zq|bD+b^30-8I^~YFk^^g8qfaZEwAgmGQOrKK(_Yce%Od#M*Ul7w;Br6yNxa zwM2KhVpY#+(2|C^WpO^YUj-cFZu{(KQ+Qu|txD5z{kDo%{kyxv_V-DwPfjlXyz}C_ zWmf_$G|T@?il5B(nkl`+sZ#m+U$$NE=7lQ9Htb@bQjqI4VgKq-o%@f&4{g1zU*ePZ zRy;0Z$Gv~cclv)dvr${PH`USJX<>0-=M|gPH&4~7$M$blmjG39;d`}yRu>`j$y zwrnDi_c!_-;n;1e`Q{_1;`9rdAubbyolc5WsNIM?a!S@TLu#2^n3 zh*gBdpLqMdHS2ijf@$ykULR#Xsds_pvF)a|>n^UwuYVtnV)+-ki$A{Ws?v!)taVE? zu9d|zsLwA^4VfnL@k4XIBg@I6-#_myHgp? zOKusw<@MBwnAu@vC^|99@)_@@h-;dU99a4I+?rkvrl&JJo*v1`aXqBEsO@r5 z(UkeuR5r(d->bi&#c^SQL4Qoq^4SL#bMm>zY2KaQC!9a~@5I{oD^Fee8oQ)JU!PA` z^7{nYo0|Wh*jNXhe>?rO%_99dT8+;?u_dL4PI}hRaX{IfMJ_I2X8+3Ye(v%wrx$L1 z`(hVwp~uv|rDC^qq;4my5p%q?aqG0hn{0OFHn-+&etU}3c;S{M4gAl~Br~eA2&$ZZ z`&jdMTFZgYheFkl?CG_OFDO5A;iO~Pl`6p+w)(e%PYQBF8r~RJ{OuHwoj&uMenl^* zT4?5<)hjo%1sryL=@e>`A6Dz8zhA&6RQT)kH+ga=MJlvH4*pOOYjh9k&Q-Uc@;aiY zzh}#I|CtIG4JLKD`^1JlcQ|hUNF{I4GXL|6&+Rt(rv`7zUbn`(@YL+!2M_O7oYOgB zUN$ZD?%I+^Ya4lYv-$tIlpWQ4U@^xs_rfLnIOclou!vgY#`WOewyBnS!PmIXt&n^E zd(Yx4GMlnf3?sBfxT@zo33Ije|HSJVHCL|aLiYS}KA!De&n^{aMwhQ~{(a$(%?Z6H z*4rok++lR~z4dKl|1k4QIuCen{rk9Tadh0mm@}6N*_D730*KX)ykyQP#f{DMn`z=#N;y3kx=t*Z*g}w%j zckweW5xvnJK6i#7)A=UJ+Ne!)+Y{UTbQukIG~Zvn`pJa&WiFp?z5ah{#x6hKj}al$ zt;JTH4n4TTS@_HAHzI4d&J;TMIO4-IR+ht8YxjSyICZWtCEv8;o@c7NZXUlV+nLy> z%FcR6!^00)tW~P0QPsP-`gz9OL+hIQoNZ=h$h<4aSQWfOey-nx=}%rGmyAnXfFAt=sB$--DIqbI$BPxlC)1=uU}S zTmS9J|MYX8smli&pJ(CUB_9W>1aApe|DUDsMOAyz4b2aCR&c2bg>@@zW|2Fo(fgGt zVn;#brIy~avrpeVrt_*!RC{rofBTP%UT+R^|4wf?5d7~?;+kxS{!D4nixO>;RtNXb zv+C+8ySsme_{*;)Wje1+0^eu58~nL@B59r4VeuQLy(PDz62vFonRDc5O$>`%`m8>+ z8F8Bm%NnoF5^?)vA%5rm#aWLyLQTsi)J_zrUOsuQ7*oGM$?2cFyg#?ysmniE7qz79 z!YPsMbIZBByVrcNKA92!x`I=nVq5$j!@KA7Y-VMATXcSo+qZOOza+Cx4%6Ok3&eWM zPabMK&~aDipsU zFrGGf|98gc>g#%cE-TXJ1?`95?3nJKcGtRp#ov4DS`Xj2=pL7~*FN`E&bP03Sl8^@ zwJfy5T7T~c&6fVK6!TX9(;SWMw=>=6NWU)gJI*t=f~#V!;QV*frdXv%e~_%sTy*a6 zqxsnzW~?mOqMsW1W%ahtdv!hk9JP}(t;pG^_vGCc|4whA5`V|Kmwa^xZMHa?IB6{s zSS54+?9t61XQh<9d1-(1>2zLk|Ne#7_iPTE#bM)?=lI;?R7d=U-x*B#vH6{~w_i0) zUUTimdyPqjTyw>(#h*wAPCT=i^W**cn_3?}tn(>7-~N6qV2lxES@*4(h(C{ z^)|O8@BZ-;*|-N&k~UuXa_4{%n8im25fl+}8FNd*UuS z=&Z|NedL$C_0gQ$Htpgf-@11_+9CI&->`7~;i+y+=Z&n)fAaf2U)fa2o%{b7+jG}V z^?NkRdkRIuCMpQC&YQud)83m|C~gq8CgQYO{CAbtK?XUyj};ZH{|w&Sl^vFUBC4?6 zU0=591FL{t)Wle;4`+G=D}F>I-#KSFW8q}sgUUAp8En5xor%8iPM5pwVuE7$U9URR zwSmFHb1y#Pkn1b|zaaXX>zw-yY1=kucV1a?@SAIN^?t{)j6F(*LJCS`LcNjl%?Wt+Y9CRG{1Q0<*vIrDOBTNl2A(D zfyYI@UraPiZszPxJypypY+UjB>-T$quXR0mS^nqgSvleUFFSu8OW(Aq)LMK)!RPE) z&_rM|%e?CI)|S`a#XY)mh1GI)|kZQ3D&+FcBRzb?>Id5ZTFj3B6aC8 zEOIxqw|?6u6S4n(Ms{m|_c4E7rOcqM3eTRO_i{DKzWrIG%enZ{L#qo`w=|QB3^u*D zn)WWOn@hBJ$*iTXWFpUXr@y~jyyduNk%w2(t`{k@4=na*a~JErJ8Aos$DgHhUzFzm zn!|oFY|Ey~yy(wz`d0PQvCG@Hzpdyi{nP!VAm#{j`VvWf9>4Xrwz1Zc>zU)4xi2Z7 z*q5as6XN_J^Oc3<_eozn-(7BqnYP%o)b;SoN;Pg9-o;thmmmDQtMNwurM*{eVnw~~ zMEExK2v+<_is=2SRPpz!{v@@$KUVdfsv2Kk7;TD7eqDUy=$ps=btj|>(_dZ+4axxc$M`s$^<2HxsH4^~DWVBK@@Z^l8}Id8O& z8{W9XC4Sv>rpAYirk9sePHTQY!Fn|BX!@3i7mZT-Ew*WfI<+dFJ-^3o>glrgG8|RA zx|#24St@Q4@XOeJKllCajsuG`ROMW+-M+NX=EggtD-p9ov}COI{lE8c?X%qUZt*bR zjJxeQ0u>^Lj~-;53h;hV*kN0@Y4;mF*Fg8Jd!}}?_whBSslMI4dXFso`;5ENQ!bsT zywD_~d$UjMZtnBVFLy}fMXs5BJA>~naDdRJa#8@G*)uhOopSLaWvojsWrzedH;jnihpW?7`>BTJ%Qrrsb22UNioE$tiyMM>+tYduZ2r)A z!#Z=K^F{8>B~3?U*S}MAza~84@ViBQs@0d~*x$+HjcLu+F;=el)AjOFjbiz~h3Pw5 z`5eQa_dVYl$XeXxn7OxQ3c_lCEK;6 zmG;g{SkAoU*2GRL(f!(s?EJ)rO3} z>h7eEe4B(k4^BARv03+Yn$tH=v!xlCWq0Q`Et1>w#X+DrcCw&N;K2upp_lsgPU&y_ zw7e$khV1GCk9&SC{N1zL!R$^f*P^D08^vp{FHN~`YVlrh=jOK_LeKwfXnSMmYsCLD z%EIXUz3qAXofj0AojDWGs4ly!%xe30{m=``Cx~`T>SI+sA=BtCde%L8C&M|(g4!q! z`GX6CXPMsH?6==6ncKkr+X-vQVrQ!r#hkZ)yh)sP*j^$tN_QJmGnuS>HLbELnmTliYtM|3mv98;AMxOLX2ncHdE ztLfL!sb-yIUqT>8>+e z^n`uqf#B-*7b}*#9lRj>$gfn}e{y#y%eu7}mwG#sZd2J@Y%|!jt8C-4mcMt ztlOx?W}O(yY2@=@<#g`bIlF8A9L`j%n=(Co)5OEe%(KrhpWihv_g>7rLMH$J!t2k> z=={jGdEMIY+~IQ@q>E*8KEF9^{I=l+!t#WN)_BH6=lwCb# z`Hl)jmU$LjHmf8~b=rj=`5?C{W7e0db&r2*1?{o;7bti}xc2AbTdRsL#4ecfLCVHc zKV+p>+w9p-bPG?{-CeBJ)_=^*^61v`IjyNb?S4gIPI zT2Icc`}UjXX5ho(*e7?^c?JI6oU=y!c=o3|%etzz|6#hTydtsx+10yWtd{*`VmcqT zSZn`9mE;NWX61)h38l3Vu|3z9Ov`?%Qx=5>LTT0wDyo;+wo9Gd!LZAHjY}$ zK07_nbjwpc_*?U`a>b4VE4vonx?=5W;uqxdIoSH^I;JM4y|cXu&n|Vp5ui___=Cb;x9Icy^Ofrr}2X3GLg?lMqbNNb9t(t>NEQD`PSR^XorCeV=qubrE z$G`c+nv&8m`*79b4O;ErRtVdl-Erk;*R{8TSA(pxLf&6p;4J&j`s4nzy5)b@h?i+) zhL_8f`0He|aoG6A8NdIXe`b+^5={ZtAuMndu6+J&O4C|K8n`RbpjI+S5wZ_kaKS z$8O)}OSiw*-T&Rot^VKP;qq(hCIJTZ8shqELZ{q+x-Zmq_tU@q?=Q`c%jimQH9EX_ zNlL`UGiM!d-sk`IP1J4SlZ_gy1U@tb-Da%F$dRl#X8Zi;O^@&&6Xtxo!`b(C#r%l6 z1CKTA<~0X&AGX{^xk{b&#n`cMUt!y=OsmDOPpS&F!^2~^Ay*` z(>BhK=AK{4dSZJlOZl@))}OVi|F2P+wb|;Jqgi(KzdsKemTMnaysK!xjIU+3+fg@N z`_xZD90!5Hk{=LV^ZHrIN7wJ7SK96#m!0_Y&1U^t-tv7v9%Zw;u_iii`9A;h z+Dr9$+S?!96si3A=wu_KjsN`p{ukehxYlOx3(jA(>DkQV)$MNzFRTx5teLw!<;TYZ zQ5^*bF5lfDe0#Is#zP;@NbnYIn_jqdQlw`@c~89HqWt9xmTgc=i{0`?KyfL*l8?>x zH9JyT1NUB>uaGQt<@sB#Xt53Re{R*9Y2Iw2;wI~-+pFHyxIJ)nn*pnh#x^$B2UNMxdV%MfQ+3yejT=@9;2L5%7mj2V${JZz>glwj;PH3c_!*S~w+=WlrypNvftNy5eON-;!t(ogh?wwWTGFlXQT0~9FFH^U|W$)v)t9#b$ zyl`FrKzdz#VzybC?Y+K@s@|&trPX&mcT3|cn%J24o9n^E4KwBQbKLc#gO~aIcACY= zAk9=6U*+&I`S>KZng72$c>Vj{_iH&USH7@(TM~I%RpR*igtM{h6lc6`-x-|YV-Q!dat|xN zk-T1-Eu5)wR)=Jha*OU72BxbWqaP#l@>c*+JYik@%%Wc~CJ;}7@u_jx5 zrPsy(dtRHy*~~0uEW4`3{-AJ5Y_G*N9%qLYv#;&fe$h3hKA}!5Mf}>Yq}Q?D`3IYm z^BA&xuO8lTd`-%=Lsjw&a;p;N-28ZL?iBlD&JD>QO4dl~Y;Q?@lH&HeA@J2{+xyq6 zszvYawqva1eBwAk+HqZ&%4?$upA*;4Ul^cmbkd5yCCz+~>=S*tA3qQMmOq+i_2$j5 z_8#la`hFc5+?C(mP_LiT9PU)8tGnDU3TpF@6vz8>AmXl)usPu{k*sR-utzm zH&z#mnXGL#Ef98mc2DDBQ=O#fvt^;JdQp)NHeZ`Ok6~w@vGAAsmlZT`ZoV_o^1(w@ z3o#qneCsNe=`khq-!uCAuc?sR@jo+uTjHNQ_g{6&dmrBT%N_P-{?3H?K645ZE(ib2 zJ-~4O%UyAi=dapzCcnSp$}FdtzBEDZ#oo%SbiSJ}xnyi^f8zfcci~V^c}LyNzy_6M z(-W;mMt-fk{<|D|Cd3-~IrQW#*7)2Xb0>$K+{1M12g8YI>3|uxMgFH{Y1%$GD8?$W zj+@Pp{lM48c}+9tZ(J$F#j*Dv&#|(DeActJHXq+9=I*i4Ynz(M&9kqXlg~bE3-mQz zl@qXfnY5Nn!}M0Q`v-T}GhaG$K#Z?~i^axh^54!0Aw2K;zcW=$ztyyS^$olKjeB45 zt&&)MV#fNUY@?XV5*dpfUhD}LIjPi@;oX&SEM}wY^Xd~h3^zTMzq$O_ZCT*!tbS)A zd-~?jrL~OJdEfpiMjDlcCvmesQQ%p8;yc@e#0Hg`xXtW^+H+02x1Kxkd*(*AW6wn= zuqWzA@fJUm`SH&y(kS!IGUbOnn|SugJ-oZV>AofV6R976*&pP`w>`fcUUWSy>a`Y! zzf{w@Rabj+tpdL;Arfm9ATb`GSsr zn7b>Tm2b9UQ;h3!zq!f^Iad#VKbOK+^R%ve*1Km3|25A(W9@qUSYi7iQ`-oAc1xd$ zJ)K=E`V~En3ten?6F%F#>8-TspA%BPTV9)9yd%n&`eFAo!w=?P)_!+Z&^O3`^O?Wy zr5$_4il=s{3SOUKYdRUsEn_wP|C7GS zg?d|RSx@-P&=+?F zqR+GK&*LtU;&A(1vYaz14}VqhAv$g`S+%D|xuZygXx47pKUZ2a2a< zzPr`EdUEu8gL~*%-LBXKf7_o?3PC;bbuaU|@`Shk_!wOIckB0BUfDAF$!GpNzhzs! z=1+b3^B-sKocwID<7J~wRn-QD^Oqzycjdm8tXclf@PXi_w8Jct%nzzIzS-xRyM95m z-*RRNjqd!07*+3Soi4>DU1#qL?Ykd;cx7EkW3Y=YlTaJ;R)NWr?HOc0MtEIt_!v4d zmYaQZorYEVg0@mgk#C1z-1ho$Uw_r!`K@XN4a*P59xe7^3Wug+)svgKaB5?Bw5+p8EU0t;F}4Ua{)iB^TbQ_6T6) z*T^wF^OaGdyn5Z7rDtT0YwS9eG9^um z!~gT~IlKP+lY1iJYE~W`Qfw`;{-2gnR_3m3r!yRiWfF_02uM#9SyB}&VgG5zgrNMk z7q)JmjCJj7P5l`j?H>)Y7RLWRo_sPl^yCfBDgQ(j_$%}(r?T0+Nw4F|{Udt9kJE5! zYn3#2n4HOh!`Gg_H<*`xAi_svsq5`an^|kO9jaV>t#Y4$zzuDO7ZR7A2><2Gy61Du zS6O7v9G36yT9Zn*e;4Zd+hb~QEa<=cl&=2Mej4UyB#gKv$l(s;r;q~Q*PFl7`L0Vo=Q$<;r<}A|G@>NTH zD1D{%nbV>(rJE+Lm=qhWb%443lgGJRhi{$Uye(sWLcx`qA3xY7X732LKPfclhvyBA z-*!R~2bv?kD=2e*3XwbFGkw9l*R~yB!hh^p%5|yz>GkXb9m-A`vwA8%7+q4FudQubRw;bE|i?NI&o|2yS+XMPm7ovzgE^d+?~9>Ii0De-!t}U($Wa?mK&uT z*1zHWZ|}O+D>P;ALNR%n-P?6eZ(5ghWz*J`k0UiBuQF78VlMm7eC^2252bBxCAqI- z`(J-7cFCI~6zH;9?%Kl&3u}|+In%cv^P8}7Z>Z9(H>^e5S?WGLIr_iBeN&lc*}~et zU$@U(Fy&|QIgJ&*QX7r_*ZmK^f93Z=_W55`a~ha4p6v_0Y$nOHr|RUrnRBHpUVUlW zlrdLQ=Rk2-mfOT!({~T7OBJVa?pi(L=$-AwEh;OE0v30DJ9nYqKm3QTR&DmLZRxja z9^A+{?Xc}#a;mY=@xCDc`^$E631}!NB+guR_LxcOHpPZV`bU!2o(mThQM?-3r6kg} z;=^|rpHiz0kKUejEU=i(xapSVkDZN%+B4a9CpEact&_CWkV%v=c8xUZZ#rMQ{7`%8 zH)Ht;fkw-&%r;oJa?*@F!a3Z$Uwsq(6D1kCBDfzvyA-z2^5VSbn=0&1lo^%j^c?bP zWDK-?@KWf*nu15!IX7aLZ-1u$cg|tQo5c_K`qZ*zRx?JFOKm&Ex#oL@F8}>+2G3vr zW1lDOrJ;M}72ma1=GrVn&$2D^ue`i{;QNBP&x$L5M~7vcnU|cToSyb@<>&1YneV3V zZ9Vp8V&_|qw&%UU>Xxj(H>MhH?APLCdhx67DyN?Q%*R$&Yl|b2m00uKbhoB4b_R&s zOC9I@cG^Jj;ZNnj16h~9d~7P_Kd<~lR!i{L?=O|A*2ziN6D+T;^fyg0KTx;ZtMvFY z@A)aut=R>OrCoFuI&M)8O*y#v+UaTKv&6Pq?{qmP>#e={TJ^GB2~&3!B`H6O_;|s% z-K9RSaYVE&|YR-x~C66DoZhtHO{O;4)RhKQ?>sqYM<;?e9cQ_D#Ztf&WS#ygcH#=+u4Gt9l ze8?^)o+3HB|MT2k1qYn@7p^OH65?B8rSa)lVBD{}MRKP;CEMT8^cFjt7R+z7V)d{7 zDbp>kacx>+9WR>sIE>pwSAWa-!o-Uw7U}Uvve@x^x3TOum;d+vv1p%duqC(Ln}6$B zG&+O!20BmJI9a@T(WD3cbG!8&tDkEb7E6bg+;v~!>iqtI?%|DX3vVW0i4Z;`I#=ud z?&g@2XPUhokhkylS=qv+ zs;d@+E|fK!t;usYZQ1UAMQsuFGL6Qsakrk=TgX4Cjut*Si!Hu>`tx_+1vS>YYiL+b zxoKQ2DiL(RIl0ENaNn!htdY4(uenQYJ+b;i+-*a>P1@U6otW_|MWCltc8$z0j%UA) z1-BpGeD$vPhL;?@kuLwIMV+~JT>Sro@6rL;F3#V|q>R2bY%O-3w>rTx`OwB{Ya8aZ zl}6jPFS&Vc#n;ZRHw6rg{?+qRZ^(i0PA#)zC3{??Hu*T5QmSKks>&c{bHht)^NozOURg+n$1lPtL4Uo-Feu$KhYew#-;bH!=Iy1GpnEqS0SLmGFyHfdVlNa-sAd9B6bsyXAFq(!Oq#uCouDP&!v_*{eHS zQyk}hRbZU=r{V7X!*l9=rk(lOHRp~JyIi7NQr5)K9lt|-=1NXF$#kK4a#OZs&eg0* zGf$g|=gO_<+;vAm_eN5`?lO*ow+wQ1ub!w%h}qmY>2~R{YxxJp)kcomzuy(t9DG~- z{O$iAi_V4r{goKc8uIV6<{GsAv9}J-x=jS%+*=mwo(Dgrrif^>YxmyrdZ&Jn)rO6^qUW-MSL(v4-TRd#&o}8z zOg{OK>zCmg-Nnmf8aFP!eU5+P3_+(IJJYtzDY>l^wfW)Z|16~z@*<(G6UALbrZknV zx4!iHclO?!j1_w-jblpJq**!6Es^|wY0c`#sS!qt)~Uv5&Dr`uM1@`C-Iyz`Ts8`tMc&$lc*xbTW^yYpO|N~45^=jQga!*Zhk9|T25 zx%uTOZan(OP8?c0_sW8&#*@?UU#VM^kUZ6?MD_fx6P<|xeg~MniX&9R-Yt-_icnaf zb7=EBHP-MewkzK>9Dm90pBe6^Bh-=dR?qudmVTY%uBOJe&42wD-VXfuE5ttRV8Bve z?j+6TBkCdp|EubO$#=f>*ZPAOce!J7+6ijEo}5|-9BXTcBNO_ zvOgZ5n-1s+tvH?MInR25k9BeDjL7Y$3gfuG7d2k~^Kofm{W+G-=ZCwz7~~?i9S&4| zZs*g{`C#YgWYgznd-wT7Zi_hZS@TNbMy1)Z>#Dgve!Ep=%+YXMI{S5O9I` zB&~K6tu}vQSa@fL)o!z*1Bbmo=u}Mj*qtPH_tVnpEPNmC)IN{r%9}saW_t98hBrHt zi=Xuge?Pru_v$?b4ZCvA-kZHVFunDg=LClHkJEj0&AP9#*DTj#`S2u(L&L(g*}kv! z-|j|Bk+r&yZ(r^fIeB1q=+J_SRvB?2pU1Ojh~D&H#qwcJ z^dX60%L>InZMn~4ky+C|y>epSb*WCoZPBG8FEfs;d&*a4yzcLU_>8c1lkCd(h+H&k zJl{Vv+QELtywg$DGY`00<*$obD6sk9OKD5jW|i;#i=S=Ub@9mFii(>?YP&9M)Nx)V zZDXf*^Q(cnU-Pbg@yF8rt&P+7H`S(^o{*D{3btdhc{}&+{+9RU zM;?3^-V;_Ci2zaKfMbnmoVwLaRa!Q-1_v?=~Q^_{J-}7lzEbp z_F;jSdUK5yO=X&1*Y$dzipispsdnXeHcm*s!h2>(WQG3u(-Kql7N?%ljo1|8dX6zx z<>tPiW5lP*U zYbtY_#p5boihc07cJ`>wp2^{H^Es0ww_omRy8Yvm_x~4@^`feKGSB_F=+0OhzrRMG z-QsKMfzKDe3Ny1-%-dfuKk4=_)B7vUV@#6DWsYTd#ojpX-Vn3G!=&5GbLtHf;;t*pM&{~)c=@}X)Q^udt1)fx4iHvNL8PBq&+sJYW19I zIivNyC%z?I7dl+aZEF=6sQ_w~88#fZ;$5<~_2L4h_ zZ8Hwd`zyICCZC$|NPCjvVa6AN6MQ=FFWJO?sb2erk*MX?-k|$ZVG|~^YP{KAw@6HJ zLBrJ}qPA5cR=amPS8X=iSY z6prj27Usf{n=*ac1g6bCv{=Y9G34&0#swedPkJsjZDl~TK{NHZb9Nc#Ke#H$TsS_2Vo6FbeAC@}B z^t1l-q^$vKqD>6?G;bIkUUFnz=20<~%cZHDoKMo$P1?9~JNwaV(>XtEsJ^#0>x#_6 z31W`d?ri1z-o($yzq{;y)AOX&LdNbVR9MgFpFUY})W2cRhfRV9l)aB7AJsX~t-niu zLc{YJi`QxF=(Ue8b)MgU(Adk=X!=?UgU^5GK5b?0pJ!)koS0+GZhgjrh4WoN<;z8j z82!~mqC}tV{_3{I=tvmbhZok)`}dxeIpmeSS9#|BhCA%1ro5`D`~O&T!H2gq5-tAp zd+(U(%yDTuOKXndJ_oqXFd{aM(% zL^cKWI$oK7p*dVGeq4n&C6*)>bNzW=UBj?xQ`gJ#u9UkBMiU!__63A7aaX7ubM-dz zXZX2?=bh8f|KIBs);9!n|4_Y9&l`KKbagW;@5`;LUY`nG?cug|dZYFtFELSd1B0%} zL*JamerEAcO60_?q%HA2$^8zImv!{aderQ0*(B^!aa6 zQj*qGlzo__xM$||bwTNG7k^^Ek-tn+Wvk`p|1Gsy2PaRx?a4DQ-1*D~t=SXVwt5D* zHt$aRc<8j7ZpDU=DTdq=n%Whnb?fa}{=m%atbKR-#ZM{!rtF+q9~WAcJ@0(}oEer` z*L?U&cX zh39$e{Hu$8DLgmu_#&S&_e1P_MO&XFT{#-CFc8oQX}LNCR-{+z)5qT=tA zUCv_ymoN#hp=Qd?+))2t6f=Oe);&) zmLs`yV>Wjf`UcNtJ6ag*GAYOL;LaJb1@CvK^RwCrWi8$KDJxO$OMCUKtM8W?9c@a! z8}xr!!fTG!h{Ht%XUZGe|MpCGNN{*x^mt0*;C%;tI&r#)#rm*w-+ADb1yt__(UJ4?dJ=czo+qVfJb5WmghZkSk8U3e0ysnvyFay z%pO*LC*A0ecdMMca`aU5n{Vyv?OJ;@&Sz@Sudn5MIHzp<>8&$cDO%mhCw_lu@C3oc zb*U5Lt0fPdlC0RisG*%rK5bok(;wCMy!pG%JZKI*lKJ4d$l`=AH%c}2UduMLcbN5> ze#sCwtk>Guo<4P2c}L3Is~gKw@0zb;JIHDirdyobWHWvFm1)&}o0I)6ue`tB>)H8` zl(V57wqo;gqxogZLLTS{L~#4Mnrv_T&FYsk<&y)~xBB%#Wmc!|8YiEerSMfT?9DBi z9P@CC1Kam-3fcv1sGR8Jqsv}#rlX&Mf3ncsO1-(O6s%80NEe=#jjPa<_;A5|qpQvl z>Gl6>6&TNdKEF4#K>G2f*pEI@bxbxPAvtO1wxp+AyLWZ>$AovWrn<&f=aS!+sBiu@ zPu$|b=ToY)Qx0x6y}#CcNzcjE<+xnJ8^VfgaUd`}LY(~8o`--D% z>^8GGDpFFO2B~^q^Vw0~vQbU*`i|}S32)bYTyx?@+-HNi%nITSGRr;M_=*(P{N|jM zVcx(K{^~4m)cRwI`@hEQk$FAOLSygaa{Glc4eIPnNolrsF8{F3pAzvi{rSD^8|?YsZHcI3 zuz9)XGdJtM-`X!;YzUW^D{mK%tI9a|xyZ2l&>Y{jdD_eFrx$ zWH=Ll*J@#S(8AyZmxF@rmUp`iYyBQDoL@bgp~8(f^H$IMYtnW(*^d^zxqZd6pnmQ2 z+(jY{?SJE6JM8>ntkLj0%}H%-`vVQ>NiTof1$RE@YM#`zXv;ON8*)!Iqo4LT2PcZJ zU2;Leyj>)TbCpTGfQASAR7cO(o&p!DIkrhXE?SjV;v@d*b+oflkMzxkPt)dBuVG^F z;$~XN_2PNYr1+OtUYtL9!bdPAPkep5rG$4Y$JsScgpZo+Xkfp_yXfO|pQ^q6Z;l=O zxvfde=C$qX=B?6^{{P#)eiu1mYtX(vMD=yagvAej`)*3{Yqi|ZU1QIvl&N$^@`~*b z$$fu)tjbhxTW365uk#>N!r1)DmD9Zszg9|KzEsbW{Y#{EMqy-5pGDrmBWou&@z&d~ zpZN0a{;C_ZR~w{H{>o)Et?I-2Df*ID>*I6w@fuCNrY@Mvx6*?Bk~U9;RkMam@}HT> z+&7sk9=1+j$G~5vk=ZZ)`0e(;h7%f|ALifxBG5Yi=k47xA4-(leP>Po`Q_N7;PT4{ z&K^H&-NQVG>B$G@bXL9(d&(j|HY{J57-qjOI$z+!6@%-W9qZ2Txb8;3+mhYdN=ph!Xu-(vGylG~H&bb{{dVx7?tGSNdD-0@MFC-miQLHbQ-S;Cumc@jP z|D5Jh7qjh+zg2r~Km5fNnYZzP$?JVrpD`XUWz{}4N%?+-ATDY0v`e_Hdv(^dAR*Qbn@K}CoA_My$12s2~X=@t`Y8g)KP3?2NGL`$_FFtL< zoVWIi_Fa9+Et9+Mpmc+Ekx%(l^`z?>f!-M%FP$IO-wAjq^5NyV3Wgk*?BK_?e~vVdQO294-)CB0eRuh}6?e#ONr5S?j=8Z{V@q43e%#^yeYD>+ zr8obIST1|AS-`sM-bucZ`g)5JEW{VYZu*vbXY;kG8Rt{}E#J!aBuO$dTJy+~-(|i5 z%U_W%aemv8nZ|9j@uzTIZj5cFug(#m;V9 zei+Sj31B=gvGHV}kLZ-G8YXUszH(QJ31-fBo>yXiAlIq^NeZ+97Lnh{c z^W%FiAvd~BbG|gcwp7@k{H-S_Qf8as|Nk=Ugr-aJ;#x!r|$>)7;{I@4PcF?SE$d)oqI5$ydh@%=_%%HM3RutjrVrsT*(l z7vFcfrgefRZdd%1T~aL&q@+KLF38oC%W@qP! zc^sI0cE!@9!)JL~C!2oYxyAZUKV3J+i?<>qC1$DTuXUU&Err}YXI<1jqgzq&aOp3d zihogOl45GlDmS)Rzbv`sSs#~SE&uzkx6Fqx3a!5$r~AAOdstupb;+4?yjJXHt|!{B zzm>E&@cByRZ;{*g^ndv66-?#1{dZ>bjrCp`YbU&$a+GDJJ^Rb|wRsAR{j*qY-hE|M z*q=IOsvUF1r$-iVpB?@2V;9dlCh6Mb^&Z!x6^?5yvYikmtEu-ua2Jz}mfzJkcP6yj z9#hem3tic~|C>u>B~zzZ-immQBaSEKma4IaUaH-p*Th{BlGEyV=fLhGay-|yZB9=s zVO5M*eRjtxpe1wLN^6I-MG;xvx70Z=iaqk$l))sV^U<@;*k#W7 zx61X}Ov+mr! zQk7l3Jip^5gIw+o>;JCjbtiOOxvMD3mphH?=%Qz73w~^^y4m5ivf`V=I&u8K!juD0H2;wr8>bK!97jybDZe_mRl@OT&3?w3<9?^3UySO0eI z+w*KQFMa$Yu`MLPmHArGxh9{fX7iF_K8yTs%h)V1-DhT6P)dW#%4;{H)Y8sqsl2vM zm?Cn=_KZN&FP`OkQIj$_lRwsE9G(*JcSA~jq>;Gcfi+JWj-Q;@6O=TsSB3ufgNXceR%nMg%YM`4<@c z-~pegl=+GkX33wK_XX#(d^xeM``U`cv$HEBExdRuZY9LXF-LNo3fASW*um;=pj)xx zYI8cHvdOBoTMPDgKVKKNGU4>k`9F0u7TYh%?4-Naf%d~#yw73qin_82kG6D?glOGcvc z$`pqVMV6aQ?oGA}7^NrrJUjL#&TKi4Ved6#z5M=W6Tv6@G*0c>Q+VJow|*!~#e;(C ze8cXKYmD=_RUTBn-|Js>z(ky1? ztjM!YtlF4c*ZuyA^u|LIc-$WbdUmm$*t^rELH(@Ei)&hwDpxy+vYxcfxVXOH1y{`d z!UK=D-rdX=TzKkdjm|mQ1q!c&CiqAOeG?Pf{VaaL68V)1u}fwa$1GJ&uVnC==rBR@ z;soAP^DG&bxR%{5I<>GzWKy36bJxqv2(z}HDR)Emv&?1^dn>?hvy{=sWVg^_fp^@W zR*3iP(0#D6qAJUFTK?J3<+E}Fm3JizhF{+iqF^E|n(WK={qA>dGuZ=+FP!PTA;VoH z$erM|#lK{S&dNzQWvo)wm+M7r&EBx4y@|IXNb&6L^1GGUKJ8`yw#PWGcKpYb)Xekc zTKJLE?~a0N1HlJ77tT1k@HvX7;DcU*+kPZ}O@<Xw?N9>ABYvdr6Dt-A4{_s#il?)~E7Si_qv z04~?|Hay>yB=*?QJyy`;z~>k5<8*fP?*9AYzgFD+*J1WjCzy}v^`1O4>-I70u77)K zpA`md<}5n+sF|6wLeKusfu!4iH14n5&1#dU5E$5B@%`|^{qHma*L3(H~3NI4Av_sU*3U({eV`?XoG> zo+a;vZ*-U}3Y+D&m$B~udC_h6JfAm)y?VFp_>07DhsaHH7d(-3dEBr(MK5aYxjhGp zHs9tmRIQTSdM(`G!8Z8>|80NgubfmUWMls%kyGDe=X6I+uOR7L#)seSxu9HNJ1;F! zOZmaV>d!qHi!80hf2D2?WWC;`zFoc|chSLz_s`wleMyzY#zerD*X0~ICFwpmc>$D`YR`t1uYj}<61KHBr;l8$mtAnUVtMhtSV7L~5mnpv%tbk4P#dENtNxqxpi ziXV?UEw0+JpySxI1s`6XW3(<`u)Zbe%(=(KkB;0q$ZRur&2f{QHen6Z5Z7r3H9V&J z_4*zEkt-Q#&$_aBs_Jg$pn?OzRtGkhzbFxJNj6vg@glc>!A4iUhW+&h{4!g<8y+~k z?fZO%)hai8Pbw=bJ57-55}Y;nRKXMWYicLn`is6aS^GUs;rjE`om#0pa#>T$6GWNh z%(@@4ObFl2V7T;D(2w#7$s#Aq82hI$oxm&?SFuv*0JHR0&!alWT0fuTQF&nb^ocZg z-s8^W@xdQ|-7C5*+-Mu|N2k)`*|o>Ze=oCB*RwI1@b|~Fr`4=NALgXV1?9|M|IE7d zdMle#iHLzdH`g8I7wfA}F;rY@465Zn(WvtvP(`j;{e@nXvTLlvk#`n1j;kiiTzNm0 zH|2$<_=A&m@BS9-nC~MpS<-y<{Ely5MFg|WC(O3d2<4Lq^k{hD&bjHB@)W5R71#J# zZZ^#qy`05hBfDRJY0H}Ei7ip)?fc^yJ{8@QSo=O%%vR?{RjAss)~Q#*WNY1e<`k_r zDd((su$sN%jPxmviWfB&0hTA;FxYe@vHgp@$NHdhX-&iN8r~-ou3Oj)&liDfvRMrM zJ!hXD)j9U}yRG4bhUHiHEqGuSSNrs^==%4dzWxUGu4Xlz6EfEC4dSmnz2N!Tc>Wy& zL*uB+%+r6YnVD#U>c>dhQ)`e@5-fMnWx)aY?`e#eiftJ5}wx_1d^VwiIr6TgSNDlu5)w^4- zIk*Yc{i}I09zEp2+k5#!IWa~Y3b)Xo3@MNb?IWv|$9 zl8t3P%cr!PF~fdchRx!i}oKTN2k0D_Sf0b3tD=5 z^mX<3x%HfH8k2Hj&g;y0)c?c!``V(rbHdEnPj@{(@R6H|vwVe&X?9v+gS%1vF;Hn- zp>jL8pp<{pjl&Dh=jk&)h~)SBa@izU@S{lB&Cj==H>lreIR29T{8P_PwUt8KSv3y( zE;wQvdT`3iOtu>rLay%*PhK|7`pxZke~w=ZGcxjjV@A^{`&Ft6q7rs~#P@Hk*f?C#AsUHfbH>|Tt`f%i>so8g?pW&T~ zUhEGV|9-NZu(aTs)Rfw3r)}6kJr~Ibj@OE~R( zF!(d2Z)T6Y?KuB#7~6r%eC!9VJNBuZc=KGAD?5d+f{DLE%Ei`YNys%LH;HYBHzq1= z)hICgQOfx|)IXNx|5~qSAudJjHK92xTdm&3C2?M1Uvw5cKzIIJ$YqywmN+s8cC&!V#+%HmiZw(^`6`(54jzs;w&Szbwk>T6k3A%3W_!Ic+puxx^tbERr{v6Q*tk+- z!J5YsbIY#gqLMw-IHvnR`t{W9#h0d+tTL z9pmIOihFlxS8D9k_YFva3E*7KCtY*@th{-~4q`Gr!hCVMTj&IS}!zr7{&HQimHBGhO; zgXgM6rUzu>jTe2A0QFR28`R~pCBxn{)%@N1otd>lZ`X%;O}ERgNpAnS=p4WO7a4!i zeVvW-J|ATEy>C~%{%7Ng|H{jKd37GlG}yLtM^5@AQ&6)pFv{h0;1Ox*Nz4n*=jj+e zI4NiLlJSd^;kFg>ip!Y2OcoV$&wIsW^Y_@X^h&SxLIsUO=bs$aS}F42r{A$>$^r9d z_JlevR*K}W_`GS-#_}6g|72fT9&FsWNN$p(O5fXC6r}L*+^*f%#M4(*-^}k$;ufdv$BxGmXX@D{=#7Sv~;Om|h0LIXC?d{Czp^Y1lHSgVSzqT<~&I z{V|4lFAM&!=Sp|UI{!BMSh&LV<__6YbEcg3Il}0l%IL2?;U(+p?`y)!8h%H9kSwog zJgRMaU0d|XpJK_#aDkNm(~caggX>pYnD?75>TlTpbLI2C2(JT|Coc7#dSt(tBOO9-T(RWO8P)=WUDeQWbl$zAMGYr7Ki9yPMU&llMxZ zHUEb<{GSBaZRUMiy^X72Lfft#nv+v+p1f3MU%X*G%Y(?{J*N*F{Qovp^|ir--L~J? zl(35CnyvVpB`LXB!0%h|9v>Hz1YJcob{o6R4?Z4K<9QsiI$^Q!nF}2^Iqt_Z71`Fj z<5qZYsIXC>{94Jp?A7y+-o0)3#oo++#q~RN*HX&+r=(ox_3D1fbTR7C$B0wRBFAce z?9M;MRYk=Gv#)!a&P+kyMF~e&wc6I(Pq6;P;IoqP{Lfh# z8N6TLeQslti!$c}4FJ4i=r43P#(#j`&1Af$aUkOwtn?VMn01V4aYSrjvs#M7ytmfcYU$gc%j|(%FQ0X=gIy#5M&n$(gOY_FKWez3@iUj*sP8Qxwt)N*Mbcj03Wia@$P> z)=fXH=b`h}D*AYesU+(tCGNQnhsXdJ`tjikQ=e5$QKtQ@>vN!&p#1@y)rPKAnyaZrzbF2@^27FPW~H z$6@5xX7S)*>vSy^n@uUQv1gi=Tu|Y7deJ>jV~6kc_}?qmzxwyaIKL<%c*z^PU6!9; zsmAWkzPq<5>Y`)A>8Wx5_M7{2=sei@ZpPp4+~4s(^#(J4{S>>Cw<{8yUSkv`mcdsWdp5U*TRA^-4m} zkvHhKp@#hPMkAXjw_ObZ#-VOISp6Ls`n!tP*{#aCTBrB#ZGQVU`E`QD)xKt%Y9F(jUbF6Hl$Aan zySZ+%T3jyIp`XTIMT{d)B- z`~QD#%UsK8>XDvV^621(4-fx^UY2Qi^t?*mCb2HzA>Y(^9lI4fws~K>TQgnHK`=eW z;XxJOiT9uH9h+vfnzMKBhCR*?1;09IZr(J#@r?D6bxBj^l%}#U?_;wmoWk-{@2Sj+ zU}o%p5t<*x3By~iJ>*B4*wOR3`y;<%7-Fu>CB z>3+u*ru9196;s~S6fpG9PRy77l3aC{qi8exR?Hn8L zCLEpg^nH4t`SFm8T*^n6Tw6TR<&4alOO}t%^#^@Rw0d8ExwdA%&Vz}C>IKKaBMZV0 z0#&O+Hg7B~|d;N_IZ|bekogV*7`u^4bOU3^mbUdqe>dRg6lzqFdZf&c4 z-sL${R(RL@*EK%ie%M`Bov z7HzbX?Ch>OQhbr?$n6jzUpF^1Ipq~M?#bLLFuc6%{co!WANelLdo+7h=T-f4QJKAk zF%~g9uE(*p);pi_+ReX4>Bf}9{Hubro}JO1vSgjzLbk%f2j4{GnxD5Xb-c3BUnX1b zmCzOcV{W>6F&A|Abn{tlYU|KdHc9Dk-8Y--b){nK!#-A-vr|+o-|o7-`PPrc(-*#* zA2<89`J-hauj=9-U-eD?W$WZ|t@e15j_n(XJHh2!%TKYWU*hvQbj(&kxcI9tTHijCO#CHA{c$~?|Zkzy_v zZYU@xpSFCklIcLO&%uqZO>J)sz9b$Aop&~htNp<)`x6Zpdrxnh@H9l$@tMU9WATPH zE${C$?EgGxQuomyy#te_qu-=lu98dPSsZdbg|YvQ=6~H&D+I38O7Ad#@P6K_W2@GR zd{$no^q70X(k*h<@6CjtWo|MHXlQ@&cS_awoxHUc{^!=NFPVAg&WpOQ*%R2r9)YLM zR|Xw;oO0>VGj>_NNALLjusuG-&0I%oh3xkG-`wN3J^Xt8-t&fK;wjq$Po60j z_u=!Y{a^ZZi39Vz%-y;g>)zhTdAPfT@znMi)w8`8tz0wGHmr6QJl zulODxkHq{vE3Z6oIsbisg^p!}a4FO4x;fJ?o@cR?exi7OlXCDQTU+zxcYPMUcidf; zJ7w}cy9C2`)|TA9KF#ZuYs-K9y>IY&@f64Nx9_O%oco%kbgOrT-BtgiRrWJ=78*o- z__>+=)%JJ~H8*=9e>V=p*9+&LpIm%?PLN&o4WH)qnv)7;JZz2EC4#HwF2?>Y+4zTY z`lnL6^*`x8u>5sGo9i6gwR`ivrAptcdd_5v*E^rO&fM4o@P71SZ?>_{_@lSwusjY%2RV>R|r%ytaVSmB9t=8T0!X1 z`ZZA^A6%Svr>=Kz4p4oyTWG&6?;2Kr_x(q6me(b#%lNDntUK5+cbgqo#f=%h_Q6k< ztkAk*x;ipo!)lgnjiZeI@|zxH-dJe4%JT8>RA?V%~HSLd+yYi~HWQXp`BX!6Nf z8}Bwa2wiPI^!{wi{+%LEMD+`bHJJ}=H!l7qEahMoEzTRA=$TiPIAyxmZaMw(Z$DVF ziGp0m%WXmE;9$7fBZ~VECakIx;xKW@mOEY^m`QtleARSg3cx897tQu5^9oh z>lXhE=LpUF-_oY`MB2R{zj!}fIJ`P!yTO~d=ys!d;lzM{;n@PEv) zV+&=)+-`6sP7;Y(@c8JXgPQ5~60KqZrB_HeY?GN=E}+SK>>fmUAyms@n53^SRZ`Zb#q~`Djr<}Ksjuu2r ze!l7Y^rZJ|zBl>WGZxoAzq6-k^T~?}ryX_fG%dGDeZSX_PjG9fLf)HA1|IHf0QOIvwXdd)68K3p6Nu z$gwG@A$#J~l50~s#32)LycH967H{O9yzJArV9S>+S6Q##zI6WdlFOO#%yL|-Exx?4 z7E*V6_Ib7%i`0iZ+q0rJzWpzMe<8QFiEGX`12O51Gvc}x%qzRnE$?w{O1tlW;)E50 za`k3Div}IXW83t9t*Uzw!{|PH?%{_k+LlH;)SlOFzU_Q9;zP!fG9%~AEF%-6eHGpT zA11|lJ@t`iwpmm3d70Ks^{hpa`jekcSY&c}dWv9x5%cclUtAAN4)94dFY$R^r|{Hw zr*8L7OX0h9jm7hMuk|iAiQU~fM<*t4+u9(FTRIPZ%J=--bdCOPC)ct5Yx;5Tt{&VXReun-ha>qJmENb7VXu=#R z+03@N-LPs$JqzD@VXG-(Iajo9bZHd5)782&eee7^Z%jAxo$k_I$oE;~dv1lLLPRLT zyp~5S{Cv$uqGgZ2MlOE%Z++^iPmk3TV`b&8sZ?LDIP*gO=w?=~iU)QwPi&X!GswM~ zX?49j|4{RG`EKX2`EhO#ZF&AkcQ{d`6ph#q5$-j^b&x+6<+0gd~1BftcZ!GEsK5MRIR$|(QTYB-9EE9yiNAG%0*IZ$reeyT2 z&E)q?GiO!1{V(u^q(hF1ITki^#fuJZ`n3G|Po}mDQ{Hl_ZoT-c@BZ5pXV~Ry=5c!6 zOunvc@#lrI(?gF7jOR~1bMDUBU&wxcVKs~Xjl(ydDQtF-(+&1%k=KwaNZ7-~H^I7B zZktKC{>Gb~_6O4zX>zs%#6C^Fo!ifm$nJ5YN#^X%PC-l8-|21B=RVVZw%`<_e`2xj zq`CQsyUat);m|AeX*UeS8G~hb>DaHziIv+7Crp8qYSq`io7-kaxZ!Y6)Yx^l3=a(C>uKIM-|ihidiM|z)Z(x1ZP zJ7Y`JIqk+HlXj*n7;bO)CA-AyS-vgz?UK0p(|65(@wNWp%|FRH>aK!2^&*dKKI62a z6gwRnONtaVd4X{NMOzTlepu&Ka~Ya-YSY7t)_@rB<`c zeOSUbBQ~e&OYslByTYkQlWL!{_TJ!~mo!B$St|QlRr9fq9kEl_7;f>qeZ>8sv1|Dy zj>}gLJ!P_C5y@q}DOfRYry-k^*wqs^tZsOJjD5)y;K*v@WcGJEyYR}{*MwtP|0w z_oeq~_^2o=CELbm9#HnodvM#LEV*4#)0F2`MZF}$`EV0wQK7dP>~AJnEw7h1JyfFm z%SM7BUbALbx$-+l)!t6l&tKLnc(>(We{yWPVwtbgHl6SE^csFwz8T2)}ze- zg?)lG<2h4fN4W$YhTErqtU1|t*qSAB^2AGTdTv@&Ha6_?&^-^%^lMrBKTck>N!V2H z+?K6t*K_=d*uT&Ab9eE39gXB`Pi*_o?)m-h_{;@UT~~5j{CUB=>3QGT#&+wFW$*Vb zh!9`9w5I)C;~}S<_p2Yu)ChTThwcdvb$FS^KPkR?V*_7c%5LQZGbzjY%Ou?a^_E#pmB^>EH=NsB#A%w*8jKo&4b5pxBr_w)B5$6rdQ^_Hv}8a zuluy$N8t2H?e)(YmhhkGpuDw?e^NNni^fhQ;TFlX&Lv_WA*gL~i7 ztQpMTh19O*N6RzVeEDU+Kx5mne$kXab0(-XTH35o5|Ei!KI@W^ z$UOVg3*zHmq{(Y;-SGGSny#Lsg;QjDIAR~06l^~TO0Hj24>)(khQE2V=hb!o>AV$t zzTZ=yIbWta)KM?;OSzuK|GeXu)K@&cb#?z^P`Ks$-!%Ww@O*FfH=SL7i_Y|Ymfgj1 zv?uCV34>g+(uucMp6_G&7+fBQWuz9YkTjOs9(PeQ+1;t^-l}htCq{AyF}M4v zAM4h+nW+2iaDI@xd?%rH;kv=Cd>Kg9*tOHoqF-mwTp#IQ#+FPF21EW^@cr*%@I){ z<1DFgi|csy;CW9XURxA9ioLycW7=tn@Ru|HCf3Qa*zcBma+m9y z^YPPSFWS9@H6*2bpS8CyTzaVOp$NF2zm-+&BKuLDb4$DRv^1am_;6T%4o~FgwE6QI z*#i!SS?k;U_fvcE^n88QROSg<(`ALL($3~SX6CP$XHlGWvHi@$!_jOXWp}729RD;W z@4BmZ=!)MyMJzG~(<|Fgd+ktO`<^GHXT9UQcIS^13l_{{y5IeH*ON5qtnj}xB$*WN zDc)+8GcBCd^n3Ac+c`ovm%Kdheri%FgGPLT##TAj%=4?mcl)n#Y~rq%z}VF*oFY0! ziMe@E3D;z!y_44EFJ-gIJb(0kn&j=nd(6KkhF&@xx_N$x-wYiU8vUf(M{G+nH0N2HZV}@le4#6aN~JG|Ciz{p_G=;jqfwdeasY4eqWsDd#r5HI64F6qD7RuRT@Bf3o#@zL@I4 zzJrGuc$e~tNPEsdwZvz=Texo6!i8^hSZ`hmZeUT>KPQ_>bq@k?n{78OeW>Oh2C$;4N-Y_x#61r=}ghEd*!3Uhw~XNUZh|AI{?o=FWV+ z<-PCHdvE4Pf4=+PQ|s}o#fr5Scl6u$*Y7>_+tKR$q^GG9ibGG%@o`Bm$nOhRva2@^ zy|%LTs@IPm^}Q0YQ)|9$`25&ga%Oo$Px>0YY;gH*7xln#+nyV&zs}wM|FU$$#{+-; z)BP-e9y_y;{r{Ti>1*899zFS_^xe(v)`gX;HU=~tbM*LnO8(reAkl{O&`%SRem~IU ztLV~>ik-YVC+T`9C*PCXRa}>(-e0&9*LXOsywU7_c+7UWUrG;aEDGfKiVQT^tq<0U zxQgfRTe!{a#ItQuo1`5RZ(rx0;>G;;-c#9i_HW!A{fiG=_7+gM=5v=Jlc&97xl6N9 z!kt4At|8I8O0HdfcwNd!g!QFq>7}GuE6;M?Gn^1^#*;Tm9Jl~GbOF=l3Z=S;ne zao658&e+1dMdk5<7uUG&irktdt7f=(wfMn46Ad?yiDxIUou9YkMu>LzE|%&WbNy@< ze`1Z+`?Aw{^S`?e#{-rLa7X?=usKCVFuHBgi%E^rQerDNuH5hTMONhEl?Aybx;4Tl zPTsRwdT{;ycS-j`9^E`#_iVM18K3G2nVv=6cc$A%^cR5EX4Y;INb1!sYrI$WSy;7I z=JmC$srM@W)*erNxt-sBdgZg1=0^E@zdEM9y|v9}tN1+6)m5pW@w4ycee1anOrB%5 z{RB_cwX-rGa+*?GvwZ`YPpPxonw)EJZPxyEpj7RxCV%$sAc2Pq{<59et7W9I!qRV= zPVm$<%nq6NnBN^gkhp6i*TgS#MN?!JBo(jhT4TKThgJRSXA^duYfo6N#n5kkUFJY= z_td3LI@?_yRH>d&50z?Hf1R~4%=Ott_JuF|CMhnOwkp_c72}712iuY#*IZvDD61}D zD9U{F1Ji{5>KA&G@4A1|4Y+n7-9}CMwYZT{RKmUYD?O^1HU3G|naQ*q`l@T-#Vc^J zi(|FL^{d;jDMn22 z_5rSo4?k+wD0BR5(^tH4=K}qVgx}`gIi(>QFCwqI-mg`*mb|h?`axuom`(AD-vvVJ z(laLQHMgl(JM_aZM0NJ5xPn@NGxOC%rQ4l%`;~55t9WDgU5oPAx8k{?x0oY$xF##c zE?wn&dso6ey@J2ss0xiqbKPSuW*fKQxNLE1+w@CQy!H3ZKRiKe>U>rHm0UZVTbEVo zZ;}ryI3p+a*Q}~ozPI5_d^P*C^A2{Oj2ic{x4#T`{P&F2 z``-!6A3O6O?dM*x)x(rwUf-1moSPyVUS}$lS3I8a(J)5)*jeS(Cbs-V4(&@nyT!;_ zJXk65U}dj&8+%0%%RB$Ds{2PX=Wb`OeRgF2q%+SQUi`7=f1)wrr_rWwQvuZ`Qu`XnfAW2b!FX>Yl6@Yb39e*625@fJfa>Rja=CRVknI zs*rhqkJ^PYmhan@TH|)~=+{QAnVT&iycE6gSmr`q<%@G6k=8ByCZ4rUSzcSfk^A}8 z=0a7sPfIqKzkSQd!0p(7@Ud3O39(}WujZw=KB$_}wMIE{U+wB=kMdY#KWs5Bo~Bi? zpJl0yoYL|3kV`(IYO`F`r<{#D&glP>v0o?h%~l83pu1DHfk5@4%#Yd&2In36!2;yzck3uE{2U%PM@DZ~rggyK%Ui`Md4oZ`a;Ue(sn5 zEq%7Xj@#v)tq;Yo-11QVWsv&9C_=o8Lkiib^n=c)J=AvdZAo9n#kFo$@yXT89-KPAa#CrA zQ_jtx=8p%zdOLjFdg$u5a+a%2?Z+ILTEi0FZEw7v|B=CK->2KRe}_ivfAyYq^Yy8n z+c$px!^;}2u6;yqUhpdJ`Oii8a&~_`B(Q4p%gyN>>1V%e;x~S?ba&;uC-&8!<~#m; zJo#H-^``40EAB2%NG@UjBN9AuUXLQU`E|6Rz2n%tqk5OL>%J;1H&oSHYu0}_IJd~} z^uf@_^Gd&;%5;@Cat)L}z<7R{xa4!b3?n1<1Apz}Ufo(3#Qa;Ax76x=Zhp16y?6ej z+ZB8jRh%C(dT!cZ+0FPw@BH4~d%2kOHg}!5D`nR9*60hzFS9*uSMR%^q6`wd-~g0(}w3wN7==1^XFfz zZD*c$PrGx5w9MvphL#M!zCVw9GiwV!qksJ51I%*bY!Ql)Mk}ku_KJlFPGLM5!}hjn zsaMdl^r;qH(&uEheqY!3NZGq=txUGys>1UeOb-HI7r%45;_BTy)5+%A8@7GxqC*yY zJ^QzcTk59e486lKC%3;kx$9kt`cJXWsZ4BVJ|Ea6^=!M+v1x80{ukSS%bhO`y_7Ba zVR!rFQ0?!QFSX8W34dv;RqbzFse7;1`@5u)@Z#eRE#MK5q63eQJal~azAJJGr|Q(7 zuU6+<_G&u*{BT&`NNVGc=Cxia=bluCGMBH=nQgdw#}DfhJEk+wi`(|(W7C7G>%oqB zpP3}Xlb0#;RcHm69MeskZ`j`Y_=d#erIWs`-WuV5yyMz`#R)UJ#1A*I&0RW;OXkB5 zrVkuhXLyb#pLP2C&HHe{{>8%1YV2ayKmT2}K*H$ezF(XZgB}Wgb-%dt!5hzl`=4tW zpDvi7G)qwQX0@TDtmT83cTFe9&X(7{@q_bCa1vjKgl(DfQ&1lK#q?mJ%>SE<#9le` z+nvtz={)tvGW`R4^wblv>TknC8qw6AV? z`^u)B1qUwYO0Il0PwBy-BY~&>Z!ed75E$}W`h1?yhhN)crmns;X|8du`9hCC!HJ7F z!dXQ`0u)Z3JEO+P!f0(45U?hEuh#W4!8cvJzeP88Cmf6m*tlrX3X{yt=l|#4^fZbT zZppoQchCOkmGkfV-nC3mD?4W$&)Ie2{>?8{*~+b5mtU@{c)O^4_Ng=6uB$pP{k!rZ zw(#bcS49C=9yA`j+@l#~q@BM(BJ@c>`SN==c8RR{Iq6jFBdxi4LHVmDRqoOLq_#}2 za8c~uQ%&FYg)HzB30xwUnduTG_)Kw|NYL{c8|$`lSDtRmlb2rg^x~1qQ&&TU+4vb2 zoO=-X*|{-;|H|icdzt-sE^S#L`{Kem)j$2MwMNJ8{Cs(CUtmA0?yM$*ibwq-8edOE z?eIu*Jib7qs&et>3a_h8rwuBWeCg&}$m+J5@9pWi*RF&KzP#K~s~yWb&s59UU*PuD z-z(-V`Zl5cq@w?_dHRlulKwEs+70| zKYU`Z7*D!cVg1)g=C9aPj7#{%3z=d#Aqj{r^1V()P@nJP+06b>9lJ zcSO%N4LCh}&f2G)&$z<2N&J6t?x{fq!-WokGI=< z_XOx4W{;iqdKHh>I-M{3Z#FMBuc$e;(4a!-bi#Ro{`ph0O8oX)E_%$?=PqHhQPL); z)@p%TYt^OK(Q{m@d}m2^zB?)`D}5{}==v|#{)rd4drAx{{=A*@OR1_t@zr~cTMK7j zpLp%ij(bl(u9IDEE^_k4xldH@g>}bG|5H(|6JPcYx*JeGj#Me?x=cMQY-+k=dToxz(Kz3A{;5~yd^YBh zV{lMr^Z#%oga2B0`W;?%HAkKJZ<2N&E=8m|yQNRrshlEXY<25n#9O=P4JLPj^}e-+Kn%LUw7aA%eiy!pFi4Dzxa?-tZ3M*wU>6b z_r)%+o+f!ay-&N#C0Te=$juXc8-jlS2%M4-UZorKs?PV~w#O@FVl%Gm2f1DmHExcK zWN+~`y?W~EoD6N34-5?dENTf!szs|Cru|#|?nbCfmKcv5!vQ}j8$a23%NN-u zOyB!abIuHwk1v$hKW^6Ny}I9B@cuWCG}rev`xn&fBpb1_`4@fgE)kqzesA9iy%Wa& z7kZ~z_WoBF6c4+q`(|ARFV8#)@#0B)7YH1+-8*aj72$+sI}A^F^hcIcs`NlpLh`K3ZEDdoiy$V#b*g zUpUuM+CL<1$2nog{@rT*_r<5Ys6A6~^~u3KrnNp-1<-^4?cX{W@fH#2=8}^>+ z`S_(S?A3~smm}6Lx%K!Fe{W=oPs^{%pRe_8eA)ZIQihG6LHF2$mvhV$=Vxf&{r<|5 zH8YCy$-Uk24VjbNUfo>TvVQmfjvnKRcXq#byf}VT{LA47p6d<QG5v%x*S z-+311w~b9^E#EbBv8U0c2P^k`%kSg3Uw;2`amY!d7hm6J2DBgldoS#A)8>$5)&Cb% z56_;S-`VQALyyzg_}85YYMN1z>saSsmla&3ndbg$pCk7}caKH2TdM1#7i9)K&e>l5 z|KFTvUvg%}@2$Jnn{zW%!t&yZFF%!6e|2j*)WrT`Qs$-KRZh7(uC2&v{2X@o#GG26 zU2WfQv|duZ%o$3{gD?AD3LNvbxp$ZK?SEcHzkiT3T8^5Bz(6$i`DR z*81O%9^>+sH0i^OWhd)KMk;Sk{GcMf-{0kZ#LYVOM~f8J=kQn6F1j3>9n5W|W^N^0 zaZT;wmy;#U>et`v9C}%4UoCyOyJ?nNSL?Dx2S4{IMcckTA#n1;g(Q>ilZyV{7Wzw~ zS^9;KCM?gsxN}Wj;;Pe2-7h5lP`tEWQ_gal59i}2#+P(v&ivhaTyKW(gxBI-xWyw6tZhkW!9xX>sC~6{I0AUw|?2%y**!wj+k9nIs9PZ|KCz8Tc*6$*-+&1 zGWUal_U>~J5~sP$^1c15H+%Q$AcIo}#bTf5D4Y#X$^L(fm38m8H-Z*>UFV2jy73^h z;0((Z&b1*oFPNOPnfs;e$p4$ym=-y$`X6EvZzle_E7a%w?Q4C(jV8y03$Es5vNbP1 zxX1J7(%moZ76(i}Gt(r0zI)F3Zybv)E-S6oQnH(WA7qf`A%Q|F{dLVYZkG4b&(uhVmy_VK=Yv!iG3`WGMVs$@hpWcDwYnEuY>e3$-=*H+KfOYMur zi+?D@7Ts#dYMe%{Ipdpd6wJlgXoR5&yJrCq-1x<&6M23o(L^lVaJpTYTJ6U`?^ z32RtlkhRr>yZJW0G%oz@6$Iq>o&z0Q0GkII7BUkz5RV(Hf zzfNnNT9%sQR90c0N4=Vash^60})Kf^tm)#o_MA8o$Bd|LRst17F7a}WDmI@Pgr)yV@Lt8Y1-@_&6s zi*JSM@2h>?ml7wbR~9pX%JTFN9*_KzOW5Dp_uI+q`84f((PW*tY{`mq($-4Ji_7or zDNDHW*h&0hZQ=)$=`(}tzn_24-FZE(^=iCB=9&ugUdg*AH#I_M7FxgTR@+sRQ!aM$ zY^nJj*_FXBxt{i~+4^~g+RV!#H*I{bo!$FNeCxJ@n|Da^%zImM@RvyDj%60CDKD9( zO_5~l4-P*OsWIpElFU%E;Ot*>4J)=?I+WF<(|dpK-uF5EkBm2&-v6O9b!ytlO;?T1 zaY`nAcu{m>v3KfCLGJ6@b_q#&+%8^NyeB{RQB2rOwv`U1>$Y5eJ8{`cIluf_t503? ztTR2Kv}oQdpA`GI*BjIBY`^*^V2S4Q)Tk8?_h>F#bMhL;t$ETRCx0yIe7s%h=;Do8 z{~mqLlzkaEYr5y62}KR9x30KaluACnxFlek$K9(}gwpwhr+a~G!i*0s?~k~~HhjJ- zy86K(l?lAB%|G93_i0@?DTzx%Ext7Re~)Q}%kA$O`c*Ek)OqHGL_fKBXxi2G_qWRr zKAkjA%Q)@Q99#cimm(XkzB%HW>&qne#rk*o(bg~iOGKC4E;%W6Y3JdVyRY@cKA(-* zIAiHs*;_t>HisTWx=4AQl4xFTmfNtc|J&lA{#%Mq1Xr>xX^*)mymCXsy3Q9HzO2-n z-F&3_{W|%%OMmp-nO6QL-H*lpMvBRvLm5|=EZVj6&6*j%FMhOZ`*&}ypVk@wdcn|X zT5V6_=YDQU{3`o0G}b8nN9|YcWZUz0>z};Hs&ZNU((dv+^Qod&&+dH0Ew*Bk;oNQc z8$U~aiRMY3dgkY?K*pz67p<^B?R&b|_3}q4<7ZZPZC90Dn6>>cci=O}fS-bFBsi1eI>bmZT$1In;1-HDS6791 z*->u`RS%<0cS~I_-F~rhz3P&Ui)vqf4Vi8jF_kH{XjM${rl{#_rT=)HcyF|-DC$<> zyi7L#;2)76^1pn)mC;q&p*8#YWy9}mE3??2Z(P;EF>zK}Pr#a-6uBciUq{|*EAlnv z{%cqK-cQyiU+wMeecO`vl+3$xdi~UA7dKjkq+g3Uej)44j=Xi2-GAn+$~k1S=7Q>H zOW#dunmSi}C!W4_;d+#7-e#57Sr7N{HnTT~Wq$aPy14vGsmB+iecz^kEl3nP^JVk% zgAtw!U(7w9Cplr`vvb?r`JV5!n(kt1W@Y@Y@!;fH8mB}yI(+Y1wEy3pwv~Lp8jr3u z6|b9Q03>F2{1)Ixhl^MieJQsX4UPAs+eZ65Oxh zVe5|R_5!I%;x_}`L%0g*9#4mpzR{YyB8F6Er0mj@nYka3VXLZ zFBf-T+wI}~@+be2ujg35iyd$1T)NLSg}3>v-;{Gz$0MqmRBjdWUsl;%+w9tuliFRP z7?^zGtK_qF{K1dzEL`Kc>EPz6dJ)er&G@EJIhQ*kf0@e%i#EryiXMN_V~tK#9{zLn zqTX57U6>!Aq`xg?t4U75lj#*vw>IAEI9R;z-*Y{;lNZ?3TVHLckhIwm?Y;N*ho>LRx!q8rRJKd==(pr9mk%*6zxGIe z`}wux&t2bFOwH^KDhs`?&AceLrTzP?W7=9;4-cspDgcwE4M4>%k>$Bo785eUwWPW z;v-+Ek-YL=kMs90r+&7+6m@*T&8+)7C!fg@Dqnf%=e_)IvZXV{OgGh*|8@Muv#U*R z`6buS9+w2a>@oH%IdyaK%I>h(Rr%$<^~*ym4D+{#>8aQ5Z$4VT$G7#C$HKE;>h-^_ zp7tu-c^=bJ8}INnXN)SA{NgU!pYqOmy}EnL@8wM{Ru_)W{U#-r_g=64b`JZk-rUOT zWsfA%+b%nvxzSuAm|1FkrZZy3eeYMY`OVK2_D@X}Uz6Iq+<-MDoBvmwooxHvmxTs@ zb0lr&`~E&Ux5Te%>X+O}m$_Yg7c5;B@usUp@}y0x9KUL2m(Z>E{%I%nO>>`NEt&q| zLS^7op8h*^cjj+ft#Cd^bV74(<~lBsldrt(!+qx4Ze;pI&yOB6aCiUv z`$&=8s%zVQmo_$u#Irn`e&Of}jUzeFUKM3(_xk1)DJquhWIp~rv$f=4b&J+aXOHBU zKUeO`iF25hdtWpCS?MBYuG+t9mux0Qc-(c`b@1oC)Fm%VHrh0*h?zI1#IKk$eR<~cUcai7b2%^7q}#ny|JyF2bW*QbU9WrhAHIXXYD2c#WbN2c zxlT)Z?X7L{KbIG~mYc1bx@?Be)U|abep+gsQ>M;tx#&}M%JXGa@0V$l)xPYU{OD`% z^*oX0_j~tK3VD&o$ZF?J2hRJR?`O#9WOR$K5La zY^vRNT2Qb-vU+jif5UId)87TZu-Ix-vQA{a<6051Q+a>>1fR*UVVlr3$I57Xr(dYw z#Ao5sGuOY5DgN3$nJL`b-`?u;m#xBzUsejce%)mp?mzk6!rI40mt1_8#A_a0rxWmJ zxisHzwzkTYr7GduyX&LlI;U1|;;UUzXB`>&dDf?*zt5M(bH4Lk-@iO}_bJPGy9Y9> zJ8jJGKYx+->(4xgS1*hz_Czu?9C;`bU8ixY`_AL`GfzxSO+l$dGC05S<|Q7hEybMi z<^KgGb3g1UyL!>^&P$_bhaa4*iP)XgA7A3~apujg^JfB8qgp%;Nf<>>jOvcED4mq^ z_SvN0ztVD)_f^%sr|~T`||zS5=kpjPY>=@YqIa^MK1hfyX>h* z*3-U9u~RFyC`28(V#{A*a`z-$x9~lOJ`_B9>4fg)59uu>+7E#>zdmP zHm<)qqm=ojz0#zMW*ZKdo^J8{qHo0?=eOT@ZryU9iDxb@Y3#~j5jlF}x@3FUDgz@C zO`tGOYI`=1grrP;(%DxGAI)1@Q&Exrs%cW_tOHN*s z`E|4Yrn&TtO6{|knSR}~nPedSU$-aW_wyO0e3?yp(aL`|RkqA${mZyD!#-C3*X+-d z)4b=GTh0>tvhBU8)!%bB*NMHUQ+=`Uy5%{`qU|a3H-G)$E7gDc!NERWhef<{N7KJ_ zKXv;4bh=%`?@27%a#ufC?7v53nfD~q|7R2>bKi0JO4%9xGMscaNy6r(r~3aZyPj9R z?H7F+skG?0U!{QVHP@)VSK4Rwt}IupF}&ob|NVqtuk6SFt~0D-6RKY@?Y=mF`qhG2 z)8@#>Ss0zodhEjeK}Fi;QDL*zOy#Wm(F=|4tuE0x%E#s~>BH%-ng*>|J1cyj^ILbF z&KCcEwftT}Qf2Dy%&wjCE1MV1@JV)TU2^fh)|H4A#w)ewPOrN3(^G`~o#QK|eb2*v z?Em)l*QM9T@7wvk@ol+`@@gbZX>@%j;O@oB#1!qA=I{?fLmkraOYu za<3l>j{g1jT~`{rJ@?63LYFe-xaOAJ&r6#3qT1FYcJBJ0Z|5g3thv<4xV5Bzlk)#t z1>8S$)_15}eS6;hwDbfo+q<{tw>9%KOqj*iKS9_}>T2sc16hZnh}~t(@9#hNk{rJ%1G8Uv=cv!MX0&`rX-sB{&xOO%E4!E3#k;e6xS2SDmN!P4P2R zQK^q2>=7WokPxm}c`o7jY{7|i@(ERl=XRFI*voNc1MwvXG(bIo#=dL$DBra`U zb9|OjdBTU3DF-Jn{l!x~QD00c>xbTI*~G=C6K@KdZ=EDH@%Qgd%DYYkt9Xn2KCktK zwftpm=9z+04$gPZ>yKM~HNJBiu>M1^?1JLs-1hjztgG< zSk7sXD6~TG@9`T)+l_DD`T25P-Ja|N9g4Q6C3u_JP5O5xr0kwot`IAFQE`b@<8$`% z=W})aCO%}!Tz_&acjo$8@`qFyC$PnQzjJJj++4Np4C$yl(Q=`itRLJAR4#j%wp(DY z?B5BC4Yk+&;#gqy(l2Gv4_Q~Ey-u5Q5)x-C)CBEx5B9RHQ+=}a@-i3i%D@egE*U4+ z?G&82tnbycQ+jV?rs+k;?a!%VT5}|L#r#QEQzQI&^cWP3dE~6S+_$A2FS_{q)oJ-d zm7YrLFI0%X%9uZo<7aB~{@S|-K=HJrqC)5K7x(GRUH|ym_%r01kC#Z=q?s>g_VN|F zwb*@@`+cjqo}Xghvu8RnfWU%E-gypnkLn3wO{lgsda$x(s5hkM!!*;voEYDvmj-v5x1>;Cl8K8stK_0ie;uBiNw z34Cfa@!mz1WTln9a}O#@8dQL+VQ}~?W#go^`Ps3ehrf?JKK)>qisZ$kYl;`h9P^R6 zy=eX3?fGuAt#8Y}J?t-8;dQ1vX0D{Lz`@D8G|mV;FX3N#PomJ)KV0g{{fS;CRkd?9 ze#l)D!@gh6dle+E7YfJzNcdo)Uzb`lO>g?G^Qrf5u$g9A zZQi)3+qJAwMDNGU<>qC<$-&N$+C)p4zXEWIw)C^2eiY`HK7Z(hscZbYeXE zK>oa~(c-gnd-=aSJpN5?cE`utLKD{d8B_%EA3G(rgnMZ*OTX6D@^h1{*uDqU9Fw^9 z-OeK@yX2mS;PdmI#~!X_OV8Z5z~E|B=l3=aIgq6c6O7pUWsfS~EO%YJPrxqum2N@w ztWMMHy64J$`&pKq)q7R+x~0dg+$H7G6hUQS%QpQ34>ILU!tym(Tuz6p=DhM)Q{vy+ z$|a$F+jmdi@@Z9>^P(>Qw2eNg@45TiUMu71pOF`&@1<>)fBCedWTngU^qv3zxk-I* zU84N<`(xuHHct*b0DGQ6GyMaLiuQcHgzZO*4EKEb>i=MyiEy-8{^QB&_gEgMsb5`R zZFv8~15lGEM$k&};o}t^&lDtVJWa|}HsAX*;k1p^<~wT-*J>H7UwX{Dq{!Yl^!xj= z*LAx*Zl4ll?G_DtlQ~^{%l>=8{=FRY8?tus=v-cX(rw$DE%MhRRxsDTEeb#4mpbX3 z=(`iwOijvyO~NKU4{zEQxonZG6*!nYT$|PJ1fJewHm&>N;{G}3w&$*HxP8gz%9Sf0 zqVwx!{TAg`JJ_zAUmqx1l`+1BR1Qq$&qoN|^u@q-C}UFsh9>E5}`?%#Uvc0WtD zdYPS`x9{pq%jFJ6e;)m=tlF9DA@Zv!Q}F0w!_%uRTNjy%-I_gbe%f~R{5ut%(++-~ zc*tq*k!r1kquVwfj+n__tpstBXS4g|Q@1X-zG;il>&V~rk<&u=>=|SKy+?P8Ejk<) zJj?Fcksh=1ytzguGav5^4LtUawOPGV5;eDC?G17N~{_yv*b}R7+EX?=lpM36O z%`?S!`wfrOhl~BK)9#Jjsl}bTI_Sh|?WLaP+MjMuztL~JZik8Y#Admvk~fu>u2Fa& zQhnRXxVU`oO!bS`r&+T0RSI!yU%w(+G;NLVlGd(}TVZ?jrT*ts@fNd#9lXSBS5BP7 zY1fSzcmJNfp8a5xiLrd$2UGh;1|so&A^+#O*8F95PsuquH)TnxsgR&41Dk)5fM%4G z_UTDi3y$mP-p^TUn9P#7F?hC{%bXCGD=UA_t9$PjYSg#%dDpR~-zGo5yB^oN`Q^nb zO(m z;IB2={Jm2vOf*@Vp>DNnZ54jWbnTDfH15a2Zf74}y=qzzIcw(D{cD~dT(mmzjm4+a zZ1sO%EjHd9eXZI3!)2l9fJ%0wWd|R4&N}$`>e317_5rT~YZiumsbV;L@NnPC&5N%r z^ z^ZLDomI8<5IKcrPG^;>YF572~_;K!c5j^dxtO9Pg>+`B6v0n-3Hm^uyZBJja^YhWQ z%s&d--LG@)EnYC^e%9W9y^woT3~ zW6!D;Gm~fgw}nqWyJ5y(o7-Z-n=Y>}3R!6q{x0HLMqyGb>*p8gZIjn2?vyPbkpLzaxQK3%Oftt@}&lf}lwTQm2 zIr#g#<>sTatedNPH!qGcI(sLw;@`$U3iSr73?(mo&-r(;{r-*L?yjz?_IV#z1OLiS zmAroNfv3&CKLuy!uFF-gV*1&(%=?&3AoEGl+)az4jIQ=M23@{zU_;#Xh!JTwBX|dtTmL-~L(YTRs_o&Y#QWRGr;*aq73Fw^q!3W>R)1 zStaQ1Jk_sa?{@}BYTHQ{Z!~8Dd78!4i zY4*>p%*Ll~{7iJTw@bXSV#V<#6>mcST#q-u@>cqO?RW1lfpMEEO?l=WQ4E@Ca`aQi zz5wmpFFxKp`eD7o-v_hWU(MB#xDz2Y$>ZeTjxRp+D_?%m{RN484$IeY_|NY^=pR`cv6~D%?Q_t`H zI5NNIdiS|_fv-PSosr>hRxez!FCaE*+OjEc)*EJX_5RkHzA<=X_eIUnzRLKUEs5K` zrk;JC5tqOGx_H*4%t>t*{>wVNTIv}lepLLDh}fy`#+!9l^4i|_{yek#XYIlL*Em)FJKd{@%=JJe@o)LgX>>-Rf_uMK&8 zJHLM0Vhf|YcP#&P7}k7tt~FZZeC@IE|C9YzJ2ZZoY+bzUGylE+t1c$pStRDar9S+l zz=$JE$Wr@bl-G#YVXZ~D$Y>mICx5>#}?o-#i z4oN$^V7&u@UzDDh@CmDK8K3k~bEK)3X(P4@9%ltcTQ7%!Azm9t9t~Q;yD)>oF z;NwX*{`s2-#y-*Bl>PR~vb0Qgo`Zgn{C?^D1J3J@IOU{Yw|@SRD1O`2&LJx>IQX#q zp8u)$J}>Z;%zRg(&+#dA`&a+${1yEY`7nr6NJY!dm}Ek9yOrEI3@ zOYTeUOSXHgw~X?NdUxfbpzEY?tF|cyrDpG4qCdS#>$=t1(;Xj6V8k{oY@OcczFud#Qf^$zJ}s(*&QLb9Gg|bm_*=H`5sZbY0;5 z^T1Elu>atXORsZ3=eeJmRq5~Z-}z%})xigz9S4K2uk@UDwsBRCo!8^NswZ39vz4Z- z*gu!+d5_oi>dN5BYlA{{@9w^K;o{7|^c@GGjiJok4;-44OiiwJzqr@#^ECT&!5 zQZ7lWiEfd(yKYxqlJVs%)wpnK&(_j%k*!yn_X>l8 zj@aY>JaXL4>hIY6cPud4yobw}`O7c0<-I$1|1fNSl=p!n_sYzV1}pc^V7{5|{_cuv zkS5ozG;msJ@G+{`vdOGO@K5m?@vEPo+p@E(`c{-5)v|karx3673FXmjobd95z{*5$|evi&|;qiaB+t;UEOR#oQuQ-!&PglU_ zhFN#Xns%{$r&g~nx6@j;YSk=$`KrR4|IudMi7$@ad@xZb|Hg}ryz`np%S|pNavk&e z6C%&6I#Yh4T3YJaEzb|=&)zvJD$3q`Q>&~9*eNM;Wh(tCXZnvH`tscL^@Dj!1P{6V zxWD=Rkvu*X2U{+$AD^f5KAV59M&fL3t4*W0udbsBDGTq%a{pI&}sx14z*68!+Ti0GF^gXMb_cF14 z*1QcTlb>cyvTQr{`?A}s`v>N9`ZqqmHSI-q>B$)}7dBS(-`e@X+wsaBaI`V#9)6&y zGiOJ8#FL{{3u`}}I9-vN8hT4ow*Jq?tw(3u>hJwwCtrIhRdaWTb_nmi?a_kv3*Se? znfyqxOT3{`cWuSwA7>^#EStV+w!ic(hw^!qGV{xS3H;t^kaOfbN31~}%Y2(86?yX) zDRp-qCRfhSJGy6bn#ujLm#KdqNtXXlEnl-tVE^^c3p@?o-b*Z=IBBXxg#YG!(UZ?- ztUh@y!q;qm@Y7vN2ez^{CVp|rf0t9fH*Tq6c3V7GnLx7Lj_U>dz1zdH!)46w&S;z~ z1PSzQ2b0}DZg_BTo%Mpi*OO+JFQ2>fm?HK@89mNhG(z0Y?4l@)qme@A|_$e>H2NzlPh-NqQxy*v(I-b9k394wCcXgah*ur zeC1<1H$1)}qp3e-sm6;LtYAx{W*ihYt2{IF()Kpb*B6r}uFZZdxD(PZ{v$K z(^tf*boJ`*NH+Zc^!Zm#$=~(y-y>}9z4q;u{jKmZqig2fIIFXdjVixgQ2ESp>8XkS zeAhFc<@h=FR`T zX74gnGJ3S+;by5l|0Tum9o?RHm(M+&z2@i9{+f5a{vQgL$0UmL=Vf;LC$e?r&bNPW z`R}s-e)Wwb<~nLUW_pdJE=vA9vmL zLTrj&$1TT++!+%mZTYnM{lh13SfaP(Z1?TZ){j5Y&cEA2d3WJg`$sx;zotb0n9Duy zxqRJ+(n~yN+63Hyyw-Q_j^7oqOLi&Z%Od3s~4HiJzd_ne`0*~S>-=B)9*dA zUmtm0=kbop&t`USc1V7_u`c{>?#09pCi;w9udF+{SSu~hd9nJHu;gnRm#&(m>s#p_ zkoJyVWm;pT*t%LRY4)MEB{}w89n08FPwIT%JYj#?9ImH9`3@V4zMV9Bd;P8T>>Dc~ zj?Un6HRSK!(Jb{f$ujoV^6p)ZzLJ8Ez1KhA|96|9JIw||Q6FzqP-rWCl%98(+yIkGe`Y#X7H7Czf4`M%B-ECWR{Eke5 zW2yTxy$m~(`7ak%p8BS(TACSMS$S-pmT}wV>+cn6y>7YOy5^)*(Q@*0yWVVV&+C~# z!|DQ*qnTbw1wSOn8-ER7B`KILd0)~(8 zQ@-6T-+v={|A)r#AHL=`=bpz@+0U;#XJeOnsCQ4ro6wpc7rNi(rQ7cI%?eWL^>n${ zA!2wjH|x`p*+0H)e*V!r{cgIrhFE>{l8K$S9F>;pT;2I+6Z@l031O+lM(5Hrb3b&5 z==wY8W(BB3Bp$SV`FfU?n#VxVwWUaAM;Axa~Rf#qGT_=*j z#_77m>{w>tUnbdO5dzCs?L7R7n`!f&&PQ9PPl;YJ?fH)Sb;&v(3D$Ppyx#|1^mxdk6K}TnGDcyF> z{qVcVZb`yY`%ULRX`DH;%raMWf)o3zGjCQKKK7}qwSSp>+~a286`!wsXJ%BmKL0cI z=H(mzZ~5H*xad@mWLJuu=2P+ai;axK(|DujmD@fMdBXy6!x83-Q8#x+%Y1fnSaPZ6 zTjs0sWkPeiU3#bOU!~uYAt&HJud1l>PL|NhGg8e`)-OHlju%F18Naz@!?^T~ziIj} zgTU?THzw}8-(A9#Y4lR-lJb(LvvO)5FL8M5HSsam@^k)cx0x@z_5N*9xc=0c`!79? zUeYRM{A|K0Pi9fEDYv)eMntPGh=_|d+iW`j>VY?bpPg?wLLAn?x@!gB1=pXyDs*hB z<&CD+|GheEN_2z!R#xVwDD&Ymx-FRL#}sV~h=wkSGg zmYMY^LLySaCEBGiMbLIx>sgtLy@4})=Xd($ZQ(Va>=!(AIH~xbUvlOD}UbaCw}GJ!WV5`2J8`E zBG+&Csmf+|=iZeXlcNtmI4RTYKIx3k)Yfjp_t6HcPTs#f2x*?)c>@^jt&Gt$8S^UTkk5WHPQuoLBy14Vd zzgrehKf9Xsv-sy%ucUTJE?@*Tm(EMM$!E3?><*-4XA&6wLiEjP%n#(^Kx4F z_S`R?(?6aMU)Q#pF{kc-t+2jSacaz2*;uo6S*Jjj6a`jKOMf|Q=2Go8_w=^hA2%{% zH_g2@b+w`Qim4u-m9~ln)w*dP4WG1ng-Mi#jD7u)?1{H51I~%tZ&z9> z{ybmj?~%IsyLaWXMy-r(sXW|vw!k9FK)&`1@B8u}GYd`2U23PSSu-;ZIs?SOz_7@u zqD4acc69Q!-5;aYbon`Pzl)f2sr!rgv_0w4W*2krC$jrL<-hme=+d?*1%BO{spXmy z^7+N-o#(~x@TO}_;P}$#*#C^XO2qO>;FTbkM+{&v;UrssfWrPYy1FLz-OmqwJ@i@Z z!>U!Uro^uK|NdrqQdV~Dp$RkpfaX!ZW-$Kwc>LOq>$eNHtl3cTNUq}o#Fq>V0_I=( z(yXV>oPWHS(R$$CjG^82N%*ZpF!G>K|iH@LA`%c}Xky+ikRYRiazon0ZmJ`t~+1 z3nj^>4!hl7pDnLk7&oos#N2+@D>7Mcz@A}XVDRWTsJujDA)nOdeLc^c${g-*e#;lO z^3><<@DG0fZis(!|7xxNwA-h7uEvr-56%DlP&5`!d{f|S6rK?kDJ>Gn1~q3wK(o8Y zwBEjtW|g2$g#FhipQglK(U1Qo`TkmEQp~o}FP+mrmEZfvgA=dZ^4(!_x*q$xzAJsd6#4z*sZ*<_#5~yl{hijm-xF3A-t-Hc zpZ@B}?4R53|KoYCa#T?E;=A47RqH&yUlvMzGoj}>XjS-PUdZeV14BcKl+C(#QSXme zZ>&DG!t&2^)7L>o8~6YEz4gzIgW^YWqg{EYzkGh?@%-9fwsWS4Jh>~n`p1VvarFx~ z1T!O#{Q(VS{p7Q{2@7%tPUDIetLMHq+Qn9`KU#h9_U{XsWlmmfaeYz%E}GJtY?%PyIKFZEjjYOU^d z#|OTCvMqZ4LsQe0=@lI&noSUrSv$rKnrAshAE3coh>Wl=c-ADIn!b$Z(4grzB*;)T{c_6+jkegURiT)>G#j? z_x)x)=iT)4*J{&0XD;V^r!b~&56hlZ$`P~y8kY=9G>rG|N{O4VvH9`8-*H|ySM*n0 zdz$I9i0@3(&+ebQN@qVfc0};v**9A}izlbP;n=Hl+V%J1V+d!sPd@WGLos@C2kWON z!OuV4ulKi|?7Sqnb6rHI8HW}7sa)qrt*Y-q8`dmL0ypGFx5`$UKAF)yDfJac=<7|U z(Js3db{Ih&)3C*Hde7vHz~AmlVy8ii7UQcv{+(jNF9#Z%T>JURk{N7CGV5l1W{CeO zX;tu+M}E(L(fOvI<*cQ2I*;{TIA1s4P$G5{)G&r6s$EH?i;uo7?~E|%e&;4W$9#VN ztu3mm(JT9{pLQ*o#lif0{^|7iBK`WmTl*_rT~+VMT#va^@>uxKeE0eDbApdg)0*8a za`XrLIatDHxFqGPovr+;PsOO~=||VoLFXa_)}DKQzPhhmQ(}Heu%nAWXUMj8`KsRk zzh1YAMH>A47Crx|djB3B$=q|ZFID-Poi2czHNjBh)RSwoQuI8YG)1|bsrmPlUo*0y z^3BQV=F?9mZ2v7+`Y7k$``2=h^L75Vo>vnxd-Q06;+;+6qYv6+xYs4%0|`iek;SzYucp5oL;{xNwNOP z!|OlW8ZlTl)IhJc~=P z%(sQ3+1+DW_qXuW&o^E4_jDhNIeEXjJoJ>L?flwbwU@dL+=Gwo%71VCLv!)751Y^J z=#8&`oBU@Xd;C-VeINGS?R|AzzJBxV-s6XUmDYdA-xqiIe(c6B``>47mz`bYoqqSY zy8oWe`4xwBKxIi#RgaABwWR^#G7n)Mjhb^%IFH-bC_4F=Yp2i6Rabv(s$ZU>{c(Ey zuLjU?oYf2oU#}(ZvbP?a)P?LX{_)W{?bE}lcv$9MirIZQXw}u{m5C>-Yfc!7 zFWvd@V$zqw9p}!Sc+=JfnPp*MV7So4E4Q#fyY%(S6rBsMf2Vz3etvs!Pf+KoRayH! zG=^9FS3cL&Icbrgb+4o~bUg+G14Bbwg~>^qX0glBlQt%Ry11Lu?j7F$KQH%`q;0=` z)U*x{{%?h+uD%dl zd*Eob@XJ(#s7b51d^gDjt7NY*kV||3^CZLWj1Lywnf+?BQ@32WH|0$0-WPYiJP|%! znVK5vC21R9^O4#9{T=Iy>xVuX#0$m5yQg2XTy$DmsRU*~g9PuqT^Vxm7u>twJ)gYy z!u#46CUu9ucE2&ZxAMp`-{>F7{W@E%gB4e)=lu$te&)64(N}FEIxyV|jBNfcCgo<+GM;^NYAY zSx@}v?ukW^CMyF2!;+4J!f|uXf63Yu>BId!e%56hfwkAveRo{U%ifoG_}LVr>Q$?= z-(1VJ^0|D*df>)W3_X`q2t^B+GiKmWX#ea`I8WqzhFO(!;iGi4IYum%&u zidVaGHzuS<-O=|{J^XOTLGcfk)9;;+uYIb#uk`h|DaJK*zh0K_R=)nF_}RzB{d4BM zi`}WkJZWk_2ecc@z@Wg)qY(Atd!Xv$qanr$yF_}nN6)_hdZ6mm(!VR_NF&0ltXzto%Z(3xA|mL_jC^X zPhoST}_W#*)Pw&y57Z;b7rA+GhC0YIH z>7N^p;vWw4%{#@MrIDGu-20dgqBZK!CS_CV)}JG{*swTq*VSn+%YV#-7KsaE6@Fg! z-}hKlbgE74jSY3i|DU}!tvTAI|8sHwo^$#$E6wFCleWno6nVZ;iUZzMV#rWrn{Yp! z=h}>qbB>*vRidk*^Xd_3G&9cTo7QfJPrl}MiS8G5svigc`}y4dLwx<+1KC+wLV5Q> zLPI~7zBc=qzyI&d`X2|c|M_~|?wP;s&N=!s_RskL`c?be4ms@~`gTudbXS~jR#PqV zx%TC~y)io?bS^mY$dzB$ojdchNm}0N17#1(szTm7FTeTh6lnHPs{E8_*}~^hGgYsD zxU6Pg)S^HF{JD`7`)mk z{)0oTnf{d-KV#LTY;_%?nSZ9{Lg!FGDs^u>SZU7emiD@v=vU03sU_& zY1wng^afa!n614^=9xJQ4PSfD(9q0zXYkqCb>}eZ#Qe=wmZ(mn9uakaWl2a8G^~ zZjfDODtp>yJt#{DxJZIc0ug7Ol9#opwDMgLeO%Eyv$*7Dji(4t+r-NbQ>BdO1p(QWw*QfO@inwcXlH=v1%B5PfyG_#8sf2D(2u@#m?o8#Z9I@Yj zkN!S*03JpTw>B((F$L64{<|h5SfzM^LA^;@?38Ekq}LYFI8+pF`=!ow%_D>^~?zaaZ{#S;5YY`Pd2*z;~z&fnnaJ%xuHE~b8V z>YX|L^_d-!Gn}-H#V@@+x+_I*>GAE^uJQlp?AyO8(C~HaN8@wvb;LIP@5{0CIRAdZ z^Xpk>c6dzfy=-z)D`=*nO3tk*Vp&mzzsug9Dm%^tcR9#P((#SVfAWVW2t1$AauvJ^ Nz|+;wWt~$(699Q?sR;l8 literal 0 HcmV?d00001 diff --git a/test/kubernetes/benchmarks/ruby_dev_test.go b/test/kubernetes/benchmarks/ruby_dev_test.go new file mode 100644 index 000000000..5546c0200 --- /dev/null +++ b/test/kubernetes/benchmarks/ruby_dev_test.go @@ -0,0 +1,225 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ruby_dev_test holds a benchmark to time a build job of a ruby application. +package ruby_dev_test + +import ( + "bytes" + "context" + "fmt" + "io" + "path" + "strings" + "testing" + + "gvisor.dev/gvisor/test/benchmarks/tools" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/rubydev_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/rubydev_aarch64:latest" + builderContainerName = "builder" +) + +// TestRubyDev benchmarks a build job on k8s clusters. +func TestRubyDev(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("RubyDev", func(t *testing.T) { + t.Parallel() + doRubyDevTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doRubyDevTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "ruby-dev" + + // create a persistent volume on which to store the code. + persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi") + persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + for _, test := range []struct { + name string + volume *v13.Volume + }{ + { + name: "RootFS", + volume: nil, + }, + { + name: "EmptyDir", + volume: &v13.Volume{ + Name: "emptydir", + VolumeSource: v13.VolumeSource{ + EmptyDir: &v13.EmptyDirVolumeSource{}, + }, + }, + }, + { + name: "PersistentVolume", + volume: &v13.Volume{ + Name: persistentVol.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: persistentVol.GetName(), + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + // create a new RubyDevPod and set it to run on the runtime under test nodepool. + pod := newRubyDevPod(benchmarkNS, name, image, test.volume) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("failed to configure pod for test runtime node: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, builderContainerName, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("failed to set container resources: %v", err) + } + + if pod, err = cluster.CreatePod(ctx, pod); err != nil { + t.Fatalf("failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, builderContainerName) + if err != nil { + t.Fatalf("failed to get container duration: %v", err) + } + + reader, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer reader.Close() + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + + output := buf.String() + testTime, err := tools.ExtractRubyTestTime(output) + if err != nil { + t.Fatalf("ExtractRubyTestTime failed: %v", err) + } + loadTime, err := tools.ExtractRubyLoadTime(output) + if err != nil { + t.Fatalf("ExtractRubyLoadTime failed: %v", err) + } + + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record(ctx, fmt.Sprintf("RubyDev/%s", test.name), + benchmetric.BenchmarkDuration(containerDuration), + benchmetric.SpecificDuration(testTime, "test"), + benchmetric.SpecificDuration(loadTime, "load"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } +} + +// newRubyDevPod creates a new ruby dev pod spec for benchmarks. +func newRubyDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod { + const workdir = "/workdir" + const fastlane = "/fastlane" + initCommand := []string{ + "sh", + "-c", + strings.Join([]string{ + "mkdir", "-p", workdir, + "&&", + "cp", "-r", fastlane, fmt.Sprintf("%s/.", workdir), + }, " "), + } + command := []string{"bash", "/files/run_fastlane_tests.sh"} + var volumes []v13.Volume + var volumeMounts []v13.VolumeMount + if volume != nil { + volumes = []v13.Volume{*volume} + volumeMounts = []v13.VolumeMount{{ + MountPath: workdir, + Name: volume.Name, + }} + } + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Volumes: volumes, + Containers: []v13.Container{ + { + Name: builderContainerName, + Image: image, + Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, fastlane), command), + VolumeMounts: volumeMounts, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestRubyDev": TestRubyDev, + }) +} diff --git a/test/kubernetes/benchmarks/stablediffusion_test.go b/test/kubernetes/benchmarks/stablediffusion_test.go new file mode 100644 index 000000000..8b288d15b --- /dev/null +++ b/test/kubernetes/benchmarks/stablediffusion_test.go @@ -0,0 +1,239 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stablediffusion_test + +import ( + "context" + "fmt" + "hash/fnv" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/gpu/stablediffusion" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // Container image for Stable Diffusion XL. + stableDiffusionImage = "gcr.io/gvisor-presubmit/gpu/stable-diffusion-xl" +) + +func TestStableDiffusionXL(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("stable_diffusion_xl", func(t *testing.T) { + t.Parallel() + doStableDiffusionXLTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +// kubernetesPodRunner implements `stablediffusion.ContainerRunner`. +type kubernetesPodRunner struct { + cluster *testcluster.TestCluster + namespace *testcluster.Namespace +} + +// Run implements `stablediffusion.ContainerRunner.Run`. +func (r *kubernetesPodRunner) Run(ctx context.Context, image string, argv []string) ([]byte, error) { + // Build pod spec. + const stableDiffusionXLPodName = "stable-diffusion-xl" + stableDiffusionXLPod := &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: stableDiffusionXLPodName, + Namespace: r.namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: stableDiffusionXLPodName, + Image: image, + Args: argv, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } + stableDiffusionXLPod, err := r.cluster.ConfigurePodForRuntimeTestNodepool(stableDiffusionXLPod) + if err != nil { + return nil, fmt.Errorf("failed to configure pod: %v", err) + } + stableDiffusionXLPod, err = testcluster.MaybeSetContainerResources(stableDiffusionXLPod, stableDiffusionXLPod.ObjectMeta.Name, testcluster.ContainerResourcesRequest{GPU: true}) + if err != nil { + return nil, fmt.Errorf("failed to set container resources: %v", err) + } + + // Delete pod that may possibly exist from a previous iteration. + // Ignore errors since it most likely doesn't exist. + r.cluster.DeletePod(ctx, stableDiffusionXLPod) + + // Start new client pod and wait for it. + stableDiffusionXLPod, err = r.cluster.CreatePod(ctx, stableDiffusionXLPod) + if err != nil { + return nil, fmt.Errorf("failed to create stable diffusion XL pod: %v", err) + } + defer r.cluster.DeletePod(ctx, stableDiffusionXLPod) + if err := r.cluster.WaitForPodCompleted(ctx, stableDiffusionXLPod); err != nil { + logs, logsErr := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) + logs = strings.TrimSpace(logs) + if logsErr != nil { + return nil, fmt.Errorf("failed to run Stable Diffusion XL (%w) and to read logs from the pod: %v", err, logsErr) + } + if logs == "" { + return nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs are empty)", err) + } + return nil, fmt.Errorf("failed to run Stable Diffusion XL: %w (pod logs: %v)", err, logs) + } + + // All good, get logs. + logs, err := r.cluster.ReadPodLogs(ctx, stableDiffusionXLPod) + if err != nil { + return nil, fmt.Errorf("failed to read logs from pod %q: %v", stableDiffusionXLPod.GetName(), err) + } + return []byte(logs), nil +} + +// doStableDiffusionXLTest runs Stable Diffusion XL benchmarks for a single cluster. +func doStableDiffusionXLTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + imageName, err := k8sCtx.ResolveImage(ctx, stableDiffusionImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + xl := stablediffusion.NewXL(imageName, &kubernetesPodRunner{ + cluster: cluster, + namespace: benchmarkNS, + }) + + // The refiner model uses a lot of VRAM, and not all GPUs have enough of + // that to make it work. + // So we try each prompt without the refiner first. If it fails, then we + // don't try the same prompt with the refiner, as there is no way it will + // work. Similarly, if the benchmark does work without the refiner but + // does not work with the refiner, then future prompts will all have their + // refiner model attempt skipped. + refinerFailed := false + + for _, test := range []struct { + name string + query string + useRefiner bool + noiseFraction float64 + steps int + }{ + { + name: "BoringCorporateLogo", + query: `A boring flat corporate logo that says "gVisor"`, + useRefiner: true, + noiseFraction: 0.9, + steps: 32, + }, + { + name: "Androids", + query: "Photorealistic image of two androids playing chess aboard a spaceship", + useRefiner: true, + noiseFraction: 0.85, + steps: 64, + }, + } { + t.Run(test.name, func(t *testing.T) { + failedWithoutRefiner := false + for _, useRefiner := range []bool{false, true} { + t.Run(fmt.Sprintf("refiner=%t", useRefiner), func(t *testing.T) { + if useRefiner { + if failedWithoutRefiner { + t.Skipf("benchmark failed without refiner; skipping benchmark with refiner") + } + if refinerFailed { + t.Skipf("refiner failed in previous benchmark; skipping benchmark with refiner") + } + } + testCtx, testCancel := context.WithTimeout(ctx, 15*time.Minute) + defer testCancel() + prompt := &stablediffusion.XLPrompt{ + Query: test.query, + AllowCPUOffload: false, + NoiseFraction: test.noiseFraction, + Steps: test.steps, + Warm: true, + UseRefiner: useRefiner, + } + image, err := xl.Generate(testCtx, prompt) + if err != nil { + if useRefiner { + refinerFailed = true + t.Skipf("Failed to generate image with Refiner; will skip future attempts to run any prompt with the refiner.") + } + failedWithoutRefiner = true + t.Fatalf("Failed to generate images: %v", err) + } + ascii, err := image.ASCII() + if err != nil { + t.Fatalf("Failed to get ASCII: %v", err) + } + t.Logf("Generated image:\n\n%s\n", ascii) + hash := fnv.New32() + hash.Write([]byte(ascii)) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + metrics := []benchmetric.MetricValue{ + benchmetric.BenchmarkDuration(image.TotalDuration()), + benchmetric.SpecificDuration(image.ColdBaseDuration(), "base-cold"), + benchmetric.SpecificDuration(image.WarmBaseDuration(), "base-warm"), + } + if coldRefinerDuration := image.ColdRefinerDuration(); coldRefinerDuration >= 0 { + metrics = append(metrics, benchmetric.SpecificDuration(coldRefinerDuration, "refiner-cold")) + } + if warmRefinerDuration := image.WarmRefinerDuration(); warmRefinerDuration >= 0 { + metrics = append(metrics, benchmetric.SpecificDuration(warmRefinerDuration, "refiner-warm")) + } + // The image-hash metric should never change; it is still useful to + // report as a metric in order to detect instability across benchmark + // runs. + metrics = append(metrics, benchmetric.Checksum(hash, "image")) + if err := recorder.Record(ctx, fmt.Sprintf("StableDiffusionXL/%s/refiner=%t/steps=%d", test.name, useRefiner, test.steps), metrics...); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + } + }) + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestStableDiffusionXL": TestStableDiffusionXL, + }) +} diff --git a/test/kubernetes/benchmarks/startup_test.go b/test/kubernetes/benchmarks/startup_test.go new file mode 100644 index 000000000..bcec1867f --- /dev/null +++ b/test/kubernetes/benchmarks/startup_test.go @@ -0,0 +1,132 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package startup_test benchmarks the time it takes for an empty alpine container to complete successfully. +package startup_test + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" +) + +const ( + benchName = "StartUp" +) + +var ( + command = []string{"/bin/sh", "-c", "echo hello"} +) + +// TestStartup benchmarks the time it takes for an empty alpine container to complete successfully. +// Note: WRT gVisor startup latency, this is not a meaningful benchmark. Startup time is dominated +// by Kubernetes control plane API calls and not actual container startups. This benchmark is +// provided for illustrative purposes only. +func TestStartup(t *testing.T) { + t.Logf("Warning: This is not a meaningful benchmark. Read the comments.") + + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run(benchName, func(t *testing.T) { + cluster := cluster + t.Parallel() + + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + podName := "startup" + image, err := k8sCtx.ResolveImage(ctx, "alpine") + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + p, err := cluster.ConfigurePodForRuntimeTestNodepool(benchmarkNS.NewAlpinePod(podName, image, command)) + if err != nil { + t.Fatalf("failed to set pod for test nodepool: %v", err) + } + + start := time.Now() + p, err = cluster.CreatePod(ctx, p) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, p) + if err := cluster.WaitForPodCompleted(ctx, p); err != nil { + t.Fatalf("Failed to wait for pod to complete: %v", err) + } + reader, err := cluster.GetLogReader(ctx, p, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer reader.Close() + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + if strings.TrimSpace(buf.String()) != "hello" { + t.Fatalf("Mistmatch output: got: %q want: %q", buf.String(), "hello") + } + + // For longer running containers, and where the desired duration to + // measure is the time it takes to run a command within a container, + // this should use `GetTimedContainerDuration` instead. + // However, since this benchmark's goal is to measure container runtime + // overhead, it uses the Kubernetes-level metrics for container + // duration. + containerDuration, err := cluster.ContainerDurationSecondsByName(ctx, p, p.GetName()) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + overallDuration := time.Since(start) + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + err = recorder.Record(ctx, benchName, + benchmetric.BenchmarkDuration(overallDuration), + benchmetric.SpecificDuration(containerDuration, "container-runtime"), + ) + if err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + }) + }) +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestStartup": TestStartup, + }) +} diff --git a/test/kubernetes/benchmarks/tensorflow_test.go b/test/kubernetes/benchmarks/tensorflow_test.go new file mode 100644 index 000000000..03dd6f175 --- /dev/null +++ b/test/kubernetes/benchmarks/tensorflow_test.go @@ -0,0 +1,170 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tensorflow_test + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/benchmetric" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + imageAMD = "gcr.io/gvisor-presubmit/benchmarks/tensorflow_x86_64:latest" + imageARM = "gcr.io/gvisor-presubmit/benchmarks/tensorflow_aarch64:latest" +) + +var workloads = map[string]string{ + "Kmeans": "2_BasicModels/kmeans.py", + "LogisticRegression": "2_BasicModels/logistic_regression.py", + "NearestNeighbor": "2_BasicModels/nearest_neighbor.py", + "RandomForest": "2_BasicModels/random_forest.py", + "ConvolutionalNetwork": "3_NeuralNetworks/convolutional_network.py", + "MultilayerPerceptron": "3_NeuralNetworks/multilayer_perceptron.py", + "NeuralNetwork": "3_NeuralNetworks/neural_network.py", +} + +func TestTensorflowOnCPU(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("TensorflowOnCPU", func(t *testing.T) { + t.Parallel() + doTensorflowOnCPU(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doTensorflowOnCPU(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + const name = "tensorflow" + recorder, err := benchmetric.GetRecorder(ctx) + if err != nil { + t.Fatalf("Failed to initialize benchmark recorder: %v", err) + } + + image := imageAMD + if cluster.RuntimeTestNodepoolIsARM() { + image = imageARM + } + if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + + workloadPaths := make([]string, 0, len(workloads)) + workloadPathToName := make(map[string]string, len(workloads)) + for name, path := range workloads { + workloadPaths = append(workloadPaths, path) + workloadPathToName[path] = name + } + sort.Strings(workloadPaths) + + var total time.Duration + for _, workloadPath := range workloadPaths { + workloadName := workloadPathToName[workloadPath] + t.Run(workloadName, func(t *testing.T) { + pod := newTensorflowOnCPUPod(benchmarkNS, name, image, workloadPath) + pod, err := cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod for test runtime: %v", err) + } + + pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, pod) + + containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name) + if err != nil { + t.Fatalf("Failed to get container duration: %v", err) + } + if err := recorder.Record(ctx, fmt.Sprintf("TensorflowOnCPU/%s", workloadName), benchmetric.BenchmarkDuration(containerDuration)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + total += containerDuration + }) + if t.Failed() { + break + } + } + if !t.Failed() { + if err := recorder.Record(ctx, "TensorflowOnCPU", benchmetric.BenchmarkDuration(total)); err != nil { + t.Fatalf("Failed to record benchmark data: %v", err) + } + } +} + +func newTensorflowOnCPUPod(namespace *testcluster.Namespace, name, image, workloadPath string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Command: benchmetric.TimedCommand("python", workloadPath), + WorkingDir: "/TensorFlow-Examples/examples", + Env: []v13.EnvVar{ + { + Name: "PYTHONPATH", + Value: "/TensorFlow-Examples/examples", + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestTensorflowOnCPU": TestTensorflowOnCPU, + }) +} diff --git a/test/kubernetes/benchmarks/wordpress_test.go b/test/kubernetes/benchmarks/wordpress_test.go new file mode 100644 index 000000000..dce15a78e --- /dev/null +++ b/test/kubernetes/benchmarks/wordpress_test.go @@ -0,0 +1,398 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wordpress_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench" + "gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + wordpressImage = "wordpress:6.2.0-php8.2-apache" + mariaDBImage = "mariadb:10.11.3-jammy" + wordpressPort = 80 + mariaDBPort = 3306 + wordpressBenchmarkDuration = 70 * time.Second + wordpressRequestTimeout = 10 * time.Second + wordpressLoginPage = "/wp-login.php" + mariaDBName = "wpbench" + mariaDBUser = "wpuser" + mariaDBPassword = "wppassword" + mariaDBRootPassword = "hunter2" + mariaDBVolumeName = "wpdata" + mariaDBVolumeDirectory = "/var/lib/mysql" + + wordpressServerLabelKey = "app.kubernetes.io/name" + wordpressServerLabelValue = "wordpress" + mariaDBServerLabelKey = "app.kubernetes.io/name" + mariaDBServerLabelValue = "mariadb" +) + +var ( + threads = []int{1, 8, 64, 1000} + targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS} + wantPercentiles = []int{50, 95, 99} +) + +func TestWordpress(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) { + t.Run("wordpress", func(t *testing.T) { + t.Parallel() + doWordpressTest(ctx, t, k8sCtx, cluster) + }) + }) +} + +func doWordpressTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { + benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS) + if err != nil { + t.Fatalf("Failed to setup profiling: %v", err) + } + defer endProfiling() + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) + + // Create a persistent volume on which to store the database data. + dbVolume := benchmarkNS.GetPersistentVolume(mariaDBVolumeName, "30Gi") + dbVolume, err = cluster.CreatePersistentVolume(ctx, dbVolume) + if err != nil { + t.Fatalf("failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, dbVolume) + + databaseName := "mariadb" + mariaDBImg, err := k8sCtx.ResolveImage(ctx, mariaDBImage) + if err != nil { + t.Fatalf("failed to resolve image: %v", err) + } + database := newMariaDBServer(benchmarkNS, databaseName, mariaDBImg, dbVolume) + database, err = cluster.ConfigurePodForTertiaryNodepool(database) + if err != nil { + t.Fatalf("Failed to configure pod for tertiary nodepool: %v", err) + } + database, err = cluster.CreatePod(ctx, database) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, database) + if err := cluster.WaitForPodRunning(ctx, database); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + databaseService := newMariaDBService(benchmarkNS, databaseName) + databaseService, err = cluster.CreateService(ctx, databaseService) + if err != nil { + t.Fatalf("Failed to create database service: %v", err) + } + defer cluster.DeleteService(ctx, databaseService) + mariaDBIP := testcluster.GetIPFromService(databaseService) + + name := "wordpress" + wordpressImg, err := k8sCtx.ResolveImage(ctx, wordpressImage) + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + server := newWordpressServer(benchmarkNS, name, wordpressImg, mariaDBIP) + server, err = cluster.ConfigurePodForRuntimeTestNodepool(server) + if err != nil { + t.Fatalf("Failed to configure pod for runtime nodepool: %v", err) + } + server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{}) + if err != nil { + t.Fatalf("Failed to set container resources: %v", err) + } + server, err = cluster.CreatePod(ctx, server) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, server) + + if err := cluster.WaitForPodRunning(ctx, server); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + + service := newWordpressService(benchmarkNS, name) + service, err = cluster.CreateService(ctx, service) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer cluster.DeleteService(ctx, service) + wordpressIP := testcluster.GetIPFromService(service) + + // Install WordPress. + installWordpressPod := newWordpressInstall(benchmarkNS, "install-wordpress", wordpressIP) + installWordpressPod, err = cluster.ConfigurePodForClientNodepool(installWordpressPod) + if err != nil { + t.Fatalf("Failed to configure pod for client nodepool: %v", err) + } + installWordpressPod, err = cluster.CreatePod(ctx, installWordpressPod) + if err != nil { + t.Fatalf("Failed to create pod: %v", err) + } + defer cluster.DeletePod(ctx, installWordpressPod) + if err := cluster.WaitForPodCompleted(ctx, installWordpressPod); err != nil { + t.Fatalf("Failed to wait for pod: %v", err) + } + cluster.DeletePod(ctx, installWordpressPod) + + var rounds []httpbench.Round + for _, numThreads := range threads { + for _, qps := range targetQPS { + if qps < numThreads { + continue + } + onlyReport := []httpbench.MetricType{httpbench.RequestsPerSecond} + // If we're testing at max QPS, don't report latency, + // because all requests will hit the timeout. + // Otherwise, only report latency, because the throughput + // is exactly determined by the QPS target anyway. + if qps != httpbench.InfiniteQPS { + onlyReport = append(onlyReport, httpbench.Latency) + } + rounds = append(rounds, httpbench.Round{ + NumThreads: numThreads, + TargetQPS: qps, + Duration: wordpressBenchmarkDuration, + OnlyReport: onlyReport, + }) + } + } + benchmark := &httpbench.HTTPBenchmark{ + Name: "wordpress", + Cluster: cluster, + Namespace: benchmarkNS, + Service: service, + Port: wordpressPort, + Path: wordpressLoginPage, + Rounds: rounds, + Timeout: wordpressRequestTimeout, + WantPercentiles: wantPercentiles, + } + benchmark.Run(ctx, t) +} + +func newMariaDBServer(namespace *testcluster.Namespace, name, image string, volume *v13.PersistentVolumeClaim) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{mariaDBServerLabelKey: mariaDBServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: mariaDBPort, + }, + }, + Env: []v13.EnvVar{ + { + Name: "MARIADB_ROOT_PASSWORD", + Value: mariaDBRootPassword, + }, + { + Name: "MARIADB_DATABASE", + Value: mariaDBName, + }, + { + Name: "MARIADB_USER", + Value: mariaDBUser, + }, + { + Name: "MARIADB_PASSWORD", + Value: mariaDBPassword, + }, + }, + VolumeMounts: []v13.VolumeMount{{ + Name: volume.GetName(), + MountPath: mariaDBVolumeDirectory, + }}, + }, + }, + Volumes: []v13.Volume{{ + Name: volume.GetName(), + VolumeSource: v13.VolumeSource{ + PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{ + ClaimName: volume.GetName(), + }, + }, + }}, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newMariaDBService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{mariaDBServerLabelKey: mariaDBServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: mariaDBPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func newWordpressServer(namespace *testcluster.Namespace, name, image, mariaDBHost string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + Labels: map[string]string{wordpressServerLabelKey: wordpressServerLabelValue}, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: name, + Image: image, + Ports: []v13.ContainerPort{ + { + Name: name, + ContainerPort: wordpressPort, + }, + }, + Env: []v13.EnvVar{ + { + Name: "WORDPRESS_DB_HOST", + Value: mariaDBHost, + }, + { + Name: "WORDPRESS_DB_USER", + Value: mariaDBUser, + }, + { + Name: "WORDPRESS_DB_PASSWORD", + Value: mariaDBPassword, + }, + { + Name: "WORDPRESS_DB_NAME", + Value: mariaDBName, + }, + { + Name: "WORDPRESS_TABLE_PREFIX", + Value: "wp_", + }, + }, + }, + }, + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +func newWordpressService(namespace *testcluster.Namespace, name string) *v13.Service { + return namespace.GetService(name, v13.ServiceSpec{ + Selector: map[string]string{wordpressServerLabelKey: wordpressServerLabelValue}, + Ports: []v13.ServicePort{ + { + Name: name, + Protocol: v13.ProtocolTCP, + Port: wordpressPort, + TargetPort: intstr.FromString(name), + }, + }, + }) +} + +func newWordpressInstall(namespace *testcluster.Namespace, name, wpHost string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace.Namespace, + }, + Spec: v13.PodSpec{ + Containers: []v13.Container{ + { + Name: "install-wordpress", + Image: "debian:latest", + // This command installs WordPress through the web UI. + // Source of the parameters: + // https://github.com/GoogleCloudPlatform/click-to-deploy/blob/master/k8s/wordpress/chart/wordpress/templates/wordpress-configmap.yaml + Command: []string{ + "sh", "-c", + strings.Join([]string{ + "apt-get update -y " { + if strings.ContainsRune(cmd, c) { + hasAnySpecialCharacter = true + break + } + } + if !hasAnySpecialCharacter { + return cmd + } + for _, c := range "\\\"$`" { + cmd = strings.ReplaceAll(cmd, string(c), fmt.Sprintf("\\%s", string(c))) + } + return fmt.Sprintf(`"%s"`, cmd) +} + +// TimedCommand takes in a command-line and computes a high-fidelity +// duration of how long it took. A pod using this command-line should +// have its duration measured using GetTimedContainerDuration. +// This is more reliable than looking at Kubernetes-tracked metrics +// for container start/end times, because Kubernetes only tracks these +// with second-level granularity, and because they include the overhead +// of starting/stopping a container. For benchmarks that want to measure +// the time of a particular command, TimedCommand provides better +// precision (nanosecond resolution) and accuracy (only measure the +// duration of the command, not the container runtime overhead). +// This must run in a container that has either `sh` or `bash` +// installed. +func TimedCommand(argv ...string) []string { + escapedCmd := make([]string, len(argv)) + for i, arg := range argv { + escapedCmd[i] = shellEscape(arg) + } + escapedCmd = append(escapedCmd, "||", "echo", shellEscape(timingDataLineFailure)) + nowNanos := strings.ReplaceAll(nowNanosShellFunc, "\n", " ") + for strings.Contains(nowNanos, " ") { + nowNanos = strings.ReplaceAll(nowNanos, " ", " ") + } + innerCommand := strings.Join([]string{ + // Define the now_nanos function. + nowNanos, + // Get the timestamp before the command. + `before="$(now_nanos)"`, + // Run the command. + strings.Join(escapedCmd, " "), + // Get the timestamp after the command. + `after="$(now_nanos)"`, + // Print out the before/after timestamps. + // We cannot use shsprintf here because we *want* to be able to + // use variables that shsprintf considers to "go out of scope". + // This is safe because all of the strings below are completely + // static. + // We use this weird string concatenation to avoid triggering + // the linter. + fmt.Sprintf(`ec`+`ho "%s"`, strings.Join([]string{ + timingDataLinePrefix, + "${before}", + "${after}", + }, timingDataSeparator)), + }, "; ") + + return []string{ + // We can't assume that bash is installed, but we also can't + // do conditionals in the top-level command, so just spawn sh + // and we'll check if bash exists within that. + "sh", + "-c", + fmt.Sprintf( + // Use bash if possible, otherwise use sh. + "if hash bash > /dev/null 2>/dev/null; then bash -c %s; else sh -c %s; fi", + shellEscape(innerCommand), + shellEscape(innerCommand), + ), + } +} + +// CommandThenTimed returns a command-line that runs a given command as +// initialization, then cd's into the given directory, then runs another +// command there under TimedCommand. +// When using GetTimedContainerDuration, only the duration of the +// `timedCmd` command will be measured. +// If `cd` is empty, no directory change happens. +func CommandThenTimed(initCmd []string, cd string, timedCmd []string) []string { + escapedInitCmd := make([]string, len(initCmd)) + for i, arg := range initCmd { + escapedInitCmd[i] = shellEscape(arg) + } + timedCmd = TimedCommand(timedCmd...) + escapedTimedCmd := make([]string, len(timedCmd)) + for i, arg := range timedCmd { + escapedTimedCmd[i] = shellEscape(arg) + } + commands := make([]string, 0, 3) + commands = append(commands, strings.Join(escapedInitCmd, " ")) + if cd != "" { + commands = append(commands, fmt.Sprintf("c"+"d %s", shellEscape(cd))) + } + commands = append(commands, strings.Join(escapedTimedCmd, " ")) + return []string{ + "sh", + "-c", + strings.Join(commands, " && "), + } +} + +// GetTimedContainerDuration waits for the given pod to exit, then parses its +// output and looks for duration information as expected from a command-line +// generated using `TimedCommand`. +func GetTimedContainerDuration(ctx context.Context, c *testcluster.TestCluster, pod *v13.Pod, containerName string) (time.Duration, error) { + if err := c.WaitForPodCompleted(ctx, pod); err != nil { + return 0, fmt.Errorf("failed to wait for pod to complete: %v", err) + } + rdr, err := c.GetLogReader(ctx, pod, v13.PodLogOptions{ + Container: containerName, + }) + if err != nil { + return 0, fmt.Errorf("GetLogReader on cluster %q pod %v: %v", c.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return 0, fmt.Errorf("failed to read from pod: %q: %v", pod.GetName(), err) + } + return ParseTimedContainerOutput(string(out)) +} + +// ParseTimedContainerOutput parses the output of a TimedContainer. +func ParseTimedContainerOutput(out string) (time.Duration, error) { + found := false + var duration time.Duration + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, timingDataLinePrefix) { + continue + } + if line == timingDataLineFailure { + return 0, fmt.Errorf("command failed; output: %s", out) + } + if found { + return 0, fmt.Errorf("output has multiple lines that look like duration information: %s", out) + } + data := strings.Split(line, timingDataSeparator) + if len(data) != 4 { + return 0, fmt.Errorf("malformed timing duration data line: %q", line) + } + beforeString, afterString := data[2], data[3] + beforeNanos, err := strconv.ParseInt(beforeString, 10, 64) + if err != nil { + return 0, fmt.Errorf("malformed timing duration data line %q: %v", line, err) + } + beforeTime := time.Unix(beforeNanos/1e9, beforeNanos%1e9) + afterNanos, err := strconv.ParseInt(afterString, 10, 64) + if err != nil { + return 0, fmt.Errorf("malformed timing duration data line %q: %v", line, err) + } + afterTime := time.Unix(afterNanos/1e9, afterNanos%1e9) + duration = afterTime.Sub(beforeTime) + if duration <= 0 { + return 0, fmt.Errorf("duration is zero or negative: got before=%d (%v) / after=%d (%v)", beforeNanos, beforeTime, afterNanos, afterTime) + } + found = true + } + if !found { + return 0, fmt.Errorf("output did not contain duration information: %s", out) + } + return duration, nil +} diff --git a/test/kubernetes/k8sctx/BUILD b/test/kubernetes/k8sctx/BUILD new file mode 100644 index 000000000..7f7769618 --- /dev/null +++ b/test/kubernetes/k8sctx/BUILD @@ -0,0 +1,25 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "k8sctx", + srcs = [ + "k8sctx.go", + "k8sctx_impl.go", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + "//runsc/flag", + "//test/kubernetes:test_range_config_go_proto", + "//test/kubernetes/testcluster", + "//tools/gvisor_k8s_tool/provider/kubectl", + "@googleapis//google/container/v1:container_go_proto", + "@org_golang_google_protobuf//encoding/prototext:go_default_library", + ], +) diff --git a/test/kubernetes/k8sctx/k8sctx.go b/test/kubernetes/k8sctx/k8sctx.go new file mode 100644 index 000000000..554bd705c --- /dev/null +++ b/test/kubernetes/k8sctx/k8sctx.go @@ -0,0 +1,111 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package k8sctx is used to manage the lifecycle of a Kubernetes test or +// benchmark running in one or more Kubernetes clusters. +// It is used to control the behavior of Kubernetes-based tests and benchmarks +// at runtime and to abstract away how the Kubernetes test clusters are +// created and managed from the test code. +package k8sctx + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "gvisor.dev/gvisor/test/kubernetes/testcluster" +) + +// KubernetesContext represents the Kubernetes execution context. +// It is used to keep track of available Kubernetes clusters to test on. +// Tests are expected to call `RegisterTest` for every of their test function, +// then `TestMain`. +type KubernetesContext interface { + // TestMain should be called inside tests' `TestMain` function, after having + // registered all tests with `RegisterTest`. + TestMain(m *testing.M) + + // RegisterTest registers a test. + // It should be called for every `Test*(*testing.T)` function in the test. + // Note that the `k8sctx.TestMain` helper function below will call this for + // you given a map of tests. + RegisterTest(name string, fn TestFunc) + + // AcquireCluster returns a single cluster for the test or benchmark to use. + // The cluster is guaranteed to not be in use by other tests or benchmarks + // until the `ReleaseCluster` method is called. + // This method should block if there are no available clusters. + AcquireCluster(ctx context.Context, t *testing.T) *testcluster.TestCluster + + // ReleaseCluster unlocks the given cluster for use by other tests or + // benchmarks. + ReleaseCluster(ctx context.Context, t *testing.T, cluster *testcluster.TestCluster) + + // ForEachCluster reserves as many test clusters as are available, calls + // `fn` on each of them, and releases each of them when `fn` finishes. + ForEachCluster(ctx context.Context, t *testing.T, fn func(cluster *testcluster.TestCluster)) + + // ResolveImage resolves a container image name (possibly with a label) + // to a fully-qualified image name. It can also return an `image:label` + // string if the Kubernetes cluster the test runs in will resolve it on + // its own. + ResolveImage(ctx context.Context, imageName string) (string, error) +} + +// TestFunc is a test function that is expected to call `Context` and run a +// test or benchmark within a Kubernetes context. +type TestFunc func(t *testing.T) + +var ( + kubernetesCtxMu sync.Mutex + kubernetesCtxOnce sync.Once + kubernetesCtxFn func(context.Context) (KubernetesContext, error) + kubernetesCtx KubernetesContext + kubernetesCtxErr error +) + +// Context gets the global Kubernetes context. +// It must be called after SetContext has already been called. +func Context(ctx context.Context) (KubernetesContext, error) { + kubernetesCtxMu.Lock() + defer kubernetesCtxMu.Unlock() + if kubernetesCtxFn == nil { + return nil, errors.New("k8sctx.Context called prior to k8sctx.SetContextConstructor") + } + kubernetesCtxOnce.Do(func() { + kubernetesCtx, kubernetesCtxErr = kubernetesCtxFn(ctx) + }) + return kubernetesCtx, kubernetesCtxErr +} + +// SetContextConstructor sets the global Kubernetes context constructor. +func SetContextConstructor(fn func(context.Context) (KubernetesContext, error)) { + kubernetesCtxMu.Lock() + defer kubernetesCtxMu.Unlock() + kubernetesCtxFn = fn +} + +// TestMain is a helper to write the TestMain function of tests. +func TestMain(m *testing.M, testFuncs map[string]TestFunc) { + k8sCtx, err := Context(context.Background()) + if err != nil { + panic(fmt.Sprintf("failed to get k8sctx: %v", err)) + } + for name, fn := range testFuncs { + k8sCtx.RegisterTest(name, fn) + } + k8sCtx.TestMain(m) +} diff --git a/test/kubernetes/k8sctx/k8sctx_impl.go b/test/kubernetes/k8sctx/k8sctx_impl.go new file mode 100644 index 000000000..72e9e3a3e --- /dev/null +++ b/test/kubernetes/k8sctx/k8sctx_impl.go @@ -0,0 +1,99 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !false +// +build !false + +package k8sctx + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + + cspb "cloud.google.com/go/container/apiv1/containerpb" + "google.golang.org/protobuf/encoding/prototext" + "gvisor.dev/gvisor/runsc/flag" + testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl" +) + +var ( + kubectlContextName = flag.String("kubectl-context-name", "", "Name of the kubectl context to use within the kubectl config") + clusterProtoPath = flag.String("cluter-proto-path", "", "Path to a `google.container.v1.Cluster` textproto file") + testNodepoolRuntime = flag.String("test-nodepool-runtime", "", "if set, override the runtime used for pods scheduled on the 'test' nodepool. If unset, the nodepool default is used") +) + +// kubectlContext implements KubernetesContext using a named `kubectl` context +// from the user's kubectl config. +type kubectlContext struct { + cluster *testcluster.TestCluster +} + +func newKubectlContext(ctx context.Context) (KubernetesContext, error) { + if *kubectlContextName == "" { + return nil, errors.New("no kubectl context name specified") + } + if *clusterProtoPath == "" { + return nil, errors.New("no cluster proto path specified") + } + cluster, err := kubectl.NewCluster(*kubectlContextName) + if err != nil { + return nil, fmt.Errorf("cannot initialize cluster %q: %w", *kubectlContextName, err) + } + var clusterPB cspb.Cluster + clusterBytes, err := os.ReadFile(*clusterProtoPath) + if err != nil { + return nil, fmt.Errorf("cannot read cluster textproto file %q: %w", *clusterProtoPath, err) + } + if err = prototext.Unmarshal(clusterBytes, &clusterPB); err != nil { + return nil, fmt.Errorf("cannot unmarshal cluster textproto file %q: %w", *clusterProtoPath, err) + } + testCluster := testcluster.NewTestClusterWithClient(&testpb.Cluster{Cluster: &clusterPB}, cluster.Client()) + if *testNodepoolRuntime != "" { + testCluster.OverrideTestNodepoolRuntime(testcluster.RuntimeType(*testNodepoolRuntime)) + } + return &kubectlContext{cluster: testCluster}, nil +} + +func (c *kubectlContext) AcquireCluster(ctx context.Context, t *testing.T) *testcluster.TestCluster { + return c.cluster +} + +func (c *kubectlContext) ReleaseCluster(ctx context.Context, t *testing.T, cluster *testcluster.TestCluster) { + // Nothing to do. +} + +func (c *kubectlContext) ForEachCluster(ctx context.Context, t *testing.T, fn func(cluster *testcluster.TestCluster)) { + fn(c.cluster) +} + +func (c *kubectlContext) ResolveImage(ctx context.Context, imageName string) (string, error) { + return imageName, nil +} + +func (c *kubectlContext) RegisterTest(name string, fn TestFunc) { + // Nothing to do here, we use the regular testing library. +} + +func (c *kubectlContext) TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func init() { + SetContextConstructor(newKubectlContext) +} diff --git a/test/kubernetes/test_range_config.proto b/test/kubernetes/test_range_config.proto new file mode 100644 index 000000000..0a36e8b57 --- /dev/null +++ b/test/kubernetes/test_range_config.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package k8s_tester.test_cluster_config; + +import "google/container/v1/cluster_service.proto"; + +// TestRangeSpec is a description of the test environment to be created. It is +// the input of the setup step which creates the required clusters. +message TestRangeSpec { + // Name for clusters. This name will be used as a template for all created + // clusters (e.g. my-cluster will have clusters named something like + // "my-cluster-0" with an ascending index). Cluster names are limited to + // 40 charecters, so names will be truncated to fit this constraint. + string name = 1; + + // A nodepool built with the runtime under test. + google.container.v1.NodePool test_runtime = 2; + + // clients is another nodepool in the cluster to use against the test_runtime. + // For example, in most client-server tests, the runtime under test is the + // server and the clients are the client. Clients always use the default + // runtime runc. + google.container.v1.NodePool clients = 3; + + // tertiary is a third nodepool in the cluster, used by some benchmarks that + // need it for isolation. + // For example, the WordPress benchmark needs to run the MySQL database on + // a separate machine in order to force network traffic to flow across the + // host's non-local network stack for a fair comparison between runsc/runc. + // The tertiary nodepool may use gVisor or runc as a runtime, depending on + // user configuration. + google.container.v1.NodePool tertiary = 8; + + // versions are the GKE patch versions to use for the clusters. The number + // of clusters created will be num(versions) * replicas. + repeated string versions = 4; + + // zones are the availability zones in which to create clusters. clusters will + // be created in each zone in a round robin fashion until the requested amount + // of clusters is created. This is provided to expand quota. + // Note: please check that given zones actual have resources available (e.g + // ARM machines are not available in all zones). + repeated string zones = 5; + + // project is the project under which clusters should be created. + string project = 6; + + // project service account to use to create clusters. + string gke_service_account = 7; +} + +// TestRange contains the created clusters. This is an output from the setup +// phase and an input for the test phase. +message TestRange { + repeated Cluster clusters = 1; +} + +// Cluster holds the created cluster and its credential file. +message Cluster { + // Created Cluster proto. + google.container.v1.Cluster cluster = 1; + + // The setup step will create individual credential files for each created + // cluster. + string credential_file = 2; +} diff --git a/test/kubernetes/testcluster/BUILD b/test/kubernetes/testcluster/BUILD new file mode 100644 index 000000000..b49dc8aa8 --- /dev/null +++ b/test/kubernetes/testcluster/BUILD @@ -0,0 +1,31 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "testcluster", + srcs = [ + "objects.go", + "testcluster.go", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + "//test/kubernetes:test_range_config_go_proto", + "@googleapis//google/container/v1:container_go_proto", + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/fields:go_default_library", + "@io_k8s_apimachinery//pkg/watch:go_default_library", + "@io_k8s_client_go//kubernetes:go_default_library", + "@io_k8s_client_go//tools/clientcmd:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + "@org_golang_x_sync//errgroup:go_default_library", + ], +) diff --git a/test/kubernetes/testcluster/objects.go b/test/kubernetes/testcluster/objects.go new file mode 100644 index 000000000..9c63d6f46 --- /dev/null +++ b/test/kubernetes/testcluster/objects.go @@ -0,0 +1,335 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcluster + +import ( + "context" + "fmt" + "strconv" + + cspb "cloud.google.com/go/container/apiv1/containerpb" + "google.golang.org/protobuf/proto" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + pod = "Pod" + apiV1 = "v1" +) + +// Namespace represents a Kubernetes object namespace. +// It can contain pods or services or other Kubernetes objects. +// It is useful in tests that create multiple objects and want to ensure +// their mutual destruction, as well as for re-running tests and ensuring +// that the objects from past runs are cleaned up properly on the next run. +type Namespace struct { + testCluster *TestCluster + Namespace string +} + +// Namespace returns a new namespace in this cluster. +func (t *TestCluster) Namespace(namespace string) *Namespace { + return &Namespace{ + testCluster: t, + Namespace: namespace, + } +} + +// Reset deletes this namespace if it exists, and unconditionally +// creates a new namespace of this name. +// This should be used in the beginning of tests, such that the namespace +// is empty and ready to be used. +func (n *Namespace) Reset(ctx context.Context) error { + n.Cleanup(ctx) + _, err := n.testCluster.createNamespace(ctx, &v13.Namespace{ + TypeMeta: v1.TypeMeta{ + Kind: "namespace", + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: n.Namespace, + }, + }) + return err +} + +// Cleanup deletes this namespace if it exists. +func (n *Namespace) Cleanup(ctx context.Context) error { + return n.testCluster.deleteNamespace(ctx, n.Namespace) +} + +// NewAlpinePod returns an alpine pod template. +func (n *Namespace) NewAlpinePod(name, image string, cmd []string) *v13.Pod { + container := v13.Container{Name: name, Image: image, Command: cmd} + pod := n.NewPod(name) + pod.Spec.Containers = []v13.Container{container} + return pod +} + +// NewPod returns a pod template. +func (n *Namespace) NewPod(name string) *v13.Pod { + return &v13.Pod{ + TypeMeta: v1.TypeMeta{ + Kind: pod, + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: n.Namespace, + }, + Spec: v13.PodSpec{ + RestartPolicy: v13.RestartPolicyNever, + }, + } +} + +// GetPersistentVolume gets a persistent volume spec for benchmarks. +func (n *Namespace) GetPersistentVolume(name, size string) *v13.PersistentVolumeClaim { + return &v13.PersistentVolumeClaim{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: n.Namespace, + }, + Spec: v13.PersistentVolumeClaimSpec{ + AccessModes: []v13.PersistentVolumeAccessMode{v13.ReadWriteOnce}, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceStorage: resource.MustParse(size), + }, + }, + }, + } +} + +// GetService gets a service spec for benchmarks. +func (n *Namespace) GetService(name string, spec v13.ServiceSpec) *v13.Service { + return &v13.Service{ + TypeMeta: v1.TypeMeta{ + Kind: "Service", + APIVersion: apiV1, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: n.Namespace, + }, + Spec: spec, + } +} + +// ContainerResourcesRequest holds arguments to set requested resource on a container. +type ContainerResourcesRequest struct { + CPUResources string // CPUResources to request. Note: Will be overridden by flag above. + MemoryResources string // MemoryResources to request. Note: Will be overridden by flag above. + GPU bool +} + +// MaybeSetContainerResources sets container resources if flags are given. Sets both the resource +// limits and requests as container runtimes honor them differently. +func MaybeSetContainerResources(pod *v13.Pod, containerName string, requests ContainerResourcesRequest) (*v13.Pod, error) { + resourceList := v13.ResourceList{} + if requests.CPUResources != "" { + resourceList[v13.ResourceCPU] = resource.MustParse(requests.CPUResources) + } + if requests.MemoryResources != "" { + resourceList[v13.ResourceMemory] = resource.MustParse(requests.MemoryResources) + } + + if requests.GPU { + acceleratorCount, ok := pod.Spec.NodeSelector[NodepoolNumAcceleratorsKey] + if !ok { + return nil, fmt.Errorf("cannot determine number of accelerators that the pod should use, make sure to call ConfigurePodForRuntimeTestNodepool first") + } + resourceList[v13.ResourceName("nvidia.com/gpu")] = resource.MustParse(acceleratorCount) + } + + requirements := v13.ResourceRequirements{ + Limits: resourceList, + Requests: resourceList, + } + + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == containerName { + pod.Spec.Containers[i].Resources = requirements + return pod, nil + } + } + return nil, fmt.Errorf("container %q not found", containerName) +} + +// RuntimeType is a supported runtime for the test nodepool. +type RuntimeType string + +// List of known runtime types. +const ( + RuntimeTypeGVisor = RuntimeType("gvisor") + RuntimeTypeUnsandboxed = RuntimeType("runc") + RuntimeTypeGVisorNvidia = RuntimeType("gvisor-nvidia") + RuntimeTypeGVisorTPU = RuntimeType("gvisor-tpu") + RuntimeTypeUnsandboxedNvidia = RuntimeType("runc-nvidia") + RuntimeTypeUnsandboxedTPU = RuntimeType("runc-tpu") +) + +// ApplyNodepool modifies the nodepool to configure it to use the runtime. +func (t RuntimeType) ApplyNodepool(nodepool *cspb.NodePool, accelType AcceleratorType, accelShape string, accelRes string) { + if nodepool.GetConfig().GetLabels() == nil { + nodepool.GetConfig().Labels = map[string]string{} + } + + switch t { + case RuntimeTypeGVisor: + nodepool.Config.SandboxConfig = &cspb.SandboxConfig{ + Type: cspb.SandboxConfig_GVISOR, + } + nodepool.GetConfig().Labels[NodepoolRuntimeKey] = string(RuntimeTypeGVisor) + case RuntimeTypeUnsandboxed: + nodepool.GetConfig().Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxed) + // Do nothing. + case RuntimeTypeGVisorNvidia: + nodepool.Config.SandboxConfig = &cspb.SandboxConfig{ + Type: cspb.SandboxConfig_GVISOR, + } + accelCount, err := strconv.Atoi(accelShape) + if err != nil { + panic(fmt.Sprintf("GPU count must be a valid number, got %v", accelShape)) + } + if accelCount == 0 { + panic("GPU count needs to be >=1") + } + nodepool.Config.MachineType = DefaultNvidiaMachineType + nodepool.Config.Accelerators = []*cspb.AcceleratorConfig{ + { + AcceleratorType: string(accelType), + AcceleratorCount: int64(accelCount), + }, + } + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeGVisorNvidia) + nodepool.Config.Labels[NodepoolNumAcceleratorsKey] = strconv.Itoa(accelCount) + case RuntimeTypeGVisorTPU: + nodepool.Config.MachineType = TPUAcceleratorMachineTypeMap[accelType] + nodepool.PlacementPolicy = &cspb.NodePool_PlacementPolicy{ + TpuTopology: accelShape, + Type: cspb.NodePool_PlacementPolicy_COMPACT, + } + + nodepool.Config.Labels[gvisorNodepoolKey] = gvisorRuntimeClass + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeGVisorTPU) + nodepool.Config.Labels[NodepoolTPUTopologyKey] = accelShape + nodepool.Config.Taints = append(nodepool.Config.Taints, &cspb.NodeTaint{ + Key: gvisorNodepoolKey, + Value: gvisorRuntimeClass, + Effect: cspb.NodeTaint_NO_SCHEDULE, + }) + case RuntimeTypeUnsandboxedNvidia: + accelCount, err := strconv.Atoi(accelShape) + if err != nil { + panic(fmt.Sprintf("GPU count must be a valid number, got %v", accelShape)) + } + if accelCount == 0 { + panic("GPU count needs to be >=1") + } + nodepool.Config.MachineType = DefaultNvidiaMachineType + nodepool.Config.Accelerators = []*cspb.AcceleratorConfig{ + { + AcceleratorType: string(accelType), + AcceleratorCount: int64(accelCount), + }, + } + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxedNvidia) + nodepool.Config.Labels[NodepoolNumAcceleratorsKey] = strconv.Itoa(accelCount) + case RuntimeTypeUnsandboxedTPU: + nodepool.Config.MachineType = TPUAcceleratorMachineTypeMap[accelType] + nodepool.PlacementPolicy = &cspb.NodePool_PlacementPolicy{ + TpuTopology: accelShape, + Type: cspb.NodePool_PlacementPolicy_COMPACT, + } + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxedTPU) + nodepool.Config.Labels[NodepoolTPUTopologyKey] = accelShape + default: + panic(fmt.Sprintf("unsupported runtime %q", t)) + } + if accelRes != "" { + nodepool.Config.ReservationAffinity = &cspb.ReservationAffinity{ + ConsumeReservationType: cspb.ReservationAffinity_SPECIFIC_RESERVATION, + Key: "compute.googleapis.com/reservation-name", + Values: []string{accelRes}, + } + } +} + +// ApplyPodSpec modifies a PodSpec to use this runtime. +func (t RuntimeType) ApplyPodSpec(podSpec *v13.PodSpec) { + switch t { + case RuntimeTypeGVisor: + podSpec.RuntimeClassName = proto.String(gvisorRuntimeClass) + podSpec.NodeSelector[NodepoolRuntimeKey] = string(RuntimeTypeGVisor) + case RuntimeTypeUnsandboxed: + // Allow the pod to schedule on gVisor nodes as well. + // This enables the use of `--test-nodepool-runtime=runc` to run + // unsandboxed benchmarks on gVisor test clusters. + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Effect: v13.TaintEffectNoSchedule, + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + }) + case RuntimeTypeGVisorNvidia: + podSpec.RuntimeClassName = proto.String(gvisorRuntimeClass) + podSpec.NodeSelector[NodepoolRuntimeKey] = string(RuntimeTypeGVisorNvidia) + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "nvidia.com/gpu", + Operator: v13.TolerationOpExists, + }) + case RuntimeTypeGVisorTPU: + podSpec.RuntimeClassName = proto.String(gvisorRuntimeClass) + podSpec.NodeSelector[NodepoolRuntimeKey] = string(RuntimeTypeGVisorTPU) + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "google.com/tpu", + Operator: v13.TolerationOpExists, + }) + case RuntimeTypeUnsandboxedNvidia: + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "nvidia.com/gpu", + Operator: v13.TolerationOpExists, + }) + // Allow the pod to schedule on gVisor nodes as well. + // This enables the use of `--test-nodepool-runtime=runc-nvidia` to run + // unsandboxed benchmarks on gVisor test clusters. + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Effect: v13.TaintEffectNoSchedule, + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + }) + case RuntimeTypeUnsandboxedTPU: + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: "google.com/tpu", + Operator: v13.TolerationOpExists, + }) + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Effect: v13.TaintEffectNoSchedule, + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + }) + default: + panic(fmt.Sprintf("unsupported runtime %q", t)) + } +} diff --git a/test/kubernetes/testcluster/testcluster.go b/test/kubernetes/testcluster/testcluster.go new file mode 100644 index 000000000..ace25e600 --- /dev/null +++ b/test/kubernetes/testcluster/testcluster.go @@ -0,0 +1,791 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testcluster wraps the Kubernetes library for common test operations. +// It also provides a TestCluster abstraction for interacting with clusters. +package testcluster + +import ( + "context" + "fmt" + "io" + "strconv" + "strings" + "time" + + cspb "cloud.google.com/go/container/apiv1/containerpb" + "golang.org/x/sync/errgroup" + testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto" + appsv1 "k8s.io/api/apps/v1" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + // archKey is given to nodepools to mark their architecture. Used here to mark ARM nodepools. + archKey = "kubernetes.io/arch" + // armValue marks an ARM nodepool. + armValue = "arm64" + + // k8sApp is used as a label to distinguish between applications. + k8sApp = "k8s-app" +) + +// Common namespace names. +const ( + // NamespaceDefault is the default namespace for Kubernetes. + NamespaceDefault = v13.NamespaceDefault + + // NamespaceSanity is used for objects that are part of sanity checks. + NamespaceSanity = "sanity" + + // NamespaceBenchmark is used for objects that are part of benchmarks. + NamespaceBenchmark = "benchmark" +) + +// Nodepool names. +const ( + // TestRuntimeNodepoolName is the value that marks a "test-runtime-nodepool", or a nodepool where + // w/ the runtime under test. + TestRuntimeNodepoolName = "test-runtime-nodepool" + // ClientNodepoolName is the value that marks a client nodepool. Usually this is a plain GKE + // nodepool + ClientNodepoolName = "client-nodepool" + // TertiaryNodepoolName is the value that marks the tertiary nodepool. + // This could either be a plain GKE nodepool or could be gVisor-enabled, + // as configured during test range creation. + TertiaryNodepoolName = "tertiary-nodepool" +) + +// Nodepool keys. +const ( + // NodePoolTypeKey is the key to mark a nodepool as a "test-runtime-nodepool" or a "client-nodepool" + NodePoolTypeKey = "nodepool-type" + // NodepoolRuntimeKey is the key to mark the runtime used by a nodepool. + NodepoolRuntimeKey = "runtime" + // NodepoolNumAcceleratorsKey is the key to mark the number of accelerators in a nodepool. + NodepoolNumAcceleratorsKey = "num-accelerators" + // NodepoolTPUTopologyKey is the key to mark the TPU topology used by a nodepool. + NodepoolTPUTopologyKey = "tpu-topology" + // Name of the nodepool key used in Pod.Spec.NodeSelector. + NodePoolSelectorKey = "cloud.google.com/gke-nodepool" + // Name of the TPU accelerator key used in Pod.Spec.NodeSelector. + NodepoolTPUAcceleratorSelectorKey = "cloud.google.com/gke-tpu-accelerator" + // Name of the TPU topology key used in Pod.Spec.NodeSelector. + NodepoolTPUTopologySelectorKey = "cloud.google.com/gke-tpu-topology" +) + +// Default machine types. +var ( + // DefaultMachineType is the default machine type to use for specs and create-default. + DefaultMachineType = "n2-standard-4" + // DefaultNvidiaMachineType is the default machine type for nvidia. + DefaultNvidiaMachineType = "n1-standard-4" + // TPUAcceleratorMachineTypeMap maps TPU types to the machine type to use. + TPUAcceleratorMachineTypeMap = map[AcceleratorType]string{ + AcceleratorTypeV4PodTPU: "ct4p-hightpu-4t", + } +) + +// GKE Sandbox gVisor runtime. +const ( + // gvisorNodepoolKey the key for the label given to GKE Sandbox nodepools. + gvisorNodepoolKey = "sandbox.gke.io/runtime" + // gvisorRuntimeClass the runtimeClassName used for GKE Sandbox pods. + gvisorRuntimeClass = "gvisor" +) + +// AcceleratorType is the gpu type to be used. +type AcceleratorType string + +// List of supported GPUs. +const ( + AcceleratorTypeTeslaT4GPU = AcceleratorType("nvidia-tesla-t4") + AcceleratorTypeA100GPU = AcceleratorType("nvidia-tesla-a100") + AcceleratorTypeL4GPU = AcceleratorType("nvidia-tesla-l4") + AcceleratorTypeV4PodTPU = AcceleratorType("tpu-v4-pod") +) + +// TestCluster wraps clusters with their individual ClientSets so that helper methods can be called. +type TestCluster struct { + cluster *testpb.Cluster + client kubernetes.Interface + + // testNodepoolRuntimeOverride, if set, overrides the runtime used for pods + // running on the test nodepool. If unset, the test nodepool's default + // runtime is used. + testNodepoolRuntimeOverride RuntimeType +} + +// NewTestCluster returns a new TestCluster client. +func NewTestCluster(cluster *testpb.Cluster) (*TestCluster, error) { + config, err := clientcmd.BuildConfigFromFlags("" /*masterURL*/, cluster.GetCredentialFile()) + if err != nil { + return nil, fmt.Errorf("BuildConfigFromFlags: %w", err) + } + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("kubernetes.NewForConfig: %w", err) + } + return NewTestClusterWithClient(cluster, client), nil +} + +// NewTestClusterWithClient returns a new TestCluster client with a given client. +func NewTestClusterWithClient(cluster *testpb.Cluster, client kubernetes.Interface) *TestCluster { + return &TestCluster{ + cluster: cluster, + client: client, + testNodepoolRuntimeOverride: "", + } +} + +// Cluster returns the underlying cluster proto for tests. +func (t *TestCluster) Cluster() *testpb.Cluster { + return t.cluster +} + +// GetName returns this cluster's name. +func (t *TestCluster) GetName() string { + return t.cluster.GetCluster().GetName() +} + +// GetGVisorRuntimeLabelMap returns the gVisor runtime key-value pair used +// on gVisor-runtime-enabled nodes. +func (t *TestCluster) GetGVisorRuntimeLabelMap() map[string]string { + return map[string]string{ + gvisorNodepoolKey: gvisorRuntimeClass, + } +} + +// GetGVisorRuntimeToleration returns a pod scheduling toleration that +// allows the pod to schedule on gVisor-runtime-enabled nodes. +func (t *TestCluster) GetGVisorRuntimeToleration() v13.Toleration { + return v13.Toleration{ + Key: gvisorNodepoolKey, + Operator: v13.TolerationOpEqual, + Value: gvisorRuntimeClass, + Effect: v13.TaintEffectNoSchedule, + } +} + +// OverrideTestNodepoolRuntime overrides the runtime used for pods running on +// the test nodepool. If unset, the test nodepool's default runtime is used. +func (t *TestCluster) OverrideTestNodepoolRuntime(testRuntime RuntimeType) { + t.testNodepoolRuntimeOverride = testRuntime +} + +// createNamespace creates a namespace. +func (t *TestCluster) createNamespace(ctx context.Context, namespace *v13.Namespace) (*v13.Namespace, error) { + return t.client.CoreV1().Namespaces().Create(ctx, namespace, v1.CreateOptions{}) +} + +// getNamespace returns the given namespace in the cluster if it exists. +func (t *TestCluster) getNamespace(ctx context.Context, namespaceName string) (*v13.Namespace, error) { + return t.client.CoreV1().Namespaces().Get(ctx, namespaceName, v1.GetOptions{}) +} + +// deleteNamespace is a helper method to delete a namespace. +func (t *TestCluster) deleteNamespace(ctx context.Context, namespaceName string) error { + if err := t.client.CoreV1().Namespaces().Delete(ctx, namespaceName, v1.DeleteOptions{}); err != nil { + return err + } + // Wait for the namespace to disappear or for the context to expire. + for ctx.Err() == nil { + if _, err := t.getNamespace(ctx, namespaceName); err != nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + return ctx.Err() +} + +// ListNodes is a helper method to list nodes in a cluster. +func (t *TestCluster) ListNodes(ctx context.Context) (*v13.NodeList, error) { + return t.client.CoreV1().Nodes().List(ctx, v1.ListOptions{}) +} + +// HasGVisorTestRuntime returns whether the test nodes in this cluster +// use the gVisor runtime. +func (t *TestCluster) HasGVisorTestRuntime(ctx context.Context) (bool, error) { + nodes, err := t.ListNodes(ctx) + if err != nil { + return false, fmt.Errorf("cannot list nodes: %w", err) + } + var foundRuntime RuntimeType + for _, n := range nodes.Items { + if n.Labels[NodePoolTypeKey] != TestRuntimeNodepoolName { + continue + } + nodeRuntime := RuntimeType(n.Labels[NodepoolRuntimeKey]) + if nodeRuntime == "" { + return false, fmt.Errorf("node %q has no runtime label", n.GetName()) + } + if foundRuntime == "" { + foundRuntime = nodeRuntime + continue + } + if nodeRuntime != foundRuntime { + return false, fmt.Errorf("found conflicting runtimes in the same cluster: %q vs %q", foundRuntime, nodeRuntime) + } + } + return foundRuntime == RuntimeTypeGVisor || foundRuntime == RuntimeTypeGVisorNvidia, nil +} + +// CreatePod is a helper to create a pod. +func (t *TestCluster) CreatePod(ctx context.Context, pod *v13.Pod) (*v13.Pod, error) { + if pod.GetObjectMeta().GetNamespace() == "" { + pod.SetNamespace(NamespaceDefault) + } + return t.client.CoreV1().Pods(pod.GetNamespace()).Create(ctx, pod, v1.CreateOptions{}) +} + +// GetPod is a helper method to Get a pod's metadata. +func (t *TestCluster) GetPod(ctx context.Context, pod *v13.Pod) (*v13.Pod, error) { + return t.client.CoreV1().Pods(pod.GetNamespace()).Get(ctx, pod.GetName(), v1.GetOptions{}) +} + +// ListPods is a helper method to List pods in a cluster. +func (t *TestCluster) ListPods(ctx context.Context, namespace string) (*v13.PodList, error) { + return t.client.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{}) +} + +// DeletePod is a helper method to delete a pod. +func (t *TestCluster) DeletePod(ctx context.Context, pod *v13.Pod) error { + if err := t.client.CoreV1().Pods(pod.GetNamespace()).Delete(ctx, pod.GetName(), v1.DeleteOptions{}); err != nil { + return err + } + // Wait for the pod to disappear or for the context to expire. + for ctx.Err() == nil { + if _, err := t.GetPod(ctx, pod); err != nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + return ctx.Err() +} + +// GetLogReader gets an io.ReadCloser from which logs can be read. It is the caller's +// responsibility to close it. +func (t *TestCluster) GetLogReader(ctx context.Context, pod *v13.Pod, opts v13.PodLogOptions) (io.ReadCloser, error) { + return t.client.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &opts).Stream(ctx) +} + +// ReadPodLogs reads logs from a pod. +func (t *TestCluster) ReadPodLogs(ctx context.Context, pod *v13.Pod) (string, error) { + rdr, err := t.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + return "", fmt.Errorf("GetLogReader on cluster %q pod %q: %v", t.GetName(), pod.GetName(), err) + } + out, err := io.ReadAll(rdr) + if err != nil { + return "", fmt.Errorf("failed to read from pod %q: %v", pod.GetName(), err) + } + return string(out), nil +} + +// WaitForPodRunning is a helper method to wait for a pod to be running. +func (t *TestCluster) WaitForPodRunning(ctx context.Context, pod *v13.Pod) error { + return t.doWaitForPod(ctx, pod, v13.PodRunning) +} + +// WaitForPodCompleted is a helper method to wait for a pod to be completed. +func (t *TestCluster) WaitForPodCompleted(ctx context.Context, pod *v13.Pod) error { + return t.doWaitForPod(ctx, pod, v13.PodSucceeded) +} + +// doWaitForPod waits for a pod to complete based on a given v13.PodPhase. +func (t *TestCluster) doWaitForPod(ctx context.Context, pod *v13.Pod, phase v13.PodPhase) error { + w, err := t.client.CoreV1().Pods(pod.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: pod.GetName()}).String(), + }) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + + var p *v13.Pod + for { + select { + case <-ctx.Done(): + return ctx.Err() + case e := <-w.ResultChan(): + var ok bool + p, ok = e.Object.(*v13.Pod) + if !ok { + return fmt.Errorf("invalid object watched: %T", p) + } + case <-time.After(10 * time.Second): + p, err = t.GetPod(ctx, pod) + if err != nil { + return fmt.Errorf("failed to poll pod: %w", err) + } + } + if ctx.Err() != nil { + return fmt.Errorf("context expired waiting for pod %q failed: %s", pod.GetName(), ctx.Err()) + } + if pod.Status.Reason == v13.PodReasonUnschedulable { + return fmt.Errorf("pod %q failed: reason: %q message: %q", pod.GetName(), pod.Status.Reason, pod.Status.Message) + } + + for _, c := range p.Status.Conditions { + if strings.Contains(c.Reason, "Unschedulable") { + return fmt.Errorf("pod %q failed: reason: %q message: %q", pod.GetName(), c.Reason, c.Message) + } + } + + switch p.Status.Phase { + case v13.PodFailed: + return fmt.Errorf("pod %q failed: %s", pod.GetName(), p.Status.Message) + case phase: + return nil + } + } +} + +// RuntimeTestNodepoolIsARM returns true if the runtime undertest nodepool is an ARM nodepool. +func (t *TestCluster) RuntimeTestNodepoolIsARM() bool { + np, err := t.getNodePoolByName(TestRuntimeNodepoolName) + if err != nil { + return false + } + return strings.HasPrefix(np.GetConfig().GetMachineType(), "t2a") +} + +// configureDaemonSetForNodepool configures the DaemonSet to run on a given nodepool. +func (t *TestCluster) configureDaemonSetForNodepool(ds *appsv1.DaemonSet, nodepoolName string) error { + np, err := t.getNodePoolByName(nodepoolName) + if err != nil { + return err + } + if ds.Labels == nil { + ds.Labels = make(map[string]string) + } + return t.applyCommonPodConfigurations(np, &ds.Spec.Template.Spec) +} + +// configurePodForNodepool configures the pod to run on a given nodepool. +func (t *TestCluster) configurePodForNodepool(pod *v13.Pod, nodepoolName string) (*v13.Pod, error) { + np, err := t.getNodePoolByName(nodepoolName) + if err != nil { + return nil, err + } + if pod.Labels == nil { + pod.Labels = make(map[string]string) + } + if err := t.applyCommonPodConfigurations(np, &pod.Spec); err != nil { + return nil, err + } + return pod, nil +} + +// ConfigureDaemonSetForRuntimeTestNodepool configures the DaemonSet to run +// on the test runtime. +func (t *TestCluster) ConfigureDaemonSetForRuntimeTestNodepool(ds *appsv1.DaemonSet) error { + return t.configureDaemonSetForNodepool(ds, TestRuntimeNodepoolName) +} + +// ConfigurePodForRuntimeTestNodepool configures the pod to run on the test runtime. +func (t *TestCluster) ConfigurePodForRuntimeTestNodepool(pod *v13.Pod) (*v13.Pod, error) { + return t.configurePodForNodepool(pod, TestRuntimeNodepoolName) +} + +// ConfigurePodForClientNodepool configures the pod to run on the client +// nodepool. +func (t *TestCluster) ConfigurePodForClientNodepool(pod *v13.Pod) (*v13.Pod, error) { + return t.configurePodForNodepool(pod, ClientNodepoolName) +} + +// ConfigurePodForTertiaryNodepool configures the pod to run on the tertiary +// nodepool. +func (t *TestCluster) ConfigurePodForTertiaryNodepool(pod *v13.Pod) (*v13.Pod, error) { + return t.configurePodForNodepool(pod, TertiaryNodepoolName) +} + +func (t *TestCluster) getNodePoolByName(name string) (*cspb.NodePool, error) { + for _, np := range t.cluster.GetCluster().GetNodePools() { + if np.GetName() == name { + return np, nil + } + } + return nil, fmt.Errorf("failed to find nodepool %q: %+v", name, t.cluster.GetCluster().GetNodePools()) +} + +func (t *TestCluster) applyCommonPodConfigurations(np *cspb.NodePool, podSpec *v13.PodSpec) error { + // Apply GKE Sandbox configurations if the nodepool is a GKE Sandbox nodepool. + if podSpec.NodeSelector == nil { + podSpec.NodeSelector = make(map[string]string) + } + + np.GetConfig().GetLabels()[NodePoolTypeKey] = np.GetName() + + // Force the pod to run on this nodepool. + podSpec.NodeSelector[NodePoolSelectorKey] = np.GetName() + + // Figure out which runtime to use for this pod, either by flag override or + // autodetection based on the nodepool configuration. + var applyRuntime = RuntimeTypeUnsandboxed + if np.GetName() == TestRuntimeNodepoolName && t.testNodepoolRuntimeOverride != "" { + applyRuntime = t.testNodepoolRuntimeOverride + } else if nodePoolRuntime, ok := np.GetConfig().GetLabels()[NodepoolRuntimeKey]; ok { + applyRuntime = RuntimeType(nodePoolRuntime) + } + + // Apply the runtime we've chosen, whether by override or autodetection. + applyRuntime.ApplyPodSpec(podSpec) + + // If the nodepool has accelerators, copy the number of them as a node + // selector option. + // This doesn't really constrain the pod further, but allows + // this number to be carried over when setting pod resources. + if len(np.GetConfig().GetAccelerators()) > 0 { + totalAccels := 0 + for _, accelCfg := range np.GetConfig().GetAccelerators() { + totalAccels += int(accelCfg.GetAcceleratorCount()) + } + if accelCount, ok := np.GetConfig().GetLabels()[NodepoolNumAcceleratorsKey]; !ok || accelCount != strconv.Itoa(totalAccels) { + return fmt.Errorf("unexpected %s=%q label on nodepool with %d total accelerators", NodepoolNumAcceleratorsKey, accelCount, totalAccels) + } + podSpec.NodeSelector[NodepoolNumAcceleratorsKey] = strconv.Itoa(totalAccels) + } else { + for accelType, machineType := range TPUAcceleratorMachineTypeMap { + if machineType == np.GetConfig().GetMachineType() { + topology, ok := np.GetConfig().GetLabels()[NodepoolTPUTopologyKey] + if !ok { + return fmt.Errorf("unexpected %s=%q label on nodepool with no accelerators", NodepoolTPUTopologyKey, topology) + } + podSpec.NodeSelector[NodepoolTPUAcceleratorSelectorKey] = string(accelType) + podSpec.NodeSelector[NodepoolTPUTopologySelectorKey] = np.GetConfig().GetLabels()[NodepoolTPUTopologyKey] + } + } + } + + // If the nodepool is an ARM nodepool, apply ARM tolerations. + for key, val := range np.GetConfig().GetLabels() { + if key == archKey && val == armValue { + podSpec.NodeSelector[archKey] = armValue + podSpec.Tolerations = append(podSpec.Tolerations, v13.Toleration{ + Key: archKey, + Value: armValue, + Operator: v13.TolerationOpEqual, + Effect: v13.TaintEffectNoSchedule, + }) + } + } + return nil +} + +// ContainerDurationSecondsByName gets the runtime of a container reported by the kubelet by name. +// The kubelet reports runtime at second granularity. +func (t *TestCluster) ContainerDurationSecondsByName(ctx context.Context, pod *v13.Pod, containerName string) (time.Duration, error) { + p, err := t.GetPod(ctx, pod) + if err != nil { + return 0, fmt.Errorf("GetPod: %w", err) + } + for _, c := range p.Status.ContainerStatuses { + fmt.Println(c.Name) + if c.Name != containerName { + continue + } + if c.State.Terminated == nil { + return 0, fmt.Errorf("failed to get runtime seconds: terminated is nil: %+v", c.State) + } + start := c.State.Terminated.StartedAt + end := c.State.Terminated.FinishedAt + result := end.Unix() - start.Unix() + if result < 0 { + return 0, fmt.Errorf("invalid result %d: %+v", result, c.State) + } + return time.Duration(result) * time.Second, nil + } + return 0, fmt.Errorf("container %q not found: %+v", containerName, pod.Status.ContainerStatuses) +} + +// CreateService is a helper method to create a service in a cluster. +func (t *TestCluster) CreateService(ctx context.Context, service *v13.Service) (*v13.Service, error) { + return t.client.CoreV1().Services(service.GetNamespace()).Create(ctx, service, v1.CreateOptions{}) +} + +// ListServices is a helper method to List services in a cluster. +func (t *TestCluster) ListServices(ctx context.Context, namespace string) (*v13.ServiceList, error) { + return t.client.CoreV1().Services(namespace).List(ctx, v1.ListOptions{}) +} + +// DeleteService is a helper to delete a given service. +func (t *TestCluster) DeleteService(ctx context.Context, service *v13.Service) error { + if err := t.client.CoreV1().Services(service.GetNamespace()).Delete(ctx, service.GetName(), v1.DeleteOptions{}); err != nil { + return err + } + // Wait for the service to disappear or for the context to expire. + for ctx.Err() == nil { + if _, err := t.client.CoreV1().Services(service.GetNamespace()).Get(ctx, service.GetName(), v1.GetOptions{}); err != nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + return ctx.Err() +} + +// WaitForServiceReady waits until a service is ready. +func (t *TestCluster) WaitForServiceReady(ctx context.Context, service *v13.Service) error { + w, err := t.client.CoreV1().Services(service.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: service.GetName()}).String(), + }) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case e := <-w.ResultChan(): + s, ok := e.Object.(*v13.Service) + if !ok { + return fmt.Errorf("invalid object watched: %T", s) + } + if e.Type == watch.Added { + return nil + } + } + } +} + +// GetIPFromService returns the IP on a service. +func GetIPFromService(service *v13.Service) string { + return service.Spec.ClusterIP +} + +// CreatePersistentVolume creates a persistent volume. +func (t *TestCluster) CreatePersistentVolume(ctx context.Context, volume *v13.PersistentVolumeClaim) (*v13.PersistentVolumeClaim, error) { + if volume.GetObjectMeta().GetNamespace() == "" { + volume.SetNamespace(NamespaceDefault) + } + return t.client.CoreV1().PersistentVolumeClaims(volume.GetNamespace()).Create(ctx, volume, v1.CreateOptions{}) +} + +// DeletePersistentVolume deletes a persistent volume. +func (t *TestCluster) DeletePersistentVolume(ctx context.Context, volume *v13.PersistentVolumeClaim) error { + return t.client.CoreV1().PersistentVolumeClaims(volume.GetNamespace()).Delete(ctx, volume.GetName(), v1.DeleteOptions{}) +} + +// CreateDaemonset creates a daemonset with default options. +func (t *TestCluster) CreateDaemonset(ctx context.Context, ds *appsv1.DaemonSet) (*appsv1.DaemonSet, error) { + if ds.GetObjectMeta().GetNamespace() == "" { + ds.SetNamespace(NamespaceDefault) + } + return t.client.AppsV1().DaemonSets(ds.GetNamespace()).Create(ctx, ds, v1.CreateOptions{}) +} + +// DeleteDaemonset deletes a daemonset from this cluster. +func (t *TestCluster) DeleteDaemonset(ctx context.Context, ds *appsv1.DaemonSet) error { + return t.client.AppsV1().DaemonSets(ds.GetNamespace()).Delete(ctx, ds.GetName(), v1.DeleteOptions{}) +} + +// GetPodsInDaemonSet returns the list of pods of the given DaemonSet. +func (t *TestCluster) GetPodsInDaemonSet(ctx context.Context, ds *appsv1.DaemonSet) ([]v13.Pod, error) { + listOptions := v1.ListOptions{} + if appLabel, found := ds.Spec.Template.Labels[k8sApp]; found { + listOptions.LabelSelector = fmt.Sprintf("%s=%s", k8sApp, appLabel) + } + pods, err := t.client.CoreV1().Pods(ds.ObjectMeta.Namespace).List(ctx, listOptions) + if err != nil { + return nil, err + } + var dsPods []v13.Pod + for _, pod := range pods.Items { + if !strings.HasPrefix(pod.Name, ds.ObjectMeta.Name) { + continue // Not part of the DaemonSet. + } + dsPods = append(dsPods, pod) + } + return dsPods, nil +} + +// WaitForDaemonset waits until a daemonset has propagated containers across the affected nodes. +func (t *TestCluster) WaitForDaemonset(ctx context.Context, ds *appsv1.DaemonSet) error { + w, err := t.client.AppsV1().DaemonSets(ds.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: ds.ObjectMeta.Name}).String(), + }) + + if err != nil { + return fmt.Errorf("failed to watch daemon: %v", err) + } + defer w.Stop() + var lastDS *appsv1.DaemonSet + + for daemonSetReady := false; !daemonSetReady; { + select { + case <-ctx.Done(): + if lastDS != nil { + return fmt.Errorf("context canceled before healthy; last DaemonSet status: %#v", lastDS.Status) + } + return fmt.Errorf("context canceled before healthy") + case e, ok := <-w.ResultChan(): + d, ok := e.Object.(*appsv1.DaemonSet) + if !ok { + return fmt.Errorf("invalid object type: %T", d) + } + lastDS = d + if d.Status.NumberReady == d.Status.DesiredNumberScheduled && d.Status.DesiredNumberScheduled > 0 && d.Status.NumberUnavailable == 0 { + daemonSetReady = true + } + } + } + + // Now wait for the pods to be running. + for ctx.Err() == nil { + pods, err := t.GetPodsInDaemonSet(ctx, ds) + if err != nil { + return fmt.Errorf("failed to get pods in daemonset: %v", err) + } + if len(pods) == 0 { + return fmt.Errorf("DaemonSet has no pods: %v", lastDS) + } + allOK := true + for _, pod := range pods { + switch pod.Status.Phase { + case v13.PodRunning, v13.PodSucceeded: + // OK, do nothing. + default: + allOK = false + } + } + if allOK { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } + return nil +} + +// StreamDaemonSetLogs streams the contents of a container from the given +// DaemonSet. The callback function is called once per node that the DaemonSet +// schedules on, with the reader corresponding to that node. The callback +// function is expected to close the reader. +// StreamDaemonSetLogs returns once the DaemonSet is ready everywhere that +// it is meant to be scheduled. +func (t *TestCluster) StreamDaemonSetLogs(ctx context.Context, ds *appsv1.DaemonSet, opts v13.PodLogOptions, fn func(context.Context, v13.Pod, io.ReadCloser) error) error { + errGroup, groupCtx := errgroup.WithContext(ctx) + nodesSeen := make(map[string]struct{}) + nodesErr := make(map[string]error) + + // refreshPods queries all matching pods in the cluster and starts new + // log streams for every pod that schedules on a node we haven't seen yet. + refreshPods := func() error { + pods, err := t.GetPodsInDaemonSet(ctx, ds) + if err != nil { + return err + } + for _, pod := range pods { + pod := pod + if pod.Spec.NodeName == "" { + continue // No node assigned yet. + } + if _, seen := nodesSeen[pod.Spec.NodeName]; seen { + continue // Node already seen. + } + logReader, err := t.client.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &opts).Stream(ctx) + if err != nil { + // This can happen if the container hasn't run yet, for example + // because other init containers that run earlier are still executing. + // We retain this error in `nodesErr` but clear it if it later becomes + // OK for this node. + nodesErr[pod.Spec.NodeName] = fmt.Errorf("failed to stream logs from pod %q/%q on node %q: %v", pod.GetNamespace(), pod.GetName(), pod.Spec.NodeName, err) + continue + } + nodesSeen[pod.Spec.NodeName] = struct{}{} + nodesErr[pod.Spec.NodeName] = nil + errGroup.Go(func() error { + return fn(groupCtx, pod, logReader) + }) + } + return nil + } + + // Watch the DaemonSet. + // We'll periodically refresh pods: either when the DaemonSet changes + // state, or periodically every second. + dsWatch, err := t.client.AppsV1().DaemonSets(ds.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: ds.ObjectMeta.Name}).String(), + }) + if err != nil { + return fmt.Errorf("failed to watch DaemonSet: %v", err) + } + timeTicker := time.NewTicker(time.Second) + defer timeTicker.Stop() + defer dsWatch.Stop() + + // Iterate and stop once the DaemonSet is fully Ready. + var loopError error + var lastDS *appsv1.DaemonSet +Outer: + for { + select { + case <-ctx.Done(): + if lastDS != nil { + loopError = fmt.Errorf("context canceled before healthy; last status: %#v", lastDS.Status) + } else { + loopError = fmt.Errorf("context canceled before healthy") + } + break Outer + case <-timeTicker.C: + if err := refreshPods(); err != nil { + loopError = err + break Outer + } + case e, ok := <-dsWatch.ResultChan(): + d, ok := e.Object.(*appsv1.DaemonSet) + if !ok { + loopError = fmt.Errorf("invalid object type: %T", d) + break Outer + } + lastDS = d + if err := refreshPods(); err != nil { + loopError = err + break Outer + } + if d.Status.NumberReady == d.Status.DesiredNumberScheduled && d.Status.DesiredNumberScheduled > 0 && d.Status.NumberUnavailable == 0 { + break Outer + } + } + } + groupErr := errGroup.Wait() + for _, err := range nodesErr { + if err != nil { + return err + } + } + if loopError != nil { + return loopError + } + return groupErr +} diff --git a/test/kubernetes/tests/BUILD b/test/kubernetes/tests/BUILD new file mode 100644 index 000000000..5f6235a06 --- /dev/null +++ b/test/kubernetes/tests/BUILD @@ -0,0 +1,21 @@ +load("//tools:defs.bzl", "go_test") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_test( + name = "hello_test", + srcs = ["hello_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + deps = [ + "//test/kubernetes/k8sctx", + "//test/kubernetes/testcluster", + "@io_k8s_api//core/v1:go_default_library", + ], +) diff --git a/test/kubernetes/tests/hello_test.go b/test/kubernetes/tests/hello_test.go new file mode 100644 index 000000000..7f4704c97 --- /dev/null +++ b/test/kubernetes/tests/hello_test.go @@ -0,0 +1,78 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hello_test + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + v13 "k8s.io/api/core/v1" +) + +// TestHello tests that a trivial alpine container runs correctly. +func TestHello(t *testing.T) { + ctx := context.Background() + k8sCtx, err := k8sctx.Context(ctx) + if err != nil { + t.Fatalf("Failed to get kubernetes context: %v", err) + } + cluster := k8sCtx.AcquireCluster(ctx, t) + defer k8sCtx.ReleaseCluster(ctx, t, cluster) + + ns := cluster.Namespace(testcluster.NamespaceDefault) + image, err := k8sCtx.ResolveImage(ctx, "alpine") + if err != nil { + t.Fatalf("Failed to resolve image: %v", err) + } + pod := ns.NewAlpinePod(fmt.Sprintf("hello-%d", time.Now().UnixNano()), image, []string{"/bin/sh", "-c", "echo hello"}) + pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod) + if err != nil { + t.Fatalf("Failed to set pod on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + pod, err = cluster.CreatePod(ctx, pod) + if err != nil { + t.Fatalf("Failed to create pod on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer cluster.DeletePod(ctx, pod) + if err := cluster.WaitForPodCompleted(ctx, pod); err != nil { + t.Fatalf("Failed to wait for pod on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + reader, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{}) + if err != nil { + t.Fatalf("Failed to get log reader on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + defer reader.Close() + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, reader); err != nil { + t.Fatalf("Failed to read log on cluster %q: %v", cluster.Cluster().GetCluster().GetName(), err) + } + if strings.TrimSpace(buf.String()) != "hello" { + t.Fatalf("Mistmatch output: got: %q want: %q", buf.String(), "hello") + } +} + +func TestMain(m *testing.M) { + k8sctx.TestMain(m, map[string]k8sctx.TestFunc{ + "TestHello": TestHello, + }) +} diff --git a/tools/gvisor_k8s_tool/cluster/cluster.go b/tools/gvisor_k8s_tool/cluster/cluster.go index 434e5cca7..a7d68d54b 100644 --- a/tools/gvisor_k8s_tool/cluster/cluster.go +++ b/tools/gvisor_k8s_tool/cluster/cluster.go @@ -45,6 +45,11 @@ func New(config *rest.Config) (*Cluster, error) { return &Cluster{clientSet}, nil } +// Client returns the underlying Kubernetes client. +func (c *Cluster) Client() kubernetes.Interface { + return c.client +} + // CreateDaemonset creates a daemonset with default options. func (c *Cluster) CreateDaemonset(ctx context.Context, ds *appsv1.DaemonSet) (*appsv1.DaemonSet, error) { if ds.GetObjectMeta().GetNamespace() == "" {