From 6abc135b3b8f80f2423a168d365e26961650f38d Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Wed, 4 Dec 2024 17:00:42 -0800 Subject: [PATCH] Kubernetes benchmarks: Create "benchmark" namespace prior to using it. Prior to this change, in certain benchmarks, the profiling steps (which require the existence of the `benchmark` namespace) happened before the `benchmark` namespace is ever created. This would fail, unless prior iterations of *other* benchmarks in the same cluster had already created the `benchmark` namespace. So when running broken benchmarks in their own individual clusters, they would consistently fail due to the lack of the `benchmark` namespace. Now all benchmarks consistently create this namespace first. (Diffbased) PiperOrigin-RevId: 702909244 --- test/kubernetes/benchmarks/pytorch.go | 8 ++++---- test/kubernetes/benchmarks/startup.go | 8 ++++---- test/kubernetes/benchmarks/tensorflow.go | 8 ++++---- test/kubernetes/benchmarks/wordpress.go | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/kubernetes/benchmarks/pytorch.go b/test/kubernetes/benchmarks/pytorch.go index c53f49e4b..f85cea64b 100644 --- a/test/kubernetes/benchmarks/pytorch.go +++ b/test/kubernetes/benchmarks/pytorch.go @@ -257,15 +257,15 @@ func RunPytorch(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesConte // doPytorchRun runs a single PyTorch test. func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) { benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("Failed to reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) } defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("Failed to reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) image, err := k8sCtx.ResolveImage(ctx, pytorchImage) if err != nil { diff --git a/test/kubernetes/benchmarks/startup.go b/test/kubernetes/benchmarks/startup.go index a57ca9126..fcc50a171 100644 --- a/test/kubernetes/benchmarks/startup.go +++ b/test/kubernetes/benchmarks/startup.go @@ -48,15 +48,15 @@ func MeasureStartup(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC t.Logf("Warning: This is not a meaningful benchmark. Read the comments.") benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) } defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) podName := "startup" image, err := k8sCtx.ResolveImage(ctx, "alpine") diff --git a/test/kubernetes/benchmarks/tensorflow.go b/test/kubernetes/benchmarks/tensorflow.go index da8e4f869..57b232863 100644 --- a/test/kubernetes/benchmarks/tensorflow.go +++ b/test/kubernetes/benchmarks/tensorflow.go @@ -48,15 +48,15 @@ var workloads = map[string]string{ // RunTensorflowOnCPU runs the Tensorflow example workloads on CPU. func RunTensorflowOnCPU(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) } defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) const name = "tensorflow" recorder, err := benchmetric.GetRecorder(ctx) diff --git a/test/kubernetes/benchmarks/wordpress.go b/test/kubernetes/benchmarks/wordpress.go index b515ffbce..781514300 100644 --- a/test/kubernetes/benchmarks/wordpress.go +++ b/test/kubernetes/benchmarks/wordpress.go @@ -60,15 +60,15 @@ var ( // BenchmarkWordpress runs a benchmark of WordPress performance. func BenchmarkWordpress(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) { benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark) + if err := benchmarkNS.Reset(ctx); err != nil { + t.Fatalf("cannot reset namespace: %v", err) + } + defer benchmarkNS.Cleanup(ctx) endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) } defer endProfiling() - if err := benchmarkNS.Reset(ctx); err != nil { - t.Fatalf("cannot reset namespace: %v", err) - } - defer benchmarkNS.Cleanup(ctx) // Create a persistent volume on which to store the database data. dbVolume := benchmarkNS.GetPersistentVolume(mariaDBVolumeName, "30Gi")