netbird/gvisor
0
0
Fork 0
mirror of https://github.com/netbirdio/gvisor.git synced 2026-05-22 17:12:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

Kubernetes tests: Librarify all the tests and benchmarks.

Browse Source 
These benchmarks are used as libraries within Google to run continuous
benchmarks.

PiperOrigin-RevId: 700146722
This commit is contained in:
Etienne Perot
2024-11-25 17:12:46 -08:00
committed by gVisor bot
parent 2267c24a41
commit 8079a6cb03
32 changed files with 4359 additions and 3817 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/kubernetes/benchmarks/BUILD
+220 -66
View File
@@ -1,4 +1,4 @@
load("//tools:defs.bzl", "go_test", "pkg_tar")
load("//tools:defs.bzl", "go_library", "go_test")
package(
default_applicable_licenses = ["//:license"],
@@ -30,28 +30,11 @@ filegroup(
srcs = _ALL_BENCHMARK_TARGETS,
)
[pkg_tar(
name = "%s_tar" % (src[src.index(":") + 1:],),
go_library(
name = "abslbuild",
testonly = True,
srcs = [src],
extension = "tar.bz2",
) for src in _ALL_BENCHMARK_TARGETS]
filegroup(
name = "all_benchmark_test_binaries_tar",
testonly = True,
srcs = ["%s_tar" % (src[src.index(":") + 1:],) for src in _ALL_BENCHMARK_TARGETS],
)
go_test(
name = "abslbuild_test",
srcs = ["abslbuild_test.go"],
srcs = ["abslbuild.go"],
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
@@ -64,14 +47,26 @@ go_test(
)
go_test(
name = "startup_test",
srcs = ["startup_test.go"],
name = "abslbuild_test",
srcs = ["abslbuild_test.go"],
library = ":abslbuild",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "startup",
testonly = True,
srcs = ["startup.go"],
nogo = False,
deps = [
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
@@ -82,14 +77,26 @@ go_test(
)
go_test(
name = "redis_test",
srcs = ["redis_test.go"],
name = "startup_test",
srcs = ["startup_test.go"],
library = ":startup",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "redis",
testonly = True,
srcs = ["redis.go"],
nogo = False,
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
@@ -102,8 +109,9 @@ go_test(
)
go_test(
name = "ruby_dev_test",
srcs = ["ruby_dev_test.go"],
name = "redis_test",
srcs = ["redis_test.go"],
library = ":redis",
nogo = False,
tags = [
"local",
@@ -111,7 +119,17 @@ go_test(
"notap",
],
deps = [
"//test/benchmarks/tools",
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "ffmpeg",
testonly = True,
srcs = ["ffmpeg.go"],
nogo = False,
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
@@ -125,12 +143,24 @@ go_test(
go_test(
name = "ffmpeg_test",
srcs = ["ffmpeg_test.go"],
library = ":ffmpeg",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "grpc",
testonly = True,
srcs = ["grpc.go"],
nogo = False,
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
@@ -145,6 +175,7 @@ go_test(
go_test(
name = "grpc_test",
srcs = ["grpc_test.go"],
library = ":grpc",
nogo = False,
tags = [
"local",
@@ -152,25 +183,16 @@ go_test(
"notap",
],
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
],
)
go_test(
name = "nginx_test",
srcs = ["nginx_test.go"],
go_library(
name = "nginx",
testonly = True,
srcs = ["nginx.go"],
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/httpbench",
@@ -183,9 +205,41 @@ go_test(
],
)
go_test(
name = "nginx_test",
srcs = ["nginx_test.go"],
library = ":nginx",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "postgresql",
testonly = True,
srcs = ["postgresql.go"],
nogo = False,
deps = [
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apimachinery//pkg/util/intstr:go_default_library",
],
)
go_test(
name = "postgresql_test",
srcs = ["postgresql_test.go"],
library = ":postgresql",
nogo = False,
tags = [
"local",
@@ -193,24 +247,16 @@ go_test(
"notap",
],
deps = [
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apimachinery//pkg/util/intstr:go_default_library",
],
)
go_test(
name = "tensorflow_test",
srcs = ["tensorflow_test.go"],
go_library(
name = "tensorflow",
testonly = True,
srcs = ["tensorflow.go"],
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
@@ -223,14 +269,26 @@ go_test(
)
go_test(
name = "wordpress_test",
srcs = ["wordpress_test.go"],
name = "tensorflow_test",
srcs = ["tensorflow_test.go"],
library = ":tensorflow",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "wordpress",
testonly = True,
srcs = ["wordpress.go"],
nogo = False,
deps = [
"//test/kubernetes/benchmarks/httpbench",
"//test/kubernetes/benchmarks/profiling",
@@ -243,14 +301,26 @@ go_test(
)
go_test(
name = "pytorch_test",
srcs = ["pytorch_test.go"],
name = "wordpress_test",
srcs = ["wordpress_test.go"],
library = ":wordpress",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "pytorch",
testonly = True,
srcs = ["pytorch.go"],
nogo = False,
deps = [
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
@@ -262,17 +332,29 @@ go_test(
)
go_test(
name = "ollama_test",
srcs = ["ollama_test.go"],
embedsrcs = [
"//test/kubernetes/benchmarks/resources:files", # keep
],
name = "pytorch_test",
srcs = ["pytorch_test.go"],
library = ":pytorch",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "ollama",
testonly = True,
srcs = ["ollama.go"],
embedsrcs = [
"//test/kubernetes/benchmarks/resources:files", # keep
],
nogo = False,
deps = [
"//test/gpu/ollama",
"//test/kubernetes",
@@ -287,14 +369,26 @@ go_test(
)
go_test(
name = "stablediffusion_test",
srcs = ["stablediffusion_test.go"],
name = "ollama_test",
srcs = ["ollama_test.go"],
library = ":ollama",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "stablediffusion",
testonly = True,
srcs = ["stablediffusion.go"],
nogo = False,
deps = [
"//test/gpu/stablediffusion",
"//test/kubernetes",
@@ -307,14 +401,26 @@ go_test(
)
go_test(
name = "gsutil_test",
srcs = ["gsutil_test.go"],
name = "stablediffusion_test",
srcs = ["stablediffusion_test.go"],
library = ":stablediffusion",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "gsutil",
testonly = True,
srcs = ["gsutil.go"],
nogo = False,
deps = [
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
@@ -324,3 +430,51 @@ go_test(
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
],
)
go_test(
name = "gsutil_test",
srcs = ["gsutil_test.go"],
library = ":gsutil",
nogo = False,
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
go_library(
name = "rubydev",
testonly = True,
srcs = ["rubydev.go"],
nogo = False,
deps = [
"//test/benchmarks/tools",
"//test/kubernetes",
"//test/kubernetes/benchmarks/profiling",
"//test/kubernetes/benchmetric",
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
],
)
go_test(
name = "rubydev_test",
srcs = ["rubydev_test.go"],
library = ":rubydev",
tags = [
"local",
"noguitar",
"notap",
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
],
)
test/kubernetes/benchmarks/abslbuild.go
+193
View File
@@ -0,0 +1,193 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package abslbuild
import (
"context"
"fmt"
"path"
"strings"
"testing"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
imageAMD = k8s.ImageRepoPrefix + "benchmarks/absl_x86_64:latest"
)
// BuildABSL runs the ABSL build benchmark.
// This benchmark builds the ABSL library from source.
func BuildABSL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "absl"
persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
t.Skipf("Building ABSL is not supported on ARM")
return
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
pod := newABSLPod(benchmarkNS, name, image, test.volume)
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
if err != nil {
t.Fatalf("Failed to set pod for test runtime: %v", err)
}
pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
pod, err = cluster.CreatePod(ctx, pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, pod)
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
if err := recorder.Record(ctx, fmt.Sprintf("ABSL/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
func newABSLPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
const workdir = "/workdir"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{
"mkdir", "-p", workdir,
"&&",
"cp", "-r", "/abseil-cpp", fmt.Sprintf("%s/.", workdir),
}, " "),
}
command := []string{
"bazel",
"build",
"//absl/base",
"//absl/algorithm",
"//absl/container/...",
"//absl/debugging/...",
"//absl/flags:flag",
"//absl/hash",
"//absl/memory",
"//absl/meta:type_traits",
"//absl/numeric:int128",
"//absl/strings",
"//absl/synchronization",
"//absl/time",
"//absl/types/...",
"//absl/utility",
}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: workdir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: name,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "abseil-cpp"), command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
test/kubernetes/benchmarks/abslbuild_test.go
+2 -170
View File
@@ -12,26 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package abslbuild_test
package abslbuild
import (
"context"
"fmt"
"path"
"strings"
"testing"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
imageAMD = k8s.ImageRepoPrefix + "benchmarks/absl_x86_64:latest"
)
// TestABSLBuild benchmarks building various Abseil C++ targets.
@@ -44,167 +32,11 @@ func TestABSLBuild(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("ABSL", func(t *testing.T) {
t.Parallel()
doABSLBuild(ctx, t, k8sCtx, cluster)
BuildABSL(ctx, t, k8sCtx, cluster)
})
})
}
func doABSLBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "absl"
persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
t.Skipf("Building ABSL is not supported on ARM")
return
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
pod := newABSLPod(benchmarkNS, name, image, test.volume)
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
if err != nil {
t.Fatalf("Failed to set pod for test runtime: %v", err)
}
pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
pod, err = cluster.CreatePod(ctx, pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, pod)
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
if err := recorder.Record(ctx, fmt.Sprintf("ABSL/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
func newABSLPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
const workdir = "/workdir"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{
"mkdir", "-p", workdir,
"&&",
"cp", "-r", "/abseil-cpp", fmt.Sprintf("%s/.", workdir),
}, " "),
}
command := []string{
"bazel",
"build",
"//absl/base",
"//absl/algorithm",
"//absl/container/...",
"//absl/debugging/...",
"//absl/flags:flag",
"//absl/hash",
"//absl/memory",
"//absl/meta:type_traits",
"//absl/numeric:int128",
"//absl/strings",
"//absl/synchronization",
"//absl/time",
"//absl/types/...",
"//absl/utility",
}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: workdir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: name,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "abseil-cpp"), command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestABSLBuild": TestABSLBuild,
test/kubernetes/benchmarks/ffmpeg.go
+181
View File
@@ -0,0 +1,181 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ffmpeg
import (
"context"
"fmt"
"strings"
"testing"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
ffmpegContainerName = "ffmpeg"
imageAMD = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_x86_64:latest"
imageARM = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_aarch64:latest"
)
// RunFFMPEG runs the ffmpeg benchmark.
func RunFFMPEG(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "ffmpeg"
// create persistent volume
persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = imageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
p := newFfmpegDevPod(benchmarkNS, name, image, test.volume)
p, err = cluster.ConfigurePodForRuntimeTestNodepool(p)
if err != nil {
t.Fatalf("Failed to configure pod for runtime: %v", err)
}
p, err = testcluster.MaybeSetContainerResources(p, ffmpegContainerName, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
p, err = cluster.CreatePod(ctx, p)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, p)
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, ffmpegContainerName)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
if recorder.Record(ctx, fmt.Sprintf("FFMPEG/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
// newFfmpegPod creates a new ffmpeg dev pod spec for benchmarks.
func newFfmpegDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
const workdir = "/workdir"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{
"mkdir", "-p", workdir,
"&&",
"cp", "/media/video.mp4", fmt.Sprintf("%s/.", workdir),
}, " "),
}
command := []string{
"ffmpeg",
"-i", "video.mp4",
"-c:v", "libx264",
"-preset", "veryslow",
"output.mp4",
}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: workdir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: ffmpegContainerName,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, workdir, command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
test/kubernetes/benchmarks/ffmpeg_test.go
+2 -159
View File
@@ -12,27 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package ffmpeg_test
package ffmpeg
import (
"context"
"fmt"
"strings"
"testing"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
ffmpegContainerName = "ffmpeg"
imageAMD = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_x86_64:latest"
imageARM = k8s.ImageRepoPrefix + "benchmarks/ffmpeg_aarch64:latest"
)
func TestFfmpeg(t *testing.T) {
@@ -44,155 +31,11 @@ func TestFfmpeg(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("ffmpeg", func(t *testing.T) {
t.Parallel()
doFfmpegTest(ctx, t, k8sCtx, cluster)
RunFFMPEG(ctx, t, k8sCtx, cluster)
})
})
}
func doFfmpegTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "ffmpeg"
// create persistent volume
persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = imageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
p := newFfmpegDevPod(benchmarkNS, name, image, test.volume)
p, err = cluster.ConfigurePodForRuntimeTestNodepool(p)
if err != nil {
t.Fatalf("Failed to configure pod for runtime: %v", err)
}
p, err = testcluster.MaybeSetContainerResources(p, ffmpegContainerName, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
p, err = cluster.CreatePod(ctx, p)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, p)
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, ffmpegContainerName)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
if recorder.Record(ctx, fmt.Sprintf("FFMPEG/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
// newFfmpegPod creates a new ffmpeg dev pod spec for benchmarks.
func newFfmpegDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
const workdir = "/workdir"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{
"mkdir", "-p", workdir,
"&&",
"cp", "/media/video.mp4", fmt.Sprintf("%s/.", workdir),
}, " "),
}
command := []string{
"ffmpeg",
"-i", "video.mp4",
"-c:v", "libx264",
"-preset", "veryslow",
"output.mp4",
}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: workdir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: ffmpegContainerName,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, workdir, command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestFfmpeg": TestFfmpeg,
test/kubernetes/benchmarks/grpc.go
+175
View File
@@ -0,0 +1,175 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package grpc
import (
"context"
"fmt"
"path"
"strings"
"testing"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
imageAMD = k8s.ImageRepoPrefix + "benchmarks/build-grpc_x86_64:latest"
imageARM = k8s.ImageRepoPrefix + "benchmarks/build-grpc_aarch64:latest"
)
// BuildGRPC runs the GRPC benchmark.
// This benchmark builds the gRPC library using bazel.
func BuildGRPC(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "grpc"
persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = imageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
pod := newGRPCPod(benchmarkNS, name, image, test.volume)
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
if err != nil {
t.Fatalf("Failed to set pod for test runtime: %v", err)
}
pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
pod, err = cluster.CreatePod(ctx, pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, pod)
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
if err := recorder.Record(ctx, fmt.Sprintf("gRPC/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
func newGRPCPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
const workdir = "/workdir"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{
"mkdir", "-p", workdir,
"&&",
"cp", "-r", "/grpc", fmt.Sprintf("%s/.", workdir),
}, " "),
}
command := []string{"bazel", "build", ":grpc"}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: workdir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: name,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "grpc"), command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
test/kubernetes/benchmarks/grpc_test.go
+2 -152
View File
@@ -12,27 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package grpc_test
package grpc
import (
"context"
"fmt"
"path"
"strings"
"testing"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
imageAMD = k8s.ImageRepoPrefix + "benchmarks/build-grpc_x86_64:latest"
imageARM = k8s.ImageRepoPrefix + "benchmarks/build-grpc_aarch64:latest"
)
func TestGRPCBuild(t *testing.T) {
@@ -44,148 +31,11 @@ func TestGRPCBuild(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("gRPC", func(t *testing.T) {
t.Parallel()
doGRPCBuild(ctx, t, k8sCtx, cluster)
BuildGRPC(ctx, t, k8sCtx, cluster)
})
})
}
func doGRPCBuild(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "grpc"
persistentVol := benchmarkNS.GetPersistentVolume(name, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = imageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
pod := newGRPCPod(benchmarkNS, name, image, test.volume)
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
if err != nil {
t.Fatalf("Failed to set pod for test runtime: %v", err)
}
pod, err = testcluster.MaybeSetContainerResources(pod, name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
pod, err = cluster.CreatePod(ctx, pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, pod)
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, pod, name)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
if err := recorder.Record(ctx, fmt.Sprintf("gRPC/%s", test.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
func newGRPCPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
const workdir = "/workdir"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{
"mkdir", "-p", workdir,
"&&",
"cp", "-r", "/grpc", fmt.Sprintf("%s/.", workdir),
}, " "),
}
command := []string{"bazel", "build", ":grpc"}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: workdir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: name,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, path.Join(workdir, "grpc"), command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestGRPCBuild": TestGRPCBuild,
test/kubernetes/benchmarks/gsutil.go
+200
View File
@@ -0,0 +1,200 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gsutil is used to benchmark the speed of large (10GB)
// downloads. It is intended for comparing runsc with runc.
package gsutil
import (
"context"
"fmt"
"path/filepath"
"strings"
"testing"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
imageAMD = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_x86_64:7eba9c02d11172d4"
imageARM = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_aarch64:7eba9c02d11172d4"
bigfile = "gs://gvisor-benchmark-testdata/bigrandomfile"
containerName = "gsutil"
)
// RunGSUtil runs a series of gsutil speed benchmarks.
func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "gsutil"
// Create persistent volume.
persistentVol := benchmarkNS.GetPersistentVolume(name, "15Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = imageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
// Run tests with different volume types.
// TODO(b/361182379): Use gsutil parallel sliced downloads as a test
// dimension.
for _, storage := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(storage.name, func(t *testing.T) {
for _, slicing := range []struct {
name string
option string
}{
{
name: "slicing=false",
option: `-o "GSUtil:sliced_object_download_threshold=0"`,
},
{
// Slicing is enabled by default, so we
// don't set any extra options.
name: "slicing=true",
},
} {
t.Run(slicing.name, func(t *testing.T) {
// Setup profiling if requested by the user.
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
// Create a pod that performs setup, then times
// downloading.
p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option)
p, err = cluster.ConfigurePodForRuntimeTestNodepool(p)
if err != nil {
t.Fatalf("Failed to configure pod for runtime: %v", err)
}
p, err = testcluster.MaybeSetContainerResources(p, containerName, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
// GetTimedContainerDuration waits for the container to
// finish.
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, containerName)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
if err := recorder.Record(ctx, fmt.Sprintf("GSUtil/%s/%s", storage.name, slicing.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
})
}
}
// newGSUtilPod creates a new gsutil dev pod spec for benchmarks.
func newGSUtilDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume, gsutilFlags string) *v13.Pod {
const downloadDir = "/downloads"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{"mkdir", "-p", downloadDir}, " "),
}
command := []string{
"sh", "-c",
fmt.Sprintf("gsutil %s cp %s %s && sync",
gsutilFlags,
bigfile,
filepath.Join(downloadDir, "randombigfile"),
),
}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: downloadDir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: containerName,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, "", command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
test/kubernetes/benchmarks/gsutil_test.go
+3 -177
View File
@@ -12,30 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// package gsutil_test is used to benchmark the speed of large (10GB)
// package gsutil is used to benchmark the speed of large (10GB)
// downloads. It is intended for comparing runsc with runc.
package gsutil_test
package gsutil
import (
"context"
"fmt"
"path/filepath"
"strings"
"testing"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
imageAMD = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_x86_64:7eba9c02d11172d4"
imageARM = "us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images/benchmarks/gsutil_aarch64:7eba9c02d11172d4"
bigfile = "gs://gvisor-benchmark-testdata/bigrandomfile"
containerName = "gsutil"
)
func TestGSUtil(t *testing.T) {
@@ -47,171 +33,11 @@ func TestGSUtil(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("GSUtil", func(t *testing.T) {
t.Parallel()
doGSUtilTest(ctx, t, k8sCtx, cluster)
RunGSUtil(ctx, t, k8sCtx, cluster)
})
})
}
func doGSUtilTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
const name = "gsutil"
// Create persistent volume.
persistentVol := benchmarkNS.GetPersistentVolume(name, "15Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := imageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = imageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
// Run tests with different volume types.
// TODO(b/361182379): Use gsutil parallel sliced downloads as a test
// dimension.
for _, storage := range []struct {
name string
volume *v13.Volume
}{
{
name: "RootFS",
volume: nil,
},
{
name: "EmptyDir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(storage.name, func(t *testing.T) {
for _, slicing := range []struct {
name string
option string
}{
{
name: "slicing=false",
option: `-o "GSUtil:sliced_object_download_threshold=0"`,
},
{
// Slicing is enabled by default, so we
// don't set any extra options.
name: "slicing=true",
},
} {
t.Run(slicing.name, func(t *testing.T) {
// Setup profiling if requested by the user.
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
// Create a pod that performs setup, then times
// downloading.
p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option)
p, err = cluster.ConfigurePodForRuntimeTestNodepool(p)
if err != nil {
t.Fatalf("Failed to configure pod for runtime: %v", err)
}
p, err = testcluster.MaybeSetContainerResources(p, containerName, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
// GetTimedContainerDuration waits for the container to
// finish.
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
containerDuration, err := benchmetric.GetTimedContainerDuration(ctx, cluster, p, containerName)
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
if err := recorder.Record(ctx, fmt.Sprintf("GSUtil/%s/%s", storage.name, slicing.name), benchmetric.BenchmarkDuration(containerDuration)); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
})
}
}
// newGSUtilPod creates a new gsutil dev pod spec for benchmarks.
func newGSUtilDevPod(namespace *testcluster.Namespace, name, image string, volume *v13.Volume, gsutilFlags string) *v13.Pod {
const downloadDir = "/downloads"
initCommand := []string{
"sh",
"-c",
strings.Join([]string{"mkdir", "-p", downloadDir}, " "),
}
command := []string{
"sh", "-c",
fmt.Sprintf("gsutil %s cp %s %s && sync",
gsutilFlags,
bigfile,
filepath.Join(downloadDir, "randombigfile"),
),
}
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: downloadDir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
},
Spec: v13.PodSpec{
Volumes: volumes,
Containers: []v13.Container{
{
Name: containerName,
Image: image,
Command: benchmetric.CommandThenTimed(initCommand, "", command),
VolumeMounts: volumeMounts,
},
},
RestartPolicy: v13.RestartPolicyNever,
},
}
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestGSUtil": TestGSUtil,
test/kubernetes/benchmarks/nginx.go
+291
View File
@@ -0,0 +1,291 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nginx
import (
"context"
"fmt"
"strings"
"testing"
"time"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
nginxPort = 80
nginxBenchmarkDuration = 70 * time.Second
nginxRequestTimeout = 3 * time.Second
nginxServingDir = "/tmp/html"
nginxServerLabelKey = "app.kubernetes.io/name"
nginxServerLabelValue = "nginx-server"
nginxImageAMD = k8s.ImageRepoPrefix + "benchmarks/nginx_x86_64:latest"
nginxImageARM = k8s.ImageRepoPrefix + "benchmarks/nginx_aarch64:latest"
)
var (
// nginxCommand is the main server command.
// The test expects that it contains the files to be served at /local,
// and will serve files out of `nginxServingDir`.
nginxCommand = []string{"nginx", "-c", "/etc/nginx/nginx.conf"}
nginxDocKibibytes = []int{1, 10, 100, 10240}
threads = []int{1, 8, 64, 1000}
targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS}
wantPercentiles = []int{50, 95, 99}
)
// BenchmarkNginx runs a series of benchmarks against an nginx server.
func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
nginxImage := nginxImageAMD
if cluster.RuntimeTestNodepoolIsARM() {
nginxImage = nginxImageARM
}
nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage)
if err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi"))
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
for _, test := range []struct {
// Name of the test.
name string
// Suffix for pod names, must be short enough.
suffix string
// Volume to mount at /tmp/root.
volume *v13.Volume
}{
{
name: "RootFS",
suffix: "rootfs",
volume: nil,
},
{
name: "EmptyDir",
suffix: "emdir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
suffix: "pvol",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
name := fmt.Sprintf("nginx-%s", test.suffix)
server := newNginxServer(benchmarkNS, name, nginxImage, test.volume)
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
if err != nil {
t.Fatalf("Failed to configure pod for runtime nodepool: %v", err)
}
server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
server, err = cluster.CreatePod(ctx, server)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, server)
if err := cluster.WaitForPodRunning(ctx, server); err != nil {
t.Fatalf("Failed to wait for pod: %v", err)
}
service := newNginxService(benchmarkNS, name)
service, err = cluster.CreateService(ctx, service)
if err != nil {
t.Fatalf("Failed to create service: %v", err)
}
defer cluster.DeleteService(ctx, service)
var rounds []httpbench.Round
for _, numThreads := range threads {
for _, qps := range targetQPS {
if qps < numThreads {
continue
}
var onlyReport []httpbench.MetricType
// If we're testing at max QPS, only report throughput,
// because all requests will time out.
// Otherwise, only report latency, because the throughput
// is exactly determined by the QPS target anyway.
if qps == httpbench.InfiniteQPS {
onlyReport = append(onlyReport, httpbench.RequestsPerSecond)
onlyReport = append(onlyReport, httpbench.BytesPerSecond)
} else {
onlyReport = append(onlyReport, httpbench.Latency)
}
rounds = append(rounds, httpbench.Round{
NumThreads: numThreads,
TargetQPS: qps,
Duration: nginxBenchmarkDuration,
OnlyReport: onlyReport,
})
}
}
t.Run("0KiB", func(t *testing.T) {
benchmark := &httpbench.HTTPBenchmark{
Name: fmt.Sprintf("nginx/%s/0KiB", test.name),
Cluster: cluster,
Namespace: benchmarkNS,
Service: service,
Port: nginxPort,
Path: "/index.html",
Rounds: rounds,
Timeout: nginxRequestTimeout,
WantPercentiles: wantPercentiles,
}
benchmark.Run(ctx, t)
})
for _, docKibibytes := range nginxDocKibibytes {
t.Run(fmt.Sprintf("%dKiB", docKibibytes), func(t *testing.T) {
benchmark := &httpbench.HTTPBenchmark{
Name: fmt.Sprintf("nginx/%s/%dKiB", test.name, docKibibytes),
Cluster: cluster,
Namespace: benchmarkNS,
Service: service,
Port: nginxPort,
Path: fmt.Sprintf("/latin%dk.txt", docKibibytes),
Rounds: rounds,
Timeout: nginxRequestTimeout,
WantPercentiles: wantPercentiles,
}
benchmark.Run(ctx, t)
})
}
t.Run("HTTP404", func(t *testing.T) {
benchmark := &httpbench.HTTPBenchmark{
Name: fmt.Sprintf("nginx/%s/HTTP404", test.name),
Cluster: cluster,
Namespace: benchmarkNS,
Service: service,
Port: nginxPort,
Path: "/404-this-page-does-not-exist.html",
Rounds: rounds,
Timeout: nginxRequestTimeout,
WantPercentiles: wantPercentiles,
}
benchmark.Run(ctx, t)
})
})
if t.Failed() {
break
}
}
}
func newNginxServer(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: nginxServingDir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
Labels: map[string]string{nginxServerLabelKey: nginxServerLabelValue},
},
Spec: v13.PodSpec{
Containers: []v13.Container{
{
Name: name,
Image: image,
Command: []string{
"sh",
"-c",
strings.Join([]string{
strings.Join([]string{"mkdir", "-p", nginxServingDir}, " "),
strings.Join([]string{
"cp", "-r", "/local/*", fmt.Sprintf("%s/.", nginxServingDir),
}, " "),
strings.Join(nginxCommand, " "),
}, " && "),
},
VolumeMounts: volumeMounts,
Ports: []v13.ContainerPort{
{
Name: name,
ContainerPort: nginxPort,
},
},
},
},
Volumes: volumes,
RestartPolicy: v13.RestartPolicyNever,
},
}
}
func newNginxService(namespace *testcluster.Namespace, name string) *v13.Service {
return namespace.GetService(name, v13.ServiceSpec{
Selector: map[string]string{nginxServerLabelKey: nginxServerLabelValue},
Ports: []v13.ServicePort{
{
Name: name,
Protocol: v13.ProtocolTCP,
Port: nginxPort,
TargetPort: intstr.FromString(name),
},
},
})
}
test/kubernetes/benchmarks/nginx_test.go
+2 -269
View File
@@ -12,46 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package nginx_test
package nginx
import (
"context"
"fmt"
"strings"
"testing"
"time"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/httpbench"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
nginxPort = 80
nginxBenchmarkDuration = 70 * time.Second
nginxRequestTimeout = 3 * time.Second
nginxServingDir = "/tmp/html"
nginxServerLabelKey = "app.kubernetes.io/name"
nginxServerLabelValue = "nginx-server"
nginxImageAMD = k8s.ImageRepoPrefix + "benchmarks/nginx_x86_64:latest"
nginxImageARM = k8s.ImageRepoPrefix + "benchmarks/nginx_aarch64:latest"
)
var (
// nginxCommand is the main server command.
// The test expects that it contains the files to be served at /local,
// and will serve files out of `nginxServingDir`.
nginxCommand = []string{"nginx", "-c", "/etc/nginx/nginx.conf"}
nginxDocKibibytes = []int{1, 10, 100, 10240}
threads = []int{1, 8, 64, 1000}
targetQPS = []int{1, 8, 64, httpbench.InfiniteQPS}
wantPercentiles = []int{50, 95, 99}
)
func TestNginx(t *testing.T) {
@@ -63,246 +31,11 @@ func TestNginx(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("nginx", func(t *testing.T) {
t.Parallel()
doNginxTest(ctx, t, k8sCtx, cluster)
BenchmarkNginx(ctx, t, k8sCtx, cluster)
})
})
}
func doNginxTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
nginxImage := nginxImageAMD
if cluster.RuntimeTestNodepoolIsARM() {
nginxImage = nginxImageARM
}
nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage)
if err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi"))
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
for _, test := range []struct {
// Name of the test.
name string
// Suffix for pod names, must be short enough.
suffix string
// Volume to mount at /tmp/root.
volume *v13.Volume
}{
{
name: "RootFS",
suffix: "rootfs",
volume: nil,
},
{
name: "EmptyDir",
suffix: "emdir",
volume: &v13.Volume{
Name: "emptydir",
VolumeSource: v13.VolumeSource{
EmptyDir: &v13.EmptyDirVolumeSource{},
},
},
},
{
name: "PersistentVolume",
suffix: "pvol",
volume: &v13.Volume{
Name: persistentVol.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: persistentVol.GetName(),
},
},
},
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
name := fmt.Sprintf("nginx-%s", test.suffix)
server := newNginxServer(benchmarkNS, name, nginxImage, test.volume)
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
if err != nil {
t.Fatalf("Failed to configure pod for runtime nodepool: %v", err)
}
server, err = testcluster.MaybeSetContainerResources(server, name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
server, err = cluster.CreatePod(ctx, server)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, server)
if err := cluster.WaitForPodRunning(ctx, server); err != nil {
t.Fatalf("Failed to wait for pod: %v", err)
}
service := newNginxService(benchmarkNS, name)
service, err = cluster.CreateService(ctx, service)
if err != nil {
t.Fatalf("Failed to create service: %v", err)
}
defer cluster.DeleteService(ctx, service)
var rounds []httpbench.Round
for _, numThreads := range threads {
for _, qps := range targetQPS {
if qps < numThreads {
continue
}
var onlyReport []httpbench.MetricType
// If we're testing at max QPS, only report throughput,
// because all requests will time out.
// Otherwise, only report latency, because the throughput
// is exactly determined by the QPS target anyway.
if qps == httpbench.InfiniteQPS {
onlyReport = append(onlyReport, httpbench.RequestsPerSecond)
onlyReport = append(onlyReport, httpbench.BytesPerSecond)
} else {
onlyReport = append(onlyReport, httpbench.Latency)
}
rounds = append(rounds, httpbench.Round{
NumThreads: numThreads,
TargetQPS: qps,
Duration: nginxBenchmarkDuration,
OnlyReport: onlyReport,
})
}
}
t.Run("0KiB", func(t *testing.T) {
benchmark := &httpbench.HTTPBenchmark{
Name: fmt.Sprintf("nginx/%s/0KiB", test.name),
Cluster: cluster,
Namespace: benchmarkNS,
Service: service,
Port: nginxPort,
Path: "/index.html",
Rounds: rounds,
Timeout: nginxRequestTimeout,
WantPercentiles: wantPercentiles,
}
benchmark.Run(ctx, t)
})
for _, docKibibytes := range nginxDocKibibytes {
t.Run(fmt.Sprintf("%dKiB", docKibibytes), func(t *testing.T) {
benchmark := &httpbench.HTTPBenchmark{
Name: fmt.Sprintf("nginx/%s/%dKiB", test.name, docKibibytes),
Cluster: cluster,
Namespace: benchmarkNS,
Service: service,
Port: nginxPort,
Path: fmt.Sprintf("/latin%dk.txt", docKibibytes),
Rounds: rounds,
Timeout: nginxRequestTimeout,
WantPercentiles: wantPercentiles,
}
benchmark.Run(ctx, t)
})
}
t.Run("HTTP404", func(t *testing.T) {
benchmark := &httpbench.HTTPBenchmark{
Name: fmt.Sprintf("nginx/%s/HTTP404", test.name),
Cluster: cluster,
Namespace: benchmarkNS,
Service: service,
Port: nginxPort,
Path: "/404-this-page-does-not-exist.html",
Rounds: rounds,
Timeout: nginxRequestTimeout,
WantPercentiles: wantPercentiles,
}
benchmark.Run(ctx, t)
})
})
if t.Failed() {
break
}
}
}
func newNginxServer(namespace *testcluster.Namespace, name, image string, volume *v13.Volume) *v13.Pod {
var volumes []v13.Volume
var volumeMounts []v13.VolumeMount
if volume != nil {
volumes = []v13.Volume{*volume}
volumeMounts = []v13.VolumeMount{{
MountPath: nginxServingDir,
Name: volume.Name,
}}
}
return &v13.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace.Namespace,
Labels: map[string]string{nginxServerLabelKey: nginxServerLabelValue},
},
Spec: v13.PodSpec{
Containers: []v13.Container{
{
Name: name,
Image: image,
Command: []string{
"sh",
"-c",
strings.Join([]string{
strings.Join([]string{"mkdir", "-p", nginxServingDir}, " "),
strings.Join([]string{
"cp", "-r", "/local/*", fmt.Sprintf("%s/.", nginxServingDir),
}, " "),
strings.Join(nginxCommand, " "),
}, " && "),
},
VolumeMounts: volumeMounts,
Ports: []v13.ContainerPort{
{
Name: name,
ContainerPort: nginxPort,
},
},
},
},
Volumes: volumes,
RestartPolicy: v13.RestartPolicyNever,
},
}
}
func newNginxService(namespace *testcluster.Namespace, name string) *v13.Service {
return namespace.GetService(name, v13.ServiceSpec{
Selector: map[string]string{nginxServerLabelKey: nginxServerLabelValue},
Ports: []v13.ServicePort{
{
Name: name,
Protocol: v13.ProtocolTCP,
Port: nginxPort,
TargetPort: intstr.FromString(name),
},
},
})
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestNginx": TestNginx,
test/kubernetes/benchmarks/ollama.go
+859
View File
File diff suppressed because it is too large Load Diff
test/kubernetes/benchmarks/ollama_test.go
+2 -836
View File
File diff suppressed because it is too large Load Diff
test/kubernetes/benchmarks/postgresql.go
+371
View File
@@ -0,0 +1,371 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package postgresql benchmarks a PostgreSQL database.
package postgresql
import (
"context"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"testing"
"time"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
postgresServerLabelKey = "app.kubernetes.io/name"
postgresServerLabelValue = "postgresql-server"
postgresPort = 5432
postgresImage = "postgres:15.3-alpine"
postgresUser = "benchman"
postgresPassword = "hunter2"
postgresDatabase = "benchpress"
postgresVolumeDir = "/var/lib/postgresql/data"
postgresDataDir = "/var/lib/postgresql/data/pgdata"
)
var (
numConnections = []int{1, 2, 6, 16, 32, 64}
)
// BenchmarkPostgresPGBench runs a PostgreSQL pgbench test.
func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
// Create a persistent volume on which to store the database data.
persistentVol := benchmarkNS.GetPersistentVolume("pgdata", "30Gi")
persistentVol, err = cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
// Create a server on the runtime under test nodepool.
image, err := k8sCtx.ResolveImage(ctx, postgresImage)
if err != nil {
t.Fatalf("failed to resolve image: %v", err)
}
server := newPostgresPod(benchmarkNS, "postgresql", image, nil, true /* withPort */, persistentVol)
if server.ObjectMeta.Labels == nil {
server.ObjectMeta.Labels = make(map[string]string)
}
server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
if err != nil {
t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err)
}
server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err)
}
server, err = cluster.CreatePod(ctx, server)
if err != nil {
t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, server)
if err := cluster.WaitForPodRunning(ctx, server); err != nil {
t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err)
}
// Create a service to service traffic to the pod.
service := newPostgresService(benchmarkNS, server.GetName())
service, err = cluster.CreateService(ctx, service)
if err != nil {
t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeleteService(ctx, service)
if err := cluster.WaitForServiceReady(ctx, service); err != nil {
t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err)
}
ip := testcluster.GetIPFromService(service)
if ip == "" {
t.Fatalf("did not get valid ip: %s", ip)
}
// Run the 'pg_isready' command to ping the server and make sure it is up.
ensureUp := func() error {
pgIsReadyName := "pgisready"
pgIsReady := newPostgresPod(benchmarkNS, pgIsReadyName, image, []string{
"pg_isready",
"--timeout=10",
fmt.Sprintf("--host=%s", ip),
fmt.Sprintf("--port=%d", postgresPort),
fmt.Sprintf("--username=%s", postgresUser),
fmt.Sprintf("--dbname=%s", postgresDatabase),
}, false /* withPort */, nil /* pvc */)
pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady)
if err != nil {
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err)
}
pgIsReady, err = cluster.CreatePod(ctx, pgIsReady)
if err != nil {
return fmt.Errorf("CreatePod %q on cluster %q: %v", pgIsReady.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, pgIsReady)
waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
defer waitCancel()
if err := cluster.WaitForPodCompleted(waitCtx, pgIsReady); err != nil {
return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), pgIsReadyName, err)
}
return nil
}
var isUpErr error
for i := 0; i < 5; i++ {
if isUpErr = ensureUp(); isUpErr == nil {
break
}
}
if isUpErr != nil {
t.Fatalf("postgresql did not come up: %v", isUpErr)
}
// pgbench has two steps: an "init step" which create and fills up a
// database with stuff, and then a main phase which does queries on that
// stuff.
// The initialization only needs to be done once per database.
initDatabase := func() error {
initDBName := "initdb"
initDB := newPostgresPod(benchmarkNS, initDBName, image, []string{
"pgbench",
"--initialize",
fmt.Sprintf("--host=%s", ip),
fmt.Sprintf("--port=%d", postgresPort),
fmt.Sprintf("--username=%s", postgresUser),
postgresDatabase,
}, false /* withPort */, nil /* pvc */)
initDB, err = cluster.ConfigurePodForClientNodepool(initDB)
if err != nil {
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err)
}
initDB, err = cluster.CreatePod(ctx, initDB)
if err != nil {
return fmt.Errorf("CreatePod %q on cluster %q: %v", initDB.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, initDB)
waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
defer waitCancel()
if err := cluster.WaitForPodCompleted(waitCtx, initDB); err != nil {
return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), initDBName, err)
}
return nil
}
if err := initDatabase(); err != nil {
t.Fatalf("cannot initialize database: %v", err)
}
for _, connections := range numConnections {
t.Run(fmt.Sprintf("%dClients", connections), func(t *testing.T) {
clientCmd := []string{
"pgbench",
"--time=90", // In seconds
"--report-per-command",
fmt.Sprintf("--host=%s", ip),
fmt.Sprintf("--port=%d", postgresPort),
fmt.Sprintf("--username=%s", postgresUser),
fmt.Sprintf("--client=%d", connections),
fmt.Sprintf("--jobs=%d", connections),
postgresDatabase,
}
client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */)
client, err = cluster.ConfigurePodForClientNodepool(client)
if err != nil {
t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
client, err = cluster.CreatePod(ctx, client)
if err != nil {
t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, client)
if err := cluster.WaitForPodCompleted(ctx, client); err != nil {
t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
// get and parse the logs from the client to get the results
rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{})
if err != nil {
t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err)
}
out, err := io.ReadAll(rdr)
if err != nil {
t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err)
}
postgresBenchmarkName := fmt.Sprintf("PostgresPGBench/%dClients", connections)
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
metrics, err := getMeasurements(string(out))
if err != nil {
t.Fatalf("failed to get metrics: out:\n\n%s\n\nerr: %v", string(out), err)
}
if err := recorder.Record(ctx, postgresBenchmarkName, metrics...); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
// newPostgresService gets a service to serve traffic to the PostgreSQL server.
func newPostgresService(namespace *testcluster.Namespace, containerName string) *v13.Service {
name := fmt.Sprintf("postgresql-service-%d", time.Now().UnixNano())
return namespace.GetService(name, v13.ServiceSpec{
Selector: map[string]string{postgresServerLabelKey: postgresServerLabelValue},
Ports: []v13.ServicePort{
{
Name: name,
Protocol: v13.ProtocolTCP,
Port: postgresPort,
TargetPort: intstr.FromInt(postgresPort),
},
},
})
}
func newPostgresPod(namespace *testcluster.Namespace, containerName, image string, argv []string, withPort bool, pvc *v13.PersistentVolumeClaim) *v13.Pod {
pod := namespace.NewPod(containerName)
container := v13.Container{
Name: containerName,
Image: image,
Command: argv,
Env: []v13.EnvVar{
// Used by postgres server:
{Name: "POSTGRES_USER", Value: postgresUser},
{Name: "POSTGRES_PASSWORD", Value: postgresPassword},
{Name: "POSTGRES_DB", Value: postgresDatabase},
{Name: "PGDATA", Value: postgresDataDir},
// Used by pgbench:
{Name: "PGPASSWORD", Value: postgresPassword},
{Name: "sslmode", Value: "disable"},
},
}
if withPort {
container.Ports = append(container.Ports, v13.ContainerPort{ContainerPort: postgresPort})
}
if pvc != nil {
pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{
Name: pvc.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.GetName(),
},
},
})
container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{
MountPath: postgresVolumeDir,
Name: pvc.GetName(),
})
}
pod.Spec.Containers = append(pod.Spec.Containers, container)
return pod
}
var (
latencyRegex = regexp.MustCompile("^latency average = ([-,.\\d]+ .?s)$")
initialConnectionRegex = regexp.MustCompile("^initial connection time = ([-,.\\d]+ .?s)$")
tpsRegex = regexp.MustCompile("^tps = ([-,.\\d]+) \\(without initial connection time\\)$")
)
func stringToFloat64(s string) float64 {
f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64)
if err != nil {
panic(fmt.Sprintf("cannot convert float %q: %v", s, err))
}
return f
}
func stringToDuration(s string) time.Duration {
parts := strings.SplitN(s, " ", 2)
floatStr, unit := parts[0], parts[1]
floatPart := stringToFloat64(floatStr)
switch unit {
case "s":
return time.Duration(floatPart * float64(time.Second))
case "ms":
return time.Duration(floatPart * float64(time.Millisecond))
case "us", "μs":
return time.Duration(floatPart * float64(time.Microsecond))
case "ns":
return time.Duration(floatPart * float64(time.Nanosecond))
default:
panic(fmt.Sprintf("unknown time unit %q", unit))
}
}
// getMeasurements parses the output of pgbench to get the stats.
func getMeasurements(out string) ([]benchmetric.MetricValue, error) {
var foundLatency, foundInitialConnection, foundTPS benchmetric.MetricValue
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if latencyMatch := latencyRegex.FindStringSubmatch(line); latencyMatch != nil {
if foundLatency != nil {
return nil, fmt.Errorf("found duplicate latency data: %v vs %q", foundLatency, line)
}
foundLatency = benchmetric.SpecificDuration(stringToDuration(latencyMatch[1]), "avg")
}
if initialConnectionMatch := initialConnectionRegex.FindStringSubmatch(line); initialConnectionMatch != nil {
if foundInitialConnection != nil {
return nil, fmt.Errorf("found duplicate initial connection data: %v vs %q", foundInitialConnection, line)
}
foundInitialConnection = benchmetric.SpecificDuration(stringToDuration(initialConnectionMatch[1]), "init")
}
if tpsMatch := tpsRegex.FindStringSubmatch(line); tpsMatch != nil {
if foundTPS != nil {
return nil, fmt.Errorf("found duplicate TPS data: %v vs %q", foundTPS, line)
}
foundTPS = benchmetric.RequestsPerSecond(stringToFloat64(tpsMatch[1]))
}
}
if foundLatency == nil || foundInitialConnection == nil || foundTPS == nil {
return nil, fmt.Errorf("did not find the data we wanted: foundLatency=%v foundInitialConnection=%v foundTPS=%v", foundLatency, foundInitialConnection, foundTPS)
}
return []benchmetric.MetricValue{
foundLatency,
foundInitialConnection,
foundTPS,
}, nil
}
test/kubernetes/benchmarks/postgresql_test.go
+3 -350
View File
@@ -12,41 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package postgresql_test benchmarks a PostgreSQL database.
package postgresql_test
// Package postgresql benchmarks a PostgreSQL database.
package postgresql
import (
"context"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"testing"
"time"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
postgresServerLabelKey = "app.kubernetes.io/name"
postgresServerLabelValue = "postgresql-server"
postgresPort = 5432
postgresImage = "postgres:15.3-alpine"
postgresUser = "benchman"
postgresPassword = "hunter2"
postgresDatabase = "benchpress"
postgresVolumeDir = "/var/lib/postgresql/data"
postgresDataDir = "/var/lib/postgresql/data/pgdata"
)
var (
numConnections = []int{1, 2, 6, 16, 32, 64}
)
// TestPostgresPGBench benchmarks a PostgreSQL database with pgbench.
@@ -59,332 +33,11 @@ func TestPostgresPGBench(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("PostgresPGBench", func(t *testing.T) {
t.Parallel()
doPostgresPGBenchTest(ctx, t, k8sCtx, cluster)
BenchmarkPostgresPGBench(ctx, t, k8sCtx, cluster)
})
})
}
// doPostgresTest runs a PostgreSQL pgbench test.
func doPostgresPGBenchTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
// Create a persistent volume on which to store the database data.
persistentVol := benchmarkNS.GetPersistentVolume("pgdata", "30Gi")
persistentVol, err = cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
// Create a server on the runtime under test nodepool.
image, err := k8sCtx.ResolveImage(ctx, postgresImage)
if err != nil {
t.Fatalf("failed to resolve image: %v", err)
}
server := newPostgresPod(benchmarkNS, "postgresql", image, nil, true /* withPort */, persistentVol)
if server.ObjectMeta.Labels == nil {
server.ObjectMeta.Labels = make(map[string]string)
}
server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
if err != nil {
t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err)
}
server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err)
}
server, err = cluster.CreatePod(ctx, server)
if err != nil {
t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, server)
if err := cluster.WaitForPodRunning(ctx, server); err != nil {
t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err)
}
// Create a service to service traffic to the pod.
service := newPostgresService(benchmarkNS, server.GetName())
service, err = cluster.CreateService(ctx, service)
if err != nil {
t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeleteService(ctx, service)
if err := cluster.WaitForServiceReady(ctx, service); err != nil {
t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err)
}
ip := testcluster.GetIPFromService(service)
if ip == "" {
t.Fatalf("did not get valid ip: %s", ip)
}
// Run the 'pg_isready' command to ping the server and make sure it is up.
ensureUp := func() error {
pgIsReadyName := "pgisready"
pgIsReady := newPostgresPod(benchmarkNS, pgIsReadyName, image, []string{
"pg_isready",
"--timeout=10",
fmt.Sprintf("--host=%s", ip),
fmt.Sprintf("--port=%d", postgresPort),
fmt.Sprintf("--username=%s", postgresUser),
fmt.Sprintf("--dbname=%s", postgresDatabase),
}, false /* withPort */, nil /* pvc */)
pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady)
if err != nil {
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err)
}
pgIsReady, err = cluster.CreatePod(ctx, pgIsReady)
if err != nil {
return fmt.Errorf("CreatePod %q on cluster %q: %v", pgIsReady.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, pgIsReady)
waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
defer waitCancel()
if err := cluster.WaitForPodCompleted(waitCtx, pgIsReady); err != nil {
return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), pgIsReadyName, err)
}
return nil
}
var isUpErr error
for i := 0; i < 5; i++ {
if isUpErr = ensureUp(); isUpErr == nil {
break
}
}
if isUpErr != nil {
t.Fatalf("postgresql did not come up: %v", isUpErr)
}
// pgbench has two steps: an "init step" which create and fills up a
// database with stuff, and then a main phase which does queries on that
// stuff.
// The initialization only needs to be done once per database.
initDatabase := func() error {
initDBName := "initdb"
initDB := newPostgresPod(benchmarkNS, initDBName, image, []string{
"pgbench",
"--initialize",
fmt.Sprintf("--host=%s", ip),
fmt.Sprintf("--port=%d", postgresPort),
fmt.Sprintf("--username=%s", postgresUser),
postgresDatabase,
}, false /* withPort */, nil /* pvc */)
initDB, err = cluster.ConfigurePodForClientNodepool(initDB)
if err != nil {
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err)
}
initDB, err = cluster.CreatePod(ctx, initDB)
if err != nil {
return fmt.Errorf("CreatePod %q on cluster %q: %v", initDB.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, initDB)
waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
defer waitCancel()
if err := cluster.WaitForPodCompleted(waitCtx, initDB); err != nil {
return fmt.Errorf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), initDBName, err)
}
return nil
}
if err := initDatabase(); err != nil {
t.Fatalf("cannot initialize database: %v", err)
}
for _, connections := range numConnections {
t.Run(fmt.Sprintf("%dClients", connections), func(t *testing.T) {
clientCmd := []string{
"pgbench",
"--time=90", // In seconds
"--report-per-command",
fmt.Sprintf("--host=%s", ip),
fmt.Sprintf("--port=%d", postgresPort),
fmt.Sprintf("--username=%s", postgresUser),
fmt.Sprintf("--client=%d", connections),
fmt.Sprintf("--jobs=%d", connections),
postgresDatabase,
}
client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */)
client, err = cluster.ConfigurePodForClientNodepool(client)
if err != nil {
t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
client, err = cluster.CreatePod(ctx, client)
if err != nil {
t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, client)
if err := cluster.WaitForPodCompleted(ctx, client); err != nil {
t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
// get and parse the logs from the client to get the results
rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{})
if err != nil {
t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err)
}
out, err := io.ReadAll(rdr)
if err != nil {
t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err)
}
postgresBenchmarkName := fmt.Sprintf("PostgresPGBench/%dClients", connections)
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
metrics, err := getMeasurements(string(out))
if err != nil {
t.Fatalf("failed to get metrics: out:\n\n%s\n\nerr: %v", string(out), err)
}
if err := recorder.Record(ctx, postgresBenchmarkName, metrics...); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
})
}
}
// newPostgresService gets a service to serve traffic to the PostgreSQL server.
func newPostgresService(namespace *testcluster.Namespace, containerName string) *v13.Service {
name := fmt.Sprintf("postgresql-service-%d", time.Now().UnixNano())
return namespace.GetService(name, v13.ServiceSpec{
Selector: map[string]string{postgresServerLabelKey: postgresServerLabelValue},
Ports: []v13.ServicePort{
{
Name: name,
Protocol: v13.ProtocolTCP,
Port: postgresPort,
TargetPort: intstr.FromInt(postgresPort),
},
},
})
}
func newPostgresPod(namespace *testcluster.Namespace, containerName, image string, argv []string, withPort bool, pvc *v13.PersistentVolumeClaim) *v13.Pod {
pod := namespace.NewPod(containerName)
container := v13.Container{
Name: containerName,
Image: image,
Command: argv,
Env: []v13.EnvVar{
// Used by postgres server:
{Name: "POSTGRES_USER", Value: postgresUser},
{Name: "POSTGRES_PASSWORD", Value: postgresPassword},
{Name: "POSTGRES_DB", Value: postgresDatabase},
{Name: "PGDATA", Value: postgresDataDir},
// Used by pgbench:
{Name: "PGPASSWORD", Value: postgresPassword},
{Name: "sslmode", Value: "disable"},
},
}
if withPort {
container.Ports = append(container.Ports, v13.ContainerPort{ContainerPort: postgresPort})
}
if pvc != nil {
pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{
Name: pvc.GetName(),
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.GetName(),
},
},
})
container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{
MountPath: postgresVolumeDir,
Name: pvc.GetName(),
})
}
pod.Spec.Containers = append(pod.Spec.Containers, container)
return pod
}
var (
latencyRegex = regexp.MustCompile("^latency average = ([-,.\\d]+ .?s)$")
initialConnectionRegex = regexp.MustCompile("^initial connection time = ([-,.\\d]+ .?s)$")
tpsRegex = regexp.MustCompile("^tps = ([-,.\\d]+) \\(without initial connection time\\)$")
)
func stringToFloat64(s string) float64 {
f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64)
if err != nil {
panic(fmt.Sprintf("cannot convert float %q: %v", s, err))
}
return f
}
func stringToDuration(s string) time.Duration {
parts := strings.SplitN(s, " ", 2)
floatStr, unit := parts[0], parts[1]
floatPart := stringToFloat64(floatStr)
switch unit {
case "s":
return time.Duration(floatPart * float64(time.Second))
case "ms":
return time.Duration(floatPart * float64(time.Millisecond))
case "us", "μs":
return time.Duration(floatPart * float64(time.Microsecond))
case "ns":
return time.Duration(floatPart * float64(time.Nanosecond))
default:
panic(fmt.Sprintf("unknown time unit %q", unit))
}
}
// getMeasurements parses the output of pgbench to get the stats.
func getMeasurements(out string) ([]benchmetric.MetricValue, error) {
var foundLatency, foundInitialConnection, foundTPS benchmetric.MetricValue
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if latencyMatch := latencyRegex.FindStringSubmatch(line); latencyMatch != nil {
if foundLatency != nil {
return nil, fmt.Errorf("found duplicate latency data: %v vs %q", foundLatency, line)
}
foundLatency = benchmetric.SpecificDuration(stringToDuration(latencyMatch[1]), "avg")
}
if initialConnectionMatch := initialConnectionRegex.FindStringSubmatch(line); initialConnectionMatch != nil {
if foundInitialConnection != nil {
return nil, fmt.Errorf("found duplicate initial connection data: %v vs %q", foundInitialConnection, line)
}
foundInitialConnection = benchmetric.SpecificDuration(stringToDuration(initialConnectionMatch[1]), "init")
}
if tpsMatch := tpsRegex.FindStringSubmatch(line); tpsMatch != nil {
if foundTPS != nil {
return nil, fmt.Errorf("found duplicate TPS data: %v vs %q", foundTPS, line)
}
foundTPS = benchmetric.RequestsPerSecond(stringToFloat64(tpsMatch[1]))
}
}
if foundLatency == nil || foundInitialConnection == nil || foundTPS == nil {
return nil, fmt.Errorf("did not find the data we wanted: foundLatency=%v foundInitialConnection=%v foundTPS=%v", foundLatency, foundInitialConnection, foundTPS)
}
return []benchmetric.MetricValue{
foundLatency,
foundInitialConnection,
foundTPS,
}, nil
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestPostgresPGBench": TestPostgresPGBench,
test/kubernetes/benchmarks/pytorch.go
+381
View File
@@ -0,0 +1,381 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package pytorch contains benchmarks using the pytorch "torchbench" repo.
package pytorch
// These tests use pytorch's "torchbench" suite (https://github.com/pytorch/benchmark/tree/main).
// The Authors describe the benchmarks in this paper: https://arxiv.org/pdf/2304.14226.pdf
// The Authors list both the type of model and its profile (how GPU intensive).
// Note: The image for this test is about 7-8 GB as of writing. After you get your clusters up and
// running, start the test and make sure that the pods show the event of downloading the image. Then
// get a cup of coffee, chat with your co-workers for 5 min, and it will be about done 5 min after
// that. You'll only need to do this once for each cluster (in parallel).
import (
"context"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"testing"
"time"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
)
// pytorchTestType is the method used, either training or evaluation, for the model.
type pytorchTestType string
const (
train = pytorchTestType("train")
eval = pytorchTestType("eval")
pytorchImage = k8s.ImageRepoPrefix + "benchmarks/pytorch_x86_64:f6f280aeb1b07989"
)
type pytorchMode string
// pytorchMode is the pytorch mode used, either script mode (jit) or eager mode.
// See: https://towardsdatascience.com/pytorch-jit-and-torchscript-c2a77bac0fff
const (
jit = pytorchMode("jit")
eager = pytorchMode("eager")
)
type pytorchTest struct {
module string
test pytorchTestType
mode pytorchMode
}
// Sets of tests.
var (
// FastNLPBert uses the fastNLP_Bert module, which is classified as a NLP Language Model.
// fastNLP_Bert taxes the GPU heavily with low data movement. See Figure 2 on
// page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/fastNLP_Bert
// Bert Blog Post: https://towardsdatascience.com/bert-explained-state-of-the-art-language-model-for-nlp-f8b21a9b6270
// Paper: https://arxiv.org/abs/1810.04805
FastNLPBert = []pytorchTest{
{
module: "fastNLP_Bert",
test: train,
mode: eager,
},
{
module: "fastNLP_Bert",
test: eval,
mode: eager,
},
}
// BigBird uses the hf_BigBird module, which is classified as a NLP Language Model.
// hf_BigBird taxes the GPU moderately with low data movement. See Figure 2 on
// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/hf_BigBird
// Paper: https://arxiv.org/abs/2007.14062
BigBird = []pytorchTest{
{
module: "hf_BigBird",
test: train,
mode: eager,
},
{
module: "hf_BigBird",
test: eval,
mode: eager,
},
}
// SpeechTransformer uses the speech_transformer module classified as "Speech Recognition"
// model. speech_transformer has a lot of idle time for the GPU. See Figure 2 on
// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/pull/374
// Paper: https://arxiv.org/abs/1706.03762
SpeechTransformer = []pytorchTest{
{
module: "speech_transformer",
test: train,
mode: eager,
},
{
module: "speech_transformer",
test: eval,
mode: eager,
},
}
// LearningToPaint uses the LearningToPaint module classified as "neural renderer in model-based
// Deep Reinforcement Learning (DRL)".
// Learning to paint has a lot of "data movement" and doesn't tax the GPU a lot. See Figure 2 on
// page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/LearningToPaint
LearningToPaint = []pytorchTest{
{
module: "LearningToPaint",
test: train,
mode: jit,
},
{
module: "LearningToPaint",
test: eval,
mode: jit,
},
}
// MobileNetV2 uses the mobilenet_v2 module classified as "Computer Vision: Image Classification".
// MobileNet has a lot of taxes the GPU. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/mobilenet_v2
// Paper: https://paperswithcode.com/method/mobilenetv2
MobileNetV2 = []pytorchTest{
{
module: "mobilenet_v2",
test: train,
mode: jit,
},
{
module: "mobilenet_v2",
test: eval,
mode: jit,
},
}
// BackgroundMatting uses the Background_Matting module classified as "Computer Vision: Pattern Recognition".
// BackgroundMatting has a lot of GPU idle time. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/Background_Matting (see README)
BackgroundMatting = []pytorchTest{
{
module: "Background_Matting",
test: train,
mode: eager,
},
{
module: "Background_Matting",
test: eval,
mode: eager,
},
}
)
// Name returns the name of the test with the argument parameters included. It is formatted so
// that it can be used for the name of the pod.
func (p pytorchTest) Name() string {
// Kubernetes pod names cannot contain "_".
module := strings.ReplaceAll(strings.ToLower(p.module), "_", "-")
return fmt.Sprintf("%s-%s-%s", module, p.test, p.mode)
}
var snakeCase = regexp.MustCompile("_.")
// BenchName returns the name of the test with the argument parameters included.
// It is formatted so that it can be used for benchstat output.
func (p pytorchTest) BenchName() string {
// First letter of the module should be capitalized, as it will be
// concatenated with "Benchmark" and it's useful to mark it as a different
// word.
// Some modules use a lowercase first letter, e.g. "fastNLP_Bert".
moduleName := strings.ToUpper(p.module[:1]) + p.module[1:]
// We also replace "snake_case" with "snakeCase". Sorry snakes.
moduleName = snakeCase.ReplaceAllStringFunc(moduleName, func(s string) string {
return strings.ToUpper(strings.TrimPrefix(s, "_"))
})
test := strings.ToUpper(string(p.test)[:1]) + string(p.test[1:])
var mode string
switch p.mode {
case eager:
mode = "Eager"
case jit:
mode = "JIT"
default:
panic(fmt.Sprintf("Unknown mode: %v", p.mode))
}
return fmt.Sprintf("%s/%s/%s", moduleName, test, mode)
}
func (p pytorchTest) toPod(namespace *testcluster.Namespace, image string) (*v13.Pod, error) {
pod := namespace.NewPod(p.Name())
pod.Spec = v13.PodSpec{
RestartPolicy: v13.RestartPolicyNever,
Containers: []v13.Container{
{
Name: p.Name(),
Image: pytorchImage,
Command: benchmetric.TimedCommand(p.command()...),
},
},
}
return pod, nil
}
func (p pytorchTest) command() []string {
return []string{
"python",
"run.py",
p.module,
"--device", "cuda",
"--test", string(p.test),
"--mode", string(p.mode),
}
}
// RunPytorch runs the given PyTorch tests sequentially on a single cluster.
func RunPytorch(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, tests []pytorchTest) {
t.Helper()
for _, test := range tests {
t.Run(test.Name(), func(t *testing.T) {
doPytorchRun(ctx, t, k8sCtx, cluster, test)
})
}
}
// doPytorchRun runs a single PyTorch test.
func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("Failed to reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
image, err := k8sCtx.ResolveImage(ctx, pytorchImage)
if err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
pod, err := params.toPod(benchmarkNS, image)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
if err != nil {
t.Fatalf("Failed to configure pod for test-nodepool: %v", err)
}
pod, err = testcluster.MaybeSetContainerResources(pod, pod.Name, testcluster.ContainerResourcesRequest{GPU: true})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
pod, err = cluster.CreatePod(ctx, pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, pod)
if err := cluster.WaitForPodCompleted(ctx, pod); err != nil {
t.Fatalf("Failed to wait for pod to complete: %v", err)
}
rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{})
if err != nil {
t.Fatalf("GetLogReader on cluster %q pod %v: %v", cluster.GetName(), pod.GetName(), err)
}
out, err := io.ReadAll(rdr)
if err != nil {
t.Fatalf("failed to read from pod: %q: %v", pod.GetName(), err)
}
metrics, err := parseStandardOutput(string(out))
if err != nil {
t.Fatalf("parseStandardOutput: %v", err)
}
containerDuration, err := benchmetric.ParseTimedContainerOutput(string(out))
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
metrics = append(metrics, benchmetric.BenchmarkDuration(containerDuration))
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
if err := recorder.Record(ctx, params.BenchName(), metrics...); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
}
func parseStandardOutput(output string) ([]benchmetric.MetricValue, error) {
gpuTimeMillis, err := parseGPUTime(output)
if err != nil {
return nil, fmt.Errorf("parseGPUTime: %v", err)
}
gpuPeakMemoryGB, err := parseGPUPeakMemoryGB(output)
if err != nil {
return nil, fmt.Errorf("parseGPUPeakMemory: %v", err)
}
cpuPeakMemoryGB, err := parseCPUPeakMemoryGB(output)
if err != nil {
return nil, fmt.Errorf("parseCPUPeakMemory: %v", err)
}
return []benchmetric.MetricValue{
benchmetric.SpecificDuration(time.Duration(gpuTimeMillis)*time.Millisecond, "gpu-runtime"),
benchmetric.SpecificBytes(gpuPeakMemoryGB*1024*1024*1024, "gpu-peak-memory"),
benchmetric.SpecificBytes(cpuPeakMemoryGB*1024*1024*1024, "cpu-peak-memory"),
}, nil
}
var gpuTimeRegex = regexp.MustCompile(`GPU\sTime:\s*(\d+\.\d+)\smilliseconds`)
func parseGPUTime(output string) (float64, error) {
match := gpuTimeRegex.FindStringSubmatch(output)
if len(match) < 2 {
return 0, fmt.Errorf("failed to find GPU Time: %s", output)
}
return strconv.ParseFloat(match[1], 64)
}
var gpuPeakMemoryRegex = regexp.MustCompile(`GPU\s0\sPeak\sMemory:\s*(\d+\.\d+)\sGB`)
func parseGPUPeakMemoryGB(output string) (float64, error) {
match := gpuPeakMemoryRegex.FindStringSubmatch(output)
if len(match) < 2 {
return 0, fmt.Errorf("failed to find GPU Peak Memory: %s", output)
}
return strconv.ParseFloat(match[1], 64)
}
var cpuPeakMemoryRegex = regexp.MustCompile(`CPU\sPeak\sMemory:\s*(\d+\.\d+)\sGB`)
func parseCPUPeakMemoryGB(output string) (float64, error) {
match := cpuPeakMemoryRegex.FindStringSubmatch(output)
if len(match) < 2 {
return 0, fmt.Errorf("failed to find CPU Peak Memory: %s", output)
}
return strconv.ParseFloat(match[1], 64)
}
test/kubernetes/benchmarks/pytorch_test.go
+8 -356
View File
@@ -12,263 +12,44 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package pytorch_test contains benchmarks using the pytorch "torchbench" repo.
package pytorch_test
// These tests use pytorch's "torchbench" suite (https://github.com/pytorch/benchmark/tree/main).
// The Authors describe the benchmarks in this paper: https://arxiv.org/pdf/2304.14226.pdf
// The Authors list both the type of model and its profile (how GPU intensive).
// Note: The image for this test is about 7-8 GB as of writing. After you get your clusters up and
// running, start the test and make sure that the pods show the event of downloading the image. Then
// get a cup of coffee, chat with your co-workers for 5 min, and it will be about done 5 min after
// that. You'll only need to do this once for each cluster (in parallel).
package pytorch
import (
"context"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"testing"
"time"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
)
// pytorchTestType is the method used, either training or evaluation, for the model.
type pytorchTestType string
const (
train = pytorchTestType("train")
eval = pytorchTestType("eval")
pytorchImage = k8s.ImageRepoPrefix + "benchmarks/pytorch_x86_64:f6f280aeb1b07989"
)
type pytorchMode string
// pytorchMode is the pytorch mode used, either script mode (jit) or eager mode.
// See: https://towardsdatascience.com/pytorch-jit-and-torchscript-c2a77bac0fff
const (
jit = pytorchMode("jit")
eager = pytorchMode("eager")
)
type pytorchTest struct {
module string
test pytorchTestType
mode pytorchMode
}
// Name returns the name of the test with the argument parameters included. It is formatted so
// that it can be used for the name of the pod.
func (p pytorchTest) Name() string {
// Kubernetes pod names cannot contain "_".
module := strings.ReplaceAll(strings.ToLower(p.module), "_", "-")
return fmt.Sprintf("%s-%s-%s", module, p.test, p.mode)
}
var snakeCase = regexp.MustCompile("_.")
// BenchName returns the name of the test with the argument parameters included.
// It is formatted so that it can be used for benchstat output.
func (p pytorchTest) BenchName() string {
// First letter of the module should be capitalized, as it will be
// concatenated with "Benchmark" and it's useful to mark it as a different
// word.
// Some modules use a lowercase first letter, e.g. "fastNLP_Bert".
moduleName := strings.ToUpper(p.module[:1]) + p.module[1:]
// We also replace "snake_case" with "snakeCase". Sorry snakes.
moduleName = snakeCase.ReplaceAllStringFunc(moduleName, func(s string) string {
return strings.ToUpper(strings.TrimPrefix(s, "_"))
})
test := strings.ToUpper(string(p.test)[:1]) + string(p.test[1:])
var mode string
switch p.mode {
case eager:
mode = "Eager"
case jit:
mode = "JIT"
default:
panic(fmt.Sprintf("Unknown mode: %v", p.mode))
}
return fmt.Sprintf("%s/%s/%s", moduleName, test, mode)
}
func (p pytorchTest) toPod(namespace *testcluster.Namespace, image string) (*v13.Pod, error) {
pod := namespace.NewPod(p.Name())
pod.Spec = v13.PodSpec{
RestartPolicy: v13.RestartPolicyNever,
Containers: []v13.Container{
{
Name: p.Name(),
Image: pytorchImage,
Command: benchmetric.TimedCommand(p.command()...),
},
},
}
return pod, nil
}
func (p pytorchTest) command() []string {
return []string{
"python",
"run.py",
p.module,
"--device", "cuda",
"--test", string(p.test),
"--mode", string(p.mode),
}
}
// TestFastNLPBert uses the fastNLP_Bert module, which is classified as a NLP Language Model.
// fastNLP_Bert taxes the GPU heavily with low data movement. See Figure 2 on
// page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/fastNLP_Bert
// Bert Blog Post: https://towardsdatascience.com/bert-explained-state-of-the-art-language-model-for-nlp-f8b21a9b6270
// Paper: https://arxiv.org/abs/1810.04805
func TestFastNLPBert(t *testing.T) {
ctx := context.Background()
const module = "fastNLP_Bert"
tests := []pytorchTest{
{
module: module,
test: train,
mode: eager,
},
{
module: module,
test: eval,
mode: eager,
},
}
runTests(ctx, t, tests)
runTests(ctx, t, FastNLPBert)
}
// TestBigBird uses the hf_BigBird module, which is classified as a NLP Language Model.
// hf_BigBird taxes the GPU moderately with low data movement. See Figure 2 on
// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/hf_BigBird
// Paper: https://arxiv.org/abs/2007.14062
func TestBigBird(t *testing.T) {
ctx := context.Background()
const module = "hf_BigBird"
tests := []pytorchTest{
{
module: module,
test: train,
mode: eager,
},
{
module: module,
test: eval,
mode: eager,
},
}
runTests(ctx, t, tests)
runTests(ctx, t, BigBird)
}
// TestSpeechTransformer uses the speech_transformer module classified as "Speech Recognition"
// model. speech_transformer has a lot of idle time for the GPU. See Figure 2 on
// page 5 (speech_tf): https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/pull/374
// Paper: https://arxiv.org/abs/1706.03762
func TestSpeechTransformer(t *testing.T) {
ctx := context.Background()
const module = "speech_transformer"
tests := []pytorchTest{
{
module: module,
test: train,
mode: eager,
},
{
module: module,
test: eval,
mode: eager,
},
}
runTests(ctx, t, tests)
runTests(ctx, t, SpeechTransformer)
}
// TestLearningToPaint uses the LearningToPaint module classified as "neural renderer in model-based
// Deep Reinforcement Learning (DRL)".
// Learning to paint has a lot of "data movement" and doesn't tax the GPU a lot. See Figure 2 on
// page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/LearningToPaint
func TestLearningToPaint(t *testing.T) {
ctx := context.Background()
const module = "LearningToPaint"
tests := []pytorchTest{
{
module: module,
test: train,
mode: jit,
},
{
module: module,
test: eval,
mode: jit,
},
}
runTests(ctx, t, tests)
runTests(ctx, t, LearningToPaint)
}
// TestMobileNetV2 uses the mobilenet_v2 module classified as "Computer Vision: Image Classification".
// MobileNet has a lot of taxes the GPU. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/mobilenet_v2
// Paper: https://paperswithcode.com/method/mobilenetv2
func TestMobileNetV2(t *testing.T) {
ctx := context.Background()
const module = "mobilenet_v2"
tests := []pytorchTest{
{
module: module,
test: train,
mode: jit,
},
{
module: module,
test: eval,
mode: jit,
},
}
runTests(ctx, t, tests)
runTests(ctx, t, MobileNetV2)
}
// TestBackgroundMatting uses the Background_Matting module classified as "Computer Vision: Pattern Recognition".
// BackgroundMatting has a lot of GPU idle time. See Figure 2 on page 5: https://arxiv.org/pdf/2304.14226.pdf
//
// https://github.com/pytorch/benchmark/tree/main/torchbenchmark/models/Background_Matting (see README)
func TestBackgroundMatting(t *testing.T) {
ctx := context.Background()
const module = "Background_Matting"
tests := []pytorchTest{
{
module: module,
test: train,
mode: eager,
},
{
module: module,
test: eval,
mode: eager,
},
}
runTests(ctx, t, tests)
runTests(ctx, t, BackgroundMatting)
}
func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) {
@@ -279,140 +60,11 @@ func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("PyTorch", func(t *testing.T) {
t.Parallel()
for _, p := range tests {
t.Run(p.Name(), func(t *testing.T) {
doPytorchRun(ctx, t, k8sCtx, cluster, p)
})
}
RunPytorch(ctx, t, k8sCtx, cluster, tests)
})
})
}
func doPytorchRun(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster, params pytorchTest) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("Failed to reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
image, err := k8sCtx.ResolveImage(ctx, pytorchImage)
if err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
pod, err := params.toPod(benchmarkNS, image)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
if err != nil {
t.Fatalf("Failed to configure pod for test-nodepool: %v", err)
}
pod, err = testcluster.MaybeSetContainerResources(pod, pod.Name, testcluster.ContainerResourcesRequest{GPU: true})
if err != nil {
t.Fatalf("Failed to set container resources: %v", err)
}
pod, err = cluster.CreatePod(ctx, pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
defer cluster.DeletePod(ctx, pod)
if err := cluster.WaitForPodCompleted(ctx, pod); err != nil {
t.Fatalf("Failed to wait for pod to complete: %v", err)
}
rdr, err := cluster.GetLogReader(ctx, pod, v13.PodLogOptions{})
if err != nil {
t.Fatalf("GetLogReader on cluster %q pod %v: %v", cluster.GetName(), pod.GetName(), err)
}
out, err := io.ReadAll(rdr)
if err != nil {
t.Fatalf("failed to read from pod: %q: %v", pod.GetName(), err)
}
metrics, err := parseStandardOutput(string(out))
if err != nil {
t.Fatalf("parseStandardOutput: %v", err)
}
containerDuration, err := benchmetric.ParseTimedContainerOutput(string(out))
if err != nil {
t.Fatalf("Failed to get container duration: %v", err)
}
metrics = append(metrics, benchmetric.BenchmarkDuration(containerDuration))
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
if err := recorder.Record(ctx, params.BenchName(), metrics...); err != nil {
t.Fatalf("Failed to record benchmark data: %v", err)
}
}
func parseStandardOutput(output string) ([]benchmetric.MetricValue, error) {
gpuTimeMillis, err := parseGPUTime(output)
if err != nil {
return nil, fmt.Errorf("parseGPUTime: %v", err)
}
gpuPeakMemoryGB, err := parseGPUPeakMemoryGB(output)
if err != nil {
return nil, fmt.Errorf("parseGPUPeakMemory: %v", err)
}
cpuPeakMemoryGB, err := parseCPUPeakMemoryGB(output)
if err != nil {
return nil, fmt.Errorf("parseCPUPeakMemory: %v", err)
}
return []benchmetric.MetricValue{
benchmetric.SpecificDuration(time.Duration(gpuTimeMillis)*time.Millisecond, "gpu-runtime"),
benchmetric.SpecificBytes(gpuPeakMemoryGB*1024*1024*1024, "gpu-peak-memory"),
benchmetric.SpecificBytes(cpuPeakMemoryGB*1024*1024*1024, "cpu-peak-memory"),
}, nil
}
var gpuTimeRegex = regexp.MustCompile(`GPU\sTime:\s*(\d+\.\d+)\smilliseconds`)
func parseGPUTime(output string) (float64, error) {
match := gpuTimeRegex.FindStringSubmatch(output)
if len(match) < 2 {
return 0, fmt.Errorf("failed to find GPU Time: %s", output)
}
return strconv.ParseFloat(match[1], 64)
}
var gpuPeakMemoryRegex = regexp.MustCompile(`GPU\s0\sPeak\sMemory:\s*(\d+\.\d+)\sGB`)
func parseGPUPeakMemoryGB(output string) (float64, error) {
match := gpuPeakMemoryRegex.FindStringSubmatch(output)
if len(match) < 2 {
return 0, fmt.Errorf("failed to find GPU Peak Memory: %s", output)
}
return strconv.ParseFloat(match[1], 64)
}
var cpuPeakMemoryRegex = regexp.MustCompile(`CPU\sPeak\sMemory:\s*(\d+\.\d+)\sGB`)
func parseCPUPeakMemoryGB(output string) (float64, error) {
match := cpuPeakMemoryRegex.FindStringSubmatch(output)
if len(match) < 2 {
return 0, fmt.Errorf("failed to find CPU Peak Memory: %s", output)
}
return strconv.ParseFloat(match[1], 64)
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestFastNLPBert": TestFastNLPBert,
test/kubernetes/benchmarks/redis.go
+454
View File
@@ -0,0 +1,454 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package redis holds the redis test where the runtime under test runs a redis server and the
// native runtime runs a client making requests against it.
package redis
import (
"context"
"fmt"
"io"
"math"
"regexp"
"strconv"
"strings"
"testing"
"time"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
redisPort = 6379
defaultRequestsPerConnection = 50000
redisServerLabelKey = "app.kubernetes.io/name"
redisServerLabelValue = "redis-server"
redisVolumeName = "redis-data"
redisDataDirectory = "/redis-data"
redisImageAMD = k8s.ImageRepoPrefix + "benchmarks/redis_x86_64:latest"
redisImageARM = k8s.ImageRepoPrefix + "benchmarks/redis_aarch64:latest"
)
var (
numConnections = []int{1, 2, 4, 8, 16, 32}
latencyPercentiles = []int{50, 95, 99}
operations = []string{"SET", "GET", "MSET", "LPUSH", "LRANGE_500"}
)
// BenchmarkRedis runs the Redis performance benchmark using redis-benchmark.
func BenchmarkRedis(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
persistentVol := benchmarkNS.GetPersistentVolume(redisVolumeName, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := redisImageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = redisImageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
// Benchmark name
name string
// Suffix for the redis server, must be short to fit in pod name.
suffix string
// redis-server command-line.
serverCommand []string
// Volume to use for persistence, if any.
volume *v13.PersistentVolumeClaim
}{
{
name: "Persistence",
suffix: "persist",
serverCommand: []string{
"redis-server",
"--dir", redisDataDirectory,
// Default save settings per
// https://redis.io/docs/management/config-file/
"--save", "3600 1 300 100 60 10000",
},
volume: persistentVol,
},
{
name: "NoPersistence",
suffix: "nopersist",
serverCommand: []string{
"redis-server",
"--appendonly", "no",
"--save", "",
},
volume: nil,
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
// Create a server on the runtime under test nodepool.
server := newRedisPodWithPort(benchmarkNS, fmt.Sprintf("redis-%s", test.suffix), image, test.serverCommand, redisPort, test.volume)
if server.ObjectMeta.Labels == nil {
server.ObjectMeta.Labels = make(map[string]string)
}
server.ObjectMeta.Labels[redisServerLabelKey] = redisServerLabelValue
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
if err != nil {
t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err)
}
server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err)
}
server, err = cluster.CreatePod(ctx, server)
if err != nil {
t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, server)
if err := cluster.WaitForPodRunning(ctx, server); err != nil {
t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err)
}
// create a service to service traffic to the pod.
service := newRedisService(benchmarkNS, server.GetName())
service, err = cluster.CreateService(ctx, service)
if err != nil {
t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeleteService(ctx, service)
if err := cluster.WaitForServiceReady(ctx, service); err != nil {
t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err)
}
ip := testcluster.GetIPFromService(service)
if ip == "" {
t.Fatalf("did not get valid ip: %s", ip)
}
// run the 'redis-cli' command to ping the server and make sure it is up. The "ping" request comes
// back with a "PONG" response. We repeat -r=5 times with a -i=1 second interval. If we
// get one PONG back then the server is considered up.
pingCmd := []string{"redis-cli", "-h", ip, "-r", "5", "-i", "1", "ping"}
ensureUp := func() error {
pinger := newRedisPod(benchmarkNS, fmt.Sprintf("rpinger-%s", test.suffix), image, pingCmd)
pinger, err = cluster.ConfigurePodForClientNodepool(pinger)
if err != nil {
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pinger.GetName(), err)
}
pinger, err = cluster.CreatePod(ctx, pinger)
if err != nil {
return fmt.Errorf("CreatePod %q on cluster %q: %v", pinger.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, pinger)
waitCtx, waitCancel := context.WithTimeout(ctx, 30*time.Second)
var podWaitSuffix string
if err := cluster.WaitForPodCompleted(waitCtx, pinger); err != nil {
podWaitSuffix = fmt.Sprintf(" (pod wait error: %v)", err)
}
waitCancel()
rdr, err := cluster.GetLogReader(ctx, pinger, v13.PodLogOptions{})
if err != nil {
return fmt.Errorf("GetLogReader on cluster %q: %v%s", cluster.GetName(), err, podWaitSuffix)
}
out, err := io.ReadAll(rdr)
if err != nil {
return fmt.Errorf("failed to read from pod: %q: %v%s", pinger.GetName(), err, podWaitSuffix)
}
if !strings.Contains(string(out), "PONG") {
return fmt.Errorf("mismatched output: wanted: PONG got: %q%s", string(out), podWaitSuffix)
}
return nil
}
var isUpErr error
serverUpCtx, serverUpCancel := context.WithTimeout(ctx, 100*time.Second)
defer serverUpCancel()
for serverUpCtx.Err() == nil {
if isUpErr = ensureUp(); isUpErr == nil {
break
}
}
if isUpErr != nil {
t.Fatalf("%s at IP %s did not come up: %v", server.GetName(), ip, isUpErr)
}
for _, connections := range numConnections {
t.Run(fmt.Sprintf("Connections_%d", connections), func(t *testing.T) {
for _, operation := range operations {
t.Run(operation, func(t *testing.T) {
// Create a client for this client run w/ the specified number of connections.
// Sadly the --csv mode only reports QPS, not latency. In order to report both,
// we need to parse the human-readable version of the output.
clientCmd := []string{
"redis-benchmark",
"-t", operation, // RPC to benchmark
"-h", ip, // Redis server IP
"-n", fmt.Sprintf("%d", defaultRequestsPerConnection*connections), // Number of total requests to do
"-c", fmt.Sprintf("%d", connections), // Number of threads to spread them over.
"-r", "1000", // Key space size (larger = more memory faults)
"--precision", "4", // Floating-point precision for reporting latency (in ms)
}
client := newRedisPod(benchmarkNS, "client", image, clientCmd)
client, err = cluster.ConfigurePodForClientNodepool(client)
if err != nil {
t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
client, err = cluster.CreatePod(ctx, client)
if err != nil {
t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, client)
if err := cluster.WaitForPodCompleted(ctx, client); err != nil {
t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
// get and parse the logs from the client to get the results
rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{})
if err != nil {
t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err)
}
out, err := io.ReadAll(rdr)
if err != nil {
t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err)
}
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
redisBenchmarkName := fmt.Sprintf("Redis/%s/%dClients/%s", test.name, connections, operation)
metrics, err := getMeasurements(string(out), operation)
if err != nil {
// Redis uses '\r' to update its status by overwriting the current line.
// If printed directly, this messes up the output.
// To make that clear, we replace '\r' with a literal
// backslash + 'r', and add a newline.
humanReadableOut := strings.ReplaceAll(string(out), "\r", "\\r\n")
t.Fatalf("failed to get metric for op %q: out:\n\n%s\n\nerr: %v", operation, humanReadableOut, err)
}
// We don't multiply `defaultRequestsPerConnection` by `connections` here
// because the number of "samples" we're testing is the number of times we
// can call an RPC from *that many connections* (which is part of the
// benchmark name).
// Adding 5x the number of connections does not make the sample size of this
// benchmark go 5x higher.
if err := recorder.RecordIters(ctx, redisBenchmarkName, defaultRequestsPerConnection, metrics...); err != nil {
t.Fatalf("Failed to record benchmark data for op %q: %v", operation, err)
}
})
if t.Failed() {
break
}
}
})
if t.Failed() {
break
}
}
})
if t.Failed() {
break
}
}
}
// newRedisService gets a service to serve traffic to the redis server.
func newRedisService(namespace *testcluster.Namespace, containerName string) *v13.Service {
name := fmt.Sprintf("redis-service-%d", time.Now().UnixNano())
return namespace.GetService(name, v13.ServiceSpec{
Selector: map[string]string{redisServerLabelKey: redisServerLabelValue},
Ports: []v13.ServicePort{
{
Name: name,
Protocol: v13.ProtocolTCP,
Port: redisPort,
TargetPort: intstr.FromString(containerName),
},
},
})
}
var (
latencyPercentileRegex = regexp.MustCompile("^([-,.\\d]+)% <=? ([-,.\\d]+) milliseconds(?: \\(cumulative count .*\\))?$")
latencyStartHeader = "Latency by percentile distribution:"
queriesPerSecondRegex = regexp.MustCompile("^throughput summary: ([-,.\\d]+) requests per second$")
)
func stringToFloat64(s string) float64 {
f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64)
if err != nil {
panic(fmt.Sprintf("cannot convert float %q: %v", s, err))
}
return f
}
// getMeasurements parses the output of redis-benchmark to get the stats.
func getMeasurements(out, operation string) ([]benchmetric.MetricValue, error) {
var currentOperation string
var returned []benchmetric.MetricValue
inLatencyBlock := false
foundPercentiles := make(map[int]bool, len(latencyPercentiles))
foundQPS := false
lastPercentile := -1.0
lastPercentileLatencyMs := math.NaN()
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
// The human-readable output contains a bunch of data like
// "OPERATION: number\r" which are used to update the result
// interactively. Strip them out here.
if strings.Contains(line, "\r") {
line = line[strings.LastIndex(line, "\r")+1:]
}
if strings.HasPrefix(line, "====== ") {
currentOperation = strings.SplitN(strings.Trim(line, "= "), " ", 2)[0]
}
if currentOperation != operation {
continue
}
if line == latencyStartHeader {
inLatencyBlock = true
continue
}
if inLatencyBlock {
latencyMatch := latencyPercentileRegex.FindStringSubmatch(line)
if latencyMatch != nil {
percentile := stringToFloat64(latencyMatch[1])
if percentile < lastPercentile {
continue
}
latencyMs := stringToFloat64(latencyMatch[2])
if percentile == 0 {
lastPercentile = 0
lastPercentileLatencyMs = latencyMs
continue
}
// Look for all percentiles in `wantPercentiles` that are in the range [lastPercentile,
// percentile].
var recordPercentiles []int
for _, wantPercentile := range latencyPercentiles {
if float64(wantPercentile) < lastPercentile {
continue
}
if float64(wantPercentile) > percentile {
continue
}
if foundPercentiles[wantPercentile] {
continue
}
recordPercentiles = append(recordPercentiles, wantPercentile)
}
for _, recordPercentile := range recordPercentiles {
// Linear interpolation of the latency value from within the latency range in the two
// percentile values that we got.
// For example, given p50=1.0ms and p70=2.0ms, we infer that p60=1.5ms.
// This isn't bulletproof but it is better than rounding to either end of the bucket.
rangeFraction := (float64(recordPercentile) - lastPercentile) / (percentile - lastPercentile)
pctileLatency := rangeFraction*(latencyMs-lastPercentileLatencyMs) + lastPercentileLatencyMs
returned = append(returned, benchmetric.SpecificDuration(time.Duration(pctileLatency*float64(time.Millisecond)), fmt.Sprintf("p%d", recordPercentile)))
foundPercentiles[recordPercentile] = true
}
// Update values for next round.
lastPercentile = percentile
lastPercentileLatencyMs = latencyMs
} else {
inLatencyBlock = false
}
continue
}
qpsMatch := queriesPerSecondRegex.FindStringSubmatch(line)
if qpsMatch != nil {
if foundQPS {
return nil, fmt.Errorf("found QPS value multiple times: %q", line)
}
foundQPS = true
returned = append(returned, benchmetric.RequestsPerSecond(stringToFloat64(qpsMatch[1])))
}
}
if !foundQPS || len(foundPercentiles) != len(latencyPercentiles) {
return nil, fmt.Errorf("did not find the data we wanted: foundQPS=%v foundPercentiles=%v", foundQPS, foundPercentiles)
}
return returned, nil
}
// newRedisPodWithPort returns a redis pod template.
func newRedisPodWithPort(namespace *testcluster.Namespace, name, image string, cmd []string, port int32, pvc *v13.PersistentVolumeClaim) *v13.Pod {
container := newRedisContainer(name, image, cmd)
container.Ports = append(container.Ports, v13.ContainerPort{Name: name, ContainerPort: port})
if pvc != nil {
container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{
Name: redisVolumeName,
MountPath: redisDataDirectory,
})
}
pod := namespace.NewPod(name)
pod.Spec.Containers = []v13.Container{container}
if pvc != nil {
pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{
Name: redisVolumeName,
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.GetName(),
},
},
})
}
return pod
}
// newRedisPod returns a redis pod template.
func newRedisPod(namespace *testcluster.Namespace, name, image string, cmd []string) *v13.Pod {
pod := namespace.NewPod(name)
pod.Spec.Containers = []v13.Container{newRedisContainer(name, image, cmd)}
return pod
}
// newRedisContainer returns a new redis container.
func newRedisContainer(name, image string, cmd []string) v13.Container {
return v13.Container{
Name: name,
Image: image,
Command: cmd,
}
}
test/kubernetes/benchmarks/redis_test.go
+2 -433
View File
@@ -12,46 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package redis_test holds the redis test where the runtime under test runs a redis server and the
// native runtime runs a client making requests against it.
package redis_test
package redis
import (
"context"
"fmt"
"io"
"math"
"regexp"
"strconv"
"strings"
"testing"
"time"
k8s "gvisor.dev/gvisor/test/kubernetes"
"gvisor.dev/gvisor/test/kubernetes/benchmarks/profiling"
"gvisor.dev/gvisor/test/kubernetes/benchmetric"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
v13 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
redisPort = 6379
defaultRequestsPerConnection = 50000
redisServerLabelKey = "app.kubernetes.io/name"
redisServerLabelValue = "redis-server"
redisVolumeName = "redis-data"
redisDataDirectory = "/redis-data"
redisImageAMD = k8s.ImageRepoPrefix + "benchmarks/redis_x86_64:latest"
redisImageARM = k8s.ImageRepoPrefix + "benchmarks/redis_aarch64:latest"
)
var (
numConnections = []int{1, 2, 4, 8, 16, 32}
latencyPercentiles = []int{50, 95, 99}
operations = []string{"SET", "GET", "MSET", "LPUSH", "LRANGE_500"}
)
// TestRedis benchmarks redis servers on k8s clusters.
@@ -64,410 +32,11 @@ func TestRedis(t *testing.T) {
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("Redis", func(t *testing.T) {
t.Parallel()
doRedisTest(ctx, t, k8sCtx, cluster)
BenchmarkRedis(ctx, t, k8sCtx, cluster)
})
})
}
// doRedisTest runs redis test.
func doRedisTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
}
defer benchmarkNS.Cleanup(ctx)
persistentVol := benchmarkNS.GetPersistentVolume(redisVolumeName, "30Gi")
persistentVol, err := cluster.CreatePersistentVolume(ctx, persistentVol)
if err != nil {
t.Fatalf("Failed to create persistent volume: %v", err)
}
defer cluster.DeletePersistentVolume(ctx, persistentVol)
image := redisImageAMD
if cluster.RuntimeTestNodepoolIsARM() {
image = redisImageARM
}
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
t.Fatalf("Failed to resolve image: %v", err)
}
for _, test := range []struct {
// Benchmark name
name string
// Suffix for the redis server, must be short to fit in pod name.
suffix string
// redis-server command-line.
serverCommand []string
// Volume to use for persistence, if any.
volume *v13.PersistentVolumeClaim
}{
{
name: "Persistence",
suffix: "persist",
serverCommand: []string{
"redis-server",
"--dir", redisDataDirectory,
// Default save settings per
// https://redis.io/docs/management/config-file/
"--save", "3600 1 300 100 60 10000",
},
volume: persistentVol,
},
{
name: "NoPersistence",
suffix: "nopersist",
serverCommand: []string{
"redis-server",
"--appendonly", "no",
"--save", "",
},
volume: nil,
},
} {
t.Run(test.name, func(t *testing.T) {
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
if err != nil {
t.Fatalf("Failed to setup profiling: %v", err)
}
defer endProfiling()
// Create a server on the runtime under test nodepool.
server := newRedisPodWithPort(benchmarkNS, fmt.Sprintf("redis-%s", test.suffix), image, test.serverCommand, redisPort, test.volume)
if server.ObjectMeta.Labels == nil {
server.ObjectMeta.Labels = make(map[string]string)
}
server.ObjectMeta.Labels[redisServerLabelKey] = redisServerLabelValue
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
if err != nil {
t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err)
}
server, err = testcluster.MaybeSetContainerResources(server, server.Spec.Containers[0].Name, testcluster.ContainerResourcesRequest{})
if err != nil {
t.Fatalf("MaybeSetContainerResources on cluster %q: %v", cluster.GetName(), err)
}
server, err = cluster.CreatePod(ctx, server)
if err != nil {
t.Fatalf("CreatePod on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, server)
if err := cluster.WaitForPodRunning(ctx, server); err != nil {
t.Fatalf("WaitForPodRunning on cluster %q pod: %q: %v", cluster.GetName(), server.GetName(), err)
}
// create a service to service traffic to the pod.
service := newRedisService(benchmarkNS, server.GetName())
service, err = cluster.CreateService(ctx, service)
if err != nil {
t.Fatalf("CreateService on cluster %q: %v", cluster.GetName(), err)
}
defer cluster.DeleteService(ctx, service)
if err := cluster.WaitForServiceReady(ctx, service); err != nil {
t.Fatalf("WaitForServiceReady on cluster %q: %v", cluster.GetName(), err)
}
ip := testcluster.GetIPFromService(service)
if ip == "" {
t.Fatalf("did not get valid ip: %s", ip)
}
// run the 'redis-cli' command to ping the server and make sure it is up. The "ping" request comes
// back with a "PONG" response. We repeat -r=5 times with a -i=1 second interval. If we
// get one PONG back then the server is considered up.
pingCmd := []string{"redis-cli", "-h", ip, "-r", "5", "-i", "1", "ping"}
ensureUp := func() error {
pinger := newRedisPod(benchmarkNS, fmt.Sprintf("rpinger-%s", test.suffix), image, pingCmd)
pinger, err = cluster.ConfigurePodForClientNodepool(pinger)
if err != nil {
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pinger.GetName(), err)
}
pinger, err = cluster.CreatePod(ctx, pinger)
if err != nil {
return fmt.Errorf("CreatePod %q on cluster %q: %v", pinger.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, pinger)
waitCtx, waitCancel := context.WithTimeout(ctx, 30*time.Second)
var podWaitSuffix string
if err := cluster.WaitForPodCompleted(waitCtx, pinger); err != nil {
podWaitSuffix = fmt.Sprintf(" (pod wait error: %v)", err)
}
waitCancel()
rdr, err := cluster.GetLogReader(ctx, pinger, v13.PodLogOptions{})
if err != nil {
return fmt.Errorf("GetLogReader on cluster %q: %v%s", cluster.GetName(), err, podWaitSuffix)
}
out, err := io.ReadAll(rdr)
if err != nil {
return fmt.Errorf("failed to read from pod: %q: %v%s", pinger.GetName(), err, podWaitSuffix)
}
if !strings.Contains(string(out), "PONG") {
return fmt.Errorf("mismatched output: wanted: PONG got: %q%s", string(out), podWaitSuffix)
}
return nil
}
var isUpErr error
serverUpCtx, serverUpCancel := context.WithTimeout(ctx, 100*time.Second)
defer serverUpCancel()
for serverUpCtx.Err() == nil {
if isUpErr = ensureUp(); isUpErr == nil {
break
}
}
if isUpErr != nil {
t.Fatalf("%s at IP %s did not come up: %v", server.GetName(), ip, isUpErr)
}
for _, connections := range numConnections {
t.Run(fmt.Sprintf("Connections_%d", connections), func(t *testing.T) {
for _, operation := range operations {
t.Run(operation, func(t *testing.T) {
// Create a client for this client run w/ the specified number of connections.
// Sadly the --csv mode only reports QPS, not latency. In order to report both,
// we need to parse the human-readable version of the output.
clientCmd := []string{
"redis-benchmark",
"-t", operation, // RPC to benchmark
"-h", ip, // Redis server IP
"-n", fmt.Sprintf("%d", defaultRequestsPerConnection*connections), // Number of total requests to do
"-c", fmt.Sprintf("%d", connections), // Number of threads to spread them over.
"-r", "1000", // Key space size (larger = more memory faults)
"--precision", "4", // Floating-point precision for reporting latency (in ms)
}
client := newRedisPod(benchmarkNS, "client", image, clientCmd)
client, err = cluster.ConfigurePodForClientNodepool(client)
if err != nil {
t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
client, err = cluster.CreatePod(ctx, client)
if err != nil {
t.Fatalf("CreatePod %q on cluster %q: %v", client.GetName(), cluster.GetName(), err)
}
defer cluster.DeletePod(ctx, client)
if err := cluster.WaitForPodCompleted(ctx, client); err != nil {
t.Fatalf("WaitForPodCompleted on cluster %q pod: %q: %v", cluster.GetName(), client.GetName(), err)
}
// get and parse the logs from the client to get the results
rdr, err := cluster.GetLogReader(ctx, client, v13.PodLogOptions{})
if err != nil {
t.Fatalf("GetLogReader on cluster %q: %v", cluster.GetName(), err)
}
out, err := io.ReadAll(rdr)
if err != nil {
t.Fatalf("failed to read from pod: %q: %v", client.GetName(), err)
}
recorder, err := benchmetric.GetRecorder(ctx)
if err != nil {
t.Fatalf("Failed to initialize benchmark recorder: %v", err)
}
redisBenchmarkName := fmt.Sprintf("Redis/%s/%dClients/%s", test.name, connections, operation)
metrics, err := getMeasurements(string(out), operation)
if err != nil {
// Redis uses '\r' to update its status by overwriting the current line.
// If printed directly, this messes up the output.
// To make that clear, we replace '\r' with a literal
// backslash + 'r', and add a newline.
humanReadableOut := strings.ReplaceAll(string(out), "\r", "\\r\n")
t.Fatalf("failed to get metric for op %q: out:\n\n%s\n\nerr: %v", operation, humanReadableOut, err)
}
// We don't multiply `defaultRequestsPerConnection` by `connections` here
// because the number of "samples" we're testing is the number of times we
// can call an RPC from *that many connections* (which is part of the
// benchmark name).
// Adding 5x the number of connections does not make the sample size of this
// benchmark go 5x higher.
if err := recorder.RecordIters(ctx, redisBenchmarkName, defaultRequestsPerConnection, metrics...); err != nil {
t.Fatalf("Failed to record benchmark data for op %q: %v", operation, err)
}
})
if t.Failed() {
break
}
}
})
if t.Failed() {
break
}
}
})
if t.Failed() {
break
}
}
}
// newRedisService gets a service to serve traffic to the redis server.
func newRedisService(namespace *testcluster.Namespace, containerName string) *v13.Service {
name := fmt.Sprintf("redis-service-%d", time.Now().UnixNano())
return namespace.GetService(name, v13.ServiceSpec{
Selector: map[string]string{redisServerLabelKey: redisServerLabelValue},
Ports: []v13.ServicePort{
{
Name: name,
Protocol: v13.ProtocolTCP,
Port: redisPort,
TargetPort: intstr.FromString(containerName),
},
},
})
}
var (
latencyPercentileRegex = regexp.MustCompile("^([-,.\\d]+)% <=? ([-,.\\d]+) milliseconds(?: \\(cumulative count .*\\))?$")
latencyStartHeader = "Latency by percentile distribution:"
queriesPerSecondRegex = regexp.MustCompile("^throughput summary: ([-,.\\d]+) requests per second$")
)
func stringToFloat64(s string) float64 {
f, err := strconv.ParseFloat(strings.ReplaceAll(s, ",", ""), 64)
if err != nil {
panic(fmt.Sprintf("cannot convert float %q: %v", s, err))
}
return f
}
// getMeasurements parses the output of redis-benchmark to get the stats.
func getMeasurements(out, operation string) ([]benchmetric.MetricValue, error) {
var currentOperation string
var returned []benchmetric.MetricValue
inLatencyBlock := false
foundPercentiles := make(map[int]bool, len(latencyPercentiles))
foundQPS := false
lastPercentile := -1.0
lastPercentileLatencyMs := math.NaN()
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
// The human-readable output contains a bunch of data like
// "OPERATION: number\r" which are used to update the result
// interactively. Strip them out here.
if strings.Contains(line, "\r") {
line = line[strings.LastIndex(line, "\r")+1:]
}
if strings.HasPrefix(line, "====== ") {
currentOperation = strings.SplitN(strings.Trim(line, "= "), " ", 2)[0]
}
if currentOperation != operation {
continue
}
if line == latencyStartHeader {
inLatencyBlock = true
continue
}
if inLatencyBlock {
latencyMatch := latencyPercentileRegex.FindStringSubmatch(line)
if latencyMatch != nil {
percentile := stringToFloat64(latencyMatch[1])
if percentile < lastPercentile {
continue
}
latencyMs := stringToFloat64(latencyMatch[2])
if percentile == 0 {
lastPercentile = 0
lastPercentileLatencyMs = latencyMs
continue
}
// Look for all percentiles in `wantPercentiles` that are in the range [lastPercentile,
// percentile].
var recordPercentiles []int
for _, wantPercentile := range latencyPercentiles {
if float64(wantPercentile) < lastPercentile {
continue
}
if float64(wantPercentile) > percentile {
continue
}
if foundPercentiles[wantPercentile] {
continue
}
recordPercentiles = append(recordPercentiles, wantPercentile)
}
for _, recordPercentile := range recordPercentiles {
// Linear interpolation of the latency value from within the latency range in the two
// percentile values that we got.
// For example, given p50=1.0ms and p70=2.0ms, we infer that p60=1.5ms.
// This isn't bulletproof but it is better than rounding to either end of the bucket.
rangeFraction := (float64(recordPercentile) - lastPercentile) / (percentile - lastPercentile)
pctileLatency := rangeFraction*(latencyMs-lastPercentileLatencyMs) + lastPercentileLatencyMs
returned = append(returned, benchmetric.SpecificDuration(time.Duration(pctileLatency*float64(time.Millisecond)), fmt.Sprintf("p%d", recordPercentile)))
foundPercentiles[recordPercentile] = true
}
// Update values for next round.
lastPercentile = percentile
lastPercentileLatencyMs = latencyMs
} else {
inLatencyBlock = false
}
continue
}
qpsMatch := queriesPerSecondRegex.FindStringSubmatch(line)
if qpsMatch != nil {
if foundQPS {
return nil, fmt.Errorf("found QPS value multiple times: %q", line)
}
foundQPS = true
returned = append(returned, benchmetric.RequestsPerSecond(stringToFloat64(qpsMatch[1])))
}
}
if !foundQPS || len(foundPercentiles) != len(latencyPercentiles) {
return nil, fmt.Errorf("did not find the data we wanted: foundQPS=%v foundPercentiles=%v", foundQPS, foundPercentiles)
}
return returned, nil
}
// newRedisPodWithPort returns a redis pod template.
func newRedisPodWithPort(namespace *testcluster.Namespace, name, image string, cmd []string, port int32, pvc *v13.PersistentVolumeClaim) *v13.Pod {
container := newRedisContainer(name, image, cmd)
container.Ports = append(container.Ports, v13.ContainerPort{Name: name, ContainerPort: port})
if pvc != nil {
container.VolumeMounts = append(container.VolumeMounts, v13.VolumeMount{
Name: redisVolumeName,
MountPath: redisDataDirectory,
})
}
pod := namespace.NewPod(name)
pod.Spec.Containers = []v13.Container{container}
if pvc != nil {
pod.Spec.Volumes = append(pod.Spec.Volumes, v13.Volume{
Name: redisVolumeName,
VolumeSource: v13.VolumeSource{
PersistentVolumeClaim: &v13.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.GetName(),
},
},
})
}
return pod
}
// newRedisPod returns a redis pod template.
func newRedisPod(namespace *testcluster.Namespace, name, image string, cmd []string) *v13.Pod {
pod := namespace.NewPod(name)
pod.Spec.Containers = []v13.Container{newRedisContainer(name, image, cmd)}
return pod
}
// newRedisContainer returns a new redis container.
func newRedisContainer(name, image string, cmd []string) v13.Container {
return v13.Container{
Name: name,
Image: image,
Command: cmd,
}
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestRedis": TestRedis,
test/kubernetes/benchmarks/ruby_dev_test.go → test/kubernetes/benchmarks/rubydev.go
+5 -24
View File
@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ruby_dev_test holds a benchmark to time a build job of a ruby application.
package ruby_dev_test
// Package rubydev holds a benchmark to time a build job of a ruby application.
package rubydev
import (
"bytes"
@@ -40,22 +40,9 @@ const (
imageARM = k8s.ImageRepoPrefix + "benchmarks/rubydev_aarch64:latest"
)
// TestRubyDev benchmarks a build job on k8s clusters.
func TestRubyDev(t *testing.T) {
ctx := context.Background()
k8sCtx, err := k8sctx.Context(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
k8sCtx.ForEachCluster(ctx, t, func(cluster *testcluster.TestCluster) {
t.Run("RubyDev", func(t *testing.T) {
t.Parallel()
doRubyDevTest(ctx, t, k8sCtx, cluster)
})
})
}
func doRubyDevTest(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
// RunRubyDev runs a benchmark measuring the time to build and test a
// popular Ruby library.
func RunRubyDev(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, cluster *testcluster.TestCluster) {
benchmarkNS := cluster.Namespace(testcluster.NamespaceBenchmark)
if err := benchmarkNS.Reset(ctx); err != nil {
t.Fatalf("cannot reset namespace: %v", err)
@@ -218,9 +205,3 @@ func newRubyDevPod(namespace *testcluster.Namespace, name, image string, volume
},
}
}
func TestMain(m *testing.M) {
k8sctx.TestMain(m, map[string]k8sctx.TestFunc{
"TestRubyDev": TestRubyDev,
})
}

Some files were not shown because too many files have changed in this diff Show More