Kubernetes benchmarks: Refactor kubectl context to allow sets of clusters.

Also add sanity check to make sure each cluster works as part of
initialization, by running a sample pod within.

PiperOrigin-RevId: 710786751
This commit is contained in:
Etienne Perot
2024-12-30 13:47:04 -08:00
committed by gVisor bot
parent 1db44ead0c
commit 4dbba85b07
25 changed files with 283 additions and 133 deletions
+14 -14
View File
@@ -58,7 +58,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -89,7 +89,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -122,7 +122,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -155,7 +155,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -188,7 +188,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -222,7 +222,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -254,7 +254,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -287,7 +287,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -320,7 +320,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -352,7 +352,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -392,7 +392,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -426,7 +426,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -458,7 +458,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
@@ -491,7 +491,7 @@ go_test(
],
deps = [
"//test/kubernetes/k8sctx",
"//test/kubernetes/k8sctx/autok8sctx",
"//test/kubernetes/k8sctx/kubectlctx",
"//test/kubernetes/testcluster",
],
)
+2 -2
View File
@@ -19,14 +19,14 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
// TestABSLBuild benchmarks building various Abseil C++ targets.
func TestABSLBuild(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestFfmpeg(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestGRPCBuild(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -21,13 +21,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestGSUtil(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestNginx(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -22,7 +22,7 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
@@ -30,7 +30,7 @@ func TestOllama(t *testing.T) {
fmt.Fprint(os.Stderr, "HEADS UP: This test uses a huge container image which may take up to 30 minutes to download onto nodes the first time you run it.\n")
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
@@ -20,14 +20,14 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
// TestPostgresPGBench benchmarks a PostgreSQL database with pgbench.
func TestPostgresPGBench(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,7 +19,7 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
@@ -49,7 +49,7 @@ func TestMobileNetV2(t *testing.T) {
}
func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) {
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,14 +19,14 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
// TestRedis benchmarks redis servers on k8s clusters.
func TestRedis(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,14 +19,14 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
// TestRubyDev benchmarks a build job on k8s clusters.
func TestRubyDev(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestStableDiffusionXL(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestStartup(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestTensorflowOnCPU(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
+2 -2
View File
@@ -19,13 +19,13 @@ import (
"testing"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx"
"gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
)
func TestWordpress(t *testing.T) {
ctx := context.Background()
k8sCtx, err := autok8sctx.New(ctx)
k8sCtx, err := kubectlctx.New(ctx)
if err != nil {
t.Fatalf("Failed to get kubernetes context: %v", err)
}
-1
View File
@@ -7,7 +7,6 @@ package(
go_library(
name = "k8sctx",
testonly = True,
srcs = [
"k8sctx.go",
],
@@ -1,50 +0,0 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !false
// +build !false
// Package autok8sctx provides a KubernetesContext that uses a kubectl config
// and context to determine the test cluster to use for tests and benchmarks.
// See parent package (`k8sctx`) for more info.
package autok8sctx
import (
"context"
"errors"
"fmt"
"gvisor.dev/gvisor/runsc/flag"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
"gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl"
)
var (
kubectlContextName = flag.String("kubectl-context-name", "", "Name of the kubectl context to use within the kubectl config")
)
// New creates a KubernetesContext using flags to determine which kubectl
// config and kubectl context to use as the test cluster.
func New(ctx context.Context) (k8sctx.KubernetesContext, error) {
if *kubectlContextName == "" {
return nil, errors.New("no kubectl context name specified")
}
cluster, err := kubectl.NewCluster(*kubectlContextName)
if err != nil {
return nil, fmt.Errorf("cannot initialize cluster %q: %w", *kubectlContextName, err)
}
testCluster := testcluster.NewTestClusterFromClient(*kubectlContextName, cluster.Client())
return k8sctx.NewSingleCluster(testCluster), nil
}
+18 -20
View File
@@ -21,7 +21,6 @@ package k8sctx
import (
"context"
"sync"
"testing"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
@@ -66,34 +65,33 @@ func ForEachCluster(ctx context.Context, t *testing.T, k8sCtx KubernetesContext,
}
}
// singleCluster implements KubernetesContext using a single cluster.
type singleCluster struct {
mu sync.Mutex
cluster *testcluster.TestCluster
// clusters implements KubernetesContext using a set of clusters.
type clusters struct {
clustersCh chan *testcluster.TestCluster
}
// Cluster implements KubernetesContext.Cluster.
func (sc *singleCluster) Cluster(ctx context.Context, t *testing.T) (*testcluster.TestCluster, func()) {
sc.mu.Lock()
defer sc.mu.Unlock()
cl := sc.cluster
sc.cluster = nil
return cl, func() {
if cl != nil {
sc.mu.Lock()
defer sc.mu.Unlock()
sc.cluster = cl
func (cs *clusters) Cluster(ctx context.Context, t *testing.T) (*testcluster.TestCluster, func()) {
select {
case cluster := <-cs.clustersCh:
return cluster, func() {
cs.clustersCh <- cluster
}
default:
return nil, func() {}
}
}
// ResolveImage implements KubernetesContext.ResolveImage.
func (*singleCluster) ResolveImage(ctx context.Context, imageName string) (string, error) {
func (*clusters) ResolveImage(ctx context.Context, imageName string) (string, error) {
return imageName, nil
}
// NewSingleCluster creates a KubernetesContext that uses a single, static
// test cluster.
func NewSingleCluster(cluster *testcluster.TestCluster) KubernetesContext {
return &singleCluster{cluster: cluster}
// New creates a KubernetesContext that set of test clusters.
func New(testClusters ...*testcluster.TestCluster) KubernetesContext {
clustersCh := make(chan *testcluster.TestCluster, len(testClusters))
for _, cluster := range testClusters {
clustersCh <- cluster
}
return &clusters{clustersCh: clustersCh}
}
@@ -6,8 +6,7 @@ package(
)
go_library(
name = "autok8sctx",
testonly = True,
name = "kubectlctx",
srcs = ["kubectlctx.go"],
nogo = False,
visibility = [
@@ -15,8 +14,13 @@ go_library(
],
deps = [
"//runsc/flag",
"//test/kubernetes:test_range_config_go_proto",
"//test/kubernetes/k8sctx",
"//test/kubernetes/testcluster",
"//tools/gvisor_k8s_tool/provider/kubectl",
"@io_k8s_client_go//kubernetes:go_default_library",
"@io_k8s_client_go//tools/clientcmd:go_default_library",
"@org_golang_google_protobuf//encoding/prototext:go_default_library",
"@org_golang_x_sync//errgroup:go_default_library",
],
)
@@ -0,0 +1,142 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package kubectlctx provides a KubernetesContext that uses one or more
// kubectl configs to determine the cluster(s) to use for tests and benchmarks.
// See parent package (`k8sctx`) for more info.
package kubectlctx
import (
"context"
"fmt"
"os"
"path/filepath"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/encoding/prototext"
"gvisor.dev/gvisor/runsc/flag"
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto"
"gvisor.dev/gvisor/test/kubernetes/testcluster"
"gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
var (
rangeDir = flag.String("range-dir", "", "A directory containing a test_range.textproto text file describing multiple clusters to use for tests and benchmarks; takes precedence over --kubectl-context-name")
kubectlContext = flag.String("kubectl-context", "", "The name of the kubectl context to use; if unset, use the default context within the kubectl config at KUBECONFIG")
testNodepoolRuntime = flag.String("test-nodepool-runtime", "", "if set, override the runtime used for pods scheduled on the 'test' nodepool. If unset, the nodepool default is used")
)
// New creates a KubernetesContext using flags to determine which clusters
// to use for tests and benchmarks.
func New(ctx context.Context) (k8sctx.KubernetesContext, error) {
if *rangeDir != "" && *kubectlContext != "" {
return nil, fmt.Errorf("cannot use --range-dir and --kubectl-context at the same time")
}
var clusters []*testcluster.TestCluster
var err error
if *rangeDir != "" {
clusters, err = NewFromRangeDir(ctx, *rangeDir)
} else {
clusters, err = NewFromKubectlContext(ctx, *kubectlContext)
}
if err != nil {
return nil, fmt.Errorf("cannot initialize test clusters: %w", err)
}
if *testNodepoolRuntime != "" {
overriddenRuntime := testcluster.RuntimeType(*testNodepoolRuntime)
if !overriddenRuntime.IsValid() {
return nil, fmt.Errorf("invalid runtime type %q", *testNodepoolRuntime)
}
for _, cluster := range clusters {
cluster.OverrideTestNodepoolRuntime(overriddenRuntime)
}
}
if err := verifyClusters(ctx, clusters); err != nil {
return nil, fmt.Errorf("cannot verify clusters are working: %w", err)
}
return k8sctx.New(clusters...), nil
}
// NewFromRangeDir creates a set of test clusters from a test range directory.
func NewFromRangeDir(ctx context.Context, rangeDir string) ([]*testcluster.TestCluster, error) {
rangeFile := filepath.Join(rangeDir, "test_range.textproto")
rangeFileData, err := os.ReadFile(rangeFile)
if err != nil {
return nil, fmt.Errorf("cannot read range file %q: %w", rangeFile, err)
}
var testRange testpb.TestRange
if err := prototext.Unmarshal(rangeFileData, &testRange); err != nil {
return nil, fmt.Errorf("error unmarshalling range file %q: %v", rangeFile, err)
}
if len(testRange.GetClusters()) == 0 {
return nil, fmt.Errorf("range file %q has no clusters", rangeFile)
}
clusters := make([]*testcluster.TestCluster, len(testRange.GetClusters()))
for i, cluster := range testRange.GetClusters() {
configPath := cluster.GetKubectlConfig()
if configPath == "" {
return nil, fmt.Errorf("cluster %q has no kubectl config path", cluster.GetCluster())
}
cfg, err := clientcmd.LoadFromFile(configPath)
if err != nil {
return nil, fmt.Errorf("cannot load kubectl config at %q for cluster %q: %w", configPath, cluster.GetCluster(), err)
}
contextName := cluster.GetKubectlContext()
if contextName == "" {
contextName = cfg.CurrentContext
}
restConfig, err := clientcmd.NewNonInteractiveClientConfig(*cfg, contextName, nil, clientcmd.NewDefaultClientConfigLoadingRules()).ClientConfig()
if err != nil {
return nil, fmt.Errorf("cannot load REST client config for cluster %q: %w", cluster.GetCluster(), err)
}
kubeClient, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return nil, fmt.Errorf("cannot create Kubernetes client for cluster %q: %w", cluster.GetCluster(), err)
}
clusters[i] = testcluster.NewTestClusterFromClient(cluster.GetCluster(), kubeClient)
}
return clusters, nil
}
// NewFromKubectlContext creates a test cluster from a kubectl config.
//
// If the kubectl config is not specified, the default kubectl config is used.
// If the kubectl context is not specified, the default context within the
// kubectl config is used.
func NewFromKubectlContext(ctx context.Context, kubectlContext string) ([]*testcluster.TestCluster, error) {
cluster, err := kubectl.NewCluster(kubectlContext)
if err != nil {
return nil, fmt.Errorf("cannot initialize cluster from kubectl config: %w", err)
}
clusterName := "test-cluster" // Default name.
if kubectlContext != "" {
clusterName = kubectlContext
}
return []*testcluster.TestCluster{testcluster.NewTestClusterFromClient(clusterName, cluster.Client())}, nil
}
// verifyClusters verifies that all clusters are working.
func verifyClusters(ctx context.Context, clusters []*testcluster.TestCluster) error {
var g errgroup.Group
for _, cluster := range clusters {
c := cluster
g.Go(func() error {
return c.SanityCheck(ctx)
})
}
return g.Wait()
}

Some files were not shown because too many files have changed in this diff Show More