From 23c8b4b0425949865adb6496913688551a638794 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Wed, 6 Nov 2024 08:23:37 -0800 Subject: [PATCH] Add test to check COS drivers as they are posted. Our current check of COS drivers often lags behind COS releases. This is due to needing to preload GPU docker images onto the images that run in our CI pipelines. In addition, COS can be a bit more complex than originally thought releasing driver versions both across GPU types and release branches. Thus, this test searches the latest COS images on each family for new drivers. It does this by looking at COS's published release notes which include a proto of LATEST/DEFAULT drivers selected for each device. This will flag new versions faster with more coverage than our CI pipeline currently. Due to this not actually needing a GPU to run, this can run on any VM. PiperOrigin-RevId: 693736100 --- .buildkite/pipeline.yaml | 6 +- .buildkite/release.yaml | 4 + pkg/sentry/devices/nvproxy/BUILD | 1 + test/gpu/BUILD | 22 +++- test/gpu/cos_gpu_compatibility_test.go | 142 +++++++++++++++++++++++++ test/gpu/gpu_driver_versions.proto | 54 ++++++++++ tools/gpu/cos_drivers_test.sh | 27 +++++ 7 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 test/gpu/cos_gpu_compatibility_test.go create mode 100644 test/gpu/gpu_driver_versions.proto create mode 100755 tools/gpu/cos_drivers_test.sh diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index e2bda4374..4d05b894e 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -177,6 +177,11 @@ steps: - make gpu-smoke-tests agents: queue: gpu + - <<: *common + <<: *source_test_continuous + label: ":bun: COS Latest Driver Compatibility Test" + commands: + - tools/gpu/cos_drivers_test.sh - <<: *common <<: *source_test_continuous label: ":screwdriver: GPU Tests" @@ -212,7 +217,6 @@ steps: - tools/gpu/all_drivers_test.sh agents: queue: gpu - # Release workflow. - <<: *common <<: *source_test diff --git a/.buildkite/release.yaml b/.buildkite/release.yaml index e913feb68..186a1da9f 100644 --- a/.buildkite/release.yaml +++ b/.buildkite/release.yaml @@ -54,6 +54,10 @@ steps: commands: # The built website image must be x86_64. - make BAZEL_OPTIONS=--config=x86_64 website-deploy + - <<: *common + label: ":bun: COS Latest Driver Compatibility Test" + commands: + - tools/gpu/cos_drivers_test.sh - <<: *common label: ":female_supervillain: COS GPU Tests" commands: diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index bc4bc3346..65a24d687 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -63,6 +63,7 @@ go_library( ], visibility = [ "//pkg/sentry:internal", + "//test/gpu:__subpackages__", "//tools:__subpackages__", ], deps = [ diff --git a/test/gpu/BUILD b/test/gpu/BUILD index 290060f3e..fa819b278 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_test") +load("//tools:defs.bzl", "go_test", "proto_library") package( default_applicable_licenses = ["//:license"], @@ -140,3 +140,23 @@ go_test( visibility = ["//:sandbox"], deps = ["//pkg/test/dockerutil"], ) + +proto_library( + name = "gpu_driver_versions", + srcs = ["gpu_driver_versions.proto"], +) + +go_test( + name = "cos_gpu_compatibility_test", + srcs = ["cos_gpu_compatibility_test.go"], + tags = [ + "manual", + "noguitar", + "notap", + ], + deps = [ + ":gpu_driver_versions_go_proto", + "//pkg/sentry/devices/nvproxy", + "@org_golang_google_protobuf//encoding/prototext:go_default_library", + ], +) diff --git a/test/gpu/cos_gpu_compatibility_test.go b/test/gpu/cos_gpu_compatibility_test.go new file mode 100644 index 000000000..909436e51 --- /dev/null +++ b/test/gpu/cos_gpu_compatibility_test.go @@ -0,0 +1,142 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cos_gpu_compatibility_test checks the latest COS images' GPU drivers for +// gVisor compatibility. +package cos_gpu_compatibility_test + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "testing" + + "google.golang.org/protobuf/encoding/prototext" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" + cospb "gvisor.dev/gvisor/test/gpu/gpu_driver_versions_go_proto" +) + +var ( + imageJSON = flag.String("image_json", "", "Path to file containing the list of COS images") + unsupportedDevices = map[string]any{ + "NVIDIA_TESLA_V100": true, + "NVIDIA_TESLA_P100": true, + "NVIDIA_TESLA_P4": true, + "NO_GPU": true, + } +) + +func TestGPUDriversCompatibility(t *testing.T) { + content, err := os.ReadFile(*imageJSON) + if err != nil { + t.Fatalf("Failed to read image JSON file: %v", err) + } + images := []map[string]any{} + if err := json.Unmarshal(content, &images); err != nil { + t.Fatalf("Failed to unmarshal image JSON file: %v", err) + } + + for _, image := range images { + name := image["name"].(string) + family := image["family"].(string) + + t.Run(fmt.Sprintf("%s", name), func(t *testing.T) { + cosBranch, version, err := imageNameToCosPatchVersion(name, family) + if err != nil { + t.Fatalf("Failed to convert image name to COS version: %v", err) + } + + if cosBranch < 109 { + // As of writing, GKE is only on cos-109 and above. + t.Skipf("Skipping COS branch %d image: %q family: %q", cosBranch, name, family) + } + + driverVersions, err := listedDriverVersions(version) + if err != nil { + t.Fatalf("Failed to get listed driver versions: %v", err) + } + + list := cospb.GPUDriverVersionInfoList{} + if err := prototext.Unmarshal(driverVersions, &list); err != nil { + t.Fatalf("Failed to unmarshal driver versions: %v", err) + } + + supportedDrivers := map[string]bool{} + for _, driver := range nvproxy.SupportedDrivers() { + supportedDrivers[driver.String()] = true + } + + for _, info := range list.GetGpuDriverVersionInfo() { + if _, ok := unsupportedDevices[info.GetGpuDevice().GetGpuType()]; ok { + continue + } + + for _, driver := range info.GetSupportedDriverVersions() { + switch strings.ToLower(driver.GetLabel()) { + case "default": + case "latest": + default: + continue + } + + if !supportedDrivers[driver.GetVersion()] { + t.Errorf("Unsupported driver patch: %q gpu: %q version: %q", driver.GetVersion(), info.GetGpuDevice().GetGpuType(), driver.GetLabel()) + continue + } + t.Logf("Supported driver patch: %q gpu: %q version: %q", driver.GetVersion(), info.GetGpuDevice().GetGpuType(), driver.GetLabel()) + + } + } + }) + } +} + +func imageNameToCosPatchVersion(imageName string, family string) (int, string, error) { + var cosVersionRegex = regexp.MustCompile(`^cos-(?:arm64-)?(?:beta-|dev-|stable-)?(\d+)-(\d+)-(\d+)-(\d+)$`) + matches := cosVersionRegex.FindStringSubmatch(imageName) + if len(matches) != 5 { + return 0, "", fmt.Errorf("image name %q does not match regex %q", imageName, cosVersionRegex.String()) + } + cosBranch, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, "", fmt.Errorf("failed to convert COS branch to int: %w", err) + } + return cosBranch, fmt.Sprintf("%s.%s.%s", matches[2], matches[3], matches[4]), nil +} + +// listedDriverVersions returns the list of GPU driver versions listed for the given COS version. +func listedDriverVersions(cosVersion string) ([]byte, error) { + // Each entry on the COS release list has a corresponding textproto file with the list of GPU + // driver versions supported in that release. + // See: https://cloud.google.com/container-optimized-os/docs/release-notes + url := fmt.Sprintf("https://storage.googleapis.com/cos-tools/%s/lakitu/gpu_driver_versions.textproto", cosVersion) + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to get driver versions for release %q: %w", cosVersion, err) + } + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) +} + +func TestMain(m *testing.M) { + flag.Parse() + nvproxy.Init() + os.Exit(m.Run()) +} diff --git a/test/gpu/gpu_driver_versions.proto b/test/gpu/gpu_driver_versions.proto new file mode 100644 index 000000000..1c11da941 --- /dev/null +++ b/test/gpu/gpu_driver_versions.proto @@ -0,0 +1,54 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This proto file is copied from +// https://cos.googlesource.com/cos/tools/+/refs/heads/master/src/pkg/gpuconfig/proto/gpu_driver_versions.proto + +// gpu_driver_versions.proto schema +syntax = "proto3"; + +package cos_gpu; + +// Represents all the information about a GPU driver version. +// It contains the GPU driver version label (e.g., "default", "latest"), and the +// GPU driver version that is associated with that label. +message DriverVersion { + // GPU driver version label. + string label = 1; + + // The driver version that is associated with this label. + string version = 2; + // more metadata about the driver version. +} +// Represents the information about a GPU device. +message GPUDevice { + // GPU type. + string gpu_type = 1; + // reserved for other GPU Device info needed to be included in the proto + // e.g. string pci_value = 2; // PCI identifier for the GPU +} +// Represents GPU driver version information for a specific GPU type, mapping +// from GPU type to GPU driver version information. +message GPUDriverVersionInfo { + // Encapsulated GPU device info + GPUDevice gpu_device = 1; + // A repeated field to hold list of GPU driver versions with the associated + // labels that are built by cos. it is something like: + // <{"latest","535.129.11"}, ...> + repeated DriverVersion supported_driver_versions = 2; +} +// Represents a list of GPU driver version information. +message GPUDriverVersionInfoList { + repeated GPUDriverVersionInfo gpu_driver_version_info = 1; +} diff --git a/tools/gpu/cos_drivers_test.sh b/tools/gpu/cos_drivers_test.sh new file mode 100755 index 000000000..9bac01345 --- /dev/null +++ b/tools/gpu/cos_drivers_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2024 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to easily run gpu tests on all supported driver versions. This should +# be run from the gVisor repo root directory. +set -ueo pipefail + +json_file=$(mktemp /tmp/cos_gpu_compatibility_test.XXXXXX) +trap "rm -f ${json_file}" EXIT + +gcloud compute images list --project cos-cloud \ + --filter="family:cos*" --format json > "${json_file}" + +make run TARGETS=test/gpu:cos_gpu_compatibility_test ARGS="--image_json=${json_file}"