diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index e2bda4374..4d05b894e 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -177,6 +177,11 @@ steps: - make gpu-smoke-tests agents: queue: gpu + - <<: *common + <<: *source_test_continuous + label: ":bun: COS Latest Driver Compatibility Test" + commands: + - tools/gpu/cos_drivers_test.sh - <<: *common <<: *source_test_continuous label: ":screwdriver: GPU Tests" @@ -212,7 +217,6 @@ steps: - tools/gpu/all_drivers_test.sh agents: queue: gpu - # Release workflow. - <<: *common <<: *source_test diff --git a/.buildkite/release.yaml b/.buildkite/release.yaml index e913feb68..186a1da9f 100644 --- a/.buildkite/release.yaml +++ b/.buildkite/release.yaml @@ -54,6 +54,10 @@ steps: commands: # The built website image must be x86_64. - make BAZEL_OPTIONS=--config=x86_64 website-deploy + - <<: *common + label: ":bun: COS Latest Driver Compatibility Test" + commands: + - tools/gpu/cos_drivers_test.sh - <<: *common label: ":female_supervillain: COS GPU Tests" commands: diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index bc4bc3346..65a24d687 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -63,6 +63,7 @@ go_library( ], visibility = [ "//pkg/sentry:internal", + "//test/gpu:__subpackages__", "//tools:__subpackages__", ], deps = [ diff --git a/test/gpu/BUILD b/test/gpu/BUILD index 290060f3e..fa819b278 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_test") +load("//tools:defs.bzl", "go_test", "proto_library") package( default_applicable_licenses = ["//:license"], @@ -140,3 +140,23 @@ go_test( visibility = ["//:sandbox"], deps = ["//pkg/test/dockerutil"], ) + +proto_library( + name = "gpu_driver_versions", + srcs = ["gpu_driver_versions.proto"], +) + +go_test( + name = "cos_gpu_compatibility_test", + srcs = ["cos_gpu_compatibility_test.go"], + tags = [ + "manual", + "noguitar", + "notap", + ], + deps = [ + ":gpu_driver_versions_go_proto", + "//pkg/sentry/devices/nvproxy", + "@org_golang_google_protobuf//encoding/prototext:go_default_library", + ], +) diff --git a/test/gpu/cos_gpu_compatibility_test.go b/test/gpu/cos_gpu_compatibility_test.go new file mode 100644 index 000000000..909436e51 --- /dev/null +++ b/test/gpu/cos_gpu_compatibility_test.go @@ -0,0 +1,142 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cos_gpu_compatibility_test checks the latest COS images' GPU drivers for +// gVisor compatibility. +package cos_gpu_compatibility_test + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "testing" + + "google.golang.org/protobuf/encoding/prototext" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" + cospb "gvisor.dev/gvisor/test/gpu/gpu_driver_versions_go_proto" +) + +var ( + imageJSON = flag.String("image_json", "", "Path to file containing the list of COS images") + unsupportedDevices = map[string]any{ + "NVIDIA_TESLA_V100": true, + "NVIDIA_TESLA_P100": true, + "NVIDIA_TESLA_P4": true, + "NO_GPU": true, + } +) + +func TestGPUDriversCompatibility(t *testing.T) { + content, err := os.ReadFile(*imageJSON) + if err != nil { + t.Fatalf("Failed to read image JSON file: %v", err) + } + images := []map[string]any{} + if err := json.Unmarshal(content, &images); err != nil { + t.Fatalf("Failed to unmarshal image JSON file: %v", err) + } + + for _, image := range images { + name := image["name"].(string) + family := image["family"].(string) + + t.Run(fmt.Sprintf("%s", name), func(t *testing.T) { + cosBranch, version, err := imageNameToCosPatchVersion(name, family) + if err != nil { + t.Fatalf("Failed to convert image name to COS version: %v", err) + } + + if cosBranch < 109 { + // As of writing, GKE is only on cos-109 and above. + t.Skipf("Skipping COS branch %d image: %q family: %q", cosBranch, name, family) + } + + driverVersions, err := listedDriverVersions(version) + if err != nil { + t.Fatalf("Failed to get listed driver versions: %v", err) + } + + list := cospb.GPUDriverVersionInfoList{} + if err := prototext.Unmarshal(driverVersions, &list); err != nil { + t.Fatalf("Failed to unmarshal driver versions: %v", err) + } + + supportedDrivers := map[string]bool{} + for _, driver := range nvproxy.SupportedDrivers() { + supportedDrivers[driver.String()] = true + } + + for _, info := range list.GetGpuDriverVersionInfo() { + if _, ok := unsupportedDevices[info.GetGpuDevice().GetGpuType()]; ok { + continue + } + + for _, driver := range info.GetSupportedDriverVersions() { + switch strings.ToLower(driver.GetLabel()) { + case "default": + case "latest": + default: + continue + } + + if !supportedDrivers[driver.GetVersion()] { + t.Errorf("Unsupported driver patch: %q gpu: %q version: %q", driver.GetVersion(), info.GetGpuDevice().GetGpuType(), driver.GetLabel()) + continue + } + t.Logf("Supported driver patch: %q gpu: %q version: %q", driver.GetVersion(), info.GetGpuDevice().GetGpuType(), driver.GetLabel()) + + } + } + }) + } +} + +func imageNameToCosPatchVersion(imageName string, family string) (int, string, error) { + var cosVersionRegex = regexp.MustCompile(`^cos-(?:arm64-)?(?:beta-|dev-|stable-)?(\d+)-(\d+)-(\d+)-(\d+)$`) + matches := cosVersionRegex.FindStringSubmatch(imageName) + if len(matches) != 5 { + return 0, "", fmt.Errorf("image name %q does not match regex %q", imageName, cosVersionRegex.String()) + } + cosBranch, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, "", fmt.Errorf("failed to convert COS branch to int: %w", err) + } + return cosBranch, fmt.Sprintf("%s.%s.%s", matches[2], matches[3], matches[4]), nil +} + +// listedDriverVersions returns the list of GPU driver versions listed for the given COS version. +func listedDriverVersions(cosVersion string) ([]byte, error) { + // Each entry on the COS release list has a corresponding textproto file with the list of GPU + // driver versions supported in that release. + // See: https://cloud.google.com/container-optimized-os/docs/release-notes + url := fmt.Sprintf("https://storage.googleapis.com/cos-tools/%s/lakitu/gpu_driver_versions.textproto", cosVersion) + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to get driver versions for release %q: %w", cosVersion, err) + } + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) +} + +func TestMain(m *testing.M) { + flag.Parse() + nvproxy.Init() + os.Exit(m.Run()) +} diff --git a/test/gpu/gpu_driver_versions.proto b/test/gpu/gpu_driver_versions.proto new file mode 100644 index 000000000..1c11da941 --- /dev/null +++ b/test/gpu/gpu_driver_versions.proto @@ -0,0 +1,54 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This proto file is copied from +// https://cos.googlesource.com/cos/tools/+/refs/heads/master/src/pkg/gpuconfig/proto/gpu_driver_versions.proto + +// gpu_driver_versions.proto schema +syntax = "proto3"; + +package cos_gpu; + +// Represents all the information about a GPU driver version. +// It contains the GPU driver version label (e.g., "default", "latest"), and the +// GPU driver version that is associated with that label. +message DriverVersion { + // GPU driver version label. + string label = 1; + + // The driver version that is associated with this label. + string version = 2; + // more metadata about the driver version. +} +// Represents the information about a GPU device. +message GPUDevice { + // GPU type. + string gpu_type = 1; + // reserved for other GPU Device info needed to be included in the proto + // e.g. string pci_value = 2; // PCI identifier for the GPU +} +// Represents GPU driver version information for a specific GPU type, mapping +// from GPU type to GPU driver version information. +message GPUDriverVersionInfo { + // Encapsulated GPU device info + GPUDevice gpu_device = 1; + // A repeated field to hold list of GPU driver versions with the associated + // labels that are built by cos. it is something like: + // <{"latest","535.129.11"}, ...> + repeated DriverVersion supported_driver_versions = 2; +} +// Represents a list of GPU driver version information. +message GPUDriverVersionInfoList { + repeated GPUDriverVersionInfo gpu_driver_version_info = 1; +} diff --git a/tools/gpu/cos_drivers_test.sh b/tools/gpu/cos_drivers_test.sh new file mode 100755 index 000000000..9bac01345 --- /dev/null +++ b/tools/gpu/cos_drivers_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2024 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to easily run gpu tests on all supported driver versions. This should +# be run from the gVisor repo root directory. +set -ueo pipefail + +json_file=$(mktemp /tmp/cos_gpu_compatibility_test.XXXXXX) +trap "rm -f ${json_file}" EXIT + +gcloud compute images list --project cos-cloud \ + --filter="family:cos*" --format json > "${json_file}" + +make run TARGETS=test/gpu:cos_gpu_compatibility_test ARGS="--image_json=${json_file}"