Add test to check COS drivers as they are posted.

Our current check of COS drivers often lags behind COS releases.
This is due to needing to preload GPU docker images onto the
images that run in our CI pipelines.

In addition, COS can be a bit more complex than originally thought
releasing driver versions both across GPU types and release branches.

Thus, this test searches the latest COS images on each family for
new drivers. It does this by looking at COS's published release notes
which include a proto of LATEST/DEFAULT drivers selected for each device.

This will flag new versions faster with more coverage than our
CI pipeline currently. Due to this not actually needing a GPU
to run, this can run on any VM.

PiperOrigin-RevId: 693736100

.buildkite/pipeline.yaml

@@ -177,6 +177,11 @@ steps:
       - make gpu-smoke-tests
     agents:
       queue: gpu
+  - <<: *common
+    <<: *source_test_continuous
+    label: ":bun: COS Latest Driver Compatibility Test"
+    commands:
+      - tools/gpu/cos_drivers_test.sh
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: GPU Tests"
@@ -212,7 +217,6 @@ steps:
       - tools/gpu/all_drivers_test.sh
     agents:
       queue: gpu
-
   # Release workflow.
   - <<: *common
     <<: *source_test

.buildkite/release.yaml

@@ -54,6 +54,10 @@ steps:
     commands:
       # The built website image must be x86_64.
       - make BAZEL_OPTIONS=--config=x86_64 website-deploy
+  - <<: *common
+    label: ":bun: COS Latest Driver Compatibility Test"
+    commands:
+      - tools/gpu/cos_drivers_test.sh
   - <<: *common
     label: ":female_supervillain: COS GPU Tests"
     commands:

pkg/sentry/devices/nvproxy/BUILD

@@ -63,6 +63,7 @@ go_library(
     ],
     visibility = [
         "//pkg/sentry:internal",
+        "//test/gpu:__subpackages__",
         "//tools:__subpackages__",
     ],
     deps = [

test/gpu/BUILD

@@ -1,4 +1,4 @@
-load("//tools:defs.bzl", "go_test")
+load("//tools:defs.bzl", "go_test", "proto_library")
 
 package(
     default_applicable_licenses = ["//:license"],
@@ -140,3 +140,23 @@ go_test(
     visibility = ["//:sandbox"],
     deps = ["//pkg/test/dockerutil"],
 )
+
+proto_library(
+    name = "gpu_driver_versions",
+    srcs = ["gpu_driver_versions.proto"],
+)
+
+go_test(
+    name = "cos_gpu_compatibility_test",
+    srcs = ["cos_gpu_compatibility_test.go"],
+    tags = [
+        "manual",
+        "noguitar",
+        "notap",
+    ],
+    deps = [
+        ":gpu_driver_versions_go_proto",
+        "//pkg/sentry/devices/nvproxy",
+        "@org_golang_google_protobuf//encoding/prototext:go_default_library",
+    ],
+)

test/gpu/cos_gpu_compatibility_test.go

@@ -0,0 +1,142 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cos_gpu_compatibility_test checks the latest COS images' GPU drivers for
+// gVisor compatibility.
+package cos_gpu_compatibility_test
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+
+	"google.golang.org/protobuf/encoding/prototext"
+	"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy"
+	cospb "gvisor.dev/gvisor/test/gpu/gpu_driver_versions_go_proto"
+)
+
+var (
+	imageJSON          = flag.String("image_json", "", "Path to file containing the list of COS images")
+	unsupportedDevices = map[string]any{
+		"NVIDIA_TESLA_V100": true,
+		"NVIDIA_TESLA_P100": true,
+		"NVIDIA_TESLA_P4":   true,
+		"NO_GPU":            true,
+	}
+)
+
+func TestGPUDriversCompatibility(t *testing.T) {
+	content, err := os.ReadFile(*imageJSON)
+	if err != nil {
+		t.Fatalf("Failed to read image JSON file: %v", err)
+	}
+	images := []map[string]any{}
+	if err := json.Unmarshal(content, &images); err != nil {
+		t.Fatalf("Failed to unmarshal image JSON file: %v", err)
+	}
+
+	for _, image := range images {
+		name := image["name"].(string)
+		family := image["family"].(string)
+
+		t.Run(fmt.Sprintf("%s", name), func(t *testing.T) {
+			cosBranch, version, err := imageNameToCosPatchVersion(name, family)
+			if err != nil {
+				t.Fatalf("Failed to convert image name to COS version: %v", err)
+			}
+
+			if cosBranch < 109 {
+				// As of writing, GKE is only on cos-109 and above.
+				t.Skipf("Skipping COS branch %d image: %q family: %q", cosBranch, name, family)
+			}
+
+			driverVersions, err := listedDriverVersions(version)
+			if err != nil {
+				t.Fatalf("Failed to get listed driver versions: %v", err)
+			}
+
+			list := cospb.GPUDriverVersionInfoList{}
+			if err := prototext.Unmarshal(driverVersions, &list); err != nil {
+				t.Fatalf("Failed to unmarshal driver versions: %v", err)
+			}
+
+			supportedDrivers := map[string]bool{}
+			for _, driver := range nvproxy.SupportedDrivers() {
+				supportedDrivers[driver.String()] = true
+			}
+
+			for _, info := range list.GetGpuDriverVersionInfo() {
+				if _, ok := unsupportedDevices[info.GetGpuDevice().GetGpuType()]; ok {
+					continue
+				}
+
+				for _, driver := range info.GetSupportedDriverVersions() {
+					switch strings.ToLower(driver.GetLabel()) {
+					case "default":
+					case "latest":
+					default:
+						continue
+					}
+
+					if !supportedDrivers[driver.GetVersion()] {
+						t.Errorf("Unsupported driver patch: %q gpu: %q version: %q", driver.GetVersion(), info.GetGpuDevice().GetGpuType(), driver.GetLabel())
+						continue
+					}
+					t.Logf("Supported driver patch: %q gpu: %q version: %q", driver.GetVersion(), info.GetGpuDevice().GetGpuType(), driver.GetLabel())
+
+				}
+			}
+		})
+	}
+}
+
+func imageNameToCosPatchVersion(imageName string, family string) (int, string, error) {
+	var cosVersionRegex = regexp.MustCompile(`^cos-(?:arm64-)?(?:beta-|dev-|stable-)?(\d+)-(\d+)-(\d+)-(\d+)$`)
+	matches := cosVersionRegex.FindStringSubmatch(imageName)
+	if len(matches) != 5 {
+		return 0, "", fmt.Errorf("image name %q does not match regex %q", imageName, cosVersionRegex.String())
+	}
+	cosBranch, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return 0, "", fmt.Errorf("failed to convert COS branch to int: %w", err)
+	}
+	return cosBranch, fmt.Sprintf("%s.%s.%s", matches[2], matches[3], matches[4]), nil
+}
+
+// listedDriverVersions returns the list of GPU driver versions listed for the given COS version.
+func listedDriverVersions(cosVersion string) ([]byte, error) {
+	// Each entry on the COS release list has a corresponding textproto file with the list of GPU
+	// driver versions supported in that release.
+	// See: https://cloud.google.com/container-optimized-os/docs/release-notes
+	url := fmt.Sprintf("https://storage.googleapis.com/cos-tools/%s/lakitu/gpu_driver_versions.textproto", cosVersion)
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get driver versions for release %q: %w", cosVersion, err)
+	}
+	defer resp.Body.Close()
+	return ioutil.ReadAll(resp.Body)
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	nvproxy.Init()
+	os.Exit(m.Run())
+}

test/gpu/gpu_driver_versions.proto

@@ -0,0 +1,54 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This proto file is copied from
+// https://cos.googlesource.com/cos/tools/+/refs/heads/master/src/pkg/gpuconfig/proto/gpu_driver_versions.proto
+
+// gpu_driver_versions.proto schema
+syntax = "proto3";
+
+package cos_gpu;
+
+// Represents all the information about a GPU driver version.
+// It contains the GPU driver version label (e.g., "default", "latest"), and the
+// GPU driver version that is associated with that label.
+message DriverVersion {
+  // GPU driver version label.
+  string label = 1;
+
+  // The driver version that is associated with this label.
+  string version = 2;
+  // more metadata about the driver version.
+}
+// Represents the information about a GPU device.
+message GPUDevice {
+  // GPU type.
+  string gpu_type = 1;
+  // reserved for other GPU Device info needed to be included in the proto
+  // e.g. string pci_value = 2; // PCI identifier for the GPU
+}
+// Represents GPU driver version information for a specific GPU type, mapping
+// from GPU type to GPU driver version information.
+message GPUDriverVersionInfo {
+  // Encapsulated GPU device info
+  GPUDevice gpu_device = 1;
+  // A repeated field to hold list of GPU driver versions with the associated
+  // labels that are built by cos. it is something like:
+  // <{"latest","535.129.11"}, ...>
+  repeated DriverVersion supported_driver_versions = 2;
+}
+// Represents a list of GPU driver version information.
+message GPUDriverVersionInfoList {
+  repeated GPUDriverVersionInfo gpu_driver_version_info = 1;
+}

tools/gpu/cos_drivers_test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2024 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script to easily run gpu tests on all supported driver versions. This should
+# be run from the gVisor repo root directory.
+set -ueo pipefail
+
+json_file=$(mktemp /tmp/cos_gpu_compatibility_test.XXXXXX)
+trap "rm -f ${json_file}" EXIT
+
+gcloud compute images list --project cos-cloud \
+  --filter="family:cos*"  --format json > "${json_file}"
+
+make run TARGETS=test/gpu:cos_gpu_compatibility_test ARGS="--image_json=${json_file}"