Add script to run gpu test for all supported driver versions.

Add script and subsequent calls in the buildkite pipeline to run
gpu tests on all supported drivers.

In addition, add an outfile flag to the "list" command for the
driver installer which allows us to get the list of drivers
and use it in the script (instead of a bunch of output from
make/bazel/other stuff).

PiperOrigin-RevId: 575870051

.buildkite/pipeline.yaml

@@ -171,10 +171,9 @@ steps:
   # GPU workflow.
   - <<: *common
     <<: *source_test
-    label: "GPU tests"
+    label: ":screwdriver: All GPU Drivers Test"
     commands:
-      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
-      - make gpu-tests
+      - tools/gpu/all_drivers_test.sh
     agents:
       queue: gpu
   # Release workflow.

tools/gpu/all_drivers_test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2023 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script to easily run gpu tests on all supported driver versions. This should
+# be run from the gVisor repo root directory.
+set -ueo pipefail
+
+tmp_file=$(mktemp)
+trap "rm -f ${tmp_file}" EXIT
+
+make sudo TARGETS=tools/gpu:main ARGS="list --outfile=${tmp_file}"
+read -r -a versions <<< "$(cat "${tmp_file}")"
+
+for driver in "${versions[@]}"; do
+  make sudo TARGETS=tools/gpu:main ARGS="install --version ${driver}"
+  make gpu-tests
+done

tools/gpu/drivers/install_driver.go

@@ -194,13 +194,26 @@ func (i *Installer) getRequestedDriver() (nvproxy.DriverVersion, bool) {
 
 // ListSupportedDrivers prints the driver to stderr in a format that can be
 // consumed by the Makefile to iterate tests across drivers.
-func ListSupportedDrivers() {
+func ListSupportedDrivers(outfile string) error {
+	out := os.Stdout
+	if outfile != "" {
+		f, err := os.OpenFile(outfile, os.O_WRONLY, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to open outfile: %w", err)
+		}
+		defer f.Close()
+		out = f
+	}
+
 	supportedDrivers := nvproxy.GetSupportedDriversAndChecksums()
 	list := make([]string, 0, len(supportedDrivers))
-	for version := range nvproxy.GetSupportedDriversAndChecksums() {
+	for version := range supportedDrivers {
 		list = append(list, version.String())
 	}
-	fmt.Println(strings.Join(list, " "))
+	if _, err := out.WriteString(strings.Join(list, " ") + "\n"); err != nil {
+		return fmt.Errorf("failed to write to outfile: %w", err)
+	}
+	return nil
 }
 
 // ChecksumDriver downloads and returns the SHA265 checksum of the driver.
@@ -251,14 +264,8 @@ func installDriver(driverPath string) error {
 	cmd := exec.Command(driverPath, driverArgs...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	/*
-		cmd.Env = append(os.Environ(),
-			"IGNORE_CC_MISMATCH=1",
-			"LLVM=1",
-			"LLVM_IS=1",
-		)
-	*/
 	if err := cmd.Run(); err != nil {
+		tryToPrintFailureLogs()
 		return fmt.Errorf("failed to run nvidia-install: %w out: %s", err, string(out))
 	}
 
@@ -266,7 +273,28 @@ func installDriver(driverPath string) error {
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed to run nvidia-install: %w out: %s", err, string(out))
+		return fmt.Errorf("failed to run nvidia-smi post install: %w out: %s", err, string(out))
 	}
 	return nil
 }
+
+func tryToPrintFailureLogs() {
+	// nvidia driver installers print failure logs to this path.
+	const logPath = "/var/log/nvidia-installer.log"
+	f, err := os.OpenFile(logPath, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Warningf("failed to stat nvidia-installer.log: %v", err)
+		return
+	}
+	defer f.Close()
+
+	out, err := io.ReadAll(f)
+	if err != nil {
+		log.Warningf("failed to read nvidia-installer.log: %v", err)
+		return
+	}
+
+	for _, line := range strings.Split(string(out), "\n") {
+		fmt.Printf("[nvidia-installer]: %s\n", line)
+	}
+}

tools/gpu/main.go

@@ -46,6 +46,7 @@ var (
 
 	// The list command returns the list of supported drivers from this tool.
 	listCmd = flag.NewFlagSet(listCmdStr, flag.ContinueOnError)
+	outfile = listCmd.String("outfile", "", "if set, write the list output to this file")
 
 	commandSet = map[*flag.FlagSet]string{
 		installCmd:  installDescription,
@@ -110,7 +111,10 @@ func main() {
 			log.Warningf("%s failed with: %v", listCmdStr, err)
 			os.Exit(1)
 		}
-		drivers.ListSupportedDrivers()
+		if err := drivers.ListSupportedDrivers(*outfile); err != nil {
+			log.Warningf("Failed to list drivers: %v", err)
+			os.Exit(1)
+		}
 	default:
 		printUsage()
 		os.Exit(1)