From 59af1edc780ed48963fbf67e690bcb1bb7d93384 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Mon, 23 Oct 2023 11:01:48 -0700 Subject: [PATCH] Add script to run gpu test for all supported driver versions. Add script and subsequent calls in the buildkite pipeline to run gpu tests on all supported drivers. In addition, add an outfile flag to the "list" command for the driver installer which allows us to get the list of drivers and use it in the script (instead of a bunch of output from make/bazel/other stuff). PiperOrigin-RevId: 575870051 --- .buildkite/pipeline.yaml | 5 ++- tools/gpu/all_drivers_test.sh | 30 +++++++++++++++++ tools/gpu/drivers/install_driver.go | 50 ++++++++++++++++++++++------- tools/gpu/main.go | 6 +++- 4 files changed, 76 insertions(+), 15 deletions(-) create mode 100755 tools/gpu/all_drivers_test.sh diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 4fb677d1b..ed6d7889a 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -171,10 +171,9 @@ steps: # GPU workflow. - <<: *common <<: *source_test - label: "GPU tests" + label: ":screwdriver: All GPU Drivers Test" commands: - - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log - - make gpu-tests + - tools/gpu/all_drivers_test.sh agents: queue: gpu # Release workflow. diff --git a/tools/gpu/all_drivers_test.sh b/tools/gpu/all_drivers_test.sh new file mode 100755 index 000000000..e9a560642 --- /dev/null +++ b/tools/gpu/all_drivers_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2023 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to easily run gpu tests on all supported driver versions. This should +# be run from the gVisor repo root directory. +set -ueo pipefail + +tmp_file=$(mktemp) +trap "rm -f ${tmp_file}" EXIT + +make sudo TARGETS=tools/gpu:main ARGS="list --outfile=${tmp_file}" +read -r -a versions <<< "$(cat "${tmp_file}")" + +for driver in "${versions[@]}"; do + make sudo TARGETS=tools/gpu:main ARGS="install --version ${driver}" + make gpu-tests +done \ No newline at end of file diff --git a/tools/gpu/drivers/install_driver.go b/tools/gpu/drivers/install_driver.go index 48e98336b..eb1f3dd14 100644 --- a/tools/gpu/drivers/install_driver.go +++ b/tools/gpu/drivers/install_driver.go @@ -194,13 +194,26 @@ func (i *Installer) getRequestedDriver() (nvproxy.DriverVersion, bool) { // ListSupportedDrivers prints the driver to stderr in a format that can be // consumed by the Makefile to iterate tests across drivers. -func ListSupportedDrivers() { +func ListSupportedDrivers(outfile string) error { + out := os.Stdout + if outfile != "" { + f, err := os.OpenFile(outfile, os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open outfile: %w", err) + } + defer f.Close() + out = f + } + supportedDrivers := nvproxy.GetSupportedDriversAndChecksums() list := make([]string, 0, len(supportedDrivers)) - for version := range nvproxy.GetSupportedDriversAndChecksums() { + for version := range supportedDrivers { list = append(list, version.String()) } - fmt.Println(strings.Join(list, " ")) + if _, err := out.WriteString(strings.Join(list, " ") + "\n"); err != nil { + return fmt.Errorf("failed to write to outfile: %w", err) + } + return nil } // ChecksumDriver downloads and returns the SHA265 checksum of the driver. @@ -251,14 +264,8 @@ func installDriver(driverPath string) error { cmd := exec.Command(driverPath, driverArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - /* - cmd.Env = append(os.Environ(), - "IGNORE_CC_MISMATCH=1", - "LLVM=1", - "LLVM_IS=1", - ) - */ if err := cmd.Run(); err != nil { + tryToPrintFailureLogs() return fmt.Errorf("failed to run nvidia-install: %w out: %s", err, string(out)) } @@ -266,7 +273,28 @@ func installDriver(driverPath string) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to run nvidia-install: %w out: %s", err, string(out)) + return fmt.Errorf("failed to run nvidia-smi post install: %w out: %s", err, string(out)) } return nil } + +func tryToPrintFailureLogs() { + // nvidia driver installers print failure logs to this path. + const logPath = "/var/log/nvidia-installer.log" + f, err := os.OpenFile(logPath, os.O_RDONLY, 0644) + if err != nil { + log.Warningf("failed to stat nvidia-installer.log: %v", err) + return + } + defer f.Close() + + out, err := io.ReadAll(f) + if err != nil { + log.Warningf("failed to read nvidia-installer.log: %v", err) + return + } + + for _, line := range strings.Split(string(out), "\n") { + fmt.Printf("[nvidia-installer]: %s\n", line) + } +} diff --git a/tools/gpu/main.go b/tools/gpu/main.go index d2e5fec84..fb663dfda 100644 --- a/tools/gpu/main.go +++ b/tools/gpu/main.go @@ -46,6 +46,7 @@ var ( // The list command returns the list of supported drivers from this tool. listCmd = flag.NewFlagSet(listCmdStr, flag.ContinueOnError) + outfile = listCmd.String("outfile", "", "if set, write the list output to this file") commandSet = map[*flag.FlagSet]string{ installCmd: installDescription, @@ -110,7 +111,10 @@ func main() { log.Warningf("%s failed with: %v", listCmdStr, err) os.Exit(1) } - drivers.ListSupportedDrivers() + if err := drivers.ListSupportedDrivers(*outfile); err != nil { + log.Warningf("Failed to list drivers: %v", err) + os.Exit(1) + } default: printUsage() os.Exit(1)