From 4810afc36c5bcd977659b31119e37e1847b1a24c Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Fri, 19 Apr 2024 19:10:11 -0700 Subject: [PATCH] GPU support: Add NVIDIA CUDA sample tests. This is a set of CUDA tests defined by NVIDIA in this repository: https://github.com/NVIDIA/cuda-samples This change introduces a large new test (`cuda_test`) which runs each CUDA sample test in a container. There are many subtleties involved due to how the CUDA samples repository isn't always meant to be run as a test, some of it involves graphical applications, and a lot of them require this or that CUDA feature which not all NVIDIA GPUs support, some require multiple GPUs to be on the machine, etc. Therefore, the test maps each test to their `Compatibility` data which determines whether or not a sample test is expected to fail when run in a certain environment. The overall test also has a `--cuda_verify_compatibility` flag to verify the veracity of this mapping, by running expected-to-be-broken tests and verifying that their failure matches how this expected failure typically manifests. Because there are a lot of CUDA sample tests (213 as of the CUDA 12.3 release of the cuda-samples repo), and they don't all require the whole GPU to themselves, and spawning a GPU-using container is expensive (~seconds), the test uses a pool of reusable containers in which it `exec`s tests (at most one per container at any given time, but in parallel across containers). If any test unexpectedly fails, we drain the entire pool of containers and only run this one test without anything else running on other containers. This de-flakes tests, especially those that fail because they require more resources than the GPU has when other tests are using it at the same time. However, this removes parallelism and therefore increases test time significantly. Despite these optimizations, the test is very long and has the maximum deadline of 1 hour. Because it may get close to the timeout (especially when `--cuda_verify_compatibility` is on, because that needs to run all the tests even if they are known to fail, and the ones that do fail can hang for a while rather than crash), the test also has more logging and debugging than the typical test, as enabled with the `--cuda_log_successful_tests` and `--cuda_test_debug` flag. It periodically logs the status of each container and the pool's utilization ratio. This is useful when debugging the test to see which pooled container is doing what, and/or to `docker exec` into containers while they are running a certain test. The test also does its own timekeeping, which is useful so that it can print a more helpful failure message that distinguishes between tests failing due to actual failure reasons vs those that are failing purely because the test timed out. To run the test manually (from a VM with the repo checked out): ``` $ docker build -t gvisor.dev/images/gpu/cuda-tests images/gpu/cuda-tests -f images/gpu/cuda-tests/Dockerfile.x86_64 && mkdir -p bin && make copy TARGETS=runsc DESTINATION=bin/ && ./bin/runsc install -- --nvproxy=true --debug=true --debug-log=/tmp/runsc/ && systemctl reload docker && make test TEST_OPTIONS='--test_output=streamed --verbose_failures=true' TARGETS=//test/gpu:cuda_test OPTIONS='--test_env=RUNTIME=runsc --test_arg=--cuda_test_debug=true --test_arg=cuda_verify_compatibility=true --test_arg=--cuda_log_successful_tests=true' ``` PiperOrigin-RevId: 626528982 --- images/gpu/cuda-tests/Dockerfile.x86_64 | 47 +- images/gpu/cuda-tests/cuda_test_util.h | 12 + images/gpu/cuda-tests/list_features.cu | 52 + images/gpu/cuda-tests/list_features.sh | 32 + images/gpu/cuda-tests/list_sample_tests.sh | 40 + images/gpu/cuda-tests/run_sample.go | 1057 +++++++++++++++++ .../gpu/cuda-tests/{run.sh => run_smoke.sh} | 0 pkg/test/dockerutil/gpu.go | 8 +- test/gpu/BUILD | 17 + test/gpu/cuda_test.go | 823 +++++++++++++ test/gpu/smoke_test.go | 8 +- 11 files changed, 2086 insertions(+), 10 deletions(-) create mode 100644 images/gpu/cuda-tests/list_features.cu create mode 100644 images/gpu/cuda-tests/list_features.sh create mode 100644 images/gpu/cuda-tests/list_sample_tests.sh create mode 100644 images/gpu/cuda-tests/run_sample.go rename images/gpu/cuda-tests/{run.sh => run_smoke.sh} (100%) create mode 100644 test/gpu/cuda_test.go diff --git a/images/gpu/cuda-tests/Dockerfile.x86_64 b/images/gpu/cuda-tests/Dockerfile.x86_64 index d335c691b..76198083b 100644 --- a/images/gpu/cuda-tests/Dockerfile.x86_64 +++ b/images/gpu/cuda-tests/Dockerfile.x86_64 @@ -1,8 +1,45 @@ -FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 +FROM nvidia/cuda:12.3.2-devel-ubuntu22.04 + +# From: https://github.com/NVIDIA/cuda-samples/releases +# Ideally, pick a release that matches the CUDA version of the image above. +ARG CUDA_SAMPLES_VERSION=v12.3 WORKDIR / -COPY cuda_malloc_managed.cu . -COPY cuda_test_util.h . -COPY run.sh . +COPY *.cu *.h *.sh *.go / ENV PATH=$PATH:/usr/local/nvidia/bin:/bin/nvidia/bin -ENTRYPOINT ["/run.sh"] +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && \ + apt-get install -y \ + build-essential \ + cmake \ + freeglut3 freeglut3-dev \ + git \ + golang \ + imagemagick \ + libegl-dev \ + libfreeimage3 libfreeimage-dev \ + libfreeimageplus3 libfreeimageplus-dev \ + libgles2-mesa-dev \ + libglfw3 libglfw3-dev \ + libglu1-mesa libglu1-mesa-dev \ + libxi-dev \ + libxmu-dev \ + llvm \ + mpich \ + pkg-config \ + x11-xserver-utils \ + xdotool \ + xvfb \ + zlib1g zlib1g-dev \ + && \ + chmod 555 /*.sh && \ + git clone --depth=1 --branch="$CUDA_SAMPLES_VERSION" --single-branch \ + https://github.com/NVIDIA/cuda-samples.git /cuda-samples && \ + go install \ + github.com/TheZoraiz/ascii-image-converter@d05a757c5e02ab23e97b6f6fca4e1fbeb10ab559 && \ + mv "$HOME/go/bin/ascii-image-converter" /usr/bin/ && \ + go build -o /run_sample /run_sample.go + +# Override entrypoint to nothing, otherwise all invocations will have +# a copyright notice printed, which breaks parsing the stdout logs. +ENTRYPOINT [] diff --git a/images/gpu/cuda-tests/cuda_test_util.h b/images/gpu/cuda-tests/cuda_test_util.h index 0d0e4a7af..2877f78b1 100644 --- a/images/gpu/cuda-tests/cuda_test_util.h +++ b/images/gpu/cuda-tests/cuda_test_util.h @@ -17,6 +17,7 @@ #include +// cudaError_t is returned by CUDA runtime functions. #define CHECK_CUDA(expr) \ do { \ cudaError_t code = (expr); \ @@ -27,4 +28,15 @@ } \ } while (0) +// CUresult is returned by CUDA driver functions. +#define CHECK_CUDA_RESULT(expr) \ + do { \ + CUresult code = (expr); \ + if (code != CUDA_SUCCESS) { \ + std::cout << "Check failed at " << __FILE__ << ":" << __LINE__ << ": " \ + << #expr << ": " << code << std::endl; \ + abort(); \ + } \ + } while (0) + #endif // THIRD_PARTY_GVISOR_IMAGES_GPU_CUDA_TESTS_CUDA_TEST_UTIL_H_ diff --git a/images/gpu/cuda-tests/list_features.cu b/images/gpu/cuda-tests/list_features.cu new file mode 100644 index 000000000..060740304 --- /dev/null +++ b/images/gpu/cuda-tests/list_features.cu @@ -0,0 +1,52 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This program lists the features of the CUDA device that are available. +// It is used as part of the list_features.sh script. +// Each line it outputs is a CUDA feature name, prefixed by either +// "PRESENT: " or "ABSENT: ". + +#include +#include +#include + +#include "cuda_test_util.h" // NOLINT(build/include) + +void printFeature(const char* feature, bool have) { + if (have) { + printf("PRESENT: %s\n", feature); + } else { + printf("ABSENT: %s\n", feature); + } +} + +int main(int argc, char *argv[]) { + int cuda_device; + CHECK_CUDA(cudaGetDevice(&cuda_device)); + cudaDeviceProp properties; + CHECK_CUDA(cudaGetDeviceProperties(&properties, cuda_device)); + bool cdpCapable = + (properties.major == 3 && properties.minor >= 5) || properties.major >= 4; + printFeature("DYNAMIC_PARALLELISM", cdpCapable); + printFeature( + "PERSISTENT_L2_CACHING", properties.persistingL2CacheMaxSize > 0); + // Tensor cores are a thing in Volta (SM8X) + printFeature("TENSOR_CORES", properties.major >= 8); + int isCompressionAvailable; + CHECK_CUDA_RESULT( + cuDeviceGetAttribute(&isCompressionAvailable, + CU_DEVICE_ATTRIBUTE_GENERIC_COMPRESSION_SUPPORTED, + cuda_device)); + printFeature("COMPRESSIBLE_MEMORY", isCompressionAvailable != 0); +} diff --git a/images/gpu/cuda-tests/list_features.sh b/images/gpu/cuda-tests/list_features.sh new file mode 100644 index 000000000..32ec98a10 --- /dev/null +++ b/images/gpu/cuda-tests/list_features.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2024 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script outputs a list of CUDA features that are present or absent, +# one per line. Each line begins with either "PRESENT: " or "ABSENT: ", +# followed by the feature name. + +set -euo pipefail + +cd / +nvcc list_features.cu -lcuda -o list_features +./list_features + +# Detect GL by using a simple test that uses it as reference. +if xvfb-run make -C /cuda-samples/Samples/0_Introduction/simpleCUDA2GL TARGET_ARCH="$(uname -m)" testrun &>/dev/null; then + echo "PRESENT: GL" +else + echo "ABSENT: GL" +fi diff --git a/images/gpu/cuda-tests/list_sample_tests.sh b/images/gpu/cuda-tests/list_sample_tests.sh new file mode 100644 index 000000000..e34772ccc --- /dev/null +++ b/images/gpu/cuda-tests/list_sample_tests.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Copyright 2024 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script outputs a sorted list of CUDA sample tests, one per line. + +set -euo pipefail + +( + while IFS= read -r makefile_path; do + dirname "$makefile_path" + done < <(find /cuda-samples -type f -name Makefile) \ + | grep -vE '^/cuda-samples$' | grep -vE '/7_libNVVM' + + # cuda-samples/Samples/7_libNVVM is not structured like the other tests. + # It is built with `cmake` and generates multiple test binaries. + # The generated ones all follow the pattern of being named after their + # parent directory name, so we look for that. + pushd /cuda-samples/Samples/7_libNVVM &>/dev/null + cmake . &>/dev/null + make TARGET_ARCH="$(uname -m)" all &>/dev/null + popd &>/dev/null + while IFS= read -r dir_path; do + if [[ -x "$dir_path/$(basename "$dir_path")" ]]; then + echo "$dir_path" + fi + done < <(find /cuda-samples/Samples/7_libNVVM -type d) | sort | uniq +) | sed 's~/cuda-samples/Samples/~~' | sort diff --git a/images/gpu/cuda-tests/run_sample.go b/images/gpu/cuda-tests/run_sample.go new file mode 100644 index 000000000..1d5e9ca1d --- /dev/null +++ b/images/gpu/cuda-tests/run_sample.go @@ -0,0 +1,1057 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// run_sample runs a CUDA sample test. +// These tests are complicated because some of them involve X windows, +// as opposed to traditional command-line-only tests. +// This binary handles all types of CUDA sample tests. +// +// To run: /run_sample [--timeout=15m] test1 test2 test3 ... +package main + +import ( + "bufio" + "bytes" + "context" + "errors" + "flag" + "fmt" + "image" + "image/draw" + "image/png" + "io" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// Flags. +var ( + timeoutFlag = flag.Duration("timeout", 15*time.Minute, "Timeout for the program before it must clean up") +) + +const ( + // xDisplay is the X server address. + xDisplay = ":0" +) + +// logMu protects log output. +var logMu sync.Mutex + +// log logs a message to stderr. `format` should not have a newline. +// This does not use the standard logging library because this program needs +// to support logging multiple lines atomically. +func log(format string, values ...any) { + logDo(func() { + fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, values...)) + }) +} + +// logDo runs a function while logging the log lock. +// This is useful to log multiple lines at a time. +func logDo(fn func()) { + logMu.Lock() + defer logMu.Unlock() + fn() +} + +// logWriter implements io.Writer and logs to stderr. +type logWriter struct{} + +func (w *logWriter) Write(p []byte) (n int, err error) { + logDo(func() { + n, err = os.Stderr.Write(p) + }) + return n, err +} + +// Command wraps a command with some niceties for stdout/stderr handling. +type Command struct { + // Cmd is the wrapped command. + Cmd *exec.Cmd + + // Option fields. + // If non-nil, this data will be fed to the command's stdin. + Stdin []byte + // ForwardStdout and ForwardStderr control whether stdout/stderr are + // forwarded to the user's console. + ForwardStdout, ForwardStderr bool + // PrefixStdout and PrefixStderr are prefixes for forwarded logs. + PrefixStdout, PrefixStderr string + + // streamWG waits for stdout/stderr capturing goroutines. + streamWG sync.WaitGroup + + // mu protects the fields below. + mu sync.Mutex + + // started is `true` if the command has started. + started bool + + // Sets of stdout/stderr/combined output lines. + stdoutLines, stderrLines, combined []string + + // waitErr is the error returned by `Cmd.Wait`. + waitErr error + + // doneCh is closed when the command is done running. + doneCh chan struct{} +} + +// Start starts a command in the background. +func (c *Command) Start(ctx context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.started { + return errors.New("command already started") + } + for _, env := range os.Environ() { + c.Cmd.Env = append(c.Cmd.Env, env) + } + if len(c.Stdin) == 0 { + c.Cmd.Stdin = nil // Read from /dev/null + } else { + c.Cmd.Stdin = bytes.NewReader(c.Stdin) + } + stdout, err := c.Cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("cannot open stdout pipe: %w", err) + } + stderr, err := c.Cmd.StderrPipe() + if err != nil { + return fmt.Errorf("cannot open stderr pipe: %w", err) + } + if err := c.Cmd.Start(); err != nil { + return fmt.Errorf("cannot start command: %w", err) + } + c.started = true + + for _, stream := range []struct { + forward bool + prefix string + from io.ReadCloser + to io.Writer + lines *[]string + }{ + {c.ForwardStdout, c.PrefixStdout, stdout, os.Stdout, &c.stdoutLines}, + {c.ForwardStderr, c.PrefixStderr, stderr, &logWriter{}, &c.stderrLines}, + } { + c.streamWG.Add(1) + go func(forward bool, prefix string, from io.ReadCloser, to io.Writer, lines *[]string) { + defer c.streamWG.Done() + for scanner := bufio.NewScanner(from); scanner.Scan(); { + text := scanner.Text() + c.mu.Lock() + *lines = append(*lines, text) + c.combined = append(c.combined, text) + if forward { + fmt.Fprintf(to, "%s%s\n", prefix, text) + } + c.mu.Unlock() + } + }(stream.forward, stream.prefix, stream.from, stream.to, stream.lines) + } + c.doneCh = make(chan struct{}) + go func() { + c.streamWG.Wait() + c.mu.Lock() + defer c.mu.Unlock() + c.waitErr = c.Cmd.Wait() + close(c.doneCh) + }() + return nil +} + +// Stdout returns the standard output lines of the command so far. +func (c *Command) Stdout() []string { + c.mu.Lock() + defer c.mu.Unlock() + return c.stdoutLines[:] +} + +// Stderr returns the standard error lines of the command so far. +func (c *Command) Stderr() []string { + c.mu.Lock() + defer c.mu.Unlock() + return c.stderrLines[:] +} + +// Combined returns the combined stodut/stderr lines of the command so far. +// This is not the same as stdout concatenated with stderr, as it preserves +// line ordering as they were emitted. +func (c *Command) Combined() []string { + c.mu.Lock() + defer c.mu.Unlock() + return c.combined[:] +} + +// PID returns the PID of the running command. +func (c *Command) PID() int { + return c.Cmd.Process.Pid +} + +// ExitCode returns the exit code of the command. +func (c *Command) ExitCode(ctx context.Context) (int, error) { + c.mu.Lock() + if !c.started { + c.mu.Unlock() + return 0, errors.New("command not started") + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-c.Done(): + } + c.mu.Lock() + defer c.mu.Unlock() + if c.waitErr == nil { + return 0, nil + } + if exitErr := (*exec.ExitError)(nil); errors.As(c.waitErr, &exitErr) { + return exitErr.ExitCode(), nil + } + return 0, fmt.Errorf("process exit did not carry exit code: %w", c.waitErr) +} + +// Wait waits for a `Start`ed command to run to completion and returns +// stdout/stderr. +func (c *Command) Wait(ctx context.Context) ([]string, []string, error) { + c.mu.Lock() + if !c.started { + c.mu.Unlock() + return nil, nil, errors.New("command not started") + } + c.mu.Unlock() + select { + case <-ctx.Done(): + case <-c.Done(): + } + stdout := c.Stdout() + stderr := c.Stderr() + c.mu.Lock() + err := c.waitErr + c.mu.Unlock() + if err != nil { + return stdout, stderr, fmt.Errorf("command failed: %w", err) + } + return stdout, stderr, err +} + +// Run `Start`s and `Wait`s for a command to run to completion. +func (c *Command) Run(ctx context.Context) ([]string, []string, error) { + if err := c.Start(ctx); err != nil { + return nil, nil, err + } + return c.Wait(ctx) +} + +// CombinedOutput runs a command to completion and returns combined +// stdout/stderr output. +func (c *Command) CombinedOutput(ctx context.Context) (string, error) { + if err := c.Start(ctx); err != nil { + return "", err + } + _, _, err := c.Wait(ctx) + return strings.Join(c.Combined(), "\n"), err +} + +// Done returns a channel that is closed when the command terminates. +// Must be called after `Start`. +func (c *Command) Done() <-chan struct{} { + c.mu.Lock() + defer c.mu.Unlock() + if c.doneCh == nil { + panic("Command.Done called before Command.Start") + } + return c.doneCh +} + +// Terminate terminates a process. +// It does not reap the process; the caller should call wait if appropriate. +func Terminate(ctx context.Context, pid int, waitChans ...<-chan struct{}) error { + unifiedWaitChan := make(chan struct{}) + waitShutdown := make(chan struct{}) + defer close(waitShutdown) + for _, waitChan := range waitChans { + go func(waitChan <-chan struct{}) { + select { + case <-waitShutdown: + case <-waitChan: + unifiedWaitChan <- struct{}{} + } + }(waitChan) + } + // Ignore errors here because it doesn't matter; we will re-detect + // the post-signal process state later. + _ = syscall.Kill(pid, syscall.SIGTERM) + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + case <-unifiedWaitChan: + } + if _, err := os.Stat(fmt.Sprintf("/proc/%d", pid)); err != nil && os.IsNotExist(err) { + // The process is gone, so we are successful. + return nil + } + // Otherwise, send SIGKILL. + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + return fmt.Errorf("cannot send SIGKILL: %w", err) + } + return nil +} + +// XServer represents an X server. +type XServer struct { + xvfb *Command +} + +// NewXServer creates a new X server. +func NewXServer(ctx context.Context) (*XServer, error) { + xvfb := &Command{ + Cmd: exec.CommandContext(ctx, "Xvfb", xDisplay, "-screen", "0", "1920x1080x24"), + ForwardStdout: true, + PrefixStdout: "[Xvfb:stdout] ", + ForwardStderr: true, + PrefixStderr: "[Xvfb:stderr] ", + } + if err := xvfb.Start(ctx); err != nil { + return nil, fmt.Errorf("cannot start X server: %w", err) + } + x := &XServer{xvfb: xvfb} + if err := x.Probe(ctx); err != nil { + x.Shutdown(ctx) + return nil, fmt.Errorf("X server did not start in time: %w", err) + } + return x, nil +} + +// Env returns the DISPLAY environment variable to use for this X server. +func (x *XServer) Env() string { + return fmt.Sprintf("DISPLAY=%s", xDisplay) +} + +// Command returns a command that runs in the context of this X server. +func (x *XServer) Command(ctx context.Context, argv ...string) *Command { + cmd := &Command{Cmd: exec.CommandContext(ctx, argv[0], argv[1:]...)} + cmd.Cmd.Env = append(cmd.Cmd.Env, x.Env()) + return cmd +} + +// Probe probes the X server to see if it is alive. +func (x *XServer) Probe(ctx context.Context) error { + probeCtx, probeCancel := context.WithTimeout(ctx, 10*time.Second) + defer probeCancel() + lastErr := ctx.Err() + for probeCtx.Err() == nil { + output, err := x.Command(probeCtx, "xset", "q").CombinedOutput(ctx) + if err == nil { + return nil + } + lastErr = fmt.Errorf("cannot probe X server: %w: %s", err, output) + } + return lastErr +} + +// Shutdown attempts to shut down the X server. +func (x *XServer) Shutdown(ctx context.Context) error { + if err := Terminate(ctx, x.xvfb.Cmd.Process.Pid, x.xvfb.Done()); err != nil { + return fmt.Errorf("cannot shut down Xvfb: %w", err) + } + _, _, _ = x.xvfb.Wait(ctx) // Reap, ignore errors. + return nil +} + +// XWindow represents a window in the X server. +type XWindow struct { + x *XServer + id int64 +} + +// Windows returns a list of X windows. +func (x *XServer) Windows(ctx context.Context) ([]*XWindow, error) { + cmd := x.Command(ctx, "xdotool", "search", "--all", ".*") + stdout, _, err := cmd.Run(ctx) + if err != nil { + return nil, fmt.Errorf("xdotool search failed: %w (output: %v)", err, cmd.Combined()) + } + windows := make([]*XWindow, 0, len(stdout)) + for _, line := range stdout { + line = strings.TrimSpace(line) + if line == "" { + continue + } + windowID, err := strconv.Atoi(line) + if err != nil { + return nil, fmt.Errorf("unexpected xdotool output: %q (whole output: %v)", line, cmd.Combined()) + } + windows = append(windows, &XWindow{x: x, id: int64(windowID)}) + } + return windows, nil +} + +// ID returns a the window ID as a string. +func (w *XWindow) ID() string { + return fmt.Sprintf("%d", w.id) +} + +// String returns a string containing the window ID. +func (w *XWindow) String() string { + return fmt.Sprintf("window:%d", w.id) +} + +// Title returns the window title. +func (w *XWindow) Title(ctx context.Context) (string, error) { + cmd := w.x.Command(ctx, "xdotool", "getwindowname", w.ID()) + stdout, stderr, err := cmd.Wait(ctx) + if err != nil { + return "", w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s title: %w (%q)", w, err, strings.Join(stderr, "\n"))) + } + if len(stdout) != 1 || stdout[0] == "" { + return "", w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s title: unexpected output %q", w, strings.Join(stdout, "\n"))) + } + return stdout[0], nil +} + +// PID returns the PID controlling the window. +// Note that this information is only optionally specified by a process +// creating a window, and is never guaranteed to be there. +func (w *XWindow) PID(ctx context.Context) (int, error) { + cmd := w.x.Command(ctx, "xdotool", "getwindowpid", w.ID()) + stdout, stderr, err := cmd.Wait(ctx) + if err != nil { + return -1, w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s PID: %w (%q)", w, err, strings.Join(stderr, "\n"))) + } + if len(stdout) != 1 || stdout[0] == "" { + return -1, w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s PID: unexpected output %q", w, strings.Join(stdout, "\n"))) + } + pid, err := strconv.Atoi(stdout[0]) + if err != nil { + return -1, w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s PID: invalid PID %q: %w", w, stdout[0], err)) + } + return pid, nil +} + +// Activate activates or focuses the X window. +func (w *XWindow) Activate(ctx context.Context) error { + cmd := w.x.Command(ctx, "xdotool", "windowactivate", "--sync", w.ID()) + if output, err := cmd.CombinedOutput(ctx); err != nil { + return w.diagnoseErr(ctx, fmt.Errorf("xdotool windowactivate: %w (output: %q)", err, output)) + } + return nil +} + +// Keystroke sends a keystroke to the X window. +func (w *XWindow) Keystroke(ctx context.Context, keystrokes ...string) error { + cmd := w.x.Command( + ctx, + append( + []string{ + "xdotool", + "key", + "--clearmodifiers", + "--window", + w.ID(), + }, + keystrokes...)...) + if output, err := cmd.CombinedOutput(ctx); err != nil { + return w.diagnoseErr(ctx, fmt.Errorf("xdotool key: %w (output: %q)", err, output)) + } + return nil +} + +// Screenshot takes a screenshot image of the X window. +func (w *XWindow) Screenshot(ctx context.Context) (image.Image, error) { + screenshotCtx, screenshotCancel := context.WithTimeout(ctx, 10*time.Second) + // Need to use a raw `exec.Command` here because stdout is a byte stream + // as opposed to a text stream. + cmd := exec.CommandContext(screenshotCtx, "import", "-window", w.ID(), "png:-" /* Save to stdout as PNG */) + cmd.Env = append(cmd.Env, w.x.Env()) + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err := cmd.Run() + screenshotCancel() + stderr := string(stderrBuf.Bytes()) + if err != nil { + // Best-effort attempt to kill the process. + _ = Terminate(ctx, cmd.Process.Pid) + return nil, w.diagnoseErr(ctx, fmt.Errorf("imagemagick failed: %w (output: %q)", err, stderr)) + } + img, err := png.Decode(&stdoutBuf) + if err != nil { + return nil, w.diagnoseErr(ctx, fmt.Errorf("cannot decode screenshot image: %w (output: %q)", err, stderr)) + } + if size := img.Bounds().Size(); size.X == 0 || size.Y == 0 { + return nil, w.diagnoseErr(ctx, fmt.Errorf("screenshot image has zero dimension (output: %q)", stderr)) + } + return img, nil +} + +// diagnoseErr annotates an error with additional window information. +func (w *XWindow) diagnoseErr(ctx context.Context, err error) error { + if err == nil { + return nil + } + probeCtx, probeCancel := context.WithTimeout(ctx, 1*time.Second) + defer probeCancel() + if xErr := w.x.Probe(probeCtx); xErr != nil { + return fmt.Errorf("%w (X server is down: %v)", err, xErr) + } + winInfo, infoErr := w.x.Command(ctx, "xwininfo", "-id", w.ID()).CombinedOutput(ctx) + if infoErr != nil { + return fmt.Errorf("%w (cannot get window info: %v - %q)", err, infoErr, winInfo) + } + return fmt.Errorf("%w (window info: %q)", err, winInfo) +} + +// SampleTest represents a single sample test to execute. +type SampleTest struct { + TestName string + XServer *XServer +} + +// NewSampleTest creates a new SampleTest. +func NewSampleTest(testName string, x *XServer) (*SampleTest, error) { + st := &SampleTest{TestName: testName, XServer: x} + if _, err := os.Stat(st.dir()); err != nil { + return nil, fmt.Errorf("invalid test %q: directory %q: %w", st.TestName, st.dir(), err) + } + return st, nil +} + +// dir returns the test directory. +func (st *SampleTest) dir() string { + const samplesRoot = "/cuda-samples/Samples" + return path.Join(samplesRoot, st.TestName) +} + +// cmd returns a `*Command` with proper environment variables and +// working directory for the test. Its output is forwarded to the console. +func (st *SampleTest) cmd(ctx context.Context, argv ...string) *Command { + argv0Base := path.Base(argv[0]) + cmd := st.XServer.Command(ctx, argv...) + cmd.Cmd.Dir = st.dir() + cmd.ForwardStdout = true + cmd.PrefixStdout = fmt.Sprintf("[%s:%s:stdout] ", st.TestName, argv0Base) + cmd.ForwardStderr = true + cmd.PrefixStderr = fmt.Sprintf("[%s:%s:stderr] ", st.TestName, argv0Base) + return cmd +} + +// quietCmd returns a `*Command` with proper environment variables and +// working directory for the test. Its output is not forwarded to the console. +func (st *SampleTest) quietCmd(ctx context.Context, argv ...string) *Command { + cmd := st.cmd(ctx, argv...) + cmd.ForwardStdout = false + cmd.ForwardStderr = false + return cmd +} + +// SampleState captures states that is captured before a test runs, and that +// is useful to refer to while (or after) the test is running. +type SampleState struct { + // When is the timestamp at which this SampleState was taken. + When time.Time + + // Executables holds clean paths of all executable files in the test dir. + Executables map[string]struct{} + + // Windows is a list of window screenshots in the X server, mapped by ID. + Windows map[string]*XWindow + + // Screenshots is a list of screenshots mapped by window ID. + // If a screenshot fails, the window is mapped to `nil`. + Screenshots map[string]image.Image +} + +// NewExecutables returns the executables in `after` that are not in `ss`. +func (ss *SampleState) NewExecutables(after *SampleState) []string { + newExecutables := make([]string, 0, len(after.Executables)) + for e := range after.Executables { + if _, found := ss.Executables[e]; !found { + newExecutables = append(newExecutables, e) + } + } + return newExecutables +} + +// DifferentWindows returns the windows in `after` that are new or for which +// the screenshot has changed. +func (ss *SampleState) DifferentWindows(after *SampleState) []*XWindow { + diffWindows := make([]*XWindow, 0, len(after.Windows)) + for id, window := range after.Windows { + if _, found := ss.Windows[id]; !found { + diffWindows = append(diffWindows, window) + continue + } + if !imgEq(ss.Screenshots[id], after.Screenshots[id]) { + diffWindows = append(diffWindows, window) + } + } + return diffWindows +} + +// imgEq returns true if the two given images are identical in size and pixel +// values. +func imgEq(a, b image.Image) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + bounds := a.Bounds() + if bounds != b.Bounds() { + return false + } + // Convert images to RGBA so that we can compare raw pixel data directly. + imgA := image.NewRGBA(bounds) + draw.Draw(imgA, bounds, a, image.Point{0, 0}, draw.Src) + imgB := image.NewRGBA(bounds) + draw.Draw(imgB, bounds, b, image.Point{0, 0}, draw.Src) + if imgA.Stride != imgB.Stride || imgA.Rect != imgB.Rect || len(imgA.Pix) != len(imgB.Pix) { + return false + } + for i := 0; i < len(imgA.Pix); i++ { + if imgA.Pix[i] != imgB.Pix[i] { + return false + } + } + return true +} + +// logImageWithPrefix renders an image to text, frames it with the given +// title, and logs that with a given prefix. +func logImageWithFrameAndPrefix(ctx context.Context, img image.Image, title, prefix string) error { + const imageWidth = 72 + var pngBytes bytes.Buffer + if err := png.Encode(&pngBytes, img); err != nil { + return fmt.Errorf("png encoding failed: %v", err) + } + stdout, stderr, err := (&Command{ + Cmd: exec.CommandContext(ctx, "ascii-image-converter", "/dev/stdin", fmt.Sprintf("--width=%d", imageWidth), "--braille", "--dither"), + Stdin: pngBytes.Bytes(), + }).Run(ctx) + if err != nil { + return fmt.Errorf("ascii-image-converter failed: %v (output: %q)", err, strings.Join(stderr, "\n")) + } + header := "┍" + footer := "╰" + numHeaderHorizontalLines := imageWidth - len(title) - 2 + leftHeaderHorizontalLines := numHeaderHorizontalLines / 2 + rightHeaderHorizontalLines := numHeaderHorizontalLines - leftHeaderHorizontalLines + for i := 0; i < leftHeaderHorizontalLines; i++ { + header += "━" + } + header += fmt.Sprintf(" %s ", title) + for i := 0; i < rightHeaderHorizontalLines; i++ { + header += "━" + } + for i := 0; i < imageWidth; i++ { + footer += "─" + } + header += "┑" + footer += "╯" + logDo(func() { + fmt.Fprintf(os.Stderr, "%s%s\n", prefix, header) + for _, line := range stdout { + fmt.Fprintf(os.Stderr, "%s|%s|\n", prefix, line) + } + fmt.Fprintf(os.Stderr, "%s%s\n", prefix, footer) + }) + return nil +} + +// State returns the current state of the test. +func (st *SampleTest) State(ctx context.Context) (*SampleState, error) { + when := time.Now() + executables := make(map[string]struct{}) + err := filepath.Walk(st.dir(), func(path string, info fs.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("cannot walk %q (%q): %w", st.dir(), path, err) + } + if !info.IsDir() && info.Mode()&0111 != 0 { + executables[path] = struct{}{} + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("cannot list executables: %w", err) + } + windows, err := st.XServer.Windows(ctx) + if err != nil { + return nil, fmt.Errorf("cannot list windows: %w", err) + } + windowMap := make(map[string]*XWindow, len(windows)) + screenshots := make(map[string]image.Image, len(windows)) + for _, w := range windows { + windowMap[w.ID()] = w + if screenshot, err := w.Screenshot(ctx); err == nil { + screenshots[w.ID()] = screenshot + } + } + return &SampleState{ + When: when, + Executables: executables, + Windows: windowMap, + Screenshots: screenshots, + }, nil +} + +// makeRun runs `make run` or `make testrun` in the test directory. +func (st *SampleTest) makeRun(ctx context.Context) (*Command, error) { + arch, err := st.quietCmd(ctx, "uname", "-m").CombinedOutput(ctx) + if err != nil || arch == "" { + return nil, fmt.Errorf("cannot get architecture (%q): %w", arch, err) + } + + // All samples have a "testrun" make target. However, most of them have it + // set to do literally nothing. + // All samples also have a "run" make target. Unlike the "testrun" target, + // "run" always does something. + // However, when "testrun" actually does something, it is usually for the + // explicit purpose of running a test. + // For example, `0_Introduction/simpleTexture3D` has a `testrun` target that + // runs the file with an example texture file, whereas the `run` target + // opens a file passed as argument, which does not exist here. + // So we must detect the case where "testrun" does something useful vs the + // case where it does not. + // To do this, we parse the Makefile a bit to see if the `testrun` target + // contains any actual commands, as opposed to only containing build + // dependencies. + makefilePath := path.Join(st.dir(), "Makefile") + makefile, err := os.Open(makefilePath) + if err != nil { + return nil, fmt.Errorf("cannot open %q: %w", makefilePath, err) + } + defer makefile.Close() + testRunTargetHasCommands := false + for scanner := bufio.NewScanner(makefile); scanner.Scan(); { + line := scanner.Text() + if !strings.HasPrefix(line, "testrun:") { + continue + } + if !scanner.Scan() { + break + } + nextLine := scanner.Text() + if strings.HasPrefix(nextLine, "\t") && strings.TrimSpace(nextLine) != "" { + testRunTargetHasCommands = true + } + break + } + argv := []string{"make", "-C", st.dir(), fmt.Sprintf("TARGET_ARCH=%s", arch)} + if testRunTargetHasCommands { + argv = append(argv, "testrun") + } else { + argv = append(argv, "run") + } + log("[%s] Executing: %v", st.TestName, strings.Join(argv, " ")) + cmd := st.cmd(ctx, argv...) + if err := cmd.Start(ctx); err != nil { + return nil, fmt.Errorf("cannot start `make`: %w", err) + } + return cmd, nil +} + +// Run runs a single sample test. +func (st *SampleTest) Run(ctx context.Context) error { + const libNVVMTestDir = "7_libNVVM/" + if strings.HasPrefix(st.TestName, libNVVMTestDir) { + return st.RunLibNVVMTest(ctx) + } + if _, _, err := st.cmd(ctx, "make", "-C", st.dir(), "clean").Run(ctx); err != nil { + return fmt.Errorf("cannot run `make clean`: %w", err) + } + stateBefore, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get state before test: %w", err) + } + makeRun, err := st.makeRun(ctx) + if err != nil { + return fmt.Errorf("cannot run `make run`: %w", err) + } + defer Terminate(ctx, makeRun.PID()) + // There are multiple possibilities here. + // Some CUDA programs will run an X application that runs forever. + // In this case, we need to detect this and to make sure it runs, + // then kill it. + // Other programs are just command-line based and run to completion, + // and we rely on their exit code. + // To determine this, we first just wait for a few seconds and see what + // the command does. + if err := st.Monitor(ctx, makeRun, stateBefore); err != nil { + return fmt.Errorf("test failed in `make run`: %w", err) + } + + // Some `make` targets will silently exist with code 0 even if the test + // was actually unsuccessful because it cannot be built. + // To detect this case, we look for the absence of any executable file in + // the sample directory. All `make` targets should create an executable, and + // this won't happen if `make` bails out. + stateAfter, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get state after test: %w", err) + } + if len(stateBefore.NewExecutables(stateAfter)) == 0 { + return fmt.Errorf("did not find any new executable file created by `make run` in the test directory %q (existing executables: %v)", st.dir(), stateBefore.Executables) + } + return nil +} + +// Monitor monitors whether a `make run` command terminates quickly or +// produces an X window. +func (st *SampleTest) Monitor(ctx context.Context, makeRun *Command, stateBefore *SampleState) error { + fastTicker := time.NewTicker(200 * time.Millisecond) + defer fastTicker.Stop() + var currentState *SampleState + for windowsChanged := false; !windowsChanged; { + select { + case <-ctx.Done(): // Context expired. + return ctx.Err() + case <-makeRun.Done(): // `make run` finished on its own. + _, _, err := makeRun.Wait(ctx) + return err + case <-fastTicker.C: + // Check for new windows. + var err error + currentState, err = st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get test state: %w", err) + } + windowsChanged = len(stateBefore.DifferentWindows(currentState)) > 0 + } + } + + // If we get here, the test produces X windows. So we need to monitor them. + // We will consider the test a success in any of the following cases: + // - The `make run` process exits at any time with a 0 exit code. + // - The set of windows stops changing for 3 consecutive seconds, i.e. + // the test has reached a stable steady state without crashing. + // - The set of windows continuously changes for 10 consecutive seconds, + // i.e. the test is likely a visually-changing demo over time and has + // reached a steady state without crashing. + log("[%s] This appears to be a test that uses graphics and X windows.", st.TestName) + lastState := stateBefore + slowTicker := time.NewTicker(1 * time.Second) + defer slowTicker.Stop() + lastWindowChange := currentState.When + successDeadline := time.After(10 * time.Second) + for { + select { + case <-ctx.Done(): // Context expired. + return ctx.Err() + case <-makeRun.Done(): // `make run` finished on its own. + _, _, err := makeRun.Wait(ctx) + return err + case <-successDeadline: // Still no crashes after long enough. + return st.TerminateWindowTest(ctx, makeRun, stateBefore) + case <-slowTicker.C: + stateNow, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get test state: %w", err) + } + if differentWindows := lastState.DifferentWindows(stateNow); len(differentWindows) > 0 { + lastWindowChange = stateNow.When + log("[%s] [%s] Windows changed:", st.TestName, stateNow.When.Format("15:04:05")) + for _, window := range differentWindows { + title, err := window.Title(ctx) + if err != nil { + title = window.String() + } + if screenshot := stateNow.Screenshots[window.ID()]; screenshot == nil { + log("[%s:%s] ", st.TestName, title) + } else if err := logImageWithFrameAndPrefix(ctx, screenshot, title, fmt.Sprintf("[%s] ", st.TestName)); err != nil { + log("[%s:%s] ", st.TestName, title, err) + } + } + } + if currentState.When.Sub(lastWindowChange) >= 3*time.Second { + return st.TerminateWindowTest(ctx, makeRun, stateBefore) + } + lastState = stateNow + } + } +} + +// TerminateWindowTest terminates a sample test that produces X windows. +func (st *SampleTest) TerminateWindowTest(ctx context.Context, makeRun *Command, stateBefore *SampleState) error { + stateNow, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get test state: %w", err) + } + testWindows := stateBefore.DifferentWindows(stateNow) + // Most windows-based tests accept typing the letter "Q" to quit them. + // Try it first. + for _, window := range testWindows { + // Ignore error for both activation and keystrokes; this is just a + // best-effort attempt to press "Q". + _ = window.Activate(ctx) + _ = window.Keystroke(ctx, "q") + } + // Now wait a little bit to see if the program ends on its own from that. + select { + case <-ctx.Done(): + return ctx.Err() + case <-makeRun.Done(): + _, _, err = makeRun.Wait(ctx) + return err + case <-time.After(3 * time.Second): + // Didn't work, keep going. + } + // Gather a list of test PIDs. + windowPIDs := make(map[int]struct{}) + for _, window := range testWindows { + pid, err := window.PID(ctx) + if err != nil { + // X window PID information is optional; erroring out here is not + // appropriate. + continue + } + if pid == makeRun.PID() { + continue + } + windowPIDs[pid] = struct{}{} + } + if len(windowPIDs) > 0 { + // Kill all the PIDs we gathered. + for pid := range windowPIDs { + _ = Terminate(ctx, pid, makeRun.Done()) + } + // Now check if `make run` terminates on its own. + select { + case <-ctx.Done(): + return ctx.Err() + case <-makeRun.Done(): + _, _, err = makeRun.Wait(ctx) + return err + case <-time.After(3 * time.Second): + // Didn't work, keep going. + } + } + return errors.New("test did not terminate") +} + +// RunLibNVVMTest runs a `libnvvm`-based test. +// These tests are located in the `7_libNVVM/` directory. +func (st *SampleTest) RunLibNVVMTest(ctx context.Context) error { + const ptxgenTestName = "ptxgen" + + // Need to run `cmake` in the 7_libNVVM/ directory to build the test. + libNVVMTestsDir := path.Dir(st.dir()) + libNVVMTestName := path.Base(st.dir()) + cmake := st.cmd(ctx, "cmake", ".") + cmake.Cmd.Dir = libNVVMTestsDir + if _, _, err := cmake.Run(ctx); err != nil { + return fmt.Errorf("cannot run `cmake`: %w", err) + } + // Then run `make` in the test directory. + // CMake generates a make file in the parent directory. + // We `make all` rather than just the test target, because + // `cuda-c-linking` depends on the `mathfuncs` target despite not being + // declared as such in the Makefile. + arch, err := st.quietCmd(ctx, "uname", "-m").CombinedOutput(ctx) + if err != nil || arch == "" { + return fmt.Errorf("cannot get architecture (%q): %w", arch, err) + } + makeCmd := st.cmd(ctx, "make", "-C", libNVVMTestsDir, fmt.Sprintf("TARGET_ARCH=%s", arch), "all") + if _, _, err := makeCmd.Run(ctx); err != nil { + return fmt.Errorf("cannot run `make`: %w", err) + } + // `make` will create an executable in the test directory that has the same + // name as the directory does. + exePath := path.Join(st.dir(), libNVVMTestName) + if _, err := os.Stat(exePath); err != nil { + return fmt.Errorf("cannot stat executable at expected location %q: %w", exePath, err) + } + argv := []string{exePath} + if libNVVMTestName == ptxgenTestName { + // The ptxgen test binary needs a .ll file as input. + // Conveniently, there is one called "test.ll" in the test directory. + argv = append(argv, path.Join(st.dir(), "test.ll")) + } + if _, _, err := st.cmd(ctx, argv...).Run(ctx); err != nil { + return fmt.Errorf("test binary failed: %w", err) + } + return nil +} + +// Main is the main method of this program. +func Main(ctx context.Context) (int, error) { + flag.Parse() + cleanupCtx, cleanupCancel := context.WithTimeout(ctx, *timeoutFlag) + defer cleanupCancel() + deadline, _ := cleanupCtx.Deadline() + x, err := NewXServer(cleanupCtx) + if err != nil { + return 1, fmt.Errorf("failed to start X server: %s", err) + } + defer x.Shutdown(cleanupCtx) + testsCtx, testsCancel := context.WithDeadline(cleanupCtx, deadline.Add(-10*time.Second)) + defer testsCancel() + failed := false + numTests := 0 + exitCode := 1 + for _, testName := range flag.Args() { + numTests++ + st, err := NewSampleTest(testName, x) + if err != nil { + log("> Invalid test %q: %s", testName, err) + failed = true + continue + } + log("> Running test: %s", testName) + testCtx, testCancel := context.WithCancel(testsCtx) + err = st.Run(testCtx) + testCancel() + if err != nil { + log("> Test failed: %s (%s)", testName, err) + failed = true + if exitErr := (*exec.ExitError)(nil); errors.As(err, &exitErr) && exitErr.ExitCode() > 0 { + exitCode = exitErr.ExitCode() + } + continue + } + log("> Test passed: %s", testName) + } + if numTests == 0 { + return 1, fmt.Errorf("no tests to run, failing vacuously; specify test names as positional arguments") + } + if failed { + if numTests == 1 { + // If there was a single test to run, pass along its error code. + return exitCode, fmt.Errorf("test failed") + } + return 1, errors.New("one or more tests failed") + } + return 0, nil +} + +func main() { + exitCode, err := Main(context.Background()) + if err != nil { + log("%s", err) + log("FAIL") + } else { + log("PASS") + } + os.Exit(exitCode) +} diff --git a/images/gpu/cuda-tests/run.sh b/images/gpu/cuda-tests/run_smoke.sh similarity index 100% rename from images/gpu/cuda-tests/run.sh rename to images/gpu/cuda-tests/run_smoke.sh diff --git a/pkg/test/dockerutil/gpu.go b/pkg/test/dockerutil/gpu.go index f7370a0cf..eb86ecd46 100644 --- a/pkg/test/dockerutil/gpu.go +++ b/pkg/test/dockerutil/gpu.go @@ -29,14 +29,19 @@ var ( setCOSGPU = flag.Bool("cos-gpu", false, "set to configure GPU settings for COS, as opposed to Docker") ) +// AllGPUCapabilities is the environment variable that enables all NVIDIA GPU +// capabilities within a container. +const AllGPUCapabilities = "NVIDIA_DRIVER_CAPABILITIES=all" + // GPURunOpts returns Docker run options with GPU support enabled. func GPURunOpts() RunOpts { if !*setCOSGPU { return RunOpts{ + Env: []string{AllGPUCapabilities}, DeviceRequests: []container.DeviceRequest{ { Count: -1, - Capabilities: [][]string{[]string{"gpu"}}, + Capabilities: [][]string{{"gpu"}}, Options: map[string]string{}, }, }, @@ -92,6 +97,7 @@ func GPURunOpts() RunOpts { } return RunOpts{ + Env: []string{AllGPUCapabilities}, Mounts: mounts, Devices: devices, } diff --git a/test/gpu/BUILD b/test/gpu/BUILD index 747a23e7f..de5c4c1b8 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -62,6 +62,23 @@ go_test( ], ) +go_test( + name = "cuda_test", + timeout = "eternal", # YES_I_REALLY_NEED_AN_ETERNAL_TEST + srcs = ["cuda_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//pkg/test/testutil", + "@org_golang_x_sync//errgroup:go_default_library", + ], +) + go_test( name = "imagegen_test", srcs = ["imagegen_test.go"], diff --git a/test/gpu/cuda_test.go b/test/gpu/cuda_test.go new file mode 100644 index 000000000..11823b4f3 --- /dev/null +++ b/test/gpu/cuda_test.go @@ -0,0 +1,823 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cuda_test tests basic CUDA workloads. +package cuda_test + +import ( + "context" + "errors" + "flag" + "fmt" + "math" + "os" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/sync/errgroup" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +const ( + // defaultTestTimeout is the default timeout for a single CUDA sample test. + defaultTestTimeout = 20 * time.Minute + + // hangingTestTimeout is the test timeout for tests that are fast when they + // succeed, but hang forever otherwise. + hangingTestTimeout = 1 * time.Minute + + // defaultContainersPerCPU is the default number of pooled containers to + // spawn for each CPU. This can be a floating-point value. + // This value was arrived at experimentally and has no particular meaning. + // Setting it too low will cause the test to take longer than necessary + // because of insufficient parallelism. + // However, setting it too high will *also* cause the test to take longer + // than necessary, because the added resource contention will cause more + // tests to fail when run in parallel with each other, forcing them to be + // re-run serialized. + defaultContainersPerCPU = 1.75 + + // exitCodeWaived is the EXIT_WAIVED constant used in CUDA tests. + // This exit code is typically used by CUDA tests to indicate that the + // test requires a capability or condition that is not met in the current + // test environment. + exitCodeWaived = 2 +) + +// Flags. +var ( + verifyCompatibility = flag.Bool("cuda_verify_compatibility", os.Getenv("GVISOR_TEST_CUDA_VERIFY_COMPATIBILITY") == "true", "whether to verify that all tests are marked as compatible") + logSuccessfulTests = flag.Bool("cuda_log_successful_tests", false, "log console output of successful tests") + debug = flag.Bool("cuda_test_debug", false, "log more data as the test is running") + containersPerCPU = flag.Float64("cuda_containers_per_cpu", defaultContainersPerCPU, "number of parallel execution containers to spawn per CPU (floating point values allowed)") +) + +// testCompatibility maps test names to their compatibility data. +// Unmapped test names are assumed to be fully compatible. +var testCompatibility = map[string]Compatibility{ + "0_Introduction/simpleAttributes": RequiresFeatures(FeaturePersistentL2Caching), + "0_Introduction/simpleCUDA2GL": RequiresFeatures(FeatureGL), + "0_Introduction/simpleIPC": &BrokenInGVisor{OnlyWhenMultipleGPU: true}, + "0_Introduction/simpleP2P": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "0_Introduction/UnifiedMemoryStreams": &BrokenInGVisor{}, + "0_Introduction/vectorAddMMAP": &BrokenInGVisor{OnlyWhenMultipleGPU: true}, + "2_Concepts_and_Techniques/cuHook": &BrokenEverywhere{ + Reason: "Requires ancient version of glibc (<=2.33)", + }, + "2_Concepts_and_Techniques/EGLStream_CUDA_Interop": &BrokenEverywhere{ + Reason: "Requires newer version of EGL libraries than Ubuntu has (eglCreateStreamKHR)", + }, + "2_Concepts_and_Techniques/EGLStream_CUDA_CrossGPU": MultiCompatibility( + &RequiresMultiGPU{}, + &BrokenEverywhere{ + Reason: "Requires newer version of EGL libraries than Ubuntu has (eglCreateStreamKHR)", + }, + ), + "2_Concepts_and_Techniques/EGLSync_CUDAEvent_Interop": &OnlyOnWindows{}, + "2_Concepts_and_Techniques/streamOrderedAllocationIPC": &BrokenInGVisor{}, + "2_Concepts_and_Techniques/streamOrderedAllocationP2P": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "3_CUDA_Features/bf16TensorCoreGemm": RequiresFeatures(FeatureTensorCores), + "3_CUDA_Features/cdpAdvancedQuicksort": RequiresFeatures(FeatureDynamicParallelism), + "3_CUDA_Features/cudaCompressibleMemory": RequiresFeatures(FeatureCompressibleMemory), + "3_CUDA_Features/dmmaTensorCoreGemm": RequiresFeatures(FeatureTensorCores), + "3_CUDA_Features/memMapIPCDrv": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "3_CUDA_Features/tf32TensorCoreGemm": RequiresFeatures(FeatureTensorCores), + "4_CUDA_Libraries/conjugateGradientMultiDeviceCG": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "4_CUDA_Libraries/cudaNvSci": &RequiresNvSci{}, + "4_CUDA_Libraries/cudaNvSciNvMedia": &RequiresNvSci{}, + "4_CUDA_Libraries/cuDLAErrorReporting": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLAHybridMode": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLAStandaloneMode": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLALayerwiseStatsHybrid": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLALayerwiseStatsStandalone": &OnlyOnWindows{}, + "4_CUDA_Libraries/simpleCUFFT_2d_MGPU": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "4_CUDA_Libraries/simpleCUFFT_MGPU": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "5_Domain_Specific/fluidsD3D9": &OnlyOnWindows{}, + "5_Domain_Specific/fluidsGL": RequiresFeatures(FeatureGL), + "5_Domain_Specific/fluidsGLES": &OnlyOnWindows{}, + "5_Domain_Specific/nbody_opengles": &OnlyOnWindows{}, + "5_Domain_Specific/nbody_screen": &OnlyOnWindows{}, + "5_Domain_Specific/p2pBandwidthLatencyTest": &BrokenInGVisor{OnlyWhenMultipleGPU: true}, + "5_Domain_Specific/postProcessGL": RequiresFeatures(FeatureGL), + "5_Domain_Specific/simpleD3D10": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D10RenderTarget": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D10Texture": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D11": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D11Texture": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D12": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D9": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D9Texture": &OnlyOnWindows{}, + "5_Domain_Specific/simpleGLES": &OnlyOnWindows{}, + "5_Domain_Specific/simpleGLES_EGLOutput": &OnlyOnWindows{}, + "5_Domain_Specific/simpleGLES_screen": &OnlyOnWindows{}, + "5_Domain_Specific/simpleVulkan": RequiresFeatures(FeatureGL), + "5_Domain_Specific/simpleVulkanMMAP": RequiresFeatures(FeatureGL), + "5_Domain_Specific/SLID3D10Texture": &OnlyOnWindows{}, + "5_Domain_Specific/VFlockingD3D10": &OnlyOnWindows{}, + "5_Domain_Specific/vulkanImageCUDA": RequiresFeatures(FeatureGL), +} + +// flakyTests is a list of tests that are flaky. +// These will be retried up to 3 times in parallel before running serially. +var flakyTests = map[string]struct{}{} + +// exclusiveTests is a list of tests that must run exclusively (i.e. with +// no other test running on the machine at the same time), or they will +// likely fail. These tests are not attempted to be run in parallel. +// This is usually the case for performance tests or tests that use a lot +// of resources in general. +// This saves the trouble to run them in parallel, while also avoiding +// causing spurious failures for the tests that happen to be running in +// parallel with them. +var exclusiveTests = map[string]struct{}{ + "6_Performance/alignedTypes": {}, + "6_Performance/transpose": {}, + "6_Performance/UnifiedMemoryPerf": {}, +} + +// alwaysSkippedTests don't run at all, ever, and are not verified when +// --cuda_verify_compatibility is set. +// Each test is mapped to a reason why it should be skipped. +var alwaysSkippedTests = map[string]string{ + // These tests seem to flake in gVisor, but consistently within the same + // run of the overall test, so they cannot be included in `flakyTests`. + "0_Introduction/simpleAssert": "Flaky in gVisor", + "0_Introduction/simpleAssert_nvrtc": "Flaky in gVisor", +} + +// Feature is a feature as listed by /list_features.sh. +type Feature string + +// All CUDA features listed by /list_features.sh. +const ( + FeaturePersistentL2Caching Feature = "PERSISTENT_L2_CACHING" + FeatureDynamicParallelism Feature = "DYNAMIC_PARALLELISM" + FeatureGL Feature = "GL" + FeatureTensorCores Feature = "TENSOR_CORES" + FeatureCompressibleMemory Feature = "COMPRESSIBLE_MEMORY" +) + +// allFeatures is a list of all CUDA features above. +var allFeatures = []Feature{ + FeaturePersistentL2Caching, + FeatureDynamicParallelism, + FeatureGL, + FeatureTensorCores, + FeatureCompressibleMemory, +} + +// TestEnvironment represents the environment in which a sample test runs. +type TestEnvironment struct { + NumGPUs int + RuntimeIsGVisor bool + Features map[Feature]bool +} + +// Compatibility encodes the compatibility of a test depending on the +// environment it runs in. +type Compatibility interface { + // WillFail returns a string explaining why the test is expected to fail + // in the given environment, or "" if it isn't expected to fail. + WillFail(ctx context.Context, env *TestEnvironment) string + + // IsExpectedFailure checks whether the `logs` (from a failed run of the test + // in the given environment) matches the failure that this test expects in + // that environment. If they match, this function should return nil. + // It is only called when `WillFail` returns a non-empty string for the same + // environment, so it may assume that `env` is non-compatible. + IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error +} + +// BrokenEverywhere implements `Compatibility` for tests that are broken in +// all environments. +type BrokenEverywhere struct { + Reason string +} + +// WillFail implements `Compatibility.WillFail`. +func (be *BrokenEverywhere) WillFail(ctx context.Context, env *TestEnvironment) string { + return fmt.Sprintf("Known-broken test: %v", be.Reason) +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*BrokenEverywhere) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return nil +} + +// BrokenInGVisor implements `Compatibility` for tests that are broken in +// gVisor only. +type BrokenInGVisor struct { + // OnlyWhenMultipleGPU may be set to true for tests which only fail when + // multiple GPUs are present. This should not be used for tests that + // *require* multiple GPUs to run (use RequiresMultiGPU instead). + // This is for tests that can run on a single or multiple GPUs alike, + // but specifically fail in gVisor when run with multiple GPUs. + OnlyWhenMultipleGPU bool + + // KnownToHang may be set to true for short tests which can hang instead + // of failing. This avoids waiting ~forever for them to finish. + KnownToHang bool +} + +// WillFail implements `Compatibility.WillFail`. +func (big *BrokenInGVisor) WillFail(ctx context.Context, env *TestEnvironment) string { + if !env.RuntimeIsGVisor { + return "" + } + if big.OnlyWhenMultipleGPU && env.NumGPUs == 1 { + return "" + } + if big.OnlyWhenMultipleGPU { + return "Known to be broken in gVisor when multiple GPUs are present" + } + return "Known to be broken in gVisor" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*BrokenInGVisor) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return nil +} + +// RequiresMultiGPU implements `Compatibility` for tests that require multiple +// GPUs. +type RequiresMultiGPU struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*RequiresMultiGPU) WillFail(ctx context.Context, env *TestEnvironment) string { + if env.NumGPUs < 2 { + return "Requires >= 2 GPUs" + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*RequiresMultiGPU) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + if exitCode != exitCodeWaived { + return fmt.Errorf("exit code %d, expected EXIT_WAIVED (%d)", exitCode, exitCodeWaived) + } + return nil +} + +// requiresFeatures implements `Compatibility` for tests that require +// specific features. +type requiresFeatures struct { + features []Feature +} + +func RequiresFeatures(features ...Feature) Compatibility { + return &requiresFeatures{features: features} +} + +// WillFail implements `Compatibility.WillFail`. +func (r *requiresFeatures) WillFail(ctx context.Context, env *TestEnvironment) string { + for _, feature := range r.features { + if !env.Features[feature] { + return fmt.Sprintf("Requires feature %s", feature) + } + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*requiresFeatures) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + if exitCode != exitCodeWaived { + return fmt.Errorf("exit code %d, expected EXIT_WAIVED (%d)", exitCode, exitCodeWaived) + } + return nil +} + +// OnlyOnWindows implements `Compatibility` for tests that are only expected +// to only pass on Windows. +type OnlyOnWindows struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*OnlyOnWindows) WillFail(ctx context.Context, env *TestEnvironment) string { + if runtime.GOOS != "windows" { + return "Only runs on Windows" + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*OnlyOnWindows) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + if strings.Contains(logs, "is not supported on Linux") { + return nil + } + if exitCode != exitCodeWaived { + return fmt.Errorf("exit code %d, expected EXIT_WAIVED (%d)", exitCode, exitCodeWaived) + } + return nil +} + +type RequiresNvSci struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*RequiresNvSci) WillFail(ctx context.Context, env *TestEnvironment) string { + return "Requires NvSci library which is not open-source" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*RequiresNvSci) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return nil +} + +// multiCompatibility implements `Compatibility` with multiple possible +// Compatibility implementations. +type multiCompatibility struct { + compats []Compatibility +} + +// MultiCompatibility implements `Compatibility` with multiple possible +// Compatibility implementations. +func MultiCompatibility(compats ...Compatibility) Compatibility { + return &multiCompatibility{compats: compats} +} + +// WillFail implements `Compatibility.WillFail`. +func (mc *multiCompatibility) WillFail(ctx context.Context, env *TestEnvironment) string { + for _, compat := range mc.compats { + if reason := compat.WillFail(ctx, env); reason != "" { + return reason + } + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (mc *multiCompatibility) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + var possibleCompats []Compatibility + for _, compat := range mc.compats { + if reason := compat.WillFail(ctx, env); reason != "" { + possibleCompats = append(possibleCompats, compat) + } + } + if len(possibleCompats) == 0 { + return errors.New("no known explanation for this failure") + } + var errs []string + for _, compat := range possibleCompats { + err := compat.IsExpectedFailure(ctx, env, logs, exitCode) + if err == nil { + return nil + } + errs = append(errs, fmt.Sprintf("might have been broken because %s but %v", compat.WillFail(ctx, env), err)) + } + return fmt.Errorf("no known explanation for this failure: %v", strings.Join(errs, "; ")) +} + +// FullyCompatible implements `Compatibility` for tests that are expected to +// pass in any environment. +type FullyCompatible struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*FullyCompatible) WillFail(ctx context.Context, env *TestEnvironment) string { + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*FullyCompatible) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return errors.New("test is expected to pass regardless of environment") +} + +// getContainerOpts returns the container run options to run CUDA tests. +func getContainerOpts() dockerutil.RunOpts { + opts := dockerutil.GPURunOpts() + opts.Image = "gpu/cuda-tests" + return opts +} + +// testLog logs a line as a test log. +// If debug is enabled, it is also printed immediately to stderr. +// This is useful for debugging tests. +func testLog(t *testing.T, format string, values ...any) { + t.Helper() + if *debug { + fmt.Fprintf(os.Stderr, "[%s] %s\n", t.Name(), fmt.Sprintf(format, values...)) + } + t.Logf(format, values...) +} + +// multiLineLog logs a multiline string as separate log messages to `t`. +// This is useful to log multi-line container logs without them looking weird +// with line breaks in the middle. +func multiLineLog(t *testing.T, output string) { + t.Helper() + for _, line := range strings.Split(output, "\n") { + // `line` may contain % characters here, so we need to format it through + // `%s` so that `%` characters don't show up as "MISSING" in the logs. + testLog(t, "%s", line) + } +} + +// GetEnvironment returns the environment in which a sample test runs. +func GetEnvironment(ctx context.Context, t *testing.T) (*TestEnvironment, error) { + numGPU := dockerutil.NumGPU() + if numGPU == 0 { + return nil, errors.New("no GPUs detected") + } + if numGPU == 1 { + testLog(t, "1 GPU detected") + } else { + testLog(t, "%d GPUs detected", numGPU) + } + runtimeIsGVisor, err := dockerutil.IsGVisorRuntime(ctx, t) + if err != nil { + return nil, fmt.Errorf("cannot determine if runtime is gVisor or not: %w", err) + } + if runtimeIsGVisor { + testLog(t, "Runtime is detected as gVisor") + } else { + testLog(t, "Runtime is detected as not gVisor") + } + featuresContainer := dockerutil.MakeContainer(ctx, t) + defer featuresContainer.CleanUp(ctx) + featuresList, err := featuresContainer.Run(ctx, getContainerOpts(), "/list_features.sh") + if err != nil { + return nil, fmt.Errorf("cannot get list of CUDA features: %v", err) + } + features := make(map[Feature]bool) + for _, line := range strings.Split(featuresList, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + featureAvailable := false + var feature Feature + if strings.HasPrefix(line, "PRESENT: ") { + featureAvailable = true + feature = Feature(strings.TrimPrefix(line, "PRESENT: ")) + } else if strings.HasPrefix(line, "ABSENT: ") { + featureAvailable = false + feature = Feature(strings.TrimPrefix(line, "ABSENT: ")) + } else { + return nil, fmt.Errorf("unexpected CUDA feature line: %q", line) + } + found := false + for _, f := range allFeatures { + if feature == f { + features[f] = featureAvailable + if featureAvailable { + testLog(t, "CUDA feature is available: %s", string(f)) + } else { + testLog(t, "CUDA feature is *not* available: %s", string(f)) + } + found = true + break + } + } + if !found { + return nil, fmt.Errorf("unknown CUDA feature: %s", string(feature)) + } + } + for _, feature := range allFeatures { + if _, ok := features[feature]; !ok { + return nil, fmt.Errorf("CUDA feature not found in feature list: %s", string(feature)) + } + } + // Use CUDA dynamic parallelism as a litmus test to see if the features were + // enumerated correctly. + if _, hasDynamicParallelism := features[FeatureDynamicParallelism]; !hasDynamicParallelism { + return nil, errors.New("CUDA feature Dynamic Parallelism is not available yet should be available in all environments gVisor supports; this indicates a failure in the feature listing script") + } + return &TestEnvironment{ + NumGPUs: numGPU, + RuntimeIsGVisor: runtimeIsGVisor, + Features: features, + }, nil +} + +// runSampleTest runs a single CUDA sample test. +// It first tries to run in pooled container. +// If that fails, then it runs in an exclusive container. +// It returns a skip reason (or empty if the test was not skipped), and +// an error if the test fails. +func runSampleTest(ctx context.Context, t *testing.T, testName string, te *TestEnvironment, cp *dockerutil.ContainerPool) (string, error) { + compat, found := testCompatibility[testName] + if !found { + compat = &FullyCompatible{} + } + willFailReason := compat.WillFail(ctx, te) + if willFailReason != "" && !*verifyCompatibility { + return fmt.Sprintf("this test is expected to fail (%s) --cuda_verify_compatibility=true to verify compatibility)", willFailReason), nil + } + if skipReason, isAlwaysSkipped := alwaysSkippedTests[testName]; isAlwaysSkipped { + return fmt.Sprintf("this test is always skipped (%v)", skipReason), nil + } + testTimeout := defaultTestTimeout + execTestTimeout := testTimeout - 15*time.Second + testAttempts := 1 + if _, isFlakyTest := flakyTests[testName]; isFlakyTest { + testAttempts = 3 + } + parallelAttempts := testAttempts + if _, isExclusiveTest := exclusiveTests[testName]; isExclusiveTest { + parallelAttempts = 0 + } + for attempt := 0; attempt < parallelAttempts; attempt++ { + c, release, err := cp.Get(ctx) + if err != nil { + release() + return "", fmt.Errorf("failed to get container: %v", err) + } + cp.SetContainerLabel(c, fmt.Sprintf("Running %s in parallel (attempt %d/%d)", testName, attempt+1, parallelAttempts)) + testLog(t, "Running test in parallel mode in container %s (attempt %d/%d)...", c.Name, attempt+1, parallelAttempts) + parallelCtx, parallelCancel := context.WithTimeoutCause(ctx, testTimeout, errors.New("parallel execution took too long")) + testStartedAt := time.Now() + output, err := c.Exec(parallelCtx, dockerutil.ExecOpts{}, "/run_sample", fmt.Sprintf("--timeout=%v", execTestTimeout), testName) + testDuration := time.Since(testStartedAt) + parallelCancel() + release() + if err == nil { + if willFailReason != "" { + multiLineLog(t, output) + return "", fmt.Errorf("test unexpectedly succeeded, but we expected it to fail: %s; please update `testCompatibility`", willFailReason) + } + // Only log the output when the test succeeds here. + // If it fails, we'll run exclusively below, and the output from *that* + // run will be logged instead. + if *logSuccessfulTests { + multiLineLog(t, output) + } + testLog(t, "Test passed in parallel mode in %v.", testDuration) + return "", nil + } + var exitCode int + if execErr, ok := err.(*dockerutil.ExecError); ok { + exitCode = execErr.ExitStatus + } + if willFailReason != "" { + isExpectedErr := compat.IsExpectedFailure(ctx, te, output, exitCode) + if isExpectedErr == nil { + testLog(t, "Test failed as expected: %s (took %v)", willFailReason, testDuration) + return "", nil + } + } + } + if parallelAttempts > 0 { + testLog(t, "Will re-run the test in exclusive mode.") + } + c, release, err := cp.GetExclusive(ctx) + defer release() + if err != nil { + return "", fmt.Errorf("failed to get excusive container: %v", err) + } + var testErr error + for attempt := 0; attempt < testAttempts; attempt++ { + cp.SetContainerLabel(c, fmt.Sprintf("Running %s exclusively (attempt %d/%d)", testName, attempt+1, testAttempts)) + testLog(t, "Running test in exclusive mode in container %s (attempt %d/%d)...", c.Name, attempt+1, testAttempts) + exclusiveCtx, exclusiveCancel := context.WithTimeoutCause(ctx, testTimeout, errors.New("exclusive execution took too long")) + testStartedAt := time.Now() + var output string + output, testErr = c.Exec(exclusiveCtx, dockerutil.ExecOpts{}, "/run_sample", fmt.Sprintf("--timeout=%v", execTestTimeout), testName) + testDuration := time.Since(testStartedAt) + exclusiveCancel() + if testErr == nil { + if willFailReason != "" { + multiLineLog(t, output) + return "", fmt.Errorf("test unexpectedly succeeded, but we expected it to fail: %s; please update `testCompatibility`", willFailReason) + } + if *logSuccessfulTests { + multiLineLog(t, output) + } + testLog(t, "Test passed in exclusive mode in %v.", testDuration) + return "", nil + } + multiLineLog(t, output) + var exitCode int + if execErr, ok := testErr.(*dockerutil.ExecError); ok { + exitCode = execErr.ExitStatus + } + if willFailReason != "" { + isExpectedErr := compat.IsExpectedFailure(ctx, te, output, exitCode) + if isExpectedErr == nil { + testLog(t, "Test failed as expected: %s (took %v)", willFailReason, testDuration) + return "", nil + } + return "", fmt.Errorf("test was expected to fail (%s), but it failed with %v which is a different reason reason than expected: %v", willFailReason, testErr, isExpectedErr) + } + } + return "", fmt.Errorf("test failed: %v", testErr) +} + +// getDesiredTestParallelism returns the number of tests to run in parallel. +func getDesiredTestParallelism() int { + numCPU := runtime.NumCPU() + if numCPU <= 0 { + panic("cannot detect number of cores") + } + return int(math.Ceil((*containersPerCPU) * float64(numCPU))) +} + +// TestCUDA runs CUDA tests. +func TestCUDA(t *testing.T) { + const defaultMaxDuration = 59*time.Minute + 30*time.Second + + testStart := time.Now() + maxDuration := defaultMaxDuration + if timeoutFlag := flag.Lookup("timeout"); timeoutFlag != nil { + if timeoutFlagStr := timeoutFlag.Value.String(); timeoutFlagStr != "" { + timeoutFlagValue, err := time.ParseDuration(timeoutFlagStr) + if err != nil { + t.Fatalf("--timeout flag %q is not a valid duration: %v", timeoutFlagStr, err) + } + if timeoutFlagValue != 0 { + maxDuration = timeoutFlagValue + } + } + } + ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, errors.New("overall test timed out")) + defer cancel() + testDeadline, ok := ctx.Deadline() + if !ok { + t.Fatal("context had no deadline") + } + testLog(t, "Test timeout is %v; started at %v, deadline is %v", maxDuration, testStart, testDeadline) + + te, err := GetEnvironment(ctx, t) + if err != nil { + t.Fatalf("Failed to get test environment: %v", err) + } + + // Get a list of sample tests. + listContainer := dockerutil.MakeContainer(ctx, t) + defer listContainer.CleanUp(ctx) + testsList, err := listContainer.Run(ctx, getContainerOpts(), "/list_sample_tests.sh") + if err != nil { + t.Fatalf("Cannot list sample tests: %v", err) + } + testsSplit := strings.Split(testsList, "\n") + allTests := make([]string, 0, len(testsSplit)) + allTestsMap := make(map[string]struct{}, len(testsSplit)) + for _, test := range testsSplit { + testName := strings.TrimSpace(test) + if testName == "" { + continue + } + allTestsMap[testName] = struct{}{} + allTests = append(allTests, testName) + } + numTests := len(allTests) + testLog(t, "Number of CUDA sample tests detected: %d", numTests) + + // Check that all tests in test maps still exist. + t.Run("CUDA test existence", func(t *testing.T) { + for testName := range testCompatibility { + if _, ok := allTestsMap[testName]; !ok { + t.Errorf("CUDA test %q referenced in `testCompatibility` but it no longer exists, please remove it.", testName) + } + } + }) + + // In order to go through tests efficiently, we reuse containers. + // However, running tests serially within the same container would also be + // slow. So this test spawns a pool of containers, one per CPU. + // This saves time because a lot of the time here is actually spent waiting + // for compilation of the CUDA program on the CPU, and isn't actually + // blocked on the GPU. However, it is possible that two CUDA tests do end + // up running on the GPU at the same time, and that they don't work together + // for some reason (e.g. out of GPU memory). + // To address this, the test first runs every test in parallel. Then, if + // any of them failed, it will run only the failed ones serially. + numContainers := getDesiredTestParallelism() + testLog(t, "Number of cores is %d, spawning %.1f CUDA containers for each (%d containers total)...", runtime.NumCPU(), *containersPerCPU, numContainers) + spawnGroup, spawnCtx := errgroup.WithContext(ctx) + containers := make([]*dockerutil.Container, numContainers) + for i := 0; i < numContainers; i++ { + spawnGroup.Go(func() error { + c := dockerutil.MakeContainer(ctx, t) + if err := c.Spawn(spawnCtx, getContainerOpts(), "/bin/sleep", "6h"); err != nil { + return fmt.Errorf("container %v failed to spawn: %w", c.Name, err) + } + containers[i] = c + return nil + }) + } + if err := spawnGroup.Wait(); err != nil { + for _, c := range containers { + if c != nil { + c.CleanUp(ctx) + } + } + t.Fatalf("Failed to spawn containers: %v", err) + } + cp := dockerutil.NewContainerPool(containers) + defer cp.CleanUp(ctx) + var testMu sync.Mutex + testsDone := 0 + var failedTests []string + statusFn := func() { + now := time.Now() + testMu.Lock() + defer testMu.Unlock() + donePct := 100.0 * float64(testsDone) / float64(numTests) + startedAgo := now.Sub(testStart) + deadlineIn := testDeadline.Sub(now) + durationPct := 100.0 * float64(startedAgo) / float64(testDeadline.Sub(testStart)) + testLog(t, "[Timing] %d/%d tests (%.1f%%) finished executing. Test started %v ago, deadline in %v (%.1f%%).", testsDone, numTests, donePct, startedAgo.Truncate(time.Second), deadlineIn.Truncate(time.Second), durationPct) + if len(failedTests) > 0 { + testLog(t, "[Failed] %d test failed: %v", len(failedTests), strings.Join(failedTests, ", ")) + } + testLog(t, "[Pool] %v", cp.String()) + } + if *debug { + go func() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + statusFn() + } + } + }() + } + var samplesTestName string + t.Run("Samples", func(t *testing.T) { + samplesTestName = t.Name() + // Now spawn all subtests in parallel. + // All sub-tests will first try to run in parallel using one of the pooled + // containers. + // Those that failed will try to grab `serialMu` in order to run serially. + // Therefore, the main goroutine here holds `serialMu` and only releases + // when all parallel test attempts have completed. + testutil.NewTree(allTests, "/").RunParallel(t, func(t *testing.T, testName string) { + t.Helper() + skippedReason, err := runSampleTest(ctx, t, testName, te, cp) + if err != nil { + t.Errorf("%s: %v", testName, err) + } + testMu.Lock() + defer testMu.Unlock() + testsDone++ + if t.Failed() && ctx.Err() == nil { + failedTests = append(failedTests, testName) + } + if skippedReason != "" { + t.Skip(skippedReason) + } + }) + }) + statusFn() + testMu.Lock() + defer testMu.Unlock() + if len(failedTests) > 0 { + if ctx.Err() != nil { + t.Errorf("%d tests failed prior to timeout:", len(failedTests)) + for _, testName := range failedTests { + t.Errorf(" %s", testName) + } + } + if len(failedTests) > 0 { + t.Errorf("To re-run a specific test locally, either re-run this test with filtering enabled (example: --test.run=%s/%s), or:", samplesTestName, failedTests[0]) + t.Errorf( + " $ docker run --runtime=%s --gpus=all -e %s --rm %s /run_sample %s", + dockerutil.Runtime(), + dockerutil.AllGPUCapabilities, + getContainerOpts().Image, + failedTests[0], + ) + } + } else if poolUtilization := cp.Utilization(); poolUtilization < 0.6 { + testLog(t, "WARNING: Pool utilization was only %.1f%%.", poolUtilization*100.0) + testLog(t, "This test can be made faster and more efficient with proper test categorization,") + testLog(t, "by identifying flaky tests and exclusive-requiring tests.") + testLog(t, "Consider going over the logs to identify such tests and categorize them accordingly.") + } +} + +// TestMain overrides the `test.parallel` flag. +func TestMain(m *testing.M) { + dockerutil.EnsureSupportedDockerVersion() + flag.Parse() + // The Go testing library won't run more than GOMAXPROCS parallel tests by + // default, and the value of GOMAXPROCS is taken at program initialization + // time, so by the time we get here, it is already stuck at GOMAXPROCS. + // In order to run more parallel tests than there are cores, we therefore + // need to override the `test.parallel` flag here before `m.Run`. + testParallelFlag := flag.Lookup("test.parallel") + if testParallelFlag == nil { + panic("cannot find -test.parallel flag") + } + if err := testParallelFlag.Value.Set(strconv.Itoa(getDesiredTestParallelism())); err != nil { + panic(fmt.Sprintf("cannot set -test.parallel flag: %v", err)) + } + os.Exit(m.Run()) +} diff --git a/test/gpu/smoke_test.go b/test/gpu/smoke_test.go index e5a91e1ed..5e273357c 100644 --- a/test/gpu/smoke_test.go +++ b/test/gpu/smoke_test.go @@ -36,16 +36,16 @@ func TestGPUHello(t *testing.T) { t.Logf("cuda-vector-add output: %s", string(out)) } -func TestCUDATests(t *testing.T) { +func TestCUDASmokeTests(t *testing.T) { ctx := context.Background() c := dockerutil.MakeContainer(ctx, t) defer c.CleanUp(ctx) opts := dockerutil.GPURunOpts() opts.Image = "gpu/cuda-tests" - out, err := c.Run(ctx, opts) + out, err := c.Run(ctx, opts, "/run_smoke.sh") if err != nil { - t.Fatalf("could not run cuda-tests: %v", err) + t.Fatalf("could not run cuda-tests smoke tests: %v", err) } - t.Logf("cuda-tests output: %s", string(out)) + t.Logf("cuda-tests smoke tests output: %s", string(out)) }