diff --git a/images/gpu/cuda-tests/Dockerfile.x86_64 b/images/gpu/cuda-tests/Dockerfile.x86_64 index d335c691b..76198083b 100644 --- a/images/gpu/cuda-tests/Dockerfile.x86_64 +++ b/images/gpu/cuda-tests/Dockerfile.x86_64 @@ -1,8 +1,45 @@ -FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 +FROM nvidia/cuda:12.3.2-devel-ubuntu22.04 + +# From: https://github.com/NVIDIA/cuda-samples/releases +# Ideally, pick a release that matches the CUDA version of the image above. +ARG CUDA_SAMPLES_VERSION=v12.3 WORKDIR / -COPY cuda_malloc_managed.cu . -COPY cuda_test_util.h . -COPY run.sh . +COPY *.cu *.h *.sh *.go / ENV PATH=$PATH:/usr/local/nvidia/bin:/bin/nvidia/bin -ENTRYPOINT ["/run.sh"] +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && \ + apt-get install -y \ + build-essential \ + cmake \ + freeglut3 freeglut3-dev \ + git \ + golang \ + imagemagick \ + libegl-dev \ + libfreeimage3 libfreeimage-dev \ + libfreeimageplus3 libfreeimageplus-dev \ + libgles2-mesa-dev \ + libglfw3 libglfw3-dev \ + libglu1-mesa libglu1-mesa-dev \ + libxi-dev \ + libxmu-dev \ + llvm \ + mpich \ + pkg-config \ + x11-xserver-utils \ + xdotool \ + xvfb \ + zlib1g zlib1g-dev \ + && \ + chmod 555 /*.sh && \ + git clone --depth=1 --branch="$CUDA_SAMPLES_VERSION" --single-branch \ + https://github.com/NVIDIA/cuda-samples.git /cuda-samples && \ + go install \ + github.com/TheZoraiz/ascii-image-converter@d05a757c5e02ab23e97b6f6fca4e1fbeb10ab559 && \ + mv "$HOME/go/bin/ascii-image-converter" /usr/bin/ && \ + go build -o /run_sample /run_sample.go + +# Override entrypoint to nothing, otherwise all invocations will have +# a copyright notice printed, which breaks parsing the stdout logs. +ENTRYPOINT [] diff --git a/images/gpu/cuda-tests/cuda_test_util.h b/images/gpu/cuda-tests/cuda_test_util.h index 0d0e4a7af..2877f78b1 100644 --- a/images/gpu/cuda-tests/cuda_test_util.h +++ b/images/gpu/cuda-tests/cuda_test_util.h @@ -17,6 +17,7 @@ #include +// cudaError_t is returned by CUDA runtime functions. #define CHECK_CUDA(expr) \ do { \ cudaError_t code = (expr); \ @@ -27,4 +28,15 @@ } \ } while (0) +// CUresult is returned by CUDA driver functions. +#define CHECK_CUDA_RESULT(expr) \ + do { \ + CUresult code = (expr); \ + if (code != CUDA_SUCCESS) { \ + std::cout << "Check failed at " << __FILE__ << ":" << __LINE__ << ": " \ + << #expr << ": " << code << std::endl; \ + abort(); \ + } \ + } while (0) + #endif // THIRD_PARTY_GVISOR_IMAGES_GPU_CUDA_TESTS_CUDA_TEST_UTIL_H_ diff --git a/images/gpu/cuda-tests/list_features.cu b/images/gpu/cuda-tests/list_features.cu new file mode 100644 index 000000000..060740304 --- /dev/null +++ b/images/gpu/cuda-tests/list_features.cu @@ -0,0 +1,52 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This program lists the features of the CUDA device that are available. +// It is used as part of the list_features.sh script. +// Each line it outputs is a CUDA feature name, prefixed by either +// "PRESENT: " or "ABSENT: ". + +#include +#include +#include + +#include "cuda_test_util.h" // NOLINT(build/include) + +void printFeature(const char* feature, bool have) { + if (have) { + printf("PRESENT: %s\n", feature); + } else { + printf("ABSENT: %s\n", feature); + } +} + +int main(int argc, char *argv[]) { + int cuda_device; + CHECK_CUDA(cudaGetDevice(&cuda_device)); + cudaDeviceProp properties; + CHECK_CUDA(cudaGetDeviceProperties(&properties, cuda_device)); + bool cdpCapable = + (properties.major == 3 && properties.minor >= 5) || properties.major >= 4; + printFeature("DYNAMIC_PARALLELISM", cdpCapable); + printFeature( + "PERSISTENT_L2_CACHING", properties.persistingL2CacheMaxSize > 0); + // Tensor cores are a thing in Volta (SM8X) + printFeature("TENSOR_CORES", properties.major >= 8); + int isCompressionAvailable; + CHECK_CUDA_RESULT( + cuDeviceGetAttribute(&isCompressionAvailable, + CU_DEVICE_ATTRIBUTE_GENERIC_COMPRESSION_SUPPORTED, + cuda_device)); + printFeature("COMPRESSIBLE_MEMORY", isCompressionAvailable != 0); +} diff --git a/images/gpu/cuda-tests/list_features.sh b/images/gpu/cuda-tests/list_features.sh new file mode 100644 index 000000000..32ec98a10 --- /dev/null +++ b/images/gpu/cuda-tests/list_features.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2024 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script outputs a list of CUDA features that are present or absent, +# one per line. Each line begins with either "PRESENT: " or "ABSENT: ", +# followed by the feature name. + +set -euo pipefail + +cd / +nvcc list_features.cu -lcuda -o list_features +./list_features + +# Detect GL by using a simple test that uses it as reference. +if xvfb-run make -C /cuda-samples/Samples/0_Introduction/simpleCUDA2GL TARGET_ARCH="$(uname -m)" testrun &>/dev/null; then + echo "PRESENT: GL" +else + echo "ABSENT: GL" +fi diff --git a/images/gpu/cuda-tests/list_sample_tests.sh b/images/gpu/cuda-tests/list_sample_tests.sh new file mode 100644 index 000000000..e34772ccc --- /dev/null +++ b/images/gpu/cuda-tests/list_sample_tests.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Copyright 2024 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script outputs a sorted list of CUDA sample tests, one per line. + +set -euo pipefail + +( + while IFS= read -r makefile_path; do + dirname "$makefile_path" + done < <(find /cuda-samples -type f -name Makefile) \ + | grep -vE '^/cuda-samples$' | grep -vE '/7_libNVVM' + + # cuda-samples/Samples/7_libNVVM is not structured like the other tests. + # It is built with `cmake` and generates multiple test binaries. + # The generated ones all follow the pattern of being named after their + # parent directory name, so we look for that. + pushd /cuda-samples/Samples/7_libNVVM &>/dev/null + cmake . &>/dev/null + make TARGET_ARCH="$(uname -m)" all &>/dev/null + popd &>/dev/null + while IFS= read -r dir_path; do + if [[ -x "$dir_path/$(basename "$dir_path")" ]]; then + echo "$dir_path" + fi + done < <(find /cuda-samples/Samples/7_libNVVM -type d) | sort | uniq +) | sed 's~/cuda-samples/Samples/~~' | sort diff --git a/images/gpu/cuda-tests/run_sample.go b/images/gpu/cuda-tests/run_sample.go new file mode 100644 index 000000000..1d5e9ca1d --- /dev/null +++ b/images/gpu/cuda-tests/run_sample.go @@ -0,0 +1,1057 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// run_sample runs a CUDA sample test. +// These tests are complicated because some of them involve X windows, +// as opposed to traditional command-line-only tests. +// This binary handles all types of CUDA sample tests. +// +// To run: /run_sample [--timeout=15m] test1 test2 test3 ... +package main + +import ( + "bufio" + "bytes" + "context" + "errors" + "flag" + "fmt" + "image" + "image/draw" + "image/png" + "io" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// Flags. +var ( + timeoutFlag = flag.Duration("timeout", 15*time.Minute, "Timeout for the program before it must clean up") +) + +const ( + // xDisplay is the X server address. + xDisplay = ":0" +) + +// logMu protects log output. +var logMu sync.Mutex + +// log logs a message to stderr. `format` should not have a newline. +// This does not use the standard logging library because this program needs +// to support logging multiple lines atomically. +func log(format string, values ...any) { + logDo(func() { + fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, values...)) + }) +} + +// logDo runs a function while logging the log lock. +// This is useful to log multiple lines at a time. +func logDo(fn func()) { + logMu.Lock() + defer logMu.Unlock() + fn() +} + +// logWriter implements io.Writer and logs to stderr. +type logWriter struct{} + +func (w *logWriter) Write(p []byte) (n int, err error) { + logDo(func() { + n, err = os.Stderr.Write(p) + }) + return n, err +} + +// Command wraps a command with some niceties for stdout/stderr handling. +type Command struct { + // Cmd is the wrapped command. + Cmd *exec.Cmd + + // Option fields. + // If non-nil, this data will be fed to the command's stdin. + Stdin []byte + // ForwardStdout and ForwardStderr control whether stdout/stderr are + // forwarded to the user's console. + ForwardStdout, ForwardStderr bool + // PrefixStdout and PrefixStderr are prefixes for forwarded logs. + PrefixStdout, PrefixStderr string + + // streamWG waits for stdout/stderr capturing goroutines. + streamWG sync.WaitGroup + + // mu protects the fields below. + mu sync.Mutex + + // started is `true` if the command has started. + started bool + + // Sets of stdout/stderr/combined output lines. + stdoutLines, stderrLines, combined []string + + // waitErr is the error returned by `Cmd.Wait`. + waitErr error + + // doneCh is closed when the command is done running. + doneCh chan struct{} +} + +// Start starts a command in the background. +func (c *Command) Start(ctx context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.started { + return errors.New("command already started") + } + for _, env := range os.Environ() { + c.Cmd.Env = append(c.Cmd.Env, env) + } + if len(c.Stdin) == 0 { + c.Cmd.Stdin = nil // Read from /dev/null + } else { + c.Cmd.Stdin = bytes.NewReader(c.Stdin) + } + stdout, err := c.Cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("cannot open stdout pipe: %w", err) + } + stderr, err := c.Cmd.StderrPipe() + if err != nil { + return fmt.Errorf("cannot open stderr pipe: %w", err) + } + if err := c.Cmd.Start(); err != nil { + return fmt.Errorf("cannot start command: %w", err) + } + c.started = true + + for _, stream := range []struct { + forward bool + prefix string + from io.ReadCloser + to io.Writer + lines *[]string + }{ + {c.ForwardStdout, c.PrefixStdout, stdout, os.Stdout, &c.stdoutLines}, + {c.ForwardStderr, c.PrefixStderr, stderr, &logWriter{}, &c.stderrLines}, + } { + c.streamWG.Add(1) + go func(forward bool, prefix string, from io.ReadCloser, to io.Writer, lines *[]string) { + defer c.streamWG.Done() + for scanner := bufio.NewScanner(from); scanner.Scan(); { + text := scanner.Text() + c.mu.Lock() + *lines = append(*lines, text) + c.combined = append(c.combined, text) + if forward { + fmt.Fprintf(to, "%s%s\n", prefix, text) + } + c.mu.Unlock() + } + }(stream.forward, stream.prefix, stream.from, stream.to, stream.lines) + } + c.doneCh = make(chan struct{}) + go func() { + c.streamWG.Wait() + c.mu.Lock() + defer c.mu.Unlock() + c.waitErr = c.Cmd.Wait() + close(c.doneCh) + }() + return nil +} + +// Stdout returns the standard output lines of the command so far. +func (c *Command) Stdout() []string { + c.mu.Lock() + defer c.mu.Unlock() + return c.stdoutLines[:] +} + +// Stderr returns the standard error lines of the command so far. +func (c *Command) Stderr() []string { + c.mu.Lock() + defer c.mu.Unlock() + return c.stderrLines[:] +} + +// Combined returns the combined stodut/stderr lines of the command so far. +// This is not the same as stdout concatenated with stderr, as it preserves +// line ordering as they were emitted. +func (c *Command) Combined() []string { + c.mu.Lock() + defer c.mu.Unlock() + return c.combined[:] +} + +// PID returns the PID of the running command. +func (c *Command) PID() int { + return c.Cmd.Process.Pid +} + +// ExitCode returns the exit code of the command. +func (c *Command) ExitCode(ctx context.Context) (int, error) { + c.mu.Lock() + if !c.started { + c.mu.Unlock() + return 0, errors.New("command not started") + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-c.Done(): + } + c.mu.Lock() + defer c.mu.Unlock() + if c.waitErr == nil { + return 0, nil + } + if exitErr := (*exec.ExitError)(nil); errors.As(c.waitErr, &exitErr) { + return exitErr.ExitCode(), nil + } + return 0, fmt.Errorf("process exit did not carry exit code: %w", c.waitErr) +} + +// Wait waits for a `Start`ed command to run to completion and returns +// stdout/stderr. +func (c *Command) Wait(ctx context.Context) ([]string, []string, error) { + c.mu.Lock() + if !c.started { + c.mu.Unlock() + return nil, nil, errors.New("command not started") + } + c.mu.Unlock() + select { + case <-ctx.Done(): + case <-c.Done(): + } + stdout := c.Stdout() + stderr := c.Stderr() + c.mu.Lock() + err := c.waitErr + c.mu.Unlock() + if err != nil { + return stdout, stderr, fmt.Errorf("command failed: %w", err) + } + return stdout, stderr, err +} + +// Run `Start`s and `Wait`s for a command to run to completion. +func (c *Command) Run(ctx context.Context) ([]string, []string, error) { + if err := c.Start(ctx); err != nil { + return nil, nil, err + } + return c.Wait(ctx) +} + +// CombinedOutput runs a command to completion and returns combined +// stdout/stderr output. +func (c *Command) CombinedOutput(ctx context.Context) (string, error) { + if err := c.Start(ctx); err != nil { + return "", err + } + _, _, err := c.Wait(ctx) + return strings.Join(c.Combined(), "\n"), err +} + +// Done returns a channel that is closed when the command terminates. +// Must be called after `Start`. +func (c *Command) Done() <-chan struct{} { + c.mu.Lock() + defer c.mu.Unlock() + if c.doneCh == nil { + panic("Command.Done called before Command.Start") + } + return c.doneCh +} + +// Terminate terminates a process. +// It does not reap the process; the caller should call wait if appropriate. +func Terminate(ctx context.Context, pid int, waitChans ...<-chan struct{}) error { + unifiedWaitChan := make(chan struct{}) + waitShutdown := make(chan struct{}) + defer close(waitShutdown) + for _, waitChan := range waitChans { + go func(waitChan <-chan struct{}) { + select { + case <-waitShutdown: + case <-waitChan: + unifiedWaitChan <- struct{}{} + } + }(waitChan) + } + // Ignore errors here because it doesn't matter; we will re-detect + // the post-signal process state later. + _ = syscall.Kill(pid, syscall.SIGTERM) + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + case <-unifiedWaitChan: + } + if _, err := os.Stat(fmt.Sprintf("/proc/%d", pid)); err != nil && os.IsNotExist(err) { + // The process is gone, so we are successful. + return nil + } + // Otherwise, send SIGKILL. + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + return fmt.Errorf("cannot send SIGKILL: %w", err) + } + return nil +} + +// XServer represents an X server. +type XServer struct { + xvfb *Command +} + +// NewXServer creates a new X server. +func NewXServer(ctx context.Context) (*XServer, error) { + xvfb := &Command{ + Cmd: exec.CommandContext(ctx, "Xvfb", xDisplay, "-screen", "0", "1920x1080x24"), + ForwardStdout: true, + PrefixStdout: "[Xvfb:stdout] ", + ForwardStderr: true, + PrefixStderr: "[Xvfb:stderr] ", + } + if err := xvfb.Start(ctx); err != nil { + return nil, fmt.Errorf("cannot start X server: %w", err) + } + x := &XServer{xvfb: xvfb} + if err := x.Probe(ctx); err != nil { + x.Shutdown(ctx) + return nil, fmt.Errorf("X server did not start in time: %w", err) + } + return x, nil +} + +// Env returns the DISPLAY environment variable to use for this X server. +func (x *XServer) Env() string { + return fmt.Sprintf("DISPLAY=%s", xDisplay) +} + +// Command returns a command that runs in the context of this X server. +func (x *XServer) Command(ctx context.Context, argv ...string) *Command { + cmd := &Command{Cmd: exec.CommandContext(ctx, argv[0], argv[1:]...)} + cmd.Cmd.Env = append(cmd.Cmd.Env, x.Env()) + return cmd +} + +// Probe probes the X server to see if it is alive. +func (x *XServer) Probe(ctx context.Context) error { + probeCtx, probeCancel := context.WithTimeout(ctx, 10*time.Second) + defer probeCancel() + lastErr := ctx.Err() + for probeCtx.Err() == nil { + output, err := x.Command(probeCtx, "xset", "q").CombinedOutput(ctx) + if err == nil { + return nil + } + lastErr = fmt.Errorf("cannot probe X server: %w: %s", err, output) + } + return lastErr +} + +// Shutdown attempts to shut down the X server. +func (x *XServer) Shutdown(ctx context.Context) error { + if err := Terminate(ctx, x.xvfb.Cmd.Process.Pid, x.xvfb.Done()); err != nil { + return fmt.Errorf("cannot shut down Xvfb: %w", err) + } + _, _, _ = x.xvfb.Wait(ctx) // Reap, ignore errors. + return nil +} + +// XWindow represents a window in the X server. +type XWindow struct { + x *XServer + id int64 +} + +// Windows returns a list of X windows. +func (x *XServer) Windows(ctx context.Context) ([]*XWindow, error) { + cmd := x.Command(ctx, "xdotool", "search", "--all", ".*") + stdout, _, err := cmd.Run(ctx) + if err != nil { + return nil, fmt.Errorf("xdotool search failed: %w (output: %v)", err, cmd.Combined()) + } + windows := make([]*XWindow, 0, len(stdout)) + for _, line := range stdout { + line = strings.TrimSpace(line) + if line == "" { + continue + } + windowID, err := strconv.Atoi(line) + if err != nil { + return nil, fmt.Errorf("unexpected xdotool output: %q (whole output: %v)", line, cmd.Combined()) + } + windows = append(windows, &XWindow{x: x, id: int64(windowID)}) + } + return windows, nil +} + +// ID returns a the window ID as a string. +func (w *XWindow) ID() string { + return fmt.Sprintf("%d", w.id) +} + +// String returns a string containing the window ID. +func (w *XWindow) String() string { + return fmt.Sprintf("window:%d", w.id) +} + +// Title returns the window title. +func (w *XWindow) Title(ctx context.Context) (string, error) { + cmd := w.x.Command(ctx, "xdotool", "getwindowname", w.ID()) + stdout, stderr, err := cmd.Wait(ctx) + if err != nil { + return "", w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s title: %w (%q)", w, err, strings.Join(stderr, "\n"))) + } + if len(stdout) != 1 || stdout[0] == "" { + return "", w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s title: unexpected output %q", w, strings.Join(stdout, "\n"))) + } + return stdout[0], nil +} + +// PID returns the PID controlling the window. +// Note that this information is only optionally specified by a process +// creating a window, and is never guaranteed to be there. +func (w *XWindow) PID(ctx context.Context) (int, error) { + cmd := w.x.Command(ctx, "xdotool", "getwindowpid", w.ID()) + stdout, stderr, err := cmd.Wait(ctx) + if err != nil { + return -1, w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s PID: %w (%q)", w, err, strings.Join(stderr, "\n"))) + } + if len(stdout) != 1 || stdout[0] == "" { + return -1, w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s PID: unexpected output %q", w, strings.Join(stdout, "\n"))) + } + pid, err := strconv.Atoi(stdout[0]) + if err != nil { + return -1, w.diagnoseErr(ctx, fmt.Errorf("cannot get window %s PID: invalid PID %q: %w", w, stdout[0], err)) + } + return pid, nil +} + +// Activate activates or focuses the X window. +func (w *XWindow) Activate(ctx context.Context) error { + cmd := w.x.Command(ctx, "xdotool", "windowactivate", "--sync", w.ID()) + if output, err := cmd.CombinedOutput(ctx); err != nil { + return w.diagnoseErr(ctx, fmt.Errorf("xdotool windowactivate: %w (output: %q)", err, output)) + } + return nil +} + +// Keystroke sends a keystroke to the X window. +func (w *XWindow) Keystroke(ctx context.Context, keystrokes ...string) error { + cmd := w.x.Command( + ctx, + append( + []string{ + "xdotool", + "key", + "--clearmodifiers", + "--window", + w.ID(), + }, + keystrokes...)...) + if output, err := cmd.CombinedOutput(ctx); err != nil { + return w.diagnoseErr(ctx, fmt.Errorf("xdotool key: %w (output: %q)", err, output)) + } + return nil +} + +// Screenshot takes a screenshot image of the X window. +func (w *XWindow) Screenshot(ctx context.Context) (image.Image, error) { + screenshotCtx, screenshotCancel := context.WithTimeout(ctx, 10*time.Second) + // Need to use a raw `exec.Command` here because stdout is a byte stream + // as opposed to a text stream. + cmd := exec.CommandContext(screenshotCtx, "import", "-window", w.ID(), "png:-" /* Save to stdout as PNG */) + cmd.Env = append(cmd.Env, w.x.Env()) + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err := cmd.Run() + screenshotCancel() + stderr := string(stderrBuf.Bytes()) + if err != nil { + // Best-effort attempt to kill the process. + _ = Terminate(ctx, cmd.Process.Pid) + return nil, w.diagnoseErr(ctx, fmt.Errorf("imagemagick failed: %w (output: %q)", err, stderr)) + } + img, err := png.Decode(&stdoutBuf) + if err != nil { + return nil, w.diagnoseErr(ctx, fmt.Errorf("cannot decode screenshot image: %w (output: %q)", err, stderr)) + } + if size := img.Bounds().Size(); size.X == 0 || size.Y == 0 { + return nil, w.diagnoseErr(ctx, fmt.Errorf("screenshot image has zero dimension (output: %q)", stderr)) + } + return img, nil +} + +// diagnoseErr annotates an error with additional window information. +func (w *XWindow) diagnoseErr(ctx context.Context, err error) error { + if err == nil { + return nil + } + probeCtx, probeCancel := context.WithTimeout(ctx, 1*time.Second) + defer probeCancel() + if xErr := w.x.Probe(probeCtx); xErr != nil { + return fmt.Errorf("%w (X server is down: %v)", err, xErr) + } + winInfo, infoErr := w.x.Command(ctx, "xwininfo", "-id", w.ID()).CombinedOutput(ctx) + if infoErr != nil { + return fmt.Errorf("%w (cannot get window info: %v - %q)", err, infoErr, winInfo) + } + return fmt.Errorf("%w (window info: %q)", err, winInfo) +} + +// SampleTest represents a single sample test to execute. +type SampleTest struct { + TestName string + XServer *XServer +} + +// NewSampleTest creates a new SampleTest. +func NewSampleTest(testName string, x *XServer) (*SampleTest, error) { + st := &SampleTest{TestName: testName, XServer: x} + if _, err := os.Stat(st.dir()); err != nil { + return nil, fmt.Errorf("invalid test %q: directory %q: %w", st.TestName, st.dir(), err) + } + return st, nil +} + +// dir returns the test directory. +func (st *SampleTest) dir() string { + const samplesRoot = "/cuda-samples/Samples" + return path.Join(samplesRoot, st.TestName) +} + +// cmd returns a `*Command` with proper environment variables and +// working directory for the test. Its output is forwarded to the console. +func (st *SampleTest) cmd(ctx context.Context, argv ...string) *Command { + argv0Base := path.Base(argv[0]) + cmd := st.XServer.Command(ctx, argv...) + cmd.Cmd.Dir = st.dir() + cmd.ForwardStdout = true + cmd.PrefixStdout = fmt.Sprintf("[%s:%s:stdout] ", st.TestName, argv0Base) + cmd.ForwardStderr = true + cmd.PrefixStderr = fmt.Sprintf("[%s:%s:stderr] ", st.TestName, argv0Base) + return cmd +} + +// quietCmd returns a `*Command` with proper environment variables and +// working directory for the test. Its output is not forwarded to the console. +func (st *SampleTest) quietCmd(ctx context.Context, argv ...string) *Command { + cmd := st.cmd(ctx, argv...) + cmd.ForwardStdout = false + cmd.ForwardStderr = false + return cmd +} + +// SampleState captures states that is captured before a test runs, and that +// is useful to refer to while (or after) the test is running. +type SampleState struct { + // When is the timestamp at which this SampleState was taken. + When time.Time + + // Executables holds clean paths of all executable files in the test dir. + Executables map[string]struct{} + + // Windows is a list of window screenshots in the X server, mapped by ID. + Windows map[string]*XWindow + + // Screenshots is a list of screenshots mapped by window ID. + // If a screenshot fails, the window is mapped to `nil`. + Screenshots map[string]image.Image +} + +// NewExecutables returns the executables in `after` that are not in `ss`. +func (ss *SampleState) NewExecutables(after *SampleState) []string { + newExecutables := make([]string, 0, len(after.Executables)) + for e := range after.Executables { + if _, found := ss.Executables[e]; !found { + newExecutables = append(newExecutables, e) + } + } + return newExecutables +} + +// DifferentWindows returns the windows in `after` that are new or for which +// the screenshot has changed. +func (ss *SampleState) DifferentWindows(after *SampleState) []*XWindow { + diffWindows := make([]*XWindow, 0, len(after.Windows)) + for id, window := range after.Windows { + if _, found := ss.Windows[id]; !found { + diffWindows = append(diffWindows, window) + continue + } + if !imgEq(ss.Screenshots[id], after.Screenshots[id]) { + diffWindows = append(diffWindows, window) + } + } + return diffWindows +} + +// imgEq returns true if the two given images are identical in size and pixel +// values. +func imgEq(a, b image.Image) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + bounds := a.Bounds() + if bounds != b.Bounds() { + return false + } + // Convert images to RGBA so that we can compare raw pixel data directly. + imgA := image.NewRGBA(bounds) + draw.Draw(imgA, bounds, a, image.Point{0, 0}, draw.Src) + imgB := image.NewRGBA(bounds) + draw.Draw(imgB, bounds, b, image.Point{0, 0}, draw.Src) + if imgA.Stride != imgB.Stride || imgA.Rect != imgB.Rect || len(imgA.Pix) != len(imgB.Pix) { + return false + } + for i := 0; i < len(imgA.Pix); i++ { + if imgA.Pix[i] != imgB.Pix[i] { + return false + } + } + return true +} + +// logImageWithPrefix renders an image to text, frames it with the given +// title, and logs that with a given prefix. +func logImageWithFrameAndPrefix(ctx context.Context, img image.Image, title, prefix string) error { + const imageWidth = 72 + var pngBytes bytes.Buffer + if err := png.Encode(&pngBytes, img); err != nil { + return fmt.Errorf("png encoding failed: %v", err) + } + stdout, stderr, err := (&Command{ + Cmd: exec.CommandContext(ctx, "ascii-image-converter", "/dev/stdin", fmt.Sprintf("--width=%d", imageWidth), "--braille", "--dither"), + Stdin: pngBytes.Bytes(), + }).Run(ctx) + if err != nil { + return fmt.Errorf("ascii-image-converter failed: %v (output: %q)", err, strings.Join(stderr, "\n")) + } + header := "┍" + footer := "╰" + numHeaderHorizontalLines := imageWidth - len(title) - 2 + leftHeaderHorizontalLines := numHeaderHorizontalLines / 2 + rightHeaderHorizontalLines := numHeaderHorizontalLines - leftHeaderHorizontalLines + for i := 0; i < leftHeaderHorizontalLines; i++ { + header += "━" + } + header += fmt.Sprintf(" %s ", title) + for i := 0; i < rightHeaderHorizontalLines; i++ { + header += "━" + } + for i := 0; i < imageWidth; i++ { + footer += "─" + } + header += "┑" + footer += "╯" + logDo(func() { + fmt.Fprintf(os.Stderr, "%s%s\n", prefix, header) + for _, line := range stdout { + fmt.Fprintf(os.Stderr, "%s|%s|\n", prefix, line) + } + fmt.Fprintf(os.Stderr, "%s%s\n", prefix, footer) + }) + return nil +} + +// State returns the current state of the test. +func (st *SampleTest) State(ctx context.Context) (*SampleState, error) { + when := time.Now() + executables := make(map[string]struct{}) + err := filepath.Walk(st.dir(), func(path string, info fs.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("cannot walk %q (%q): %w", st.dir(), path, err) + } + if !info.IsDir() && info.Mode()&0111 != 0 { + executables[path] = struct{}{} + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("cannot list executables: %w", err) + } + windows, err := st.XServer.Windows(ctx) + if err != nil { + return nil, fmt.Errorf("cannot list windows: %w", err) + } + windowMap := make(map[string]*XWindow, len(windows)) + screenshots := make(map[string]image.Image, len(windows)) + for _, w := range windows { + windowMap[w.ID()] = w + if screenshot, err := w.Screenshot(ctx); err == nil { + screenshots[w.ID()] = screenshot + } + } + return &SampleState{ + When: when, + Executables: executables, + Windows: windowMap, + Screenshots: screenshots, + }, nil +} + +// makeRun runs `make run` or `make testrun` in the test directory. +func (st *SampleTest) makeRun(ctx context.Context) (*Command, error) { + arch, err := st.quietCmd(ctx, "uname", "-m").CombinedOutput(ctx) + if err != nil || arch == "" { + return nil, fmt.Errorf("cannot get architecture (%q): %w", arch, err) + } + + // All samples have a "testrun" make target. However, most of them have it + // set to do literally nothing. + // All samples also have a "run" make target. Unlike the "testrun" target, + // "run" always does something. + // However, when "testrun" actually does something, it is usually for the + // explicit purpose of running a test. + // For example, `0_Introduction/simpleTexture3D` has a `testrun` target that + // runs the file with an example texture file, whereas the `run` target + // opens a file passed as argument, which does not exist here. + // So we must detect the case where "testrun" does something useful vs the + // case where it does not. + // To do this, we parse the Makefile a bit to see if the `testrun` target + // contains any actual commands, as opposed to only containing build + // dependencies. + makefilePath := path.Join(st.dir(), "Makefile") + makefile, err := os.Open(makefilePath) + if err != nil { + return nil, fmt.Errorf("cannot open %q: %w", makefilePath, err) + } + defer makefile.Close() + testRunTargetHasCommands := false + for scanner := bufio.NewScanner(makefile); scanner.Scan(); { + line := scanner.Text() + if !strings.HasPrefix(line, "testrun:") { + continue + } + if !scanner.Scan() { + break + } + nextLine := scanner.Text() + if strings.HasPrefix(nextLine, "\t") && strings.TrimSpace(nextLine) != "" { + testRunTargetHasCommands = true + } + break + } + argv := []string{"make", "-C", st.dir(), fmt.Sprintf("TARGET_ARCH=%s", arch)} + if testRunTargetHasCommands { + argv = append(argv, "testrun") + } else { + argv = append(argv, "run") + } + log("[%s] Executing: %v", st.TestName, strings.Join(argv, " ")) + cmd := st.cmd(ctx, argv...) + if err := cmd.Start(ctx); err != nil { + return nil, fmt.Errorf("cannot start `make`: %w", err) + } + return cmd, nil +} + +// Run runs a single sample test. +func (st *SampleTest) Run(ctx context.Context) error { + const libNVVMTestDir = "7_libNVVM/" + if strings.HasPrefix(st.TestName, libNVVMTestDir) { + return st.RunLibNVVMTest(ctx) + } + if _, _, err := st.cmd(ctx, "make", "-C", st.dir(), "clean").Run(ctx); err != nil { + return fmt.Errorf("cannot run `make clean`: %w", err) + } + stateBefore, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get state before test: %w", err) + } + makeRun, err := st.makeRun(ctx) + if err != nil { + return fmt.Errorf("cannot run `make run`: %w", err) + } + defer Terminate(ctx, makeRun.PID()) + // There are multiple possibilities here. + // Some CUDA programs will run an X application that runs forever. + // In this case, we need to detect this and to make sure it runs, + // then kill it. + // Other programs are just command-line based and run to completion, + // and we rely on their exit code. + // To determine this, we first just wait for a few seconds and see what + // the command does. + if err := st.Monitor(ctx, makeRun, stateBefore); err != nil { + return fmt.Errorf("test failed in `make run`: %w", err) + } + + // Some `make` targets will silently exist with code 0 even if the test + // was actually unsuccessful because it cannot be built. + // To detect this case, we look for the absence of any executable file in + // the sample directory. All `make` targets should create an executable, and + // this won't happen if `make` bails out. + stateAfter, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get state after test: %w", err) + } + if len(stateBefore.NewExecutables(stateAfter)) == 0 { + return fmt.Errorf("did not find any new executable file created by `make run` in the test directory %q (existing executables: %v)", st.dir(), stateBefore.Executables) + } + return nil +} + +// Monitor monitors whether a `make run` command terminates quickly or +// produces an X window. +func (st *SampleTest) Monitor(ctx context.Context, makeRun *Command, stateBefore *SampleState) error { + fastTicker := time.NewTicker(200 * time.Millisecond) + defer fastTicker.Stop() + var currentState *SampleState + for windowsChanged := false; !windowsChanged; { + select { + case <-ctx.Done(): // Context expired. + return ctx.Err() + case <-makeRun.Done(): // `make run` finished on its own. + _, _, err := makeRun.Wait(ctx) + return err + case <-fastTicker.C: + // Check for new windows. + var err error + currentState, err = st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get test state: %w", err) + } + windowsChanged = len(stateBefore.DifferentWindows(currentState)) > 0 + } + } + + // If we get here, the test produces X windows. So we need to monitor them. + // We will consider the test a success in any of the following cases: + // - The `make run` process exits at any time with a 0 exit code. + // - The set of windows stops changing for 3 consecutive seconds, i.e. + // the test has reached a stable steady state without crashing. + // - The set of windows continuously changes for 10 consecutive seconds, + // i.e. the test is likely a visually-changing demo over time and has + // reached a steady state without crashing. + log("[%s] This appears to be a test that uses graphics and X windows.", st.TestName) + lastState := stateBefore + slowTicker := time.NewTicker(1 * time.Second) + defer slowTicker.Stop() + lastWindowChange := currentState.When + successDeadline := time.After(10 * time.Second) + for { + select { + case <-ctx.Done(): // Context expired. + return ctx.Err() + case <-makeRun.Done(): // `make run` finished on its own. + _, _, err := makeRun.Wait(ctx) + return err + case <-successDeadline: // Still no crashes after long enough. + return st.TerminateWindowTest(ctx, makeRun, stateBefore) + case <-slowTicker.C: + stateNow, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get test state: %w", err) + } + if differentWindows := lastState.DifferentWindows(stateNow); len(differentWindows) > 0 { + lastWindowChange = stateNow.When + log("[%s] [%s] Windows changed:", st.TestName, stateNow.When.Format("15:04:05")) + for _, window := range differentWindows { + title, err := window.Title(ctx) + if err != nil { + title = window.String() + } + if screenshot := stateNow.Screenshots[window.ID()]; screenshot == nil { + log("[%s:%s] ", st.TestName, title) + } else if err := logImageWithFrameAndPrefix(ctx, screenshot, title, fmt.Sprintf("[%s] ", st.TestName)); err != nil { + log("[%s:%s] ", st.TestName, title, err) + } + } + } + if currentState.When.Sub(lastWindowChange) >= 3*time.Second { + return st.TerminateWindowTest(ctx, makeRun, stateBefore) + } + lastState = stateNow + } + } +} + +// TerminateWindowTest terminates a sample test that produces X windows. +func (st *SampleTest) TerminateWindowTest(ctx context.Context, makeRun *Command, stateBefore *SampleState) error { + stateNow, err := st.State(ctx) + if err != nil { + return fmt.Errorf("cannot get test state: %w", err) + } + testWindows := stateBefore.DifferentWindows(stateNow) + // Most windows-based tests accept typing the letter "Q" to quit them. + // Try it first. + for _, window := range testWindows { + // Ignore error for both activation and keystrokes; this is just a + // best-effort attempt to press "Q". + _ = window.Activate(ctx) + _ = window.Keystroke(ctx, "q") + } + // Now wait a little bit to see if the program ends on its own from that. + select { + case <-ctx.Done(): + return ctx.Err() + case <-makeRun.Done(): + _, _, err = makeRun.Wait(ctx) + return err + case <-time.After(3 * time.Second): + // Didn't work, keep going. + } + // Gather a list of test PIDs. + windowPIDs := make(map[int]struct{}) + for _, window := range testWindows { + pid, err := window.PID(ctx) + if err != nil { + // X window PID information is optional; erroring out here is not + // appropriate. + continue + } + if pid == makeRun.PID() { + continue + } + windowPIDs[pid] = struct{}{} + } + if len(windowPIDs) > 0 { + // Kill all the PIDs we gathered. + for pid := range windowPIDs { + _ = Terminate(ctx, pid, makeRun.Done()) + } + // Now check if `make run` terminates on its own. + select { + case <-ctx.Done(): + return ctx.Err() + case <-makeRun.Done(): + _, _, err = makeRun.Wait(ctx) + return err + case <-time.After(3 * time.Second): + // Didn't work, keep going. + } + } + return errors.New("test did not terminate") +} + +// RunLibNVVMTest runs a `libnvvm`-based test. +// These tests are located in the `7_libNVVM/` directory. +func (st *SampleTest) RunLibNVVMTest(ctx context.Context) error { + const ptxgenTestName = "ptxgen" + + // Need to run `cmake` in the 7_libNVVM/ directory to build the test. + libNVVMTestsDir := path.Dir(st.dir()) + libNVVMTestName := path.Base(st.dir()) + cmake := st.cmd(ctx, "cmake", ".") + cmake.Cmd.Dir = libNVVMTestsDir + if _, _, err := cmake.Run(ctx); err != nil { + return fmt.Errorf("cannot run `cmake`: %w", err) + } + // Then run `make` in the test directory. + // CMake generates a make file in the parent directory. + // We `make all` rather than just the test target, because + // `cuda-c-linking` depends on the `mathfuncs` target despite not being + // declared as such in the Makefile. + arch, err := st.quietCmd(ctx, "uname", "-m").CombinedOutput(ctx) + if err != nil || arch == "" { + return fmt.Errorf("cannot get architecture (%q): %w", arch, err) + } + makeCmd := st.cmd(ctx, "make", "-C", libNVVMTestsDir, fmt.Sprintf("TARGET_ARCH=%s", arch), "all") + if _, _, err := makeCmd.Run(ctx); err != nil { + return fmt.Errorf("cannot run `make`: %w", err) + } + // `make` will create an executable in the test directory that has the same + // name as the directory does. + exePath := path.Join(st.dir(), libNVVMTestName) + if _, err := os.Stat(exePath); err != nil { + return fmt.Errorf("cannot stat executable at expected location %q: %w", exePath, err) + } + argv := []string{exePath} + if libNVVMTestName == ptxgenTestName { + // The ptxgen test binary needs a .ll file as input. + // Conveniently, there is one called "test.ll" in the test directory. + argv = append(argv, path.Join(st.dir(), "test.ll")) + } + if _, _, err := st.cmd(ctx, argv...).Run(ctx); err != nil { + return fmt.Errorf("test binary failed: %w", err) + } + return nil +} + +// Main is the main method of this program. +func Main(ctx context.Context) (int, error) { + flag.Parse() + cleanupCtx, cleanupCancel := context.WithTimeout(ctx, *timeoutFlag) + defer cleanupCancel() + deadline, _ := cleanupCtx.Deadline() + x, err := NewXServer(cleanupCtx) + if err != nil { + return 1, fmt.Errorf("failed to start X server: %s", err) + } + defer x.Shutdown(cleanupCtx) + testsCtx, testsCancel := context.WithDeadline(cleanupCtx, deadline.Add(-10*time.Second)) + defer testsCancel() + failed := false + numTests := 0 + exitCode := 1 + for _, testName := range flag.Args() { + numTests++ + st, err := NewSampleTest(testName, x) + if err != nil { + log("> Invalid test %q: %s", testName, err) + failed = true + continue + } + log("> Running test: %s", testName) + testCtx, testCancel := context.WithCancel(testsCtx) + err = st.Run(testCtx) + testCancel() + if err != nil { + log("> Test failed: %s (%s)", testName, err) + failed = true + if exitErr := (*exec.ExitError)(nil); errors.As(err, &exitErr) && exitErr.ExitCode() > 0 { + exitCode = exitErr.ExitCode() + } + continue + } + log("> Test passed: %s", testName) + } + if numTests == 0 { + return 1, fmt.Errorf("no tests to run, failing vacuously; specify test names as positional arguments") + } + if failed { + if numTests == 1 { + // If there was a single test to run, pass along its error code. + return exitCode, fmt.Errorf("test failed") + } + return 1, errors.New("one or more tests failed") + } + return 0, nil +} + +func main() { + exitCode, err := Main(context.Background()) + if err != nil { + log("%s", err) + log("FAIL") + } else { + log("PASS") + } + os.Exit(exitCode) +} diff --git a/images/gpu/cuda-tests/run.sh b/images/gpu/cuda-tests/run_smoke.sh similarity index 100% rename from images/gpu/cuda-tests/run.sh rename to images/gpu/cuda-tests/run_smoke.sh diff --git a/pkg/test/dockerutil/gpu.go b/pkg/test/dockerutil/gpu.go index f7370a0cf..eb86ecd46 100644 --- a/pkg/test/dockerutil/gpu.go +++ b/pkg/test/dockerutil/gpu.go @@ -29,14 +29,19 @@ var ( setCOSGPU = flag.Bool("cos-gpu", false, "set to configure GPU settings for COS, as opposed to Docker") ) +// AllGPUCapabilities is the environment variable that enables all NVIDIA GPU +// capabilities within a container. +const AllGPUCapabilities = "NVIDIA_DRIVER_CAPABILITIES=all" + // GPURunOpts returns Docker run options with GPU support enabled. func GPURunOpts() RunOpts { if !*setCOSGPU { return RunOpts{ + Env: []string{AllGPUCapabilities}, DeviceRequests: []container.DeviceRequest{ { Count: -1, - Capabilities: [][]string{[]string{"gpu"}}, + Capabilities: [][]string{{"gpu"}}, Options: map[string]string{}, }, }, @@ -92,6 +97,7 @@ func GPURunOpts() RunOpts { } return RunOpts{ + Env: []string{AllGPUCapabilities}, Mounts: mounts, Devices: devices, } diff --git a/test/gpu/BUILD b/test/gpu/BUILD index 747a23e7f..de5c4c1b8 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -62,6 +62,23 @@ go_test( ], ) +go_test( + name = "cuda_test", + timeout = "eternal", # YES_I_REALLY_NEED_AN_ETERNAL_TEST + srcs = ["cuda_test.go"], + tags = [ + "local", + "noguitar", + "notap", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//pkg/test/testutil", + "@org_golang_x_sync//errgroup:go_default_library", + ], +) + go_test( name = "imagegen_test", srcs = ["imagegen_test.go"], diff --git a/test/gpu/cuda_test.go b/test/gpu/cuda_test.go new file mode 100644 index 000000000..11823b4f3 --- /dev/null +++ b/test/gpu/cuda_test.go @@ -0,0 +1,823 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cuda_test tests basic CUDA workloads. +package cuda_test + +import ( + "context" + "errors" + "flag" + "fmt" + "math" + "os" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/sync/errgroup" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +const ( + // defaultTestTimeout is the default timeout for a single CUDA sample test. + defaultTestTimeout = 20 * time.Minute + + // hangingTestTimeout is the test timeout for tests that are fast when they + // succeed, but hang forever otherwise. + hangingTestTimeout = 1 * time.Minute + + // defaultContainersPerCPU is the default number of pooled containers to + // spawn for each CPU. This can be a floating-point value. + // This value was arrived at experimentally and has no particular meaning. + // Setting it too low will cause the test to take longer than necessary + // because of insufficient parallelism. + // However, setting it too high will *also* cause the test to take longer + // than necessary, because the added resource contention will cause more + // tests to fail when run in parallel with each other, forcing them to be + // re-run serialized. + defaultContainersPerCPU = 1.75 + + // exitCodeWaived is the EXIT_WAIVED constant used in CUDA tests. + // This exit code is typically used by CUDA tests to indicate that the + // test requires a capability or condition that is not met in the current + // test environment. + exitCodeWaived = 2 +) + +// Flags. +var ( + verifyCompatibility = flag.Bool("cuda_verify_compatibility", os.Getenv("GVISOR_TEST_CUDA_VERIFY_COMPATIBILITY") == "true", "whether to verify that all tests are marked as compatible") + logSuccessfulTests = flag.Bool("cuda_log_successful_tests", false, "log console output of successful tests") + debug = flag.Bool("cuda_test_debug", false, "log more data as the test is running") + containersPerCPU = flag.Float64("cuda_containers_per_cpu", defaultContainersPerCPU, "number of parallel execution containers to spawn per CPU (floating point values allowed)") +) + +// testCompatibility maps test names to their compatibility data. +// Unmapped test names are assumed to be fully compatible. +var testCompatibility = map[string]Compatibility{ + "0_Introduction/simpleAttributes": RequiresFeatures(FeaturePersistentL2Caching), + "0_Introduction/simpleCUDA2GL": RequiresFeatures(FeatureGL), + "0_Introduction/simpleIPC": &BrokenInGVisor{OnlyWhenMultipleGPU: true}, + "0_Introduction/simpleP2P": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "0_Introduction/UnifiedMemoryStreams": &BrokenInGVisor{}, + "0_Introduction/vectorAddMMAP": &BrokenInGVisor{OnlyWhenMultipleGPU: true}, + "2_Concepts_and_Techniques/cuHook": &BrokenEverywhere{ + Reason: "Requires ancient version of glibc (<=2.33)", + }, + "2_Concepts_and_Techniques/EGLStream_CUDA_Interop": &BrokenEverywhere{ + Reason: "Requires newer version of EGL libraries than Ubuntu has (eglCreateStreamKHR)", + }, + "2_Concepts_and_Techniques/EGLStream_CUDA_CrossGPU": MultiCompatibility( + &RequiresMultiGPU{}, + &BrokenEverywhere{ + Reason: "Requires newer version of EGL libraries than Ubuntu has (eglCreateStreamKHR)", + }, + ), + "2_Concepts_and_Techniques/EGLSync_CUDAEvent_Interop": &OnlyOnWindows{}, + "2_Concepts_and_Techniques/streamOrderedAllocationIPC": &BrokenInGVisor{}, + "2_Concepts_and_Techniques/streamOrderedAllocationP2P": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "3_CUDA_Features/bf16TensorCoreGemm": RequiresFeatures(FeatureTensorCores), + "3_CUDA_Features/cdpAdvancedQuicksort": RequiresFeatures(FeatureDynamicParallelism), + "3_CUDA_Features/cudaCompressibleMemory": RequiresFeatures(FeatureCompressibleMemory), + "3_CUDA_Features/dmmaTensorCoreGemm": RequiresFeatures(FeatureTensorCores), + "3_CUDA_Features/memMapIPCDrv": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "3_CUDA_Features/tf32TensorCoreGemm": RequiresFeatures(FeatureTensorCores), + "4_CUDA_Libraries/conjugateGradientMultiDeviceCG": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "4_CUDA_Libraries/cudaNvSci": &RequiresNvSci{}, + "4_CUDA_Libraries/cudaNvSciNvMedia": &RequiresNvSci{}, + "4_CUDA_Libraries/cuDLAErrorReporting": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLAHybridMode": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLAStandaloneMode": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLALayerwiseStatsHybrid": &OnlyOnWindows{}, + "4_CUDA_Libraries/cuDLALayerwiseStatsStandalone": &OnlyOnWindows{}, + "4_CUDA_Libraries/simpleCUFFT_2d_MGPU": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "4_CUDA_Libraries/simpleCUFFT_MGPU": MultiCompatibility(&RequiresMultiGPU{}, &BrokenInGVisor{}), + "5_Domain_Specific/fluidsD3D9": &OnlyOnWindows{}, + "5_Domain_Specific/fluidsGL": RequiresFeatures(FeatureGL), + "5_Domain_Specific/fluidsGLES": &OnlyOnWindows{}, + "5_Domain_Specific/nbody_opengles": &OnlyOnWindows{}, + "5_Domain_Specific/nbody_screen": &OnlyOnWindows{}, + "5_Domain_Specific/p2pBandwidthLatencyTest": &BrokenInGVisor{OnlyWhenMultipleGPU: true}, + "5_Domain_Specific/postProcessGL": RequiresFeatures(FeatureGL), + "5_Domain_Specific/simpleD3D10": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D10RenderTarget": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D10Texture": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D11": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D11Texture": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D12": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D9": &OnlyOnWindows{}, + "5_Domain_Specific/simpleD3D9Texture": &OnlyOnWindows{}, + "5_Domain_Specific/simpleGLES": &OnlyOnWindows{}, + "5_Domain_Specific/simpleGLES_EGLOutput": &OnlyOnWindows{}, + "5_Domain_Specific/simpleGLES_screen": &OnlyOnWindows{}, + "5_Domain_Specific/simpleVulkan": RequiresFeatures(FeatureGL), + "5_Domain_Specific/simpleVulkanMMAP": RequiresFeatures(FeatureGL), + "5_Domain_Specific/SLID3D10Texture": &OnlyOnWindows{}, + "5_Domain_Specific/VFlockingD3D10": &OnlyOnWindows{}, + "5_Domain_Specific/vulkanImageCUDA": RequiresFeatures(FeatureGL), +} + +// flakyTests is a list of tests that are flaky. +// These will be retried up to 3 times in parallel before running serially. +var flakyTests = map[string]struct{}{} + +// exclusiveTests is a list of tests that must run exclusively (i.e. with +// no other test running on the machine at the same time), or they will +// likely fail. These tests are not attempted to be run in parallel. +// This is usually the case for performance tests or tests that use a lot +// of resources in general. +// This saves the trouble to run them in parallel, while also avoiding +// causing spurious failures for the tests that happen to be running in +// parallel with them. +var exclusiveTests = map[string]struct{}{ + "6_Performance/alignedTypes": {}, + "6_Performance/transpose": {}, + "6_Performance/UnifiedMemoryPerf": {}, +} + +// alwaysSkippedTests don't run at all, ever, and are not verified when +// --cuda_verify_compatibility is set. +// Each test is mapped to a reason why it should be skipped. +var alwaysSkippedTests = map[string]string{ + // These tests seem to flake in gVisor, but consistently within the same + // run of the overall test, so they cannot be included in `flakyTests`. + "0_Introduction/simpleAssert": "Flaky in gVisor", + "0_Introduction/simpleAssert_nvrtc": "Flaky in gVisor", +} + +// Feature is a feature as listed by /list_features.sh. +type Feature string + +// All CUDA features listed by /list_features.sh. +const ( + FeaturePersistentL2Caching Feature = "PERSISTENT_L2_CACHING" + FeatureDynamicParallelism Feature = "DYNAMIC_PARALLELISM" + FeatureGL Feature = "GL" + FeatureTensorCores Feature = "TENSOR_CORES" + FeatureCompressibleMemory Feature = "COMPRESSIBLE_MEMORY" +) + +// allFeatures is a list of all CUDA features above. +var allFeatures = []Feature{ + FeaturePersistentL2Caching, + FeatureDynamicParallelism, + FeatureGL, + FeatureTensorCores, + FeatureCompressibleMemory, +} + +// TestEnvironment represents the environment in which a sample test runs. +type TestEnvironment struct { + NumGPUs int + RuntimeIsGVisor bool + Features map[Feature]bool +} + +// Compatibility encodes the compatibility of a test depending on the +// environment it runs in. +type Compatibility interface { + // WillFail returns a string explaining why the test is expected to fail + // in the given environment, or "" if it isn't expected to fail. + WillFail(ctx context.Context, env *TestEnvironment) string + + // IsExpectedFailure checks whether the `logs` (from a failed run of the test + // in the given environment) matches the failure that this test expects in + // that environment. If they match, this function should return nil. + // It is only called when `WillFail` returns a non-empty string for the same + // environment, so it may assume that `env` is non-compatible. + IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error +} + +// BrokenEverywhere implements `Compatibility` for tests that are broken in +// all environments. +type BrokenEverywhere struct { + Reason string +} + +// WillFail implements `Compatibility.WillFail`. +func (be *BrokenEverywhere) WillFail(ctx context.Context, env *TestEnvironment) string { + return fmt.Sprintf("Known-broken test: %v", be.Reason) +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*BrokenEverywhere) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return nil +} + +// BrokenInGVisor implements `Compatibility` for tests that are broken in +// gVisor only. +type BrokenInGVisor struct { + // OnlyWhenMultipleGPU may be set to true for tests which only fail when + // multiple GPUs are present. This should not be used for tests that + // *require* multiple GPUs to run (use RequiresMultiGPU instead). + // This is for tests that can run on a single or multiple GPUs alike, + // but specifically fail in gVisor when run with multiple GPUs. + OnlyWhenMultipleGPU bool + + // KnownToHang may be set to true for short tests which can hang instead + // of failing. This avoids waiting ~forever for them to finish. + KnownToHang bool +} + +// WillFail implements `Compatibility.WillFail`. +func (big *BrokenInGVisor) WillFail(ctx context.Context, env *TestEnvironment) string { + if !env.RuntimeIsGVisor { + return "" + } + if big.OnlyWhenMultipleGPU && env.NumGPUs == 1 { + return "" + } + if big.OnlyWhenMultipleGPU { + return "Known to be broken in gVisor when multiple GPUs are present" + } + return "Known to be broken in gVisor" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*BrokenInGVisor) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return nil +} + +// RequiresMultiGPU implements `Compatibility` for tests that require multiple +// GPUs. +type RequiresMultiGPU struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*RequiresMultiGPU) WillFail(ctx context.Context, env *TestEnvironment) string { + if env.NumGPUs < 2 { + return "Requires >= 2 GPUs" + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*RequiresMultiGPU) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + if exitCode != exitCodeWaived { + return fmt.Errorf("exit code %d, expected EXIT_WAIVED (%d)", exitCode, exitCodeWaived) + } + return nil +} + +// requiresFeatures implements `Compatibility` for tests that require +// specific features. +type requiresFeatures struct { + features []Feature +} + +func RequiresFeatures(features ...Feature) Compatibility { + return &requiresFeatures{features: features} +} + +// WillFail implements `Compatibility.WillFail`. +func (r *requiresFeatures) WillFail(ctx context.Context, env *TestEnvironment) string { + for _, feature := range r.features { + if !env.Features[feature] { + return fmt.Sprintf("Requires feature %s", feature) + } + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*requiresFeatures) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + if exitCode != exitCodeWaived { + return fmt.Errorf("exit code %d, expected EXIT_WAIVED (%d)", exitCode, exitCodeWaived) + } + return nil +} + +// OnlyOnWindows implements `Compatibility` for tests that are only expected +// to only pass on Windows. +type OnlyOnWindows struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*OnlyOnWindows) WillFail(ctx context.Context, env *TestEnvironment) string { + if runtime.GOOS != "windows" { + return "Only runs on Windows" + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*OnlyOnWindows) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + if strings.Contains(logs, "is not supported on Linux") { + return nil + } + if exitCode != exitCodeWaived { + return fmt.Errorf("exit code %d, expected EXIT_WAIVED (%d)", exitCode, exitCodeWaived) + } + return nil +} + +type RequiresNvSci struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*RequiresNvSci) WillFail(ctx context.Context, env *TestEnvironment) string { + return "Requires NvSci library which is not open-source" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*RequiresNvSci) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return nil +} + +// multiCompatibility implements `Compatibility` with multiple possible +// Compatibility implementations. +type multiCompatibility struct { + compats []Compatibility +} + +// MultiCompatibility implements `Compatibility` with multiple possible +// Compatibility implementations. +func MultiCompatibility(compats ...Compatibility) Compatibility { + return &multiCompatibility{compats: compats} +} + +// WillFail implements `Compatibility.WillFail`. +func (mc *multiCompatibility) WillFail(ctx context.Context, env *TestEnvironment) string { + for _, compat := range mc.compats { + if reason := compat.WillFail(ctx, env); reason != "" { + return reason + } + } + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (mc *multiCompatibility) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + var possibleCompats []Compatibility + for _, compat := range mc.compats { + if reason := compat.WillFail(ctx, env); reason != "" { + possibleCompats = append(possibleCompats, compat) + } + } + if len(possibleCompats) == 0 { + return errors.New("no known explanation for this failure") + } + var errs []string + for _, compat := range possibleCompats { + err := compat.IsExpectedFailure(ctx, env, logs, exitCode) + if err == nil { + return nil + } + errs = append(errs, fmt.Sprintf("might have been broken because %s but %v", compat.WillFail(ctx, env), err)) + } + return fmt.Errorf("no known explanation for this failure: %v", strings.Join(errs, "; ")) +} + +// FullyCompatible implements `Compatibility` for tests that are expected to +// pass in any environment. +type FullyCompatible struct{} + +// WillFail implements `Compatibility.WillFail`. +func (*FullyCompatible) WillFail(ctx context.Context, env *TestEnvironment) string { + return "" +} + +// IsExpectedFailure implements `Compatibility.IsExpectedFailure`. +func (*FullyCompatible) IsExpectedFailure(ctx context.Context, env *TestEnvironment, logs string, exitCode int) error { + return errors.New("test is expected to pass regardless of environment") +} + +// getContainerOpts returns the container run options to run CUDA tests. +func getContainerOpts() dockerutil.RunOpts { + opts := dockerutil.GPURunOpts() + opts.Image = "gpu/cuda-tests" + return opts +} + +// testLog logs a line as a test log. +// If debug is enabled, it is also printed immediately to stderr. +// This is useful for debugging tests. +func testLog(t *testing.T, format string, values ...any) { + t.Helper() + if *debug { + fmt.Fprintf(os.Stderr, "[%s] %s\n", t.Name(), fmt.Sprintf(format, values...)) + } + t.Logf(format, values...) +} + +// multiLineLog logs a multiline string as separate log messages to `t`. +// This is useful to log multi-line container logs without them looking weird +// with line breaks in the middle. +func multiLineLog(t *testing.T, output string) { + t.Helper() + for _, line := range strings.Split(output, "\n") { + // `line` may contain % characters here, so we need to format it through + // `%s` so that `%` characters don't show up as "MISSING" in the logs. + testLog(t, "%s", line) + } +} + +// GetEnvironment returns the environment in which a sample test runs. +func GetEnvironment(ctx context.Context, t *testing.T) (*TestEnvironment, error) { + numGPU := dockerutil.NumGPU() + if numGPU == 0 { + return nil, errors.New("no GPUs detected") + } + if numGPU == 1 { + testLog(t, "1 GPU detected") + } else { + testLog(t, "%d GPUs detected", numGPU) + } + runtimeIsGVisor, err := dockerutil.IsGVisorRuntime(ctx, t) + if err != nil { + return nil, fmt.Errorf("cannot determine if runtime is gVisor or not: %w", err) + } + if runtimeIsGVisor { + testLog(t, "Runtime is detected as gVisor") + } else { + testLog(t, "Runtime is detected as not gVisor") + } + featuresContainer := dockerutil.MakeContainer(ctx, t) + defer featuresContainer.CleanUp(ctx) + featuresList, err := featuresContainer.Run(ctx, getContainerOpts(), "/list_features.sh") + if err != nil { + return nil, fmt.Errorf("cannot get list of CUDA features: %v", err) + } + features := make(map[Feature]bool) + for _, line := range strings.Split(featuresList, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + featureAvailable := false + var feature Feature + if strings.HasPrefix(line, "PRESENT: ") { + featureAvailable = true + feature = Feature(strings.TrimPrefix(line, "PRESENT: ")) + } else if strings.HasPrefix(line, "ABSENT: ") { + featureAvailable = false + feature = Feature(strings.TrimPrefix(line, "ABSENT: ")) + } else { + return nil, fmt.Errorf("unexpected CUDA feature line: %q", line) + } + found := false + for _, f := range allFeatures { + if feature == f { + features[f] = featureAvailable + if featureAvailable { + testLog(t, "CUDA feature is available: %s", string(f)) + } else { + testLog(t, "CUDA feature is *not* available: %s", string(f)) + } + found = true + break + } + } + if !found { + return nil, fmt.Errorf("unknown CUDA feature: %s", string(feature)) + } + } + for _, feature := range allFeatures { + if _, ok := features[feature]; !ok { + return nil, fmt.Errorf("CUDA feature not found in feature list: %s", string(feature)) + } + } + // Use CUDA dynamic parallelism as a litmus test to see if the features were + // enumerated correctly. + if _, hasDynamicParallelism := features[FeatureDynamicParallelism]; !hasDynamicParallelism { + return nil, errors.New("CUDA feature Dynamic Parallelism is not available yet should be available in all environments gVisor supports; this indicates a failure in the feature listing script") + } + return &TestEnvironment{ + NumGPUs: numGPU, + RuntimeIsGVisor: runtimeIsGVisor, + Features: features, + }, nil +} + +// runSampleTest runs a single CUDA sample test. +// It first tries to run in pooled container. +// If that fails, then it runs in an exclusive container. +// It returns a skip reason (or empty if the test was not skipped), and +// an error if the test fails. +func runSampleTest(ctx context.Context, t *testing.T, testName string, te *TestEnvironment, cp *dockerutil.ContainerPool) (string, error) { + compat, found := testCompatibility[testName] + if !found { + compat = &FullyCompatible{} + } + willFailReason := compat.WillFail(ctx, te) + if willFailReason != "" && !*verifyCompatibility { + return fmt.Sprintf("this test is expected to fail (%s) --cuda_verify_compatibility=true to verify compatibility)", willFailReason), nil + } + if skipReason, isAlwaysSkipped := alwaysSkippedTests[testName]; isAlwaysSkipped { + return fmt.Sprintf("this test is always skipped (%v)", skipReason), nil + } + testTimeout := defaultTestTimeout + execTestTimeout := testTimeout - 15*time.Second + testAttempts := 1 + if _, isFlakyTest := flakyTests[testName]; isFlakyTest { + testAttempts = 3 + } + parallelAttempts := testAttempts + if _, isExclusiveTest := exclusiveTests[testName]; isExclusiveTest { + parallelAttempts = 0 + } + for attempt := 0; attempt < parallelAttempts; attempt++ { + c, release, err := cp.Get(ctx) + if err != nil { + release() + return "", fmt.Errorf("failed to get container: %v", err) + } + cp.SetContainerLabel(c, fmt.Sprintf("Running %s in parallel (attempt %d/%d)", testName, attempt+1, parallelAttempts)) + testLog(t, "Running test in parallel mode in container %s (attempt %d/%d)...", c.Name, attempt+1, parallelAttempts) + parallelCtx, parallelCancel := context.WithTimeoutCause(ctx, testTimeout, errors.New("parallel execution took too long")) + testStartedAt := time.Now() + output, err := c.Exec(parallelCtx, dockerutil.ExecOpts{}, "/run_sample", fmt.Sprintf("--timeout=%v", execTestTimeout), testName) + testDuration := time.Since(testStartedAt) + parallelCancel() + release() + if err == nil { + if willFailReason != "" { + multiLineLog(t, output) + return "", fmt.Errorf("test unexpectedly succeeded, but we expected it to fail: %s; please update `testCompatibility`", willFailReason) + } + // Only log the output when the test succeeds here. + // If it fails, we'll run exclusively below, and the output from *that* + // run will be logged instead. + if *logSuccessfulTests { + multiLineLog(t, output) + } + testLog(t, "Test passed in parallel mode in %v.", testDuration) + return "", nil + } + var exitCode int + if execErr, ok := err.(*dockerutil.ExecError); ok { + exitCode = execErr.ExitStatus + } + if willFailReason != "" { + isExpectedErr := compat.IsExpectedFailure(ctx, te, output, exitCode) + if isExpectedErr == nil { + testLog(t, "Test failed as expected: %s (took %v)", willFailReason, testDuration) + return "", nil + } + } + } + if parallelAttempts > 0 { + testLog(t, "Will re-run the test in exclusive mode.") + } + c, release, err := cp.GetExclusive(ctx) + defer release() + if err != nil { + return "", fmt.Errorf("failed to get excusive container: %v", err) + } + var testErr error + for attempt := 0; attempt < testAttempts; attempt++ { + cp.SetContainerLabel(c, fmt.Sprintf("Running %s exclusively (attempt %d/%d)", testName, attempt+1, testAttempts)) + testLog(t, "Running test in exclusive mode in container %s (attempt %d/%d)...", c.Name, attempt+1, testAttempts) + exclusiveCtx, exclusiveCancel := context.WithTimeoutCause(ctx, testTimeout, errors.New("exclusive execution took too long")) + testStartedAt := time.Now() + var output string + output, testErr = c.Exec(exclusiveCtx, dockerutil.ExecOpts{}, "/run_sample", fmt.Sprintf("--timeout=%v", execTestTimeout), testName) + testDuration := time.Since(testStartedAt) + exclusiveCancel() + if testErr == nil { + if willFailReason != "" { + multiLineLog(t, output) + return "", fmt.Errorf("test unexpectedly succeeded, but we expected it to fail: %s; please update `testCompatibility`", willFailReason) + } + if *logSuccessfulTests { + multiLineLog(t, output) + } + testLog(t, "Test passed in exclusive mode in %v.", testDuration) + return "", nil + } + multiLineLog(t, output) + var exitCode int + if execErr, ok := testErr.(*dockerutil.ExecError); ok { + exitCode = execErr.ExitStatus + } + if willFailReason != "" { + isExpectedErr := compat.IsExpectedFailure(ctx, te, output, exitCode) + if isExpectedErr == nil { + testLog(t, "Test failed as expected: %s (took %v)", willFailReason, testDuration) + return "", nil + } + return "", fmt.Errorf("test was expected to fail (%s), but it failed with %v which is a different reason reason than expected: %v", willFailReason, testErr, isExpectedErr) + } + } + return "", fmt.Errorf("test failed: %v", testErr) +} + +// getDesiredTestParallelism returns the number of tests to run in parallel. +func getDesiredTestParallelism() int { + numCPU := runtime.NumCPU() + if numCPU <= 0 { + panic("cannot detect number of cores") + } + return int(math.Ceil((*containersPerCPU) * float64(numCPU))) +} + +// TestCUDA runs CUDA tests. +func TestCUDA(t *testing.T) { + const defaultMaxDuration = 59*time.Minute + 30*time.Second + + testStart := time.Now() + maxDuration := defaultMaxDuration + if timeoutFlag := flag.Lookup("timeout"); timeoutFlag != nil { + if timeoutFlagStr := timeoutFlag.Value.String(); timeoutFlagStr != "" { + timeoutFlagValue, err := time.ParseDuration(timeoutFlagStr) + if err != nil { + t.Fatalf("--timeout flag %q is not a valid duration: %v", timeoutFlagStr, err) + } + if timeoutFlagValue != 0 { + maxDuration = timeoutFlagValue + } + } + } + ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, errors.New("overall test timed out")) + defer cancel() + testDeadline, ok := ctx.Deadline() + if !ok { + t.Fatal("context had no deadline") + } + testLog(t, "Test timeout is %v; started at %v, deadline is %v", maxDuration, testStart, testDeadline) + + te, err := GetEnvironment(ctx, t) + if err != nil { + t.Fatalf("Failed to get test environment: %v", err) + } + + // Get a list of sample tests. + listContainer := dockerutil.MakeContainer(ctx, t) + defer listContainer.CleanUp(ctx) + testsList, err := listContainer.Run(ctx, getContainerOpts(), "/list_sample_tests.sh") + if err != nil { + t.Fatalf("Cannot list sample tests: %v", err) + } + testsSplit := strings.Split(testsList, "\n") + allTests := make([]string, 0, len(testsSplit)) + allTestsMap := make(map[string]struct{}, len(testsSplit)) + for _, test := range testsSplit { + testName := strings.TrimSpace(test) + if testName == "" { + continue + } + allTestsMap[testName] = struct{}{} + allTests = append(allTests, testName) + } + numTests := len(allTests) + testLog(t, "Number of CUDA sample tests detected: %d", numTests) + + // Check that all tests in test maps still exist. + t.Run("CUDA test existence", func(t *testing.T) { + for testName := range testCompatibility { + if _, ok := allTestsMap[testName]; !ok { + t.Errorf("CUDA test %q referenced in `testCompatibility` but it no longer exists, please remove it.", testName) + } + } + }) + + // In order to go through tests efficiently, we reuse containers. + // However, running tests serially within the same container would also be + // slow. So this test spawns a pool of containers, one per CPU. + // This saves time because a lot of the time here is actually spent waiting + // for compilation of the CUDA program on the CPU, and isn't actually + // blocked on the GPU. However, it is possible that two CUDA tests do end + // up running on the GPU at the same time, and that they don't work together + // for some reason (e.g. out of GPU memory). + // To address this, the test first runs every test in parallel. Then, if + // any of them failed, it will run only the failed ones serially. + numContainers := getDesiredTestParallelism() + testLog(t, "Number of cores is %d, spawning %.1f CUDA containers for each (%d containers total)...", runtime.NumCPU(), *containersPerCPU, numContainers) + spawnGroup, spawnCtx := errgroup.WithContext(ctx) + containers := make([]*dockerutil.Container, numContainers) + for i := 0; i < numContainers; i++ { + spawnGroup.Go(func() error { + c := dockerutil.MakeContainer(ctx, t) + if err := c.Spawn(spawnCtx, getContainerOpts(), "/bin/sleep", "6h"); err != nil { + return fmt.Errorf("container %v failed to spawn: %w", c.Name, err) + } + containers[i] = c + return nil + }) + } + if err := spawnGroup.Wait(); err != nil { + for _, c := range containers { + if c != nil { + c.CleanUp(ctx) + } + } + t.Fatalf("Failed to spawn containers: %v", err) + } + cp := dockerutil.NewContainerPool(containers) + defer cp.CleanUp(ctx) + var testMu sync.Mutex + testsDone := 0 + var failedTests []string + statusFn := func() { + now := time.Now() + testMu.Lock() + defer testMu.Unlock() + donePct := 100.0 * float64(testsDone) / float64(numTests) + startedAgo := now.Sub(testStart) + deadlineIn := testDeadline.Sub(now) + durationPct := 100.0 * float64(startedAgo) / float64(testDeadline.Sub(testStart)) + testLog(t, "[Timing] %d/%d tests (%.1f%%) finished executing. Test started %v ago, deadline in %v (%.1f%%).", testsDone, numTests, donePct, startedAgo.Truncate(time.Second), deadlineIn.Truncate(time.Second), durationPct) + if len(failedTests) > 0 { + testLog(t, "[Failed] %d test failed: %v", len(failedTests), strings.Join(failedTests, ", ")) + } + testLog(t, "[Pool] %v", cp.String()) + } + if *debug { + go func() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + statusFn() + } + } + }() + } + var samplesTestName string + t.Run("Samples", func(t *testing.T) { + samplesTestName = t.Name() + // Now spawn all subtests in parallel. + // All sub-tests will first try to run in parallel using one of the pooled + // containers. + // Those that failed will try to grab `serialMu` in order to run serially. + // Therefore, the main goroutine here holds `serialMu` and only releases + // when all parallel test attempts have completed. + testutil.NewTree(allTests, "/").RunParallel(t, func(t *testing.T, testName string) { + t.Helper() + skippedReason, err := runSampleTest(ctx, t, testName, te, cp) + if err != nil { + t.Errorf("%s: %v", testName, err) + } + testMu.Lock() + defer testMu.Unlock() + testsDone++ + if t.Failed() && ctx.Err() == nil { + failedTests = append(failedTests, testName) + } + if skippedReason != "" { + t.Skip(skippedReason) + } + }) + }) + statusFn() + testMu.Lock() + defer testMu.Unlock() + if len(failedTests) > 0 { + if ctx.Err() != nil { + t.Errorf("%d tests failed prior to timeout:", len(failedTests)) + for _, testName := range failedTests { + t.Errorf(" %s", testName) + } + } + if len(failedTests) > 0 { + t.Errorf("To re-run a specific test locally, either re-run this test with filtering enabled (example: --test.run=%s/%s), or:", samplesTestName, failedTests[0]) + t.Errorf( + " $ docker run --runtime=%s --gpus=all -e %s --rm %s /run_sample %s", + dockerutil.Runtime(), + dockerutil.AllGPUCapabilities, + getContainerOpts().Image, + failedTests[0], + ) + } + } else if poolUtilization := cp.Utilization(); poolUtilization < 0.6 { + testLog(t, "WARNING: Pool utilization was only %.1f%%.", poolUtilization*100.0) + testLog(t, "This test can be made faster and more efficient with proper test categorization,") + testLog(t, "by identifying flaky tests and exclusive-requiring tests.") + testLog(t, "Consider going over the logs to identify such tests and categorize them accordingly.") + } +} + +// TestMain overrides the `test.parallel` flag. +func TestMain(m *testing.M) { + dockerutil.EnsureSupportedDockerVersion() + flag.Parse() + // The Go testing library won't run more than GOMAXPROCS parallel tests by + // default, and the value of GOMAXPROCS is taken at program initialization + // time, so by the time we get here, it is already stuck at GOMAXPROCS. + // In order to run more parallel tests than there are cores, we therefore + // need to override the `test.parallel` flag here before `m.Run`. + testParallelFlag := flag.Lookup("test.parallel") + if testParallelFlag == nil { + panic("cannot find -test.parallel flag") + } + if err := testParallelFlag.Value.Set(strconv.Itoa(getDesiredTestParallelism())); err != nil { + panic(fmt.Sprintf("cannot set -test.parallel flag: %v", err)) + } + os.Exit(m.Run()) +} diff --git a/test/gpu/smoke_test.go b/test/gpu/smoke_test.go index e5a91e1ed..5e273357c 100644 --- a/test/gpu/smoke_test.go +++ b/test/gpu/smoke_test.go @@ -36,16 +36,16 @@ func TestGPUHello(t *testing.T) { t.Logf("cuda-vector-add output: %s", string(out)) } -func TestCUDATests(t *testing.T) { +func TestCUDASmokeTests(t *testing.T) { ctx := context.Background() c := dockerutil.MakeContainer(ctx, t) defer c.CleanUp(ctx) opts := dockerutil.GPURunOpts() opts.Image = "gpu/cuda-tests" - out, err := c.Run(ctx, opts) + out, err := c.Run(ctx, opts, "/run_smoke.sh") if err != nil { - t.Fatalf("could not run cuda-tests: %v", err) + t.Fatalf("could not run cuda-tests smoke tests: %v", err) } - t.Logf("cuda-tests output: %s", string(out)) + t.Logf("cuda-tests smoke tests output: %s", string(out)) }