GPU support: Add NVIDIA CUDA sample tests.

This is a set of CUDA tests defined by NVIDIA in this repository:
https://github.com/NVIDIA/cuda-samples

This change introduces a large new test (`cuda_test`) which runs each CUDA
sample test in a container.

There are many subtleties involved due to how the CUDA samples repository
isn't always meant to be run as a test, some of it involves graphical
applications, and a lot of them require this or that CUDA feature which not
all NVIDIA GPUs support, some require multiple GPUs to be on the machine, etc.
Therefore, the test maps each test to their `Compatibility` data which
determines whether or not a sample test is expected to fail when run in a
certain environment. The overall test also has a
`--cuda_verify_compatibility` flag to verify the veracity of this mapping,
by running expected-to-be-broken tests and verifying that their failure
matches how this expected failure typically manifests.

Because there are a lot of CUDA sample tests (213 as of the CUDA 12.3 release
of the cuda-samples repo), and they don't all require the whole GPU to
themselves, and spawning a GPU-using container is expensive (~seconds),
the test uses a pool of reusable containers in which it `exec`s tests (at
most one per container at any given time, but in parallel across containers).
If any test unexpectedly fails, we drain the entire pool of containers and
only run this one test without anything else running on other containers.
This de-flakes tests, especially those that fail because they require more
resources than the GPU has when other tests are using it at the same time.
However, this removes parallelism and therefore increases test time
significantly.

Despite these optimizations, the test is very long and has the maximum deadline
of 1 hour. Because it may get close to the timeout (especially when
`--cuda_verify_compatibility` is on, because that needs to run all the tests
even if they are known to fail, and the ones that do fail can hang for a
while rather than crash), the test also has more logging and debugging than
the typical test, as enabled with the `--cuda_log_successful_tests` and
`--cuda_test_debug` flag. It periodically logs the status of each container
and the pool's utilization ratio. This is useful when debugging the test to
see which pooled container is doing what, and/or to `docker exec` into
containers while they are running a certain test.
The test also does its own timekeeping, which is useful so that it can print
a more helpful failure message that distinguishes between tests failing due
to actual failure reasons vs those that are failing purely because the test
timed out.

To run the test manually (from a VM with the repo checked out):

```
$ docker build -t gvisor.dev/images/gpu/cuda-tests images/gpu/cuda-tests -f images/gpu/cuda-tests/Dockerfile.x86_64 && mkdir -p bin && make copy TARGETS=runsc DESTINATION=bin/ && ./bin/runsc install -- --nvproxy=true --debug=true --debug-log=/tmp/runsc/ && systemctl reload docker && make test TEST_OPTIONS='--test_output=streamed --verbose_failures=true' TARGETS=//test/gpu:cuda_test OPTIONS='--test_env=RUNTIME=runsc --test_arg=--cuda_test_debug=true --test_arg=cuda_verify_compatibility=true --test_arg=--cuda_log_successful_tests=true'
```

PiperOrigin-RevId: 626528982

images/gpu/cuda-tests/Dockerfile.x86_64

@@ -1,8 +1,45 @@
-FROM nvidia/cuda:12.2.0-devel-ubuntu20.04
+FROM nvidia/cuda:12.3.2-devel-ubuntu22.04
+
+# From: https://github.com/NVIDIA/cuda-samples/releases
+# Ideally, pick a release that matches the CUDA version of the image above.
+ARG CUDA_SAMPLES_VERSION=v12.3
 
 WORKDIR /
-COPY cuda_malloc_managed.cu .
-COPY cuda_test_util.h .
-COPY run.sh .
+COPY *.cu *.h *.sh *.go /
 ENV PATH=$PATH:/usr/local/nvidia/bin:/bin/nvidia/bin
-ENTRYPOINT ["/run.sh"]
+RUN export DEBIAN_FRONTEND=noninteractive; \
+    apt-get update && \
+    apt-get install -y \
+      build-essential \
+      cmake \
+      freeglut3 freeglut3-dev \
+      git \
+      golang \
+      imagemagick \
+      libegl-dev \
+      libfreeimage3 libfreeimage-dev \
+      libfreeimageplus3 libfreeimageplus-dev \
+      libgles2-mesa-dev \
+      libglfw3 libglfw3-dev \
+      libglu1-mesa libglu1-mesa-dev \
+      libxi-dev \
+      libxmu-dev \
+      llvm \
+      mpich \
+      pkg-config \
+      x11-xserver-utils \
+      xdotool \
+      xvfb \
+      zlib1g zlib1g-dev \
+    && \
+    chmod 555 /*.sh && \
+    git clone --depth=1 --branch="$CUDA_SAMPLES_VERSION" --single-branch \
+      https://github.com/NVIDIA/cuda-samples.git /cuda-samples && \
+    go install \
+      github.com/TheZoraiz/ascii-image-converter@d05a757c5e02ab23e97b6f6fca4e1fbeb10ab559 && \
+    mv "$HOME/go/bin/ascii-image-converter" /usr/bin/ && \
+    go build -o /run_sample /run_sample.go
+
+# Override entrypoint to nothing, otherwise all invocations will have
+# a copyright notice printed, which breaks parsing the stdout logs.
+ENTRYPOINT []

images/gpu/cuda-tests/cuda_test_util.h

@@ -17,6 +17,7 @@
 
 #include <iostream>
 
+// cudaError_t is returned by CUDA runtime functions.
 #define CHECK_CUDA(expr)                                                     \
   do {                                                                       \
     cudaError_t code = (expr);                                               \
@@ -27,4 +28,15 @@
     }                                                                        \
   } while (0)
 
+// CUresult is returned by CUDA driver functions.
+#define CHECK_CUDA_RESULT(expr)                                              \
+  do {                                                                       \
+    CUresult code = (expr);                                                  \
+    if (code != CUDA_SUCCESS) {                                              \
+      std::cout << "Check failed at " << __FILE__ << ":" << __LINE__ << ": " \
+                << #expr << ": " << code << std::endl;                       \
+      abort();                                                               \
+    }                                                                        \
+  } while (0)
+
 #endif  // THIRD_PARTY_GVISOR_IMAGES_GPU_CUDA_TESTS_CUDA_TEST_UTIL_H_

images/gpu/cuda-tests/list_features.cu

@@ -0,0 +1,52 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This program lists the features of the CUDA device that are available.
+// It is used as part of the list_features.sh script.
+// Each line it outputs is a CUDA feature name, prefixed by either
+// "PRESENT: " or "ABSENT: ".
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <stdio.h>
+
+#include "cuda_test_util.h"  // NOLINT(build/include)
+
+void printFeature(const char* feature, bool have) {
+  if (have) {
+    printf("PRESENT: %s\n", feature);
+  } else {
+    printf("ABSENT: %s\n", feature);
+  }
+}
+
+int main(int argc, char *argv[]) {
+  int cuda_device;
+  CHECK_CUDA(cudaGetDevice(&cuda_device));
+  cudaDeviceProp properties;
+  CHECK_CUDA(cudaGetDeviceProperties(&properties, cuda_device));
+  bool cdpCapable =
+      (properties.major == 3 && properties.minor >= 5) || properties.major >= 4;
+  printFeature("DYNAMIC_PARALLELISM", cdpCapable);
+  printFeature(
+      "PERSISTENT_L2_CACHING", properties.persistingL2CacheMaxSize > 0);
+  // Tensor cores are a thing in Volta (SM8X)
+  printFeature("TENSOR_CORES", properties.major >= 8);
+  int isCompressionAvailable;
+  CHECK_CUDA_RESULT(
+      cuDeviceGetAttribute(&isCompressionAvailable,
+                           CU_DEVICE_ATTRIBUTE_GENERIC_COMPRESSION_SUPPORTED,
+                           cuda_device));
+  printFeature("COMPRESSIBLE_MEMORY", isCompressionAvailable != 0);
+}

images/gpu/cuda-tests/list_features.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2024 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script outputs a list of CUDA features that are present or absent,
+# one per line. Each line begins with either "PRESENT: " or "ABSENT: ",
+# followed by the feature name.
+
+set -euo pipefail
+
+cd /
+nvcc list_features.cu -lcuda -o list_features
+./list_features
+
+# Detect GL by using a simple test that uses it as reference.
+if xvfb-run make -C /cuda-samples/Samples/0_Introduction/simpleCUDA2GL TARGET_ARCH="$(uname -m)" testrun &>/dev/null; then
+  echo "PRESENT: GL"
+else
+  echo "ABSENT: GL"
+fi

images/gpu/cuda-tests/list_sample_tests.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Copyright 2024 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script outputs a sorted list of CUDA sample tests, one per line.
+
+set -euo pipefail
+
+(
+  while IFS= read -r makefile_path; do
+    dirname "$makefile_path"
+  done < <(find /cuda-samples -type f -name Makefile) \
+    | grep -vE '^/cuda-samples$' | grep -vE '/7_libNVVM'
+
+  # cuda-samples/Samples/7_libNVVM is not structured like the other tests.
+  # It is built with `cmake` and generates multiple test binaries.
+  # The generated ones all follow the pattern of being named after their
+  # parent directory name, so we look for that.
+  pushd /cuda-samples/Samples/7_libNVVM &>/dev/null
+    cmake . &>/dev/null
+    make TARGET_ARCH="$(uname -m)" all &>/dev/null
+  popd &>/dev/null
+  while IFS= read -r dir_path; do
+    if [[ -x "$dir_path/$(basename "$dir_path")" ]]; then
+      echo "$dir_path"
+    fi
+  done < <(find /cuda-samples/Samples/7_libNVVM -type d) | sort | uniq
+) | sed 's~/cuda-samples/Samples/~~' | sort

pkg/test/dockerutil/gpu.go

@@ -29,14 +29,19 @@ var (
 	setCOSGPU = flag.Bool("cos-gpu", false, "set to configure GPU settings for COS, as opposed to Docker")
 )
 
+// AllGPUCapabilities is the environment variable that enables all NVIDIA GPU
+// capabilities within a container.
+const AllGPUCapabilities = "NVIDIA_DRIVER_CAPABILITIES=all"
+
 // GPURunOpts returns Docker run options with GPU support enabled.
 func GPURunOpts() RunOpts {
 	if !*setCOSGPU {
 		return RunOpts{
+			Env: []string{AllGPUCapabilities},
 			DeviceRequests: []container.DeviceRequest{
 				{
 					Count:        -1,
-					Capabilities: [][]string{[]string{"gpu"}},
+					Capabilities: [][]string{{"gpu"}},
 					Options:      map[string]string{},
 				},
 			},
@@ -92,6 +97,7 @@ func GPURunOpts() RunOpts {
 	}
 
 	return RunOpts{
+		Env:     []string{AllGPUCapabilities},
 		Mounts:  mounts,
 		Devices: devices,
 	}

test/gpu/BUILD

@@ -62,6 +62,23 @@ go_test(
     ],
 )
 
+go_test(
+    name = "cuda_test",
+    timeout = "eternal",  # YES_I_REALLY_NEED_AN_ETERNAL_TEST
+    srcs = ["cuda_test.go"],
+    tags = [
+        "local",
+        "noguitar",
+        "notap",
+    ],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//pkg/test/testutil",
+        "@org_golang_x_sync//errgroup:go_default_library",
+    ],
+)
+
 go_test(
     name = "imagegen_test",
     srcs = ["imagegen_test.go"],

test/gpu/smoke_test.go

@@ -36,16 +36,16 @@ func TestGPUHello(t *testing.T) {
 	t.Logf("cuda-vector-add output: %s", string(out))
 }
 
-func TestCUDATests(t *testing.T) {
+func TestCUDASmokeTests(t *testing.T) {
 	ctx := context.Background()
 	c := dockerutil.MakeContainer(ctx, t)
 	defer c.CleanUp(ctx)
 
 	opts := dockerutil.GPURunOpts()
 	opts.Image = "gpu/cuda-tests"
-	out, err := c.Run(ctx, opts)
+	out, err := c.Run(ctx, opts, "/run_smoke.sh")
 	if err != nil {
-		t.Fatalf("could not run cuda-tests: %v", err)
+		t.Fatalf("could not run cuda-tests smoke tests: %v", err)
 	}
-	t.Logf("cuda-tests output: %s", string(out))
+	t.Logf("cuda-tests smoke tests output: %s", string(out))
 }