From 2c3974d10d86c3a0d2f0693f28bfd6517e32e8c6 Mon Sep 17 00:00:00 2001
From: Etienne Perot <eperot@google.com>
Date: Wed, 8 Jan 2025 11:48:11 -0800
Subject: [PATCH] Add CUDA tests to release pipeline.

These run in compatibility-checking mode, meaning that they run tests which
are expected to fail and ensure that these tests still fail. This takes
even longer than when not checking this, so these run only as part of the
release pipeline.

PiperOrigin-RevId: 713365006
---
 .buildkite/release.yaml | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/.buildkite/release.yaml b/.buildkite/release.yaml
index 53f4a6276..a355d5b14 100644
--- a/.buildkite/release.yaml
+++ b/.buildkite/release.yaml
@@ -1,15 +1,16 @@
 agents:
   queue: release
 _templates:
+  retry_settings: &retry_settings
+    automatic:
+      - exit_status: -1
+        limit: 10
+      - exit_status: "*"
+        limit: 2
   common: &common
     timeout_in_minutes: 180
     retry:
-      automatic:
-        - exit_status: -1
-          limit: 10
-        - exit_status: "*"
-          limit: 2
-
+      <<: *retry_settings
 notify:
   - email: "gvisor-eng+buildkite@google.com"
     if: build.state == "failed"
@@ -75,6 +76,18 @@ steps:
       - make gpu-all-tests
     agents:
       queue: gpu
+  - label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 30 minutes to run.
+    parallelism: 32
+    timeout_in_minutes: 120
+    retry:
+      <<: *retry_settings
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu
   - <<: *common
     label: ":screwdriver: All GPU Drivers Test"
     parallelism: 8