From 0c4a709bc15702ec9d01fc13b6701376e1099115 Mon Sep 17 00:00:00 2001
From: Etienne Perot <eperot@google.com>
Date: Wed, 30 Oct 2024 12:35:47 -0700
Subject: [PATCH] Run CUDA tests as part of GPU tests.

Attempt #2.

This runs in continuous mode only.

PiperOrigin-RevId: 691516066
---
 .buildkite/pipeline.yaml | 10 ++++++++++
 .buildkite/release.yaml  |  9 +++++++++
 Makefile                 |  5 +++++
 test/gpu/cuda_test.go    |  2 +-
 4 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 030a99d92..b729e025b 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -190,6 +190,16 @@ steps:
       - make cos-gpu-all-tests
     agents:
       queue: cos-canary-gpu
+  - <<: *common
+    <<: *source_test_continuous
+    label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 15 minutes to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests
+    agents:
+      queue: gpu
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"
diff --git a/.buildkite/release.yaml b/.buildkite/release.yaml
index e913feb68..6c0473ad5 100644
--- a/.buildkite/release.yaml
+++ b/.buildkite/release.yaml
@@ -67,6 +67,15 @@ steps:
       - make gpu-all-tests
     agents:
       queue: gpu
+  - <<: *common
+    label: ":fish: CUDA tests in compatibility-verifying mode"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around an hour to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu
   - <<: *common
     label: ":screwdriver: All GPU Drivers Test"
     commands:
diff --git a/Makefile b/Makefile
index 2c7e31989..71f7f0591 100644
--- a/Makefile
+++ b/Makefile
@@ -331,6 +331,11 @@ cos-gpu-all-tests: gpu-images cos-gpu-smoke-tests $(RUNTIME_BIN)
 	@$(call sudo,test/gpu:sniffer_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
 .PHONY: cos-gpu-all-tests
 
+cuda-tests: load-gpu_cuda-tests
+	@$(call install_runtime,$(RUNTIME),--nvproxy=true --nvproxy-docker=true)
+	@$(call sudo,test/gpu:cuda_test,--runtime=$(RUNTIME) -test.v $(ARGS))
+.PHONY: cuda-tests
+
 portforward-tests: load-basic_redis load-basic_nginx $(RUNTIME_BIN)
 	@$(call install_runtime,$(RUNTIME),--network=sandbox)
 	@$(call sudo,test/root:portforward_test,--runtime=$(RUNTIME) -test.v $(ARGS))
diff --git a/test/gpu/cuda_test.go b/test/gpu/cuda_test.go
index dec522b0f..03906e68b 100644
--- a/test/gpu/cuda_test.go
+++ b/test/gpu/cuda_test.go
@@ -745,7 +745,7 @@ func TestCUDA(t *testing.T) {
 		durationPct := 100.0 * float64(startedAgo) / float64(testDeadline.Sub(testStart))
 		testLog(t, "[Timing] %d/%d tests (%.1f%%) finished executing. Test started %v ago, deadline in %v (%.1f%%).", testsDone, numTests, donePct, startedAgo.Truncate(time.Second), deadlineIn.Truncate(time.Second), durationPct)
 		if len(failedTests) > 0 {
-			testLog(t, "[Failed] %d test failed: %v", len(failedTests), strings.Join(failedTests, ", "))
+			testLog(t, "[Failed] %d test(s) failed: %v", len(failedTests), strings.Join(failedTests, ", "))
 		}
 		testLog(t, "[Pool] %v", cp.String())
 	}