Run CUDA tests as part of GPU tests.

This runs in continuous mode only.

PiperOrigin-RevId: 688755446

.buildkite/pipeline.yaml

@@ -190,6 +190,16 @@ steps:
       - make cos-gpu-all-tests
     agents:
       queue: cos-canary-gpu
+  - <<: *common
+    <<: *source_test_continuous
+    label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 15 minutes to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make sudo TARGETS=//test/gpu:cuda_test
+    agents:
+      queue: gpu
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"

.buildkite/release.yaml

@@ -73,3 +73,12 @@ steps:
       - tools/gpu/all_drivers_test.sh
     agents:
       queue: gpu
+  - <<: *common
+    label: ":fish: CUDA tests in compatibility-verifying mode"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around an hour to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make sudo TARGETS=//test/gpu:cuda_test ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu

test/gpu/cuda_test.go

@@ -745,7 +745,7 @@ func TestCUDA(t *testing.T) {
 		durationPct := 100.0 * float64(startedAgo) / float64(testDeadline.Sub(testStart))
 		testLog(t, "[Timing] %d/%d tests (%.1f%%) finished executing. Test started %v ago, deadline in %v (%.1f%%).", testsDone, numTests, donePct, startedAgo.Truncate(time.Second), deadlineIn.Truncate(time.Second), durationPct)
 		if len(failedTests) > 0 {
-			testLog(t, "[Failed] %d test failed: %v", len(failedTests), strings.Join(failedTests, ", "))
+			testLog(t, "[Failed] %d test(s) failed: %v", len(failedTests), strings.Join(failedTests, ", "))
 		}
 		testLog(t, "[Pool] %v", cp.String())
 	}