Parallelize CUDA test and run it continuously.

Tested in presubmit mode:
https://buildkite.com/gvisor/pipeline/builds/32983#0192e4d7-2a41-4505-8eea-422477b07644

PiperOrigin-RevId: 692198767

.buildkite/pipeline.yaml

@@ -1,12 +1,14 @@
 _templates:
+  retry_settings: &retry_settings
+    automatic:
+      - exit_status: -1
+        limit: 10
+      - exit_status: "*"
+        limit: 2
   common: &common
     timeout_in_minutes: 30
     retry:
-      automatic:
-        - exit_status: -1
-          limit: 10
-        - exit_status: "*"
-          limit: 2
+      <<: *retry_settings
   source_test: &source_test
     if: build.env("STAGED_BINARIES") == null
   source_test_presubmit: &source_test_presubmit
@@ -190,6 +192,19 @@ steps:
       - make cos-gpu-all-tests
     agents:
       queue: cos-canary-gpu
+  - <<: *source_test_continuous
+    label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 30 minutes to run.
+    parallelism: 8
+    timeout_in_minutes: 60
+    retry:
+      <<: *retry_settings
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests
+    agents:
+      queue: gpu
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"

test/gpu/cuda_test.go

@@ -694,6 +694,21 @@ func TestCUDA(t *testing.T) {
 		}
 	})
 
+	// Filter tests if partitioning is enabled.
+	testIndices, err := testutil.TestIndicesForShard(numTests)
+	if err != nil {
+		t.Fatalf("Failed to get test indices for shard: %v", err)
+	}
+	if len(testIndices) != numTests {
+		filteredTests := make([]string, 0, len(testIndices))
+		for _, testIndex := range testIndices {
+			filteredTests = append(filteredTests, allTests[testIndex])
+		}
+		testLog(t, "Filtered tests from sharding; %d -> %d tests.", numTests, len(filteredTests))
+		allTests = filteredTests
+		numTests = len(allTests)
+	}
+
 	// In order to go through tests efficiently, we reuse containers.
 	// However, running tests serially within the same container would also be
 	// slow. So this test spawns a pool of containers, one per CPU.