From 0fcb9b7f2ede22f8c96b2a0e15d3a5a27d7d41ad Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Wed, 11 Sep 2024 15:13:53 -0700 Subject: [PATCH] Disable GPU sniffer on all non-smoke GPU tests. The presubmit pipeline does not exercise these, so commit a689c11a76539af18c432f942494c4a43f3857b7 broke them. This change disables the sniffer on all the non-smoke tests to unbreak the release pipeline. I will then send another change to re-enable them on tests where the sniffer works fine after manual testing. Updates #10885 PiperOrigin-RevId: 673555026 --- test/gpu/cuda_test.go | 4 +++- test/gpu/nccl_test.go | 4 +++- test/gpu/ollama/ollama.go | 2 +- test/gpu/pytorch_test.go | 4 +++- test/gpu/sr_test.go | 4 +++- test/gpu/stablediffusion/stablediffusion.go | 2 +- test/gpu/vllm/vllm_test.go | 4 +++- 7 files changed, 17 insertions(+), 7 deletions(-) diff --git a/test/gpu/cuda_test.go b/test/gpu/cuda_test.go index 9762f5527..2fb25f431 100644 --- a/test/gpu/cuda_test.go +++ b/test/gpu/cuda_test.go @@ -396,7 +396,9 @@ func (*FullyCompatible) IsExpectedFailure(ctx context.Context, env *TestEnvironm // getContainerOpts returns the container run options to run CUDA tests. func getContainerOpts() (dockerutil.RunOpts, error) { - opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{AllowIncompatibleIoctl: true}) + opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", + }) if err != nil { return dockerutil.RunOpts{}, fmt.Errorf("failed to get GPU run options: %w", err) } diff --git a/test/gpu/nccl_test.go b/test/gpu/nccl_test.go index 74c7772b9..b82282711 100644 --- a/test/gpu/nccl_test.go +++ b/test/gpu/nccl_test.go @@ -27,7 +27,9 @@ import ( func runNCCL(ctx context.Context, t *testing.T, testName string) { t.Helper() c := dockerutil.MakeContainer(ctx, t) - opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{AllowIncompatibleIoctl: true}) + opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", + }) if err != nil { t.Fatalf("Failed to get GPU run options: %v", err) } diff --git a/test/gpu/ollama/ollama.go b/test/gpu/ollama/ollama.go index 172cf278e..7443d8682 100644 --- a/test/gpu/ollama/ollama.go +++ b/test/gpu/ollama/ollama.go @@ -150,7 +150,7 @@ type dockerServer struct { // in a local Docker container. func NewDocker(ctx context.Context, cont *dockerutil.Container, logger testutil.Logger) (*Ollama, error) { opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ - AllowIncompatibleIoctl: true, + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", }) if err != nil { return nil, fmt.Errorf("failed to get GPU run options: %w", err) diff --git a/test/gpu/pytorch_test.go b/test/gpu/pytorch_test.go index 15b5ffa3a..05d74a53a 100644 --- a/test/gpu/pytorch_test.go +++ b/test/gpu/pytorch_test.go @@ -26,7 +26,9 @@ import ( func runPytorch(ctx context.Context, t *testing.T, scriptPath string, args ...string) { t.Helper() c := dockerutil.MakeContainer(ctx, t) - opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{AllowIncompatibleIoctl: true}) + opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", + }) if err != nil { t.Fatalf("Failed to get GPU run options: %v", err) } diff --git a/test/gpu/sr_test.go b/test/gpu/sr_test.go index 5f9db3dd2..e8ca6cfc8 100644 --- a/test/gpu/sr_test.go +++ b/test/gpu/sr_test.go @@ -34,7 +34,9 @@ func TestGPUCheckpointRestore(t *testing.T) { c := dockerutil.MakeContainer(ctx, t) defer c.CleanUp(ctx) - opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{AllowIncompatibleIoctl: true}) + opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", + }) if err != nil { t.Fatalf("failed to get GPU run options: %v", err) } diff --git a/test/gpu/stablediffusion/stablediffusion.go b/test/gpu/stablediffusion/stablediffusion.go index 511b00d53..44499221a 100644 --- a/test/gpu/stablediffusion/stablediffusion.go +++ b/test/gpu/stablediffusion/stablediffusion.go @@ -48,7 +48,7 @@ func (dr *dockerRunner) Run(ctx context.Context, image string, argv []string) ([ cont := dockerutil.MakeContainer(ctx, dr.logger) defer cont.CleanUp(ctx) opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ - AllowIncompatibleIoctl: true, + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", }) if err != nil { return nil, fmt.Errorf("failed to get GPU run options: %w", err) diff --git a/test/gpu/vllm/vllm_test.go b/test/gpu/vllm/vllm_test.go index d4d55c138..2c84ce750 100644 --- a/test/gpu/vllm/vllm_test.go +++ b/test/gpu/vllm/vllm_test.go @@ -51,7 +51,9 @@ func doVLLMTest(b *testing.B) { } // Run vllm. - runOpts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{AllowIncompatibleIoctl: true}) + runOpts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + DisableSnifferReason: "TODO(gvisor.dev/issue/10885): Verify that this test works", + }) if err != nil { b.Fatalf("failed to get GPU run options: %v", err) }