From 18c75622863f9aec21f2c390eded9dafe829234d Mon Sep 17 00:00:00 2001 From: Anthony Cui Date: Thu, 20 Jun 2024 16:38:43 -0700 Subject: [PATCH] Add simple functionality tests for ioctl_sniffer. This also adds two new flags, --enforce_compatibility and --verbose, and fixes an issue where legacy control ioctls were being flagged as unsupported. PiperOrigin-RevId: 645193498 --- images/gpu/cuda-tests/Dockerfile.x86_64 | 3 +- images/gpu/cuda-tests/unsupported_ioctl.cc | 28 +++++++ test/gpu/BUILD | 19 +++++ test/gpu/sniffer_test.go | 85 ++++++++++++++++++++++ tools/ioctl_sniffer/BUILD | 4 + tools/ioctl_sniffer/run_sniffer.go | 12 +++ tools/ioctl_sniffer/sniffer/sniffer.go | 11 ++- 7 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 images/gpu/cuda-tests/unsupported_ioctl.cc create mode 100644 test/gpu/sniffer_test.go diff --git a/images/gpu/cuda-tests/Dockerfile.x86_64 b/images/gpu/cuda-tests/Dockerfile.x86_64 index 76198083b..a8a160a66 100644 --- a/images/gpu/cuda-tests/Dockerfile.x86_64 +++ b/images/gpu/cuda-tests/Dockerfile.x86_64 @@ -5,7 +5,7 @@ FROM nvidia/cuda:12.3.2-devel-ubuntu22.04 ARG CUDA_SAMPLES_VERSION=v12.3 WORKDIR / -COPY *.cu *.h *.sh *.go / +COPY *.cu *.h *.sh *.go *.cc / ENV PATH=$PATH:/usr/local/nvidia/bin:/bin/nvidia/bin RUN export DEBIAN_FRONTEND=noninteractive; \ apt-get update && \ @@ -38,6 +38,7 @@ RUN export DEBIAN_FRONTEND=noninteractive; \ go install \ github.com/TheZoraiz/ascii-image-converter@d05a757c5e02ab23e97b6f6fca4e1fbeb10ab559 && \ mv "$HOME/go/bin/ascii-image-converter" /usr/bin/ && \ + gcc -o /unsupported_ioctl /unsupported_ioctl.cc && \ go build -o /run_sample /run_sample.go # Override entrypoint to nothing, otherwise all invocations will have diff --git a/images/gpu/cuda-tests/unsupported_ioctl.cc b/images/gpu/cuda-tests/unsupported_ioctl.cc new file mode 100644 index 000000000..e2f81b267 --- /dev/null +++ b/images/gpu/cuda-tests/unsupported_ioctl.cc @@ -0,0 +1,28 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +// This test makes a non-existing ioctl call to the nvidia driver. +// It's used to test that ioctl_sniffer is catching unsupported ioctls. +int main() { + int fd = open("/dev/nvidiactl", O_RDWR); + if (fd < 0) { + return 1; + } + + ioctl(fd, 0, nullptr); + return 0; +} diff --git a/test/gpu/BUILD b/test/gpu/BUILD index b097d48b0..7f3558a57 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -102,3 +102,22 @@ go_test( visibility = ["//:sandbox"], deps = ["//test/gpu/stablediffusion"], ) + +go_test( + name = "sniffer_test", + srcs = ["sniffer_test.go"], + data = [ + "//tools/ioctl_sniffer:run_sniffer", + ], + tags = [ + "manual", + "noguitar", + "notap", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//pkg/test/testutil", + "@com_github_docker_docker//api/types/mount:go_default_library", + ], +) diff --git a/test/gpu/sniffer_test.go b/test/gpu/sniffer_test.go new file mode 100644 index 000000000..b79fa9605 --- /dev/null +++ b/test/gpu/sniffer_test.go @@ -0,0 +1,85 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sniffer_test tests the ioctl_sniffer against simple cuda workloads. +package sniffer_test + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types/mount" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +const maxDuration = 1 * time.Minute + +// RunCommand runs the given command via the sniffer, with the -enforce_compatibility flag. +// +// It's run in a docker container, with the cuda-tests image. +func runCommand(t *testing.T, cmd ...string) (string, error) { + // Find the sniffer binary + cliPath, err := testutil.FindFile("tools/ioctl_sniffer/run_sniffer") + if err != nil { + t.Fatalf("Failed to find run_sniffer: %v", err) + } + + // Set up our docker container + ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, errors.New("overall test timed out")) + defer cancel() + + listContainer := dockerutil.MakeContainer(ctx, t) + defer listContainer.CleanUp(ctx) + + // Mount the sniffer binary into the container + opts := dockerutil.GPURunOpts() + opts.Image = "gpu/cuda-tests" + opts.Mounts = append(opts.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: cliPath, + Target: "/run_sniffer", + ReadOnly: false, + }) + + command := append([]string{"/run_sniffer", "-enforce_compatibility", "-verbose"}, cmd...) + output, err := listContainer.Run(ctx, opts, command...) + return output, err +} + +func TestSupportedCUDAProgram(t *testing.T) { + output, err := runCommand(t, "/run_sample", "0_Introduction/vectorAdd") + t.Logf("%s", output) + if err != nil { + t.Logf("Error: %v", err) + if strings.Contains(output, "unsupported ioctls found") { + t.Fatalf("'unsupported ioctls found' found in output") + } + t.Fatalf("Failed to run vectorAdd") + } +} + +func TestUnsupportedCUDAProgram(t *testing.T) { + output, err := runCommand(t, "/unsupported_ioctl") + t.Logf("%s", output) + if err == nil { + t.Fatalf("Expected run_sniffer to fail") + } + if !strings.Contains(output, "unsupported ioctls found") { + t.Fatalf("Expected to find 'unsupported ioctls found' in output") + } +} diff --git a/tools/ioctl_sniffer/BUILD b/tools/ioctl_sniffer/BUILD index 7b06a640c..c67ecb783 100644 --- a/tools/ioctl_sniffer/BUILD +++ b/tools/ioctl_sniffer/BUILD @@ -45,6 +45,10 @@ go_binary( ":ioctl_hook", # keep ], static = True, + visibility = [ + "//test:__subpackages__", + "//tools/ioctl_sniffer:__subpackages__", + ], deps = [ "//pkg/log", "//tools/ioctl_sniffer/sniffer", diff --git a/tools/ioctl_sniffer/run_sniffer.go b/tools/ioctl_sniffer/run_sniffer.go index 1c91e2412..2c9cfa95a 100644 --- a/tools/ioctl_sniffer/run_sniffer.go +++ b/tools/ioctl_sniffer/run_sniffer.go @@ -27,6 +27,9 @@ import ( _ "embed" // Necessary to use go:embed. ) +var enforceCompatability = flag.Bool("enforce_compatibility", false, "If true, the sniffer will fail if it detects an unsupported ioctl.") +var verbose = flag.Bool("verbose", false, "If true, the sniffer will print all Nvidia ioctls it sees.") + //go:embed libioctl_hook.so var ioctlHookSharedObject []byte @@ -56,6 +59,10 @@ func Main() error { return fmt.Errorf("no command specified") } + if *verbose { + log.SetLevel(log.Debug) + } + // Init our sniffer if err := sniffer.Init(); err != nil { return fmt.Errorf("failed to init sniffer: %w", err) @@ -95,7 +102,12 @@ func Main() error { w.Close() results := sniffer.ReadHookOutput(r) + if *enforceCompatability && results.HasUnsupportedIoctl() { + return fmt.Errorf("unsupported ioctls found: %v", results) + } + // Once we've read all the output, print the list of missing ioctls. + log.Infof("============== Unsupported ioctls ==============") log.Infof("%s", results) if err := cmd.Wait(); err != nil { diff --git a/tools/ioctl_sniffer/sniffer/sniffer.go b/tools/ioctl_sniffer/sniffer/sniffer.go index 259fafc44..60548abf7 100644 --- a/tools/ioctl_sniffer/sniffer/sniffer.go +++ b/tools/ioctl_sniffer/sniffer/sniffer.go @@ -93,6 +93,11 @@ func (i Ioctl) IsSupported() bool { _, ok := suppUvmIoctls[uint32(i.nr)] return ok case control: + // Legacy ioctls are a special case where nvproxy passes them through unconditionally, + // since they are undocumented and unlikely to contain problematic arguments. + if i.cmd&nvgpu.RM_GSS_LEGACY_MASK != 0 { + return true + } _, ok := suppControlCmds[uint32(i.cmd)] return ok case alloc: @@ -165,7 +170,6 @@ func (r *Results) String() string { // each time is fine. b := new(strings.Builder) - fmt.Fprintln(b, "============== Unsupported ioctls ==============") printIoctls(b, frontend, r.unsupportedOther[frontend]) printIoctls(b, uvm, r.unsupportedOther[uvm]) printIoctls(b, control, r.unsupportedControl) @@ -175,6 +179,11 @@ func (r *Results) String() string { return b.String() } +// HasUnsupportedIoctl returns true if there are any unsupported ioctls. +func (r *Results) HasUnsupportedIoctl() bool { + return len(r.unsupportedControl) != 0 || len(r.unsupportedAlloc) != 0 || len(r.unsupportedOther) != 0 +} + // Init reads from nvproxy and sets up the supported ioctl maps. func Init() error { nvproxy.Init()