Add simple functionality tests for ioctl_sniffer.

This also adds two new flags, --enforce_compatibility and --verbose, and
fixes an issue where legacy control ioctls were being flagged as unsupported.

PiperOrigin-RevId: 645193498

images/gpu/cuda-tests/Dockerfile.x86_64

@@ -5,7 +5,7 @@ FROM nvidia/cuda:12.3.2-devel-ubuntu22.04
 ARG CUDA_SAMPLES_VERSION=v12.3
 
 WORKDIR /
-COPY *.cu *.h *.sh *.go /
+COPY *.cu *.h *.sh *.go *.cc /
 ENV PATH=$PATH:/usr/local/nvidia/bin:/bin/nvidia/bin
 RUN export DEBIAN_FRONTEND=noninteractive; \
     apt-get update && \
@@ -38,6 +38,7 @@ RUN export DEBIAN_FRONTEND=noninteractive; \
     go install \
       github.com/TheZoraiz/ascii-image-converter@d05a757c5e02ab23e97b6f6fca4e1fbeb10ab559 && \
     mv "$HOME/go/bin/ascii-image-converter" /usr/bin/ && \
+    gcc -o /unsupported_ioctl /unsupported_ioctl.cc && \
     go build -o /run_sample /run_sample.go
 
 # Override entrypoint to nothing, otherwise all invocations will have

images/gpu/cuda-tests/unsupported_ioctl.cc

@@ -0,0 +1,28 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+
+// This test makes a non-existing ioctl call to the nvidia driver.
+//  It's used to test that ioctl_sniffer is catching unsupported ioctls.
+int main() {
+  int fd = open("/dev/nvidiactl", O_RDWR);
+  if (fd < 0) {
+    return 1;
+  }
+
+  ioctl(fd, 0, nullptr);
+  return 0;
+}

test/gpu/BUILD

@@ -102,3 +102,22 @@ go_test(
     visibility = ["//:sandbox"],
     deps = ["//test/gpu/stablediffusion"],
 )
+
+go_test(
+    name = "sniffer_test",
+    srcs = ["sniffer_test.go"],
+    data = [
+        "//tools/ioctl_sniffer:run_sniffer",
+    ],
+    tags = [
+        "manual",
+        "noguitar",
+        "notap",
+    ],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//pkg/test/testutil",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
+    ],
+)

test/gpu/sniffer_test.go

@@ -0,0 +1,85 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package sniffer_test tests the ioctl_sniffer against simple cuda workloads.
+package sniffer_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types/mount"
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/pkg/test/testutil"
+)
+
+const maxDuration = 1 * time.Minute
+
+// RunCommand runs the given command via the sniffer, with the -enforce_compatibility flag.
+//
+//	It's run in a docker container, with the cuda-tests image.
+func runCommand(t *testing.T, cmd ...string) (string, error) {
+	// Find the sniffer binary
+	cliPath, err := testutil.FindFile("tools/ioctl_sniffer/run_sniffer")
+	if err != nil {
+		t.Fatalf("Failed to find run_sniffer: %v", err)
+	}
+
+	// Set up our docker container
+	ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, errors.New("overall test timed out"))
+	defer cancel()
+
+	listContainer := dockerutil.MakeContainer(ctx, t)
+	defer listContainer.CleanUp(ctx)
+
+	// Mount the sniffer binary into the container
+	opts := dockerutil.GPURunOpts()
+	opts.Image = "gpu/cuda-tests"
+	opts.Mounts = append(opts.Mounts, mount.Mount{
+		Type:     mount.TypeBind,
+		Source:   cliPath,
+		Target:   "/run_sniffer",
+		ReadOnly: false,
+	})
+
+	command := append([]string{"/run_sniffer", "-enforce_compatibility", "-verbose"}, cmd...)
+	output, err := listContainer.Run(ctx, opts, command...)
+	return output, err
+}
+
+func TestSupportedCUDAProgram(t *testing.T) {
+	output, err := runCommand(t, "/run_sample", "0_Introduction/vectorAdd")
+	t.Logf("%s", output)
+	if err != nil {
+		t.Logf("Error: %v", err)
+		if strings.Contains(output, "unsupported ioctls found") {
+			t.Fatalf("'unsupported ioctls found' found in output")
+		}
+		t.Fatalf("Failed to run vectorAdd")
+	}
+}
+
+func TestUnsupportedCUDAProgram(t *testing.T) {
+	output, err := runCommand(t, "/unsupported_ioctl")
+	t.Logf("%s", output)
+	if err == nil {
+		t.Fatalf("Expected run_sniffer to fail")
+	}
+	if !strings.Contains(output, "unsupported ioctls found") {
+		t.Fatalf("Expected to find 'unsupported ioctls found' in output")
+	}
+}

tools/ioctl_sniffer/BUILD

@@ -45,6 +45,10 @@ go_binary(
         ":ioctl_hook",  # keep
     ],
     static = True,
+    visibility = [
+        "//test:__subpackages__",
+        "//tools/ioctl_sniffer:__subpackages__",
+    ],
     deps = [
         "//pkg/log",
         "//tools/ioctl_sniffer/sniffer",

tools/ioctl_sniffer/run_sniffer.go

@@ -27,6 +27,9 @@ import (
 	_ "embed" // Necessary to use go:embed.
 )
 
+var enforceCompatability = flag.Bool("enforce_compatibility", false, "If true, the sniffer will fail if it detects an unsupported ioctl.")
+var verbose = flag.Bool("verbose", false, "If true, the sniffer will print all Nvidia ioctls it sees.")
+
 //go:embed libioctl_hook.so
 var ioctlHookSharedObject []byte
 
@@ -56,6 +59,10 @@ func Main() error {
 		return fmt.Errorf("no command specified")
 	}
 
+	if *verbose {
+		log.SetLevel(log.Debug)
+	}
+
 	// Init our sniffer
 	if err := sniffer.Init(); err != nil {
 		return fmt.Errorf("failed to init sniffer: %w", err)
@@ -95,7 +102,12 @@ func Main() error {
 	w.Close()
 	results := sniffer.ReadHookOutput(r)
 
+	if *enforceCompatability && results.HasUnsupportedIoctl() {
+		return fmt.Errorf("unsupported ioctls found: %v", results)
+	}
+
 	// Once we've read all the output, print the list of missing ioctls.
+	log.Infof("============== Unsupported ioctls ==============")
 	log.Infof("%s", results)
 
 	if err := cmd.Wait(); err != nil {

tools/ioctl_sniffer/sniffer/sniffer.go

@@ -93,6 +93,11 @@ func (i Ioctl) IsSupported() bool {
 		_, ok := suppUvmIoctls[uint32(i.nr)]
 		return ok
 	case control:
+		// Legacy ioctls are a special case where nvproxy passes them through unconditionally,
+		// since they are undocumented and unlikely to contain problematic arguments.
+		if i.cmd&nvgpu.RM_GSS_LEGACY_MASK != 0 {
+			return true
+		}
 		_, ok := suppControlCmds[uint32(i.cmd)]
 		return ok
 	case alloc:
@@ -165,7 +170,6 @@ func (r *Results) String() string {
 	// each time is fine.
 	b := new(strings.Builder)
 
-	fmt.Fprintln(b, "============== Unsupported ioctls ==============")
 	printIoctls(b, frontend, r.unsupportedOther[frontend])
 	printIoctls(b, uvm, r.unsupportedOther[uvm])
 	printIoctls(b, control, r.unsupportedControl)
@@ -175,6 +179,11 @@ func (r *Results) String() string {
 	return b.String()
 }
 
+// HasUnsupportedIoctl returns true if there are any unsupported ioctls.
+func (r *Results) HasUnsupportedIoctl() bool {
+	return len(r.unsupportedControl) != 0 || len(r.unsupportedAlloc) != 0 || len(r.unsupportedOther) != 0
+}
+
 // Init reads from nvproxy and sets up the supported ioctl maps.
 func Init() error {
 	nvproxy.Init()