From d299b3998cb25a3c8176f378ef7861a099eb2ea6 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Wed, 16 Oct 2024 15:00:04 -0700 Subject: [PATCH] Sniffer: Exit instantly on unknown ioctls in compatibility enforcement mode. This makes it easier to deal with client/server-type GPU tests, such as ollama or vLLM, where the main GPU process is a long-running one. Prior to this CL, when setting enforcement mode on this process, this would only be checked if adding explicit code to shut down the container orderly by sending a signal to the server process, in order to let the ioctl sniffer have a chance to produce its report. This is easy to miss, because there is no feedback that would suggest that the ioctl sniffer isn't being respected in such tests. By exiting instantly as soon as an unsupported ioctl is found, such code doesn't need to be added. However, the previous behavior is still useful when testing new applications, so it is still available as well. The `--enforce_compatibility` flag is changed to a tri-state flag that can handle either being turned off, or enforcing compatibility on the spot vs at exit time. Updates issue #10885. PiperOrigin-RevId: 686653876 --- pkg/test/dockerutil/gpu.go | 5 +++-- tools/ioctl_sniffer/run_sniffer.go | 14 ++++++++++---- tools/ioctl_sniffer/sniffer/sniffer.go | 11 ++++++++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/test/dockerutil/gpu.go b/pkg/test/dockerutil/gpu.go index 6419cb08c..9c46bb204 100644 --- a/pkg/test/dockerutil/gpu.go +++ b/pkg/test/dockerutil/gpu.go @@ -187,10 +187,11 @@ func (sgo *SniffGPUOpts) prepend(argv []string) []string { } snifferArgv := []string{ ioctlSnifferMountPath, - "--verbose=true", - fmt.Sprintf("--enforce_compatibility=%t", !sgo.AllowIncompatibleIoctl), // TODO(eperot): Add flag to enforce capability set here once implemented. } + if !sgo.AllowIncompatibleIoctl { + snifferArgv = append(snifferArgv, "--enforce_compatibility=INSTANT") + } return append(snifferArgv, argv...) } diff --git a/tools/ioctl_sniffer/run_sniffer.go b/tools/ioctl_sniffer/run_sniffer.go index 0d5d8dd52..d0dd56692 100644 --- a/tools/ioctl_sniffer/run_sniffer.go +++ b/tools/ioctl_sniffer/run_sniffer.go @@ -28,7 +28,7 @@ import ( _ "embed" // Necessary to use go:embed. ) -var enforceCompatability = flag.Bool("enforce_compatibility", false, "If true, the sniffer will fail if it detects an unsupported ioctl.") +var enforceCompatibility = flag.String("enforce_compatibility", "", "May be set to 'INSTANT' or 'REPORT'. If set, the sniffer will return a non-zero error code if it detects an unsupported ioctl. 'INSTANT' causes the sniffer to exit immediately when this happens. 'REPORT' causes the sniffer to report all unsupported ioctls at the end of execution.") var verbose = flag.Bool("verbose", false, "If true, the sniffer will print all Nvidia ioctls it sees.") //go:embed libioctl_hook.so @@ -60,6 +60,10 @@ func Main(ctx context.Context) error { return fmt.Errorf("no command specified") } + if *enforceCompatibility != "" && *enforceCompatibility != "INSTANT" && *enforceCompatibility != "REPORT" { + return fmt.Errorf("invalid value for --enforce_compatibility: %q", *enforceCompatibility) + } + if *verbose { log.SetLevel(log.Debug) } @@ -100,8 +104,10 @@ func Main(ctx context.Context) error { cmd.Stderr = os.Stderr // Refer to the hook file by file descriptor here as its named file no // longer exists. - cmd.Env = append(os.Environ(), fmt.Sprintf("LD_PRELOAD=/proc/%d/fd/%d", os.Getpid(), hookFile.Fd())) - cmd.Env = append(cmd.Env, fmt.Sprintf("GVISOR_IOCTL_SNIFFER_SOCKET_PATH=%v", server.Addr())) + cmd.Env = append(os.Environ(), + fmt.Sprintf("LD_PRELOAD=/proc/%d/fd/%d", os.Getpid(), hookFile.Fd()), + fmt.Sprintf("GVISOR_IOCTL_SNIFFER_SOCKET_PATH=%v", server.Addr()), + fmt.Sprintf("GVISOR_IOCTL_SNIFFER_ENFORCE_COMPATIBILITY=%s", *enforceCompatibility)) // Run the command and start reading the output. if err := cmd.Start(); err != nil { @@ -115,7 +121,7 @@ func Main(ctx context.Context) error { // Merge results from each connection. finalResults := server.AllResults() - if *enforceCompatability && finalResults.HasUnsupportedIoctl() { + if *enforceCompatibility != "" && finalResults.HasUnsupportedIoctl() { return fmt.Errorf("unsupported ioctls found: %v", finalResults) } diff --git a/tools/ioctl_sniffer/sniffer/sniffer.go b/tools/ioctl_sniffer/sniffer/sniffer.go index 642c68611..a99e6dc0a 100644 --- a/tools/ioctl_sniffer/sniffer/sniffer.go +++ b/tools/ioctl_sniffer/sniffer/sniffer.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io" + "os" "regexp" "strings" @@ -71,7 +72,8 @@ func (c ioctlClass) String() string { type ioctlSubclass uint32 var ( - supportedIoctls [_numClasses]map[uint32]struct{} + supportedIoctls [_numClasses]map[uint32]struct{} + crashOnUnsupportedIoctl bool ) // Ioctl contains the parsed ioctl protobuf information. @@ -199,6 +201,9 @@ func Init() error { alloc: suppAllocClasses, unknown: make(map[uint32]struct{}), } + if os.Getenv("GVISOR_IOCTL_SNIFFER_ENFORCE_COMPATIBILITY") == "INSTANT" { + crashOnUnsupportedIoctl = true + } return nil } @@ -226,6 +231,10 @@ func (c Connection) ReadHookOutput(ctx context.Context) *Results { if !ioctl.IsSupported() { res.AddUnsupportedIoctl(ioctl) + if crashOnUnsupportedIoctl { + log.Warningf("Unsupported ioctl found; crashing immediately: %v", ioctl) + os.Exit(1) + } } } return res