diff --git a/pkg/test/dockerutil/gpu.go b/pkg/test/dockerutil/gpu.go index 910b030f1..0da70c8ca 100644 --- a/pkg/test/dockerutil/gpu.go +++ b/pkg/test/dockerutil/gpu.go @@ -126,6 +126,7 @@ func GPURunOpts(sniffGPUOpts SniffGPUOpts) (RunOpts, error) { Type: mount.TypeBind, ReadOnly: true, }) + break } } for _, nvidiaLib64 := range []string{ @@ -139,6 +140,8 @@ func GPURunOpts(sniffGPUOpts SniffGPUOpts) (RunOpts, error) { Type: mount.TypeBind, ReadOnly: true, }) + sniffGPUOpts.addLDPath = "/usr/local/nvidia/lib64" + break } } @@ -166,6 +169,10 @@ type SniffGPUOpts struct { // If unset, defaults to `DefaultGPUCapabilities`. Capabilities string + // If set, add the given directory to the ld cache. + // Must be a directory visible from within the container. + addLDPath string + // The fields below are set internally. runSniffer *os.File } @@ -191,6 +198,9 @@ func (sgo *SniffGPUOpts) prepend(argv []string) []string { if !sgo.AllowIncompatibleIoctl { snifferArgv = append(snifferArgv, "--enforce_compatibility=INSTANT") } + if sgo.addLDPath != "" { + snifferArgv = append(snifferArgv, fmt.Sprintf("--add_ld_path=%s", sgo.addLDPath)) + } return append(snifferArgv, argv...) } diff --git a/tools/ioctl_sniffer/BUILD b/tools/ioctl_sniffer/BUILD index 130adf0ad..acefa4d80 100644 --- a/tools/ioctl_sniffer/BUILD +++ b/tools/ioctl_sniffer/BUILD @@ -34,7 +34,10 @@ cc_binary( go_binary( name = "run_sniffer", - srcs = ["run_sniffer.go"], + srcs = [ + "ld.go", + "run_sniffer.go", + ], embedsrcs = [ # The 'keep' comment is needed to prevent glaze from removing this # dependency. This is because the `:ioctl_hook` `cc_binary` rule diff --git a/tools/ioctl_sniffer/ld.go b/tools/ioctl_sniffer/ld.go new file mode 100644 index 000000000..0270d8978 --- /dev/null +++ b/tools/ioctl_sniffer/ld.go @@ -0,0 +1,38 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "os/exec" +) + +// addPathToLd adds the given path to the ld cache. +func addPathToLd(ctx context.Context, path string) error { + const myLdConfigPath = "/etc/ld.so.conf.d/gvisor.conf" + if err := os.WriteFile(myLdConfigPath, []byte(fmt.Sprintf("# Generated by gVisor ioctl sniffer\n%s", path)), 0644); err != nil { + return fmt.Errorf("failed to write to ld config file %q: %w", myLdConfigPath, err) + } + if err := os.Remove("/etc/ld.so.cache"); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove ld cache file: %w", err) + } + output, err := exec.CommandContext(ctx, "ldconfig").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to run ldconfig: %w; output: %s", err, string(output)) + } + return nil +} diff --git a/tools/ioctl_sniffer/run_sniffer.go b/tools/ioctl_sniffer/run_sniffer.go index 1a0327b89..54c9f8a1c 100644 --- a/tools/ioctl_sniffer/run_sniffer.go +++ b/tools/ioctl_sniffer/run_sniffer.go @@ -28,8 +28,11 @@ import ( _ "embed" // Necessary to use go:embed. ) -var enforceCompatibility = flag.String("enforce_compatibility", "", "May be set to 'INSTANT' or 'REPORT'. If set, the sniffer will return a non-zero error code if it detects an unsupported ioctl. 'INSTANT' causes the sniffer to exit immediately when this happens. 'REPORT' causes the sniffer to report all unsupported ioctls at the end of execution.") -var verbose = flag.Bool("verbose", false, "If true, the sniffer will print all Nvidia ioctls it sees.") +var ( + enforceCompatibility = flag.String("enforce_compatibility", "", "May be set to 'INSTANT' or 'REPORT'. If set, the sniffer will return a non-zero error code if it detects an unsupported ioctl. 'INSTANT' causes the sniffer to exit immediately when this happens. 'REPORT' causes the sniffer to report all unsupported ioctls at the end of execution.") + verbose = flag.Bool("verbose", false, "If true, the sniffer will print all Nvidia ioctls it sees.") + addLdPath = flag.String("add_ld_path", "", "If set, reconfigure the ld cache to include the given directory") +) //go:embed libioctl_hook.so var ioctlHookSharedObject []byte @@ -68,6 +71,12 @@ func Main(ctx context.Context) error { log.SetLevel(log.Debug) } + if *addLdPath != "" { + if err := addPathToLd(ctx, *addLdPath); err != nil { + return fmt.Errorf("failed to add path %q to ld: %w", *addLdPath, err) + } + } + // Init our sniffer if err := sniffer.Init(); err != nil { return fmt.Errorf("failed to init sniffer: %w", err)