From 906fb319cc3afdd7ee8f6917a3a0636bcf7d1afd Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Wed, 12 Mar 2025 17:04:52 -0700 Subject: [PATCH] nvproxy: Add option to use the device gofer optionally. We always use the device gofer in runsc, because the sandbox's filesystem does not have the GPU devices mounted in it. PiperOrigin-RevId: 736316547 --- pkg/sentry/devices/nvproxy/frontend.go | 44 ++++++++++++++++---------- pkg/sentry/devices/nvproxy/nvproxy.go | 4 ++- pkg/sentry/devices/nvproxy/uvm.go | 42 ++++++++++++++---------- runsc/boot/vfs.go | 2 +- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/pkg/sentry/devices/nvproxy/frontend.go b/pkg/sentry/devices/nvproxy/frontend.go index 9baf69e89..dee0eef1c 100644 --- a/pkg/sentry/devices/nvproxy/frontend.go +++ b/pkg/sentry/devices/nvproxy/frontend.go @@ -16,6 +16,7 @@ package nvproxy import ( "fmt" + "path/filepath" "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" @@ -55,32 +56,43 @@ func (dev *frontendDevice) basename() string { // Open implements vfs.Device.Open. func (dev *frontendDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { - devClient := devutil.GoferClientFromContext(ctx) - if devClient == nil { - log.Warningf("devutil.CtxDevGoferClient is not set") - return nil, linuxerr.ENOENT + fd := &frontendFD{ + dev: dev, } basename := dev.basename() - hostFD, err := devClient.OpenAt(ctx, basename, opts.Flags) - if err != nil { - ctx.Warningf("nvproxy: failed to open host %s: %v", basename, err) - return nil, err - } - fd := &frontendFD{ - dev: dev, - containerName: devClient.ContainerName(), - hostFD: int32(hostFD), + if dev.nvp.useDevGofer { + devClient := devutil.GoferClientFromContext(ctx) + if devClient == nil { + log.Warningf("devutil.CtxDevGoferClient is not set") + return nil, linuxerr.ENOENT + } + fd.containerName = devClient.ContainerName() + hostFD, err := devClient.OpenAt(ctx, basename, opts.Flags) + if err != nil { + ctx.Warningf("nvproxy: failed to open %s: %v", basename, err) + return nil, err + } + fd.hostFD = int32(hostFD) + } else { + devPath := filepath.Join("/dev", basename) + flags := int(opts.Flags&unix.O_ACCMODE | unix.O_NOFOLLOW) + hostFD, err := unix.Openat(-1, devPath, flags, 0) + if err != nil { + ctx.Warningf("nvproxy: failed to open host %s: %v", devPath, err) + return nil, err + } + fd.hostFD = int32(hostFD) } if err := fd.vfsfd.Init(fd, opts.Flags, mnt, vfsd, &vfs.FileDescriptionOptions{ UseDentryMetadata: true, }); err != nil { - unix.Close(hostFD) + unix.Close(int(fd.hostFD)) return nil, err } fd.internalEntry.Init(fd, waiter.AllEvents) fd.internalQueue.EventRegister(&fd.internalEntry) - if err := fdnotifier.AddFD(int32(hostFD), &fd.internalQueue); err != nil { - unix.Close(hostFD) + if err := fdnotifier.AddFD(fd.hostFD, &fd.internalQueue); err != nil { + unix.Close(int(fd.hostFD)) return nil, err } fd.memmapFile.fd = fd diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go index 14c9de7b5..5c11bfcc4 100644 --- a/pkg/sentry/devices/nvproxy/nvproxy.go +++ b/pkg/sentry/devices/nvproxy/nvproxy.go @@ -30,7 +30,7 @@ import ( ) // Register registers all devices implemented by this package in vfsObj. -func Register(vfsObj *vfs.VirtualFilesystem, version nvconf.DriverVersion, driverCaps nvconf.DriverCaps, uvmDevMajor uint32) error { +func Register(vfsObj *vfs.VirtualFilesystem, version nvconf.DriverVersion, driverCaps nvconf.DriverCaps, uvmDevMajor uint32, useDevGofer bool) error { // The kernel driver's interface is unstable, so only allow versions of the // driver that are known to be supported. log.Infof("NVIDIA driver version: %s", version) @@ -45,6 +45,7 @@ func Register(vfsObj *vfs.VirtualFilesystem, version nvconf.DriverVersion, drive abi: abiCons.cons(), version: version, capsEnabled: driverCaps, + useDevGofer: useDevGofer, frontendFDs: make(map[*frontendFD]struct{}), clients: make(map[nvgpu.Handle]*rootClient), objsFreeSet: make(map[*object]struct{}), @@ -74,6 +75,7 @@ type nvproxy struct { abi *driverABI `state:"nosave"` version nvconf.DriverVersion capsEnabled nvconf.DriverCaps + useDevGofer bool fdsMu fdsMutex `state:"nosave"` frontendFDs map[*frontendFD]struct{} diff --git a/pkg/sentry/devices/nvproxy/uvm.go b/pkg/sentry/devices/nvproxy/uvm.go index 2871672a6..8606430d9 100644 --- a/pkg/sentry/devices/nvproxy/uvm.go +++ b/pkg/sentry/devices/nvproxy/uvm.go @@ -41,29 +41,39 @@ type uvmDevice struct { // Open implements vfs.Device.Open. func (dev *uvmDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { - devClient := devutil.GoferClientFromContext(ctx) - if devClient == nil { - log.Warningf("devutil.CtxDevGoferClient is not set") - return nil, linuxerr.ENOENT - } - hostFD, err := devClient.OpenAt(ctx, "nvidia-uvm", opts.Flags) - if err != nil { - ctx.Warningf("nvproxy: failed to open host /dev/nvidia-uvm: %v", err) - return nil, err - } fd := &uvmFD{ - dev: dev, - containerName: devClient.ContainerName(), - hostFD: int32(hostFD), + dev: dev, + } + if dev.nvp.useDevGofer { + devClient := devutil.GoferClientFromContext(ctx) + if devClient == nil { + log.Warningf("devutil.CtxDevGoferClient is not set") + return nil, linuxerr.ENOENT + } + fd.containerName = devClient.ContainerName() + hostFD, err := devClient.OpenAt(ctx, "nvidia-uvm", opts.Flags) + if err != nil { + ctx.Warningf("nvproxy: failed to open nvidia-uvm: %v", err) + return nil, err + } + fd.hostFD = int32(hostFD) + } else { + flags := int(opts.Flags&unix.O_ACCMODE | unix.O_NOFOLLOW) + hostFD, err := unix.Openat(-1, "/dev/nvidia-uvm", flags, 0) + if err != nil { + ctx.Warningf("nvproxy: failed to open host /dev/nvidia-uvm: %v", err) + return nil, err + } + fd.hostFD = int32(hostFD) } if err := fd.vfsfd.Init(fd, opts.Flags, mnt, vfsd, &vfs.FileDescriptionOptions{ UseDentryMetadata: true, }); err != nil { - unix.Close(hostFD) + unix.Close(int(fd.hostFD)) return nil, err } - if err := fdnotifier.AddFD(int32(hostFD), &fd.queue); err != nil { - unix.Close(hostFD) + if err := fdnotifier.AddFD(fd.hostFD, &fd.queue); err != nil { + unix.Close(int(fd.hostFD)) return nil, err } fd.memmapFile.fd = fd diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 9cc6f2268..bdfc31ff9 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -1401,7 +1401,7 @@ func nvproxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem) if err != nil { return fmt.Errorf("reserving device major number for nvidia-uvm: %w", err) } - if err := nvproxy.Register(vfsObj, info.nvidiaDriverVersion, driverCaps, uvmDevMajor); err != nil { + if err := nvproxy.Register(vfsObj, info.nvidiaDriverVersion, driverCaps, uvmDevMajor, true /* useDevGofer */); err != nil { return fmt.Errorf("registering nvproxy driver: %w", err) } info.nvidiaUVMDevMajor = uvmDevMajor