Run nvidia-container-cli configure in the Gofer mount namespace.

This change adds a new synchronization FD to the Gofer startup sequence, passed as `sync-nvproxy-fd`. The `runsc create` process uses this to wait for the Gofer to start, then to run `nvidia-container-cli configure [...] --pid=$GOFER_PID`. This causes the mounts that `nvidia-container-cli configure` does to be performed in the mount namespace of the Gofer, rather than the `runsc create` process. This avoids polluting the main mount namespace with NVIDIA-specific mountpoints, and ties the lifetime of these mounts to the lifetime of the Gofer process, which means they are cleaned up automatically when the sandbox exits. Due to the added complexity in Gofer startup, this CL also introduces a `goferSyncFDs` struct that encodes some of the logic around these FDs, and better documents how they interact with the Gofer and the container startup sequence. One suggestion by Ayush was to move `nvidia-container-cli` to be done after `createGoferProcess` returns. Unfortunately this isn't possible without having to return the nvproxy FD in the `createGoferProcess` return signature, since FDs can only be donated before the Gofer process has started. This would make the signature uglier. So instead, this CL takes the approach of a single `nvproxyConfigureGofer` function called during Gofer initialization. It creates and donates the FD to the Gofer command, and returns a callback function called after the Gofer process is started, where it finally runs `nvidia-container-cli configure` and then notifies the Gofer through this same FD it created. This encapsulates all the logic within `nvproxyConfigureGofer` and is the cleanest I could think of. Tested manually using a fresh Debian machine, with: ```shell $ sudo mkdir -p /tmp/bundle-cuda/rootfs $ docker export $(docker create nvidia/cuda:11.6.2-base-ubuntu20.04) \ | sudo tar -xf - -C /tmp/bundle-cuda/rootfs $ sudo runc spec --bundle=/tmp/bundle-cuda $ $EDITOR /tmp/bundle-cuda/config.json # Add NVIDIA_VISIBLE_DEVICES=0 and NVIDIA_DRIVER_CAPABILITIES=all to env $ sudo ./runsc -nvproxy -nvproxy-docker create --bundle=/tmp/bundle-cuda mycuda $ sudo ./runsc -nvproxy -nvproxy-docker start mycuda $ sudo ./runsc -nvproxy -nvproxy-docker exec mycuda nvidia-smi -L (Works) $ sudo ./runsc delete --force mycuda # And verified at each step that `grep bundle-cuda /proc/mounts` was empty. ``` And also verified that regular use through Docker also works. Fixes #9142. PiperOrigin-RevId: 549427158
2026-05-22 17:12:49 -07:00 · 2023-07-19 14:33:08 -07:00
parent 2212760449
commit a455fbd2a7
4 changed files with 273 additions and 132 deletions
@@ -243,9 +243,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		}
 	}

-	if b.syncUsernsFD >= 0 {
-		syncUsernsForRootless(b.syncUsernsFD)
-	}
+	syncUsernsForRootless(b.syncUsernsFD)

 	// Get the spec from the specFD. We *must* keep this os.File alive past
 	// the call setCapsAndCallSelf, otherwise the FD will be closed and the
@@ -334,7 +332,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 	}

 	if b.syncUsernsFD >= 0 {
-		// syncUsernsFD is set, but runsc hasn't been re-exeuted with a new UID and GID.
+		// syncUsernsFD is set, but runsc hasn't been re-executed with a new UID and GID.
 		// We expect that setCapsAndCallSelf has to be called in this case.
 		panic("unreachable")
 	}
@@ -517,7 +515,7 @@ func execProcUmounter() (*exec.Cmd, *os.File) {
 // umountProc writes to syncFD signalling the process started by
 // execProcUmounter() to umount /proc.
 func umountProc(syncFD int) {
-	syncFile := os.NewFile(uintptr(syncFD), "sync file")
+	syncFile := os.NewFile(uintptr(syncFD), "procfs umount sync FD")
 	buf := make([]byte, 1)
 	if w, err := syncFile.Write(buf); err != nil || w != 1 {
 		util.Fatalf("unable to write into the proc umounter descriptor: %v", err)
@@ -57,6 +57,26 @@ var goferCaps = &specs.LinuxCapabilities{
 	Permitted: caps,
 }

+// goferSyncFDs contains file descriptors that are used for synchronization
+// of the Gofer startup process against other processes.
+type goferSyncFDs struct {
+	// nvproxyFD is a file descriptor that is used to wait until
+	// nvproxy-related setup is done. This setup involves creating mounts in the
+	// Gofer process's mount namespace.
+	// If this is set, this FD is the first that the Gofer waits for.
+	nvproxyFD int
+	// usernsFD is a file descriptor that is used to wait until
+	// user namespace ID mappings are established in the Gofer's userns.
+	// If this is set, this FD is the second that the Gofer waits for.
+	usernsFD int
+	// procMountFD is a file descriptor that has to be closed when the
+	// procfs mount isn't needed anymore. It is read by the procfs unmounter
+	// process.
+	// If this is set, this FD is the last that the Gofer interacts with and
+	// closes.
+	procMountFD int
+}
+
 // Gofer implements subcommands.Command for the "gofer" command, which starts a
 // filesystem gofer.  This command should not be called directly.
 type Gofer struct {
@@ -66,14 +86,10 @@ type Gofer struct {
 	setUpRoot      bool
 	overlayMediums boot.OverlayMediumFlags

-	specFD       int
-	mountsFD     int
-	syncUsernsFD int
-	// procMountSyncFD is a file descriptor that has to be closed when the
-	// procfs mount isn't needed anymore.
-	procMountSyncFD int
-
+	specFD        int
+	mountsFD      int
 	profileFDs    profile.FDArgs
+	syncFDs       goferSyncFDs
 	stopProfiling func()
 }

@@ -103,8 +119,9 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) {
 	f.Var(&g.overlayMediums, "overlay-mediums", "information about how the gofer mounts have been overlaid.")
 	f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).")
-	f.IntVar(&g.syncUsernsFD, "sync-userns-fd", -1, "file descriptor used to synchronize rootless user namespace initialization.")
-	f.IntVar(&g.procMountSyncFD, "proc-mount-sync-fd", -1, "file descriptor that has to be written to when /proc isn't needed anymore and can be unmounted")
+
+	// Add synchronization FD flags.
+	g.syncFDs.setFlags(f)

 	// Profiling flags.
 	g.profileFDs.SetFromFlags(f)
@@ -129,47 +146,32 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm
 		util.Fatalf("reading spec: %v", err)
 	}

-	if g.syncUsernsFD >= 0 {
-		syncUsernsForRootless(g.syncUsernsFD)
-	}
+	g.syncFDs.syncNVProxy()
+	g.syncFDs.syncUsernsForRootless()

 	if g.setUpRoot {
 		if err := g.setupRootFS(spec, conf); err != nil {
 			util.Fatalf("Error setting up root FS: %v", err)
 		}
 		if !conf.TestOnlyAllowRunAsCurrentUserWithoutChroot {
-			// /proc is umounted from a forked process, because the
-			// current one may re-execute itself without capabilities.
-			cmd, w := execProcUmounter()
-			defer cmd.Wait()
-			defer w.Close()
-			if g.procMountSyncFD != -1 {
-				panic("procMountSyncFD is set")
-			}
-			g.procMountSyncFD = int(w.Fd())
-
-			// Clear FD_CLOEXEC. This process may be re-executed. procMountSyncFD
-			// should remain open.
-			if _, _, errno := unix.RawSyscall(unix.SYS_FCNTL, w.Fd(), unix.F_SETFD, 0); errno != 0 {
-				util.Fatalf("error clearing CLOEXEC: %v", errno)
-			}
+			cleanupUnmounter := g.syncFDs.spawnProcUnmounter()
+			defer cleanupUnmounter()
 		}
 	}
 	if g.applyCaps {
 		// Disable caps when calling myself again.
 		// Note: minimal argument handling for the default case to keep it simple.
 		args := os.Args
-		args = append(args, "--apply-caps=false", "--setup-root=false", "--sync-userns-fd=-1", fmt.Sprintf("--proc-mount-sync-fd=%d", g.procMountSyncFD))
+		args = append(
+			args,
+			"--apply-caps=false",
+			"--setup-root=false",
+		)
+		args = append(args, g.syncFDs.flags()...)
 		util.Fatalf("setCapsAndCallSelf(%v, %v): %v", args, goferCaps, setCapsAndCallSelf(args, goferCaps))
 		panic("unreachable")
 	}

-	if g.syncUsernsFD >= 0 {
-		// syncUsernsFD is set, but runsc hasn't been re-exeuted with a new UID and GID.
-		// We expect that setCapsAndCallSelf has to be called in this case.
-		panic("unreachable")
-	}
-
 	// Start profiling. This will be a noop if no profiling arguments were passed.
 	profileOpts := g.profileFDs.ToOpts()
 	g.stopProfiling = profile.Start(profileOpts)
@@ -222,10 +224,9 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm
 	if err := fsgofer.OpenProcSelfFD(); err != nil {
 		util.Fatalf("failed to open /proc/self/fd: %v", err)
 	}
-	if g.procMountSyncFD != -1 {
-		// procfs isn't needed anymore.
-		umountProc(g.procMountSyncFD)
-	}
+
+	// procfs isn't needed anymore.
+	g.syncFDs.unmountProcfs()

 	if err := unix.Chroot(root); err != nil {
 		util.Fatalf("failed to chroot to %q: %v", root, err)
@@ -583,19 +584,98 @@ func adjustMountOptions(conf *config.Config, path string, opts []string) ([]stri
 	return rv, nil
 }

-// syncUsernsForRootless waits on syncUsernsFD to be closed and then sets
-// UID/GID to 0. Note that this function calls runtime.LockOSThread().
-//
-// Postcondition: All callers must re-exec themselves after this returns.
-func syncUsernsForRootless(syncUsernsFD int) {
-	f := os.NewFile(uintptr(syncUsernsFD), "sync FD")
+// setFlags sets sync FD flags on the given FlagSet.
+func (g *goferSyncFDs) setFlags(f *flag.FlagSet) {
+	f.IntVar(&g.nvproxyFD, "sync-nvproxy-fd", -1, "file descriptor that the gofer waits on until nvproxy setup is done")
+	f.IntVar(&g.usernsFD, "sync-userns-fd", -1, "file descriptor the the gofer waits on until userns mappings are set up")
+	f.IntVar(&g.procMountFD, "proc-mount-sync-fd", -1, "file descriptor that the gofer writes to when /proc isn't needed anymore and can be unmounted")
+}
+
+// flags returns the flags necessary to pass along the current sync FD values
+// to a re-executed version of this process.
+func (g *goferSyncFDs) flags() []string {
+	return []string{
+		fmt.Sprintf("--sync-nvproxy-fd=%d", g.nvproxyFD),
+		fmt.Sprintf("--sync-userns-fd=%d", g.usernsFD),
+		fmt.Sprintf("--proc-mount-sync-fd=%d", g.procMountFD),
+	}
+}
+
+// waitForFD waits for the other end of a given FD to be closed.
+// `fd` is closed unconditionally after that.
+// This should only be called for actual FDs (i.e. `fd` >= 0).
+func waitForFD(fd int, fdName string) error {
+	log.Debugf("Waiting on %s %d...", fdName, fd)
+	f := os.NewFile(uintptr(fd), fdName)
 	defer f.Close()
 	var b [1]byte
 	if n, err := f.Read(b[:]); n != 0 || err != io.EOF {
-		util.Fatalf("failed to sync: %v: %v", n, err)
+		return fmt.Errorf("failed to sync on %s: %v: %v", fdName, n, err)
+	}
+	log.Debugf("Synced on %s %d.", fdName, fd)
+	return nil
+}
+
+// spawnProcMounter executes the /proc unmounter process.
+// It returns a function to wait on the proc unmounter process, which
+// should be called (via defer) in case of errors in order to clean up the
+// unmounter process properly.
+// When procfs is no longer needed, `unmountProcfs` should be called.
+func (g *goferSyncFDs) spawnProcUnmounter() func() {
+	if g.procMountFD != -1 {
+		util.Fatalf("procMountFD is set")
+	}
+	// /proc is umounted from a forked process, because the
+	// current one may re-execute itself without capabilities.
+	cmd, w := execProcUmounter()
+	// Clear FD_CLOEXEC. This process may be re-executed. procMountFD
+	// should remain open.
+	if _, _, errno := unix.RawSyscall(unix.SYS_FCNTL, w.Fd(), unix.F_SETFD, 0); errno != 0 {
+		util.Fatalf("error clearing CLOEXEC: %v", errno)
+	}
+	g.procMountFD = int(w.Fd())
+	return func() {
+		g.procMountFD = -1
+		w.Close()
+		cmd.Wait()
+	}
+}
+
+// unmountProcfs signals the proc unmounter process that procfs is no longer
+// needed.
+func (g *goferSyncFDs) unmountProcfs() {
+	if g.procMountFD < 0 {
+		return
+	}
+	umountProc(g.procMountFD)
+	g.procMountFD = -1
+}
+
+// syncUsernsForRootless waits on usernsFD to be closed and then sets
+// UID/GID to 0. Note that this function calls runtime.LockOSThread().
+// This function is a no-op if usernsFD is -1.
+//
+// Postcondition: All callers must re-exec themselves after this returns,
+// unless usernsFD was -1.
+func (g *goferSyncFDs) syncUsernsForRootless() {
+	syncUsernsForRootless(g.usernsFD)
+	g.usernsFD = -1
+}
+
+// syncUsernsForRootless waits on usernsFD to be closed and then sets
+// UID/GID to 0. Note that this function calls runtime.LockOSThread().
+// This function is a no-op if usernsFD is -1.
+//
+// Postcondition: All callers must re-exec themselves after this returns,
+// unless fd is -1.
+func syncUsernsForRootless(fd int) {
+	if fd < 0 {
+		return
+	}
+	if err := waitForFD(fd, "userns sync FD"); err != nil {
+		util.Fatalf("failed to sync on userns FD: %v", err)
 	}

-	f.Close()
 	// SETUID changes UID on the current system thread, so we have
 	// to re-execute current binary.
 	runtime.LockOSThread()
@@ -606,3 +686,17 @@ func syncUsernsForRootless(syncUsernsFD int) {
 		util.Fatalf("failed to set GID: %v", errno)
 	}
 }
+
+// syncNVProxy waits on nvproxyFD to be closed.
+// Used for synchronization during nvproxy setup which is done from the
+// non-gofer process.
+// This function is a no-op if nvProxySyncFD is -1.
+func (g *goferSyncFDs) syncNVProxy() {
+	if g.nvproxyFD < 0 {
+		return
+	}
+	if err := waitForFD(g.nvproxyFD, "nvproxy sync FD"); err != nil {
+		util.Fatalf("failed to sync on NVProxy FD: %v", err)
+	}
+	g.nvproxyFD = -1
+}
@@ -296,11 +296,10 @@ func New(conf *config.Config, args Args) (*Container, error) {
 			return nil, err
 		}
 		c.OverlayMediums = overlayMediums
+		if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil {
+			return nil, err
+		}
 		if err := runInCgroup(containerCgroup, func() error {
-			if err := nvproxyUpdateAppRootFilesystem(args.Spec, conf); err != nil {
-				return err
-			}
-
 			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached)
 			if err != nil {
 				return fmt.Errorf("cannot create gofer process: %w", err)
@@ -1200,6 +1199,11 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 		defer syncFile.Close()
 	}

+	nvProxySetup, err := nvproxySetupAfterGoferUserns(spec, conf, cmd, &donations)
+	if err != nil {
+		return nil, nil, fmt.Errorf("setting up nvproxy for gofer: %w", err)
+	}
+
 	donations.Transfer(cmd, nextFD)

 	// Start the gofer in the given namespace.
@@ -1208,16 +1212,22 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	if err := specutils.StartInNS(cmd, nss); err != nil {
 		return nil, nil, fmt.Errorf("gofer: %v", err)
 	}
+	log.Infof("Gofer started, PID: %d", cmd.Process.Pid)
+	c.GoferPid = cmd.Process.Pid
+	c.goferIsChild = true

+	// Set up and synchronize rootless mode userns mappings.
 	if rootlessEUID {
 		if err := sandbox.SetUserMappings(spec, cmd.Process.Pid); err != nil {
 			return nil, nil, err
 		}
 	}

-	log.Infof("Gofer started, PID: %d", cmd.Process.Pid)
-	c.GoferPid = cmd.Process.Pid
-	c.goferIsChild = true
+	// Set up nvproxy within the Gofer namespace.
+	if err := nvProxySetup(); err != nil {
+		return nil, nil, fmt.Errorf("nvproxy setup: %w", err)
+	}
+
 	return sandEnds, mountsSand, nil
 }

@@ -1666,33 +1676,17 @@ func logIDMappings(mappings []specs.LinuxIDMapping, idType string) {
 	}
 }

-func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error {
+// nvProxyPreGoferHostSetup sets up nvproxy on the host. It runs before any
+// Gofers start.
+// It verifies that all the required dependencies are in place, loads kernel
+// modules, and ensures the correct device files exist and are accessible.
+// This should only be necessary once on the host. It should be run during the
+// root container setup sequence to make sure it has run at least once.
+func nvProxyPreGoferHostSetup(spec *specs.Spec, conf *config.Config) error {
 	if !specutils.GPUFunctionalityRequested(spec, conf) || !conf.NVProxyDocker {
 		return nil
 	}

-	// Delegate to nvidia-container-cli from libnvidia-container. This
-	// essentially replicates
-	// nvidia-container-toolkit:cmd/nvidia-container-runtime-hook, i.e. the
-	// binary that executeHook() is hard-coded to skip, with differences noted
-	// inline. We do this rather than move the prestart hook because the
-	// "runtime environment" in which prestart hooks execute is vaguely
-	// defined, such that nvidia-container-runtime-hook and existing runsc
-	// hooks differ in their expected environment.
-	//
-	// Note that nvidia-container-cli will set up files in /dev and /proc which
-	// are useless, since they will be hidden by sentry devtmpfs and procfs
-	// respectively (and some device files will have the wrong device numbers
-	// from the application's perspective since nvproxy may register device
-	// numbers in sentry VFS that differ from those on the host, e.g. for
-	// nvidia-uvm). These files are separately created during sandbox VFS
-	// construction. For this reason, we don't need to parse
-	// NVIDIA_VISIBLE_DEVICES or pass --device to nvidia-container-cli.
-
-	if spec.Root == nil {
-		return fmt.Errorf("spec missing root filesystem")
-	}
-
 	// Locate binaries. For security reasons, unlike
 	// nvidia-container-runtime-hook, we don't add the container's filesystem
 	// to the search path. We also don't support
@@ -1702,13 +1696,6 @@ func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error
 	if err != nil {
 		return fmt.Errorf("failed to locate nvidia-container-cli in PATH: %w", err)
 	}
-	// On Ubuntu, ldconfig is a wrapper around ldconfig.real, and we need the latter.
-	var ldconfigPath string
-	if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
-		ldconfigPath = "/sbin/ldconfig.real"
-	} else {
-		ldconfigPath = "/sbin/ldconfig"
-	}

 	// nvidia-container-cli --load-kmods seems to be a noop; load kernel modules ourselves.
 	nvproxyLoadKernelModules()
@@ -1730,48 +1717,6 @@ func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error
 	}
 	log.Debugf("nvidia-container-cli info: %v", infoOut.String())

-	deviceIDs, err := specutils.NvidiaDeviceNumbers(spec, conf)
-	if err != nil {
-		return fmt.Errorf("failed to get nvidia device numbers: %w", err)
-	}
-
-	// nvidia-container-cli does not create this directory.
-	if err := os.MkdirAll(path.Join(spec.Root.Path, "proc", "driver", "nvidia"), 0555); err != nil {
-		return fmt.Errorf("failed to create /proc/driver/nvidia in app filesystem: %w", err)
-	}
-
-	var nvidiaDevices strings.Builder
-	for i, deviceID := range deviceIDs {
-		if i > 0 {
-			nvidiaDevices.WriteRune(',')
-		}
-		nvidiaDevices.WriteString(fmt.Sprintf("%d", uint32(deviceID)))
-	}
-
-	argv = []string{
-		cliPath,
-		"--load-kmods",
-		"configure",
-		fmt.Sprintf("--ldconfig=@%s", ldconfigPath),
-		"--no-cgroups", // runsc doesn't configure device cgroups yet
-		"--utility",
-		"--compute",
-		fmt.Sprintf("--pid=%d", os.Getpid()),
-		fmt.Sprintf("--device=%s", nvidiaDevices.String()),
-		spec.Root.Path,
-	}
-	log.Debugf("Executing %q", argv)
-	var stdout, stderr strings.Builder
-	cmd = exec.Cmd{
-		Path:   argv[0],
-		Args:   argv,
-		Env:    os.Environ(),
-		Stdout: &stdout,
-		Stderr: &stderr,
-	}
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("nvidia-container-cli configure failed, err: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String())
-	}
 	return nil
 }

@@ -1801,3 +1746,107 @@ func nvproxyLoadKernelModules() {
 		}
 	}
 }
+
+// nvproxySetupAfterGoferUserns runs `nvidia-container-cli configure`.
+// This sets up the container filesystem with bind mounts that allow it to
+// use NVIDIA devices.
+//
+// This should be called during the Gofer setup process, as the bind mounts
+// are created in the Gofer's mount namespace.
+// If successful, it returns a callback function that must be called once the
+// Gofer process has started.
+// This function has no effect if nvproxy functionality is not requested.
+//
+// This function essentially replicates
+// nvidia-container-toolkit:cmd/nvidia-container-runtime-hook, i.e. the
+// binary that executeHook() is hard-coded to skip, with differences noted
+// inline. We do this rather than move the prestart hook because the
+// "runtime environment" in which prestart hooks execute is vaguely
+// defined, such that nvidia-container-runtime-hook and existing runsc
+// hooks differ in their expected environment.
+//
+// Note that nvidia-container-cli will set up files in /dev and /proc which
+// are useless, since they will be hidden by sentry devtmpfs and procfs
+// respectively (and some device files will have the wrong device numbers
+// from the application's perspective since nvproxy may register device
+// numbers in sentry VFS that differ from those on the host, e.g. for
+// nvidia-uvm). These files are separately created during sandbox VFS
+// construction. For this reason, we don't need to parse
+// NVIDIA_VISIBLE_DEVICES or pass --device to nvidia-container-cli.
+func nvproxySetupAfterGoferUserns(spec *specs.Spec, conf *config.Config, goferCmd *exec.Cmd, goferDonations *donation.Agency) (func() error, error) {
+	if !specutils.GPUFunctionalityRequested(spec, conf) || !conf.NVProxyDocker {
+		return func() error { return nil }, nil
+	}
+
+	if spec.Root == nil {
+		return nil, fmt.Errorf("spec missing root filesystem")
+	}
+
+	// nvidia-container-cli does not create this directory.
+	if err := os.MkdirAll(path.Join(spec.Root.Path, "proc", "driver", "nvidia"), 0555); err != nil {
+		return nil, fmt.Errorf("failed to create /proc/driver/nvidia in app filesystem: %w", err)
+	}
+
+	cliPath, err := exec.LookPath("nvidia-container-cli")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate nvidia-container-cli in PATH: %w", err)
+	}
+
+	// On Ubuntu, ldconfig is a wrapper around ldconfig.real, and we need the latter.
+	var ldconfigPath string
+	if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
+		ldconfigPath = "/sbin/ldconfig.real"
+	} else {
+		ldconfigPath = "/sbin/ldconfig"
+	}
+
+	var nvidiaDevices strings.Builder
+	deviceIDs, err := specutils.NvidiaDeviceNumbers(spec, conf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get nvidia device numbers: %w", err)
+	}
+	for i, deviceID := range deviceIDs {
+		if i > 0 {
+			nvidiaDevices.WriteRune(',')
+		}
+		nvidiaDevices.WriteString(fmt.Sprintf("%d", uint32(deviceID)))
+	}
+
+	// Create synchronization FD for nvproxy.
+	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
+	if err != nil {
+		return nil, err
+	}
+	ourEnd := os.NewFile(uintptr(fds[0]), "nvproxy sync runsc FD")
+	goferEnd := os.NewFile(uintptr(fds[1]), "nvproxy sync gofer FD")
+	goferDonations.DonateAndClose("sync-nvproxy-fd", goferEnd)
+
+	return func() error {
+		defer ourEnd.Close()
+		argv := []string{
+			cliPath,
+			"--load-kmods",
+			"configure",
+			fmt.Sprintf("--ldconfig=@%s", ldconfigPath),
+			"--no-cgroups", // runsc doesn't configure device cgroups yet
+			"--utility",
+			"--compute",
+			fmt.Sprintf("--pid=%d", goferCmd.Process.Pid),
+			fmt.Sprintf("--device=%s", nvidiaDevices.String()),
+			spec.Root.Path,
+		}
+		log.Debugf("Executing %q", argv)
+		var stdout, stderr strings.Builder
+		cmd := exec.Cmd{
+			Path:   argv[0],
+			Args:   argv,
+			Env:    os.Environ(),
+			Stdout: &stdout,
+			Stderr: &stderr,
+		}
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("nvidia-container-cli configure failed, err: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String())
+		}
+		return nil
+	}, nil
+}
@@ -1584,7 +1584,7 @@ func ConfigureCmdForRootless(cmd *exec.Cmd, donations *donation.Agency) (*os.Fil
 	if err != nil {
 		return nil, err
 	}
-	f := os.NewFile(uintptr(fds[1]), "sync other FD")
+	f := os.NewFile(uintptr(fds[1]), "userns sync other FD")
 	donations.DonateAndClose("sync-userns-fd", f)
 	if cmd.SysProcAttr == nil {
 		cmd.SysProcAttr = &unix.SysProcAttr{}
@@ -1605,7 +1605,7 @@ func ConfigureCmdForRootless(cmd *exec.Cmd, donations *donation.Agency) (*os.Fil
 		// Needed to be able to clear bounding set (PR_CAPBSET_DROP).
 		unix.CAP_SETPCAP,
 	}
-	return os.NewFile(uintptr(fds[0]), "sync FD"), nil
+	return os.NewFile(uintptr(fds[0]), "userns sync FD"), nil
 }

 // SetUserMappings uses newuidmap/newgidmap programs to set up user ID mappings