runsc: Only mount NVIDIA devices and register driver if GPUs are requested.

Prior to this change, setting `runsc` with `--nvproxy` and `--nvproxy-docker`
caused the NVIDIA control files to be available in all sandboxes' filesystems,
rather than only those which request GPU access.

With this change, these devices are only mounted when requested for that
sandbox, and the `nvproxy` driver isn't registered.

This also has the advantage of not requiring the existence of these device
files on the host when starting a non-GPU sandbox, which should help for
possibly enabling this feature on by default and have `runsc` still work on
machines without GPUs.

The downside is that the spec has to be read twice, due to the boot ordering
of `runsc`. `specutils.ReadSpecFromFile` already seeks to offset 0 when
reading a spec file, so this should be OK.

```shell
# On a machine without nvidia drivers loaded and no device files:
$ docker run --rm --runtime=runsc debian sh -c 'echo /dev/nvidia*'
/dev/nvidia*

# Same machine, now requesting GPUs:
$ docker run --rm --runtime=runsc --gpus=all debian sh -c 'echo /dev/nvidia*'
/dev/nvidia-uvm /dev/nvidia0 /dev/nvidiactl

# (As a side-effect of the previous container starting, kernel modules are
# now loaded and /dev/nvidia0 exists on the host)

# Same machine, but container no longer requesting GPUs:
$ docker run --rm --runtime=runsc debian sh -c 'echo /dev/nvidia*'
/dev/nvidia*

# More tests with `--gpus` flag:

$ docker run --rm --runtime=runsc --gpus=device=all debian sh -c 'echo /dev/nvidia*'
/dev/nvidia-uvm /dev/nvidia0 /dev/nvidiactl
$ docker run --rm --runtime=runsc --gpus=device=none debian sh -c 'echo /dev/nvidia*'
/dev/nvidia-uvm /dev/nvidiactl
$ docker run --rm --runtime=runsc --gpus=device=void debian sh -c 'echo /dev/nvidia*'
/dev/nvidia*
$ docker run --rm --runtime=runsc --gpus=device=0 debian sh -c 'echo /dev/nvidia*'
/dev/nvidia-uvm /dev/nvidia0 /dev/nvidiactl
$ docker run --rm --runtime=runsc --gpus=device=1 debian sh -c 'echo /dev/nvidia*'
docker: Error response from daemon: failed to create task for container: failed to create shim task: OCI runtime create failed: creating container: nvidia-container-cli configure failed, err: exit status 1
stdout:
stderr: nvidia-container-cli: device error: 1: unknown device: unknown.
$ docker run --rm --runtime=runsc debian sh -c 'echo /dev/nvidia*'
/dev/nvidia*
```
PiperOrigin-RevId: 542098578

runsc/boot/vfs.go

@@ -1108,7 +1108,7 @@ func createDeviceFiles(ctx context.Context, creds *auth.Credentials, info *conta
 }
 
 func nvproxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerInfo, k *kernel.Kernel, vfsObj *vfs.VirtualFilesystem, a *devtmpfs.Accessor) error {
-	if !info.conf.NVProxy {
+	if !specutils.GPUFunctionalityRequested(info.spec, info.conf) {
 		return nil
 	}
 	uvmDevMajor, err := k.VFS().GetDynamicCharDevMajor()
@@ -1119,14 +1119,17 @@ func nvproxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerIn
 		return fmt.Errorf("registering nvproxy driver: %w", err)
 	}
 	info.nvidiaUVMDevMajor = uvmDevMajor
-	if specutils.HaveNvidiaVisibleDevices(info.spec, info.conf) {
+	if info.conf.NVProxyDocker {
+		// In Docker mode, create all the device files now.
+		// In non-Docker mode, these are instead created as part of
+		// `createDeviceFiles`, using the spec's Device list.
+		nvd, err := specutils.NvidiaDeviceNumbers(info.spec, info.conf)
+		if err != nil {
+			return fmt.Errorf("getting nvidia devices: %w", err)
+		}
 		if err := nvproxy.CreateDriverDevtmpfsFiles(ctx, a, uvmDevMajor); err != nil {
 			return fmt.Errorf("creating nvproxy devtmpfs files: %w", err)
 		}
-		nvd, err := specutils.NvidiaVisibleDevices(info.spec, info.conf)
-		if err != nil {
-			return fmt.Errorf("getting NVIDIA_VISIBLE_DEVICES: %w", err)
-		}
 		for _, d := range nvd {
 			if err := nvproxy.CreateIndexDevtmpfsFile(ctx, a, d); err != nil {
 				return fmt.Errorf("creating nvproxy devtmpfs file for device %d: %w", d, err)

runsc/cmd/boot.go

@@ -243,8 +243,17 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		syncUsernsForRootless(b.syncUsernsFD)
 	}
 
+	// Get the spec from the specFD. We *must* keep this os.File alive past
+	// the call setCapsAndCallSelf, otherwise the FD will be closed and the
+	// child process cannot read it
+	specFile := os.NewFile(uintptr(b.specFD), "spec file")
+	spec, err := specutils.ReadSpecFromFile(b.bundleDir, specFile, conf)
+	if err != nil {
+		util.Fatalf("reading spec: %v", err)
+	}
+
 	if b.setUpRoot {
-		if err := setUpChroot(b.pidns, conf); err != nil {
+		if err := setUpChroot(b.pidns, spec, conf); err != nil {
 			util.Fatalf("error setting up chroot: %v", err)
 		}
 
@@ -280,14 +289,6 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		}
 	}
 
-	// Get the spec from the specFD. We *must* keep this os.File alive past
-	// the call setCapsAndCallSelf, otherwise the FD will be closed and the
-	// child process cannot read it
-	specFile := os.NewFile(uintptr(b.specFD), "spec file")
-	spec, err := specutils.ReadSpecFromFile(b.bundleDir, specFile, conf)
-	if err != nil {
-		util.Fatalf("reading spec: %v", err)
-	}
 	specutils.LogSpecDebug(spec, conf.OCISeccomp)
 
 	if b.applyCaps {

runsc/cmd/chroot.go

@@ -19,8 +19,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/runsc/config"
@@ -81,7 +81,7 @@ func copyFile(dst, src string) error {
 
 // setUpChroot creates an empty directory with runsc mounted at /runsc and proc
 // mounted at /proc.
-func setUpChroot(pidns bool, conf *config.Config) error {
+func setUpChroot(pidns bool, spec *specs.Spec, conf *config.Config) error {
 	// We are a new mount namespace, so we can use /tmp as a directory to
 	// construct a new root.
 	chroot := os.TempDir()
@@ -117,7 +117,7 @@ func setUpChroot(pidns bool, conf *config.Config) error {
 		}
 	}
 
-	if err := nvproxyUpdateChroot(chroot, conf); err != nil {
+	if err := nvproxyUpdateChroot(chroot, spec, conf); err != nil {
 		return fmt.Errorf("error configuring chroot for Nvidia GPUs: %w", err)
 	}
 
@@ -128,8 +128,8 @@ func setUpChroot(pidns bool, conf *config.Config) error {
 	return pivotRoot(chroot)
 }
 
-func nvproxyUpdateChroot(chroot string, conf *config.Config) error {
-	if !conf.NVProxy {
+func nvproxyUpdateChroot(chroot string, spec *specs.Spec, conf *config.Config) error {
+	if !specutils.GPUFunctionalityRequested(spec, conf) {
 		return nil
 	}
 	if err := os.Mkdir(filepath.Join(chroot, "dev"), 0755); err != nil && !errors.Is(err, os.ErrExist) {
@@ -141,18 +141,14 @@ func nvproxyUpdateChroot(chroot string, conf *config.Config) error {
 	if err := mountInChroot(chroot, "/dev/nvidia-uvm", "/dev/nvidia-uvm", "bind", unix.MS_BIND); err != nil {
 		return fmt.Errorf("error mounting /dev/nvidia-uvm in chroot: %w", err)
 	}
-	// We must bind-mount all available GPUs because in the Kubernetes case,
-	// the set of usable GPUs isn't known until time of subcontainer creation.
-	paths, err := filepath.Glob("/dev/nvidia*")
+	deviceIDs, err := specutils.NvidiaDeviceNumbers(spec, conf)
 	if err != nil {
-		return fmt.Errorf("enumerating Nvidia device files: %w", err)
+		return fmt.Errorf("enumerating nvidia device IDs: %w", err)
 	}
-	re := regexp.MustCompile(`^/dev/nvidia\d+$`)
-	for _, path := range paths {
-		if re.MatchString(path) {
-			if err := mountInChroot(chroot, path, path, "bind", unix.MS_BIND); err != nil {
-				return fmt.Errorf("error mounting %q in chroot: %v", path, err)
-			}
+	for _, deviceID := range deviceIDs {
+		path := fmt.Sprintf("/dev/nvidia%d", deviceID)
+		if err := mountInChroot(chroot, path, path, "bind", unix.MS_BIND); err != nil {
+			return fmt.Errorf("error mounting %q in chroot: %v", path, err)
 		}
 	}
 	return nil

runsc/container/container.go

@@ -1667,7 +1667,7 @@ func logIDMappings(mappings []specs.LinuxIDMapping, idType string) {
 }
 
 func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error {
-	if !specutils.HaveNvidiaVisibleDevices(spec, conf) {
+	if !specutils.GPUFunctionalityRequested(spec, conf) || !conf.NVProxyDocker {
 		return nil
 	}
 
@@ -1710,12 +1710,45 @@ func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error
 		ldconfigPath = "/sbin/ldconfig"
 	}
 
+	// nvidia-container-cli --load-kmods seems to be a noop; load kernel modules ourselves.
+	nvproxyLoadKernelModules()
+
+	// Run `nvidia-container-cli info`.
+	// This has the side-effect of automatically creating GPU device files.
+	argv := []string{cliPath, "--load-kmods", "info"}
+	log.Debugf("Executing %q", argv)
+	var infoOut, infoErr strings.Builder
+	cmd := exec.Cmd{
+		Path:   argv[0],
+		Args:   argv,
+		Env:    os.Environ(),
+		Stdout: &infoOut,
+		Stderr: &infoErr,
+	}
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("nvidia-container-cli info failed, err: %v\nstdout: %s\nstderr: %s", err, infoOut.String(), infoErr.String())
+	}
+	log.Debugf("nvidia-container-cli info: %v", infoOut.String())
+
+	deviceIDs, err := specutils.NvidiaDeviceNumbers(spec, conf)
+	if err != nil {
+		return fmt.Errorf("failed to get nvidia device numbers: %w", err)
+	}
+
 	// nvidia-container-cli does not create this directory.
 	if err := os.MkdirAll(path.Join(spec.Root.Path, "proc", "driver", "nvidia"), 0555); err != nil {
 		return fmt.Errorf("failed to create /proc/driver/nvidia in app filesystem: %w", err)
 	}
 
-	argv := []string{
+	var nvidiaDevices strings.Builder
+	for i, deviceID := range deviceIDs {
+		if i > 0 {
+			nvidiaDevices.WriteRune(',')
+		}
+		nvidiaDevices.WriteString(fmt.Sprintf("%d", uint32(deviceID)))
+	}
+
+	argv = []string{
 		cliPath,
 		"--load-kmods",
 		"configure",
@@ -1724,11 +1757,12 @@ func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error
 		"--utility",
 		"--compute",
 		fmt.Sprintf("--pid=%d", os.Getpid()),
+		fmt.Sprintf("--device=%s", nvidiaDevices.String()),
 		spec.Root.Path,
 	}
 	log.Debugf("Executing %q", argv)
 	var stdout, stderr strings.Builder
-	cmd := exec.Cmd{
+	cmd = exec.Cmd{
 		Path:   argv[0],
 		Args:   argv,
 		Env:    os.Environ(),
@@ -1736,7 +1770,34 @@ func nvproxyUpdateAppRootFilesystem(spec *specs.Spec, conf *config.Config) error
 		Stderr: &stderr,
 	}
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("nvidia-container-cli failed, err: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String())
+		return fmt.Errorf("nvidia-container-cli configure failed, err: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String())
 	}
 	return nil
 }
+
+// nvproxyLoadKernelModules loads NVIDIA-related kernel modules with modprobe.
+func nvproxyLoadKernelModules() {
+	for _, mod := range [...]string{
+		"nvidia",
+		"nvidia-uvm",
+	} {
+		argv := []string{
+			"/sbin/modprobe",
+			mod,
+		}
+		log.Debugf("Executing %q", argv)
+		var stdout, stderr strings.Builder
+		cmd := exec.Cmd{
+			Path:   argv[0],
+			Args:   argv,
+			Env:    os.Environ(),
+			Stdout: &stdout,
+			Stderr: &stderr,
+		}
+		if err := cmd.Run(); err != nil {
+			// This might not be fatal since modules may already be loaded. Log
+			// the failure but continue.
+			log.Warningf("modprobe %s failed, err: %v\nstdout: %s\nstderr: %s", mod, err, stdout.String(), stderr.String())
+		}
+	}
+}

runsc/specutils/nvidia.go

@@ -22,49 +22,96 @@ import (
 	"strings"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/runsc/config"
 )
 
 const nvdEnvVar = "NVIDIA_VISIBLE_DEVICES"
 
-// HaveNvidiaVisibleDevices returns true if the NVIDIA_VISIBLE_DEVICES
-// environment variable for the specified container enables Nvidia GPU usage.
-func HaveNvidiaVisibleDevices(spec *specs.Spec, conf *config.Config) bool {
-	if !conf.NVProxy || !conf.NVProxyDocker || spec.Process == nil {
+// GPUFunctionalityRequested returns true if the user intends for the sandbox
+// to have access to GPU functionality (e.g. access to /dev/nvidiactl),
+// irrespective of whether or not they want access to any specific GPU.
+func GPUFunctionalityRequested(spec *specs.Spec, conf *config.Config) bool {
+	if !conf.NVProxy {
+		// nvproxy disabled.
+		return false
+	}
+	if !conf.NVProxyDocker {
+		// nvproxy enabled in non-Docker mode.
+		return true
+	}
+	// nvproxy enabled in Docker mode.
+	// GPU access is only requested if NVIDIA_VISIBLE_DEVICES is non-empty
+	// and set to a value that doesn't mean "no GPU".
+	if spec.Process == nil {
 		return false
 	}
 	nvd, _ := EnvVar(spec.Process.Env, nvdEnvVar)
+	// A value of "none" means "no GPU device, but still access to driver
+	// functionality", so it is not a value we check for here.
 	return nvd != "" && nvd != "void"
 }
 
-// NvidiaVisibleDevices returns the Nvidia GPU device minor numbers enabled by
-// the NVIDIA_VISIBLE_DEVICES environment variable for the specified container.
-func NvidiaVisibleDevices(spec *specs.Spec, conf *config.Config) ([]uint32, error) {
-	if !conf.NVProxy || !conf.NVProxyDocker || spec.Process == nil {
-		return nil, nil
+// CanAccessAtLeastOneGPU returns true if the sandbox and container should
+// be able to access at least one Nvidia GPU. This is a function of the
+// sandbox configuration and the container spec's NVIDIA_VISIBLE_DEVICES
+// environment variable.
+func CanAccessAtLeastOneGPU(spec *specs.Spec, conf *config.Config) bool {
+	gpus, err := NvidiaDeviceNumbers(spec, conf)
+	if err != nil {
+		log.Warningf("Cannot determine if the container should have access to GPUs: %v", err)
+		return false
 	}
-	nvd, _ := EnvVar(spec.Process.Env, nvdEnvVar)
-	if nvd == "" || nvd == "void" || nvd == "none" {
-		return nil, nil
+	return len(gpus) > 0
+}
+
+// nvidiaDeviceRegex matches Nvidia GPU device paths.
+var nvidiaDeviceRegex = regexp.MustCompile(`^/dev/nvidia(\d+)$`)
+
+// findAllGPUDevices returns the Nvidia GPU device minor numbers of all GPUs
+// on the machine.
+func findAllGPUDevices() ([]uint32, error) {
+	paths, err := filepath.Glob("/dev/nvidia*")
+	if err != nil {
+		return nil, fmt.Errorf("enumerating Nvidia device files: %w", err)
 	}
 	var devMinors []uint32
-	if nvd == "all" {
-		paths, err := filepath.Glob("/dev/nvidia*")
-		if err != nil {
-			return nil, fmt.Errorf("enumerating Nvidia device files: %w", err)
-		}
-		re := regexp.MustCompile(`^/dev/nvidia(\d+)$`)
-		for _, path := range paths {
-			if ms := re.FindStringSubmatch(path); ms != nil {
-				index, err := strconv.ParseUint(ms[1], 10, 32)
-				if err != nil {
-					return nil, fmt.Errorf("invalid host device file %q: %w", path, err)
-				}
-				devMinors = append(devMinors, uint32(index))
+	for _, path := range paths {
+		if ms := nvidiaDeviceRegex.FindStringSubmatch(path); ms != nil {
+			index, err := strconv.ParseUint(ms[1], 10, 32)
+			if err != nil {
+				return nil, fmt.Errorf("invalid host device file %q: %w", path, err)
 			}
+			devMinors = append(devMinors, uint32(index))
 		}
-		return devMinors, nil
 	}
+	return devMinors, nil
+}
+
+// NvidiaDeviceNumbers returns the Nvidia GPU device minor numbers that
+// should be visible to the specified container.
+// In Docker mode, this is the set of devices specified in
+// NVIDIA_VISIBLE_DEVICES.
+// In non-Docker mode, this is all Nvidia devices, as we cannot know the set
+// of usable GPUs until subcontainer creation.
+func NvidiaDeviceNumbers(spec *specs.Spec, conf *config.Config) ([]uint32, error) {
+	if !GPUFunctionalityRequested(spec, conf) {
+		return nil, nil
+	}
+	if !conf.NVProxyDocker {
+		// nvproxy enabled in non-Docker mode.
+		// Return all GPUs on the machine.
+		return findAllGPUDevices()
+	}
+	// nvproxy is enabled in Docker mode.
+	nvd, _ := EnvVar(spec.Process.Env, nvdEnvVar)
+	if nvd == "none" {
+		return nil, nil
+	}
+	if nvd == "all" {
+		return findAllGPUDevices()
+	}
+	var devMinors []uint32
 	// Expect nvd to be a list of indices; UUIDs aren't supported
 	// yet.
 	for _, indexStr := range strings.Split(nvd, ",") {