From 8758e992c5ca892be1830534a62cc85298061016 Mon Sep 17 00:00:00 2001
From: Eric Zhang <ekzhang1@gmail.com>
Date: Thu, 27 Jul 2023 03:06:01 +0000
Subject: [PATCH] perf: Remove call to nvidia-container-cli info

This call is repeated on every container startup sequence, but it is expensive, consistently taking between 2-3 seconds to run each time. This adds a significant amount of startup latency to every container. From `strace` output, the main issue appears to be a call to `openat(AT_FDCWD, "/dev/nvidia0", O_RDWR|O_CLOEXEC)`. The first time this is called, it creates a new open device file descriptor and blocks for 2 seconds while doing so. This was measured on a `g4dn.2xlarge` instance on AWS, running a Tesla T4 GPU.

For comparison, the `nvidia-container-prestart-hook` for runc also runs `nvidia-container-cli` during container boot, but it only calls it once for the `configure` command. gVisor appears to call it twice, once for `info` and then for `configure`.

By removing the `info` call, or at least only running it when the GPU device files are not already present on the host, GPU container startups can be 2-3 seconds faster.

Does this sound reasonable? I'm not actually familiar with why the `/dev/nvidia0` file takes 2-3 seconds to open, or why GPU device files need to be explicitly loaded. But I tested the change, and gVisor nvproxy still works.
---
 runsc/container/container.go | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/runsc/container/container.go b/runsc/container/container.go
index 6d0cacb15..9937ebff5 100644
--- a/runsc/container/container.go
+++ b/runsc/container/container.go
@@ -1700,22 +1700,28 @@ func nvProxyPreGoferHostSetup(spec *specs.Spec, conf *config.Config) error {
 	// nvidia-container-cli --load-kmods seems to be a noop; load kernel modules ourselves.
 	nvproxyLoadKernelModules()
 
-	// Run `nvidia-container-cli info`.
-	// This has the side-effect of automatically creating GPU device files.
-	argv := []string{cliPath, "--load-kmods", "info"}
-	log.Debugf("Executing %q", argv)
-	var infoOut, infoErr strings.Builder
-	cmd := exec.Cmd{
-		Path:   argv[0],
-		Args:   argv,
-		Env:    os.Environ(),
-		Stdout: &infoOut,
-		Stderr: &infoErr,
+	if _, err := os.Stat("/dev/nvidiactl"); err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("stat(2) for /dev/nvidiactl failed: %w", err)
+		}
+
+		// Run `nvidia-container-cli info`.
+		// This has the side-effect of automatically creating GPU device files.
+		argv := []string{cliPath, "--load-kmods", "info"}
+		log.Debugf("Executing %q", argv)
+		var infoOut, infoErr strings.Builder
+		cmd := exec.Cmd{
+			Path:   argv[0],
+			Args:   argv,
+			Env:    os.Environ(),
+			Stdout: &infoOut,
+			Stderr: &infoErr,
+		}
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("nvidia-container-cli info failed, err: %v\nstdout: %s\nstderr: %s", err, infoOut.String(), infoErr.String())
+		}
+		log.Debugf("nvidia-container-cli info: %v", infoOut.String())
 	}
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("nvidia-container-cli info failed, err: %v\nstdout: %s\nstderr: %s", err, infoOut.String(), infoErr.String())
-	}
-	log.Debugf("nvidia-container-cli info: %v", infoOut.String())
 
 	return nil
 }