Add device gofer connection.

Adds a gofer connection for /dev directory on the gofer when GPU functionality
is requested. This gofer connection is currently unused. The gofer client is
owned by the kernel, which injects the connection into the context. The gofer
connection is closed on container exit. S/R should be supported with this.

PiperOrigin-RevId: 581298536

pkg/devutil/BUILD

@@ -0,0 +1,20 @@
+load("//tools:defs.bzl", "go_library")
+
+package(default_applicable_licenses = ["//:license"])
+
+licenses(["notice"])
+
+go_library(
+    name = "devutil",
+    srcs = [
+        "context.go",
+        "devutil.go",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/context",
+        "//pkg/lisafs",
+        "//pkg/unet",
+        "@org_golang_x_sys//unix:go_default_library",
+    ],
+)

pkg/devutil/context.go

@@ -0,0 +1,33 @@
+// Copyright 2023 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devutil
+
+import "gvisor.dev/gvisor/pkg/context"
+
+// contextID is this package's type for context.Context.Value keys.
+type contextID int
+
+const (
+	// CtxDevGoferClient is a Context.Value key for a /dev gofer client.
+	CtxDevGoferClient contextID = iota
+)
+
+// GoferClientFromContext returns the device gofer client used by ctx.
+func GoferClientFromContext(ctx context.Context) *GoferClient {
+	if v := ctx.Value(CtxDevGoferClient); v != nil {
+		return v.(*GoferClient)
+	}
+	return nil
+}

pkg/devutil/devutil.go

@@ -0,0 +1,60 @@
+// Copyright 2023 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package devutil provides device specific utilities.
+package devutil
+
+import (
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/lisafs"
+	"gvisor.dev/gvisor/pkg/unet"
+)
+
+// GoferClient is the lisafs client for the /dev gofer connection.
+type GoferClient struct {
+	clientFD lisafs.ClientFD
+	hostFD   int
+}
+
+// NewGoferClient establishes the LISAFS connection to the dev gofer server.
+// It takes ownership of fd.
+func NewGoferClient(ctx context.Context, fd int) (*GoferClient, error) {
+	ctx.UninterruptibleSleepStart(false)
+	defer ctx.UninterruptibleSleepFinish(false)
+
+	sock, err := unet.NewSocket(fd)
+	if err != nil {
+		ctx.Warningf("failed to create socket for dev gofer client: %v", err)
+		return nil, err
+	}
+	client, devInode, devHostFD, err := lisafs.NewClient(sock)
+	if err != nil {
+		ctx.Warningf("failed to create dev gofer client: %v", err)
+		return nil, err
+	}
+	return &GoferClient{
+		clientFD: client.NewFD(devInode.ControlFD),
+		hostFD:   devHostFD,
+	}, nil
+}
+
+// Close closes the LISAFS connection.
+func (g *GoferClient) Close() {
+	// Close the connection to the server. This implicitly closes all FDs.
+	g.clientFD.Client().Close()
+	if g.hostFD >= 0 {
+		_ = unix.Close(g.hostFD)
+	}
+}

pkg/sentry/kernel/BUILD

@@ -326,6 +326,7 @@ go_library(
         "//pkg/context",
         "//pkg/coverage",
         "//pkg/cpuid",
+        "//pkg/devutil",
         "//pkg/errors",
         "//pkg/errors/linuxerr",
         "//pkg/eventchannel",

pkg/sentry/kernel/kernel.go

@@ -42,6 +42,7 @@ import (
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/cpuid"
+	"gvisor.dev/gvisor/pkg/devutil"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
 	"gvisor.dev/gvisor/pkg/eventchannel"
 	"gvisor.dev/gvisor/pkg/fspath"
@@ -327,6 +328,10 @@ type Kernel struct {
 	// MaxFDLimit specifies the maximum file descriptor number that can be
 	// used by processes.
 	MaxFDLimit atomicbitops.Int32
+
+	// devGofers maps container ID to its device gofer client.
+	devGofers   map[string]*devutil.GoferClient `state:"nosave"`
+	devGofersMu sync.Mutex                      `state:"nosave"`
 }
 
 // InitKernelArgs holds arguments to Init.
@@ -798,6 +803,8 @@ func (ctx *createProcessContext) Value(key any) any {
 		mntns := ctx.kernel.GlobalInit().Leader().MountNamespace()
 		mntns.IncRef()
 		return mntns
+	case devutil.CtxDevGoferClient:
+		return ctx.kernel.getDevGoferClient(ctx.args.ContainerID)
 	case inet.CtxStack:
 		return ctx.kernel.RootNetworkNamespace().Stack()
 	case ktime.CtxRealtimeClock:
@@ -1683,6 +1690,7 @@ func (k *Kernel) Release() {
 	k.timekeeper.Destroy()
 	k.vdso.Release(ctx)
 	k.RootNetworkNamespace().DecRef(ctx)
+	k.cleaupDevGofers()
 }
 
 // PopulateNewCgroupHierarchy moves all tasks into a newly created cgroup
@@ -1784,3 +1792,48 @@ func (k *Kernel) GetUserCounters(uid auth.KUID) *UserCounters {
 	k.userCountersMap[uid] = uc
 	return uc
 }
+
+// AddDevGofer initializes the dev gofer connection and starts tracking it.
+// It takes ownership of goferFD.
+func (k *Kernel) AddDevGofer(cid string, goferFD int) error {
+	client, err := devutil.NewGoferClient(k.SupervisorContext(), goferFD)
+	if err != nil {
+		return err
+	}
+
+	k.devGofersMu.Lock()
+	defer k.devGofersMu.Unlock()
+	if k.devGofers == nil {
+		k.devGofers = make(map[string]*devutil.GoferClient)
+	}
+	k.devGofers[cid] = client
+	return nil
+}
+
+// RemoveDevGofer closes the dev gofer connection, if one exists, and stops
+// tracking it.
+func (k *Kernel) RemoveDevGofer(cid string) {
+	k.devGofersMu.Lock()
+	defer k.devGofersMu.Unlock()
+	client, ok := k.devGofers[cid]
+	if !ok {
+		return
+	}
+	client.Close()
+	delete(k.devGofers, cid)
+}
+
+func (k *Kernel) getDevGoferClient(cid string) *devutil.GoferClient {
+	k.devGofersMu.Lock()
+	defer k.devGofersMu.Unlock()
+	return k.devGofers[cid]
+}
+
+func (k *Kernel) cleaupDevGofers() {
+	k.devGofersMu.Lock()
+	defer k.devGofersMu.Unlock()
+	for _, client := range k.devGofers {
+		client.Close()
+	}
+	k.devGofers = nil
+}

pkg/sentry/kernel/task_context.go

@@ -20,6 +20,7 @@ import (
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/cpuid"
+	"gvisor.dev/gvisor/pkg/devutil"
 	"gvisor.dev/gvisor/pkg/sentry/inet"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/ipc"
@@ -103,6 +104,8 @@ func (t *Task) contextValue(key any, isTaskGoroutine bool) any {
 		}
 		t.mountNamespace.IncRef()
 		return t.mountNamespace
+	case devutil.CtxDevGoferClient:
+		return t.k.getDevGoferClient(t.containerID)
 	case inet.CtxStack:
 		return t.NetworkContext()
 	case ktime.CtxRealtimeClock:

runsc/boot/controller.go

@@ -279,6 +279,9 @@ type StartArgs struct {
 	// NumGoferFilestoreFDs is the number of gofer filestore FDs donated.
 	NumGoferFilestoreFDs int
 
+	// IsDevIoFilePresent indicates whether the dev gofer FD is present.
+	IsDevIoFilePresent bool
+
 	// GoferMountConfs contains information about how the gofer mounts have been
 	// configured. The first entry is for rootfs and the following entries are
 	// for bind mounts in Spec.Mounts (in the same order).
@@ -287,6 +290,7 @@ type StartArgs struct {
 	// FilePayload contains, in order:
 	//   * stdin, stdout, and stderr (optional: if terminal is disabled).
 	//   * file descriptors to gofer-backing host files (optional).
+	//   * file descriptor for /dev gofer connection (optional)
 	//   * file descriptors to connect to gofer to serve the root filesystem.
 	urpc.FilePayload
 }
@@ -309,6 +313,9 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro
 	}
 	expectedFDs := 1 // At least one FD for the root filesystem.
 	expectedFDs += args.NumGoferFilestoreFDs
+	if args.IsDevIoFilePresent {
+		expectedFDs++
+	}
 	if !args.Spec.Process.Terminal {
 		expectedFDs += 3
 	}
@@ -352,6 +359,17 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro
 		}
 	}()
 
+	var devGoferFD *fd.FD
+	if args.IsDevIoFilePresent {
+		var err error
+		devGoferFD, err = fd.NewFromFile(goferFiles[0])
+		if err != nil {
+			return fmt.Errorf("error dup'ing dev gofer file: %w", err)
+		}
+		goferFiles = goferFiles[1:]
+		defer devGoferFD.Close()
+	}
+
 	goferFDs, err := fd.NewFromFiles(goferFiles)
 	if err != nil {
 		return fmt.Errorf("error dup'ing gofer files: %w", err)
@@ -362,7 +380,7 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro
 		}
 	}()
 
-	if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, goferFilestoreFDs, args.GoferMountConfs); err != nil {
+	if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, goferFilestoreFDs, devGoferFD, args.GoferMountConfs); err != nil {
 		log.Debugf("containerManager.StartSubcontainer failed, cid: %s, args: %+v, err: %v", args.CID, args, err)
 		return err
 	}

runsc/boot/loader.go

@@ -113,6 +113,9 @@ type containerInfo struct {
 	// goferFDs are the FDs that attach the sandbox to the gofers.
 	goferFDs []*fd.FD
 
+	// devGoferFD is the FD to attach the sandbox to the dev gofer.
+	devGoferFD *fd.FD
+
 	// goferFilestoreFDs are FDs to the regular files that will back the tmpfs or
 	// overlayfs mount for certain gofer mounts.
 	goferFilestoreFDs []*fd.FD
@@ -168,9 +171,6 @@ type Loader struct {
 	// /sys/devices/virtual/dmi/id/product_name.
 	productName string
 
-	// nvidiaUVMDevMajor is the device major number used for nvidia-uvm.
-	nvidiaUVMDevMajor uint32
-
 	// mu guards the fields below.
 	mu sync.Mutex
 
@@ -251,6 +251,9 @@ type Args struct {
 	// GoferFDs is an array of FDs used to connect with the Gofer. The Loader
 	// takes ownership of these FDs and may close them at any time.
 	GoferFDs []int
+	// DevGoferFD is the FD for the dev gofer connection. The Loader takes
+	// ownership of this FD and may close it at any time.
+	DevGoferFD int
 	// StdioFDs is the stdio for the application. The Loader takes ownership of
 	// these FDs and may close them at any time.
 	StdioFDs []int
@@ -354,7 +357,9 @@ func New(args Args) (*Loader, error) {
 	for _, filestoreFD := range args.GoferFilestoreFDs {
 		info.goferFilestoreFDs = append(info.goferFilestoreFDs, fd.New(filestoreFD))
 	}
-
+	if args.DevGoferFD >= 0 {
+		info.devGoferFD = fd.New(args.DevGoferFD)
+	}
 	if args.ExecFD >= 0 {
 		info.execFD = fd.New(args.ExecFD)
 	}
@@ -526,16 +531,15 @@ func New(args Args) (*Loader, error) {
 
 	eid := execID{cid: args.ID}
 	l := &Loader{
-		k:                 k,
-		watchdog:          dog,
-		sandboxID:         args.ID,
-		processes:         map[execID]*execProcess{eid: {}},
-		mountHints:        mountHints,
-		sharedMounts:      make(map[string]*vfs.Mount),
-		root:              info,
-		stopProfiling:     stopProfiling,
-		productName:       args.ProductName,
-		nvidiaUVMDevMajor: info.nvidiaUVMDevMajor,
+		k:             k,
+		watchdog:      dog,
+		sandboxID:     args.ID,
+		processes:     map[execID]*execProcess{eid: {}},
+		mountHints:    mountHints,
+		sharedMounts:  make(map[string]*vfs.Mount),
+		root:          info,
+		stopProfiling: stopProfiling,
+		productName:   args.ProductName,
 	}
 
 	// We don't care about child signals; some platforms can generate a
@@ -636,6 +640,9 @@ func (l *Loader) Destroy() {
 	for _, f := range l.root.goferFilestoreFDs {
 		_ = f.Close()
 	}
+	if l.root.devGoferFD != nil {
+		_ = l.root.devGoferFD.Close()
+	}
 
 	l.stopProfiling()
 }
@@ -821,7 +828,7 @@ func (l *Loader) createSubcontainer(cid string, tty *fd.FD) error {
 // startSubcontainer starts a child container. It returns the thread group ID of
 // the newly created process. Used FDs are either closed or released. It's safe
 // for the caller to close any remaining files upon return.
-func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, goferMountConfs []GoferMountConf) error {
+func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, devGoferFD *fd.FD, goferMountConfs []GoferMountConf) error {
 	// Create capabilities.
 	caps, err := specutils.Capabilities(conf.EnableRaw, spec.Process.Capabilities)
 	if err != nil {
@@ -877,9 +884,10 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 		conf:              conf,
 		spec:              spec,
 		goferFDs:          goferFDs,
+		devGoferFD:        devGoferFD,
 		goferFilestoreFDs: goferFilestoreFDs,
 		goferMountConfs:   goferMountConfs,
-		nvidiaUVMDevMajor: l.nvidiaUVMDevMajor,
+		nvidiaUVMDevMajor: l.root.nvidiaUVMDevMajor,
 		nvidiaDevMinors:   l.root.nvidiaDevMinors,
 	}
 	info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns)
@@ -901,6 +909,17 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 		info.stdioFDs = stdioFDs
 	}
 
+	var cu cleanup.Cleanup
+	defer cu.Clean()
+	if devGoferFD != nil {
+		cu.Add(func() {
+			// createContainerProcess() will consume devGoferFD and initialize a gofer
+			// connection. This connection is owned by l.k. In case of failure, we want
+			// to clean up this gofer connection so that the gofer process can exit.
+			l.k.RemoveDevGofer(cid)
+		})
+	}
+
 	ep.tg, ep.tty, err = l.createContainerProcess(cid, info)
 	if err != nil {
 		return err
@@ -927,6 +946,8 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 	}
 
 	l.k.StartProcess(ep.tg)
+	// No more failures from this point on.
+	cu.Release()
 	return nil
 }
 
@@ -966,11 +987,7 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne
 	if len(info.goferFDs) < 1 {
 		return nil, nil, fmt.Errorf("rootfs gofer FD not found")
 	}
-	// TODO(ayushranjan): The gofer monitor should be started as long as the gofer
-	// process exists, even if the root mount is not backed by lisafs.
-	if info.goferMountConfs[0].ShouldUseLisafs() {
-		l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
-	}
+	l.startGoferMonitor(cid, info)
 
 	// We can share l.sharedMounts with containerMounter since l.mu is locked.
 	// Hence, mntr must only be used within this function (while l.mu is locked).
@@ -1030,18 +1047,29 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne
 
 // startGoferMonitor runs a goroutine to monitor gofer's health. It polls on
 // the gofer FD looking for disconnects, and kills the container processes if
-// the rootfs FD disconnects.
-//
-// Note that other gofer mounts are allowed to be unmounted and disconnected.
-func (l *Loader) startGoferMonitor(cid string, rootfsGoferFD int32) {
-	if rootfsGoferFD < 0 {
-		panic(fmt.Sprintf("invalid FD: %d", rootfsGoferFD))
+// the gofer connection disconnects.
+func (l *Loader) startGoferMonitor(cid string, info *containerInfo) {
+	// We need to pick a suitable gofer connection that is expected to be alive
+	// for the entire container lifecycle. Only the following can be used:
+	// 1. Rootfs gofer connection
+	// 2. Device gofer connection
+	//
+	// Note that other gofer mounts are allowed to be unmounted and disconnected.
+	goferFD := -1
+	if info.goferMountConfs[0].ShouldUseLisafs() {
+		goferFD = info.goferFDs[0].FD()
+	} else if info.devGoferFD != nil {
+		goferFD = info.devGoferFD.FD()
+	}
+	if goferFD < 0 {
+		log.Warningf("could not find a suitable gofer FD to monitor")
+		return
 	}
 	go func() {
 		log.Debugf("Monitoring gofer health for container %q", cid)
 		events := []unix.PollFd{
 			{
-				Fd:     rootfsGoferFD,
+				Fd:     int32(goferFD),
 				Events: unix.POLLHUP | unix.POLLRDHUP,
 			},
 		}
@@ -1094,13 +1122,16 @@ func (l *Loader) destroySubcontainer(cid string) error {
 		}
 	}
 
-	// No more failure from this point on. Remove all container thread groups
-	// from the map.
+	// No more failure from this point on.
+
+	// Remove all container thread groups from the map.
 	for key := range l.processes {
 		if key.cid == cid {
 			delete(l.processes, key)
 		}
 	}
+	// Cleanup the device gofer.
+	l.k.RemoveDevGofer(cid)
 
 	log.Debugf("Container destroyed, cid: %s", cid)
 	return nil

runsc/boot/loader_test.go

@@ -139,6 +139,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error
 		Conf:            conf,
 		ControllerFD:    fd,
 		GoferFDs:        []int{sandEnd},
+		DevGoferFD:      -1,
 		StdioFDs:        stdio,
 		GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}},
 		PodInitConfigFD: -1,

runsc/boot/vfs.go

@@ -358,6 +358,9 @@ type containerMounter struct {
 	// overlayfs mount for certain gofer mounts.
 	goferFilestoreFDs fdDispenser
 
+	// devGoferFD is the FD to attach the sandbox to the dev gofer.
+	devGoferFD *fd.FD
+
 	// goferMountConfs contains information about how the gofer mounts have been
 	// configured. The first entry is for rootfs and the following entries are
 	// for bind mounts in Spec.Mounts (in the same order).
@@ -376,6 +379,9 @@ type containerMounter struct {
 	// /sys/devices/virtual/dmi/id/product_name.
 	productName string
 
+	// containerID is the ID for the container.
+	containerID string
+
 	// sandboxID is the ID for the whole sandbox.
 	sandboxID string
 }
@@ -386,11 +392,13 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountH
 		mounts:            compileMounts(info.spec, info.conf),
 		goferFDs:          fdDispenser{fds: info.goferFDs},
 		goferFilestoreFDs: fdDispenser{fds: info.goferFilestoreFDs},
+		devGoferFD:        info.devGoferFD,
 		goferMountConfs:   info.goferMountConfs,
 		k:                 k,
 		hints:             hints,
 		sharedMounts:      sharedMounts,
 		productName:       productName,
+		containerID:       info.procArgs.ContainerID,
 		sandboxID:         sandboxID,
 	}
 }
@@ -402,6 +410,9 @@ func (c *containerMounter) checkDispenser() error {
 	if !c.goferFilestoreFDs.empty() {
 		return fmt.Errorf("not all gofer Filestore FDs were consumed, remaining: %v", c.goferFilestoreFDs)
 	}
+	if c.devGoferFD != nil && c.devGoferFD.FD() >= 0 {
+		return fmt.Errorf("dev gofer FD was not consumed: %d", c.devGoferFD.FD())
+	}
 	return nil
 }
 
@@ -725,6 +736,12 @@ type mountInfo struct {
 }
 
 func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
+	// If device gofer exists, connect to it.
+	if c.devGoferFD != nil {
+		if err := c.k.AddDevGofer(c.containerID, c.devGoferFD.Release()); err != nil {
+			return nil, err
+		}
+	}
 	// Associate bind mounts with their FDs before sorting since there is an
 	// undocumented assumption that FDs are dispensed in the order in which
 	// they are required by mounts.

runsc/cmd/boot.go

@@ -81,6 +81,9 @@ type Boot struct {
 	// ioFDs is the list of FDs used to connect to FS gofers.
 	ioFDs intFlags
 
+	// devIoFD is the FD to connect to dev gofer.
+	devIoFD int
+
 	// goferFilestoreFDs are FDs to the regular files that will back the tmpfs or
 	// overlayfs mount for certain gofer mounts.
 	goferFilestoreFDs intFlags
@@ -197,6 +200,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process")
 	f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file")
 	f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
+	f.IntVar(&b.devIoFD, "dev-io-fd", -1, "FD to connect dev gofer client")
 	f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order")
 	f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.")
 	f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.")
@@ -425,6 +429,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		ControllerFD:      b.controllerFD,
 		Device:            os.NewFile(uintptr(b.deviceFD), "platform device"),
 		GoferFDs:          b.ioFDs.GetArray(),
+		DevGoferFD:        b.devIoFD,
 		StdioFDs:          b.stdioFDs.GetArray(),
 		PassFDs:           b.passFDs.GetArray(),
 		ExecFD:            b.execFD,

runsc/cmd/gofer.go

@@ -82,6 +82,7 @@ type goferSyncFDs struct {
 type Gofer struct {
 	bundleDir  string
 	ioFDs      intFlags
+	devIoFD    int
 	applyCaps  bool
 	setUpRoot  bool
 	mountConfs boot.GoferMountConfFlags
@@ -117,6 +118,7 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) {
 	// Open FDs that are donated to the gofer.
 	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.")
 	f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.")
+	f.IntVar(&g.devIoFD, "dev-io-fd", -1, "optional FD to connect /dev gofer server")
 	f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).")
 
@@ -312,6 +314,14 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs))
 	}
 
+	if g.devIoFD >= 0 {
+		cfgs = append(cfgs, connectionConfig{
+			sock:      newSocket(g.devIoFD),
+			mountPath: "/dev",
+		})
+		log.Infof("Serving /dev mapped on FD %d (ro: false)", g.devIoFD)
+	}
+
 	for _, cfg := range cfgs {
 		conn, err := server.CreateConnection(cfg.sock, cfg.mountPath, cfg.readonly)
 		if err != nil {
@@ -417,6 +427,11 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error {
 		util.Fatalf("error setting up FS: %v", err)
 	}
 
+	// Set up /dev directory is needed.
+	if g.devIoFD >= 0 {
+		g.setupDev(root)
+	}
+
 	// Create working directory if needed.
 	if spec.Process.Cwd != "" {
 		dst, err := resolveSymlinks(root, spec.Process.Cwd)
@@ -497,6 +512,13 @@ func (g *Gofer) setupMounts(conf *config.Config, mounts []specs.Mount, root, pro
 	return nil
 }
 
+func (g *Gofer) setupDev(root string) error {
+	if err := os.MkdirAll(filepath.Join(root, "dev"), 0777); err != nil {
+		return fmt.Errorf("creating dev directory: %v", err)
+	}
+	return nil
+}
+
 // resolveMounts resolved relative paths and symlinks to mount points.
 //
 // Note: mount points must already be in place for resolution to work.

runsc/container/container.go

@@ -305,7 +305,7 @@ func New(conf *config.Config, args Args) (*Container, error) {
 			return nil, err
 		}
 		if err := runInCgroup(containerCgroup, func() error {
-			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint)
+			ioFiles, devIOFile, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint)
 			if err != nil {
 				return fmt.Errorf("cannot create gofer process: %w", err)
 			}
@@ -319,6 +319,7 @@ func New(conf *config.Config, args Args) (*Container, error) {
 				ConsoleSocket:       args.ConsoleSocket,
 				UserLog:             args.UserLog,
 				IOFiles:             ioFiles,
+				DevIOFile:           devIOFile,
 				MountsFile:          specFile,
 				Cgroup:              containerCgroup,
 				Attached:            args.Attached,
@@ -471,7 +472,7 @@ func (c *Container) Start(conf *config.Config) error {
 		// the start (and all their children processes).
 		if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error {
 			// Create the gofer process.
-			goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint)
+			goferFiles, devIOFile, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint)
 			if err != nil {
 				return err
 			}
@@ -479,6 +480,9 @@ func (c *Container) Start(conf *config.Config) error {
 				if mountsFile != nil {
 					_ = mountsFile.Close()
 				}
+				if devIOFile != nil {
+					_ = devIOFile.Close()
+				}
 				for _, f := range goferFiles {
 					_ = f.Close()
 				}
@@ -502,7 +506,7 @@ func (c *Container) Start(conf *config.Config) error {
 				stdios = []*os.File{os.Stdin, os.Stdout, os.Stderr}
 			}
 
-			return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, goferConfs)
+			return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, devIOFile, goferConfs)
 		}); err != nil {
 			return err
 		}
@@ -1172,14 +1176,22 @@ func (c *Container) waitForStopped() error {
 	return backoff.Retry(op, b)
 }
 
+// shouldCreateDeviceGofer indicates whether a device gofer connection should
+// be created.
+func shouldCreateDeviceGofer(spec *specs.Spec, conf *config.Config) bool {
+	return specutils.GPUFunctionalityRequested(spec, conf)
+}
+
 // shouldSpawnGofer indicates whether the gofer process should be spawned.
-func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool {
+func shouldSpawnGofer(spec *specs.Spec, conf *config.Config, goferConfs []boot.GoferMountConf) bool {
+	// Lisafs mounts need the gofer.
 	for _, cfg := range goferConfs {
 		if cfg.ShouldUseLisafs() {
 			return true
 		}
 	}
-	return false
+	// Device gofer needs a gofer process.
+	return shouldCreateDeviceGofer(spec, conf)
 }
 
 // createGoferProcess returns an IO file list and a mounts file on success.
@@ -1187,23 +1199,23 @@ func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool {
 // a gofer endpoint for the mount points using Gofers. The mounts file is the
 // file to read list of mounts after they have been resolved (direct paths,
 // no symlinks), and will be nil if there is no cleaning required for mounts.
-func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) {
-	if !shouldSpawnGofer(c.GoferMountConfs) {
+func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, *os.File, error) {
+	if !shouldSpawnGofer(spec, conf, c.GoferMountConfs) {
 		if !c.GoferMountConfs[0].ShouldUseErofs() {
 			panic("goferless mode is only possible with EROFS rootfs")
 		}
 		ioFile, err := os.Open(rootfsHint.Mount.Source)
 		if err != nil {
-			return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
+			return nil, nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
 		}
-		return []*os.File{ioFile}, nil, nil
+		return []*os.File{ioFile}, nil, nil, nil
 	}
 
 	donations := donation.Agency{}
 	defer donations.Close()
 
 	if err := donations.OpenAndDonate("log-fd", conf.LogFilename, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	if conf.DebugLog != "" {
 		test := ""
@@ -1215,7 +1227,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 		}
 		if specutils.IsDebugCommand(conf, "gofer") {
 			if err := donations.DonateDebugLogFile("debug-log-fd", conf.DebugLog, "gofer", test); err != nil {
-				return nil, nil, err
+				return nil, nil, nil, err
 			}
 		}
 	}
@@ -1242,20 +1254,20 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	// Open the spec file to donate to the sandbox.
 	specFile, err := specutils.OpenSpec(bundleDir)
 	if err != nil {
-		return nil, nil, fmt.Errorf("opening spec file: %v", err)
+		return nil, nil, nil, fmt.Errorf("opening spec file: %v", err)
 	}
 	donations.DonateAndClose("spec-fd", specFile)
 
 	// Donate any profile FDs to the gofer.
 	if err := c.donateGoferProfileFDs(conf, &donations); err != nil {
-		return nil, nil, fmt.Errorf("donating gofer profile fds: %w", err)
+		return nil, nil, nil, fmt.Errorf("donating gofer profile fds: %w", err)
 	}
 
 	// Create pipe that allows gofer to send mount list to sandbox after all paths
 	// have been resolved.
 	mountsSand, mountsGofer, err := os.Pipe()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	donations.DonateAndClose("mounts-fd", mountsGofer)
 
@@ -1273,7 +1285,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 		case cfg.ShouldUseLisafs():
 			fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
 			if err != nil {
-				return nil, nil, err
+				return nil, nil, nil, err
 			}
 			sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD"))
 
@@ -1282,15 +1294,24 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 
 		case cfg.ShouldUseErofs():
 			if i > 0 {
-				return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount")
+				return nil, nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount")
 			}
-			if f, err := os.Open(rootfsHint.Mount.Source); err != nil {
-				return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
-			} else {
-				sandEnds = append(sandEnds, f)
+			f, err := os.Open(rootfsHint.Mount.Source)
+			if err != nil {
+				return nil, nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
 			}
+			sandEnds = append(sandEnds, f)
 		}
 	}
+	var devSandEnd *os.File
+	if shouldCreateDeviceGofer(spec, conf) {
+		fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		devSandEnd = os.NewFile(uintptr(fds[0]), "sandbox dev IO FD")
+		donations.DonateAndClose("dev-io-fd", os.NewFile(uintptr(fds[1]), "gofer dev IO FD"))
+	}
 
 	if attached {
 		// The gofer is attached to the lifetime of this process, so it
@@ -1322,19 +1343,19 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	} else {
 		userNS, ok := specutils.GetNS(specs.UserNamespace, spec)
 		if !ok {
-			return nil, nil, fmt.Errorf("unable to run a rootless container without userns")
+			return nil, nil, nil, fmt.Errorf("unable to run a rootless container without userns")
 		}
 		nss = append(nss, userNS)
 		syncFile, err := sandbox.ConfigureCmdForRootless(cmd, &donations)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		defer syncFile.Close()
 	}
 
 	nvProxySetup, err := nvproxySetupAfterGoferUserns(spec, conf, cmd, &donations)
 	if err != nil {
-		return nil, nil, fmt.Errorf("setting up nvproxy for gofer: %w", err)
+		return nil, nil, nil, fmt.Errorf("setting up nvproxy for gofer: %w", err)
 	}
 
 	donations.Transfer(cmd, nextFD)
@@ -1343,7 +1364,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	donation.LogDonations(cmd)
 	log.Debugf("Starting gofer: %s %v", cmd.Path, cmd.Args)
 	if err := specutils.StartInNS(cmd, nss); err != nil {
-		return nil, nil, fmt.Errorf("gofer: %v", err)
+		return nil, nil, nil, fmt.Errorf("gofer: %v", err)
 	}
 	log.Infof("Gofer started, PID: %d", cmd.Process.Pid)
 	c.GoferPid = cmd.Process.Pid
@@ -1352,16 +1373,16 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	// Set up and synchronize rootless mode userns mappings.
 	if rootlessEUID {
 		if err := sandbox.SetUserMappings(spec, cmd.Process.Pid); err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 	}
 
 	// Set up nvproxy within the Gofer namespace.
 	if err := nvProxySetup(); err != nil {
-		return nil, nil, fmt.Errorf("nvproxy setup: %w", err)
+		return nil, nil, nil, fmt.Errorf("nvproxy setup: %w", err)
 	}
 
-	return sandEnds, mountsSand, nil
+	return sandEnds, devSandEnd, mountsSand, nil
 }
 
 // changeStatus transitions from one status to another ensuring that the

runsc/sandbox/sandbox.go

@@ -228,6 +228,9 @@ type Args struct {
 	// same order as mounts appear in the spec.
 	IOFiles []*os.File
 
+	// File that connects to a gofer endpoint for a device mount point at /dev.
+	DevIOFile *os.File
+
 	// GoferFilestoreFiles are the regular files that will back the overlayfs or
 	// tmpfs mount if a gofer mount is to be overlaid.
 	GoferFilestoreFiles []*os.File
@@ -403,7 +406,7 @@ func (s *Sandbox) StartRoot(conf *config.Config) error {
 }
 
 // StartSubcontainer starts running a sub-container inside the sandbox.
-func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, goferConfs []boot.GoferMountConf) error {
+func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, devIOFile *os.File, goferConfs []boot.GoferMountConf) error {
 	log.Debugf("Start sub-container %q in sandbox %q, PID: %d", cid, s.ID, s.Pid.load())
 
 	if err := s.configureStdios(conf, stdios); err != nil {
@@ -414,10 +417,14 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s
 	// The payload contains (in this specific order):
 	// * stdin/stdout/stderr (optional: only present when not using TTY)
 	// * The subcontainer's gofer filestore files (optional)
+	// * The subcontainer's dev gofer file (optional)
 	// * Gofer files.
 	payload := urpc.FilePayload{}
 	payload.Files = append(payload.Files, stdios...)
 	payload.Files = append(payload.Files, goferFilestores...)
+	if devIOFile != nil {
+		payload.Files = append(payload.Files, devIOFile)
+	}
 	payload.Files = append(payload.Files, goferFiles...)
 
 	// Start running the container.
@@ -426,6 +433,7 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s
 		Conf:                 conf,
 		CID:                  cid,
 		NumGoferFilestoreFDs: len(goferFilestores),
+		IsDevIoFilePresent:   devIOFile != nil,
 		GoferMountConfs:      goferConfs,
 		FilePayload:          payload,
 	}
@@ -733,6 +741,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 
 	// If there is a gofer, sends all socket ends to the sandbox.
 	donations.DonateAndClose("io-fds", args.IOFiles...)
+	donations.DonateAndClose("dev-io-fd", args.DevIOFile)
 	donations.DonateAndClose("gofer-filestore-fds", args.GoferFilestoreFiles...)
 	donations.DonateAndClose("mounts-fd", args.MountsFile)
 	donations.Donate("start-sync-fd", startSyncFile)