diff --git a/pkg/devutil/BUILD b/pkg/devutil/BUILD new file mode 100644 index 000000000..f8daac557 --- /dev/null +++ b/pkg/devutil/BUILD @@ -0,0 +1,20 @@ +load("//tools:defs.bzl", "go_library") + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +go_library( + name = "devutil", + srcs = [ + "context.go", + "devutil.go", + ], + visibility = ["//visibility:public"], + deps = [ + "//pkg/context", + "//pkg/lisafs", + "//pkg/unet", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/pkg/devutil/context.go b/pkg/devutil/context.go new file mode 100644 index 000000000..6095dad7e --- /dev/null +++ b/pkg/devutil/context.go @@ -0,0 +1,33 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devutil + +import "gvisor.dev/gvisor/pkg/context" + +// contextID is this package's type for context.Context.Value keys. +type contextID int + +const ( + // CtxDevGoferClient is a Context.Value key for a /dev gofer client. + CtxDevGoferClient contextID = iota +) + +// GoferClientFromContext returns the device gofer client used by ctx. +func GoferClientFromContext(ctx context.Context) *GoferClient { + if v := ctx.Value(CtxDevGoferClient); v != nil { + return v.(*GoferClient) + } + return nil +} diff --git a/pkg/devutil/devutil.go b/pkg/devutil/devutil.go new file mode 100644 index 000000000..b0d969783 --- /dev/null +++ b/pkg/devutil/devutil.go @@ -0,0 +1,60 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package devutil provides device specific utilities. +package devutil + +import ( + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/lisafs" + "gvisor.dev/gvisor/pkg/unet" +) + +// GoferClient is the lisafs client for the /dev gofer connection. +type GoferClient struct { + clientFD lisafs.ClientFD + hostFD int +} + +// NewGoferClient establishes the LISAFS connection to the dev gofer server. +// It takes ownership of fd. +func NewGoferClient(ctx context.Context, fd int) (*GoferClient, error) { + ctx.UninterruptibleSleepStart(false) + defer ctx.UninterruptibleSleepFinish(false) + + sock, err := unet.NewSocket(fd) + if err != nil { + ctx.Warningf("failed to create socket for dev gofer client: %v", err) + return nil, err + } + client, devInode, devHostFD, err := lisafs.NewClient(sock) + if err != nil { + ctx.Warningf("failed to create dev gofer client: %v", err) + return nil, err + } + return &GoferClient{ + clientFD: client.NewFD(devInode.ControlFD), + hostFD: devHostFD, + }, nil +} + +// Close closes the LISAFS connection. +func (g *GoferClient) Close() { + // Close the connection to the server. This implicitly closes all FDs. + g.clientFD.Client().Close() + if g.hostFD >= 0 { + _ = unix.Close(g.hostFD) + } +} diff --git a/pkg/sentry/kernel/BUILD b/pkg/sentry/kernel/BUILD index 5892f32c5..a2147fdd0 100644 --- a/pkg/sentry/kernel/BUILD +++ b/pkg/sentry/kernel/BUILD @@ -326,6 +326,7 @@ go_library( "//pkg/context", "//pkg/coverage", "//pkg/cpuid", + "//pkg/devutil", "//pkg/errors", "//pkg/errors/linuxerr", "//pkg/eventchannel", diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index b1c364ad6..3951dff99 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -42,6 +42,7 @@ import ( "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/cpuid" + "gvisor.dev/gvisor/pkg/devutil" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/eventchannel" "gvisor.dev/gvisor/pkg/fspath" @@ -327,6 +328,10 @@ type Kernel struct { // MaxFDLimit specifies the maximum file descriptor number that can be // used by processes. MaxFDLimit atomicbitops.Int32 + + // devGofers maps container ID to its device gofer client. + devGofers map[string]*devutil.GoferClient `state:"nosave"` + devGofersMu sync.Mutex `state:"nosave"` } // InitKernelArgs holds arguments to Init. @@ -798,6 +803,8 @@ func (ctx *createProcessContext) Value(key any) any { mntns := ctx.kernel.GlobalInit().Leader().MountNamespace() mntns.IncRef() return mntns + case devutil.CtxDevGoferClient: + return ctx.kernel.getDevGoferClient(ctx.args.ContainerID) case inet.CtxStack: return ctx.kernel.RootNetworkNamespace().Stack() case ktime.CtxRealtimeClock: @@ -1683,6 +1690,7 @@ func (k *Kernel) Release() { k.timekeeper.Destroy() k.vdso.Release(ctx) k.RootNetworkNamespace().DecRef(ctx) + k.cleaupDevGofers() } // PopulateNewCgroupHierarchy moves all tasks into a newly created cgroup @@ -1784,3 +1792,48 @@ func (k *Kernel) GetUserCounters(uid auth.KUID) *UserCounters { k.userCountersMap[uid] = uc return uc } + +// AddDevGofer initializes the dev gofer connection and starts tracking it. +// It takes ownership of goferFD. +func (k *Kernel) AddDevGofer(cid string, goferFD int) error { + client, err := devutil.NewGoferClient(k.SupervisorContext(), goferFD) + if err != nil { + return err + } + + k.devGofersMu.Lock() + defer k.devGofersMu.Unlock() + if k.devGofers == nil { + k.devGofers = make(map[string]*devutil.GoferClient) + } + k.devGofers[cid] = client + return nil +} + +// RemoveDevGofer closes the dev gofer connection, if one exists, and stops +// tracking it. +func (k *Kernel) RemoveDevGofer(cid string) { + k.devGofersMu.Lock() + defer k.devGofersMu.Unlock() + client, ok := k.devGofers[cid] + if !ok { + return + } + client.Close() + delete(k.devGofers, cid) +} + +func (k *Kernel) getDevGoferClient(cid string) *devutil.GoferClient { + k.devGofersMu.Lock() + defer k.devGofersMu.Unlock() + return k.devGofers[cid] +} + +func (k *Kernel) cleaupDevGofers() { + k.devGofersMu.Lock() + defer k.devGofersMu.Unlock() + for _, client := range k.devGofers { + client.Close() + } + k.devGofers = nil +} diff --git a/pkg/sentry/kernel/task_context.go b/pkg/sentry/kernel/task_context.go index 99bad23f8..1e391c281 100644 --- a/pkg/sentry/kernel/task_context.go +++ b/pkg/sentry/kernel/task_context.go @@ -20,6 +20,7 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/cpuid" + "gvisor.dev/gvisor/pkg/devutil" "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" "gvisor.dev/gvisor/pkg/sentry/kernel/ipc" @@ -103,6 +104,8 @@ func (t *Task) contextValue(key any, isTaskGoroutine bool) any { } t.mountNamespace.IncRef() return t.mountNamespace + case devutil.CtxDevGoferClient: + return t.k.getDevGoferClient(t.containerID) case inet.CtxStack: return t.NetworkContext() case ktime.CtxRealtimeClock: diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index 501f1648d..e8d9297b2 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -279,6 +279,9 @@ type StartArgs struct { // NumGoferFilestoreFDs is the number of gofer filestore FDs donated. NumGoferFilestoreFDs int + // IsDevIoFilePresent indicates whether the dev gofer FD is present. + IsDevIoFilePresent bool + // GoferMountConfs contains information about how the gofer mounts have been // configured. The first entry is for rootfs and the following entries are // for bind mounts in Spec.Mounts (in the same order). @@ -287,6 +290,7 @@ type StartArgs struct { // FilePayload contains, in order: // * stdin, stdout, and stderr (optional: if terminal is disabled). // * file descriptors to gofer-backing host files (optional). + // * file descriptor for /dev gofer connection (optional) // * file descriptors to connect to gofer to serve the root filesystem. urpc.FilePayload } @@ -309,6 +313,9 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro } expectedFDs := 1 // At least one FD for the root filesystem. expectedFDs += args.NumGoferFilestoreFDs + if args.IsDevIoFilePresent { + expectedFDs++ + } if !args.Spec.Process.Terminal { expectedFDs += 3 } @@ -352,6 +359,17 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro } }() + var devGoferFD *fd.FD + if args.IsDevIoFilePresent { + var err error + devGoferFD, err = fd.NewFromFile(goferFiles[0]) + if err != nil { + return fmt.Errorf("error dup'ing dev gofer file: %w", err) + } + goferFiles = goferFiles[1:] + defer devGoferFD.Close() + } + goferFDs, err := fd.NewFromFiles(goferFiles) if err != nil { return fmt.Errorf("error dup'ing gofer files: %w", err) @@ -362,7 +380,7 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro } }() - if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, goferFilestoreFDs, args.GoferMountConfs); err != nil { + if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, goferFilestoreFDs, devGoferFD, args.GoferMountConfs); err != nil { log.Debugf("containerManager.StartSubcontainer failed, cid: %s, args: %+v, err: %v", args.CID, args, err) return err } diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index aaa57247b..d3c64b6f0 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -113,6 +113,9 @@ type containerInfo struct { // goferFDs are the FDs that attach the sandbox to the gofers. goferFDs []*fd.FD + // devGoferFD is the FD to attach the sandbox to the dev gofer. + devGoferFD *fd.FD + // goferFilestoreFDs are FDs to the regular files that will back the tmpfs or // overlayfs mount for certain gofer mounts. goferFilestoreFDs []*fd.FD @@ -168,9 +171,6 @@ type Loader struct { // /sys/devices/virtual/dmi/id/product_name. productName string - // nvidiaUVMDevMajor is the device major number used for nvidia-uvm. - nvidiaUVMDevMajor uint32 - // mu guards the fields below. mu sync.Mutex @@ -251,6 +251,9 @@ type Args struct { // GoferFDs is an array of FDs used to connect with the Gofer. The Loader // takes ownership of these FDs and may close them at any time. GoferFDs []int + // DevGoferFD is the FD for the dev gofer connection. The Loader takes + // ownership of this FD and may close it at any time. + DevGoferFD int // StdioFDs is the stdio for the application. The Loader takes ownership of // these FDs and may close them at any time. StdioFDs []int @@ -354,7 +357,9 @@ func New(args Args) (*Loader, error) { for _, filestoreFD := range args.GoferFilestoreFDs { info.goferFilestoreFDs = append(info.goferFilestoreFDs, fd.New(filestoreFD)) } - + if args.DevGoferFD >= 0 { + info.devGoferFD = fd.New(args.DevGoferFD) + } if args.ExecFD >= 0 { info.execFD = fd.New(args.ExecFD) } @@ -526,16 +531,15 @@ func New(args Args) (*Loader, error) { eid := execID{cid: args.ID} l := &Loader{ - k: k, - watchdog: dog, - sandboxID: args.ID, - processes: map[execID]*execProcess{eid: {}}, - mountHints: mountHints, - sharedMounts: make(map[string]*vfs.Mount), - root: info, - stopProfiling: stopProfiling, - productName: args.ProductName, - nvidiaUVMDevMajor: info.nvidiaUVMDevMajor, + k: k, + watchdog: dog, + sandboxID: args.ID, + processes: map[execID]*execProcess{eid: {}}, + mountHints: mountHints, + sharedMounts: make(map[string]*vfs.Mount), + root: info, + stopProfiling: stopProfiling, + productName: args.ProductName, } // We don't care about child signals; some platforms can generate a @@ -636,6 +640,9 @@ func (l *Loader) Destroy() { for _, f := range l.root.goferFilestoreFDs { _ = f.Close() } + if l.root.devGoferFD != nil { + _ = l.root.devGoferFD.Close() + } l.stopProfiling() } @@ -821,7 +828,7 @@ func (l *Loader) createSubcontainer(cid string, tty *fd.FD) error { // startSubcontainer starts a child container. It returns the thread group ID of // the newly created process. Used FDs are either closed or released. It's safe // for the caller to close any remaining files upon return. -func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, goferMountConfs []GoferMountConf) error { +func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, devGoferFD *fd.FD, goferMountConfs []GoferMountConf) error { // Create capabilities. caps, err := specutils.Capabilities(conf.EnableRaw, spec.Process.Capabilities) if err != nil { @@ -877,9 +884,10 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st conf: conf, spec: spec, goferFDs: goferFDs, + devGoferFD: devGoferFD, goferFilestoreFDs: goferFilestoreFDs, goferMountConfs: goferMountConfs, - nvidiaUVMDevMajor: l.nvidiaUVMDevMajor, + nvidiaUVMDevMajor: l.root.nvidiaUVMDevMajor, nvidiaDevMinors: l.root.nvidiaDevMinors, } info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns) @@ -901,6 +909,17 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st info.stdioFDs = stdioFDs } + var cu cleanup.Cleanup + defer cu.Clean() + if devGoferFD != nil { + cu.Add(func() { + // createContainerProcess() will consume devGoferFD and initialize a gofer + // connection. This connection is owned by l.k. In case of failure, we want + // to clean up this gofer connection so that the gofer process can exit. + l.k.RemoveDevGofer(cid) + }) + } + ep.tg, ep.tty, err = l.createContainerProcess(cid, info) if err != nil { return err @@ -927,6 +946,8 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st } l.k.StartProcess(ep.tg) + // No more failures from this point on. + cu.Release() return nil } @@ -966,11 +987,7 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne if len(info.goferFDs) < 1 { return nil, nil, fmt.Errorf("rootfs gofer FD not found") } - // TODO(ayushranjan): The gofer monitor should be started as long as the gofer - // process exists, even if the root mount is not backed by lisafs. - if info.goferMountConfs[0].ShouldUseLisafs() { - l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) - } + l.startGoferMonitor(cid, info) // We can share l.sharedMounts with containerMounter since l.mu is locked. // Hence, mntr must only be used within this function (while l.mu is locked). @@ -1030,18 +1047,29 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne // startGoferMonitor runs a goroutine to monitor gofer's health. It polls on // the gofer FD looking for disconnects, and kills the container processes if -// the rootfs FD disconnects. -// -// Note that other gofer mounts are allowed to be unmounted and disconnected. -func (l *Loader) startGoferMonitor(cid string, rootfsGoferFD int32) { - if rootfsGoferFD < 0 { - panic(fmt.Sprintf("invalid FD: %d", rootfsGoferFD)) +// the gofer connection disconnects. +func (l *Loader) startGoferMonitor(cid string, info *containerInfo) { + // We need to pick a suitable gofer connection that is expected to be alive + // for the entire container lifecycle. Only the following can be used: + // 1. Rootfs gofer connection + // 2. Device gofer connection + // + // Note that other gofer mounts are allowed to be unmounted and disconnected. + goferFD := -1 + if info.goferMountConfs[0].ShouldUseLisafs() { + goferFD = info.goferFDs[0].FD() + } else if info.devGoferFD != nil { + goferFD = info.devGoferFD.FD() + } + if goferFD < 0 { + log.Warningf("could not find a suitable gofer FD to monitor") + return } go func() { log.Debugf("Monitoring gofer health for container %q", cid) events := []unix.PollFd{ { - Fd: rootfsGoferFD, + Fd: int32(goferFD), Events: unix.POLLHUP | unix.POLLRDHUP, }, } @@ -1094,13 +1122,16 @@ func (l *Loader) destroySubcontainer(cid string) error { } } - // No more failure from this point on. Remove all container thread groups - // from the map. + // No more failure from this point on. + + // Remove all container thread groups from the map. for key := range l.processes { if key.cid == cid { delete(l.processes, key) } } + // Cleanup the device gofer. + l.k.RemoveDevGofer(cid) log.Debugf("Container destroyed, cid: %s", cid) return nil diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index 552e13654..b32d966f4 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -139,6 +139,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error Conf: conf, ControllerFD: fd, GoferFDs: []int{sandEnd}, + DevGoferFD: -1, StdioFDs: stdio, GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}}, PodInitConfigFD: -1, diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 8a9f6d984..e1db87cf5 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -358,6 +358,9 @@ type containerMounter struct { // overlayfs mount for certain gofer mounts. goferFilestoreFDs fdDispenser + // devGoferFD is the FD to attach the sandbox to the dev gofer. + devGoferFD *fd.FD + // goferMountConfs contains information about how the gofer mounts have been // configured. The first entry is for rootfs and the following entries are // for bind mounts in Spec.Mounts (in the same order). @@ -376,6 +379,9 @@ type containerMounter struct { // /sys/devices/virtual/dmi/id/product_name. productName string + // containerID is the ID for the container. + containerID string + // sandboxID is the ID for the whole sandbox. sandboxID string } @@ -386,11 +392,13 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountH mounts: compileMounts(info.spec, info.conf), goferFDs: fdDispenser{fds: info.goferFDs}, goferFilestoreFDs: fdDispenser{fds: info.goferFilestoreFDs}, + devGoferFD: info.devGoferFD, goferMountConfs: info.goferMountConfs, k: k, hints: hints, sharedMounts: sharedMounts, productName: productName, + containerID: info.procArgs.ContainerID, sandboxID: sandboxID, } } @@ -402,6 +410,9 @@ func (c *containerMounter) checkDispenser() error { if !c.goferFilestoreFDs.empty() { return fmt.Errorf("not all gofer Filestore FDs were consumed, remaining: %v", c.goferFilestoreFDs) } + if c.devGoferFD != nil && c.devGoferFD.FD() >= 0 { + return fmt.Errorf("dev gofer FD was not consumed: %d", c.devGoferFD.FD()) + } return nil } @@ -725,6 +736,12 @@ type mountInfo struct { } func (c *containerMounter) prepareMounts() ([]mountInfo, error) { + // If device gofer exists, connect to it. + if c.devGoferFD != nil { + if err := c.k.AddDevGofer(c.containerID, c.devGoferFD.Release()); err != nil { + return nil, err + } + } // Associate bind mounts with their FDs before sorting since there is an // undocumented assumption that FDs are dispensed in the order in which // they are required by mounts. diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index ffa92f76f..c36dccd86 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -81,6 +81,9 @@ type Boot struct { // ioFDs is the list of FDs used to connect to FS gofers. ioFDs intFlags + // devIoFD is the FD to connect to dev gofer. + devIoFD int + // goferFilestoreFDs are FDs to the regular files that will back the tmpfs or // overlayfs mount for certain gofer mounts. goferFilestoreFDs intFlags @@ -197,6 +200,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process") f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file") f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") + f.IntVar(&b.devIoFD, "dev-io-fd", -1, "FD to connect dev gofer client") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.") f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.") @@ -425,6 +429,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma ControllerFD: b.controllerFD, Device: os.NewFile(uintptr(b.deviceFD), "platform device"), GoferFDs: b.ioFDs.GetArray(), + DevGoferFD: b.devIoFD, StdioFDs: b.stdioFDs.GetArray(), PassFDs: b.passFDs.GetArray(), ExecFD: b.execFD, diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index 333f513b3..11ada4e5c 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -82,6 +82,7 @@ type goferSyncFDs struct { type Gofer struct { bundleDir string ioFDs intFlags + devIoFD int applyCaps bool setUpRoot bool mountConfs boot.GoferMountConfFlags @@ -117,6 +118,7 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) { // Open FDs that are donated to the gofer. f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.") f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.") + f.IntVar(&g.devIoFD, "dev-io-fd", -1, "optional FD to connect /dev gofer server") f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).") @@ -312,6 +314,14 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs)) } + if g.devIoFD >= 0 { + cfgs = append(cfgs, connectionConfig{ + sock: newSocket(g.devIoFD), + mountPath: "/dev", + }) + log.Infof("Serving /dev mapped on FD %d (ro: false)", g.devIoFD) + } + for _, cfg := range cfgs { conn, err := server.CreateConnection(cfg.sock, cfg.mountPath, cfg.readonly) if err != nil { @@ -417,6 +427,11 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { util.Fatalf("error setting up FS: %v", err) } + // Set up /dev directory is needed. + if g.devIoFD >= 0 { + g.setupDev(root) + } + // Create working directory if needed. if spec.Process.Cwd != "" { dst, err := resolveSymlinks(root, spec.Process.Cwd) @@ -497,6 +512,13 @@ func (g *Gofer) setupMounts(conf *config.Config, mounts []specs.Mount, root, pro return nil } +func (g *Gofer) setupDev(root string) error { + if err := os.MkdirAll(filepath.Join(root, "dev"), 0777); err != nil { + return fmt.Errorf("creating dev directory: %v", err) + } + return nil +} + // resolveMounts resolved relative paths and symlinks to mount points. // // Note: mount points must already be in place for resolution to work. diff --git a/runsc/container/container.go b/runsc/container/container.go index 044280d08..76a4dc324 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -305,7 +305,7 @@ func New(conf *config.Config, args Args) (*Container, error) { return nil, err } if err := runInCgroup(containerCgroup, func() error { - ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint) + ioFiles, devIOFile, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint) if err != nil { return fmt.Errorf("cannot create gofer process: %w", err) } @@ -319,6 +319,7 @@ func New(conf *config.Config, args Args) (*Container, error) { ConsoleSocket: args.ConsoleSocket, UserLog: args.UserLog, IOFiles: ioFiles, + DevIOFile: devIOFile, MountsFile: specFile, Cgroup: containerCgroup, Attached: args.Attached, @@ -471,7 +472,7 @@ func (c *Container) Start(conf *config.Config) error { // the start (and all their children processes). if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error { // Create the gofer process. - goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint) + goferFiles, devIOFile, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint) if err != nil { return err } @@ -479,6 +480,9 @@ func (c *Container) Start(conf *config.Config) error { if mountsFile != nil { _ = mountsFile.Close() } + if devIOFile != nil { + _ = devIOFile.Close() + } for _, f := range goferFiles { _ = f.Close() } @@ -502,7 +506,7 @@ func (c *Container) Start(conf *config.Config) error { stdios = []*os.File{os.Stdin, os.Stdout, os.Stderr} } - return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, goferConfs) + return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, devIOFile, goferConfs) }); err != nil { return err } @@ -1172,14 +1176,22 @@ func (c *Container) waitForStopped() error { return backoff.Retry(op, b) } +// shouldCreateDeviceGofer indicates whether a device gofer connection should +// be created. +func shouldCreateDeviceGofer(spec *specs.Spec, conf *config.Config) bool { + return specutils.GPUFunctionalityRequested(spec, conf) +} + // shouldSpawnGofer indicates whether the gofer process should be spawned. -func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool { +func shouldSpawnGofer(spec *specs.Spec, conf *config.Config, goferConfs []boot.GoferMountConf) bool { + // Lisafs mounts need the gofer. for _, cfg := range goferConfs { if cfg.ShouldUseLisafs() { return true } } - return false + // Device gofer needs a gofer process. + return shouldCreateDeviceGofer(spec, conf) } // createGoferProcess returns an IO file list and a mounts file on success. @@ -1187,23 +1199,23 @@ func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool { // a gofer endpoint for the mount points using Gofers. The mounts file is the // file to read list of mounts after they have been resolved (direct paths, // no symlinks), and will be nil if there is no cleaning required for mounts. -func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) { - if !shouldSpawnGofer(c.GoferMountConfs) { +func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, *os.File, error) { + if !shouldSpawnGofer(spec, conf, c.GoferMountConfs) { if !c.GoferMountConfs[0].ShouldUseErofs() { panic("goferless mode is only possible with EROFS rootfs") } ioFile, err := os.Open(rootfsHint.Mount.Source) if err != nil { - return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + return nil, nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) } - return []*os.File{ioFile}, nil, nil + return []*os.File{ioFile}, nil, nil, nil } donations := donation.Agency{} defer donations.Close() if err := donations.OpenAndDonate("log-fd", conf.LogFilename, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { - return nil, nil, err + return nil, nil, nil, err } if conf.DebugLog != "" { test := "" @@ -1215,7 +1227,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu } if specutils.IsDebugCommand(conf, "gofer") { if err := donations.DonateDebugLogFile("debug-log-fd", conf.DebugLog, "gofer", test); err != nil { - return nil, nil, err + return nil, nil, nil, err } } } @@ -1242,20 +1254,20 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu // Open the spec file to donate to the sandbox. specFile, err := specutils.OpenSpec(bundleDir) if err != nil { - return nil, nil, fmt.Errorf("opening spec file: %v", err) + return nil, nil, nil, fmt.Errorf("opening spec file: %v", err) } donations.DonateAndClose("spec-fd", specFile) // Donate any profile FDs to the gofer. if err := c.donateGoferProfileFDs(conf, &donations); err != nil { - return nil, nil, fmt.Errorf("donating gofer profile fds: %w", err) + return nil, nil, nil, fmt.Errorf("donating gofer profile fds: %w", err) } // Create pipe that allows gofer to send mount list to sandbox after all paths // have been resolved. mountsSand, mountsGofer, err := os.Pipe() if err != nil { - return nil, nil, err + return nil, nil, nil, err } donations.DonateAndClose("mounts-fd", mountsGofer) @@ -1273,7 +1285,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu case cfg.ShouldUseLisafs(): fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) if err != nil { - return nil, nil, err + return nil, nil, nil, err } sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) @@ -1282,15 +1294,24 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu case cfg.ShouldUseErofs(): if i > 0 { - return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount") + return nil, nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount") } - if f, err := os.Open(rootfsHint.Mount.Source); err != nil { - return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) - } else { - sandEnds = append(sandEnds, f) + f, err := os.Open(rootfsHint.Mount.Source) + if err != nil { + return nil, nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) } + sandEnds = append(sandEnds, f) } } + var devSandEnd *os.File + if shouldCreateDeviceGofer(spec, conf) { + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, nil, err + } + devSandEnd = os.NewFile(uintptr(fds[0]), "sandbox dev IO FD") + donations.DonateAndClose("dev-io-fd", os.NewFile(uintptr(fds[1]), "gofer dev IO FD")) + } if attached { // The gofer is attached to the lifetime of this process, so it @@ -1322,19 +1343,19 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu } else { userNS, ok := specutils.GetNS(specs.UserNamespace, spec) if !ok { - return nil, nil, fmt.Errorf("unable to run a rootless container without userns") + return nil, nil, nil, fmt.Errorf("unable to run a rootless container without userns") } nss = append(nss, userNS) syncFile, err := sandbox.ConfigureCmdForRootless(cmd, &donations) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer syncFile.Close() } nvProxySetup, err := nvproxySetupAfterGoferUserns(spec, conf, cmd, &donations) if err != nil { - return nil, nil, fmt.Errorf("setting up nvproxy for gofer: %w", err) + return nil, nil, nil, fmt.Errorf("setting up nvproxy for gofer: %w", err) } donations.Transfer(cmd, nextFD) @@ -1343,7 +1364,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu donation.LogDonations(cmd) log.Debugf("Starting gofer: %s %v", cmd.Path, cmd.Args) if err := specutils.StartInNS(cmd, nss); err != nil { - return nil, nil, fmt.Errorf("gofer: %v", err) + return nil, nil, nil, fmt.Errorf("gofer: %v", err) } log.Infof("Gofer started, PID: %d", cmd.Process.Pid) c.GoferPid = cmd.Process.Pid @@ -1352,16 +1373,16 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu // Set up and synchronize rootless mode userns mappings. if rootlessEUID { if err := sandbox.SetUserMappings(spec, cmd.Process.Pid); err != nil { - return nil, nil, err + return nil, nil, nil, err } } // Set up nvproxy within the Gofer namespace. if err := nvProxySetup(); err != nil { - return nil, nil, fmt.Errorf("nvproxy setup: %w", err) + return nil, nil, nil, fmt.Errorf("nvproxy setup: %w", err) } - return sandEnds, mountsSand, nil + return sandEnds, devSandEnd, mountsSand, nil } // changeStatus transitions from one status to another ensuring that the diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 44fef8cad..1105f4eb5 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -228,6 +228,9 @@ type Args struct { // same order as mounts appear in the spec. IOFiles []*os.File + // File that connects to a gofer endpoint for a device mount point at /dev. + DevIOFile *os.File + // GoferFilestoreFiles are the regular files that will back the overlayfs or // tmpfs mount if a gofer mount is to be overlaid. GoferFilestoreFiles []*os.File @@ -403,7 +406,7 @@ func (s *Sandbox) StartRoot(conf *config.Config) error { } // StartSubcontainer starts running a sub-container inside the sandbox. -func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, goferConfs []boot.GoferMountConf) error { +func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, devIOFile *os.File, goferConfs []boot.GoferMountConf) error { log.Debugf("Start sub-container %q in sandbox %q, PID: %d", cid, s.ID, s.Pid.load()) if err := s.configureStdios(conf, stdios); err != nil { @@ -414,10 +417,14 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s // The payload contains (in this specific order): // * stdin/stdout/stderr (optional: only present when not using TTY) // * The subcontainer's gofer filestore files (optional) + // * The subcontainer's dev gofer file (optional) // * Gofer files. payload := urpc.FilePayload{} payload.Files = append(payload.Files, stdios...) payload.Files = append(payload.Files, goferFilestores...) + if devIOFile != nil { + payload.Files = append(payload.Files, devIOFile) + } payload.Files = append(payload.Files, goferFiles...) // Start running the container. @@ -426,6 +433,7 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s Conf: conf, CID: cid, NumGoferFilestoreFDs: len(goferFilestores), + IsDevIoFilePresent: devIOFile != nil, GoferMountConfs: goferConfs, FilePayload: payload, } @@ -733,6 +741,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn // If there is a gofer, sends all socket ends to the sandbox. donations.DonateAndClose("io-fds", args.IOFiles...) + donations.DonateAndClose("dev-io-fd", args.DevIOFile) donations.DonateAndClose("gofer-filestore-fds", args.GoferFilestoreFiles...) donations.DonateAndClose("mounts-fd", args.MountsFile) donations.Donate("start-sync-fd", startSyncFile)