From d7b57d2fd37f69a2be355bb0c5c63fecb67c251a Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Wed, 30 Nov 2022 12:27:15 -0800 Subject: [PATCH] Add overlay2 flag in runsc. --overlay2 flag supersedes --overlay flag. It allows more granular configuration for overlayfs in runsc. It does so in two ways: 1. Allows to apply overlay on all mounts or only the root mount. --overlay applies overlay to all mounts. 2. Allows to specify if overlay's upper layer should be backed by container memory or disk. --overlay always used container memory. Allowing tmpfs to be backed by a file on disk prevents the container memory from bloating up. Note that the tmpfs filesystem tree will still be stored in sentry memory. Using overlay on the root filesystem, helps avoid expensive communication with the gofer process. The root filesystem of the container is not preserved across container lifecycle. So we don't need to keep updating the host filesystem, which will anyways be destroyed once the container is destroyed. It is wasted effort. Instead we keep all the changes to the root filesystem in tmpfs which is directly accessible by the sentry. The host file is created as an unnamed file using O_TMPFILE. Support has been added for sub-containers too. Save/restore support is still lacking. Co-authored-by: Andrei Vagin PiperOrigin-RevId: 491988485 --- Makefile | 3 +- g3doc/user_guide/filesystem.md | 19 +++++- pkg/sentry/fsimpl/tmpfs/tmpfs.go | 7 ++ pkg/sentry/pgalloc/pgalloc.go | 5 ++ pkg/test/testutil/testutil.go | 2 +- runsc/boot/loader.go | 21 ++++++ runsc/boot/loader_test.go | 15 +++-- runsc/boot/vfs.go | 23 ++++--- runsc/cli/main.go | 4 +- runsc/cmd/boot.go | 34 ++++++---- runsc/cmd/do.go | 6 +- runsc/cmd/gofer.go | 14 ++-- runsc/config/config.go | 106 ++++++++++++++++++++++++++++++- runsc/config/config_test.go | 22 ++++++- runsc/config/flags.go | 3 +- runsc/container/container.go | 45 ++++++++++--- runsc/sandbox/sandbox.go | 5 ++ test/runner/main.go | 2 +- 18 files changed, 278 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index 23f8d13c5..548ffa47e 100644 --- a/Makefile +++ b/Makefile @@ -274,8 +274,9 @@ docker-tests: load-basic $(RUNTIME_BIN) @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test) .PHONY: docker-tests +# TODO(b/241832602): Run overlay tests with host filestore option after S/R support is added. overlay-tests: load-basic $(RUNTIME_BIN) - @$(call install_runtime,$(RUNTIME),--overlay) + @$(call install_runtime,$(RUNTIME),--overlay2=all:memory) @$(call test_runtime,$(RUNTIME),--test_env=TEST_OVERLAY=true $(INTEGRATION_TARGETS)) .PHONY: overlay-tests diff --git a/g3doc/user_guide/filesystem.md b/g3doc/user_guide/filesystem.md index 3122987fa..1774e809e 100644 --- a/g3doc/user_guide/filesystem.md +++ b/g3doc/user_guide/filesystem.md @@ -27,13 +27,30 @@ configuration (`/etc/docker/daemon.json`) and restart the Docker daemon: "runsc": { "path": "/usr/local/bin/runsc", "runtimeArgs": [ - "--overlay" + "--overlay2=all:memory" ] } } } ``` +### Root Filesystem Overlay + +Any modifications to the root filesystem is destroyed with the container. So it +almost always makes sense to apply an overlay on top of the root filesystem. +This can drastically boost performance, as runsc will handle root filesystem +changes completely in memory instead of making costly round trips to the gofer +and make syscalls to modify the host. + +However, holding so much file data in memory for the root filesystem can bloat +up container memory usage. To circumvent this, you can have root mount's upper +layer (tmpfs) be backed by a host file, so all file data is stored on disk. + +The newer `--overlay2` flag allows you to achieve these. You can specify +`--overlay2=root:/dir/path` in `runtimeArgs`. `/dir/path` can be any existing +directory inside which the tmpfs filestore file will be created. When the +container exits, this filestore file will be destroyed. + ## Shared root filesystem The root filesystem is where the image is extracted and is not generally diff --git a/pkg/sentry/fsimpl/tmpfs/tmpfs.go b/pkg/sentry/fsimpl/tmpfs/tmpfs.go index daa452797..7f45fa26a 100644 --- a/pkg/sentry/fsimpl/tmpfs/tmpfs.go +++ b/pkg/sentry/fsimpl/tmpfs/tmpfs.go @@ -129,6 +129,10 @@ type FilesystemOpts struct { // MaxFilenameLen is the maximum filename length allowed by the tmpfs. MaxFilenameLen int + + // Filestore is the MemoryFile that will be used to store file data. If this + // is nil, then MemoryFileProviderFromContext() is used. + Filestore *pgalloc.MemoryFile } // GetFilesystem implements vfs.FilesystemType.GetFilesystem. @@ -148,6 +152,9 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt if tmpfsOpts.FilesystemType != nil { newFSType = tmpfsOpts.FilesystemType } + if tmpfsOpts.Filestore != nil { + mfp = tmpfsOpts.Filestore + } } mopts := vfs.GenericParseMountOptions(opts.Data) diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index 146784ce9..e8e643768 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -1353,6 +1353,11 @@ func (f *MemoryFile) startEvictionGoroutineLocked(user EvictableMemoryUser, info }() } +// MemoryFile implements MemoryFileProvider.MemoryFile. +func (f *MemoryFile) MemoryFile() *MemoryFile { + return f +} + // WaitForEvictions blocks until f is no longer evicting any evictable // allocations. func (f *MemoryFile) WaitForEvictions() { diff --git a/pkg/test/testutil/testutil.go b/pkg/test/testutil/testutil.go index 0236a6cff..5db023969 100644 --- a/pkg/test/testutil/testutil.go +++ b/pkg/test/testutil/testutil.go @@ -55,7 +55,7 @@ var ( isRunningWithHostNet = flag.Bool("hostnet", BoolFromEnv("HOSTNET", false), "whether test is running with hostnet") runscPath = flag.String("runsc", os.Getenv("RUNTIME"), "path to runsc binary") // Note: flag overlay is already taken by runsc. - isRunningWithOverlay = flag.Bool("test-overlay", BoolFromEnv("TEST_OVERLAY", false), "whether test is running with --overlay") + isRunningWithOverlay = flag.Bool("test-overlay", BoolFromEnv("TEST_OVERLAY", false), "whether test is running with --overlay2") ) // StringFromEnv returns the value of the named environment variable, or `def` if unset/empty. diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 6ce4fb8cb..3d87e3ab8 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -101,6 +101,10 @@ type containerInfo struct { // goferFDs are the FDs that attach the sandbox to the gofers. goferFDs []*fd.FD + + // overlayFilestore is the memory file that will back the overlay mount's + // upper tmpfs layer. + overlayFilestore *pgalloc.MemoryFile } // Loader keeps state needed to start the kernel and run the container. @@ -200,6 +204,9 @@ type Args struct { // StdioFDs is the stdio for the application. The Loader takes ownership of // these FDs and may close them at any time. StdioFDs []int + // OverlayFilestoreFD is the host FD to a regular file which will be used to + // back the overlay mount's upper tmpfs layer. + OverlayFilestoreFD int // NumCPU is the number of CPUs to create inside the sandbox. NumCPU int // TotalMem is the initial amount of total memory to report back to the @@ -268,6 +275,14 @@ func New(args Args) (*Loader, error) { for _, goferFD := range args.GoferFDs { info.goferFDs = append(info.goferFDs, fd.New(goferFD)) } + if args.OverlayFilestoreFD >= 0 { + f := os.NewFile(uintptr(args.OverlayFilestoreFD), "overlay-filestore") + mf, err := pgalloc.NewMemoryFile(f, pgalloc.MemoryFileOpts{}) + if err != nil { + return nil, fmt.Errorf("tmpfs.FilesystemType.GetFilesystem: failed to create memory file from host file: %w", err) + } + info.overlayFilestore = mf + } // Create kernel and platform. p, err := createPlatform(args.Conf, args.Device) @@ -513,6 +528,9 @@ func (l *Loader) Destroy() { for _, f := range l.root.goferFDs { _ = f.Close() } + if l.root.overlayFilestore != nil { + l.root.overlayFilestore.Destroy() + } l.stopProfiling() } @@ -747,6 +765,9 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st conf: conf, spec: spec, goferFDs: goferFDs, + // Note that K8s starts all containers in a pod (root and subcontainers) + // with the same config. So overlayFilestore can be copied. + overlayFilestore: l.root.overlayFilestore, } info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns) if err != nil { diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index f6d20af93..4ef88dacd 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -125,13 +125,14 @@ func createLoader(spec *specs.Spec) (*Loader, func(), error) { } args := Args{ - ID: "foo", - Spec: spec, - Conf: conf, - ControllerFD: fd, - GoferFDs: []int{sandEnd}, - StdioFDs: stdio, - PodInitConfigFD: -1, + ID: "foo", + Spec: spec, + Conf: conf, + ControllerFD: fd, + GoferFDs: []int{sandEnd}, + StdioFDs: stdio, + OverlayFilestoreFD: -1, + PodInitConfigFD: -1, } l, err := New(args) if err != nil { diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 99a4e9132..edb85838d 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -46,6 +46,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/vfs" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/specutils" @@ -322,6 +323,10 @@ type containerMounter struct { // fds is the list of FDs to be dispensed for mounts that require it. fds fdDispenser + // overlayFilestore is the memory file that will back the overlay mount's + // upper tmpfs layer. + overlayFilestore *pgalloc.MemoryFile + k *kernel.Kernel hints *podMountHints @@ -333,12 +338,13 @@ type containerMounter struct { func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string) *containerMounter { return &containerMounter{ - root: info.spec.Root, - mounts: compileMounts(info.spec, info.conf), - fds: fdDispenser{fds: info.goferFDs}, - k: k, - hints: hints, - productName: productName, + root: info.spec.Root, + mounts: compileMounts(info.spec, info.conf), + fds: fdDispenser{fds: info.goferFDs}, + overlayFilestore: info.overlayFilestore, + k: k, + hints: hints, + productName: productName, } } @@ -425,7 +431,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi } fsName := gofer.Name - if conf.Overlay && !c.root.Readonly { + if conf.GetOverlay2().RootMount && !c.root.Readonly { log.Infof("Adding overlay on top of root") var err error var cleanup func() @@ -488,6 +494,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, creds *auth.Cre // Upper is a tmpfs mount to keep all modifications inside the sandbox. upperOpts.GetFilesystemOptions.InternalData = tmpfs.FilesystemOpts{ RootFileType: uint16(rootType), + Filestore: c.overlayFilestore, } upper, err := c.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts) if err != nil { @@ -717,7 +724,7 @@ func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountA } // If configured, add overlay to all writable mounts. - useOverlay = conf.Overlay && !parseMountOptions(m.mount.Options).ReadOnly + useOverlay = conf.GetOverlay2().SubMounts && !parseMountOptions(m.mount.Options).ReadOnly case cgroupfs.Name: var err error diff --git a/runsc/cli/main.go b/runsc/cli/main.go index 972d957ef..57adae2a8 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -224,7 +224,9 @@ func Main(version string) { log.Infof("Configuration:") log.Infof("\t\tRootDir: %s", conf.RootDir) log.Infof("\t\tPlatform: %v", conf.Platform) - log.Infof("\t\tFileAccess: %v, overlay: %t", conf.FileAccess, conf.Overlay) + log.Infof("\t\tFileAccess: %v", conf.FileAccess) + overlay2 := conf.GetOverlay2() + log.Infof("\t\tOverlay: Root=%t, SubMounts=%t, FilestoreDir=%q", overlay2.RootMount, overlay2.SubMounts, overlay2.FilestoreDir) log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets) log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls) log.Infof("\t\tLISAFS: %t", conf.Lisafs) diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 38d3f8781..63a502333 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -56,6 +56,10 @@ type Boot struct { // ioFDs is the list of FDs used to connect to FS gofers. ioFDs intFlags + // overlayFilestoreFD is the host FD to the regular file which will back the + // overlay's upper tmpfs mount for all containers. + overlayFilestoreFD int + // stdioFDs are the fds for stdin, stdout, and stderr. They must be // provided in that order. stdioFDs intFlags @@ -142,6 +146,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file") f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") + f.IntVar(&b.overlayFilestoreFD, "overlay-filestore-fd", -1, "FD to a regular file which will be used to back the overlay's tmpfs upper mount.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") @@ -302,20 +307,21 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma // Create the loader. bootArgs := boot.Args{ - ID: f.Arg(0), - Spec: spec, - Conf: conf, - ControllerFD: b.controllerFD, - Device: os.NewFile(uintptr(b.deviceFD), "platform device"), - GoferFDs: b.ioFDs.GetArray(), - StdioFDs: b.stdioFDs.GetArray(), - NumCPU: b.cpuNum, - TotalMem: b.totalMem, - UserLogFD: b.userLogFD, - ProductName: b.productName, - PodInitConfigFD: b.podInitConfigFD, - SinkFDs: b.sinkFDs.GetArray(), - ProfileOpts: b.profileFDs.ToOpts(), + ID: f.Arg(0), + Spec: spec, + Conf: conf, + ControllerFD: b.controllerFD, + Device: os.NewFile(uintptr(b.deviceFD), "platform device"), + GoferFDs: b.ioFDs.GetArray(), + StdioFDs: b.stdioFDs.GetArray(), + OverlayFilestoreFD: b.overlayFilestoreFD, + NumCPU: b.cpuNum, + TotalMem: b.totalMem, + UserLogFD: b.userLogFD, + ProductName: b.productName, + PodInitConfigFD: b.podInitConfigFD, + SinkFDs: b.sinkFDs.GetArray(), + ProfileOpts: b.profileFDs.ToOpts(), } l, err := boot.New(bootArgs) if err != nil { diff --git a/runsc/cmd/do.go b/runsc/cmd/do.go index b8e28dae0..75fa67574 100644 --- a/runsc/cmd/do.go +++ b/runsc/cmd/do.go @@ -145,8 +145,10 @@ func (c *Do) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommand return util.Errorf("Error to retrieve hostname: %v", err) } - // Map the entire host file system, optionally using an overlay. - conf.Overlay = c.overlay + // If c.overlay is set, then forcefully enable overlay. + if overlay2 := conf.GetOverlay2(); c.overlay && !overlay2.Enabled() { + conf.Overlay = true + } absRoot, err := resolvePath(c.root) if err != nil { return util.Errorf("Error resolving root: %v", err) diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index 82af6a3c7..086dbe832 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -262,12 +262,13 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string) HostUDS: conf.GetHostUDS(), HostFifo: conf.HostFifo, }) + overlay2 := conf.GetOverlay2() // Start with root mount, then add any other additional mount as needed. cfgs = append(cfgs, connectionConfig{ sock: newSocket(g.ioFDs[0]), mountPath: "/", // fsgofer process is always chroot()ed. So serve root. - readonly: spec.Root.Readonly || conf.Overlay, + readonly: spec.Root.Readonly || overlay2.RootMount, }) log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly) @@ -287,7 +288,7 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string) cfgs = append(cfgs, connectionConfig{ sock: newSocket(g.ioFDs[mountIdx]), mountPath: m.Destination, - readonly: isReadonlyMount(m.Options) || conf.Overlay, + readonly: isReadonlyMount(m.Options) || overlay2.SubMounts, }) log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, g.ioFDs[mountIdx], cfgs[mountIdx].readonly) @@ -317,9 +318,10 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string) func (g *Gofer) serve9P(spec *specs.Spec, conf *config.Config, root string) subcommands.ExitStatus { // Start with root mount, then add any other additional mount as needed. + overlay2 := conf.GetOverlay2() ats := make([]p9.Attacher, 0, len(spec.Mounts)+1) ap, err := fsgofer.NewAttachPoint("/", fsgofer.Config{ - ROMount: spec.Root.Readonly || conf.Overlay, + ROMount: spec.Root.Readonly || overlay2.RootMount, HostUDS: conf.GetHostUDS(), HostFifo: conf.HostFifo, }) @@ -333,7 +335,7 @@ func (g *Gofer) serve9P(spec *specs.Spec, conf *config.Config, root string) subc for _, m := range spec.Mounts { if specutils.IsGoferMount(m) { cfg := fsgofer.Config{ - ROMount: isReadonlyMount(m.Options) || conf.Overlay, + ROMount: isReadonlyMount(m.Options) || overlay2.SubMounts, HostUDS: conf.GetHostUDS(), HostFifo: conf.HostFifo, } @@ -480,7 +482,7 @@ func setupRootFS(spec *specs.Spec, conf *config.Config) error { } // Check if root needs to be remounted as readonly. - if spec.Root.Readonly || conf.Overlay { + if spec.Root.Readonly || conf.GetOverlay2().RootMount { // If root is a mount point but not read-only, we can change mount options // to make it read-only for extra safety. log.Infof("Remounting root as readonly: %q", root) @@ -516,7 +518,7 @@ func setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath strin } flags := specutils.OptionsToFlags(m.Options) | unix.MS_BIND - if conf.Overlay { + if conf.GetOverlay2().SubMounts { // Force mount read-only if writes are not going to be sent to it. flags |= unix.MS_RDONLY } diff --git a/runsc/config/config.go b/runsc/config/config.go index 106e99b3e..4cf7943ff 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -19,6 +19,7 @@ package config import ( "fmt" + "strings" "time" "gvisor.dev/gvisor/pkg/refs" @@ -74,14 +75,19 @@ type Config struct { // FileAccessMounts indicates how non-root volumes are accessed. FileAccessMounts FileAccessType `flag:"file-access-mounts"` - // Overlay is whether to wrap the root filesystem in an overlay. + // Overlay is whether to wrap all mounts in an overlay. The upper tmpfs layer + // will be backed by application memory. Overlay bool `flag:"overlay"` + // Overlay2 holds configuration about wrapping mounts in overlayfs. + // DO NOT call it directly, use GetOverlay2() instead. + Overlay2 Overlay2 `flag:"overlay2"` + // FSGoferHostUDS is deprecated: use host-uds=all. FSGoferHostUDS bool `flag:"fsgofer-host-uds"` // HostUDS controls permission to access host Unix-domain sockets. - // DO NOT call it directly, use GetHostComm() instead. + // DO NOT call it directly, use GetHostUDS() instead. HostUDS HostUDS `flag:"host-uds"` // HostFifo controls permission to access host FIFO (or named pipes). @@ -273,7 +279,11 @@ type Config struct { } func (c *Config) validate() error { - if c.FileAccess == FileAccessShared && c.Overlay { + if c.Overlay && c.Overlay2.Enabled() { + // Deprecated flag was used together with flag that replaced it. + return fmt.Errorf("overlay flag has been replaced with overlay2 flag") + } + if overlay2 := c.GetOverlay2(); c.FileAccess == FileAccessShared && overlay2.Enabled() { return fmt.Errorf("overlay flag is incompatible with shared file access") } if c.NumNetworkChannels <= 0 { @@ -314,6 +324,19 @@ func (c *Config) GetHostUDS() HostUDS { return c.HostUDS } +// GetOverlay2 returns the overlay configuration, taking into consideration all +// flags that affect the result. +func (c *Config) GetOverlay2() Overlay2 { + if c.Overlay { + if c.Overlay2.Enabled() { + panic(fmt.Sprintf("Overlay2 cannot be set when --overlay=true")) + } + // Using deprecated flag, honor it to avoid breaking users. + return Overlay2{RootMount: true, SubMounts: true, FilestoreDir: ""} + } + return c.Overlay2 +} + // FileAccessType tells how the filesystem is accessed. type FileAccessType int @@ -591,3 +614,80 @@ func (g HostFifo) String() string { func (g HostFifo) AllowOpen() bool { return g&HostFifoOpen != 0 } + +// Overlay2 holds the configuration for setting up overlay filesystems for the +// container. +type Overlay2 struct { + RootMount bool + SubMounts bool + FilestoreDir string +} + +func defaultOverlay2() *Overlay2 { + return &Overlay2{} +} + +// Set implements flag.Value. +func (o *Overlay2) Set(v string) error { + if v == "none" { + // Defaults are correct. + return nil + } + vs := strings.Split(v, ":") + if len(vs) != 2 { + return fmt.Errorf("expected format is --overlay2={mount}:{medium}, got %q", v) + } + + switch mount := vs[0]; mount { + case "root": + o.RootMount = true + case "all": + o.RootMount = true + o.SubMounts = true + default: + return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount) + } + + switch medium := vs[1]; medium { + case "memory": + o.FilestoreDir = "" + default: + o.FilestoreDir = medium + } + return nil +} + +// Get implements flag.Value. +func (o *Overlay2) Get() any { + return *o +} + +// String implements flag.Value. +func (o Overlay2) String() string { + if !o.RootMount && !o.SubMounts { + return "none" + } + res := "" + switch { + case o.RootMount && o.SubMounts: + res = "all" + case o.RootMount: + res = "root" + default: + panic("invalid state of subMounts = true and rootMount = false") + } + + res += ":" + switch o.FilestoreDir { + case "": + res += "memory" + default: + res += o.FilestoreDir + } + return res +} + +// Enabled returns true if overlay option is enabled for any mounts. +func (o *Overlay2) Enabled() bool { + return o.RootMount || o.SubMounts +} diff --git a/runsc/config/config_test.go b/runsc/config/config_test.go index b5b0889f3..b4f512dcb 100644 --- a/runsc/config/config_test.go +++ b/runsc/config/config_test.go @@ -178,7 +178,7 @@ func TestValidationFail(t *testing.T) { error: "num_network_channels must be > 0", }, { - name: "fsgofer-host-uds+comm:open", + name: "fsgofer-host-uds+host-uds:open", flags: map[string]string{ "fsgofer-host-uds": "true", "host-uds": "open", @@ -186,7 +186,7 @@ func TestValidationFail(t *testing.T) { error: "fsgofer-host-uds has been replaced with host-uds flag", }, { - name: "fsgofer-host-uds+comm:create", + name: "fsgofer-host-uds+host-uds:create", flags: map[string]string{ "fsgofer-host-uds": "true", "host-uds": "create", @@ -194,13 +194,29 @@ func TestValidationFail(t *testing.T) { error: "fsgofer-host-uds has been replaced with host-uds flag", }, { - name: "fsgofer-host-uds+comm:all", + name: "fsgofer-host-uds+host-uds:all", flags: map[string]string{ "fsgofer-host-uds": "true", "host-uds": "all", }, error: "fsgofer-host-uds has been replaced with host-uds flag", }, + { + name: "overlay+overlay2:root", + flags: map[string]string{ + "overlay": "true", + "overlay2": "root:memory", + }, + error: "overlay flag has been replaced with overlay2 flag", + }, + { + name: "overlay+overlay2:all", + flags: map[string]string{ + "overlay": "true", + "overlay2": "all:memory", + }, + error: "overlay flag has been replaced with overlay2 flag", + }, } { t.Run(tc.name, func(t *testing.T) { testFlags := flag.NewFlagSet("test", flag.ContinueOnError) diff --git a/runsc/config/flags.go b/runsc/config/flags.go index 48270bade..3ebe005c4 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -78,7 +78,8 @@ func RegisterFlags(flagSet *flag.FlagSet) { // Flags that control sandbox runtime behavior: FS related. flagSet.Var(fileAccessTypePtr(FileAccessExclusive), "file-access", "specifies which filesystem validation to use for the root mount: exclusive (default), shared.") flagSet.Var(fileAccessTypePtr(FileAccessShared), "file-access-mounts", "specifies which filesystem validation to use for volumes other than the root mount: shared (default), exclusive.") - flagSet.Bool("overlay", false, "wrap filesystem mounts with writable overlay. All modifications are stored in memory inside the sandbox.") + flagSet.Bool("overlay", false, "DEPRECATED: use --overlay2=all:memory to achieve the same effect") + flagSet.Var(defaultOverlay2(), "overlay2", "wrap mounts with overlayfs. Format is {mount}:{medium}, where 'mount' can be 'root' or 'all' and medium can be 'memory' or existing directory path in which filestore will be created. 'none' will turn overlay mode off.") flagSet.Bool("fsgofer-host-uds", false, "DEPRECATED: use host-uds=all") flagSet.Var(hostUDSPtr(HostUDSNone), "host-uds", "controls permission to access host Unix-domain sockets. Values: none|open|create|all, default: none") flagSet.Var(hostFifoPtr(HostFifoNone), "host-fifo", "controls permission to access host FIFOs (or named pipes). Values: none|open, default: none") diff --git a/runsc/container/container.go b/runsc/container/container.go index 9dc0d75bc..6859a5d0a 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -256,6 +256,10 @@ func New(conf *config.Config, args Args) (*Container, error) { } } c.CompatCgroup = cgroup.CgroupJSON{Cgroup: subCgroup} + overlayFilestoreFile, err := createOverlayFilestore(conf.GetOverlay2()) + if err != nil { + return nil, err + } if err := runInCgroup(parentCgroup, func() error { ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached) if err != nil { @@ -265,15 +269,16 @@ func New(conf *config.Config, args Args) (*Container, error) { // Start a new sandbox for this container. Any errors after this point // must destroy the container. sandArgs := &sandbox.Args{ - ID: sandboxID, - Spec: args.Spec, - BundleDir: args.BundleDir, - ConsoleSocket: args.ConsoleSocket, - UserLog: args.UserLog, - IOFiles: ioFiles, - MountsFile: specFile, - Cgroup: parentCgroup, - Attached: args.Attached, + ID: sandboxID, + Spec: args.Spec, + BundleDir: args.BundleDir, + ConsoleSocket: args.ConsoleSocket, + UserLog: args.UserLog, + IOFiles: ioFiles, + MountsFile: specFile, + Cgroup: parentCgroup, + Attached: args.Attached, + OverlayFilestoreFile: overlayFilestoreFile, } sand, err := sandbox.New(conf, sandArgs) if err != nil { @@ -763,6 +768,28 @@ func (c *Container) Destroy() error { return fmt.Errorf(strings.Join(errs, "\n")) } +func createOverlayFilestore(overlay2 config.Overlay2) (*os.File, error) { + if overlay2.FilestoreDir == "" { + return nil, nil + } + fileInfo, err := os.Stat(overlay2.FilestoreDir) + if err != nil { + return nil, fmt.Errorf("failed to stat overlay filestore directory %q: %v", overlay2.FilestoreDir, err) + } + if !fileInfo.IsDir() { + return nil, fmt.Errorf("overlay2 flag should specify an existing directory") + } + // Create an unnamed temporary file in filestore directory using + // O_TMPFILE. This file will be deleted when the container exits. + // Also specify O_EXCL to prevent this file from being linked into the + // filesystem. See open(2) man page's section for O_TMPFILE for details. + unnamedTmpFD, err := unix.Open(overlay2.FilestoreDir, unix.O_TMPFILE|unix.O_RDWR|unix.O_EXCL, 0666) + if err != nil { + return nil, fmt.Errorf("failed to create an unnamed temporary file inside %q", overlay2.FilestoreDir) + } + return os.NewFile(uintptr(unnamedTmpFD), "overlay-filestore"), nil +} + // saveLocked saves the container metadata to a file. // // Precondition: container must be locked with container.lock(). diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index a97464c8f..c56a9454e 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -156,6 +156,10 @@ type Args struct { // appear in the spec. IOFiles []*os.File + // OverlayFilestoreFile is the regular file that will back the tmpfs upper + // mount in the overlay mounts. + OverlayFilestoreFile *os.File + // MountsFile is a file container mount information from the spec. It's // equivalent to the mounts from the spec, except that all paths have been // resolved to their final absolute location. @@ -591,6 +595,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn // If there is a gofer, sends all socket ends to the sandbox. donations.DonateAndClose("io-fds", args.IOFiles...) + donations.DonateAndClose("overlay-filestore-fd", args.OverlayFilestoreFile) donations.DonateAndClose("mounts-fd", args.MountsFile) donations.Donate("start-sync-fd", startSyncFile) if err := donations.OpenAndDonate("user-log-fd", args.UserLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { diff --git a/test/runner/main.go b/test/runner/main.go index eeacc0024..e194a49ad 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -204,7 +204,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { "-file-access", *fileAccess, } if *overlay { - args = append(args, "-overlay") + args = append(args, "-overlay2=all:/tmp") } if *fuse { args = append(args, "-fuse")