diff --git a/Makefile b/Makefile index 23f8d13c5..548ffa47e 100644 --- a/Makefile +++ b/Makefile @@ -274,8 +274,9 @@ docker-tests: load-basic $(RUNTIME_BIN) @$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test) .PHONY: docker-tests +# TODO(b/241832602): Run overlay tests with host filestore option after S/R support is added. overlay-tests: load-basic $(RUNTIME_BIN) - @$(call install_runtime,$(RUNTIME),--overlay) + @$(call install_runtime,$(RUNTIME),--overlay2=all:memory) @$(call test_runtime,$(RUNTIME),--test_env=TEST_OVERLAY=true $(INTEGRATION_TARGETS)) .PHONY: overlay-tests diff --git a/g3doc/user_guide/filesystem.md b/g3doc/user_guide/filesystem.md index 3122987fa..1774e809e 100644 --- a/g3doc/user_guide/filesystem.md +++ b/g3doc/user_guide/filesystem.md @@ -27,13 +27,30 @@ configuration (`/etc/docker/daemon.json`) and restart the Docker daemon: "runsc": { "path": "/usr/local/bin/runsc", "runtimeArgs": [ - "--overlay" + "--overlay2=all:memory" ] } } } ``` +### Root Filesystem Overlay + +Any modifications to the root filesystem is destroyed with the container. So it +almost always makes sense to apply an overlay on top of the root filesystem. +This can drastically boost performance, as runsc will handle root filesystem +changes completely in memory instead of making costly round trips to the gofer +and make syscalls to modify the host. + +However, holding so much file data in memory for the root filesystem can bloat +up container memory usage. To circumvent this, you can have root mount's upper +layer (tmpfs) be backed by a host file, so all file data is stored on disk. + +The newer `--overlay2` flag allows you to achieve these. You can specify +`--overlay2=root:/dir/path` in `runtimeArgs`. `/dir/path` can be any existing +directory inside which the tmpfs filestore file will be created. When the +container exits, this filestore file will be destroyed. + ## Shared root filesystem The root filesystem is where the image is extracted and is not generally diff --git a/pkg/sentry/fsimpl/tmpfs/tmpfs.go b/pkg/sentry/fsimpl/tmpfs/tmpfs.go index daa452797..7f45fa26a 100644 --- a/pkg/sentry/fsimpl/tmpfs/tmpfs.go +++ b/pkg/sentry/fsimpl/tmpfs/tmpfs.go @@ -129,6 +129,10 @@ type FilesystemOpts struct { // MaxFilenameLen is the maximum filename length allowed by the tmpfs. MaxFilenameLen int + + // Filestore is the MemoryFile that will be used to store file data. If this + // is nil, then MemoryFileProviderFromContext() is used. + Filestore *pgalloc.MemoryFile } // GetFilesystem implements vfs.FilesystemType.GetFilesystem. @@ -148,6 +152,9 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt if tmpfsOpts.FilesystemType != nil { newFSType = tmpfsOpts.FilesystemType } + if tmpfsOpts.Filestore != nil { + mfp = tmpfsOpts.Filestore + } } mopts := vfs.GenericParseMountOptions(opts.Data) diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index 146784ce9..e8e643768 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -1353,6 +1353,11 @@ func (f *MemoryFile) startEvictionGoroutineLocked(user EvictableMemoryUser, info }() } +// MemoryFile implements MemoryFileProvider.MemoryFile. +func (f *MemoryFile) MemoryFile() *MemoryFile { + return f +} + // WaitForEvictions blocks until f is no longer evicting any evictable // allocations. func (f *MemoryFile) WaitForEvictions() { diff --git a/pkg/test/testutil/testutil.go b/pkg/test/testutil/testutil.go index 0236a6cff..5db023969 100644 --- a/pkg/test/testutil/testutil.go +++ b/pkg/test/testutil/testutil.go @@ -55,7 +55,7 @@ var ( isRunningWithHostNet = flag.Bool("hostnet", BoolFromEnv("HOSTNET", false), "whether test is running with hostnet") runscPath = flag.String("runsc", os.Getenv("RUNTIME"), "path to runsc binary") // Note: flag overlay is already taken by runsc. - isRunningWithOverlay = flag.Bool("test-overlay", BoolFromEnv("TEST_OVERLAY", false), "whether test is running with --overlay") + isRunningWithOverlay = flag.Bool("test-overlay", BoolFromEnv("TEST_OVERLAY", false), "whether test is running with --overlay2") ) // StringFromEnv returns the value of the named environment variable, or `def` if unset/empty. diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 6ce4fb8cb..3d87e3ab8 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -101,6 +101,10 @@ type containerInfo struct { // goferFDs are the FDs that attach the sandbox to the gofers. goferFDs []*fd.FD + + // overlayFilestore is the memory file that will back the overlay mount's + // upper tmpfs layer. + overlayFilestore *pgalloc.MemoryFile } // Loader keeps state needed to start the kernel and run the container. @@ -200,6 +204,9 @@ type Args struct { // StdioFDs is the stdio for the application. The Loader takes ownership of // these FDs and may close them at any time. StdioFDs []int + // OverlayFilestoreFD is the host FD to a regular file which will be used to + // back the overlay mount's upper tmpfs layer. + OverlayFilestoreFD int // NumCPU is the number of CPUs to create inside the sandbox. NumCPU int // TotalMem is the initial amount of total memory to report back to the @@ -268,6 +275,14 @@ func New(args Args) (*Loader, error) { for _, goferFD := range args.GoferFDs { info.goferFDs = append(info.goferFDs, fd.New(goferFD)) } + if args.OverlayFilestoreFD >= 0 { + f := os.NewFile(uintptr(args.OverlayFilestoreFD), "overlay-filestore") + mf, err := pgalloc.NewMemoryFile(f, pgalloc.MemoryFileOpts{}) + if err != nil { + return nil, fmt.Errorf("tmpfs.FilesystemType.GetFilesystem: failed to create memory file from host file: %w", err) + } + info.overlayFilestore = mf + } // Create kernel and platform. p, err := createPlatform(args.Conf, args.Device) @@ -513,6 +528,9 @@ func (l *Loader) Destroy() { for _, f := range l.root.goferFDs { _ = f.Close() } + if l.root.overlayFilestore != nil { + l.root.overlayFilestore.Destroy() + } l.stopProfiling() } @@ -747,6 +765,9 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st conf: conf, spec: spec, goferFDs: goferFDs, + // Note that K8s starts all containers in a pod (root and subcontainers) + // with the same config. So overlayFilestore can be copied. + overlayFilestore: l.root.overlayFilestore, } info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns) if err != nil { diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index f6d20af93..4ef88dacd 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -125,13 +125,14 @@ func createLoader(spec *specs.Spec) (*Loader, func(), error) { } args := Args{ - ID: "foo", - Spec: spec, - Conf: conf, - ControllerFD: fd, - GoferFDs: []int{sandEnd}, - StdioFDs: stdio, - PodInitConfigFD: -1, + ID: "foo", + Spec: spec, + Conf: conf, + ControllerFD: fd, + GoferFDs: []int{sandEnd}, + StdioFDs: stdio, + OverlayFilestoreFD: -1, + PodInitConfigFD: -1, } l, err := New(args) if err != nil { diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 99a4e9132..edb85838d 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -46,6 +46,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/vfs" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/specutils" @@ -322,6 +323,10 @@ type containerMounter struct { // fds is the list of FDs to be dispensed for mounts that require it. fds fdDispenser + // overlayFilestore is the memory file that will back the overlay mount's + // upper tmpfs layer. + overlayFilestore *pgalloc.MemoryFile + k *kernel.Kernel hints *podMountHints @@ -333,12 +338,13 @@ type containerMounter struct { func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string) *containerMounter { return &containerMounter{ - root: info.spec.Root, - mounts: compileMounts(info.spec, info.conf), - fds: fdDispenser{fds: info.goferFDs}, - k: k, - hints: hints, - productName: productName, + root: info.spec.Root, + mounts: compileMounts(info.spec, info.conf), + fds: fdDispenser{fds: info.goferFDs}, + overlayFilestore: info.overlayFilestore, + k: k, + hints: hints, + productName: productName, } } @@ -425,7 +431,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi } fsName := gofer.Name - if conf.Overlay && !c.root.Readonly { + if conf.GetOverlay2().RootMount && !c.root.Readonly { log.Infof("Adding overlay on top of root") var err error var cleanup func() @@ -488,6 +494,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, creds *auth.Cre // Upper is a tmpfs mount to keep all modifications inside the sandbox. upperOpts.GetFilesystemOptions.InternalData = tmpfs.FilesystemOpts{ RootFileType: uint16(rootType), + Filestore: c.overlayFilestore, } upper, err := c.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts) if err != nil { @@ -717,7 +724,7 @@ func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountA } // If configured, add overlay to all writable mounts. - useOverlay = conf.Overlay && !parseMountOptions(m.mount.Options).ReadOnly + useOverlay = conf.GetOverlay2().SubMounts && !parseMountOptions(m.mount.Options).ReadOnly case cgroupfs.Name: var err error diff --git a/runsc/cli/main.go b/runsc/cli/main.go index 972d957ef..57adae2a8 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -224,7 +224,9 @@ func Main(version string) { log.Infof("Configuration:") log.Infof("\t\tRootDir: %s", conf.RootDir) log.Infof("\t\tPlatform: %v", conf.Platform) - log.Infof("\t\tFileAccess: %v, overlay: %t", conf.FileAccess, conf.Overlay) + log.Infof("\t\tFileAccess: %v", conf.FileAccess) + overlay2 := conf.GetOverlay2() + log.Infof("\t\tOverlay: Root=%t, SubMounts=%t, FilestoreDir=%q", overlay2.RootMount, overlay2.SubMounts, overlay2.FilestoreDir) log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets) log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls) log.Infof("\t\tLISAFS: %t", conf.Lisafs) diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 38d3f8781..63a502333 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -56,6 +56,10 @@ type Boot struct { // ioFDs is the list of FDs used to connect to FS gofers. ioFDs intFlags + // overlayFilestoreFD is the host FD to the regular file which will back the + // overlay's upper tmpfs mount for all containers. + overlayFilestoreFD int + // stdioFDs are the fds for stdin, stdout, and stderr. They must be // provided in that order. stdioFDs intFlags @@ -142,6 +146,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file") f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") + f.IntVar(&b.overlayFilestoreFD, "overlay-filestore-fd", -1, "FD to a regular file which will be used to back the overlay's tmpfs upper mount.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") @@ -302,20 +307,21 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma // Create the loader. bootArgs := boot.Args{ - ID: f.Arg(0), - Spec: spec, - Conf: conf, - ControllerFD: b.controllerFD, - Device: os.NewFile(uintptr(b.deviceFD), "platform device"), - GoferFDs: b.ioFDs.GetArray(), - StdioFDs: b.stdioFDs.GetArray(), - NumCPU: b.cpuNum, - TotalMem: b.totalMem, - UserLogFD: b.userLogFD, - ProductName: b.productName, - PodInitConfigFD: b.podInitConfigFD, - SinkFDs: b.sinkFDs.GetArray(), - ProfileOpts: b.profileFDs.ToOpts(), + ID: f.Arg(0), + Spec: spec, + Conf: conf, + ControllerFD: b.controllerFD, + Device: os.NewFile(uintptr(b.deviceFD), "platform device"), + GoferFDs: b.ioFDs.GetArray(), + StdioFDs: b.stdioFDs.GetArray(), + OverlayFilestoreFD: b.overlayFilestoreFD, + NumCPU: b.cpuNum, + TotalMem: b.totalMem, + UserLogFD: b.userLogFD, + ProductName: b.productName, + PodInitConfigFD: b.podInitConfigFD, + SinkFDs: b.sinkFDs.GetArray(), + ProfileOpts: b.profileFDs.ToOpts(), } l, err := boot.New(bootArgs) if err != nil { diff --git a/runsc/cmd/do.go b/runsc/cmd/do.go index b8e28dae0..75fa67574 100644 --- a/runsc/cmd/do.go +++ b/runsc/cmd/do.go @@ -145,8 +145,10 @@ func (c *Do) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommand return util.Errorf("Error to retrieve hostname: %v", err) } - // Map the entire host file system, optionally using an overlay. - conf.Overlay = c.overlay + // If c.overlay is set, then forcefully enable overlay. + if overlay2 := conf.GetOverlay2(); c.overlay && !overlay2.Enabled() { + conf.Overlay = true + } absRoot, err := resolvePath(c.root) if err != nil { return util.Errorf("Error resolving root: %v", err) diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index 82af6a3c7..086dbe832 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -262,12 +262,13 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string) HostUDS: conf.GetHostUDS(), HostFifo: conf.HostFifo, }) + overlay2 := conf.GetOverlay2() // Start with root mount, then add any other additional mount as needed. cfgs = append(cfgs, connectionConfig{ sock: newSocket(g.ioFDs[0]), mountPath: "/", // fsgofer process is always chroot()ed. So serve root. - readonly: spec.Root.Readonly || conf.Overlay, + readonly: spec.Root.Readonly || overlay2.RootMount, }) log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly) @@ -287,7 +288,7 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string) cfgs = append(cfgs, connectionConfig{ sock: newSocket(g.ioFDs[mountIdx]), mountPath: m.Destination, - readonly: isReadonlyMount(m.Options) || conf.Overlay, + readonly: isReadonlyMount(m.Options) || overlay2.SubMounts, }) log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, g.ioFDs[mountIdx], cfgs[mountIdx].readonly) @@ -317,9 +318,10 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string) func (g *Gofer) serve9P(spec *specs.Spec, conf *config.Config, root string) subcommands.ExitStatus { // Start with root mount, then add any other additional mount as needed. + overlay2 := conf.GetOverlay2() ats := make([]p9.Attacher, 0, len(spec.Mounts)+1) ap, err := fsgofer.NewAttachPoint("/", fsgofer.Config{ - ROMount: spec.Root.Readonly || conf.Overlay, + ROMount: spec.Root.Readonly || overlay2.RootMount, HostUDS: conf.GetHostUDS(), HostFifo: conf.HostFifo, }) @@ -333,7 +335,7 @@ func (g *Gofer) serve9P(spec *specs.Spec, conf *config.Config, root string) subc for _, m := range spec.Mounts { if specutils.IsGoferMount(m) { cfg := fsgofer.Config{ - ROMount: isReadonlyMount(m.Options) || conf.Overlay, + ROMount: isReadonlyMount(m.Options) || overlay2.SubMounts, HostUDS: conf.GetHostUDS(), HostFifo: conf.HostFifo, } @@ -480,7 +482,7 @@ func setupRootFS(spec *specs.Spec, conf *config.Config) error { } // Check if root needs to be remounted as readonly. - if spec.Root.Readonly || conf.Overlay { + if spec.Root.Readonly || conf.GetOverlay2().RootMount { // If root is a mount point but not read-only, we can change mount options // to make it read-only for extra safety. log.Infof("Remounting root as readonly: %q", root) @@ -516,7 +518,7 @@ func setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath strin } flags := specutils.OptionsToFlags(m.Options) | unix.MS_BIND - if conf.Overlay { + if conf.GetOverlay2().SubMounts { // Force mount read-only if writes are not going to be sent to it. flags |= unix.MS_RDONLY } diff --git a/runsc/config/config.go b/runsc/config/config.go index 106e99b3e..4cf7943ff 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -19,6 +19,7 @@ package config import ( "fmt" + "strings" "time" "gvisor.dev/gvisor/pkg/refs" @@ -74,14 +75,19 @@ type Config struct { // FileAccessMounts indicates how non-root volumes are accessed. FileAccessMounts FileAccessType `flag:"file-access-mounts"` - // Overlay is whether to wrap the root filesystem in an overlay. + // Overlay is whether to wrap all mounts in an overlay. The upper tmpfs layer + // will be backed by application memory. Overlay bool `flag:"overlay"` + // Overlay2 holds configuration about wrapping mounts in overlayfs. + // DO NOT call it directly, use GetOverlay2() instead. + Overlay2 Overlay2 `flag:"overlay2"` + // FSGoferHostUDS is deprecated: use host-uds=all. FSGoferHostUDS bool `flag:"fsgofer-host-uds"` // HostUDS controls permission to access host Unix-domain sockets. - // DO NOT call it directly, use GetHostComm() instead. + // DO NOT call it directly, use GetHostUDS() instead. HostUDS HostUDS `flag:"host-uds"` // HostFifo controls permission to access host FIFO (or named pipes). @@ -273,7 +279,11 @@ type Config struct { } func (c *Config) validate() error { - if c.FileAccess == FileAccessShared && c.Overlay { + if c.Overlay && c.Overlay2.Enabled() { + // Deprecated flag was used together with flag that replaced it. + return fmt.Errorf("overlay flag has been replaced with overlay2 flag") + } + if overlay2 := c.GetOverlay2(); c.FileAccess == FileAccessShared && overlay2.Enabled() { return fmt.Errorf("overlay flag is incompatible with shared file access") } if c.NumNetworkChannels <= 0 { @@ -314,6 +324,19 @@ func (c *Config) GetHostUDS() HostUDS { return c.HostUDS } +// GetOverlay2 returns the overlay configuration, taking into consideration all +// flags that affect the result. +func (c *Config) GetOverlay2() Overlay2 { + if c.Overlay { + if c.Overlay2.Enabled() { + panic(fmt.Sprintf("Overlay2 cannot be set when --overlay=true")) + } + // Using deprecated flag, honor it to avoid breaking users. + return Overlay2{RootMount: true, SubMounts: true, FilestoreDir: ""} + } + return c.Overlay2 +} + // FileAccessType tells how the filesystem is accessed. type FileAccessType int @@ -591,3 +614,80 @@ func (g HostFifo) String() string { func (g HostFifo) AllowOpen() bool { return g&HostFifoOpen != 0 } + +// Overlay2 holds the configuration for setting up overlay filesystems for the +// container. +type Overlay2 struct { + RootMount bool + SubMounts bool + FilestoreDir string +} + +func defaultOverlay2() *Overlay2 { + return &Overlay2{} +} + +// Set implements flag.Value. +func (o *Overlay2) Set(v string) error { + if v == "none" { + // Defaults are correct. + return nil + } + vs := strings.Split(v, ":") + if len(vs) != 2 { + return fmt.Errorf("expected format is --overlay2={mount}:{medium}, got %q", v) + } + + switch mount := vs[0]; mount { + case "root": + o.RootMount = true + case "all": + o.RootMount = true + o.SubMounts = true + default: + return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount) + } + + switch medium := vs[1]; medium { + case "memory": + o.FilestoreDir = "" + default: + o.FilestoreDir = medium + } + return nil +} + +// Get implements flag.Value. +func (o *Overlay2) Get() any { + return *o +} + +// String implements flag.Value. +func (o Overlay2) String() string { + if !o.RootMount && !o.SubMounts { + return "none" + } + res := "" + switch { + case o.RootMount && o.SubMounts: + res = "all" + case o.RootMount: + res = "root" + default: + panic("invalid state of subMounts = true and rootMount = false") + } + + res += ":" + switch o.FilestoreDir { + case "": + res += "memory" + default: + res += o.FilestoreDir + } + return res +} + +// Enabled returns true if overlay option is enabled for any mounts. +func (o *Overlay2) Enabled() bool { + return o.RootMount || o.SubMounts +} diff --git a/runsc/config/config_test.go b/runsc/config/config_test.go index b5b0889f3..b4f512dcb 100644 --- a/runsc/config/config_test.go +++ b/runsc/config/config_test.go @@ -178,7 +178,7 @@ func TestValidationFail(t *testing.T) { error: "num_network_channels must be > 0", }, { - name: "fsgofer-host-uds+comm:open", + name: "fsgofer-host-uds+host-uds:open", flags: map[string]string{ "fsgofer-host-uds": "true", "host-uds": "open", @@ -186,7 +186,7 @@ func TestValidationFail(t *testing.T) { error: "fsgofer-host-uds has been replaced with host-uds flag", }, { - name: "fsgofer-host-uds+comm:create", + name: "fsgofer-host-uds+host-uds:create", flags: map[string]string{ "fsgofer-host-uds": "true", "host-uds": "create", @@ -194,13 +194,29 @@ func TestValidationFail(t *testing.T) { error: "fsgofer-host-uds has been replaced with host-uds flag", }, { - name: "fsgofer-host-uds+comm:all", + name: "fsgofer-host-uds+host-uds:all", flags: map[string]string{ "fsgofer-host-uds": "true", "host-uds": "all", }, error: "fsgofer-host-uds has been replaced with host-uds flag", }, + { + name: "overlay+overlay2:root", + flags: map[string]string{ + "overlay": "true", + "overlay2": "root:memory", + }, + error: "overlay flag has been replaced with overlay2 flag", + }, + { + name: "overlay+overlay2:all", + flags: map[string]string{ + "overlay": "true", + "overlay2": "all:memory", + }, + error: "overlay flag has been replaced with overlay2 flag", + }, } { t.Run(tc.name, func(t *testing.T) { testFlags := flag.NewFlagSet("test", flag.ContinueOnError) diff --git a/runsc/config/flags.go b/runsc/config/flags.go index 48270bade..3ebe005c4 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -78,7 +78,8 @@ func RegisterFlags(flagSet *flag.FlagSet) { // Flags that control sandbox runtime behavior: FS related. flagSet.Var(fileAccessTypePtr(FileAccessExclusive), "file-access", "specifies which filesystem validation to use for the root mount: exclusive (default), shared.") flagSet.Var(fileAccessTypePtr(FileAccessShared), "file-access-mounts", "specifies which filesystem validation to use for volumes other than the root mount: shared (default), exclusive.") - flagSet.Bool("overlay", false, "wrap filesystem mounts with writable overlay. All modifications are stored in memory inside the sandbox.") + flagSet.Bool("overlay", false, "DEPRECATED: use --overlay2=all:memory to achieve the same effect") + flagSet.Var(defaultOverlay2(), "overlay2", "wrap mounts with overlayfs. Format is {mount}:{medium}, where 'mount' can be 'root' or 'all' and medium can be 'memory' or existing directory path in which filestore will be created. 'none' will turn overlay mode off.") flagSet.Bool("fsgofer-host-uds", false, "DEPRECATED: use host-uds=all") flagSet.Var(hostUDSPtr(HostUDSNone), "host-uds", "controls permission to access host Unix-domain sockets. Values: none|open|create|all, default: none") flagSet.Var(hostFifoPtr(HostFifoNone), "host-fifo", "controls permission to access host FIFOs (or named pipes). Values: none|open, default: none") diff --git a/runsc/container/container.go b/runsc/container/container.go index 9dc0d75bc..6859a5d0a 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -256,6 +256,10 @@ func New(conf *config.Config, args Args) (*Container, error) { } } c.CompatCgroup = cgroup.CgroupJSON{Cgroup: subCgroup} + overlayFilestoreFile, err := createOverlayFilestore(conf.GetOverlay2()) + if err != nil { + return nil, err + } if err := runInCgroup(parentCgroup, func() error { ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached) if err != nil { @@ -265,15 +269,16 @@ func New(conf *config.Config, args Args) (*Container, error) { // Start a new sandbox for this container. Any errors after this point // must destroy the container. sandArgs := &sandbox.Args{ - ID: sandboxID, - Spec: args.Spec, - BundleDir: args.BundleDir, - ConsoleSocket: args.ConsoleSocket, - UserLog: args.UserLog, - IOFiles: ioFiles, - MountsFile: specFile, - Cgroup: parentCgroup, - Attached: args.Attached, + ID: sandboxID, + Spec: args.Spec, + BundleDir: args.BundleDir, + ConsoleSocket: args.ConsoleSocket, + UserLog: args.UserLog, + IOFiles: ioFiles, + MountsFile: specFile, + Cgroup: parentCgroup, + Attached: args.Attached, + OverlayFilestoreFile: overlayFilestoreFile, } sand, err := sandbox.New(conf, sandArgs) if err != nil { @@ -763,6 +768,28 @@ func (c *Container) Destroy() error { return fmt.Errorf(strings.Join(errs, "\n")) } +func createOverlayFilestore(overlay2 config.Overlay2) (*os.File, error) { + if overlay2.FilestoreDir == "" { + return nil, nil + } + fileInfo, err := os.Stat(overlay2.FilestoreDir) + if err != nil { + return nil, fmt.Errorf("failed to stat overlay filestore directory %q: %v", overlay2.FilestoreDir, err) + } + if !fileInfo.IsDir() { + return nil, fmt.Errorf("overlay2 flag should specify an existing directory") + } + // Create an unnamed temporary file in filestore directory using + // O_TMPFILE. This file will be deleted when the container exits. + // Also specify O_EXCL to prevent this file from being linked into the + // filesystem. See open(2) man page's section for O_TMPFILE for details. + unnamedTmpFD, err := unix.Open(overlay2.FilestoreDir, unix.O_TMPFILE|unix.O_RDWR|unix.O_EXCL, 0666) + if err != nil { + return nil, fmt.Errorf("failed to create an unnamed temporary file inside %q", overlay2.FilestoreDir) + } + return os.NewFile(uintptr(unnamedTmpFD), "overlay-filestore"), nil +} + // saveLocked saves the container metadata to a file. // // Precondition: container must be locked with container.lock(). diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index a97464c8f..c56a9454e 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -156,6 +156,10 @@ type Args struct { // appear in the spec. IOFiles []*os.File + // OverlayFilestoreFile is the regular file that will back the tmpfs upper + // mount in the overlay mounts. + OverlayFilestoreFile *os.File + // MountsFile is a file container mount information from the spec. It's // equivalent to the mounts from the spec, except that all paths have been // resolved to their final absolute location. @@ -591,6 +595,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn // If there is a gofer, sends all socket ends to the sandbox. donations.DonateAndClose("io-fds", args.IOFiles...) + donations.DonateAndClose("overlay-filestore-fd", args.OverlayFilestoreFile) donations.DonateAndClose("mounts-fd", args.MountsFile) donations.Donate("start-sync-fd", startSyncFile) if err := donations.OpenAndDonate("user-log-fd", args.UserLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { diff --git a/test/runner/main.go b/test/runner/main.go index eeacc0024..e194a49ad 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -204,7 +204,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { "-file-access", *fileAccess, } if *overlay { - args = append(args, "-overlay") + args = append(args, "-overlay2=all:/tmp") } if *fuse { args = append(args, "-fuse")