Add overlay2 flag in runsc.

--overlay2 flag supersedes --overlay flag. It allows more granular
configuration for overlayfs in runsc. It does so in two ways:

1. Allows to apply overlay on all mounts or only the root mount.
   --overlay applies overlay to all mounts.
2. Allows to specify if overlay's upper layer should be backed by
   container memory or disk. --overlay always used container memory.

Allowing tmpfs to be backed by a file on disk prevents the container
memory from bloating up. Note that the tmpfs filesystem tree will
still be stored in sentry memory.

Using overlay on the root filesystem, helps avoid expensive
communication with the gofer process. The root filesystem of the
container is not preserved across container lifecycle. So we don't
need to keep updating the host filesystem, which will anyways be
destroyed once the container is destroyed. It is wasted effort.
Instead we keep all the changes to the root filesystem in tmpfs which
is directly accessible by the sentry.

The host file is created as an unnamed file using O_TMPFILE. Support
has been added for sub-containers too. Save/restore support is
still lacking.

Co-authored-by: Andrei Vagin <avagin@gmail.com>
PiperOrigin-RevId: 491988485

Makefile

@@ -274,8 +274,9 @@ docker-tests: load-basic $(RUNTIME_BIN)
 	@$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test)
 .PHONY: docker-tests
 
+# TODO(b/241832602): Run overlay tests with host filestore option after S/R support is added.
 overlay-tests: load-basic $(RUNTIME_BIN)
-	@$(call install_runtime,$(RUNTIME),--overlay)
+	@$(call install_runtime,$(RUNTIME),--overlay2=all:memory)
 	@$(call test_runtime,$(RUNTIME),--test_env=TEST_OVERLAY=true $(INTEGRATION_TARGETS))
 .PHONY: overlay-tests
 

g3doc/user_guide/filesystem.md

@@ -27,13 +27,30 @@ configuration (`/etc/docker/daemon.json`) and restart the Docker daemon:
         "runsc": {
             "path": "/usr/local/bin/runsc",
             "runtimeArgs": [
-                "--overlay"
+                "--overlay2=all:memory"
             ]
        }
     }
 }
 ```
 
+### Root Filesystem Overlay
+
+Any modifications to the root filesystem is destroyed with the container. So it
+almost always makes sense to apply an overlay on top of the root filesystem.
+This can drastically boost performance, as runsc will handle root filesystem
+changes completely in memory instead of making costly round trips to the gofer
+and make syscalls to modify the host.
+
+However, holding so much file data in memory for the root filesystem can bloat
+up container memory usage. To circumvent this, you can have root mount's upper
+layer (tmpfs) be backed by a host file, so all file data is stored on disk.
+
+The newer `--overlay2` flag allows you to achieve these. You can specify
+`--overlay2=root:/dir/path` in `runtimeArgs`. `/dir/path` can be any existing
+directory inside which the tmpfs filestore file will be created. When the
+container exits, this filestore file will be destroyed.
+
 ## Shared root filesystem
 
 The root filesystem is where the image is extracted and is not generally

pkg/sentry/fsimpl/tmpfs/tmpfs.go

@@ -129,6 +129,10 @@ type FilesystemOpts struct {
 
 	// MaxFilenameLen is the maximum filename length allowed by the tmpfs.
 	MaxFilenameLen int
+
+	// Filestore is the MemoryFile that will be used to store file data. If this
+	// is nil, then MemoryFileProviderFromContext() is used.
+	Filestore *pgalloc.MemoryFile
 }
 
 // GetFilesystem implements vfs.FilesystemType.GetFilesystem.
@@ -148,6 +152,9 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 		if tmpfsOpts.FilesystemType != nil {
 			newFSType = tmpfsOpts.FilesystemType
 		}
+		if tmpfsOpts.Filestore != nil {
+			mfp = tmpfsOpts.Filestore
+		}
 	}
 
 	mopts := vfs.GenericParseMountOptions(opts.Data)

pkg/sentry/pgalloc/pgalloc.go

@@ -1353,6 +1353,11 @@ func (f *MemoryFile) startEvictionGoroutineLocked(user EvictableMemoryUser, info
 	}()
 }
 
+// MemoryFile implements MemoryFileProvider.MemoryFile.
+func (f *MemoryFile) MemoryFile() *MemoryFile {
+	return f
+}
+
 // WaitForEvictions blocks until f is no longer evicting any evictable
 // allocations.
 func (f *MemoryFile) WaitForEvictions() {

pkg/test/testutil/testutil.go

@@ -55,7 +55,7 @@ var (
 	isRunningWithHostNet = flag.Bool("hostnet", BoolFromEnv("HOSTNET", false), "whether test is running with hostnet")
 	runscPath            = flag.String("runsc", os.Getenv("RUNTIME"), "path to runsc binary")
 	// Note: flag overlay is already taken by runsc.
-	isRunningWithOverlay = flag.Bool("test-overlay", BoolFromEnv("TEST_OVERLAY", false), "whether test is running with --overlay")
+	isRunningWithOverlay = flag.Bool("test-overlay", BoolFromEnv("TEST_OVERLAY", false), "whether test is running with --overlay2")
 )
 
 // StringFromEnv returns the value of the named environment variable, or `def` if unset/empty.

runsc/boot/loader.go

@@ -101,6 +101,10 @@ type containerInfo struct {
 
 	// goferFDs are the FDs that attach the sandbox to the gofers.
 	goferFDs []*fd.FD
+
+	// overlayFilestore is the memory file that will back the overlay mount's
+	// upper tmpfs layer.
+	overlayFilestore *pgalloc.MemoryFile
 }
 
 // Loader keeps state needed to start the kernel and run the container.
@@ -200,6 +204,9 @@ type Args struct {
 	// StdioFDs is the stdio for the application. The Loader takes ownership of
 	// these FDs and may close them at any time.
 	StdioFDs []int
+	// OverlayFilestoreFD is the host FD to a regular file which will be used to
+	// back the overlay mount's upper tmpfs layer.
+	OverlayFilestoreFD int
 	// NumCPU is the number of CPUs to create inside the sandbox.
 	NumCPU int
 	// TotalMem is the initial amount of total memory to report back to the
@@ -268,6 +275,14 @@ func New(args Args) (*Loader, error) {
 	for _, goferFD := range args.GoferFDs {
 		info.goferFDs = append(info.goferFDs, fd.New(goferFD))
 	}
+	if args.OverlayFilestoreFD >= 0 {
+		f := os.NewFile(uintptr(args.OverlayFilestoreFD), "overlay-filestore")
+		mf, err := pgalloc.NewMemoryFile(f, pgalloc.MemoryFileOpts{})
+		if err != nil {
+			return nil, fmt.Errorf("tmpfs.FilesystemType.GetFilesystem: failed to create memory file from host file: %w", err)
+		}
+		info.overlayFilestore = mf
+	}
 
 	// Create kernel and platform.
 	p, err := createPlatform(args.Conf, args.Device)
@@ -513,6 +528,9 @@ func (l *Loader) Destroy() {
 	for _, f := range l.root.goferFDs {
 		_ = f.Close()
 	}
+	if l.root.overlayFilestore != nil {
+		l.root.overlayFilestore.Destroy()
+	}
 
 	l.stopProfiling()
 }
@@ -747,6 +765,9 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 		conf:     conf,
 		spec:     spec,
 		goferFDs: goferFDs,
+		// Note that K8s starts all containers in a pod (root and subcontainers)
+		// with the same config. So overlayFilestore can be copied.
+		overlayFilestore: l.root.overlayFilestore,
 	}
 	info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns)
 	if err != nil {

runsc/boot/loader_test.go

@@ -125,13 +125,14 @@ func createLoader(spec *specs.Spec) (*Loader, func(), error) {
 	}
 
 	args := Args{
-		ID:              "foo",
-		Spec:            spec,
-		Conf:            conf,
-		ControllerFD:    fd,
-		GoferFDs:        []int{sandEnd},
-		StdioFDs:        stdio,
-		PodInitConfigFD: -1,
+		ID:                 "foo",
+		Spec:               spec,
+		Conf:               conf,
+		ControllerFD:       fd,
+		GoferFDs:           []int{sandEnd},
+		StdioFDs:           stdio,
+		OverlayFilestoreFD: -1,
+		PodInitConfigFD:    -1,
 	}
 	l, err := New(args)
 	if err != nil {

runsc/boot/vfs.go

@@ -46,6 +46,7 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/inet"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 	"gvisor.dev/gvisor/runsc/config"
 	"gvisor.dev/gvisor/runsc/specutils"
@@ -322,6 +323,10 @@ type containerMounter struct {
 	// fds is the list of FDs to be dispensed for mounts that require it.
 	fds fdDispenser
 
+	// overlayFilestore is the memory file that will back the overlay mount's
+	// upper tmpfs layer.
+	overlayFilestore *pgalloc.MemoryFile
+
 	k *kernel.Kernel
 
 	hints *podMountHints
@@ -333,12 +338,13 @@ type containerMounter struct {
 
 func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string) *containerMounter {
 	return &containerMounter{
-		root:        info.spec.Root,
-		mounts:      compileMounts(info.spec, info.conf),
-		fds:         fdDispenser{fds: info.goferFDs},
-		k:           k,
-		hints:       hints,
-		productName: productName,
+		root:             info.spec.Root,
+		mounts:           compileMounts(info.spec, info.conf),
+		fds:              fdDispenser{fds: info.goferFDs},
+		overlayFilestore: info.overlayFilestore,
+		k:                k,
+		hints:            hints,
+		productName:      productName,
 	}
 }
 
@@ -425,7 +431,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	}
 
 	fsName := gofer.Name
-	if conf.Overlay && !c.root.Readonly {
+	if conf.GetOverlay2().RootMount && !c.root.Readonly {
 		log.Infof("Adding overlay on top of root")
 		var err error
 		var cleanup func()
@@ -488,6 +494,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, creds *auth.Cre
 	// Upper is a tmpfs mount to keep all modifications inside the sandbox.
 	upperOpts.GetFilesystemOptions.InternalData = tmpfs.FilesystemOpts{
 		RootFileType: uint16(rootType),
+		Filestore:    c.overlayFilestore,
 	}
 	upper, err := c.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts)
 	if err != nil {
@@ -717,7 +724,7 @@ func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountA
 		}
 
 		// If configured, add overlay to all writable mounts.
-		useOverlay = conf.Overlay && !parseMountOptions(m.mount.Options).ReadOnly
+		useOverlay = conf.GetOverlay2().SubMounts && !parseMountOptions(m.mount.Options).ReadOnly
 
 	case cgroupfs.Name:
 		var err error

runsc/cli/main.go

@@ -224,7 +224,9 @@ func Main(version string) {
 	log.Infof("Configuration:")
 	log.Infof("\t\tRootDir: %s", conf.RootDir)
 	log.Infof("\t\tPlatform: %v", conf.Platform)
-	log.Infof("\t\tFileAccess: %v, overlay: %t", conf.FileAccess, conf.Overlay)
+	log.Infof("\t\tFileAccess: %v", conf.FileAccess)
+	overlay2 := conf.GetOverlay2()
+	log.Infof("\t\tOverlay: Root=%t, SubMounts=%t, FilestoreDir=%q", overlay2.RootMount, overlay2.SubMounts, overlay2.FilestoreDir)
 	log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets)
 	log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls)
 	log.Infof("\t\tLISAFS: %t", conf.Lisafs)

runsc/cmd/boot.go

@@ -56,6 +56,10 @@ type Boot struct {
 	// ioFDs is the list of FDs used to connect to FS gofers.
 	ioFDs intFlags
 
+	// overlayFilestoreFD is the host FD to the regular file which will back the
+	// overlay's upper tmpfs mount for all containers.
+	overlayFilestoreFD int
+
 	// stdioFDs are the fds for stdin, stdout, and stderr. They must be
 	// provided in that order.
 	stdioFDs intFlags
@@ -142,6 +146,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file")
 	f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
 	f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order")
+	f.IntVar(&b.overlayFilestoreFD, "overlay-filestore-fd", -1, "FD to a regular file which will be used to back the overlay's tmpfs upper mount.")
 	f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.")
 	f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup")
 	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
@@ -302,20 +307,21 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 
 	// Create the loader.
 	bootArgs := boot.Args{
-		ID:              f.Arg(0),
-		Spec:            spec,
-		Conf:            conf,
-		ControllerFD:    b.controllerFD,
-		Device:          os.NewFile(uintptr(b.deviceFD), "platform device"),
-		GoferFDs:        b.ioFDs.GetArray(),
-		StdioFDs:        b.stdioFDs.GetArray(),
-		NumCPU:          b.cpuNum,
-		TotalMem:        b.totalMem,
-		UserLogFD:       b.userLogFD,
-		ProductName:     b.productName,
-		PodInitConfigFD: b.podInitConfigFD,
-		SinkFDs:         b.sinkFDs.GetArray(),
-		ProfileOpts:     b.profileFDs.ToOpts(),
+		ID:                 f.Arg(0),
+		Spec:               spec,
+		Conf:               conf,
+		ControllerFD:       b.controllerFD,
+		Device:             os.NewFile(uintptr(b.deviceFD), "platform device"),
+		GoferFDs:           b.ioFDs.GetArray(),
+		StdioFDs:           b.stdioFDs.GetArray(),
+		OverlayFilestoreFD: b.overlayFilestoreFD,
+		NumCPU:             b.cpuNum,
+		TotalMem:           b.totalMem,
+		UserLogFD:          b.userLogFD,
+		ProductName:        b.productName,
+		PodInitConfigFD:    b.podInitConfigFD,
+		SinkFDs:            b.sinkFDs.GetArray(),
+		ProfileOpts:        b.profileFDs.ToOpts(),
 	}
 	l, err := boot.New(bootArgs)
 	if err != nil {

runsc/cmd/do.go

@@ -145,8 +145,10 @@ func (c *Do) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommand
 		return util.Errorf("Error to retrieve hostname: %v", err)
 	}
 
-	// Map the entire host file system, optionally using an overlay.
-	conf.Overlay = c.overlay
+	// If c.overlay is set, then forcefully enable overlay.
+	if overlay2 := conf.GetOverlay2(); c.overlay && !overlay2.Enabled() {
+		conf.Overlay = true
+	}
 	absRoot, err := resolvePath(c.root)
 	if err != nil {
 		return util.Errorf("Error resolving root: %v", err)

runsc/cmd/gofer.go

@@ -262,12 +262,13 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string)
 		HostUDS:  conf.GetHostUDS(),
 		HostFifo: conf.HostFifo,
 	})
+	overlay2 := conf.GetOverlay2()
 
 	// Start with root mount, then add any other additional mount as needed.
 	cfgs = append(cfgs, connectionConfig{
 		sock:      newSocket(g.ioFDs[0]),
 		mountPath: "/", // fsgofer process is always chroot()ed. So serve root.
-		readonly:  spec.Root.Readonly || conf.Overlay,
+		readonly:  spec.Root.Readonly || overlay2.RootMount,
 	})
 	log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly)
 
@@ -287,7 +288,7 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string)
 		cfgs = append(cfgs, connectionConfig{
 			sock:      newSocket(g.ioFDs[mountIdx]),
 			mountPath: m.Destination,
-			readonly:  isReadonlyMount(m.Options) || conf.Overlay,
+			readonly:  isReadonlyMount(m.Options) || overlay2.SubMounts,
 		})
 
 		log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, g.ioFDs[mountIdx], cfgs[mountIdx].readonly)
@@ -317,9 +318,10 @@ func (g *Gofer) serveLisafs(spec *specs.Spec, conf *config.Config, root string)
 
 func (g *Gofer) serve9P(spec *specs.Spec, conf *config.Config, root string) subcommands.ExitStatus {
 	// Start with root mount, then add any other additional mount as needed.
+	overlay2 := conf.GetOverlay2()
 	ats := make([]p9.Attacher, 0, len(spec.Mounts)+1)
 	ap, err := fsgofer.NewAttachPoint("/", fsgofer.Config{
-		ROMount:  spec.Root.Readonly || conf.Overlay,
+		ROMount:  spec.Root.Readonly || overlay2.RootMount,
 		HostUDS:  conf.GetHostUDS(),
 		HostFifo: conf.HostFifo,
 	})
@@ -333,7 +335,7 @@ func (g *Gofer) serve9P(spec *specs.Spec, conf *config.Config, root string) subc
 	for _, m := range spec.Mounts {
 		if specutils.IsGoferMount(m) {
 			cfg := fsgofer.Config{
-				ROMount:  isReadonlyMount(m.Options) || conf.Overlay,
+				ROMount:  isReadonlyMount(m.Options) || overlay2.SubMounts,
 				HostUDS:  conf.GetHostUDS(),
 				HostFifo: conf.HostFifo,
 			}
@@ -480,7 +482,7 @@ func setupRootFS(spec *specs.Spec, conf *config.Config) error {
 	}
 
 	// Check if root needs to be remounted as readonly.
-	if spec.Root.Readonly || conf.Overlay {
+	if spec.Root.Readonly || conf.GetOverlay2().RootMount {
 		// If root is a mount point but not read-only, we can change mount options
 		// to make it read-only for extra safety.
 		log.Infof("Remounting root as readonly: %q", root)
@@ -516,7 +518,7 @@ func setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath strin
 		}
 
 		flags := specutils.OptionsToFlags(m.Options) | unix.MS_BIND
-		if conf.Overlay {
+		if conf.GetOverlay2().SubMounts {
 			// Force mount read-only if writes are not going to be sent to it.
 			flags |= unix.MS_RDONLY
 		}

runsc/config/config.go

@@ -19,6 +19,7 @@ package config
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"gvisor.dev/gvisor/pkg/refs"
@@ -74,14 +75,19 @@ type Config struct {
 	// FileAccessMounts indicates how non-root volumes are accessed.
 	FileAccessMounts FileAccessType `flag:"file-access-mounts"`
 
-	// Overlay is whether to wrap the root filesystem in an overlay.
+	// Overlay is whether to wrap all mounts in an overlay. The upper tmpfs layer
+	// will be backed by application memory.
 	Overlay bool `flag:"overlay"`
 
+	// Overlay2 holds configuration about wrapping mounts in overlayfs.
+	// DO NOT call it directly, use GetOverlay2() instead.
+	Overlay2 Overlay2 `flag:"overlay2"`
+
 	// FSGoferHostUDS is deprecated: use host-uds=all.
 	FSGoferHostUDS bool `flag:"fsgofer-host-uds"`
 
 	// HostUDS controls permission to access host Unix-domain sockets.
-	// DO NOT call it directly, use GetHostComm() instead.
+	// DO NOT call it directly, use GetHostUDS() instead.
 	HostUDS HostUDS `flag:"host-uds"`
 
 	// HostFifo controls permission to access host FIFO (or named pipes).
@@ -273,7 +279,11 @@ type Config struct {
 }
 
 func (c *Config) validate() error {
-	if c.FileAccess == FileAccessShared && c.Overlay {
+	if c.Overlay && c.Overlay2.Enabled() {
+		// Deprecated flag was used together with flag that replaced it.
+		return fmt.Errorf("overlay flag has been replaced with overlay2 flag")
+	}
+	if overlay2 := c.GetOverlay2(); c.FileAccess == FileAccessShared && overlay2.Enabled() {
 		return fmt.Errorf("overlay flag is incompatible with shared file access")
 	}
 	if c.NumNetworkChannels <= 0 {
@@ -314,6 +324,19 @@ func (c *Config) GetHostUDS() HostUDS {
 	return c.HostUDS
 }
 
+// GetOverlay2 returns the overlay configuration, taking into consideration all
+// flags that affect the result.
+func (c *Config) GetOverlay2() Overlay2 {
+	if c.Overlay {
+		if c.Overlay2.Enabled() {
+			panic(fmt.Sprintf("Overlay2 cannot be set when --overlay=true"))
+		}
+		// Using deprecated flag, honor it to avoid breaking users.
+		return Overlay2{RootMount: true, SubMounts: true, FilestoreDir: ""}
+	}
+	return c.Overlay2
+}
+
 // FileAccessType tells how the filesystem is accessed.
 type FileAccessType int
 
@@ -591,3 +614,80 @@ func (g HostFifo) String() string {
 func (g HostFifo) AllowOpen() bool {
 	return g&HostFifoOpen != 0
 }
+
+// Overlay2 holds the configuration for setting up overlay filesystems for the
+// container.
+type Overlay2 struct {
+	RootMount    bool
+	SubMounts    bool
+	FilestoreDir string
+}
+
+func defaultOverlay2() *Overlay2 {
+	return &Overlay2{}
+}
+
+// Set implements flag.Value.
+func (o *Overlay2) Set(v string) error {
+	if v == "none" {
+		// Defaults are correct.
+		return nil
+	}
+	vs := strings.Split(v, ":")
+	if len(vs) != 2 {
+		return fmt.Errorf("expected format is --overlay2={mount}:{medium}, got %q", v)
+	}
+
+	switch mount := vs[0]; mount {
+	case "root":
+		o.RootMount = true
+	case "all":
+		o.RootMount = true
+		o.SubMounts = true
+	default:
+		return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount)
+	}
+
+	switch medium := vs[1]; medium {
+	case "memory":
+		o.FilestoreDir = ""
+	default:
+		o.FilestoreDir = medium
+	}
+	return nil
+}
+
+// Get implements flag.Value.
+func (o *Overlay2) Get() any {
+	return *o
+}
+
+// String implements flag.Value.
+func (o Overlay2) String() string {
+	if !o.RootMount && !o.SubMounts {
+		return "none"
+	}
+	res := ""
+	switch {
+	case o.RootMount && o.SubMounts:
+		res = "all"
+	case o.RootMount:
+		res = "root"
+	default:
+		panic("invalid state of subMounts = true and rootMount = false")
+	}
+
+	res += ":"
+	switch o.FilestoreDir {
+	case "":
+		res += "memory"
+	default:
+		res += o.FilestoreDir
+	}
+	return res
+}
+
+// Enabled returns true if overlay option is enabled for any mounts.
+func (o *Overlay2) Enabled() bool {
+	return o.RootMount || o.SubMounts
+}

runsc/config/config_test.go

@@ -178,7 +178,7 @@ func TestValidationFail(t *testing.T) {
 			error: "num_network_channels must be > 0",
 		},
 		{
-			name: "fsgofer-host-uds+comm:open",
+			name: "fsgofer-host-uds+host-uds:open",
 			flags: map[string]string{
 				"fsgofer-host-uds": "true",
 				"host-uds":         "open",
@@ -186,7 +186,7 @@ func TestValidationFail(t *testing.T) {
 			error: "fsgofer-host-uds has been replaced with host-uds flag",
 		},
 		{
-			name: "fsgofer-host-uds+comm:create",
+			name: "fsgofer-host-uds+host-uds:create",
 			flags: map[string]string{
 				"fsgofer-host-uds": "true",
 				"host-uds":         "create",
@@ -194,13 +194,29 @@ func TestValidationFail(t *testing.T) {
 			error: "fsgofer-host-uds has been replaced with host-uds flag",
 		},
 		{
-			name: "fsgofer-host-uds+comm:all",
+			name: "fsgofer-host-uds+host-uds:all",
 			flags: map[string]string{
 				"fsgofer-host-uds": "true",
 				"host-uds":         "all",
 			},
 			error: "fsgofer-host-uds has been replaced with host-uds flag",
 		},
+		{
+			name: "overlay+overlay2:root",
+			flags: map[string]string{
+				"overlay":  "true",
+				"overlay2": "root:memory",
+			},
+			error: "overlay flag has been replaced with overlay2 flag",
+		},
+		{
+			name: "overlay+overlay2:all",
+			flags: map[string]string{
+				"overlay":  "true",
+				"overlay2": "all:memory",
+			},
+			error: "overlay flag has been replaced with overlay2 flag",
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			testFlags := flag.NewFlagSet("test", flag.ContinueOnError)

runsc/config/flags.go

@@ -78,7 +78,8 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	// Flags that control sandbox runtime behavior: FS related.
 	flagSet.Var(fileAccessTypePtr(FileAccessExclusive), "file-access", "specifies which filesystem validation to use for the root mount: exclusive (default), shared.")
 	flagSet.Var(fileAccessTypePtr(FileAccessShared), "file-access-mounts", "specifies which filesystem validation to use for volumes other than the root mount: shared (default), exclusive.")
-	flagSet.Bool("overlay", false, "wrap filesystem mounts with writable overlay. All modifications are stored in memory inside the sandbox.")
+	flagSet.Bool("overlay", false, "DEPRECATED: use --overlay2=all:memory to achieve the same effect")
+	flagSet.Var(defaultOverlay2(), "overlay2", "wrap mounts with overlayfs. Format is {mount}:{medium}, where 'mount' can be 'root' or 'all' and medium can be 'memory' or existing directory path in which filestore will be created. 'none' will turn overlay mode off.")
 	flagSet.Bool("fsgofer-host-uds", false, "DEPRECATED: use host-uds=all")
 	flagSet.Var(hostUDSPtr(HostUDSNone), "host-uds", "controls permission to access host Unix-domain sockets. Values: none|open|create|all, default: none")
 	flagSet.Var(hostFifoPtr(HostFifoNone), "host-fifo", "controls permission to access host FIFOs (or named pipes). Values: none|open, default: none")

runsc/container/container.go

@@ -256,6 +256,10 @@ func New(conf *config.Config, args Args) (*Container, error) {
 			}
 		}
 		c.CompatCgroup = cgroup.CgroupJSON{Cgroup: subCgroup}
+		overlayFilestoreFile, err := createOverlayFilestore(conf.GetOverlay2())
+		if err != nil {
+			return nil, err
+		}
 		if err := runInCgroup(parentCgroup, func() error {
 			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached)
 			if err != nil {
@@ -265,15 +269,16 @@ func New(conf *config.Config, args Args) (*Container, error) {
 			// Start a new sandbox for this container. Any errors after this point
 			// must destroy the container.
 			sandArgs := &sandbox.Args{
-				ID:            sandboxID,
-				Spec:          args.Spec,
-				BundleDir:     args.BundleDir,
-				ConsoleSocket: args.ConsoleSocket,
-				UserLog:       args.UserLog,
-				IOFiles:       ioFiles,
-				MountsFile:    specFile,
-				Cgroup:        parentCgroup,
-				Attached:      args.Attached,
+				ID:                   sandboxID,
+				Spec:                 args.Spec,
+				BundleDir:            args.BundleDir,
+				ConsoleSocket:        args.ConsoleSocket,
+				UserLog:              args.UserLog,
+				IOFiles:              ioFiles,
+				MountsFile:           specFile,
+				Cgroup:               parentCgroup,
+				Attached:             args.Attached,
+				OverlayFilestoreFile: overlayFilestoreFile,
 			}
 			sand, err := sandbox.New(conf, sandArgs)
 			if err != nil {
@@ -763,6 +768,28 @@ func (c *Container) Destroy() error {
 	return fmt.Errorf(strings.Join(errs, "\n"))
 }
 
+func createOverlayFilestore(overlay2 config.Overlay2) (*os.File, error) {
+	if overlay2.FilestoreDir == "" {
+		return nil, nil
+	}
+	fileInfo, err := os.Stat(overlay2.FilestoreDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to stat overlay filestore directory %q: %v", overlay2.FilestoreDir, err)
+	}
+	if !fileInfo.IsDir() {
+		return nil, fmt.Errorf("overlay2 flag should specify an existing directory")
+	}
+	// Create an unnamed temporary file in filestore directory using
+	// O_TMPFILE. This file will be deleted when the container exits.
+	// Also specify O_EXCL to prevent this file from being linked into the
+	// filesystem. See open(2) man page's section for O_TMPFILE for details.
+	unnamedTmpFD, err := unix.Open(overlay2.FilestoreDir, unix.O_TMPFILE|unix.O_RDWR|unix.O_EXCL, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create an unnamed temporary file inside %q", overlay2.FilestoreDir)
+	}
+	return os.NewFile(uintptr(unnamedTmpFD), "overlay-filestore"), nil
+}
+
 // saveLocked saves the container metadata to a file.
 //
 // Precondition: container must be locked with container.lock().

runsc/sandbox/sandbox.go

@@ -156,6 +156,10 @@ type Args struct {
 	// appear in the spec.
 	IOFiles []*os.File
 
+	// OverlayFilestoreFile is the regular file that will back the tmpfs upper
+	// mount in the overlay mounts.
+	OverlayFilestoreFile *os.File
+
 	// MountsFile is a file container mount information from the spec. It's
 	// equivalent to the mounts from the spec, except that all paths have been
 	// resolved to their final absolute location.
@@ -591,6 +595,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 
 	// If there is a gofer, sends all socket ends to the sandbox.
 	donations.DonateAndClose("io-fds", args.IOFiles...)
+	donations.DonateAndClose("overlay-filestore-fd", args.OverlayFilestoreFile)
 	donations.DonateAndClose("mounts-fd", args.MountsFile)
 	donations.Donate("start-sync-fd", startSyncFile)
 	if err := donations.OpenAndDonate("user-log-fd", args.UserLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil {

test/runner/main.go

@@ -204,7 +204,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error {
 		"-file-access", *fileAccess,
 	}
 	if *overlay {
-		args = append(args, "-overlay")
+		args = append(args, "-overlay2=all:/tmp")
 	}
 	if *fuse {
 		args = append(args, "-fuse")