Fix overlay filestore ordering while assigning to VFS overlay mounts.

boot.containerMounter.prepareMounts() sorts the Spec.Mounts before the VFS
layer attempts to mount them. However, the filestore FDs donated to the sandbox
process are based on the original order in Spec.Mounts. So we should assign the
filestore FDs before this sorting happens. Having the filestore FD assigned to
the right overlay mount is important for the "self" overlay medium.

This change mimics the same approach taken for gofer FDs (also called ioFDs in
codebase). We assign the overlay filestore FD before sorting and save the
assignment in the `mountInfo` abstraction. The sorting happens on this
`mountInfo` slice.

Furthermore, this change plumbs an array to the container process and gofer
process which contains information about how each gofer mount is overlaid.
How a gofer mount is overlaid depends on various factors like:
* conf.Overlay2 setting.
* Readonly-ness of mount.
* If the mount source is a directory.
* (In the future, mount hints will also matter.)

This is getting increasingly complex. It is also not possible to figure out
all these factors in prepareMounts() because the container does not have the
container filesystem mounted locally and has not connected to the gofer yet.
So we figure out which gofer mounts are overlaid outside the sandbox (while
creating overlay filestores) and pass that information everywhere.

PiperOrigin-RevId: 529228922

runsc/boot/BUILD

@@ -18,6 +18,7 @@ go_library(
         "loader.go",
         "mount_hints.go",
         "network.go",
+        "overlay.go",
         "seccheck.go",
         "strace.go",
         "vfs.go",
@@ -132,6 +133,7 @@ go_test(
         "compat_test.go",
         "loader_test.go",
         "mount_hints_test.go",
+        "overlay_test.go",
         "vfs_test.go",
     ],
     library = ":boot",

runsc/boot/controller.go

@@ -269,6 +269,11 @@ type StartArgs struct {
 	// Optionally configured with the overlay2 flag.
 	NumOverlayFilestoreFDs int
 
+	// OverlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in Spec.Mounts (in the same order).
+	OverlayMediums []OverlayMedium
+
 	// FilePayload contains, in order:
 	//   * stdin, stdout, and stderr (optional: if terminal is disabled).
 	//   * file descriptors to overlay-backing host files (optional: for overlay2).
@@ -342,7 +347,7 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro
 		}
 	}()
 
-	if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, overlayFilestoreFDs); err != nil {
+	if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, overlayFilestoreFDs, args.OverlayMediums); err != nil {
 		log.Debugf("containerManager.StartSubcontainer failed, cid: %s, args: %+v, err: %v", args.CID, args, err)
 		return err
 	}

runsc/boot/loader.go

@@ -113,6 +113,11 @@ type containerInfo struct {
 	// overlayFilestoreFDs are the FDs to the regular files that will back the
 	// tmpfs upper mount in the overlay mounts.
 	overlayFilestoreFDs []*fd.FD
+
+	// overlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in spec.Mounts (in the same order).
+	overlayMediums []OverlayMedium
 }
 
 // Loader keeps state needed to start the kernel and run the container.
@@ -240,6 +245,10 @@ type Args struct {
 	// OverlayFilestoreFDs are the FDs to the regular files that will back the
 	// tmpfs upper mount in the overlay mounts.
 	OverlayFilestoreFDs []int
+	// OverlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in Spec.Mounts (in the same order).
+	OverlayMediums []OverlayMedium
 	// NumCPU is the number of CPUs to create inside the sandbox.
 	NumCPU int
 	// TotalMem is the initial amount of total memory to report back to the
@@ -287,7 +296,7 @@ func New(args Args) (*Loader, error) {
 	// Make host FDs stable between invocations. Host FDs must map to the exact
 	// same number when the sandbox is restored. Otherwise the wrong FD will be
 	// used.
-	info := containerInfo{}
+	info := containerInfo{overlayMediums: args.OverlayMediums}
 	newfd := startingStdioFD
 
 	for _, stdioFD := range args.StdioFDs {
@@ -755,7 +764,7 @@ func (l *Loader) createSubcontainer(cid string, tty *fd.FD) error {
 // startSubcontainer starts a child container. It returns the thread group ID of
 // the newly created process. Used FDs are either closed or released. It's safe
 // for the caller to close any remaining files upon return.
-func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, overlayFilestoreFDs []*fd.FD) error {
+func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, overlayFilestoreFDs []*fd.FD, overlayMediums []OverlayMedium) error {
 	// Create capabilities.
 	caps, err := specutils.Capabilities(conf.EnableRaw, spec.Process.Capabilities)
 	if err != nil {
@@ -810,6 +819,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 		spec:                spec,
 		goferFDs:            goferFDs,
 		overlayFilestoreFDs: overlayFilestoreFDs,
+		overlayMediums:      overlayMediums,
 	}
 	info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns)
 	if err != nil {

runsc/boot/loader_test.go

@@ -135,6 +135,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error
 		ControllerFD:    fd,
 		GoferFDs:        []int{sandEnd},
 		StdioFDs:        stdio,
+		OverlayMediums:  []OverlayMedium{NoOverlay},
 		PodInitConfigFD: -1,
 		ExecFD:          -1,
 	}

runsc/boot/overlay.go

@@ -0,0 +1,98 @@
+// Copyright 2023 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boot
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// OverlayMedium describes the medium that will be used to back the
+// overlay mount's upper layer.
+type OverlayMedium int
+
+const (
+	// NoOverlay indicates that this mount should not be overlaid.
+	NoOverlay OverlayMedium = iota
+
+	// MemoryMedium indicates that this mount should be overlaid with an
+	// upper layer backed by application memory.
+	MemoryMedium
+
+	// SelfMedium indicates that this mount should be overlaid with an upper
+	// layer backed by a host file in the mount's source directory.
+	SelfMedium
+
+	// AnonDirMedium indicates that this mount should be overlaid with an upper
+	// layer backed by a host file in an anonymous directory.
+	AnonDirMedium
+)
+
+// IsBackedByHostFile returns true if the overlay is backed by a host file.
+func (o *OverlayMedium) IsBackedByHostFile() bool {
+	return *o == SelfMedium || *o == AnonDirMedium
+}
+
+// IsEnabled returns true if an overlay is applied.
+func (o *OverlayMedium) IsEnabled() bool {
+	return *o != NoOverlay
+}
+
+// OverlayMediumFlags can be used with OverlayMedium flags that appear
+// multiple times.
+type OverlayMediumFlags []OverlayMedium
+
+// String implements flag.Value.
+func (o *OverlayMediumFlags) String() string {
+	return fmt.Sprintf("%v", *o)
+}
+
+// Get implements flag.Value.
+func (o *OverlayMediumFlags) Get() any {
+	return o
+}
+
+// GetArray returns an array of mappings.
+func (o *OverlayMediumFlags) GetArray() []OverlayMedium {
+	return *o
+}
+
+// Set implements flag.Value and appends an overlay medium from the command
+// line to the mediums array.
+func (o *OverlayMediumFlags) Set(s string) error {
+	mediums := strings.Split(s, ",")
+	for _, medium := range mediums {
+		mediumVal, err := strconv.Atoi(medium)
+		if err != nil {
+			return fmt.Errorf("invalid OverlayMedium value (%d): %v", mediumVal, err)
+		}
+		if mediumVal > int(AnonDirMedium) {
+			return fmt.Errorf("invalid OverlayMedium value (%d)", mediumVal)
+		}
+		*o = append(*o, OverlayMedium(mediumVal))
+	}
+	return nil
+}
+
+// ToOverlayMediumFlags converts []OverlayMedium to string format which can be
+// unpacked by OverlayMediumFlags.Set().
+func ToOverlayMediumFlags(mediums []OverlayMedium) string {
+	mediumVals := make([]string, 0, len(mediums))
+	for _, medium := range mediums {
+		mediumVals = append(mediumVals, strconv.Itoa(int(medium)))
+	}
+	return strings.Join(mediumVals, ",")
+}

runsc/boot/overlay_test.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boot
+
+import (
+	"testing"
+)
+
+func TestOverlayMedium(t *testing.T) {
+	tcs := []struct {
+		ovl          OverlayMedium
+		wantEnabled  bool
+		wantHostFile bool
+	}{{
+		ovl:          NoOverlay,
+		wantEnabled:  false,
+		wantHostFile: false,
+	}, {
+		ovl:          MemoryMedium,
+		wantEnabled:  true,
+		wantHostFile: false,
+	}, {
+		ovl:          SelfMedium,
+		wantEnabled:  true,
+		wantHostFile: true,
+	}, {
+		ovl:          AnonDirMedium,
+		wantEnabled:  true,
+		wantHostFile: true,
+	}}
+	for _, tc := range tcs {
+		if got := tc.ovl.IsEnabled(); got != tc.wantEnabled {
+			t.Errorf("overlay medium = %d, IsEnabled() = %t, want = %t", tc.ovl, got, tc.wantEnabled)
+		}
+		if got := tc.ovl.IsBackedByHostFile(); got != tc.wantHostFile {
+			t.Errorf("overlay medium = %d, IsBackedByHostFile() = %t, want = %t", tc.ovl, got, tc.wantHostFile)
+		}
+	}
+}
+
+func TestOverlayMediumFlags(t *testing.T) {
+	want := []OverlayMedium{MemoryMedium, SelfMedium, AnonDirMedium, NoOverlay}
+	var got OverlayMediumFlags
+	got.Set(ToOverlayMediumFlags(want))
+	if len(got) != len(want) {
+		t.Fatalf("overlay medium flags is incorrect length: want = %d, got = %d", len(want), len(got))
+	}
+	for i := range want {
+		if want[i] != got[i] {
+			t.Errorf("overlay medium is incorrect: want = %d, got = %d", want[i], got[i])
+		}
+	}
+}

runsc/boot/vfs.go

@@ -194,6 +194,8 @@ func setupContainerVFS(ctx context.Context, conf *config.Config, mntr *container
 
 // compileMounts returns the supported mounts from the mount spec, adding any
 // mandatory mounts that are required by the OCI specification.
+//
+// This function must NOT add/remove any gofer mounts or change their order.
 func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount {
 	// Keep track of whether proc and sys were mounted.
 	var procMounted, sysMounted, devMounted, devptsMounted bool
@@ -349,6 +351,11 @@ type containerMounter struct {
 	// tmpfs upper mount in the overlay mounts.
 	overlayFilestoreFDs fdDispenser
 
+	// overlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in `mounts` slice above (in the same order).
+	overlayMediums []OverlayMedium
+
 	k *kernel.Kernel
 
 	hints *PodMountHints
@@ -367,6 +374,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountH
 		mounts:              compileMounts(info.spec, info.conf),
 		fds:                 fdDispenser{fds: info.goferFDs},
 		overlayFilestoreFDs: fdDispenser{fds: info.overlayFilestoreFDs},
+		overlayMediums:      info.overlayMediums,
 		k:                   k,
 		hints:               hints,
 		productName:         productName,
@@ -433,8 +441,8 @@ func (c *containerMounter) mountAll(conf *config.Config, procArgs *kernel.Create
 
 // createMountNamespace creates the container's root mount and namespace.
 func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) {
-	fd := c.fds.remove()
-	data := goferMountData(fd, conf.FileAccess, conf)
+	ioFD := c.fds.remove()
+	data := goferMountData(ioFD, conf.FileAccess, conf)
 
 	// We can't check for overlayfs here because sandbox is chroot'ed and gofer
 	// can only send mount options for specs.Mounts (specs.Root is missing
@@ -444,7 +452,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	// Configure the gofer dentry cache size.
 	gofer.SetDentryCacheSize(conf.DCache)
 
-	log.Infof("Mounting root with gofer, ioFD: %d", fd)
+	log.Infof("Mounting root with gofer, ioFD: %d", ioFD)
 	opts := &vfs.MountOptions{
 		ReadOnly: c.root.Readonly,
 		GetFilesystemOptions: vfs.GetFilesystemOptions{
@@ -457,11 +465,17 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	}
 
 	fsName := gofer.Name
-	if conf.GetOverlay2().RootMount && !c.root.Readonly {
+	if c.overlayMediums[0].IsEnabled() {
 		log.Infof("Adding overlay on top of root")
-		var err error
-		var cleanup func()
-		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, useOverlayFilestoreFD)
+		var (
+			err              error
+			cleanup          func()
+			overlayFilestore *fd.FD
+		)
+		if c.overlayMediums[0].IsBackedByHostFile() {
+			overlayFilestore = c.overlayFilestoreFDs.removeAsFD()
+		}
+		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, overlayFilestore, c.overlayMediums[0])
 		if err != nil {
 			return nil, fmt.Errorf("mounting root with overlay: %w", err)
 		}
@@ -476,22 +490,11 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	return mns, nil
 }
 
-func useOverlayFilestoreFD(overlay2 config.Overlay2, isDir bool) bool {
-	if !overlay2.IsBackedByHostFile() {
-		return false
-	}
-	if overlay2.IsBackedBySelf() {
-		// Only directory mounts can be backed by a self filestore.
-		return isDir
-	}
-	return true
-}
-
 // configureOverlay mounts the lower layer using "lowerOpts", mounts the upper
 // layer using tmpfs, and return overlay mount options. "cleanup" must be called
 // after the options have been used to mount the overlay, to release refs on
 // lower and upper mounts.
-func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(overlay2 config.Overlay2, isDir bool) bool) (*vfs.MountOptions, func(), error) {
+func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, filestoreFD *fd.FD, medium OverlayMedium) (*vfs.MountOptions, func(), error) {
 	// First copy options from lower layer to upper layer and overlay. Clear
 	// filesystem specific options.
 	upperOpts := *lowerOpts
@@ -531,10 +534,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 	// Upper is a tmpfs mount to keep all modifications inside the sandbox.
 	tmpfsOpts := tmpfs.FilesystemOpts{
 		RootFileType: uint16(rootType),
-	}
-	overlay2 := conf.GetOverlay2()
-	if useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) {
-		tmpfsOpts.FilestoreFD = c.overlayFilestoreFDs.removeAsFD()
+		FilestoreFD:  filestoreFD,
 	}
 	upperOpts.GetFilesystemOptions.InternalData = tmpfsOpts
 	upper, err := c.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts)
@@ -573,9 +573,8 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 		}
 	}
 
-	// If host filestore is being used and it is backed by self, then we need to
-	// hide the filestore from the containerized application.
-	if overlay2.IsBackedBySelf() && useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) {
+	// We need to hide the filestore from the containerized application.
+	if medium == SelfMedium {
 		if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{
 			Root:  upperRootVD,
 			Start: upperRootVD,
@@ -657,9 +656,11 @@ func (c *containerMounter) mountSubmounts(ctx context.Context, conf *config.Conf
 }
 
 type mountInfo struct {
-	mount *specs.Mount
-	fd    int
-	hint  *MountHint
+	mount              *specs.Mount
+	fd                 int
+	hint               *MountHint
+	overlayMedium      OverlayMedium
+	overlayFilestoreFD *fd.FD
 }
 
 func newNonGoferMountInfo(mount *specs.Mount) *mountInfo {
@@ -671,21 +672,28 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
 	// undocumented assumption that FDs are dispensed in the order in which
 	// they are required by mounts.
 	var mounts []mountInfo
+	goferMntIdx := 1 // First index is for rootfs.
 	for i := range c.mounts {
 		m := &c.mounts[i]
 		specutils.MaybeConvertToBindMount(m)
 
 		// Only bind mounts use host FDs; see
 		// containerMounter.getMountNameAndOptions.
-		fd := -1
-		if m.Type == Bind {
-			fd = c.fds.remove()
+		info := mountInfo{
+			mount:         m,
+			fd:            -1,
+			hint:          c.hints.FindMount(m),
+			overlayMedium: NoOverlay,
 		}
-		mounts = append(mounts, mountInfo{
-			mount: m,
-			fd:    fd,
-			hint:  c.hints.FindMount(m),
-		})
+		if specutils.IsGoferMount(*m) {
+			info.fd = c.fds.remove()
+			info.overlayMedium = c.overlayMediums[goferMntIdx]
+			if info.overlayMedium.IsBackedByHostFile() {
+				info.overlayFilestoreFD = c.overlayFilestoreFDs.removeAsFD()
+			}
+			goferMntIdx++
+		}
+		mounts = append(mounts, info)
 	}
 	if err := c.checkDispenser(); err != nil {
 		return nil, err
@@ -700,7 +708,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
 }
 
 func (c *containerMounter) mountSubmount(ctx context.Context, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *mountInfo) (*vfs.Mount, error) {
-	fsName, opts, useOverlay, err := c.getMountNameAndOptions(conf, submount)
+	fsName, opts, err := c.getMountNameAndOptions(conf, submount)
 	if err != nil {
 		return nil, fmt.Errorf("mountOptions failed: %w", err)
 	}
@@ -713,10 +721,10 @@ func (c *containerMounter) mountSubmount(ctx context.Context, conf *config.Confi
 		return nil, fmt.Errorf("creating mount point %q: %w", submount.mount.Destination, err)
 	}
 
-	if useOverlay {
+	if submount.overlayMedium.IsEnabled() {
 		log.Infof("Adding overlay on top of mount %q", submount.mount.Destination)
 		var cleanup func()
-		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, useOverlayFilestoreFD)
+		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, submount.overlayFilestoreFD, submount.overlayMedium)
 		if err != nil {
 			return nil, fmt.Errorf("mounting volume with overlay at %q: %w", submount.mount.Destination, err)
 		}
@@ -742,9 +750,8 @@ func (c *containerMounter) mountSubmount(ctx context.Context, conf *config.Confi
 
 // getMountNameAndOptions retrieves the fsName, opts, and useOverlay values
 // used for mounts.
-func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountInfo) (string, *vfs.MountOptions, bool, error) {
+func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountInfo) (string, *vfs.MountOptions, error) {
 	fsName := m.mount.Type
-	useOverlay := false
 	var (
 		data         []string
 		internalData any
@@ -767,33 +774,30 @@ func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountI
 		var err error
 		data, err = parseAndFilterOptions(m.mount.Options, tmpfsAllowedData...)
 		if err != nil {
-			return "", nil, false, err
+			return "", nil, err
 		}
 
 	case Bind:
 		fsName = gofer.Name
 		if m.fd < 0 {
 			// Check that an FD was provided to fails fast.
-			return "", nil, false, fmt.Errorf("gofer mount requires a connection FD")
+			return "", nil, fmt.Errorf("gofer mount requires a connection FD")
 		}
 		data = goferMountData(m.fd, c.getMountAccessType(conf, m.mount, m.hint), conf)
 		internalData = gofer.InternalFilesystemOptions{
 			UniqueID: m.mount.Destination,
 		}
 
-		// If configured, add overlay to all writable mounts.
-		useOverlay = conf.GetOverlay2().SubMounts && !ParseMountOptions(m.mount.Options).ReadOnly
-
 	case cgroupfs.Name:
 		var err error
 		data, err = parseAndFilterOptions(m.mount.Options, cgroupfs.SupportedMountOptions...)
 		if err != nil {
-			return "", nil, false, err
+			return "", nil, err
 		}
 
 	default:
 		log.Warningf("ignoring unknown filesystem type %q", m.mount.Type)
-		return "", nil, false, nil
+		return "", nil, nil
 	}
 
 	opts := ParseMountOptions(m.mount.Options)
@@ -802,7 +806,7 @@ func (c *containerMounter) getMountNameAndOptions(conf *config.Config, m *mountI
 		InternalData: internalData,
 	}
 
-	return fsName, opts, useOverlay, nil
+	return fsName, opts, nil
 }
 
 // ParseMountOptions converts specs.Mount.Options to vfs.MountOptions.
@@ -938,28 +942,13 @@ func (c *containerMounter) mountSharedMaster(ctx context.Context, conf *config.C
 	// Map mount type to filesystem name, and parse out the options that we are
 	// capable of dealing with.
 	mntInfo := newNonGoferMountInfo(&hint.mount)
-	fsName, opts, useOverlay, err := c.getMountNameAndOptions(conf, mntInfo)
+	fsName, opts, err := c.getMountNameAndOptions(conf, mntInfo)
 	if err != nil {
 		return nil, err
 	}
 	if len(fsName) == 0 {
 		return nil, fmt.Errorf("mount type not supported %q", hint.mount.Type)
 	}
-
-	if useOverlay {
-		log.Infof("Adding overlay on top of shared mount %q", mntInfo.mount.Destination)
-		var cleanup func()
-		// TODO(b/142076984): Use an overlay for a shared EmptyDir mount. Such a
-		// mount should be backed by a self filestore, so limits can be enforced
-		// by k8s on the host. For now pass nil for useFilestoreFD.
-		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, nil /* useFilestoreFD */)
-		if err != nil {
-			return nil, fmt.Errorf("mounting shared volume with overlay at %q: %w", mntInfo.mount.Destination, err)
-		}
-		defer cleanup()
-		fsName = overlay.Name
-	}
-
 	return c.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
 }
 
@@ -972,7 +961,7 @@ func (c *containerMounter) mountSharedSubmount(ctx context.Context, conf *config
 
 	// Ignore data and useOverlay because these were already applied to
 	// the master mount.
-	_, opts, _, err := c.getMountNameAndOptions(conf, newNonGoferMountInfo(mount))
+	_, opts, err := c.getMountNameAndOptions(conf, newNonGoferMountInfo(mount))
 	if err != nil {
 		return nil, err
 	}

runsc/cmd/boot.go

@@ -82,6 +82,11 @@ type Boot struct {
 	// upper mount in the overlay mounts.
 	overlayFilestoreFDs intFlags
 
+	// overlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in Spec.Mounts (in the same order).
+	overlayMediums boot.OverlayMediumFlags
+
 	// stdioFDs are the fds for stdin, stdout, and stderr. They must be
 	// provided in that order.
 	stdioFDs intFlags
@@ -177,6 +182,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.")
 	f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.")
 	f.Var(&b.overlayFilestoreFDs, "overlay-filestore-fds", "FDs to the regular files that will back the tmpfs upper mount in the overlay mounts.")
+	f.Var(&b.overlayMediums, "overlay-mediums", "information about how the gofer mounts have been overlaid.")
 	f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.")
 	f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup")
 	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
@@ -379,6 +385,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		PassFDs:             b.passFDs.GetArray(),
 		ExecFD:              b.execFD,
 		OverlayFilestoreFDs: b.overlayFilestoreFDs.GetArray(),
+		OverlayMediums:      b.overlayMediums.GetArray(),
 		NumCPU:              b.cpuNum,
 		TotalMem:            b.totalMem,
 		UserLogFD:           b.userLogFD,

runsc/cmd/gofer.go

@@ -30,6 +30,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/unet"
+	"gvisor.dev/gvisor/runsc/boot"
 	"gvisor.dev/gvisor/runsc/cmd/util"
 	"gvisor.dev/gvisor/runsc/config"
 	"gvisor.dev/gvisor/runsc/flag"
@@ -59,10 +60,11 @@ var goferCaps = &specs.LinuxCapabilities{
 // Gofer implements subcommands.Command for the "gofer" command, which starts a
 // filesystem gofer.  This command should not be called directly.
 type Gofer struct {
-	bundleDir string
-	ioFDs     intFlags
-	applyCaps bool
-	setUpRoot bool
+	bundleDir      string
+	ioFDs          intFlags
+	applyCaps      bool
+	setUpRoot      bool
+	overlayMediums boot.OverlayMediumFlags
 
 	specFD       int
 	mountsFD     int
@@ -98,6 +100,7 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) {
 
 	// Open FDs that are donated to the gofer.
 	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec")
+	f.Var(&g.overlayMediums, "overlay-mediums", "information about how the gofer mounts have been overlaid.")
 	f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).")
 	f.IntVar(&g.syncUsernsFD, "sync-userns-fd", -1, "file descriptor used to synchronize rootless user namespace initialization.")
@@ -147,7 +150,7 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm
 	}
 
 	if g.setUpRoot {
-		if err := setupRootFS(spec, conf); err != nil {
+		if err := g.setupRootFS(spec, conf); err != nil {
 			util.Fatalf("Error setting up root FS: %v", err)
 		}
 		if !conf.TestOnlyAllowRunAsCurrentUserWithoutChroot {
@@ -283,13 +286,12 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		HostFifo:           conf.HostFifo,
 		DonateMountPointFD: conf.DirectFS,
 	})
-	overlay2 := conf.GetOverlay2()
 
 	// Start with root mount, then add any other additional mount as needed.
 	cfgs = append(cfgs, connectionConfig{
 		sock:      newSocket(g.ioFDs[0]),
 		mountPath: "/", // fsgofer process is always chroot()ed. So serve root.
-		readonly:  spec.Root.Readonly || overlay2.RootMount,
+		readonly:  spec.Root.Readonly || g.overlayMediums[0].IsEnabled(),
 	})
 	log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly)
 
@@ -309,7 +311,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		cfgs = append(cfgs, connectionConfig{
 			sock:      newSocket(g.ioFDs[mountIdx]),
 			mountPath: m.Destination,
-			readonly:  isReadonlyMount(m.Options) || overlay2.SubMounts,
+			readonly:  specutils.IsReadonlyMount(m.Options) || g.overlayMediums[mountIdx].IsEnabled(),
 		})
 
 		log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, g.ioFDs[mountIdx], cfgs[mountIdx].readonly)
@@ -356,16 +358,7 @@ func (g *Gofer) writeMounts(mounts []specs.Mount) error {
 	return nil
 }
 
-func isReadonlyMount(opts []string) bool {
-	for _, o := range opts {
-		if o == "ro" {
-			return true
-		}
-	}
-	return false
-}
-
-func setupRootFS(spec *specs.Spec, conf *config.Config) error {
+func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error {
 	// Convert all shared mounts into slaves to be sure that nothing will be
 	// propagated outside of our namespace.
 	procPath := "/proc"
@@ -428,7 +421,7 @@ func setupRootFS(spec *specs.Spec, conf *config.Config) error {
 	}
 
 	// Replace the current spec, with the clean spec with symlinks resolved.
-	if err := setupMounts(conf, spec.Mounts, root, procPath); err != nil {
+	if err := g.setupMounts(conf, spec.Mounts, root, procPath); err != nil {
 		util.Fatalf("error setting up FS: %v", err)
 	}
 
@@ -445,7 +438,7 @@ func setupRootFS(spec *specs.Spec, conf *config.Config) error {
 	}
 
 	// Check if root needs to be remounted as readonly.
-	if spec.Root.Readonly || conf.GetOverlay2().RootMount {
+	if spec.Root.Readonly || g.overlayMediums[0].IsEnabled() {
 		// If root is a mount point but not read-only, we can change mount options
 		// to make it read-only for extra safety.
 		log.Infof("Remounting root as readonly: %q", root)
@@ -469,7 +462,8 @@ func setupRootFS(spec *specs.Spec, conf *config.Config) error {
 // setupMounts bind mounts all mounts specified in the spec in their correct
 // location inside root. It will resolve relative paths and symlinks. It also
 // creates directories as needed.
-func setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath string) error {
+func (g *Gofer) setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath string) error {
+	goferMntIdx := 1 // First index is for rootfs.
 	for _, m := range mounts {
 		if !specutils.IsGoferMount(m) {
 			continue
@@ -481,7 +475,7 @@ func setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath strin
 		}
 
 		flags := specutils.OptionsToFlags(m.Options) | unix.MS_BIND
-		if conf.GetOverlay2().SubMounts {
+		if g.overlayMediums[goferMntIdx].IsEnabled() {
 			// Force mount read-only if writes are not going to be sent to it.
 			flags |= unix.MS_RDONLY
 		}
@@ -498,6 +492,7 @@ func setupMounts(conf *config.Config, mounts []specs.Mount, root, procPath strin
 				return fmt.Errorf("mount dst: %q, flags: %#x, err: %v", dst, flags, err)
 			}
 		}
+		goferMntIdx++
 	}
 	return nil
 }

runsc/config/config.go

@@ -767,12 +767,12 @@ func (o *Overlay2) Enabled() bool {
 	return o.RootMount || o.SubMounts
 }
 
-// IsBackedByHostFile indicates whether the overlay is backed by a host file.
-func (o *Overlay2) IsBackedByHostFile() bool {
-	return o.Enabled() && o.Medium != "memory"
+// IsBackedByMemory indicates whether the overlay is backed by app memory.
+func (o *Overlay2) IsBackedByMemory() bool {
+	return o.Enabled() && o.Medium == "memory"
 }
 
-// IsBackedBySelf indicates whether the overlayed mounts are backed by
+// IsBackedBySelf indicates whether the overlaid mounts are backed by
 // themselves.
 func (o *Overlay2) IsBackedBySelf() bool {
 	return o.Enabled() && o.Medium == "self"

runsc/container/container.go

@@ -135,6 +135,11 @@ type Container struct {
 	// started.
 	OverlayConf config.Overlay2 `json:"overlayConf"`
 
+	// OverlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in Spec.Mounts (in the same order).
+	OverlayMediums []boot.OverlayMedium `json:"overlayMediums"`
+
 	//
 	// Fields below this line are not saved in the state file and will not
 	// be preserved across commands.
@@ -280,10 +285,11 @@ func New(conf *config.Config, args Args) (*Container, error) {
 			}
 		}
 		c.CompatCgroup = cgroup.CgroupJSON{Cgroup: subCgroup}
-		overlayFilestoreFiles, err := c.createOverlayFilestores()
+		overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores()
 		if err != nil {
 			return nil, err
 		}
+		c.OverlayMediums = overlayMediums
 		if err := runInCgroup(containerCgroup, func() error {
 			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached)
 			if err != nil {
@@ -303,6 +309,7 @@ func New(conf *config.Config, args Args) (*Container, error) {
 				Cgroup:                containerCgroup,
 				Attached:              args.Attached,
 				OverlayFilestoreFiles: overlayFilestoreFiles,
+				OverlayMediums:        overlayMediums,
 				PassFiles:             args.PassFiles,
 				ExecFile:              args.ExecFile,
 			}
@@ -419,12 +426,11 @@ func (c *Container) Start(conf *config.Config) error {
 			return err
 		}
 	} else {
-		// Create an overlay filestore for the subcontainer if its overlay is
-		// backed by a host file.
-		overlayFilestoreFiles, err := c.createOverlayFilestores()
+		overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores()
 		if err != nil {
 			return err
 		}
+		c.OverlayMediums = overlayMediums
 		// Join cgroup to start gofer process to ensure it's part of the cgroup from
 		// the start (and all their children processes).
 		if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error {
@@ -453,7 +459,7 @@ func (c *Container) Start(conf *config.Config) error {
 				stdios = []*os.File{os.Stdin, os.Stdout, os.Stderr}
 			}
 
-			return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, overlayFilestoreFiles)
+			return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, overlayFilestoreFiles, overlayMediums)
 		}); err != nil {
 			return err
 		}
@@ -785,31 +791,15 @@ func (c *Container) Destroy() error {
 		errs = append(errs, err.Error())
 	}
 
-	if c.OverlayConf.IsBackedBySelf() {
-		// Clean up overlay filestore files created in their respective mounts.
-		c.forEachOverlayMount(func(mountSrc string) error {
-			// Persevere through errors. Always return nil. A non-nil error will halt
-			// forEachOverlayMount(). But we want to clean up as much as we can.
-			mountSrcInfo, err := os.Stat(mountSrc)
-			if err != nil {
-				err = fmt.Errorf("failed to stat mount %q to see if it were a dirctory: %v", mountSrc, err)
-				log.Warningf("%v", err)
-				errs = append(errs, err.Error())
-				return nil
-			}
-			// mountSrc only contains the filestore file if it is a directory.
-			if !mountSrcInfo.IsDir() {
-				return nil
-			}
-			filestorePath := boot.SelfOverlayFilestorePath(mountSrc, c.sandboxID())
-			if err := os.Remove(filestorePath); err != nil {
-				err = fmt.Errorf("failed to delete filestore file %q: %v", filestorePath, err)
-				log.Warningf("%v", err)
-				errs = append(errs, err.Error())
-			}
-			return nil
-		})
-	}
+	// Clean up overlay filestore files created in their respective mounts.
+	c.forEachSelfOverlay(func(mountSrc string) {
+		filestorePath := boot.SelfOverlayFilestorePath(mountSrc, c.sandboxID())
+		if err := os.Remove(filestorePath); err != nil {
+			err = fmt.Errorf("failed to delete filestore file %q: %v", filestorePath, err)
+			log.Warningf("%v", err)
+			errs = append(errs, err.Error())
+		}
+	})
 
 	c.changeStatus(Stopped)
 
@@ -848,129 +838,136 @@ func (c *Container) sandboxID() string {
 	return c.Saver.ID.SandboxID
 }
 
+func (c *Container) forEachSelfOverlay(fn func(mountSrc string)) {
+	if c.OverlayMediums == nil {
+		// Sub container not started? Skip.
+		return
+	}
+	if c.OverlayMediums[0] == boot.SelfMedium {
+		fn(c.Spec.Root.Path)
+	}
+	goferMntIdx := 1 // First index is for rootfs.
+	for i := range c.Spec.Mounts {
+		if !specutils.IsGoferMount(c.Spec.Mounts[i]) {
+			continue
+		}
+		if c.OverlayMediums[goferMntIdx] == boot.SelfMedium {
+			fn(c.Spec.Mounts[i].Source)
+		}
+		goferMntIdx++
+	}
+}
+
 // createOverlayFilestores creates the regular files that will back the tmpfs
-// upper mount for overlay mounts. It may return (nil, nil) if overlay is not
-// configured to be backed by host files.
-func (c *Container) createOverlayFilestores() ([]*os.File, error) {
-	if !c.OverlayConf.IsBackedByHostFile() {
-		return nil, nil
-	}
-	var (
-		filestoreFiles []*os.File
-		err            error
-	)
-	if c.OverlayConf.IsBackedBySelf() {
-		filestoreFiles, err = c.createOverlayFilestoreInSelf()
-	} else {
-		filestoreFiles, err = c.createOverlayFilestoreInDir()
-	}
+// upper mount for overlay mounts. It also returns information about the
+// overlay medium used for each bind mount.
+func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium, error) {
+	var filestoreFiles []*os.File
+	var overlayMediums []boot.OverlayMedium
+
+	// Handle root mount first.
+	shouldOverlay := c.OverlayConf.RootMount && !c.Spec.Root.Readonly
+	filestore, medium, err := c.createOverlayFilestore(c.Spec.Root.Path, shouldOverlay)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	for _, f := range filestoreFiles {
+	if filestore != nil {
+		filestoreFiles = append(filestoreFiles, filestore)
+	}
+	overlayMediums = append(overlayMediums, medium)
+
+	// Handle bind mounts.
+	for i := range c.Spec.Mounts {
+		if !specutils.IsGoferMount(c.Spec.Mounts[i]) {
+			continue
+		}
+		shouldOverlay := c.OverlayConf.SubMounts && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options)
+		filestore, medium, err := c.createOverlayFilestore(c.Spec.Mounts[i].Source, shouldOverlay)
+		if err != nil {
+			return nil, nil, err
+		}
+		if filestore != nil {
+			filestoreFiles = append(filestoreFiles, filestore)
+		}
+		overlayMediums = append(overlayMediums, medium)
+	}
+	for _, filestore := range filestoreFiles {
 		// Perform this work around outside the sandbox. The sandbox may already be
 		// running with seccomp filters that do not allow this.
-		pgalloc.IMAWorkAroundForMemFile(f.Fd())
+		pgalloc.IMAWorkAroundForMemFile(filestore.Fd())
 	}
-	return filestoreFiles, nil
+	return filestoreFiles, overlayMediums, nil
 }
 
-// Precondition: Overlay2.IsBackedByHostFile() && Overlay2.IsBackedBySelf().
-func (c *Container) createOverlayFilestoreInSelf() ([]*os.File, error) {
-	var filestoreFiles []*os.File
-	err := c.forEachOverlayMount(func(mountSrc string) error {
-		mountSrcInfo, err := os.Stat(mountSrc)
-		if err != nil {
-			return fmt.Errorf("failed to stat mount %q to see if it were a dirctory: %v", mountSrc, err)
-		}
-		if !mountSrcInfo.IsDir() {
-			log.Warningf("overlay2 self medium is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc)
-			return nil
-		}
-		// Create the self overlay filestore file.
-		filestorePath := boot.SelfOverlayFilestorePath(mountSrc, c.sandboxID())
-		filestoreFD, err := unix.Open(filestorePath, unix.O_RDWR|unix.O_CREAT|unix.O_EXCL, 0666)
-		if err != nil {
-			if err == unix.EEXIST {
-				// Note that if the same submount is mounted multiple times within the
-				// same sandbox, then the overlay option doesn't work correctly.
-				// Because each overlay mount is independent and changes to one are not
-				// visible to the other. Given "overlay on repeated submounts" is
-				// already broken, we don't support such a scenario with the self
-				// medium. The filestore file will already exist for such a case.
-				return fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not suppported with self medium", mountSrc, filestorePath)
-			}
-			return fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err)
-		}
-		// Filestore in self should be a named path because it needs to be
-		// discoverable via path traversal so that k8s can scan the filesystem
-		// and apply any limits appropriately (like local ephemeral storage
-		// limits). So don't delte it. These files will be unlinked when the
-		// container is destroyed. This makes self medium appropriate for k8s.
-		filestoreFiles = append(filestoreFiles, os.NewFile(uintptr(filestoreFD), filestorePath))
-		return nil
-	})
-	return filestoreFiles, err
+func (c *Container) createOverlayFilestore(mountSrc string, shouldOverlay bool) (*os.File, boot.OverlayMedium, error) {
+	switch {
+	case !shouldOverlay:
+		return nil, boot.NoOverlay, nil
+	case c.OverlayConf.IsBackedByMemory():
+		return nil, boot.MemoryMedium, nil
+	case c.OverlayConf.IsBackedBySelf():
+		return c.createOverlayFilestoreInSelf(mountSrc)
+	default:
+		return c.createOverlayFilestoreInDir()
+	}
 }
 
-// Precondition: Overlay2.IsBackedByHostFile() && !Overlay2.IsBackedBySelf().
-func (c *Container) createOverlayFilestoreInDir() ([]*os.File, error) {
+func (c *Container) createOverlayFilestoreInSelf(mountSrc string) (*os.File, boot.OverlayMedium, error) {
+	mountSrcInfo, err := os.Stat(mountSrc)
+	if err != nil {
+		return nil, boot.NoOverlay, fmt.Errorf("failed to stat mount %q to see if it were a dirctory: %v", mountSrc, err)
+	}
+	if !mountSrcInfo.IsDir() {
+		log.Warningf("overlay2 self medium is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc)
+		return nil, boot.MemoryMedium, nil
+	}
+	// Create the self overlay filestore file.
+	filestorePath := boot.SelfOverlayFilestorePath(mountSrc, c.sandboxID())
+	filestoreFD, err := unix.Open(filestorePath, unix.O_RDWR|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC, 0666)
+	if err != nil {
+		if err == unix.EEXIST {
+			// Note that if the same submount is mounted multiple times within the
+			// same sandbox, then the overlay option doesn't work correctly.
+			// Because each overlay mount is independent and changes to one are not
+			// visible to the other. Given "overlay on repeated submounts" is
+			// already broken, we don't support such a scenario with the self
+			// medium. The filestore file will already exist for such a case.
+			return nil, boot.NoOverlay, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not suppported with self medium", mountSrc, filestorePath)
+		}
+		return nil, boot.NoOverlay, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err)
+	}
+	log.Debugf("Created overlay filestore file at %q for mount source %q", filestorePath, mountSrc)
+	// Filestore in self should be a named path because it needs to be
+	// discoverable via path traversal so that k8s can scan the filesystem
+	// and apply any limits appropriately (like local ephemeral storage
+	// limits). So don't delete it. These files will be unlinked when the
+	// container is destroyed. This makes self medium appropriate for k8s.
+	return os.NewFile(uintptr(filestoreFD), filestorePath), boot.SelfMedium, nil
+}
+
+func (c *Container) createOverlayFilestoreInDir() (*os.File, boot.OverlayMedium, error) {
 	filestoreDir := c.OverlayConf.HostFileDir()
 	fileInfo, err := os.Stat(filestoreDir)
 	if err != nil {
-		return nil, fmt.Errorf("failed to stat overlay filestore directory %q: %v", filestoreDir, err)
+		return nil, boot.NoOverlay, fmt.Errorf("failed to stat overlay filestore directory %q: %v", filestoreDir, err)
 	}
 	if !fileInfo.IsDir() {
-		return nil, fmt.Errorf("overlay2 flag should specify an existing directory")
+		return nil, boot.NoOverlay, fmt.Errorf("overlay2 flag should specify an existing directory")
 	}
-	var filestoreFiles []*os.File
-	if err := c.forEachOverlayMount(func(_ string) error {
-		// Create an unnamed temporary file in filestore directory which will be
-		// deleted when the last FD on it is closed. We don't use O_TMPFILE because
-		// it is not supported on all filesystems. So we simulate it by creating a
-		// named file and then immediately unlinking it while keeping an FD on it.
-		// This file will be deleted when the container exits.
-		filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-overlay-filestore-")
-		if err != nil {
-			return fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err)
-		}
-		if err := unix.Unlink(filestoreFile.Name()); err != nil {
-			return fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err)
-		}
-		filestoreFiles = append(filestoreFiles, filestoreFile)
-		return nil
-	}); err != nil {
-		return nil, err
+	// Create an unnamed temporary file in filestore directory which will be
+	// deleted when the last FD on it is closed. We don't use O_TMPFILE because
+	// it is not supported on all filesystems. So we simulate it by creating a
+	// named file and then immediately unlinking it while keeping an FD on it.
+	// This file will be deleted when the container exits.
+	filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-overlay-filestore-")
+	if err != nil {
+		return nil, boot.NoOverlay, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err)
 	}
-	return filestoreFiles, nil
-}
-
-// forEachOverlayMount calls fn on all mounts that runsc/boot/vfs.go will
-// configure filestore-based overlays on. See containerMounter.configureOverlay().
-//
-// Precondition: Overlay2.IsBackedByHostFile().
-func (c *Container) forEachOverlayMount(fn func(srcDir string) error) error {
-	if c.OverlayConf.RootMount && !c.Spec.Root.Readonly {
-		if err := fn(c.Spec.Root.Path); err != nil {
-			return err
-		}
+	if err := unix.Unlink(filestoreFile.Name()); err != nil {
+		return nil, boot.NoOverlay, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err)
 	}
-	if !c.OverlayConf.SubMounts {
-		return nil
-	}
-	for i := range c.Spec.Mounts {
-		if c.Spec.Mounts[i].Type != boot.Bind {
-			continue
-		}
-		if boot.ParseMountOptions(c.Spec.Mounts[i].Options).ReadOnly {
-			continue
-		}
-		mountSrc := c.Spec.Mounts[i].Source
-		if err := fn(mountSrc); err != nil {
-			return err
-		}
-	}
-	return nil
+	log.Debugf("Created an unnamed overlay filestore file at %q", filestoreDir)
+	return filestoreFile, boot.AnonDirMedium, nil
 }
 
 // saveLocked saves the container metadata to a file.
@@ -1105,6 +1102,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	nextFD := donations.Transfer(cmd, 3)
 
 	cmd.Args = append(cmd.Args, "gofer", "--bundle", bundleDir)
+	cmd.Args = append(cmd.Args, "--overlay-mediums="+boot.ToOverlayMediumFlags(c.OverlayMediums))
 
 	// Open the spec file to donate to the sandbox.
 	specFile, err := specutils.OpenSpec(bundleDir)

runsc/sandbox/sandbox.go

@@ -226,6 +226,11 @@ type Args struct {
 	// mount in the overlay mounts.
 	OverlayFilestoreFiles []*os.File
 
+	// OverlayMediums contains information about how the gofer mounts have been
+	// overlaid. The first entry is for rootfs and the following entries are for
+	// bind mounts in Spec.Mounts (in the same order).
+	OverlayMediums []boot.OverlayMedium
+
 	// MountsFile is a file container mount information from the spec. It's
 	// equivalent to the mounts from the spec, except that all paths have been
 	// resolved to their final absolute location.
@@ -383,7 +388,7 @@ func (s *Sandbox) StartRoot(spec *specs.Spec, conf *config.Config) error {
 }
 
 // StartSubcontainer starts running a sub-container inside the sandbox.
-func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, overlayFilestoreFiles []*os.File) error {
+func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, overlayFilestoreFiles []*os.File, overlayMediums []boot.OverlayMedium) error {
 	log.Debugf("Start sub-container %q in sandbox %q, PID: %d", cid, s.ID, s.Pid.load())
 
 	if err := s.configureStdios(conf, stdios); err != nil {
@@ -406,6 +411,7 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s
 		Conf:                   conf,
 		CID:                    cid,
 		NumOverlayFilestoreFDs: len(overlayFilestoreFiles),
+		OverlayMediums:         overlayMediums,
 		FilePayload:            payload,
 	}
 	if err := s.call(boot.ContMgrStartSubcontainer, &args, nil); err != nil {
@@ -717,6 +723,9 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 		return err
 	}
 
+	// Pass overlay mediums.
+	cmd.Args = append(cmd.Args, "--overlay-mediums="+boot.ToOverlayMediumFlags(args.OverlayMediums))
+
 	// Create a socket for the control server and donate it to the sandbox.
 	controlAddress, sockFD, err := createControlSocket(conf.RootDir, s.ID)
 	if err != nil {

runsc/specutils/fs.go

@@ -108,6 +108,16 @@ func optionsToFlags(opts []string, source map[string]mapping) uint32 {
 	return rv
 }
 
+// IsReadonlyMount returns true if the mount options has read only option.
+func IsReadonlyMount(opts []string) bool {
+	for _, o := range opts {
+		if o == "ro" {
+			return true
+		}
+	}
+	return false
+}
+
 // validateMount validates that spec mounts are correct.
 func validateMount(mnt *specs.Mount) error {
 	if !path.IsAbs(mnt.Destination) {