Merge pull request #9486 from btw616:erofs-CR-and-rootfs-support

PiperOrigin-RevId: 581036949

images/default/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \
         apt-transport-https ca-certificates gnupg-agent \
         software-properties-common \
         pkg-config libffi-dev patch diffutils libssl-dev iptables kmod \
-        clang crossbuild-essential-amd64 erofs-utils
+        clang crossbuild-essential-amd64 erofs-utils busybox-static
 
 # Install Docker client for the website build.
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -

pkg/sentry/fsimpl/erofs/save_restore.go

@@ -50,7 +50,9 @@ func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRest
 	if got, want := newImage.SuperBlock(), fs.image.SuperBlock(); got != want {
 		return fmt.Errorf("superblock mismatch detected on restore, got %+v, expected %+v", got, want)
 	}
-	fs.image = newImage
+	// We need to update the image in place, as there are other pointers
+	// pointing to this image as well.
+	*fs.image = *newImage
 	return nil
 }
 

runsc/boot/BUILD

@@ -148,6 +148,7 @@ go_test(
         "//pkg/cpuid",
         "//pkg/fspath",
         "//pkg/log",
+        "//pkg/sentry/fsimpl/erofs",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/seccheck",
         "//pkg/sentry/vfs",

runsc/boot/gofer_conf.go

@@ -16,52 +16,175 @@ package boot
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
+
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
 )
 
-// GoferMountConf describes how a gofer mount is configured in the sandbox.
-type GoferMountConf int
+// GoferMountConfUpperType describes how upper layer is configured for the gofer mount.
+type GoferMountConfUpperType byte
 
 const (
-	// VanillaGofer indicates that this gofer mount has no special configuration.
-	VanillaGofer GoferMountConf = iota
+	// NoOverlay indicates that this gofer mount has no upper layer. In this case,
+	// this gofer mount must have a lower layer (i.e. lower != NoneLower).
+	NoOverlay GoferMountConfUpperType = iota
 
-	// MemoryOverlay indicates that this gofer mount should be overlaid with an
-	// overlayfs backed by application memory.
+	// MemoryOverlay indicates that this gofer mount should be overlaid with a
+	// tmpfs backed by application memory.
 	MemoryOverlay
 
-	// SelfOverlay indicates that this gofer mount should be overlaid with an
-	// overlayfs backed by a host file in the mount's source directory.
+	// SelfOverlay indicates that this gofer mount should be overlaid with a
+	// tmpfs backed by a host file in the mount's source directory.
 	SelfOverlay
 
-	// AnonOverlay indicates that this gofer mount should be overlaid with an
-	// overlayfs backed by a host file in an anonymous directory.
+	// AnonOverlay indicates that this gofer mount should be overlaid with a
+	// tmpfs backed by a host file in an anonymous directory.
 	AnonOverlay
 
-	// SelfTmpfs indicates that this gofer mount should be overlaid with a tmpfs
-	// mount backed by a host file in the mount's source directory.
-	SelfTmpfs
+	// UpperMax indicates the number of the valid upper layer types.
+	UpperMax
 )
 
+// String returns a human-readable string representing the upper layer type.
+func (u GoferMountConfUpperType) String() string {
+	switch u {
+	case NoOverlay:
+		return "none"
+	case MemoryOverlay:
+		return "memory"
+	case SelfOverlay:
+		return "self"
+	case AnonOverlay:
+		return "anon"
+	}
+	panic(fmt.Sprintf("Invalid gofer mount config upper layer type: %d", u))
+}
+
+// Set sets the value. Set(String()) should be idempotent.
+func (u *GoferMountConfUpperType) Set(v string) error {
+	switch v {
+	case "none":
+		*u = NoOverlay
+	case "memory":
+		*u = MemoryOverlay
+	case "self":
+		*u = SelfOverlay
+	case "anon":
+		*u = AnonOverlay
+	default:
+		return fmt.Errorf("invalid gofer mount config upper layer type: %s", v)
+	}
+	return nil
+}
+
+// GoferMountConfLowerType describes how lower layer is configured for the gofer mount.
+type GoferMountConfLowerType byte
+
+const (
+	// NoneLower indicates that this gofer mount has no lower layer.
+	NoneLower GoferMountConfLowerType = iota
+
+	// Lisafs indicates that this gofer mount has a LISAFS lower layer.
+	Lisafs
+
+	// Erofs indicates that this gofer mount has an EROFS lower layer.
+	Erofs
+
+	// LowerMax indicates the number of the valid lower layer types.
+	LowerMax
+)
+
+// String returns a human-readable string representing the lower layer type.
+func (l GoferMountConfLowerType) String() string {
+	switch l {
+	case NoneLower:
+		return "none"
+	case Lisafs:
+		return "lisafs"
+	case Erofs:
+		return erofs.Name
+	}
+	panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l))
+}
+
+// Set sets the value. Set(String()) should be idempotent.
+func (l *GoferMountConfLowerType) Set(v string) error {
+	switch v {
+	case "none":
+		*l = NoneLower
+	case "lisafs":
+		*l = Lisafs
+	case erofs.Name:
+		*l = Erofs
+	default:
+		return fmt.Errorf("invalid gofer mount config lower layer type: %s", v)
+	}
+	return nil
+}
+
+// GoferMountConf describes how a gofer mount is configured in the sandbox.
+type GoferMountConf struct {
+	Upper GoferMountConfUpperType `json:"upper"`
+	Lower GoferMountConfLowerType `json:"lower"`
+}
+
+// String returns a human-readable string representing the gofer mount config.
+func (g GoferMountConf) String() string {
+	return fmt.Sprintf("%s:%s", g.Lower, g.Upper)
+}
+
+// Set sets the value. Set(String()) should be idempotent.
+func (g *GoferMountConf) Set(v string) error {
+	parts := strings.Split(v, ":")
+	if len(parts) != 2 {
+		return fmt.Errorf("invalid gofer mount config format: %q", v)
+	}
+	if err := g.Lower.Set(parts[0]); err != nil {
+		return err
+	}
+	if err := g.Upper.Set(parts[1]); err != nil {
+		return err
+	}
+	if !g.valid() {
+		return fmt.Errorf("invalid gofer mount config: %+v", g)
+	}
+	return nil
+}
+
 // IsFilestorePresent returns true if a filestore file was associated with this.
 func (g GoferMountConf) IsFilestorePresent() bool {
-	return g == SelfOverlay || g == AnonOverlay || g == SelfTmpfs
+	return g.Upper == SelfOverlay || g.Upper == AnonOverlay
 }
 
 // IsSelfBacked returns true if this mount is backed by a filestore in itself.
 func (g GoferMountConf) IsSelfBacked() bool {
-	return g == SelfOverlay || g == SelfTmpfs
+	return g.Upper == SelfOverlay
 }
 
 // ShouldUseOverlayfs returns true if an overlayfs should be applied.
 func (g GoferMountConf) ShouldUseOverlayfs() bool {
-	return g == MemoryOverlay || g == SelfOverlay || g == AnonOverlay
+	return g.Lower != NoneLower && g.Upper != NoOverlay
+}
+
+// ShouldUseTmpfs returns true if a tmpfs should be applied.
+func (g GoferMountConf) ShouldUseTmpfs() bool {
+	// g.valid() implies that g.Upper != NoOverlay.
+	return g.Lower == NoneLower
 }
 
 // ShouldUseLisafs returns true if a lisafs client/server should be set up.
 func (g GoferMountConf) ShouldUseLisafs() bool {
-	return g == VanillaGofer || g.ShouldUseOverlayfs()
+	return g.Lower == Lisafs
+}
+
+// ShouldUseErofs returns true if an EROFS should be applied.
+func (g GoferMountConf) ShouldUseErofs() bool {
+	return g.Lower == Erofs
+}
+
+// valid returns true if this is a valid gofer mount config.
+func (g GoferMountConf) valid() bool {
+	return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay)
 }
 
 // GoferMountConfFlags can be used with GoferMountConf flags that appear
@@ -70,11 +193,11 @@ type GoferMountConfFlags []GoferMountConf
 
 // String implements flag.Value.
 func (g *GoferMountConfFlags) String() string {
-	confVals := make([]string, 0, len(*g))
+	confs := make([]string, 0, len(*g))
 	for _, confVal := range *g {
-		confVals = append(confVals, strconv.Itoa(int(confVal)))
+		confs = append(confs, confVal.String())
 	}
-	return strings.Join(confVals, ",")
+	return strings.Join(confs, ",")
 }
 
 // Get implements flag.Value.
@@ -92,14 +215,11 @@ func (g *GoferMountConfFlags) GetArray() []GoferMountConf {
 func (g *GoferMountConfFlags) Set(s string) error {
 	confs := strings.Split(s, ",")
 	for _, conf := range confs {
-		confVal, err := strconv.Atoi(conf)
-		if err != nil {
-			return fmt.Errorf("invalid GoferMountConf value (%d): %v", confVal, err)
+		var confVal GoferMountConf
+		if err := confVal.Set(conf); err != nil {
+			return fmt.Errorf("invalid GoferMountConf value (%s): %v", conf, err)
 		}
-		if confVal > int(SelfTmpfs) {
-			return fmt.Errorf("invalid GoferMountConf value (%d)", confVal)
-		}
-		*g = append(*g, GoferMountConf(confVal))
+		*g = append(*g, confVal)
 	}
 	return nil
 }

runsc/boot/gofer_conf_test.go

@@ -20,51 +20,150 @@ import (
 
 func TestGoferConf(t *testing.T) {
 	tcs := []struct {
-		ovl          GoferMountConf
+		cfg          GoferMountConf
 		wantOverlay  bool
 		wantHostFile bool
 		wantLisafs   bool
+		wantTmpfs    bool
+		wantErofs    bool
+		wantValid    bool
 	}{{
-		ovl:          VanillaGofer,
+		cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay},
+		// This is not a valid config.
+		wantValid: false,
+	}, {
+		cfg:          GoferMountConf{Lower: NoneLower, Upper: MemoryOverlay},
 		wantOverlay:  false,
 		wantHostFile: false,
-		wantLisafs:   true,
+		wantLisafs:   false,
+		wantTmpfs:    true,
+		wantErofs:    false,
+		wantValid:    true,
 	}, {
-		ovl:          MemoryOverlay,
-		wantOverlay:  true,
-		wantHostFile: false,
-		wantLisafs:   true,
-	}, {
-		ovl:          SelfOverlay,
-		wantOverlay:  true,
-		wantHostFile: true,
-		wantLisafs:   true,
-	}, {
-		ovl:          AnonOverlay,
-		wantOverlay:  true,
-		wantHostFile: true,
-		wantLisafs:   true,
-	}, {
-		ovl:          SelfTmpfs,
+		cfg:          GoferMountConf{Lower: NoneLower, Upper: SelfOverlay},
 		wantOverlay:  false,
 		wantHostFile: true,
 		wantLisafs:   false,
+		wantTmpfs:    true,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: NoneLower, Upper: AnonOverlay},
+		wantOverlay:  false,
+		wantHostFile: true,
+		wantLisafs:   false,
+		wantTmpfs:    true,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Lisafs, Upper: NoOverlay},
+		wantOverlay:  false,
+		wantHostFile: false,
+		wantLisafs:   true,
+		wantTmpfs:    false,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay},
+		wantOverlay:  true,
+		wantHostFile: false,
+		wantLisafs:   true,
+		wantTmpfs:    false,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Lisafs, Upper: SelfOverlay},
+		wantOverlay:  true,
+		wantHostFile: true,
+		wantLisafs:   true,
+		wantTmpfs:    false,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Lisafs, Upper: AnonOverlay},
+		wantOverlay:  true,
+		wantHostFile: true,
+		wantLisafs:   true,
+		wantTmpfs:    false,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: NoOverlay},
+		wantOverlay:  false,
+		wantHostFile: false,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: MemoryOverlay},
+		wantOverlay:  true,
+		wantHostFile: false,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: SelfOverlay},
+		wantOverlay:  true,
+		wantHostFile: true,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: AnonOverlay},
+		wantOverlay:  true,
+		wantHostFile: true,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax},
+		// This is not a valid config.
+		wantValid: false,
 	}}
 	for _, tc := range tcs {
-		if got := tc.ovl.ShouldUseOverlayfs(); got != tc.wantOverlay {
-			t.Errorf("gofer conf = %d, ShouldUseOverlayfs() = %t, want = %t", tc.ovl, got, tc.wantOverlay)
+		if got := tc.cfg.valid(); got != tc.wantValid {
+			t.Errorf("gofer conf = %+v, valid() = %t, want = %t", tc.cfg, got, tc.wantValid)
 		}
-		if got := tc.ovl.IsFilestorePresent(); got != tc.wantHostFile {
-			t.Errorf("gofer conf = %d, IsFilestorePresent() = %t, want = %t", tc.ovl, got, tc.wantHostFile)
+		if !tc.wantValid {
+			// Skip the following tests, if this is not a valid config.
+			continue
 		}
-		if got := tc.ovl.ShouldUseLisafs(); got != tc.wantLisafs {
-			t.Errorf("gofer conf = %d, ShouldUseLisafs() = %t, want = %t", tc.ovl, got, tc.wantLisafs)
+		if got := tc.cfg.ShouldUseOverlayfs(); got != tc.wantOverlay {
+			t.Errorf("gofer conf = %+v, ShouldUseOverlayfs() = %t, want = %t", tc.cfg, got, tc.wantOverlay)
+		}
+		if got := tc.cfg.IsFilestorePresent(); got != tc.wantHostFile {
+			t.Errorf("gofer conf = %+v, IsFilestorePresent() = %t, want = %t", tc.cfg, got, tc.wantHostFile)
+		}
+		if got := tc.cfg.ShouldUseLisafs(); got != tc.wantLisafs {
+			t.Errorf("gofer conf = %+v, ShouldUseLisafs() = %t, want = %t", tc.cfg, got, tc.wantLisafs)
+		}
+		if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs {
+			t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs)
+		}
+		if got := tc.cfg.ShouldUseErofs(); got != tc.wantErofs {
+			t.Errorf("gofer conf = %+v, ShouldUseErofs() = %t, want = %t", tc.cfg, got, tc.wantErofs)
 		}
 	}
 }
 
 func TestGoferConfFlags(t *testing.T) {
-	want := GoferMountConfFlags{VanillaGofer, MemoryOverlay, SelfOverlay, AnonOverlay, SelfTmpfs}
+	want := GoferMountConfFlags{
+		{Lower: NoneLower, Upper: MemoryOverlay},
+		{Lower: NoneLower, Upper: SelfOverlay},
+		{Lower: NoneLower, Upper: AnonOverlay},
+		{Lower: Lisafs, Upper: NoOverlay},
+		{Lower: Lisafs, Upper: MemoryOverlay},
+		{Lower: Lisafs, Upper: SelfOverlay},
+		{Lower: Lisafs, Upper: AnonOverlay},
+		{Lower: Erofs, Upper: NoOverlay},
+		{Lower: Erofs, Upper: MemoryOverlay},
+		{Lower: Erofs, Upper: SelfOverlay},
+		{Lower: Erofs, Upper: AnonOverlay},
+	}
 	var got GoferMountConfFlags
 	got.Set(want.String())
 	if len(got) != len(want) {

runsc/boot/loader.go

@@ -966,7 +966,11 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne
 	if len(info.goferFDs) < 1 {
 		return nil, nil, fmt.Errorf("rootfs gofer FD not found")
 	}
-	l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
+	// TODO(ayushranjan): The gofer monitor should be started as long as the gofer
+	// process exists, even if the root mount is not backed by lisafs.
+	if info.goferMountConfs[0].ShouldUseLisafs() {
+		l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
+	}
 
 	// We can share l.sharedMounts with containerMounter since l.mu is locked.
 	// Hence, mntr must only be used within this function (while l.mu is locked).

runsc/boot/loader_test.go

@@ -140,7 +140,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error
 		ControllerFD:    fd,
 		GoferFDs:        []int{sandEnd},
 		StdioFDs:        stdio,
-		GoferMountConfs: []GoferMountConf{VanillaGofer},
+		GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}},
 		PodInitConfigFD: -1,
 		ExecFD:          -1,
 	}

runsc/boot/mount_hints.go

@@ -16,17 +16,24 @@ package boot
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
 	"gvisor.dev/gvisor/runsc/config"
 	"gvisor.dev/gvisor/runsc/specutils"
 )
 
-// MountPrefix is the annotation prefix for mount hints.
-const MountPrefix = "dev.gvisor.spec.mount."
+const (
+	// MountPrefix is the annotation prefix for mount hints applied at the pod level.
+	MountPrefix = "dev.gvisor.spec.mount."
+
+	// RootfsPrefix is the annotation prefix for rootfs hint applied at the container level.
+	RootfsPrefix = "dev.gvisor.spec.rootfs."
+)
 
 // ShareType indicates who can access/mutate the volume contents.
 type ShareType int
@@ -219,3 +226,68 @@ func (p *PodMountHints) FindMount(mountSrc string) *MountHint {
 	}
 	return nil
 }
+
+// RootfsHint represents extra information about rootfs that are provided via
+// annotations. They can provide mount source, mount type and overlay config.
+type RootfsHint struct {
+	Mount   specs.Mount
+	Overlay config.OverlayMedium
+}
+
+func (r *RootfsHint) setSource(val string) error {
+	if !filepath.IsAbs(val) {
+		return fmt.Errorf("source should be an absolute path, got %q", val)
+	}
+	r.Mount.Source = val
+	return nil
+}
+
+func (r *RootfsHint) setType(val string) error {
+	switch val {
+	case erofs.Name, Bind:
+		r.Mount.Type = val
+	default:
+		return fmt.Errorf("invalid type %q", val)
+	}
+	return nil
+}
+
+func (r *RootfsHint) setField(key, val string) error {
+	switch key {
+	case "source":
+		return r.setSource(val)
+	case "type":
+		return r.setType(val)
+	case "overlay":
+		return r.Overlay.Set(val)
+	default:
+		return fmt.Errorf("invalid rootfs annotation: %s=%s", key, val)
+	}
+}
+
+// NewRootfsHint instantiates RootfsHint using spec.
+func NewRootfsHint(spec *specs.Spec) (*RootfsHint, error) {
+	var hint *RootfsHint
+	for k, v := range spec.Annotations {
+		// Look for 'dev.gvisor.spec.rootfs' annotations and parse them.
+		if !strings.HasPrefix(k, RootfsPrefix) {
+			continue
+		}
+		// Remove the prefix.
+		k = k[len(RootfsPrefix):]
+		if hint == nil {
+			hint = &RootfsHint{}
+		}
+		if err := hint.setField(k, v); err != nil {
+			return nil, fmt.Errorf("invalid rootfs annotation (key = %q, value = %q): %v", k, v, err)
+		}
+	}
+	// Validate the parsed hint.
+	if hint != nil {
+		log.Infof("Rootfs annotations found, source: %q, type: %q, overlay: %q", hint.Mount.Source, hint.Mount.Type, hint.Overlay)
+		if len(hint.Mount.Source) == 0 || len(hint.Mount.Type) == 0 {
+			return nil, fmt.Errorf("rootfs annotations missing required field(s): %+v", hint)
+		}
+	}
+	return hint, nil
+}

runsc/boot/mount_hints_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
+	"gvisor.dev/gvisor/runsc/config"
 )
 
 func TestPodMountHintsHappy(t *testing.T) {
@@ -247,3 +249,104 @@ func TestHintsCheckCompatible(t *testing.T) {
 		})
 	}
 }
+
+// TestRootfsHintHappy tests that valid rootfs annotations can be parsed correctly.
+func TestRootfsHintHappy(t *testing.T) {
+	const imagePath = "/tmp/rootfs.img"
+	spec := &specs.Spec{
+		Annotations: map[string]string{
+			RootfsPrefix + "source":  imagePath,
+			RootfsPrefix + "type":    erofs.Name,
+			RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+		},
+	}
+	hint, err := NewRootfsHint(spec)
+	if err != nil {
+		t.Fatalf("NewRootfsHint failed: %v", err)
+	}
+
+	// Check that fields were set correctly.
+	if hint.Mount.Source != imagePath {
+		t.Errorf("rootfs source, want: %q, got: %q", imagePath, hint.Mount.Source)
+	}
+	if hint.Mount.Type != erofs.Name {
+		t.Errorf("rootfs type, want: %q, got: %q", erofs.Name, hint.Mount.Type)
+	}
+	if hint.Overlay != config.MemoryOverlay {
+		t.Errorf("rootfs overlay, want: %q, got: %q", config.MemoryOverlay, hint.Overlay)
+	}
+}
+
+// TestRootfsHintErrors tests that proper errors will be returned when parsing
+// invalid rootfs annotations.
+func TestRootfsHintErrors(t *testing.T) {
+	const imagePath = "/tmp/rootfs.img"
+	for _, tst := range []struct {
+		name        string
+		annotations map[string]string
+		error       string
+	}{
+		{
+			name: "invalid source",
+			annotations: map[string]string{
+				RootfsPrefix + "source": "invalid",
+				RootfsPrefix + "type":   erofs.Name,
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "invalid type",
+			annotations: map[string]string{
+				RootfsPrefix + "source": imagePath,
+				RootfsPrefix + "type":   "invalid",
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "invalid overlay",
+			annotations: map[string]string{
+				RootfsPrefix + "source":  imagePath,
+				RootfsPrefix + "type":    erofs.Name,
+				RootfsPrefix + "overlay": "invalid",
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "invalid key",
+			annotations: map[string]string{
+				RootfsPrefix + "invalid": "invalid",
+				RootfsPrefix + "source":  imagePath,
+				RootfsPrefix + "type":    erofs.Name,
+				RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "missing source",
+			annotations: map[string]string{
+				RootfsPrefix + "type":    erofs.Name,
+				RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+			},
+			error: "rootfs annotations missing required field",
+		},
+		{
+			name: "missing type",
+			annotations: map[string]string{
+				RootfsPrefix + "source":  imagePath,
+				RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+			},
+			error: "rootfs annotations missing required field",
+		},
+	} {
+		t.Run(tst.name, func(t *testing.T) {
+			spec := &specs.Spec{Annotations: tst.annotations}
+			hint, err := NewRootfsHint(spec)
+			if err == nil || !strings.Contains(err.Error(), tst.error) {
+				t.Errorf("NewRootfsHint invalid error, want: .*%s.*, got: %v", tst.error, err)
+			}
+			if hint != nil {
+				t.Errorf("NewRootfsHint must return nil on failure: %+v", hint)
+			}
+		})
+	}
+}

runsc/boot/vfs.go

@@ -447,33 +447,56 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre
 // createMountNamespace creates the container's root mount and namespace.
 func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) {
 	ioFD := c.goferFDs.remove()
-	data := goferMountData(ioFD, conf.FileAccess, conf)
-
-	// We can't check for overlayfs here because sandbox is chroot'ed and gofer
-	// can only send mount options for specs.Mounts (specs.Root is missing
-	// Options field). So assume root is always on top of overlayfs.
-	data = append(data, "overlayfs_stale_read")
-
-	// Configure the gofer dentry cache size.
-	gofer.SetDentryCacheSize(conf.DCache)
-
-	log.Infof("Mounting root with gofer, ioFD: %d", ioFD)
-	opts := &vfs.MountOptions{
-		ReadOnly: c.root.Readonly,
-		GetFilesystemOptions: vfs.GetFilesystemOptions{
-			InternalMount: true,
-			Data:          strings.Join(data, ","),
-			InternalData: gofer.InternalFilesystemOptions{
-				UniqueID: "/",
-			},
-		},
-	}
-
-	fsName := gofer.Name
 	rootfsConf := c.goferMountConfs[0]
-	if rootfsConf == SelfTmpfs {
-		panic("SelfTmpfs is not possible for rootfs")
+
+	var (
+		fsName string
+		opts   *vfs.MountOptions
+	)
+	switch {
+	case rootfsConf.ShouldUseLisafs():
+		fsName = gofer.Name
+
+		data := goferMountData(ioFD, conf.FileAccess, conf)
+
+		// We can't check for overlayfs here because sandbox is chroot'ed and gofer
+		// can only send mount options for specs.Mounts (specs.Root is missing
+		// Options field). So assume root is always on top of overlayfs.
+		data = append(data, "overlayfs_stale_read")
+
+		// Configure the gofer dentry cache size.
+		gofer.SetDentryCacheSize(conf.DCache)
+
+		opts = &vfs.MountOptions{
+			ReadOnly: c.root.Readonly,
+			GetFilesystemOptions: vfs.GetFilesystemOptions{
+				InternalMount: true,
+				Data:          strings.Join(data, ","),
+				InternalData: gofer.InternalFilesystemOptions{
+					UniqueID: "/",
+				},
+			},
+		}
+
+	case rootfsConf.ShouldUseErofs():
+		fsName = erofs.Name
+		opts = &vfs.MountOptions{
+			ReadOnly: c.root.Readonly,
+			GetFilesystemOptions: vfs.GetFilesystemOptions{
+				InternalMount: true,
+				Data:          fmt.Sprintf("ifd=%d", ioFD),
+				InternalData: erofs.InternalFilesystemOptions{
+					UniqueID: "/",
+				},
+			},
+		}
+
+	default:
+		return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf)
 	}
+
+	log.Infof("Mounting root with %s, ioFD: %d", fsName, ioFD)
+
 	if rootfsConf.ShouldUseOverlayfs() {
 		log.Infof("Adding overlay on top of root")
 		var (
@@ -608,7 +631,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 	}
 
 	// We need to hide the filestore from the containerized application.
-	if mountConf == SelfOverlay {
+	if mountConf.IsSelfBacked() {
 		if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{
 			Root:  upperRootVD,
 			Start: upperRootVD,
@@ -721,7 +744,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
 			if info.goferMountConf.IsFilestorePresent() {
 				info.filestoreFD = c.goferFilestoreFDs.removeAsFD()
 			}
-			if info.goferMountConf == SelfTmpfs {
+			if info.goferMountConf.ShouldUseTmpfs() {
 				specutils.ChangeMountType(info.mount, tmpfs.Name)
 			}
 			goferMntIdx++

runsc/cli/cli_test.go

@@ -31,6 +31,7 @@ var fakeFlagValues = [...]string{
 	"2h45m",
 	"1:1,2:2",
 	"0 0 1,100000 100000 65536",
+	"lisafs:self,lisafs:none",
 }
 
 func dupFlag(t *testing.T, cmd subcommands.Command, flagName string) *flag.Flag {

runsc/cmd/boot.go

@@ -125,7 +125,9 @@ type Boot struct {
 
 	// mountsFD is the file descriptor to read list of mounts after they have
 	// been resolved (direct paths, no symlinks). They are resolved outside the
-	// sandbox (e.g. gofer) and sent through this FD.
+	// sandbox (e.g. gofer) and sent through this FD. When mountsFD is not
+	// provided, there is no cleaning required for mounts and the mounts in
+	// the spec can be used as is.
 	mountsFD int
 
 	podInitConfigFD int
@@ -194,7 +196,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process")
 	f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file")
-	f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
+	f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
 	f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order")
 	f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.")
 	f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.")
@@ -202,7 +204,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.")
 	f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.")
 	f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup")
-	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
+	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
 	f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.")
 	f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.")
 	f.Var(&b.nvidiaDevMinors, "nvidia-dev-minors", "list of device minors for Nvidia GPU devices exposed to the sandbox.")
@@ -378,15 +380,18 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		}
 	}
 
-	// Read resolved mount list and replace the original one from the spec.
-	mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file")
-	cleanMounts, err := specutils.ReadMounts(mountsFile)
-	if err != nil {
+	// When mountsFD is not provided, there is no cleaning required.
+	if b.mountsFD >= 0 {
+		// Read resolved mount list and replace the original one from the spec.
+		mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file")
+		cleanMounts, err := specutils.ReadMounts(mountsFile)
+		if err != nil {
+			mountsFile.Close()
+			util.Fatalf("Error reading mounts file: %v", err)
+		}
 		mountsFile.Close()
-		util.Fatalf("Error reading mounts file: %v", err)
+		spec.Mounts = cleanMounts
 	}
-	mountsFile.Close()
-	spec.Mounts = cleanMounts
 
 	if conf.DirectFS {
 		// sandbox should run with a umask of 0, because we want to preserve file

runsc/cmd/gofer.go

@@ -115,8 +115,8 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) {
 	f.BoolVar(&g.setUpRoot, "setup-root", true, "if true, set up an empty root for the process")
 
 	// Open FDs that are donated to the gofer.
-	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec")
-	f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured")
+	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.")
+	f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.")
 	f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).")
 
@@ -267,16 +267,20 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		DonateMountPointFD: conf.DirectFS,
 	})
 
-	// Start with root mount, then add any other additional mount as needed.
-	cfgs = append(cfgs, connectionConfig{
-		sock:      newSocket(g.ioFDs[0]),
-		mountPath: "/", // fsgofer process is always chroot()ed. So serve root.
-		readonly:  spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs(),
-	})
-	log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly)
+	ioFDs := g.ioFDs
+	rootfsConf := g.mountConfs[0]
+	if rootfsConf.ShouldUseLisafs() {
+		// Start with root mount, then add any other additional mount as needed.
+		cfgs = append(cfgs, connectionConfig{
+			sock:      newSocket(ioFDs[0]),
+			mountPath: "/", // fsgofer process is always chroot()ed. So serve root.
+			readonly:  spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs(),
+		})
+		log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, ioFDs[0], cfgs[0].readonly)
+		ioFDs = ioFDs[1:]
+	}
 
 	mountIdx := 1 // first one is the root
-	submountIoFDs := g.ioFDs[1:]
 	for _, m := range spec.Mounts {
 		if !specutils.IsGoferMount(m) {
 			continue
@@ -290,11 +294,11 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 			util.Fatalf("mount destination must be absolute: %q", m.Destination)
 		}
 
-		if len(submountIoFDs) == 0 {
+		if len(ioFDs) == 0 {
 			util.Fatalf("no FD found for mount. Did you forget --io-fd? FDs: %d, Mount: %+v", len(g.ioFDs), m)
 		}
-		ioFD := submountIoFDs[0]
-		submountIoFDs = submountIoFDs[1:]
+		ioFD := ioFDs[0]
+		ioFDs = ioFDs[1:]
 		readonly := specutils.IsReadonlyMount(m.Options) || mountConf.ShouldUseOverlayfs()
 		cfgs = append(cfgs, connectionConfig{
 			sock:      newSocket(ioFD),
@@ -304,7 +308,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly)
 	}
 
-	if len(submountIoFDs) > 0 {
+	if len(ioFDs) > 0 {
 		util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs))
 	}
 
@@ -392,17 +396,20 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error {
 		procPath = "/proc/proc"
 	}
 
-	// Mount root path followed by submounts.
-	if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil {
-		return fmt.Errorf("mounting root on root (%q) err: %v", root, err)
-	}
+	rootfsConf := g.mountConfs[0]
+	if rootfsConf.ShouldUseLisafs() {
+		// Mount root path followed by submounts.
+		if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil {
+			return fmt.Errorf("mounting root on root (%q) err: %v", root, err)
+		}
 
-	flags := uint32(unix.MS_SLAVE | unix.MS_REC)
-	if spec.Linux != nil && spec.Linux.RootfsPropagation != "" {
-		flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation})
-	}
-	if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil {
-		return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err)
+		flags := uint32(unix.MS_SLAVE | unix.MS_REC)
+		if spec.Linux != nil && spec.Linux.RootfsPropagation != "" {
+			flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation})
+		}
+		if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil {
+			return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err)
+		}
 	}
 
 	// Replace the current spec, with the clean spec with symlinks resolved.
@@ -423,7 +430,7 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error {
 	}
 
 	// Check if root needs to be remounted as readonly.
-	if spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs() {
+	if rootfsConf.ShouldUseLisafs() && (spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs()) {
 		// If root is a mount point but not read-only, we can change mount options
 		// to make it read-only for extra safety.
 		// unix.MS_NOSUID and unix.MS_NODEV are included here not only

runsc/config/config.go

@@ -737,17 +737,73 @@ func (g HostFifo) AllowOpen() bool {
 	return g&HostFifoOpen != 0
 }
 
+// OverlayMedium describes how overlay medium is configured.
+type OverlayMedium string
+
+const (
+	// NoOverlay indicates that no overlay will be applied.
+	NoOverlay = OverlayMedium("")
+
+	// MemoryOverlay indicates that the overlay is backed by app memory.
+	MemoryOverlay = OverlayMedium("memory")
+
+	// SelfOverlay indicates that the overlaid mount is backed by itself.
+	SelfOverlay = OverlayMedium("self")
+
+	// AnonOverlayPrefix is the prefix that users should specify in the
+	// config for the anonymous overlay.
+	AnonOverlayPrefix = "dir="
+)
+
+// String returns a human-readable string representing the overlay medium config.
+func (m OverlayMedium) String() string {
+	return string(m)
+}
+
+// Set sets the value. Set(String()) should be idempotent.
+func (m *OverlayMedium) Set(v string) error {
+	switch OverlayMedium(v) {
+	case NoOverlay, MemoryOverlay, SelfOverlay: // OK
+	default:
+		if !strings.HasPrefix(v, AnonOverlayPrefix) {
+			return fmt.Errorf("unexpected medium: %q", v)
+		}
+		if hostFileDir := strings.TrimPrefix(v, AnonOverlayPrefix); !filepath.IsAbs(hostFileDir) {
+			return fmt.Errorf("overlay host file directory should be an absolute path, got %q", hostFileDir)
+		}
+	}
+	*m = OverlayMedium(v)
+	return nil
+}
+
+// IsBackedByAnon indicates whether the overlaid mount is backed by a host file
+// in an anonymous directory.
+func (m OverlayMedium) IsBackedByAnon() bool {
+	return strings.HasPrefix(string(m), AnonOverlayPrefix)
+}
+
+// HostFileDir indicates the directory in which the overlay-backing host file
+// should be created.
+//
+// Precondition: m.IsBackedByAnon().
+func (m OverlayMedium) HostFileDir() string {
+	if !m.IsBackedByAnon() {
+		panic(fmt.Sprintf("anonymous overlay medium = %q does not have %v prefix", m, AnonOverlayPrefix))
+	}
+	return strings.TrimPrefix(string(m), AnonOverlayPrefix)
+}
+
 // Overlay2 holds the configuration for setting up overlay filesystems for the
 // container.
 type Overlay2 struct {
 	rootMount bool
 	subMounts bool
-	medium    string
+	medium    OverlayMedium
 }
 
 func defaultOverlay2() *Overlay2 {
 	// Rootfs overlay is enabled by default and backed by a file in rootfs itself.
-	return &Overlay2{rootMount: true, subMounts: false, medium: "self"}
+	return &Overlay2{rootMount: true, subMounts: false, medium: SelfOverlay}
 }
 
 // Set implements flag.Value. Set(String()) should be idempotent.
@@ -755,7 +811,7 @@ func (o *Overlay2) Set(v string) error {
 	if v == "none" {
 		o.rootMount = false
 		o.subMounts = false
-		o.medium = ""
+		o.medium = NoOverlay
 		return nil
 	}
 	vs := strings.Split(v, ":")
@@ -773,18 +829,7 @@ func (o *Overlay2) Set(v string) error {
 		return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount)
 	}
 
-	o.medium = vs[1]
-	switch o.medium {
-	case "memory", "self": // OK
-	default:
-		if !strings.HasPrefix(o.medium, "dir=") {
-			return fmt.Errorf("unexpected medium specifier for --overlay2: %q", o.medium)
-		}
-		if hostFileDir := strings.TrimPrefix(o.medium, "dir="); !filepath.IsAbs(hostFileDir) {
-			return fmt.Errorf("overlay host file directory should be an absolute path, got %q", hostFileDir)
-		}
-	}
-	return nil
+	return o.medium.Set(vs[1])
 }
 
 // Get implements flag.Value.
@@ -806,47 +851,26 @@ func (o Overlay2) String() string {
 	default:
 		panic("invalid state of subMounts = true and rootMount = false")
 	}
-
-	return res + ":" + o.medium
+	return res + ":" + o.medium.String()
 }
 
 // Enabled returns true if the overlay option is enabled for any mounts.
 func (o *Overlay2) Enabled() bool {
-	return o.rootMount || o.subMounts
+	return o.medium != NoOverlay
 }
 
-// RootEnabled returns true if the overlay is enabled for the root mount.
-func (o *Overlay2) RootEnabled() bool {
-	return o.rootMount
-}
-
-// SubMountEnabled returns true if the overlay is enabled for submounts.
-func (o *Overlay2) SubMountEnabled() bool {
-	return o.subMounts
-}
-
-// IsBackedByMemory indicates whether the overlay is backed by app memory.
-func (o *Overlay2) IsBackedByMemory() bool {
-	return o.Enabled() && o.medium == "memory"
-}
-
-// IsBackedBySelf indicates whether the overlaid mounts are backed by
-// themselves.
-func (o *Overlay2) IsBackedBySelf() bool {
-	return o.Enabled() && o.medium == "self"
-}
-
-// HostFileDir indicates the directory in which the overlay-backing host file
-// should be created.
-//
-// Precondition: o.IsBackedByHostFile() && !o.IsBackedBySelf().
-func (o *Overlay2) HostFileDir() string {
-	if !strings.HasPrefix(o.medium, "dir=") {
-		panic(fmt.Sprintf("Overlay2.Medium = %q does not have dir= prefix when overlay is backed by a host file", o.medium))
+// RootOverlayMedium returns the overlay medium config of the root mount.
+func (o *Overlay2) RootOverlayMedium() OverlayMedium {
+	if !o.rootMount {
+		return NoOverlay
 	}
-	hostFileDir := strings.TrimPrefix(o.medium, "dir=")
-	if !filepath.IsAbs(hostFileDir) {
-		panic(fmt.Sprintf("overlay host file directory should be an absolute path, got %q", hostFileDir))
+	return o.medium
+}
+
+// SubMountOverlayMedium returns the overlay medium config of submounts.
+func (o *Overlay2) SubMountOverlayMedium() OverlayMedium {
+	if !o.subMounts {
+		return NoOverlay
 	}
-	return hostFileDir
+	return o.medium
 }

runsc/config/config_test.go

@@ -224,7 +224,7 @@ func TestInvalidFlags(t *testing.T) {
 		{
 			name:  "overlay2",
 			value: "root:/tmp",
-			error: "unexpected medium specifier for --overlay2: \"/tmp\"",
+			error: "unexpected medium: \"/tmp\"",
 		},
 		{
 			name:  "overlay2",

runsc/container/BUILD

@@ -22,6 +22,8 @@ go_library(
         "//pkg/cleanup",
         "//pkg/log",
         "//pkg/sentry/control",
+        "//pkg/sentry/fsimpl/erofs",
+        "//pkg/sentry/fsimpl/tmpfs",
         "//pkg/sentry/pgalloc",
         "//pkg/sighandling",
         "//pkg/state/statefile",
@@ -74,6 +76,7 @@ go_test(
         "//pkg/cleanup",
         "//pkg/log",
         "//pkg/sentry/control",
+        "//pkg/sentry/fsimpl/erofs",
         "//pkg/sentry/kernel",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/limits",

runsc/container/container.go

@@ -37,6 +37,8 @@ import (
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/control"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
 	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sighandling"
 	"gvisor.dev/gvisor/pkg/state/statefile"
@@ -287,16 +289,23 @@ func New(conf *config.Config, args Args) (*Container, error) {
 		if err != nil {
 			return nil, fmt.Errorf("error creating pod mount hints: %w", err)
 		}
-		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints)
+		rootfsHint, err := boot.NewRootfsHint(args.Spec)
+		if err != nil {
+			return nil, fmt.Errorf("error creating rootfs hint: %w", err)
+		}
+		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints, rootfsHint)
 		if err != nil {
 			return nil, err
 		}
+		if !goferConfs[0].ShouldUseLisafs() && conf.NVProxyDocker {
+			return nil, fmt.Errorf("--nvproxy-docker cannot be used together with non-lisafs backed root mount")
+		}
 		c.GoferMountConfs = goferConfs
 		if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil {
 			return nil, err
 		}
 		if err := runInCgroup(containerCgroup, func() error {
-			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached)
+			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint)
 			if err != nil {
 				return fmt.Errorf("cannot create gofer process: %w", err)
 			}
@@ -449,7 +458,11 @@ func (c *Container) Start(conf *config.Config) error {
 			return err
 		}
 	} else {
-		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints)
+		rootfsHint, err := boot.NewRootfsHint(c.Spec)
+		if err != nil {
+			return fmt.Errorf("error creating rootfs hint: %w", err)
+		}
+		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints, rootfsHint)
 		if err != nil {
 			return err
 		}
@@ -458,12 +471,14 @@ func (c *Container) Start(conf *config.Config) error {
 		// the start (and all their children processes).
 		if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error {
 			// Create the gofer process.
-			goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false)
+			goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint)
 			if err != nil {
 				return err
 			}
 			defer func() {
-				_ = mountsFile.Close()
+				if mountsFile != nil {
+					_ = mountsFile.Close()
+				}
 				for _, f := range goferFiles {
 					_ = f.Close()
 				}
@@ -472,11 +487,13 @@ func (c *Container) Start(conf *config.Config) error {
 				}
 			}()
 
-			cleanMounts, err := specutils.ReadMounts(mountsFile)
-			if err != nil {
-				return fmt.Errorf("reading mounts file: %v", err)
+			if mountsFile != nil {
+				cleanMounts, err := specutils.ReadMounts(mountsFile)
+				if err != nil {
+					return fmt.Errorf("reading mounts file: %v", err)
+				}
+				c.Spec.Mounts = cleanMounts
 			}
-			c.Spec.Mounts = cleanMounts
 
 			// Setup stdios if the container is not using terminal. Otherwise TTY was
 			// already setup in create.
@@ -912,13 +929,23 @@ func (c *Container) forEachSelfMount(fn func(mountSrc string)) {
 // createGoferFilestores creates the regular files that will back the
 // tmpfs/overlayfs mounts that will overlay some gofer mounts. It also returns
 // information about how each gofer mount is configured.
-func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.GoferMountConf, error) {
+func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints, rootfsHint *boot.RootfsHint) ([]*os.File, []boot.GoferMountConf, error) {
 	var goferFilestores []*os.File
 	var goferConfs []boot.GoferMountConf
 
 	// Handle root mount first.
-	shouldOverlay := ovlConf.RootEnabled() && !c.Spec.Root.Readonly
-	filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Root.Path, shouldOverlay, nil /* hint */)
+	overlayMedium := ovlConf.RootOverlayMedium()
+	mountType := boot.Bind
+	if rootfsHint != nil {
+		overlayMedium = rootfsHint.Overlay
+		if !specutils.IsGoferMount(rootfsHint.Mount) {
+			mountType = rootfsHint.Mount.Type
+		}
+	}
+	if c.Spec.Root.Readonly {
+		overlayMedium = config.NoOverlay
+	}
+	filestore, goferConf, err := c.createGoferFilestore(overlayMedium, c.Spec.Root.Path, mountType, false /* isShared */)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -932,9 +959,22 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b
 		if !specutils.IsGoferMount(c.Spec.Mounts[i]) {
 			continue
 		}
-		hint := mountHints.FindMount(c.Spec.Mounts[i].Source)
-		shouldOverlay := ovlConf.SubMountEnabled() && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options)
-		filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Mounts[i].Source, shouldOverlay, hint)
+		overlayMedium = ovlConf.SubMountOverlayMedium()
+		mountType = boot.Bind
+		isShared := false
+		if specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) {
+			overlayMedium = config.NoOverlay
+		}
+		if hint := mountHints.FindMount(c.Spec.Mounts[i].Source); hint != nil {
+			// Note that we want overlayMedium=self even if this is a read-only mount so that
+			// the shared mount is created correctly. Future containers may mount this writably.
+			overlayMedium = config.SelfOverlay
+			if !specutils.IsGoferMount(hint.Mount) {
+				mountType = hint.Mount.Type
+			}
+			isShared = hint.ShouldShareMount()
+		}
+		filestore, goferConf, err := c.createGoferFilestore(overlayMedium, c.Spec.Mounts[i].Source, mountType, isShared)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -951,41 +991,45 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b
 	return goferFilestores, goferConfs, nil
 }
 
-func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.GoferMountConf, error) {
-	// MountHint information takes precedence over shouldOverlay.
-	if hint != nil && !specutils.IsGoferMount(hint.Mount) {
-		switch hint.Mount.Type {
-		case "tmpfs":
-			// Create self-backed tmpfs.
-			return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfTmpfs)
-		default:
-			return nil, boot.VanillaGofer, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type)
-		}
-	}
-	switch {
-	case !shouldOverlay:
-		return nil, boot.VanillaGofer, nil
-	case ovlConf.IsBackedByMemory():
-		return nil, boot.MemoryOverlay, nil
-	case ovlConf.IsBackedBySelf():
-		return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfOverlay)
+func (c *Container) createGoferFilestore(overlayMedium config.OverlayMedium, mountSrc string, mountType string, isShared bool) (*os.File, boot.GoferMountConf, error) {
+	var lower boot.GoferMountConfLowerType
+	switch mountType {
+	case boot.Bind:
+		lower = boot.Lisafs
+	case tmpfs.Name:
+		lower = boot.NoneLower
+	case erofs.Name:
+		lower = boot.Erofs
 	default:
-		return c.createGoferFilestoreInDir(ovlConf)
+		return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", mountType)
+	}
+	switch overlayMedium {
+	case config.NoOverlay:
+		return nil, boot.GoferMountConf{Lower: lower, Upper: boot.NoOverlay}, nil
+	case config.MemoryOverlay:
+		return nil, boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay}, nil
+	case config.SelfOverlay:
+		return c.createGoferFilestoreInSelf(mountSrc, isShared, boot.GoferMountConf{Lower: lower, Upper: boot.SelfOverlay})
+	default:
+		if overlayMedium.IsBackedByAnon() {
+			return c.createGoferFilestoreInDir(overlayMedium.HostFileDir(), boot.GoferMountConf{Lower: lower, Upper: boot.AnonOverlay})
+		}
+		return nil, boot.GoferMountConf{}, fmt.Errorf("unexpected overlay medium %q", overlayMedium)
 	}
 }
 
-func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.MountHint, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) {
+func (c *Container) createGoferFilestoreInSelf(mountSrc string, isShared bool, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) {
 	mountSrcInfo, err := os.Stat(mountSrc)
 	if err != nil {
-		return nil, boot.VanillaGofer, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err)
+		return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err)
 	}
 	if !mountSrcInfo.IsDir() {
 		log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc)
-		return nil, boot.MemoryOverlay, nil
+		return nil, boot.GoferMountConf{Lower: successConf.Lower, Upper: boot.MemoryOverlay}, nil
 	}
 	// Create the self filestore file.
 	createFlags := unix.O_RDWR | unix.O_CREAT | unix.O_CLOEXEC
-	if !(hint != nil && hint.ShouldShareMount()) {
+	if !isShared {
 		// Allow shared mounts to reuse existing filestore. A previous shared user
 		// may have already set up the filestore.
 		createFlags |= unix.O_EXCL
@@ -998,9 +1042,9 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount
 			// same sandbox, and is not shared, then the overlay option doesn't work
 			// correctly. Because each overlay mount is independent and changes to
 			// one are not visible to the other.
-			return nil, boot.VanillaGofer, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath)
+			return nil, boot.GoferMountConf{}, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath)
 		}
-		return nil, boot.VanillaGofer, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err)
+		return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err)
 	}
 	log.Debugf("Created filestore file at %q for mount source %q", filestorePath, mountSrc)
 	// Filestore in self should be a named path because it needs to be
@@ -1011,14 +1055,13 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount
 	return os.NewFile(uintptr(filestoreFD), filestorePath), successConf, nil
 }
 
-func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File, boot.GoferMountConf, error) {
-	filestoreDir := ovlConf.HostFileDir()
+func (c *Container) createGoferFilestoreInDir(filestoreDir string, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) {
 	fileInfo, err := os.Stat(filestoreDir)
 	if err != nil {
-		return nil, boot.VanillaGofer, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err)
+		return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err)
 	}
 	if !fileInfo.IsDir() {
-		return nil, boot.VanillaGofer, fmt.Errorf("overlay2 flag should specify an existing directory")
+		return nil, boot.GoferMountConf{}, fmt.Errorf("overlay2 flag should specify an existing directory")
 	}
 	// Create an unnamed temporary file in filestore directory which will be
 	// deleted when the last FD on it is closed. We don't use O_TMPFILE because
@@ -1027,13 +1070,13 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File
 	// This file will be deleted when the container exits.
 	filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-filestore-")
 	if err != nil {
-		return nil, boot.VanillaGofer, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err)
+		return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err)
 	}
 	if err := unix.Unlink(filestoreFile.Name()); err != nil {
-		return nil, boot.VanillaGofer, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err)
+		return nil, boot.GoferMountConf{}, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err)
 	}
 	log.Debugf("Created an unnamed filestore file at %q", filestoreDir)
-	return filestoreFile, boot.AnonOverlay, nil
+	return filestoreFile, successConf, nil
 }
 
 // saveLocked saves the container metadata to a file.
@@ -1129,7 +1172,33 @@ func (c *Container) waitForStopped() error {
 	return backoff.Retry(op, b)
 }
 
-func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool) ([]*os.File, *os.File, error) {
+// shouldSpawnGofer indicates whether the gofer process should be spawned.
+func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool {
+	for _, cfg := range goferConfs {
+		if cfg.ShouldUseLisafs() {
+			return true
+		}
+	}
+	return false
+}
+
+// createGoferProcess returns an IO file list and a mounts file on success.
+// The IO file list consists of image files and/or socket files to connect to
+// a gofer endpoint for the mount points using Gofers. The mounts file is the
+// file to read list of mounts after they have been resolved (direct paths,
+// no symlinks), and will be nil if there is no cleaning required for mounts.
+func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) {
+	if !shouldSpawnGofer(c.GoferMountConfs) {
+		if !c.GoferMountConfs[0].ShouldUseErofs() {
+			panic("goferless mode is only possible with EROFS rootfs")
+		}
+		ioFile, err := os.Open(rootfsHint.Mount.Source)
+		if err != nil {
+			return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
+		}
+		return []*os.File{ioFile}, nil, nil
+	}
+
 	donations := donation.Agency{}
 	defer donations.Close()
 
@@ -1190,24 +1259,37 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	}
 	donations.DonateAndClose("mounts-fd", mountsGofer)
 
-	// Count the number of mounts using lisafs.
-	lisafsCount := 0
+	// Count the number of mounts that needs an IO file.
+	ioFileCount := 0
 	for _, cfg := range c.GoferMountConfs {
-		if cfg.ShouldUseLisafs() {
-			lisafsCount++
+		if cfg.ShouldUseLisafs() || cfg.ShouldUseErofs() {
+			ioFileCount++
 		}
 	}
 
-	sandEnds := make([]*os.File, 0, lisafsCount)
-	for i := 0; i < lisafsCount; i++ {
-		fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
-		if err != nil {
-			return nil, nil, err
-		}
-		sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD"))
+	sandEnds := make([]*os.File, 0, ioFileCount)
+	for i, cfg := range c.GoferMountConfs {
+		switch {
+		case cfg.ShouldUseLisafs():
+			fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
+			if err != nil {
+				return nil, nil, err
+			}
+			sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD"))
 
-		goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD")
-		donations.DonateAndClose("io-fds", goferEnd)
+			goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD")
+			donations.DonateAndClose("io-fds", goferEnd)
+
+		case cfg.ShouldUseErofs():
+			if i > 0 {
+				return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount")
+			}
+			if f, err := os.Open(rootfsHint.Mount.Source); err != nil {
+				return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
+			} else {
+				sandEnds = append(sandEnds, f)
+			}
+		}
 	}
 
 	if attached {

runsc/sandbox/sandbox.go

@@ -223,9 +223,9 @@ type Args struct {
 	// UserLog is the filename to send user-visible logs to. It may be empty.
 	UserLog string
 
-	// IOFiles is the list of files that connect to a gofer endpoint for the
-	// mounts points using Gofers. They must be in the same order as mounts
-	// appear in the spec.
+	// IOFiles is the list of image files and/or socket files that connect to
+	// a gofer endpoint for the mount points using Gofers. They must be in the
+	// same order as mounts appear in the spec.
 	IOFiles []*os.File
 
 	// GoferFilestoreFiles are the regular files that will back the overlayfs or