runsc: add the EROFS rootfs support

This patch adds the EROFS rootfs support. No gofer process will be
created for the container, when there is no need to pass through host
files into a container via the gofer process. Annotations for rootfs
are also introduced to provide extra information, including the mount
source, mount type and overlay config. Additionally, busybox-static
is added to the default image and will be used to build the EROFS
rootfs images during the test.

Updates #8956

Signed-off-by: Tiwei Bie <tiwei.btw@antgroup.com>

images/default/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \
         apt-transport-https ca-certificates gnupg-agent \
         software-properties-common \
         pkg-config libffi-dev patch diffutils libssl-dev iptables kmod \
-        clang crossbuild-essential-amd64 erofs-utils
+        clang crossbuild-essential-amd64 erofs-utils busybox-static
 
 # Install Docker client for the website build.
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -

pkg/sentry/fsimpl/erofs/save_restore.go

@@ -50,7 +50,9 @@ func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRest
 	if got, want := newImage.SuperBlock(), fs.image.SuperBlock(); got != want {
 		return fmt.Errorf("superblock mismatch detected on restore, got %+v, expected %+v", got, want)
 	}
-	fs.image = newImage
+	// We need to update the image in place, as there are other pointers
+	// pointing to this image as well.
+	*fs.image = *newImage
 	return nil
 }
 

runsc/boot/BUILD

@@ -148,6 +148,7 @@ go_test(
         "//pkg/cpuid",
         "//pkg/fspath",
         "//pkg/log",
+        "//pkg/sentry/fsimpl/erofs",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/seccheck",
         "//pkg/sentry/vfs",

runsc/boot/gofer_conf.go

@@ -17,6 +17,8 @@ package boot
 import (
 	"fmt"
 	"strings"
+
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
 )
 
 // GoferMountConfUpperType describes how upper layer is configured for the gofer mount.
@@ -85,6 +87,9 @@ const (
 	// Lisafs indicates that this gofer mount has a LISAFS lower layer.
 	Lisafs
 
+	// Erofs indicates that this gofer mount has an EROFS lower layer.
+	Erofs
+
 	// LowerMax indicates the number of the valid lower layer types.
 	LowerMax
 )
@@ -96,6 +101,8 @@ func (l GoferMountConfLowerType) String() string {
 		return "none"
 	case Lisafs:
 		return "lisafs"
+	case Erofs:
+		return erofs.Name
 	}
 	panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l))
 }
@@ -107,6 +114,8 @@ func (l *GoferMountConfLowerType) Set(v string) error {
 		*l = NoneLower
 	case "lisafs":
 		*l = Lisafs
+	case erofs.Name:
+		*l = Erofs
 	default:
 		return fmt.Errorf("invalid gofer mount config lower layer type: %s", v)
 	}
@@ -168,6 +177,11 @@ func (g GoferMountConf) ShouldUseLisafs() bool {
 	return g.Lower == Lisafs
 }
 
+// ShouldUseErofs returns true if an EROFS should be applied.
+func (g GoferMountConf) ShouldUseErofs() bool {
+	return g.Lower == Erofs
+}
+
 // valid returns true if this is a valid gofer mount config.
 func (g GoferMountConf) valid() bool {
 	return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay)

runsc/boot/gofer_conf_test.go

@@ -25,6 +25,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile bool
 		wantLisafs   bool
 		wantTmpfs    bool
+		wantErofs    bool
 		wantValid    bool
 	}{{
 		cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay},
@@ -36,6 +37,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: false,
 		wantLisafs:   false,
 		wantTmpfs:    true,
+		wantErofs:    false,
 		wantValid:    true,
 	}, {
 		cfg:          GoferMountConf{Lower: NoneLower, Upper: SelfOverlay},
@@ -43,6 +45,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: true,
 		wantLisafs:   false,
 		wantTmpfs:    true,
+		wantErofs:    false,
 		wantValid:    true,
 	}, {
 		cfg:          GoferMountConf{Lower: NoneLower, Upper: AnonOverlay},
@@ -50,6 +53,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: true,
 		wantLisafs:   false,
 		wantTmpfs:    true,
+		wantErofs:    false,
 		wantValid:    true,
 	}, {
 		cfg:          GoferMountConf{Lower: Lisafs, Upper: NoOverlay},
@@ -57,6 +61,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: false,
 		wantLisafs:   true,
 		wantTmpfs:    false,
+		wantErofs:    false,
 		wantValid:    true,
 	}, {
 		cfg:          GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay},
@@ -64,6 +69,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: false,
 		wantLisafs:   true,
 		wantTmpfs:    false,
+		wantErofs:    false,
 		wantValid:    true,
 	}, {
 		cfg:          GoferMountConf{Lower: Lisafs, Upper: SelfOverlay},
@@ -71,6 +77,7 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: true,
 		wantLisafs:   true,
 		wantTmpfs:    false,
+		wantErofs:    false,
 		wantValid:    true,
 	}, {
 		cfg:          GoferMountConf{Lower: Lisafs, Upper: AnonOverlay},
@@ -78,6 +85,39 @@ func TestGoferConf(t *testing.T) {
 		wantHostFile: true,
 		wantLisafs:   true,
 		wantTmpfs:    false,
+		wantErofs:    false,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: NoOverlay},
+		wantOverlay:  false,
+		wantHostFile: false,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: MemoryOverlay},
+		wantOverlay:  true,
+		wantHostFile: false,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: SelfOverlay},
+		wantOverlay:  true,
+		wantHostFile: true,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
+		wantValid:    true,
+	}, {
+		cfg:          GoferMountConf{Lower: Erofs, Upper: AnonOverlay},
+		wantOverlay:  true,
+		wantHostFile: true,
+		wantLisafs:   false,
+		wantTmpfs:    false,
+		wantErofs:    true,
 		wantValid:    true,
 	}, {
 		cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax},
@@ -104,6 +144,9 @@ func TestGoferConf(t *testing.T) {
 		if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs {
 			t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs)
 		}
+		if got := tc.cfg.ShouldUseErofs(); got != tc.wantErofs {
+			t.Errorf("gofer conf = %+v, ShouldUseErofs() = %t, want = %t", tc.cfg, got, tc.wantErofs)
+		}
 	}
 }
 
@@ -116,6 +159,10 @@ func TestGoferConfFlags(t *testing.T) {
 		{Lower: Lisafs, Upper: MemoryOverlay},
 		{Lower: Lisafs, Upper: SelfOverlay},
 		{Lower: Lisafs, Upper: AnonOverlay},
+		{Lower: Erofs, Upper: NoOverlay},
+		{Lower: Erofs, Upper: MemoryOverlay},
+		{Lower: Erofs, Upper: SelfOverlay},
+		{Lower: Erofs, Upper: AnonOverlay},
 	}
 	var got GoferMountConfFlags
 	got.Set(want.String())

runsc/boot/loader.go

@@ -951,7 +951,11 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne
 	if len(info.goferFDs) < 1 {
 		return nil, nil, fmt.Errorf("rootfs gofer FD not found")
 	}
-	l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
+	// TODO(ayushranjan): The gofer monitor should be started as long as the gofer
+	// process exists, even if the root mount is not backed by lisafs.
+	if info.goferMountConfs[0].ShouldUseLisafs() {
+		l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
+	}
 
 	// We can share l.sharedMounts with containerMounter since l.mu is locked.
 	// Hence, mntr must only be used within this function (while l.mu is locked).

runsc/boot/mount_hints.go

@@ -16,17 +16,24 @@ package boot
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
 	"gvisor.dev/gvisor/runsc/config"
 	"gvisor.dev/gvisor/runsc/specutils"
 )
 
-// MountPrefix is the annotation prefix for mount hints.
-const MountPrefix = "dev.gvisor.spec.mount."
+const (
+	// MountPrefix is the annotation prefix for mount hints applied at the pod level.
+	MountPrefix = "dev.gvisor.spec.mount."
+
+	// RootfsPrefix is the annotation prefix for rootfs hint applied at the container level.
+	RootfsPrefix = "dev.gvisor.spec.rootfs."
+)
 
 // ShareType indicates who can access/mutate the volume contents.
 type ShareType int
@@ -219,3 +226,68 @@ func (p *PodMountHints) FindMount(mountSrc string) *MountHint {
 	}
 	return nil
 }
+
+// RootfsHint represents extra information about rootfs that are provided via
+// annotations. They can provide mount source, mount type and overlay config.
+type RootfsHint struct {
+	Mount   specs.Mount
+	Overlay config.OverlayMedium
+}
+
+func (r *RootfsHint) setSource(val string) error {
+	if !filepath.IsAbs(val) {
+		return fmt.Errorf("source should be an absolute path, got %q", val)
+	}
+	r.Mount.Source = val
+	return nil
+}
+
+func (r *RootfsHint) setType(val string) error {
+	switch val {
+	case erofs.Name, Bind:
+		r.Mount.Type = val
+	default:
+		return fmt.Errorf("invalid type %q", val)
+	}
+	return nil
+}
+
+func (r *RootfsHint) setField(key, val string) error {
+	switch key {
+	case "source":
+		return r.setSource(val)
+	case "type":
+		return r.setType(val)
+	case "overlay":
+		return r.Overlay.Set(val)
+	default:
+		return fmt.Errorf("invalid rootfs annotation: %s=%s", key, val)
+	}
+}
+
+// NewRootfsHint instantiates RootfsHint using spec.
+func NewRootfsHint(spec *specs.Spec) (*RootfsHint, error) {
+	var hint *RootfsHint
+	for k, v := range spec.Annotations {
+		// Look for 'dev.gvisor.spec.rootfs' annotations and parse them.
+		if !strings.HasPrefix(k, RootfsPrefix) {
+			continue
+		}
+		// Remove the prefix.
+		k = k[len(RootfsPrefix):]
+		if hint == nil {
+			hint = &RootfsHint{}
+		}
+		if err := hint.setField(k, v); err != nil {
+			return nil, fmt.Errorf("invalid rootfs annotation (key = %q, value = %q): %v", k, v, err)
+		}
+	}
+	// Validate the parsed hint.
+	if hint != nil {
+		log.Infof("Rootfs annotations found, source: %q, type: %q, overlay: %q", hint.Mount.Source, hint.Mount.Type, hint.Overlay)
+		if len(hint.Mount.Source) == 0 || len(hint.Mount.Type) == 0 {
+			return nil, fmt.Errorf("rootfs annotations missing required field(s): %+v", hint)
+		}
+	}
+	return hint, nil
+}

runsc/boot/mount_hints_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
+	"gvisor.dev/gvisor/runsc/config"
 )
 
 func TestPodMountHintsHappy(t *testing.T) {
@@ -247,3 +249,104 @@ func TestHintsCheckCompatible(t *testing.T) {
 		})
 	}
 }
+
+// TestRootfsHintHappy tests that valid rootfs annotations can be parsed correctly.
+func TestRootfsHintHappy(t *testing.T) {
+	const imagePath = "/tmp/rootfs.img"
+	spec := &specs.Spec{
+		Annotations: map[string]string{
+			RootfsPrefix + "source":  imagePath,
+			RootfsPrefix + "type":    erofs.Name,
+			RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+		},
+	}
+	hint, err := NewRootfsHint(spec)
+	if err != nil {
+		t.Fatalf("NewRootfsHint failed: %v", err)
+	}
+
+	// Check that fields were set correctly.
+	if hint.Mount.Source != imagePath {
+		t.Errorf("rootfs source, want: %q, got: %q", imagePath, hint.Mount.Source)
+	}
+	if hint.Mount.Type != erofs.Name {
+		t.Errorf("rootfs type, want: %q, got: %q", erofs.Name, hint.Mount.Type)
+	}
+	if hint.Overlay != config.MemoryOverlay {
+		t.Errorf("rootfs overlay, want: %q, got: %q", config.MemoryOverlay, hint.Overlay)
+	}
+}
+
+// TestRootfsHintErrors tests that proper errors will be returned when parsing
+// invalid rootfs annotations.
+func TestRootfsHintErrors(t *testing.T) {
+	const imagePath = "/tmp/rootfs.img"
+	for _, tst := range []struct {
+		name        string
+		annotations map[string]string
+		error       string
+	}{
+		{
+			name: "invalid source",
+			annotations: map[string]string{
+				RootfsPrefix + "source": "invalid",
+				RootfsPrefix + "type":   erofs.Name,
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "invalid type",
+			annotations: map[string]string{
+				RootfsPrefix + "source": imagePath,
+				RootfsPrefix + "type":   "invalid",
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "invalid overlay",
+			annotations: map[string]string{
+				RootfsPrefix + "source":  imagePath,
+				RootfsPrefix + "type":    erofs.Name,
+				RootfsPrefix + "overlay": "invalid",
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "invalid key",
+			annotations: map[string]string{
+				RootfsPrefix + "invalid": "invalid",
+				RootfsPrefix + "source":  imagePath,
+				RootfsPrefix + "type":    erofs.Name,
+				RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+			},
+			error: "invalid rootfs annotation",
+		},
+		{
+			name: "missing source",
+			annotations: map[string]string{
+				RootfsPrefix + "type":    erofs.Name,
+				RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+			},
+			error: "rootfs annotations missing required field",
+		},
+		{
+			name: "missing type",
+			annotations: map[string]string{
+				RootfsPrefix + "source":  imagePath,
+				RootfsPrefix + "overlay": config.MemoryOverlay.String(),
+			},
+			error: "rootfs annotations missing required field",
+		},
+	} {
+		t.Run(tst.name, func(t *testing.T) {
+			spec := &specs.Spec{Annotations: tst.annotations}
+			hint, err := NewRootfsHint(spec)
+			if err == nil || !strings.Contains(err.Error(), tst.error) {
+				t.Errorf("NewRootfsHint invalid error, want: .*%s.*, got: %v", tst.error, err)
+			}
+			if hint != nil {
+				t.Errorf("NewRootfsHint must return nil on failure: %+v", hint)
+			}
+		})
+	}
+}

runsc/boot/vfs.go

@@ -478,6 +478,19 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 			},
 		}
 
+	case rootfsConf.ShouldUseErofs():
+		fsName = erofs.Name
+		opts = &vfs.MountOptions{
+			ReadOnly: c.root.Readonly,
+			GetFilesystemOptions: vfs.GetFilesystemOptions{
+				InternalMount: true,
+				Data:          fmt.Sprintf("ifd=%d", ioFD),
+				InternalData: erofs.InternalFilesystemOptions{
+					UniqueID: "/",
+				},
+			},
+		}
+
 	default:
 		return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf)
 	}

runsc/cmd/boot.go

@@ -125,7 +125,9 @@ type Boot struct {
 
 	// mountsFD is the file descriptor to read list of mounts after they have
 	// been resolved (direct paths, no symlinks). They are resolved outside the
-	// sandbox (e.g. gofer) and sent through this FD.
+	// sandbox (e.g. gofer) and sent through this FD. When mountsFD is not
+	// provided, there is no cleaning required for mounts and the mounts in
+	// the spec can be used as is.
 	mountsFD int
 
 	podInitConfigFD int
@@ -194,7 +196,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process")
 	f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file")
-	f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
+	f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
 	f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order")
 	f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.")
 	f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.")
@@ -202,7 +204,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.")
 	f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.")
 	f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup")
-	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
+	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
 	f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.")
 	f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.")
 	f.Var(&b.nvidiaDevMinors, "nvidia-dev-minors", "list of device minors for Nvidia GPU devices exposed to the sandbox.")
@@ -378,15 +380,18 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		}
 	}
 
-	// Read resolved mount list and replace the original one from the spec.
-	mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file")
-	cleanMounts, err := specutils.ReadMounts(mountsFile)
-	if err != nil {
+	// When mountsFD is not provided, there is no cleaning required.
+	if b.mountsFD >= 0 {
+		// Read resolved mount list and replace the original one from the spec.
+		mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file")
+		cleanMounts, err := specutils.ReadMounts(mountsFile)
+		if err != nil {
+			mountsFile.Close()
+			util.Fatalf("Error reading mounts file: %v", err)
+		}
 		mountsFile.Close()
-		util.Fatalf("Error reading mounts file: %v", err)
+		spec.Mounts = cleanMounts
 	}
-	mountsFile.Close()
-	spec.Mounts = cleanMounts
 
 	if conf.DirectFS {
 		// sandbox should run with a umask of 0, because we want to preserve file

runsc/cmd/gofer.go

@@ -115,8 +115,8 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) {
 	f.BoolVar(&g.setUpRoot, "setup-root", true, "if true, set up an empty root for the process")
 
 	// Open FDs that are donated to the gofer.
-	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec")
-	f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured")
+	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.")
+	f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.")
 	f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec")
 	f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).")
 
@@ -267,16 +267,20 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		DonateMountPointFD: conf.DirectFS,
 	})
 
-	// Start with root mount, then add any other additional mount as needed.
-	cfgs = append(cfgs, connectionConfig{
-		sock:      newSocket(g.ioFDs[0]),
-		mountPath: "/", // fsgofer process is always chroot()ed. So serve root.
-		readonly:  spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs(),
-	})
-	log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly)
+	ioFDs := g.ioFDs
+	rootfsConf := g.mountConfs[0]
+	if rootfsConf.ShouldUseLisafs() {
+		// Start with root mount, then add any other additional mount as needed.
+		cfgs = append(cfgs, connectionConfig{
+			sock:      newSocket(ioFDs[0]),
+			mountPath: "/", // fsgofer process is always chroot()ed. So serve root.
+			readonly:  spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs(),
+		})
+		log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, ioFDs[0], cfgs[0].readonly)
+		ioFDs = ioFDs[1:]
+	}
 
 	mountIdx := 1 // first one is the root
-	submountIoFDs := g.ioFDs[1:]
 	for _, m := range spec.Mounts {
 		if !specutils.IsGoferMount(m) {
 			continue
@@ -290,11 +294,11 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 			util.Fatalf("mount destination must be absolute: %q", m.Destination)
 		}
 
-		if len(submountIoFDs) == 0 {
+		if len(ioFDs) == 0 {
 			util.Fatalf("no FD found for mount. Did you forget --io-fd? FDs: %d, Mount: %+v", len(g.ioFDs), m)
 		}
-		ioFD := submountIoFDs[0]
-		submountIoFDs = submountIoFDs[1:]
+		ioFD := ioFDs[0]
+		ioFDs = ioFDs[1:]
 		readonly := specutils.IsReadonlyMount(m.Options) || mountConf.ShouldUseOverlayfs()
 		cfgs = append(cfgs, connectionConfig{
 			sock:      newSocket(ioFD),
@@ -304,7 +308,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom
 		log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly)
 	}
 
-	if len(submountIoFDs) > 0 {
+	if len(ioFDs) > 0 {
 		util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs))
 	}
 
@@ -392,17 +396,20 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error {
 		procPath = "/proc/proc"
 	}
 
-	// Mount root path followed by submounts.
-	if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil {
-		return fmt.Errorf("mounting root on root (%q) err: %v", root, err)
-	}
+	rootfsConf := g.mountConfs[0]
+	if rootfsConf.ShouldUseLisafs() {
+		// Mount root path followed by submounts.
+		if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil {
+			return fmt.Errorf("mounting root on root (%q) err: %v", root, err)
+		}
 
-	flags := uint32(unix.MS_SLAVE | unix.MS_REC)
-	if spec.Linux != nil && spec.Linux.RootfsPropagation != "" {
-		flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation})
-	}
-	if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil {
-		return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err)
+		flags := uint32(unix.MS_SLAVE | unix.MS_REC)
+		if spec.Linux != nil && spec.Linux.RootfsPropagation != "" {
+			flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation})
+		}
+		if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil {
+			return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err)
+		}
 	}
 
 	// Replace the current spec, with the clean spec with symlinks resolved.
@@ -423,7 +430,7 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error {
 	}
 
 	// Check if root needs to be remounted as readonly.
-	if spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs() {
+	if rootfsConf.ShouldUseLisafs() && (spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs()) {
 		// If root is a mount point but not read-only, we can change mount options
 		// to make it read-only for extra safety.
 		// unix.MS_NOSUID and unix.MS_NODEV are included here not only

runsc/container/BUILD

@@ -22,6 +22,7 @@ go_library(
         "//pkg/cleanup",
         "//pkg/log",
         "//pkg/sentry/control",
+        "//pkg/sentry/fsimpl/erofs",
         "//pkg/sentry/fsimpl/tmpfs",
         "//pkg/sentry/pgalloc",
         "//pkg/sighandling",
@@ -75,6 +76,7 @@ go_test(
         "//pkg/cleanup",
         "//pkg/log",
         "//pkg/sentry/control",
+        "//pkg/sentry/fsimpl/erofs",
         "//pkg/sentry/kernel",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/limits",

runsc/container/container.go

@@ -37,6 +37,7 @@ import (
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/control"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
 	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sighandling"
@@ -288,16 +289,23 @@ func New(conf *config.Config, args Args) (*Container, error) {
 		if err != nil {
 			return nil, fmt.Errorf("error creating pod mount hints: %w", err)
 		}
-		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints)
+		rootfsHint, err := boot.NewRootfsHint(args.Spec)
+		if err != nil {
+			return nil, fmt.Errorf("error creating rootfs hint: %w", err)
+		}
+		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints, rootfsHint)
 		if err != nil {
 			return nil, err
 		}
+		if !goferConfs[0].ShouldUseLisafs() && conf.NVProxyDocker {
+			return nil, fmt.Errorf("--nvproxy-docker cannot be used together with non-lisafs backed root mount")
+		}
 		c.GoferMountConfs = goferConfs
 		if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil {
 			return nil, err
 		}
 		if err := runInCgroup(containerCgroup, func() error {
-			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached)
+			ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint)
 			if err != nil {
 				return fmt.Errorf("cannot create gofer process: %w", err)
 			}
@@ -450,7 +458,11 @@ func (c *Container) Start(conf *config.Config) error {
 			return err
 		}
 	} else {
-		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints)
+		rootfsHint, err := boot.NewRootfsHint(c.Spec)
+		if err != nil {
+			return fmt.Errorf("error creating rootfs hint: %w", err)
+		}
+		goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints, rootfsHint)
 		if err != nil {
 			return err
 		}
@@ -459,12 +471,14 @@ func (c *Container) Start(conf *config.Config) error {
 		// the start (and all their children processes).
 		if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error {
 			// Create the gofer process.
-			goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false)
+			goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint)
 			if err != nil {
 				return err
 			}
 			defer func() {
-				_ = mountsFile.Close()
+				if mountsFile != nil {
+					_ = mountsFile.Close()
+				}
 				for _, f := range goferFiles {
 					_ = f.Close()
 				}
@@ -473,11 +487,13 @@ func (c *Container) Start(conf *config.Config) error {
 				}
 			}()
 
-			cleanMounts, err := specutils.ReadMounts(mountsFile)
-			if err != nil {
-				return fmt.Errorf("reading mounts file: %v", err)
+			if mountsFile != nil {
+				cleanMounts, err := specutils.ReadMounts(mountsFile)
+				if err != nil {
+					return fmt.Errorf("reading mounts file: %v", err)
+				}
+				c.Spec.Mounts = cleanMounts
 			}
-			c.Spec.Mounts = cleanMounts
 
 			// Setup stdios if the container is not using terminal. Otherwise TTY was
 			// already setup in create.
@@ -913,13 +929,19 @@ func (c *Container) forEachSelfMount(fn func(mountSrc string)) {
 // createGoferFilestores creates the regular files that will back the
 // tmpfs/overlayfs mounts that will overlay some gofer mounts. It also returns
 // information about how each gofer mount is configured.
-func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.GoferMountConf, error) {
+func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints, rootfsHint *boot.RootfsHint) ([]*os.File, []boot.GoferMountConf, error) {
 	var goferFilestores []*os.File
 	var goferConfs []boot.GoferMountConf
 
 	// Handle root mount first.
 	overlayMedium := ovlConf.RootOverlayMedium()
 	mountType := boot.Bind
+	if rootfsHint != nil {
+		overlayMedium = rootfsHint.Overlay
+		if !specutils.IsGoferMount(rootfsHint.Mount) {
+			mountType = rootfsHint.Mount.Type
+		}
+	}
 	if c.Spec.Root.Readonly {
 		overlayMedium = config.NoOverlay
 	}
@@ -976,6 +998,8 @@ func (c *Container) createGoferFilestore(overlayMedium config.OverlayMedium, mou
 		lower = boot.Lisafs
 	case tmpfs.Name:
 		lower = boot.NoneLower
+	case erofs.Name:
+		lower = boot.Erofs
 	default:
 		return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", mountType)
 	}
@@ -1148,7 +1172,33 @@ func (c *Container) waitForStopped() error {
 	return backoff.Retry(op, b)
 }
 
-func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool) ([]*os.File, *os.File, error) {
+// shouldSpawnGofer indicates whether the gofer process should be spawned.
+func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool {
+	for _, cfg := range goferConfs {
+		if cfg.ShouldUseLisafs() {
+			return true
+		}
+	}
+	return false
+}
+
+// createGoferProcess returns an IO file list and a mounts file on success.
+// The IO file list consists of image files and/or socket files to connect to
+// a gofer endpoint for the mount points using Gofers. The mounts file is the
+// file to read list of mounts after they have been resolved (direct paths,
+// no symlinks), and will be nil if there is no cleaning required for mounts.
+func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) {
+	if !shouldSpawnGofer(c.GoferMountConfs) {
+		if !c.GoferMountConfs[0].ShouldUseErofs() {
+			panic("goferless mode is only possible with EROFS rootfs")
+		}
+		ioFile, err := os.Open(rootfsHint.Mount.Source)
+		if err != nil {
+			return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
+		}
+		return []*os.File{ioFile}, nil, nil
+	}
+
 	donations := donation.Agency{}
 	defer donations.Close()
 
@@ -1209,24 +1259,37 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu
 	}
 	donations.DonateAndClose("mounts-fd", mountsGofer)
 
-	// Count the number of mounts using lisafs.
-	lisafsCount := 0
+	// Count the number of mounts that needs an IO file.
+	ioFileCount := 0
 	for _, cfg := range c.GoferMountConfs {
-		if cfg.ShouldUseLisafs() {
-			lisafsCount++
+		if cfg.ShouldUseLisafs() || cfg.ShouldUseErofs() {
+			ioFileCount++
 		}
 	}
 
-	sandEnds := make([]*os.File, 0, lisafsCount)
-	for i := 0; i < lisafsCount; i++ {
-		fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
-		if err != nil {
-			return nil, nil, err
-		}
-		sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD"))
+	sandEnds := make([]*os.File, 0, ioFileCount)
+	for i, cfg := range c.GoferMountConfs {
+		switch {
+		case cfg.ShouldUseLisafs():
+			fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
+			if err != nil {
+				return nil, nil, err
+			}
+			sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD"))
 
-		goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD")
-		donations.DonateAndClose("io-fds", goferEnd)
+			goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD")
+			donations.DonateAndClose("io-fds", goferEnd)
+
+		case cfg.ShouldUseErofs():
+			if i > 0 {
+				return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount")
+			}
+			if f, err := os.Open(rootfsHint.Mount.Source); err != nil {
+				return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err)
+			} else {
+				sandEnds = append(sandEnds, f)
+			}
+		}
 	}
 
 	if attached {

runsc/sandbox/sandbox.go

@@ -223,9 +223,9 @@ type Args struct {
 	// UserLog is the filename to send user-visible logs to. It may be empty.
 	UserLog string
 
-	// IOFiles is the list of files that connect to a gofer endpoint for the
-	// mounts points using Gofers. They must be in the same order as mounts
-	// appear in the spec.
+	// IOFiles is the list of image files and/or socket files that connect to
+	// a gofer endpoint for the mount points using Gofers. They must be in the
+	// same order as mounts appear in the spec.
 	IOFiles []*os.File
 
 	// GoferFilestoreFiles are the regular files that will back the overlayfs or