diff --git a/images/default/Dockerfile b/images/default/Dockerfile index 128847421..d9af8639f 100644 --- a/images/default/Dockerfile +++ b/images/default/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \ apt-transport-https ca-certificates gnupg-agent \ software-properties-common \ pkg-config libffi-dev patch diffutils libssl-dev iptables kmod \ - clang crossbuild-essential-amd64 erofs-utils + clang crossbuild-essential-amd64 erofs-utils busybox-static # Install Docker client for the website build. RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - diff --git a/pkg/sentry/fsimpl/erofs/save_restore.go b/pkg/sentry/fsimpl/erofs/save_restore.go index a346e5b36..422485bd7 100644 --- a/pkg/sentry/fsimpl/erofs/save_restore.go +++ b/pkg/sentry/fsimpl/erofs/save_restore.go @@ -50,7 +50,9 @@ func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRest if got, want := newImage.SuperBlock(), fs.image.SuperBlock(); got != want { return fmt.Errorf("superblock mismatch detected on restore, got %+v, expected %+v", got, want) } - fs.image = newImage + // We need to update the image in place, as there are other pointers + // pointing to this image as well. + *fs.image = *newImage return nil } diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 3ffdf1d91..97eb48b39 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -148,6 +148,7 @@ go_test( "//pkg/cpuid", "//pkg/fspath", "//pkg/log", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/kernel/auth", "//pkg/sentry/seccheck", "//pkg/sentry/vfs", diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go index f3487346f..9d94cb8f4 100644 --- a/runsc/boot/gofer_conf.go +++ b/runsc/boot/gofer_conf.go @@ -17,6 +17,8 @@ package boot import ( "fmt" "strings" + + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" ) // GoferMountConfUpperType describes how upper layer is configured for the gofer mount. @@ -85,6 +87,9 @@ const ( // Lisafs indicates that this gofer mount has a LISAFS lower layer. Lisafs + // Erofs indicates that this gofer mount has an EROFS lower layer. + Erofs + // LowerMax indicates the number of the valid lower layer types. LowerMax ) @@ -96,6 +101,8 @@ func (l GoferMountConfLowerType) String() string { return "none" case Lisafs: return "lisafs" + case Erofs: + return erofs.Name } panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l)) } @@ -107,6 +114,8 @@ func (l *GoferMountConfLowerType) Set(v string) error { *l = NoneLower case "lisafs": *l = Lisafs + case erofs.Name: + *l = Erofs default: return fmt.Errorf("invalid gofer mount config lower layer type: %s", v) } @@ -168,6 +177,11 @@ func (g GoferMountConf) ShouldUseLisafs() bool { return g.Lower == Lisafs } +// ShouldUseErofs returns true if an EROFS should be applied. +func (g GoferMountConf) ShouldUseErofs() bool { + return g.Lower == Erofs +} + // valid returns true if this is a valid gofer mount config. func (g GoferMountConf) valid() bool { return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay) diff --git a/runsc/boot/gofer_conf_test.go b/runsc/boot/gofer_conf_test.go index 1bc454bc1..49f1f7ab2 100644 --- a/runsc/boot/gofer_conf_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -25,6 +25,7 @@ func TestGoferConf(t *testing.T) { wantHostFile bool wantLisafs bool wantTmpfs bool + wantErofs bool wantValid bool }{{ cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay}, @@ -36,6 +37,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: false, wantLisafs: false, wantTmpfs: true, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: NoneLower, Upper: SelfOverlay}, @@ -43,6 +45,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: false, wantTmpfs: true, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: NoneLower, Upper: AnonOverlay}, @@ -50,6 +53,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: false, wantTmpfs: true, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: NoOverlay}, @@ -57,6 +61,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: false, wantLisafs: true, wantTmpfs: false, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay}, @@ -64,6 +69,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: false, wantLisafs: true, wantTmpfs: false, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: SelfOverlay}, @@ -71,6 +77,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: true, wantTmpfs: false, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: AnonOverlay}, @@ -78,6 +85,39 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: true, wantTmpfs: false, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: NoOverlay}, + wantOverlay: false, + wantHostFile: false, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: MemoryOverlay}, + wantOverlay: true, + wantHostFile: false, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: SelfOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: AnonOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, wantValid: true, }, { cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax}, @@ -104,6 +144,9 @@ func TestGoferConf(t *testing.T) { if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs { t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs) } + if got := tc.cfg.ShouldUseErofs(); got != tc.wantErofs { + t.Errorf("gofer conf = %+v, ShouldUseErofs() = %t, want = %t", tc.cfg, got, tc.wantErofs) + } } } @@ -116,6 +159,10 @@ func TestGoferConfFlags(t *testing.T) { {Lower: Lisafs, Upper: MemoryOverlay}, {Lower: Lisafs, Upper: SelfOverlay}, {Lower: Lisafs, Upper: AnonOverlay}, + {Lower: Erofs, Upper: NoOverlay}, + {Lower: Erofs, Upper: MemoryOverlay}, + {Lower: Erofs, Upper: SelfOverlay}, + {Lower: Erofs, Upper: AnonOverlay}, } var got GoferMountConfFlags got.Set(want.String()) diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index e680b02d5..db7a54ecf 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -951,7 +951,11 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne if len(info.goferFDs) < 1 { return nil, nil, fmt.Errorf("rootfs gofer FD not found") } - l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) + // TODO(ayushranjan): The gofer monitor should be started as long as the gofer + // process exists, even if the root mount is not backed by lisafs. + if info.goferMountConfs[0].ShouldUseLisafs() { + l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) + } // We can share l.sharedMounts with containerMounter since l.mu is locked. // Hence, mntr must only be used within this function (while l.mu is locked). diff --git a/runsc/boot/mount_hints.go b/runsc/boot/mount_hints.go index abe903ffc..c8e428cdb 100644 --- a/runsc/boot/mount_hints.go +++ b/runsc/boot/mount_hints.go @@ -16,17 +16,24 @@ package boot import ( "fmt" + "path/filepath" "strings" specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/specutils" ) -// MountPrefix is the annotation prefix for mount hints. -const MountPrefix = "dev.gvisor.spec.mount." +const ( + // MountPrefix is the annotation prefix for mount hints applied at the pod level. + MountPrefix = "dev.gvisor.spec.mount." + + // RootfsPrefix is the annotation prefix for rootfs hint applied at the container level. + RootfsPrefix = "dev.gvisor.spec.rootfs." +) // ShareType indicates who can access/mutate the volume contents. type ShareType int @@ -219,3 +226,68 @@ func (p *PodMountHints) FindMount(mountSrc string) *MountHint { } return nil } + +// RootfsHint represents extra information about rootfs that are provided via +// annotations. They can provide mount source, mount type and overlay config. +type RootfsHint struct { + Mount specs.Mount + Overlay config.OverlayMedium +} + +func (r *RootfsHint) setSource(val string) error { + if !filepath.IsAbs(val) { + return fmt.Errorf("source should be an absolute path, got %q", val) + } + r.Mount.Source = val + return nil +} + +func (r *RootfsHint) setType(val string) error { + switch val { + case erofs.Name, Bind: + r.Mount.Type = val + default: + return fmt.Errorf("invalid type %q", val) + } + return nil +} + +func (r *RootfsHint) setField(key, val string) error { + switch key { + case "source": + return r.setSource(val) + case "type": + return r.setType(val) + case "overlay": + return r.Overlay.Set(val) + default: + return fmt.Errorf("invalid rootfs annotation: %s=%s", key, val) + } +} + +// NewRootfsHint instantiates RootfsHint using spec. +func NewRootfsHint(spec *specs.Spec) (*RootfsHint, error) { + var hint *RootfsHint + for k, v := range spec.Annotations { + // Look for 'dev.gvisor.spec.rootfs' annotations and parse them. + if !strings.HasPrefix(k, RootfsPrefix) { + continue + } + // Remove the prefix. + k = k[len(RootfsPrefix):] + if hint == nil { + hint = &RootfsHint{} + } + if err := hint.setField(k, v); err != nil { + return nil, fmt.Errorf("invalid rootfs annotation (key = %q, value = %q): %v", k, v, err) + } + } + // Validate the parsed hint. + if hint != nil { + log.Infof("Rootfs annotations found, source: %q, type: %q, overlay: %q", hint.Mount.Source, hint.Mount.Type, hint.Overlay) + if len(hint.Mount.Source) == 0 || len(hint.Mount.Type) == 0 { + return nil, fmt.Errorf("rootfs annotations missing required field(s): %+v", hint) + } + } + return hint, nil +} diff --git a/runsc/boot/mount_hints_test.go b/runsc/boot/mount_hints_test.go index 682e03f36..775db0334 100644 --- a/runsc/boot/mount_hints_test.go +++ b/runsc/boot/mount_hints_test.go @@ -20,6 +20,8 @@ import ( "testing" specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" + "gvisor.dev/gvisor/runsc/config" ) func TestPodMountHintsHappy(t *testing.T) { @@ -247,3 +249,104 @@ func TestHintsCheckCompatible(t *testing.T) { }) } } + +// TestRootfsHintHappy tests that valid rootfs annotations can be parsed correctly. +func TestRootfsHintHappy(t *testing.T) { + const imagePath = "/tmp/rootfs.img" + spec := &specs.Spec{ + Annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + } + hint, err := NewRootfsHint(spec) + if err != nil { + t.Fatalf("NewRootfsHint failed: %v", err) + } + + // Check that fields were set correctly. + if hint.Mount.Source != imagePath { + t.Errorf("rootfs source, want: %q, got: %q", imagePath, hint.Mount.Source) + } + if hint.Mount.Type != erofs.Name { + t.Errorf("rootfs type, want: %q, got: %q", erofs.Name, hint.Mount.Type) + } + if hint.Overlay != config.MemoryOverlay { + t.Errorf("rootfs overlay, want: %q, got: %q", config.MemoryOverlay, hint.Overlay) + } +} + +// TestRootfsHintErrors tests that proper errors will be returned when parsing +// invalid rootfs annotations. +func TestRootfsHintErrors(t *testing.T) { + const imagePath = "/tmp/rootfs.img" + for _, tst := range []struct { + name string + annotations map[string]string + error string + }{ + { + name: "invalid source", + annotations: map[string]string{ + RootfsPrefix + "source": "invalid", + RootfsPrefix + "type": erofs.Name, + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid type", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": "invalid", + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid overlay", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": "invalid", + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid key", + annotations: map[string]string{ + RootfsPrefix + "invalid": "invalid", + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "invalid rootfs annotation", + }, + { + name: "missing source", + annotations: map[string]string{ + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "rootfs annotations missing required field", + }, + { + name: "missing type", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "rootfs annotations missing required field", + }, + } { + t.Run(tst.name, func(t *testing.T) { + spec := &specs.Spec{Annotations: tst.annotations} + hint, err := NewRootfsHint(spec) + if err == nil || !strings.Contains(err.Error(), tst.error) { + t.Errorf("NewRootfsHint invalid error, want: .*%s.*, got: %v", tst.error, err) + } + if hint != nil { + t.Errorf("NewRootfsHint must return nil on failure: %+v", hint) + } + }) + } +} diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 6094e0d46..8a9f6d984 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -478,6 +478,19 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi }, } + case rootfsConf.ShouldUseErofs(): + fsName = erofs.Name + opts = &vfs.MountOptions{ + ReadOnly: c.root.Readonly, + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalMount: true, + Data: fmt.Sprintf("ifd=%d", ioFD), + InternalData: erofs.InternalFilesystemOptions{ + UniqueID: "/", + }, + }, + } + default: return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf) } diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 6691b0c10..ffa92f76f 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -125,7 +125,9 @@ type Boot struct { // mountsFD is the file descriptor to read list of mounts after they have // been resolved (direct paths, no symlinks). They are resolved outside the - // sandbox (e.g. gofer) and sent through this FD. + // sandbox (e.g. gofer) and sent through this FD. When mountsFD is not + // provided, there is no cleaning required for mounts and the mounts in + // the spec can be used as is. mountsFD int podInitConfigFD int @@ -194,7 +196,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process") f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file") - f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") + f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.") f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.") @@ -202,7 +204,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") - f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") + f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.") f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.") f.Var(&b.nvidiaDevMinors, "nvidia-dev-minors", "list of device minors for Nvidia GPU devices exposed to the sandbox.") @@ -378,15 +380,18 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } - // Read resolved mount list and replace the original one from the spec. - mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file") - cleanMounts, err := specutils.ReadMounts(mountsFile) - if err != nil { + // When mountsFD is not provided, there is no cleaning required. + if b.mountsFD >= 0 { + // Read resolved mount list and replace the original one from the spec. + mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file") + cleanMounts, err := specutils.ReadMounts(mountsFile) + if err != nil { + mountsFile.Close() + util.Fatalf("Error reading mounts file: %v", err) + } mountsFile.Close() - util.Fatalf("Error reading mounts file: %v", err) + spec.Mounts = cleanMounts } - mountsFile.Close() - spec.Mounts = cleanMounts if conf.DirectFS { // sandbox should run with a umask of 0, because we want to preserve file diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index aed4bc4f7..333f513b3 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -115,8 +115,8 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) { f.BoolVar(&g.setUpRoot, "setup-root", true, "if true, set up an empty root for the process") // Open FDs that are donated to the gofer. - f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec") - f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured") + f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.") + f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.") f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).") @@ -267,16 +267,20 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom DonateMountPointFD: conf.DirectFS, }) - // Start with root mount, then add any other additional mount as needed. - cfgs = append(cfgs, connectionConfig{ - sock: newSocket(g.ioFDs[0]), - mountPath: "/", // fsgofer process is always chroot()ed. So serve root. - readonly: spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs(), - }) - log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly) + ioFDs := g.ioFDs + rootfsConf := g.mountConfs[0] + if rootfsConf.ShouldUseLisafs() { + // Start with root mount, then add any other additional mount as needed. + cfgs = append(cfgs, connectionConfig{ + sock: newSocket(ioFDs[0]), + mountPath: "/", // fsgofer process is always chroot()ed. So serve root. + readonly: spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs(), + }) + log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, ioFDs[0], cfgs[0].readonly) + ioFDs = ioFDs[1:] + } mountIdx := 1 // first one is the root - submountIoFDs := g.ioFDs[1:] for _, m := range spec.Mounts { if !specutils.IsGoferMount(m) { continue @@ -290,11 +294,11 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom util.Fatalf("mount destination must be absolute: %q", m.Destination) } - if len(submountIoFDs) == 0 { + if len(ioFDs) == 0 { util.Fatalf("no FD found for mount. Did you forget --io-fd? FDs: %d, Mount: %+v", len(g.ioFDs), m) } - ioFD := submountIoFDs[0] - submountIoFDs = submountIoFDs[1:] + ioFD := ioFDs[0] + ioFDs = ioFDs[1:] readonly := specutils.IsReadonlyMount(m.Options) || mountConf.ShouldUseOverlayfs() cfgs = append(cfgs, connectionConfig{ sock: newSocket(ioFD), @@ -304,7 +308,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly) } - if len(submountIoFDs) > 0 { + if len(ioFDs) > 0 { util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs)) } @@ -392,17 +396,20 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { procPath = "/proc/proc" } - // Mount root path followed by submounts. - if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil { - return fmt.Errorf("mounting root on root (%q) err: %v", root, err) - } + rootfsConf := g.mountConfs[0] + if rootfsConf.ShouldUseLisafs() { + // Mount root path followed by submounts. + if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil { + return fmt.Errorf("mounting root on root (%q) err: %v", root, err) + } - flags := uint32(unix.MS_SLAVE | unix.MS_REC) - if spec.Linux != nil && spec.Linux.RootfsPropagation != "" { - flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation}) - } - if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil { - return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err) + flags := uint32(unix.MS_SLAVE | unix.MS_REC) + if spec.Linux != nil && spec.Linux.RootfsPropagation != "" { + flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation}) + } + if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil { + return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err) + } } // Replace the current spec, with the clean spec with symlinks resolved. @@ -423,7 +430,7 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { } // Check if root needs to be remounted as readonly. - if spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs() { + if rootfsConf.ShouldUseLisafs() && (spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs()) { // If root is a mount point but not read-only, we can change mount options // to make it read-only for extra safety. // unix.MS_NOSUID and unix.MS_NODEV are included here not only diff --git a/runsc/container/BUILD b/runsc/container/BUILD index 0d2bf7969..0e129eb4e 100644 --- a/runsc/container/BUILD +++ b/runsc/container/BUILD @@ -22,6 +22,7 @@ go_library( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/pgalloc", "//pkg/sighandling", @@ -75,6 +76,7 @@ go_test( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", "//pkg/sentry/limits", diff --git a/runsc/container/container.go b/runsc/container/container.go index 6c7f6610c..044280d08 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -37,6 +37,7 @@ import ( "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sighandling" @@ -288,16 +289,23 @@ func New(conf *config.Config, args Args) (*Container, error) { if err != nil { return nil, fmt.Errorf("error creating pod mount hints: %w", err) } - goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints) + rootfsHint, err := boot.NewRootfsHint(args.Spec) + if err != nil { + return nil, fmt.Errorf("error creating rootfs hint: %w", err) + } + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints, rootfsHint) if err != nil { return nil, err } + if !goferConfs[0].ShouldUseLisafs() && conf.NVProxyDocker { + return nil, fmt.Errorf("--nvproxy-docker cannot be used together with non-lisafs backed root mount") + } c.GoferMountConfs = goferConfs if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil { return nil, err } if err := runInCgroup(containerCgroup, func() error { - ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached) + ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint) if err != nil { return fmt.Errorf("cannot create gofer process: %w", err) } @@ -450,7 +458,11 @@ func (c *Container) Start(conf *config.Config) error { return err } } else { - goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints) + rootfsHint, err := boot.NewRootfsHint(c.Spec) + if err != nil { + return fmt.Errorf("error creating rootfs hint: %w", err) + } + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints, rootfsHint) if err != nil { return err } @@ -459,12 +471,14 @@ func (c *Container) Start(conf *config.Config) error { // the start (and all their children processes). if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error { // Create the gofer process. - goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false) + goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint) if err != nil { return err } defer func() { - _ = mountsFile.Close() + if mountsFile != nil { + _ = mountsFile.Close() + } for _, f := range goferFiles { _ = f.Close() } @@ -473,11 +487,13 @@ func (c *Container) Start(conf *config.Config) error { } }() - cleanMounts, err := specutils.ReadMounts(mountsFile) - if err != nil { - return fmt.Errorf("reading mounts file: %v", err) + if mountsFile != nil { + cleanMounts, err := specutils.ReadMounts(mountsFile) + if err != nil { + return fmt.Errorf("reading mounts file: %v", err) + } + c.Spec.Mounts = cleanMounts } - c.Spec.Mounts = cleanMounts // Setup stdios if the container is not using terminal. Otherwise TTY was // already setup in create. @@ -913,13 +929,19 @@ func (c *Container) forEachSelfMount(fn func(mountSrc string)) { // createGoferFilestores creates the regular files that will back the // tmpfs/overlayfs mounts that will overlay some gofer mounts. It also returns // information about how each gofer mount is configured. -func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.GoferMountConf, error) { +func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints, rootfsHint *boot.RootfsHint) ([]*os.File, []boot.GoferMountConf, error) { var goferFilestores []*os.File var goferConfs []boot.GoferMountConf // Handle root mount first. overlayMedium := ovlConf.RootOverlayMedium() mountType := boot.Bind + if rootfsHint != nil { + overlayMedium = rootfsHint.Overlay + if !specutils.IsGoferMount(rootfsHint.Mount) { + mountType = rootfsHint.Mount.Type + } + } if c.Spec.Root.Readonly { overlayMedium = config.NoOverlay } @@ -976,6 +998,8 @@ func (c *Container) createGoferFilestore(overlayMedium config.OverlayMedium, mou lower = boot.Lisafs case tmpfs.Name: lower = boot.NoneLower + case erofs.Name: + lower = boot.Erofs default: return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", mountType) } @@ -1148,7 +1172,33 @@ func (c *Container) waitForStopped() error { return backoff.Retry(op, b) } -func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool) ([]*os.File, *os.File, error) { +// shouldSpawnGofer indicates whether the gofer process should be spawned. +func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool { + for _, cfg := range goferConfs { + if cfg.ShouldUseLisafs() { + return true + } + } + return false +} + +// createGoferProcess returns an IO file list and a mounts file on success. +// The IO file list consists of image files and/or socket files to connect to +// a gofer endpoint for the mount points using Gofers. The mounts file is the +// file to read list of mounts after they have been resolved (direct paths, +// no symlinks), and will be nil if there is no cleaning required for mounts. +func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) { + if !shouldSpawnGofer(c.GoferMountConfs) { + if !c.GoferMountConfs[0].ShouldUseErofs() { + panic("goferless mode is only possible with EROFS rootfs") + } + ioFile, err := os.Open(rootfsHint.Mount.Source) + if err != nil { + return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + } + return []*os.File{ioFile}, nil, nil + } + donations := donation.Agency{} defer donations.Close() @@ -1209,24 +1259,37 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu } donations.DonateAndClose("mounts-fd", mountsGofer) - // Count the number of mounts using lisafs. - lisafsCount := 0 + // Count the number of mounts that needs an IO file. + ioFileCount := 0 for _, cfg := range c.GoferMountConfs { - if cfg.ShouldUseLisafs() { - lisafsCount++ + if cfg.ShouldUseLisafs() || cfg.ShouldUseErofs() { + ioFileCount++ } } - sandEnds := make([]*os.File, 0, lisafsCount) - for i := 0; i < lisafsCount; i++ { - fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) - if err != nil { - return nil, nil, err - } - sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) + sandEnds := make([]*os.File, 0, ioFileCount) + for i, cfg := range c.GoferMountConfs { + switch { + case cfg.ShouldUseLisafs(): + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) - goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") - donations.DonateAndClose("io-fds", goferEnd) + goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") + donations.DonateAndClose("io-fds", goferEnd) + + case cfg.ShouldUseErofs(): + if i > 0 { + return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount") + } + if f, err := os.Open(rootfsHint.Mount.Source); err != nil { + return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + } else { + sandEnds = append(sandEnds, f) + } + } } if attached { diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 4aa46d8b2..bef659996 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -39,12 +39,14 @@ import ( "gvisor.dev/gvisor/pkg/bits" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/cgroup" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/flag" @@ -1013,167 +1015,174 @@ func TestKillPid(t *testing.T) { } } -// TestCheckpointRestore creates a container that continuously writes successive +// testCheckpointRestore creates a container that continuously writes successive // integers to a file. To test checkpoint and restore functionality, the // container is checkpointed and the last number printed to the file is // recorded. Then, it is restored in two new containers and the first number // printed from these containers is checked. Both should be the next consecutive // number after the last number from the checkpointed container. +func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript func(string) *specs.Spec) { + dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("ioutil.TempDir failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + outputPath := filepath.Join(dir, "output") + outputFile, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile.Close() + + script := fmt.Sprintf("i=0; while true; do echo $i >> %q; sleep 1; i=$((i+1)); done", outputPath) + spec := newSpecWithScript(script) + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Create and start the container. + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(conf); err != nil { + t.Fatalf("error starting container: %v", err) + } + + // Set the image path, which is where the checkpoint image will be saved. + imagePath := filepath.Join(dir, "test-image-file") + + // Create the image file and open for writing. + file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + t.Fatalf("error opening new file at imagePath: %v", err) + } + defer file.Close() + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + // Checkpoint running container; save state into new file. + if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + t.Fatalf("error checkpointing container to empty file: %v", err) + } + defer os.RemoveAll(imagePath) + + lastNum, err := readOutputNum(outputPath, -1) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile2, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile2.Close() + + // Restore into a new container with different ID (e.g. clone). Keep the + // initial container running to ensure no conflict with it. + args2 := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont2, err := New(conf, args2) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont2.Destroy() + + if err := cont2.Restore(conf, imagePath); err != nil { + t.Fatalf("error restoring container: %v", err) + } + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile2); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) + } + cont2.Destroy() + cont2 = nil + + // Restore into a container using the same ID (e.g. save/resume). It requires + // the original container to cease to exist because they share the same identity. + cont.Destroy() + cont = nil + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile3, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile3.Close() + + cont3, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont3.Destroy() + + if err := cont3.Restore(conf, imagePath); err != nil { + t.Fatalf("error restoring container: %v", err) + } + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile3); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum2, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum2 { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) + } + cont3.Destroy() +} + +// TestCheckpointRestore does the checkpoint/restore test on each platform. func TestCheckpointRestore(t *testing.T) { // Skip overlay because test requires writing to host file. for name, conf := range configs(t, true /* noOverlay */) { t.Run(name, func(t *testing.T) { - dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test") - if err != nil { - t.Fatalf("ioutil.TempDir failed: %v", err) - } - defer os.RemoveAll(dir) - if err := os.Chmod(dir, 0777); err != nil { - t.Fatalf("error chmoding file: %q, %v", dir, err) - } - - outputPath := filepath.Join(dir, "output") - outputFile, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile.Close() - - script := fmt.Sprintf("for ((i=0; ;i++)); do echo $i >> %q; sleep 1; done", outputPath) - spec := testutil.NewSpecWithArgs("bash", "-c", script) - _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) - if err != nil { - t.Fatalf("error setting up container: %v", err) - } - defer cleanup() - - // Create and start the container. - args := Args{ - ID: testutil.RandomContainerID(), - Spec: spec, - BundleDir: bundleDir, - } - cont, err := New(conf, args) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont.Destroy() - if err := cont.Start(conf); err != nil { - t.Fatalf("error starting container: %v", err) - } - - // Set the image path, which is where the checkpoint image will be saved. - imagePath := filepath.Join(dir, "test-image-file") - - // Create the image file and open for writing. - file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - t.Fatalf("error opening new file at imagePath: %v", err) - } - defer file.Close() - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { - t.Fatalf("error checkpointing container to empty file: %v", err) - } - defer os.RemoveAll(imagePath) - - lastNum, err := readOutputNum(outputPath, -1) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Delete and recreate file before restoring. - if err := os.Remove(outputPath); err != nil { - t.Fatalf("error removing file") - } - outputFile2, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile2.Close() - - // Restore into a new container with different ID (e.g. clone). Keep the - // initial container running to ensure no conflict with it. - args2 := Args{ - ID: testutil.RandomContainerID(), - Spec: spec, - BundleDir: bundleDir, - } - cont2, err := New(conf, args2) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont2.Destroy() - - if err := cont2.Restore(conf, imagePath); err != nil { - t.Fatalf("error restoring container: %v", err) - } - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile2); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - firstNum, err := readOutputNum(outputPath, 0) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Check that lastNum is one less than firstNum and that the container - // picks up from where it left off. - if lastNum+1 != firstNum { - t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) - } - cont2.Destroy() - cont2 = nil - - // Restore into a container using the same ID (e.g. save/resume). It requires - // the original container to cease to exist because they share the same identity. - cont.Destroy() - cont = nil - - // Delete and recreate file before restoring. - if err := os.Remove(outputPath); err != nil { - t.Fatalf("error removing file") - } - outputFile3, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile3.Close() - - cont3, err := New(conf, args) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont3.Destroy() - - if err := cont3.Restore(conf, imagePath); err != nil { - t.Fatalf("error restoring container: %v", err) - } - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile3); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - firstNum2, err := readOutputNum(outputPath, 0) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Check that lastNum is one less than firstNum and that the container - // picks up from where it left off. - if lastNum+1 != firstNum2 { - t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) - } - cont3.Destroy() + testCheckpointRestore(t, conf, func(script string) *specs.Spec { + return testutil.NewSpecWithArgs("bash", "-c", script) + }) }) } } @@ -3178,3 +3187,163 @@ find $dir -type l -o -type f | sort | xargs cat | md5sum`), 0755); err != nil { } } } + +// createRootfsEROFS creates a rootfs directory and an EROFS rootfs image in +// the directory dir. +func createRootfsEROFS(dir string) (string, string, error) { + mkfs, err := exec.LookPath("mkfs.erofs") + if err != nil { + return "", "", fmt.Errorf("mkfs.erofs is not available: %v", err) + } + + busybox, err := exec.LookPath("busybox") + if err != nil { + return "", "", fmt.Errorf("busybox is not available: %v", err) + } + + // Create a rootfs directory with busybox in root. + rootfsDir := filepath.Join(dir, "rootfs") + if err := os.Mkdir(rootfsDir, 0755); err != nil { + return "", "", fmt.Errorf("os.Mkdir() failed: %v", err) + } + if err := testutil.Copy(busybox, filepath.Join(rootfsDir, "busybox")); err != nil { + return "", "", fmt.Errorf("failed to copy busybox: %v", err) + } + + // Handcraft the following mount points that the sentry mounts need, because EROFS + // does not support creating synthetic directories yet and we may not want to use + // overlay in some tests. + for _, dir := range []string{"dev", "proc", "sys", "tmp"} { + if err := os.Mkdir(filepath.Join(rootfsDir, dir), 0755); err != nil { + return "", "", fmt.Errorf("os.Mkdir() failed: %v", err) + } + } + + // Build the EROFS rootfs image. + rootfsImage := filepath.Join(dir, "rootfs.img") + cmd := fmt.Sprintf("%s -E noinline_data %s %s", mkfs, rootfsImage, rootfsDir) + if out, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { + return "", "", fmt.Errorf("exec: sh -c %q, err: %v, out: %s", cmd, err, out) + } + + return rootfsDir, rootfsImage, nil +} + +// TestRootfsEROFS starts a container using an EROFS image as the rootfs and checks that +// the rootfs in container is an EROFS. +func TestRootfsEROFS(t *testing.T) { + // Skip this test if mkfs.erofs or busybox are not available. + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skipf("mkfs.erofs is not available: %v", err) + } + if _, err := exec.LookPath("busybox"); err != nil { + t.Skipf("busybox is not available: %v", err) + } + + testDir, err := ioutil.TempDir(testutil.TmpDir(), "erofs_rootfs_test_") + if err != nil { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + rootfsDir, rootfsImage, err := createRootfsEROFS(testDir) + if err != nil { + t.Fatalf("failed to create EROFS rootfs image: %v", err) + } + + // Create the spec and set the EROFS rootfs annotations. + spec := testutil.NewSpecWithArgs("/busybox", "grep", "/ / ro - erofs", "/proc/self/mountinfo") + spec.Root.Path = rootfsDir + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.RootfsPrefix+"type"] = erofs.Name + spec.Annotations[boot.RootfsPrefix+"source"] = rootfsImage + // Disable the overlay, as we want to be sure that rootfs will always be + // shown as EROFS in mountinfo. + spec.Annotations[boot.RootfsPrefix+"overlay"] = config.NoOverlay.String() + + conf := testutil.TestConfig(t) + + for _, mounts := range [][]specs.Mount{ + // Case 1: EROFS rootfs without any other gofer mount. + nil, + + // Case 2: EROFS rootfs with a LISAFS backed gofer mount. + []specs.Mount{ + { + Type: "bind", + Destination: "/tmp", + Source: "/tmp", + }, + }, + } { + spec.Mounts = mounts + + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Create and start the container. + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + Attached: true, + } + ws, err := Run(conf, args) + if err != nil { + t.Fatalf("error running container: %v", err) + } + if ws.ExitStatus() != 0 { + t.Errorf("got exit status %v want %v", ws.ExitStatus(), 0) + } + } +} + +// TestCheckpointRestoreEROFS does the checkpoint/restore test on each platform using +// an EROFS image as the rootfs. +func TestCheckpointRestoreEROFS(t *testing.T) { + // Skip this test if mkfs.erofs or busybox are not available. + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skipf("mkfs.erofs is not available: %v", err) + } + if _, err := exec.LookPath("busybox"); err != nil { + t.Skipf("busybox is not available: %v", err) + } + + testDir, err := ioutil.TempDir(testutil.TmpDir(), "erofs_checkpoint_restore_test_") + if err != nil { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + rootfsDir, rootfsImage, err := createRootfsEROFS(testDir) + if err != nil { + t.Fatalf("failed to create EROFS rootfs image: %v", err) + } + + // Skip overlay because test requires writing to host file. + for name, conf := range configs(t, true /* noOverlay */) { + t.Run(name, func(t *testing.T) { + testCheckpointRestore(t, conf, func(script string) *specs.Spec { + spec := testutil.NewSpecWithArgs("/busybox", "sh", "-c", script) + spec.Root = &specs.Root{ + Path: rootfsDir, + Readonly: false, + } + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.RootfsPrefix+"type"] = erofs.Name + spec.Annotations[boot.RootfsPrefix+"source"] = rootfsImage + // EROFS does not support creating synthetic directories yet, so let's add + // a writeable and savable overlay for rootfs, which allows the sentry to + // create the mount point for the bind mount of the temporary directory shared + // between host and test container. + spec.Annotations[boot.RootfsPrefix+"overlay"] = config.MemoryOverlay.String() + return spec + }) + }) + } +} diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 8b4071a79..44fef8cad 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -223,9 +223,9 @@ type Args struct { // UserLog is the filename to send user-visible logs to. It may be empty. UserLog string - // IOFiles is the list of files that connect to a gofer endpoint for the - // mounts points using Gofers. They must be in the same order as mounts - // appear in the spec. + // IOFiles is the list of image files and/or socket files that connect to + // a gofer endpoint for the mount points using Gofers. They must be in the + // same order as mounts appear in the spec. IOFiles []*os.File // GoferFilestoreFiles are the regular files that will back the overlayfs or