diff --git a/images/default/Dockerfile b/images/default/Dockerfile index 128847421..d9af8639f 100644 --- a/images/default/Dockerfile +++ b/images/default/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \ apt-transport-https ca-certificates gnupg-agent \ software-properties-common \ pkg-config libffi-dev patch diffutils libssl-dev iptables kmod \ - clang crossbuild-essential-amd64 erofs-utils + clang crossbuild-essential-amd64 erofs-utils busybox-static # Install Docker client for the website build. RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - diff --git a/pkg/sentry/fsimpl/erofs/save_restore.go b/pkg/sentry/fsimpl/erofs/save_restore.go index a346e5b36..422485bd7 100644 --- a/pkg/sentry/fsimpl/erofs/save_restore.go +++ b/pkg/sentry/fsimpl/erofs/save_restore.go @@ -50,7 +50,9 @@ func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRest if got, want := newImage.SuperBlock(), fs.image.SuperBlock(); got != want { return fmt.Errorf("superblock mismatch detected on restore, got %+v, expected %+v", got, want) } - fs.image = newImage + // We need to update the image in place, as there are other pointers + // pointing to this image as well. + *fs.image = *newImage return nil } diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 3ffdf1d91..97eb48b39 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -148,6 +148,7 @@ go_test( "//pkg/cpuid", "//pkg/fspath", "//pkg/log", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/kernel/auth", "//pkg/sentry/seccheck", "//pkg/sentry/vfs", diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go index 49177accb..9d94cb8f4 100644 --- a/runsc/boot/gofer_conf.go +++ b/runsc/boot/gofer_conf.go @@ -16,52 +16,175 @@ package boot import ( "fmt" - "strconv" "strings" + + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" ) -// GoferMountConf describes how a gofer mount is configured in the sandbox. -type GoferMountConf int +// GoferMountConfUpperType describes how upper layer is configured for the gofer mount. +type GoferMountConfUpperType byte const ( - // VanillaGofer indicates that this gofer mount has no special configuration. - VanillaGofer GoferMountConf = iota + // NoOverlay indicates that this gofer mount has no upper layer. In this case, + // this gofer mount must have a lower layer (i.e. lower != NoneLower). + NoOverlay GoferMountConfUpperType = iota - // MemoryOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by application memory. + // MemoryOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by application memory. MemoryOverlay - // SelfOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by a host file in the mount's source directory. + // SelfOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by a host file in the mount's source directory. SelfOverlay - // AnonOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by a host file in an anonymous directory. + // AnonOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by a host file in an anonymous directory. AnonOverlay - // SelfTmpfs indicates that this gofer mount should be overlaid with a tmpfs - // mount backed by a host file in the mount's source directory. - SelfTmpfs + // UpperMax indicates the number of the valid upper layer types. + UpperMax ) +// String returns a human-readable string representing the upper layer type. +func (u GoferMountConfUpperType) String() string { + switch u { + case NoOverlay: + return "none" + case MemoryOverlay: + return "memory" + case SelfOverlay: + return "self" + case AnonOverlay: + return "anon" + } + panic(fmt.Sprintf("Invalid gofer mount config upper layer type: %d", u)) +} + +// Set sets the value. Set(String()) should be idempotent. +func (u *GoferMountConfUpperType) Set(v string) error { + switch v { + case "none": + *u = NoOverlay + case "memory": + *u = MemoryOverlay + case "self": + *u = SelfOverlay + case "anon": + *u = AnonOverlay + default: + return fmt.Errorf("invalid gofer mount config upper layer type: %s", v) + } + return nil +} + +// GoferMountConfLowerType describes how lower layer is configured for the gofer mount. +type GoferMountConfLowerType byte + +const ( + // NoneLower indicates that this gofer mount has no lower layer. + NoneLower GoferMountConfLowerType = iota + + // Lisafs indicates that this gofer mount has a LISAFS lower layer. + Lisafs + + // Erofs indicates that this gofer mount has an EROFS lower layer. + Erofs + + // LowerMax indicates the number of the valid lower layer types. + LowerMax +) + +// String returns a human-readable string representing the lower layer type. +func (l GoferMountConfLowerType) String() string { + switch l { + case NoneLower: + return "none" + case Lisafs: + return "lisafs" + case Erofs: + return erofs.Name + } + panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l)) +} + +// Set sets the value. Set(String()) should be idempotent. +func (l *GoferMountConfLowerType) Set(v string) error { + switch v { + case "none": + *l = NoneLower + case "lisafs": + *l = Lisafs + case erofs.Name: + *l = Erofs + default: + return fmt.Errorf("invalid gofer mount config lower layer type: %s", v) + } + return nil +} + +// GoferMountConf describes how a gofer mount is configured in the sandbox. +type GoferMountConf struct { + Upper GoferMountConfUpperType `json:"upper"` + Lower GoferMountConfLowerType `json:"lower"` +} + +// String returns a human-readable string representing the gofer mount config. +func (g GoferMountConf) String() string { + return fmt.Sprintf("%s:%s", g.Lower, g.Upper) +} + +// Set sets the value. Set(String()) should be idempotent. +func (g *GoferMountConf) Set(v string) error { + parts := strings.Split(v, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid gofer mount config format: %q", v) + } + if err := g.Lower.Set(parts[0]); err != nil { + return err + } + if err := g.Upper.Set(parts[1]); err != nil { + return err + } + if !g.valid() { + return fmt.Errorf("invalid gofer mount config: %+v", g) + } + return nil +} + // IsFilestorePresent returns true if a filestore file was associated with this. func (g GoferMountConf) IsFilestorePresent() bool { - return g == SelfOverlay || g == AnonOverlay || g == SelfTmpfs + return g.Upper == SelfOverlay || g.Upper == AnonOverlay } // IsSelfBacked returns true if this mount is backed by a filestore in itself. func (g GoferMountConf) IsSelfBacked() bool { - return g == SelfOverlay || g == SelfTmpfs + return g.Upper == SelfOverlay } // ShouldUseOverlayfs returns true if an overlayfs should be applied. func (g GoferMountConf) ShouldUseOverlayfs() bool { - return g == MemoryOverlay || g == SelfOverlay || g == AnonOverlay + return g.Lower != NoneLower && g.Upper != NoOverlay +} + +// ShouldUseTmpfs returns true if a tmpfs should be applied. +func (g GoferMountConf) ShouldUseTmpfs() bool { + // g.valid() implies that g.Upper != NoOverlay. + return g.Lower == NoneLower } // ShouldUseLisafs returns true if a lisafs client/server should be set up. func (g GoferMountConf) ShouldUseLisafs() bool { - return g == VanillaGofer || g.ShouldUseOverlayfs() + return g.Lower == Lisafs +} + +// ShouldUseErofs returns true if an EROFS should be applied. +func (g GoferMountConf) ShouldUseErofs() bool { + return g.Lower == Erofs +} + +// valid returns true if this is a valid gofer mount config. +func (g GoferMountConf) valid() bool { + return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay) } // GoferMountConfFlags can be used with GoferMountConf flags that appear @@ -70,11 +193,11 @@ type GoferMountConfFlags []GoferMountConf // String implements flag.Value. func (g *GoferMountConfFlags) String() string { - confVals := make([]string, 0, len(*g)) + confs := make([]string, 0, len(*g)) for _, confVal := range *g { - confVals = append(confVals, strconv.Itoa(int(confVal))) + confs = append(confs, confVal.String()) } - return strings.Join(confVals, ",") + return strings.Join(confs, ",") } // Get implements flag.Value. @@ -92,14 +215,11 @@ func (g *GoferMountConfFlags) GetArray() []GoferMountConf { func (g *GoferMountConfFlags) Set(s string) error { confs := strings.Split(s, ",") for _, conf := range confs { - confVal, err := strconv.Atoi(conf) - if err != nil { - return fmt.Errorf("invalid GoferMountConf value (%d): %v", confVal, err) + var confVal GoferMountConf + if err := confVal.Set(conf); err != nil { + return fmt.Errorf("invalid GoferMountConf value (%s): %v", conf, err) } - if confVal > int(SelfTmpfs) { - return fmt.Errorf("invalid GoferMountConf value (%d)", confVal) - } - *g = append(*g, GoferMountConf(confVal)) + *g = append(*g, confVal) } return nil } diff --git a/runsc/boot/gofer_conf_test.go b/runsc/boot/gofer_conf_test.go index f4e00b717..49f1f7ab2 100644 --- a/runsc/boot/gofer_conf_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -20,51 +20,150 @@ import ( func TestGoferConf(t *testing.T) { tcs := []struct { - ovl GoferMountConf + cfg GoferMountConf wantOverlay bool wantHostFile bool wantLisafs bool + wantTmpfs bool + wantErofs bool + wantValid bool }{{ - ovl: VanillaGofer, + cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay}, + // This is not a valid config. + wantValid: false, + }, { + cfg: GoferMountConf{Lower: NoneLower, Upper: MemoryOverlay}, wantOverlay: false, wantHostFile: false, - wantLisafs: true, + wantLisafs: false, + wantTmpfs: true, + wantErofs: false, + wantValid: true, }, { - ovl: MemoryOverlay, - wantOverlay: true, - wantHostFile: false, - wantLisafs: true, - }, { - ovl: SelfOverlay, - wantOverlay: true, - wantHostFile: true, - wantLisafs: true, - }, { - ovl: AnonOverlay, - wantOverlay: true, - wantHostFile: true, - wantLisafs: true, - }, { - ovl: SelfTmpfs, + cfg: GoferMountConf{Lower: NoneLower, Upper: SelfOverlay}, wantOverlay: false, wantHostFile: true, wantLisafs: false, + wantTmpfs: true, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: NoneLower, Upper: AnonOverlay}, + wantOverlay: false, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: true, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: NoOverlay}, + wantOverlay: false, + wantHostFile: false, + wantLisafs: true, + wantTmpfs: false, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay}, + wantOverlay: true, + wantHostFile: false, + wantLisafs: true, + wantTmpfs: false, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: SelfOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: true, + wantTmpfs: false, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: AnonOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: true, + wantTmpfs: false, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: NoOverlay}, + wantOverlay: false, + wantHostFile: false, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: MemoryOverlay}, + wantOverlay: true, + wantHostFile: false, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: SelfOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: AnonOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax}, + // This is not a valid config. + wantValid: false, }} for _, tc := range tcs { - if got := tc.ovl.ShouldUseOverlayfs(); got != tc.wantOverlay { - t.Errorf("gofer conf = %d, ShouldUseOverlayfs() = %t, want = %t", tc.ovl, got, tc.wantOverlay) + if got := tc.cfg.valid(); got != tc.wantValid { + t.Errorf("gofer conf = %+v, valid() = %t, want = %t", tc.cfg, got, tc.wantValid) } - if got := tc.ovl.IsFilestorePresent(); got != tc.wantHostFile { - t.Errorf("gofer conf = %d, IsFilestorePresent() = %t, want = %t", tc.ovl, got, tc.wantHostFile) + if !tc.wantValid { + // Skip the following tests, if this is not a valid config. + continue } - if got := tc.ovl.ShouldUseLisafs(); got != tc.wantLisafs { - t.Errorf("gofer conf = %d, ShouldUseLisafs() = %t, want = %t", tc.ovl, got, tc.wantLisafs) + if got := tc.cfg.ShouldUseOverlayfs(); got != tc.wantOverlay { + t.Errorf("gofer conf = %+v, ShouldUseOverlayfs() = %t, want = %t", tc.cfg, got, tc.wantOverlay) + } + if got := tc.cfg.IsFilestorePresent(); got != tc.wantHostFile { + t.Errorf("gofer conf = %+v, IsFilestorePresent() = %t, want = %t", tc.cfg, got, tc.wantHostFile) + } + if got := tc.cfg.ShouldUseLisafs(); got != tc.wantLisafs { + t.Errorf("gofer conf = %+v, ShouldUseLisafs() = %t, want = %t", tc.cfg, got, tc.wantLisafs) + } + if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs { + t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs) + } + if got := tc.cfg.ShouldUseErofs(); got != tc.wantErofs { + t.Errorf("gofer conf = %+v, ShouldUseErofs() = %t, want = %t", tc.cfg, got, tc.wantErofs) } } } func TestGoferConfFlags(t *testing.T) { - want := GoferMountConfFlags{VanillaGofer, MemoryOverlay, SelfOverlay, AnonOverlay, SelfTmpfs} + want := GoferMountConfFlags{ + {Lower: NoneLower, Upper: MemoryOverlay}, + {Lower: NoneLower, Upper: SelfOverlay}, + {Lower: NoneLower, Upper: AnonOverlay}, + {Lower: Lisafs, Upper: NoOverlay}, + {Lower: Lisafs, Upper: MemoryOverlay}, + {Lower: Lisafs, Upper: SelfOverlay}, + {Lower: Lisafs, Upper: AnonOverlay}, + {Lower: Erofs, Upper: NoOverlay}, + {Lower: Erofs, Upper: MemoryOverlay}, + {Lower: Erofs, Upper: SelfOverlay}, + {Lower: Erofs, Upper: AnonOverlay}, + } var got GoferMountConfFlags got.Set(want.String()) if len(got) != len(want) { diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index fe8c45ec1..aaa57247b 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -966,7 +966,11 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne if len(info.goferFDs) < 1 { return nil, nil, fmt.Errorf("rootfs gofer FD not found") } - l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) + // TODO(ayushranjan): The gofer monitor should be started as long as the gofer + // process exists, even if the root mount is not backed by lisafs. + if info.goferMountConfs[0].ShouldUseLisafs() { + l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) + } // We can share l.sharedMounts with containerMounter since l.mu is locked. // Hence, mntr must only be used within this function (while l.mu is locked). diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index 4abb336ce..552e13654 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -140,7 +140,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error ControllerFD: fd, GoferFDs: []int{sandEnd}, StdioFDs: stdio, - GoferMountConfs: []GoferMountConf{VanillaGofer}, + GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}}, PodInitConfigFD: -1, ExecFD: -1, } diff --git a/runsc/boot/mount_hints.go b/runsc/boot/mount_hints.go index abe903ffc..c8e428cdb 100644 --- a/runsc/boot/mount_hints.go +++ b/runsc/boot/mount_hints.go @@ -16,17 +16,24 @@ package boot import ( "fmt" + "path/filepath" "strings" specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/specutils" ) -// MountPrefix is the annotation prefix for mount hints. -const MountPrefix = "dev.gvisor.spec.mount." +const ( + // MountPrefix is the annotation prefix for mount hints applied at the pod level. + MountPrefix = "dev.gvisor.spec.mount." + + // RootfsPrefix is the annotation prefix for rootfs hint applied at the container level. + RootfsPrefix = "dev.gvisor.spec.rootfs." +) // ShareType indicates who can access/mutate the volume contents. type ShareType int @@ -219,3 +226,68 @@ func (p *PodMountHints) FindMount(mountSrc string) *MountHint { } return nil } + +// RootfsHint represents extra information about rootfs that are provided via +// annotations. They can provide mount source, mount type and overlay config. +type RootfsHint struct { + Mount specs.Mount + Overlay config.OverlayMedium +} + +func (r *RootfsHint) setSource(val string) error { + if !filepath.IsAbs(val) { + return fmt.Errorf("source should be an absolute path, got %q", val) + } + r.Mount.Source = val + return nil +} + +func (r *RootfsHint) setType(val string) error { + switch val { + case erofs.Name, Bind: + r.Mount.Type = val + default: + return fmt.Errorf("invalid type %q", val) + } + return nil +} + +func (r *RootfsHint) setField(key, val string) error { + switch key { + case "source": + return r.setSource(val) + case "type": + return r.setType(val) + case "overlay": + return r.Overlay.Set(val) + default: + return fmt.Errorf("invalid rootfs annotation: %s=%s", key, val) + } +} + +// NewRootfsHint instantiates RootfsHint using spec. +func NewRootfsHint(spec *specs.Spec) (*RootfsHint, error) { + var hint *RootfsHint + for k, v := range spec.Annotations { + // Look for 'dev.gvisor.spec.rootfs' annotations and parse them. + if !strings.HasPrefix(k, RootfsPrefix) { + continue + } + // Remove the prefix. + k = k[len(RootfsPrefix):] + if hint == nil { + hint = &RootfsHint{} + } + if err := hint.setField(k, v); err != nil { + return nil, fmt.Errorf("invalid rootfs annotation (key = %q, value = %q): %v", k, v, err) + } + } + // Validate the parsed hint. + if hint != nil { + log.Infof("Rootfs annotations found, source: %q, type: %q, overlay: %q", hint.Mount.Source, hint.Mount.Type, hint.Overlay) + if len(hint.Mount.Source) == 0 || len(hint.Mount.Type) == 0 { + return nil, fmt.Errorf("rootfs annotations missing required field(s): %+v", hint) + } + } + return hint, nil +} diff --git a/runsc/boot/mount_hints_test.go b/runsc/boot/mount_hints_test.go index 682e03f36..775db0334 100644 --- a/runsc/boot/mount_hints_test.go +++ b/runsc/boot/mount_hints_test.go @@ -20,6 +20,8 @@ import ( "testing" specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" + "gvisor.dev/gvisor/runsc/config" ) func TestPodMountHintsHappy(t *testing.T) { @@ -247,3 +249,104 @@ func TestHintsCheckCompatible(t *testing.T) { }) } } + +// TestRootfsHintHappy tests that valid rootfs annotations can be parsed correctly. +func TestRootfsHintHappy(t *testing.T) { + const imagePath = "/tmp/rootfs.img" + spec := &specs.Spec{ + Annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + } + hint, err := NewRootfsHint(spec) + if err != nil { + t.Fatalf("NewRootfsHint failed: %v", err) + } + + // Check that fields were set correctly. + if hint.Mount.Source != imagePath { + t.Errorf("rootfs source, want: %q, got: %q", imagePath, hint.Mount.Source) + } + if hint.Mount.Type != erofs.Name { + t.Errorf("rootfs type, want: %q, got: %q", erofs.Name, hint.Mount.Type) + } + if hint.Overlay != config.MemoryOverlay { + t.Errorf("rootfs overlay, want: %q, got: %q", config.MemoryOverlay, hint.Overlay) + } +} + +// TestRootfsHintErrors tests that proper errors will be returned when parsing +// invalid rootfs annotations. +func TestRootfsHintErrors(t *testing.T) { + const imagePath = "/tmp/rootfs.img" + for _, tst := range []struct { + name string + annotations map[string]string + error string + }{ + { + name: "invalid source", + annotations: map[string]string{ + RootfsPrefix + "source": "invalid", + RootfsPrefix + "type": erofs.Name, + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid type", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": "invalid", + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid overlay", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": "invalid", + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid key", + annotations: map[string]string{ + RootfsPrefix + "invalid": "invalid", + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "invalid rootfs annotation", + }, + { + name: "missing source", + annotations: map[string]string{ + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "rootfs annotations missing required field", + }, + { + name: "missing type", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "rootfs annotations missing required field", + }, + } { + t.Run(tst.name, func(t *testing.T) { + spec := &specs.Spec{Annotations: tst.annotations} + hint, err := NewRootfsHint(spec) + if err == nil || !strings.Contains(err.Error(), tst.error) { + t.Errorf("NewRootfsHint invalid error, want: .*%s.*, got: %v", tst.error, err) + } + if hint != nil { + t.Errorf("NewRootfsHint must return nil on failure: %+v", hint) + } + }) + } +} diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index aca0ed12d..8a9f6d984 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -447,33 +447,56 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre // createMountNamespace creates the container's root mount and namespace. func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) { ioFD := c.goferFDs.remove() - data := goferMountData(ioFD, conf.FileAccess, conf) - - // We can't check for overlayfs here because sandbox is chroot'ed and gofer - // can only send mount options for specs.Mounts (specs.Root is missing - // Options field). So assume root is always on top of overlayfs. - data = append(data, "overlayfs_stale_read") - - // Configure the gofer dentry cache size. - gofer.SetDentryCacheSize(conf.DCache) - - log.Infof("Mounting root with gofer, ioFD: %d", ioFD) - opts := &vfs.MountOptions{ - ReadOnly: c.root.Readonly, - GetFilesystemOptions: vfs.GetFilesystemOptions{ - InternalMount: true, - Data: strings.Join(data, ","), - InternalData: gofer.InternalFilesystemOptions{ - UniqueID: "/", - }, - }, - } - - fsName := gofer.Name rootfsConf := c.goferMountConfs[0] - if rootfsConf == SelfTmpfs { - panic("SelfTmpfs is not possible for rootfs") + + var ( + fsName string + opts *vfs.MountOptions + ) + switch { + case rootfsConf.ShouldUseLisafs(): + fsName = gofer.Name + + data := goferMountData(ioFD, conf.FileAccess, conf) + + // We can't check for overlayfs here because sandbox is chroot'ed and gofer + // can only send mount options for specs.Mounts (specs.Root is missing + // Options field). So assume root is always on top of overlayfs. + data = append(data, "overlayfs_stale_read") + + // Configure the gofer dentry cache size. + gofer.SetDentryCacheSize(conf.DCache) + + opts = &vfs.MountOptions{ + ReadOnly: c.root.Readonly, + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalMount: true, + Data: strings.Join(data, ","), + InternalData: gofer.InternalFilesystemOptions{ + UniqueID: "/", + }, + }, + } + + case rootfsConf.ShouldUseErofs(): + fsName = erofs.Name + opts = &vfs.MountOptions{ + ReadOnly: c.root.Readonly, + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalMount: true, + Data: fmt.Sprintf("ifd=%d", ioFD), + InternalData: erofs.InternalFilesystemOptions{ + UniqueID: "/", + }, + }, + } + + default: + return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf) } + + log.Infof("Mounting root with %s, ioFD: %d", fsName, ioFD) + if rootfsConf.ShouldUseOverlayfs() { log.Infof("Adding overlay on top of root") var ( @@ -608,7 +631,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co } // We need to hide the filestore from the containerized application. - if mountConf == SelfOverlay { + if mountConf.IsSelfBacked() { if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{ Root: upperRootVD, Start: upperRootVD, @@ -721,7 +744,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) { if info.goferMountConf.IsFilestorePresent() { info.filestoreFD = c.goferFilestoreFDs.removeAsFD() } - if info.goferMountConf == SelfTmpfs { + if info.goferMountConf.ShouldUseTmpfs() { specutils.ChangeMountType(info.mount, tmpfs.Name) } goferMntIdx++ diff --git a/runsc/cli/cli_test.go b/runsc/cli/cli_test.go index 1b1a74641..b45dde51e 100644 --- a/runsc/cli/cli_test.go +++ b/runsc/cli/cli_test.go @@ -31,6 +31,7 @@ var fakeFlagValues = [...]string{ "2h45m", "1:1,2:2", "0 0 1,100000 100000 65536", + "lisafs:self,lisafs:none", } func dupFlag(t *testing.T, cmd subcommands.Command, flagName string) *flag.Flag { diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 6691b0c10..ffa92f76f 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -125,7 +125,9 @@ type Boot struct { // mountsFD is the file descriptor to read list of mounts after they have // been resolved (direct paths, no symlinks). They are resolved outside the - // sandbox (e.g. gofer) and sent through this FD. + // sandbox (e.g. gofer) and sent through this FD. When mountsFD is not + // provided, there is no cleaning required for mounts and the mounts in + // the spec can be used as is. mountsFD int podInitConfigFD int @@ -194,7 +196,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process") f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file") - f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") + f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.") f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.") @@ -202,7 +204,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") - f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") + f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.") f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.") f.Var(&b.nvidiaDevMinors, "nvidia-dev-minors", "list of device minors for Nvidia GPU devices exposed to the sandbox.") @@ -378,15 +380,18 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } - // Read resolved mount list and replace the original one from the spec. - mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file") - cleanMounts, err := specutils.ReadMounts(mountsFile) - if err != nil { + // When mountsFD is not provided, there is no cleaning required. + if b.mountsFD >= 0 { + // Read resolved mount list and replace the original one from the spec. + mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file") + cleanMounts, err := specutils.ReadMounts(mountsFile) + if err != nil { + mountsFile.Close() + util.Fatalf("Error reading mounts file: %v", err) + } mountsFile.Close() - util.Fatalf("Error reading mounts file: %v", err) + spec.Mounts = cleanMounts } - mountsFile.Close() - spec.Mounts = cleanMounts if conf.DirectFS { // sandbox should run with a umask of 0, because we want to preserve file diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index aed4bc4f7..333f513b3 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -115,8 +115,8 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) { f.BoolVar(&g.setUpRoot, "setup-root", true, "if true, set up an empty root for the process") // Open FDs that are donated to the gofer. - f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec") - f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured") + f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.") + f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.") f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).") @@ -267,16 +267,20 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom DonateMountPointFD: conf.DirectFS, }) - // Start with root mount, then add any other additional mount as needed. - cfgs = append(cfgs, connectionConfig{ - sock: newSocket(g.ioFDs[0]), - mountPath: "/", // fsgofer process is always chroot()ed. So serve root. - readonly: spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs(), - }) - log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly) + ioFDs := g.ioFDs + rootfsConf := g.mountConfs[0] + if rootfsConf.ShouldUseLisafs() { + // Start with root mount, then add any other additional mount as needed. + cfgs = append(cfgs, connectionConfig{ + sock: newSocket(ioFDs[0]), + mountPath: "/", // fsgofer process is always chroot()ed. So serve root. + readonly: spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs(), + }) + log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, ioFDs[0], cfgs[0].readonly) + ioFDs = ioFDs[1:] + } mountIdx := 1 // first one is the root - submountIoFDs := g.ioFDs[1:] for _, m := range spec.Mounts { if !specutils.IsGoferMount(m) { continue @@ -290,11 +294,11 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom util.Fatalf("mount destination must be absolute: %q", m.Destination) } - if len(submountIoFDs) == 0 { + if len(ioFDs) == 0 { util.Fatalf("no FD found for mount. Did you forget --io-fd? FDs: %d, Mount: %+v", len(g.ioFDs), m) } - ioFD := submountIoFDs[0] - submountIoFDs = submountIoFDs[1:] + ioFD := ioFDs[0] + ioFDs = ioFDs[1:] readonly := specutils.IsReadonlyMount(m.Options) || mountConf.ShouldUseOverlayfs() cfgs = append(cfgs, connectionConfig{ sock: newSocket(ioFD), @@ -304,7 +308,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly) } - if len(submountIoFDs) > 0 { + if len(ioFDs) > 0 { util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs)) } @@ -392,17 +396,20 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { procPath = "/proc/proc" } - // Mount root path followed by submounts. - if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil { - return fmt.Errorf("mounting root on root (%q) err: %v", root, err) - } + rootfsConf := g.mountConfs[0] + if rootfsConf.ShouldUseLisafs() { + // Mount root path followed by submounts. + if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil { + return fmt.Errorf("mounting root on root (%q) err: %v", root, err) + } - flags := uint32(unix.MS_SLAVE | unix.MS_REC) - if spec.Linux != nil && spec.Linux.RootfsPropagation != "" { - flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation}) - } - if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil { - return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err) + flags := uint32(unix.MS_SLAVE | unix.MS_REC) + if spec.Linux != nil && spec.Linux.RootfsPropagation != "" { + flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation}) + } + if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil { + return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err) + } } // Replace the current spec, with the clean spec with symlinks resolved. @@ -423,7 +430,7 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { } // Check if root needs to be remounted as readonly. - if spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs() { + if rootfsConf.ShouldUseLisafs() && (spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs()) { // If root is a mount point but not read-only, we can change mount options // to make it read-only for extra safety. // unix.MS_NOSUID and unix.MS_NODEV are included here not only diff --git a/runsc/config/config.go b/runsc/config/config.go index 7f97ef15c..37e51a09c 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -737,17 +737,73 @@ func (g HostFifo) AllowOpen() bool { return g&HostFifoOpen != 0 } +// OverlayMedium describes how overlay medium is configured. +type OverlayMedium string + +const ( + // NoOverlay indicates that no overlay will be applied. + NoOverlay = OverlayMedium("") + + // MemoryOverlay indicates that the overlay is backed by app memory. + MemoryOverlay = OverlayMedium("memory") + + // SelfOverlay indicates that the overlaid mount is backed by itself. + SelfOverlay = OverlayMedium("self") + + // AnonOverlayPrefix is the prefix that users should specify in the + // config for the anonymous overlay. + AnonOverlayPrefix = "dir=" +) + +// String returns a human-readable string representing the overlay medium config. +func (m OverlayMedium) String() string { + return string(m) +} + +// Set sets the value. Set(String()) should be idempotent. +func (m *OverlayMedium) Set(v string) error { + switch OverlayMedium(v) { + case NoOverlay, MemoryOverlay, SelfOverlay: // OK + default: + if !strings.HasPrefix(v, AnonOverlayPrefix) { + return fmt.Errorf("unexpected medium: %q", v) + } + if hostFileDir := strings.TrimPrefix(v, AnonOverlayPrefix); !filepath.IsAbs(hostFileDir) { + return fmt.Errorf("overlay host file directory should be an absolute path, got %q", hostFileDir) + } + } + *m = OverlayMedium(v) + return nil +} + +// IsBackedByAnon indicates whether the overlaid mount is backed by a host file +// in an anonymous directory. +func (m OverlayMedium) IsBackedByAnon() bool { + return strings.HasPrefix(string(m), AnonOverlayPrefix) +} + +// HostFileDir indicates the directory in which the overlay-backing host file +// should be created. +// +// Precondition: m.IsBackedByAnon(). +func (m OverlayMedium) HostFileDir() string { + if !m.IsBackedByAnon() { + panic(fmt.Sprintf("anonymous overlay medium = %q does not have %v prefix", m, AnonOverlayPrefix)) + } + return strings.TrimPrefix(string(m), AnonOverlayPrefix) +} + // Overlay2 holds the configuration for setting up overlay filesystems for the // container. type Overlay2 struct { rootMount bool subMounts bool - medium string + medium OverlayMedium } func defaultOverlay2() *Overlay2 { // Rootfs overlay is enabled by default and backed by a file in rootfs itself. - return &Overlay2{rootMount: true, subMounts: false, medium: "self"} + return &Overlay2{rootMount: true, subMounts: false, medium: SelfOverlay} } // Set implements flag.Value. Set(String()) should be idempotent. @@ -755,7 +811,7 @@ func (o *Overlay2) Set(v string) error { if v == "none" { o.rootMount = false o.subMounts = false - o.medium = "" + o.medium = NoOverlay return nil } vs := strings.Split(v, ":") @@ -773,18 +829,7 @@ func (o *Overlay2) Set(v string) error { return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount) } - o.medium = vs[1] - switch o.medium { - case "memory", "self": // OK - default: - if !strings.HasPrefix(o.medium, "dir=") { - return fmt.Errorf("unexpected medium specifier for --overlay2: %q", o.medium) - } - if hostFileDir := strings.TrimPrefix(o.medium, "dir="); !filepath.IsAbs(hostFileDir) { - return fmt.Errorf("overlay host file directory should be an absolute path, got %q", hostFileDir) - } - } - return nil + return o.medium.Set(vs[1]) } // Get implements flag.Value. @@ -806,47 +851,26 @@ func (o Overlay2) String() string { default: panic("invalid state of subMounts = true and rootMount = false") } - - return res + ":" + o.medium + return res + ":" + o.medium.String() } // Enabled returns true if the overlay option is enabled for any mounts. func (o *Overlay2) Enabled() bool { - return o.rootMount || o.subMounts + return o.medium != NoOverlay } -// RootEnabled returns true if the overlay is enabled for the root mount. -func (o *Overlay2) RootEnabled() bool { - return o.rootMount -} - -// SubMountEnabled returns true if the overlay is enabled for submounts. -func (o *Overlay2) SubMountEnabled() bool { - return o.subMounts -} - -// IsBackedByMemory indicates whether the overlay is backed by app memory. -func (o *Overlay2) IsBackedByMemory() bool { - return o.Enabled() && o.medium == "memory" -} - -// IsBackedBySelf indicates whether the overlaid mounts are backed by -// themselves. -func (o *Overlay2) IsBackedBySelf() bool { - return o.Enabled() && o.medium == "self" -} - -// HostFileDir indicates the directory in which the overlay-backing host file -// should be created. -// -// Precondition: o.IsBackedByHostFile() && !o.IsBackedBySelf(). -func (o *Overlay2) HostFileDir() string { - if !strings.HasPrefix(o.medium, "dir=") { - panic(fmt.Sprintf("Overlay2.Medium = %q does not have dir= prefix when overlay is backed by a host file", o.medium)) +// RootOverlayMedium returns the overlay medium config of the root mount. +func (o *Overlay2) RootOverlayMedium() OverlayMedium { + if !o.rootMount { + return NoOverlay } - hostFileDir := strings.TrimPrefix(o.medium, "dir=") - if !filepath.IsAbs(hostFileDir) { - panic(fmt.Sprintf("overlay host file directory should be an absolute path, got %q", hostFileDir)) + return o.medium +} + +// SubMountOverlayMedium returns the overlay medium config of submounts. +func (o *Overlay2) SubMountOverlayMedium() OverlayMedium { + if !o.subMounts { + return NoOverlay } - return hostFileDir + return o.medium } diff --git a/runsc/config/config_test.go b/runsc/config/config_test.go index 5261fb154..d89b4dbbe 100644 --- a/runsc/config/config_test.go +++ b/runsc/config/config_test.go @@ -224,7 +224,7 @@ func TestInvalidFlags(t *testing.T) { { name: "overlay2", value: "root:/tmp", - error: "unexpected medium specifier for --overlay2: \"/tmp\"", + error: "unexpected medium: \"/tmp\"", }, { name: "overlay2", diff --git a/runsc/container/BUILD b/runsc/container/BUILD index eb93d976a..0e129eb4e 100644 --- a/runsc/container/BUILD +++ b/runsc/container/BUILD @@ -22,6 +22,8 @@ go_library( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/erofs", + "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/pgalloc", "//pkg/sighandling", "//pkg/state/statefile", @@ -74,6 +76,7 @@ go_test( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", "//pkg/sentry/limits", diff --git a/runsc/container/container.go b/runsc/container/container.go index 4025ef9d0..044280d08 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -37,6 +37,8 @@ import ( "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sighandling" "gvisor.dev/gvisor/pkg/state/statefile" @@ -287,16 +289,23 @@ func New(conf *config.Config, args Args) (*Container, error) { if err != nil { return nil, fmt.Errorf("error creating pod mount hints: %w", err) } - goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints) + rootfsHint, err := boot.NewRootfsHint(args.Spec) + if err != nil { + return nil, fmt.Errorf("error creating rootfs hint: %w", err) + } + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints, rootfsHint) if err != nil { return nil, err } + if !goferConfs[0].ShouldUseLisafs() && conf.NVProxyDocker { + return nil, fmt.Errorf("--nvproxy-docker cannot be used together with non-lisafs backed root mount") + } c.GoferMountConfs = goferConfs if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil { return nil, err } if err := runInCgroup(containerCgroup, func() error { - ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached) + ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint) if err != nil { return fmt.Errorf("cannot create gofer process: %w", err) } @@ -449,7 +458,11 @@ func (c *Container) Start(conf *config.Config) error { return err } } else { - goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints) + rootfsHint, err := boot.NewRootfsHint(c.Spec) + if err != nil { + return fmt.Errorf("error creating rootfs hint: %w", err) + } + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints, rootfsHint) if err != nil { return err } @@ -458,12 +471,14 @@ func (c *Container) Start(conf *config.Config) error { // the start (and all their children processes). if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error { // Create the gofer process. - goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false) + goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint) if err != nil { return err } defer func() { - _ = mountsFile.Close() + if mountsFile != nil { + _ = mountsFile.Close() + } for _, f := range goferFiles { _ = f.Close() } @@ -472,11 +487,13 @@ func (c *Container) Start(conf *config.Config) error { } }() - cleanMounts, err := specutils.ReadMounts(mountsFile) - if err != nil { - return fmt.Errorf("reading mounts file: %v", err) + if mountsFile != nil { + cleanMounts, err := specutils.ReadMounts(mountsFile) + if err != nil { + return fmt.Errorf("reading mounts file: %v", err) + } + c.Spec.Mounts = cleanMounts } - c.Spec.Mounts = cleanMounts // Setup stdios if the container is not using terminal. Otherwise TTY was // already setup in create. @@ -912,13 +929,23 @@ func (c *Container) forEachSelfMount(fn func(mountSrc string)) { // createGoferFilestores creates the regular files that will back the // tmpfs/overlayfs mounts that will overlay some gofer mounts. It also returns // information about how each gofer mount is configured. -func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.GoferMountConf, error) { +func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints, rootfsHint *boot.RootfsHint) ([]*os.File, []boot.GoferMountConf, error) { var goferFilestores []*os.File var goferConfs []boot.GoferMountConf // Handle root mount first. - shouldOverlay := ovlConf.RootEnabled() && !c.Spec.Root.Readonly - filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Root.Path, shouldOverlay, nil /* hint */) + overlayMedium := ovlConf.RootOverlayMedium() + mountType := boot.Bind + if rootfsHint != nil { + overlayMedium = rootfsHint.Overlay + if !specutils.IsGoferMount(rootfsHint.Mount) { + mountType = rootfsHint.Mount.Type + } + } + if c.Spec.Root.Readonly { + overlayMedium = config.NoOverlay + } + filestore, goferConf, err := c.createGoferFilestore(overlayMedium, c.Spec.Root.Path, mountType, false /* isShared */) if err != nil { return nil, nil, err } @@ -932,9 +959,22 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b if !specutils.IsGoferMount(c.Spec.Mounts[i]) { continue } - hint := mountHints.FindMount(c.Spec.Mounts[i].Source) - shouldOverlay := ovlConf.SubMountEnabled() && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) - filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Mounts[i].Source, shouldOverlay, hint) + overlayMedium = ovlConf.SubMountOverlayMedium() + mountType = boot.Bind + isShared := false + if specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) { + overlayMedium = config.NoOverlay + } + if hint := mountHints.FindMount(c.Spec.Mounts[i].Source); hint != nil { + // Note that we want overlayMedium=self even if this is a read-only mount so that + // the shared mount is created correctly. Future containers may mount this writably. + overlayMedium = config.SelfOverlay + if !specutils.IsGoferMount(hint.Mount) { + mountType = hint.Mount.Type + } + isShared = hint.ShouldShareMount() + } + filestore, goferConf, err := c.createGoferFilestore(overlayMedium, c.Spec.Mounts[i].Source, mountType, isShared) if err != nil { return nil, nil, err } @@ -951,41 +991,45 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b return goferFilestores, goferConfs, nil } -func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.GoferMountConf, error) { - // MountHint information takes precedence over shouldOverlay. - if hint != nil && !specutils.IsGoferMount(hint.Mount) { - switch hint.Mount.Type { - case "tmpfs": - // Create self-backed tmpfs. - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfTmpfs) - default: - return nil, boot.VanillaGofer, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type) - } - } - switch { - case !shouldOverlay: - return nil, boot.VanillaGofer, nil - case ovlConf.IsBackedByMemory(): - return nil, boot.MemoryOverlay, nil - case ovlConf.IsBackedBySelf(): - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfOverlay) +func (c *Container) createGoferFilestore(overlayMedium config.OverlayMedium, mountSrc string, mountType string, isShared bool) (*os.File, boot.GoferMountConf, error) { + var lower boot.GoferMountConfLowerType + switch mountType { + case boot.Bind: + lower = boot.Lisafs + case tmpfs.Name: + lower = boot.NoneLower + case erofs.Name: + lower = boot.Erofs default: - return c.createGoferFilestoreInDir(ovlConf) + return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", mountType) + } + switch overlayMedium { + case config.NoOverlay: + return nil, boot.GoferMountConf{Lower: lower, Upper: boot.NoOverlay}, nil + case config.MemoryOverlay: + return nil, boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay}, nil + case config.SelfOverlay: + return c.createGoferFilestoreInSelf(mountSrc, isShared, boot.GoferMountConf{Lower: lower, Upper: boot.SelfOverlay}) + default: + if overlayMedium.IsBackedByAnon() { + return c.createGoferFilestoreInDir(overlayMedium.HostFileDir(), boot.GoferMountConf{Lower: lower, Upper: boot.AnonOverlay}) + } + return nil, boot.GoferMountConf{}, fmt.Errorf("unexpected overlay medium %q", overlayMedium) } } -func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.MountHint, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { +func (c *Container) createGoferFilestoreInSelf(mountSrc string, isShared bool, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { mountSrcInfo, err := os.Stat(mountSrc) if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) } if !mountSrcInfo.IsDir() { log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) - return nil, boot.MemoryOverlay, nil + return nil, boot.GoferMountConf{Lower: successConf.Lower, Upper: boot.MemoryOverlay}, nil } // Create the self filestore file. createFlags := unix.O_RDWR | unix.O_CREAT | unix.O_CLOEXEC - if !(hint != nil && hint.ShouldShareMount()) { + if !isShared { // Allow shared mounts to reuse existing filestore. A previous shared user // may have already set up the filestore. createFlags |= unix.O_EXCL @@ -998,9 +1042,9 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount // same sandbox, and is not shared, then the overlay option doesn't work // correctly. Because each overlay mount is independent and changes to // one are not visible to the other. - return nil, boot.VanillaGofer, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath) + return nil, boot.GoferMountConf{}, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath) } - return nil, boot.VanillaGofer, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) } log.Debugf("Created filestore file at %q for mount source %q", filestorePath, mountSrc) // Filestore in self should be a named path because it needs to be @@ -1011,14 +1055,13 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount return os.NewFile(uintptr(filestoreFD), filestorePath), successConf, nil } -func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File, boot.GoferMountConf, error) { - filestoreDir := ovlConf.HostFileDir() +func (c *Container) createGoferFilestoreInDir(filestoreDir string, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { fileInfo, err := os.Stat(filestoreDir) if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) } if !fileInfo.IsDir() { - return nil, boot.VanillaGofer, fmt.Errorf("overlay2 flag should specify an existing directory") + return nil, boot.GoferMountConf{}, fmt.Errorf("overlay2 flag should specify an existing directory") } // Create an unnamed temporary file in filestore directory which will be // deleted when the last FD on it is closed. We don't use O_TMPFILE because @@ -1027,13 +1070,13 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File // This file will be deleted when the container exits. filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-filestore-") if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) } if err := unix.Unlink(filestoreFile.Name()); err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) } log.Debugf("Created an unnamed filestore file at %q", filestoreDir) - return filestoreFile, boot.AnonOverlay, nil + return filestoreFile, successConf, nil } // saveLocked saves the container metadata to a file. @@ -1129,7 +1172,33 @@ func (c *Container) waitForStopped() error { return backoff.Retry(op, b) } -func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool) ([]*os.File, *os.File, error) { +// shouldSpawnGofer indicates whether the gofer process should be spawned. +func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool { + for _, cfg := range goferConfs { + if cfg.ShouldUseLisafs() { + return true + } + } + return false +} + +// createGoferProcess returns an IO file list and a mounts file on success. +// The IO file list consists of image files and/or socket files to connect to +// a gofer endpoint for the mount points using Gofers. The mounts file is the +// file to read list of mounts after they have been resolved (direct paths, +// no symlinks), and will be nil if there is no cleaning required for mounts. +func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) { + if !shouldSpawnGofer(c.GoferMountConfs) { + if !c.GoferMountConfs[0].ShouldUseErofs() { + panic("goferless mode is only possible with EROFS rootfs") + } + ioFile, err := os.Open(rootfsHint.Mount.Source) + if err != nil { + return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + } + return []*os.File{ioFile}, nil, nil + } + donations := donation.Agency{} defer donations.Close() @@ -1190,24 +1259,37 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu } donations.DonateAndClose("mounts-fd", mountsGofer) - // Count the number of mounts using lisafs. - lisafsCount := 0 + // Count the number of mounts that needs an IO file. + ioFileCount := 0 for _, cfg := range c.GoferMountConfs { - if cfg.ShouldUseLisafs() { - lisafsCount++ + if cfg.ShouldUseLisafs() || cfg.ShouldUseErofs() { + ioFileCount++ } } - sandEnds := make([]*os.File, 0, lisafsCount) - for i := 0; i < lisafsCount; i++ { - fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) - if err != nil { - return nil, nil, err - } - sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) + sandEnds := make([]*os.File, 0, ioFileCount) + for i, cfg := range c.GoferMountConfs { + switch { + case cfg.ShouldUseLisafs(): + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) - goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") - donations.DonateAndClose("io-fds", goferEnd) + goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") + donations.DonateAndClose("io-fds", goferEnd) + + case cfg.ShouldUseErofs(): + if i > 0 { + return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount") + } + if f, err := os.Open(rootfsHint.Mount.Source); err != nil { + return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + } else { + sandEnds = append(sandEnds, f) + } + } } if attached { diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 7b4bbae75..66dbd81fb 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -39,12 +39,14 @@ import ( "gvisor.dev/gvisor/pkg/bits" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/cgroup" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/flag" @@ -1015,167 +1017,174 @@ func TestKillPid(t *testing.T) { } } -// TestCheckpointRestore creates a container that continuously writes successive +// testCheckpointRestore creates a container that continuously writes successive // integers to a file. To test checkpoint and restore functionality, the // container is checkpointed and the last number printed to the file is // recorded. Then, it is restored in two new containers and the first number // printed from these containers is checked. Both should be the next consecutive // number after the last number from the checkpointed container. +func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript func(string) *specs.Spec) { + dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("ioutil.TempDir failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + outputPath := filepath.Join(dir, "output") + outputFile, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile.Close() + + script := fmt.Sprintf("i=0; while true; do echo $i >> %q; sleep 1; i=$((i+1)); done", outputPath) + spec := newSpecWithScript(script) + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Create and start the container. + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(conf); err != nil { + t.Fatalf("error starting container: %v", err) + } + + // Set the image path, which is where the checkpoint image will be saved. + imagePath := filepath.Join(dir, "test-image-file") + + // Create the image file and open for writing. + file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + t.Fatalf("error opening new file at imagePath: %v", err) + } + defer file.Close() + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + // Checkpoint running container; save state into new file. + if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + t.Fatalf("error checkpointing container to empty file: %v", err) + } + defer os.RemoveAll(imagePath) + + lastNum, err := readOutputNum(outputPath, -1) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile2, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile2.Close() + + // Restore into a new container with different ID (e.g. clone). Keep the + // initial container running to ensure no conflict with it. + args2 := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont2, err := New(conf, args2) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont2.Destroy() + + if err := cont2.Restore(conf, imagePath); err != nil { + t.Fatalf("error restoring container: %v", err) + } + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile2); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) + } + cont2.Destroy() + cont2 = nil + + // Restore into a container using the same ID (e.g. save/resume). It requires + // the original container to cease to exist because they share the same identity. + cont.Destroy() + cont = nil + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile3, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile3.Close() + + cont3, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont3.Destroy() + + if err := cont3.Restore(conf, imagePath); err != nil { + t.Fatalf("error restoring container: %v", err) + } + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile3); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum2, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum2 { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) + } + cont3.Destroy() +} + +// TestCheckpointRestore does the checkpoint/restore test on each platform. func TestCheckpointRestore(t *testing.T) { // Skip overlay because test requires writing to host file. for name, conf := range configs(t, true /* noOverlay */) { t.Run(name, func(t *testing.T) { - dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test") - if err != nil { - t.Fatalf("ioutil.TempDir failed: %v", err) - } - defer os.RemoveAll(dir) - if err := os.Chmod(dir, 0777); err != nil { - t.Fatalf("error chmoding file: %q, %v", dir, err) - } - - outputPath := filepath.Join(dir, "output") - outputFile, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile.Close() - - script := fmt.Sprintf("for ((i=0; ;i++)); do echo $i >> %q; sleep 1; done", outputPath) - spec := testutil.NewSpecWithArgs("bash", "-c", script) - _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) - if err != nil { - t.Fatalf("error setting up container: %v", err) - } - defer cleanup() - - // Create and start the container. - args := Args{ - ID: testutil.RandomContainerID(), - Spec: spec, - BundleDir: bundleDir, - } - cont, err := New(conf, args) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont.Destroy() - if err := cont.Start(conf); err != nil { - t.Fatalf("error starting container: %v", err) - } - - // Set the image path, which is where the checkpoint image will be saved. - imagePath := filepath.Join(dir, "test-image-file") - - // Create the image file and open for writing. - file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - t.Fatalf("error opening new file at imagePath: %v", err) - } - defer file.Close() - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { - t.Fatalf("error checkpointing container to empty file: %v", err) - } - defer os.RemoveAll(imagePath) - - lastNum, err := readOutputNum(outputPath, -1) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Delete and recreate file before restoring. - if err := os.Remove(outputPath); err != nil { - t.Fatalf("error removing file") - } - outputFile2, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile2.Close() - - // Restore into a new container with different ID (e.g. clone). Keep the - // initial container running to ensure no conflict with it. - args2 := Args{ - ID: testutil.RandomContainerID(), - Spec: spec, - BundleDir: bundleDir, - } - cont2, err := New(conf, args2) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont2.Destroy() - - if err := cont2.Restore(conf, imagePath); err != nil { - t.Fatalf("error restoring container: %v", err) - } - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile2); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - firstNum, err := readOutputNum(outputPath, 0) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Check that lastNum is one less than firstNum and that the container - // picks up from where it left off. - if lastNum+1 != firstNum { - t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) - } - cont2.Destroy() - cont2 = nil - - // Restore into a container using the same ID (e.g. save/resume). It requires - // the original container to cease to exist because they share the same identity. - cont.Destroy() - cont = nil - - // Delete and recreate file before restoring. - if err := os.Remove(outputPath); err != nil { - t.Fatalf("error removing file") - } - outputFile3, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile3.Close() - - cont3, err := New(conf, args) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont3.Destroy() - - if err := cont3.Restore(conf, imagePath); err != nil { - t.Fatalf("error restoring container: %v", err) - } - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile3); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - firstNum2, err := readOutputNum(outputPath, 0) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Check that lastNum is one less than firstNum and that the container - // picks up from where it left off. - if lastNum+1 != firstNum2 { - t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) - } - cont3.Destroy() + testCheckpointRestore(t, conf, func(script string) *specs.Spec { + return testutil.NewSpecWithArgs("bash", "-c", script) + }) }) } } @@ -3180,3 +3189,163 @@ find $dir -type l -o -type f | sort | xargs cat | md5sum`), 0755); err != nil { } } } + +// createRootfsEROFS creates a rootfs directory and an EROFS rootfs image in +// the directory dir. +func createRootfsEROFS(dir string) (string, string, error) { + mkfs, err := exec.LookPath("mkfs.erofs") + if err != nil { + return "", "", fmt.Errorf("mkfs.erofs is not available: %v", err) + } + + busybox, err := exec.LookPath("busybox") + if err != nil { + return "", "", fmt.Errorf("busybox is not available: %v", err) + } + + // Create a rootfs directory with busybox in root. + rootfsDir := filepath.Join(dir, "rootfs") + if err := os.Mkdir(rootfsDir, 0755); err != nil { + return "", "", fmt.Errorf("os.Mkdir() failed: %v", err) + } + if err := testutil.Copy(busybox, filepath.Join(rootfsDir, "busybox")); err != nil { + return "", "", fmt.Errorf("failed to copy busybox: %v", err) + } + + // Handcraft the following mount points that the sentry mounts need, because EROFS + // does not support creating synthetic directories yet and we may not want to use + // overlay in some tests. + for _, dir := range []string{"dev", "proc", "sys", "tmp"} { + if err := os.Mkdir(filepath.Join(rootfsDir, dir), 0755); err != nil { + return "", "", fmt.Errorf("os.Mkdir() failed: %v", err) + } + } + + // Build the EROFS rootfs image. + rootfsImage := filepath.Join(dir, "rootfs.img") + cmd := fmt.Sprintf("%s -E noinline_data %s %s", mkfs, rootfsImage, rootfsDir) + if out, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { + return "", "", fmt.Errorf("exec: sh -c %q, err: %v, out: %s", cmd, err, out) + } + + return rootfsDir, rootfsImage, nil +} + +// TestRootfsEROFS starts a container using an EROFS image as the rootfs and checks that +// the rootfs in container is an EROFS. +func TestRootfsEROFS(t *testing.T) { + // Skip this test if mkfs.erofs or busybox are not available. + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skipf("mkfs.erofs is not available: %v", err) + } + if _, err := exec.LookPath("busybox"); err != nil { + t.Skipf("busybox is not available: %v", err) + } + + testDir, err := ioutil.TempDir(testutil.TmpDir(), "erofs_rootfs_test_") + if err != nil { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + rootfsDir, rootfsImage, err := createRootfsEROFS(testDir) + if err != nil { + t.Fatalf("failed to create EROFS rootfs image: %v", err) + } + + // Create the spec and set the EROFS rootfs annotations. + spec := testutil.NewSpecWithArgs("/busybox", "grep", "/ / ro - erofs", "/proc/self/mountinfo") + spec.Root.Path = rootfsDir + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.RootfsPrefix+"type"] = erofs.Name + spec.Annotations[boot.RootfsPrefix+"source"] = rootfsImage + // Disable the overlay, as we want to be sure that rootfs will always be + // shown as EROFS in mountinfo. + spec.Annotations[boot.RootfsPrefix+"overlay"] = config.NoOverlay.String() + + conf := testutil.TestConfig(t) + + for _, mounts := range [][]specs.Mount{ + // Case 1: EROFS rootfs without any other gofer mount. + nil, + + // Case 2: EROFS rootfs with a LISAFS backed gofer mount. + []specs.Mount{ + { + Type: "bind", + Destination: "/tmp", + Source: "/tmp", + }, + }, + } { + spec.Mounts = mounts + + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Create and start the container. + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + Attached: true, + } + ws, err := Run(conf, args) + if err != nil { + t.Fatalf("error running container: %v", err) + } + if ws.ExitStatus() != 0 { + t.Errorf("got exit status %v want %v", ws.ExitStatus(), 0) + } + } +} + +// TestCheckpointRestoreEROFS does the checkpoint/restore test on each platform using +// an EROFS image as the rootfs. +func TestCheckpointRestoreEROFS(t *testing.T) { + // Skip this test if mkfs.erofs or busybox are not available. + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skipf("mkfs.erofs is not available: %v", err) + } + if _, err := exec.LookPath("busybox"); err != nil { + t.Skipf("busybox is not available: %v", err) + } + + testDir, err := ioutil.TempDir(testutil.TmpDir(), "erofs_checkpoint_restore_test_") + if err != nil { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + rootfsDir, rootfsImage, err := createRootfsEROFS(testDir) + if err != nil { + t.Fatalf("failed to create EROFS rootfs image: %v", err) + } + + // Skip overlay because test requires writing to host file. + for name, conf := range configs(t, true /* noOverlay */) { + t.Run(name, func(t *testing.T) { + testCheckpointRestore(t, conf, func(script string) *specs.Spec { + spec := testutil.NewSpecWithArgs("/busybox", "sh", "-c", script) + spec.Root = &specs.Root{ + Path: rootfsDir, + Readonly: false, + } + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.RootfsPrefix+"type"] = erofs.Name + spec.Annotations[boot.RootfsPrefix+"source"] = rootfsImage + // EROFS does not support creating synthetic directories yet, so let's add + // a writeable and savable overlay for rootfs, which allows the sentry to + // create the mount point for the bind mount of the temporary directory shared + // between host and test container. + spec.Annotations[boot.RootfsPrefix+"overlay"] = config.MemoryOverlay.String() + return spec + }) + }) + } +} diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 8b4071a79..44fef8cad 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -223,9 +223,9 @@ type Args struct { // UserLog is the filename to send user-visible logs to. It may be empty. UserLog string - // IOFiles is the list of files that connect to a gofer endpoint for the - // mounts points using Gofers. They must be in the same order as mounts - // appear in the spec. + // IOFiles is the list of image files and/or socket files that connect to + // a gofer endpoint for the mount points using Gofers. They must be in the + // same order as mounts appear in the spec. IOFiles []*os.File // GoferFilestoreFiles are the regular files that will back the overlayfs or