From da2b10e207978b7a64a8698f8654fdac21371460 Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Wed, 4 Oct 2023 21:40:52 +0800 Subject: [PATCH 1/3] runsc: decouple GoferMountConf to two layers This patch decouples GoferMountConf to two layers to allow us to configure all combinations of a gofer mount in a succinct way: - Upper layer config: none, memory, self, anon. The upper layer is always tmpfs. It describes the backend for tmpfs. - Lower layer config: none, lisafs. It describes the backend for the filesystem which actually holds the image contents. The old SelfTmpfs will be represented as "upper=self,lower=none", MemoryOverlay will be "upper=memory,lower=lisafs", SelfOverlay will be "upper=self,lower=lisafs", and so on. Thanks to @ayushr2 for the suggestion on how to better decouple this. This is a preparation for adding the EROFS rootfs support. There is no functional change intended. Signed-off-by: Tiwei Bie --- runsc/boot/gofer_conf.go | 162 ++++++++++++++++++++++++++++------ runsc/boot/gofer_conf_test.go | 104 ++++++++++++++++------ runsc/boot/loader_test.go | 2 +- runsc/boot/vfs.go | 64 ++++++++------ runsc/cli/cli_test.go | 1 + runsc/container/container.go | 28 +++--- 6 files changed, 265 insertions(+), 96 deletions(-) diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go index 49177accb..f3487346f 100644 --- a/runsc/boot/gofer_conf.go +++ b/runsc/boot/gofer_conf.go @@ -16,52 +16,161 @@ package boot import ( "fmt" - "strconv" "strings" ) -// GoferMountConf describes how a gofer mount is configured in the sandbox. -type GoferMountConf int +// GoferMountConfUpperType describes how upper layer is configured for the gofer mount. +type GoferMountConfUpperType byte const ( - // VanillaGofer indicates that this gofer mount has no special configuration. - VanillaGofer GoferMountConf = iota + // NoOverlay indicates that this gofer mount has no upper layer. In this case, + // this gofer mount must have a lower layer (i.e. lower != NoneLower). + NoOverlay GoferMountConfUpperType = iota - // MemoryOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by application memory. + // MemoryOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by application memory. MemoryOverlay - // SelfOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by a host file in the mount's source directory. + // SelfOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by a host file in the mount's source directory. SelfOverlay - // AnonOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by a host file in an anonymous directory. + // AnonOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by a host file in an anonymous directory. AnonOverlay - // SelfTmpfs indicates that this gofer mount should be overlaid with a tmpfs - // mount backed by a host file in the mount's source directory. - SelfTmpfs + // UpperMax indicates the number of the valid upper layer types. + UpperMax ) +// String returns a human-readable string representing the upper layer type. +func (u GoferMountConfUpperType) String() string { + switch u { + case NoOverlay: + return "none" + case MemoryOverlay: + return "memory" + case SelfOverlay: + return "self" + case AnonOverlay: + return "anon" + } + panic(fmt.Sprintf("Invalid gofer mount config upper layer type: %d", u)) +} + +// Set sets the value. Set(String()) should be idempotent. +func (u *GoferMountConfUpperType) Set(v string) error { + switch v { + case "none": + *u = NoOverlay + case "memory": + *u = MemoryOverlay + case "self": + *u = SelfOverlay + case "anon": + *u = AnonOverlay + default: + return fmt.Errorf("invalid gofer mount config upper layer type: %s", v) + } + return nil +} + +// GoferMountConfLowerType describes how lower layer is configured for the gofer mount. +type GoferMountConfLowerType byte + +const ( + // NoneLower indicates that this gofer mount has no lower layer. + NoneLower GoferMountConfLowerType = iota + + // Lisafs indicates that this gofer mount has a LISAFS lower layer. + Lisafs + + // LowerMax indicates the number of the valid lower layer types. + LowerMax +) + +// String returns a human-readable string representing the lower layer type. +func (l GoferMountConfLowerType) String() string { + switch l { + case NoneLower: + return "none" + case Lisafs: + return "lisafs" + } + panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l)) +} + +// Set sets the value. Set(String()) should be idempotent. +func (l *GoferMountConfLowerType) Set(v string) error { + switch v { + case "none": + *l = NoneLower + case "lisafs": + *l = Lisafs + default: + return fmt.Errorf("invalid gofer mount config lower layer type: %s", v) + } + return nil +} + +// GoferMountConf describes how a gofer mount is configured in the sandbox. +type GoferMountConf struct { + Upper GoferMountConfUpperType `json:"upper"` + Lower GoferMountConfLowerType `json:"lower"` +} + +// String returns a human-readable string representing the gofer mount config. +func (g GoferMountConf) String() string { + return fmt.Sprintf("%s:%s", g.Lower, g.Upper) +} + +// Set sets the value. Set(String()) should be idempotent. +func (g *GoferMountConf) Set(v string) error { + parts := strings.Split(v, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid gofer mount config format: %q", v) + } + if err := g.Lower.Set(parts[0]); err != nil { + return err + } + if err := g.Upper.Set(parts[1]); err != nil { + return err + } + if !g.valid() { + return fmt.Errorf("invalid gofer mount config: %+v", g) + } + return nil +} + // IsFilestorePresent returns true if a filestore file was associated with this. func (g GoferMountConf) IsFilestorePresent() bool { - return g == SelfOverlay || g == AnonOverlay || g == SelfTmpfs + return g.Upper == SelfOverlay || g.Upper == AnonOverlay } // IsSelfBacked returns true if this mount is backed by a filestore in itself. func (g GoferMountConf) IsSelfBacked() bool { - return g == SelfOverlay || g == SelfTmpfs + return g.Upper == SelfOverlay } // ShouldUseOverlayfs returns true if an overlayfs should be applied. func (g GoferMountConf) ShouldUseOverlayfs() bool { - return g == MemoryOverlay || g == SelfOverlay || g == AnonOverlay + return g.Lower != NoneLower && g.Upper != NoOverlay +} + +// ShouldUseTmpfs returns true if a tmpfs should be applied. +func (g GoferMountConf) ShouldUseTmpfs() bool { + // g.valid() implies that g.Upper != NoOverlay. + return g.Lower == NoneLower } // ShouldUseLisafs returns true if a lisafs client/server should be set up. func (g GoferMountConf) ShouldUseLisafs() bool { - return g == VanillaGofer || g.ShouldUseOverlayfs() + return g.Lower == Lisafs +} + +// valid returns true if this is a valid gofer mount config. +func (g GoferMountConf) valid() bool { + return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay) } // GoferMountConfFlags can be used with GoferMountConf flags that appear @@ -70,11 +179,11 @@ type GoferMountConfFlags []GoferMountConf // String implements flag.Value. func (g *GoferMountConfFlags) String() string { - confVals := make([]string, 0, len(*g)) + confs := make([]string, 0, len(*g)) for _, confVal := range *g { - confVals = append(confVals, strconv.Itoa(int(confVal))) + confs = append(confs, confVal.String()) } - return strings.Join(confVals, ",") + return strings.Join(confs, ",") } // Get implements flag.Value. @@ -92,14 +201,11 @@ func (g *GoferMountConfFlags) GetArray() []GoferMountConf { func (g *GoferMountConfFlags) Set(s string) error { confs := strings.Split(s, ",") for _, conf := range confs { - confVal, err := strconv.Atoi(conf) - if err != nil { - return fmt.Errorf("invalid GoferMountConf value (%d): %v", confVal, err) + var confVal GoferMountConf + if err := confVal.Set(conf); err != nil { + return fmt.Errorf("invalid GoferMountConf value (%s): %v", conf, err) } - if confVal > int(SelfTmpfs) { - return fmt.Errorf("invalid GoferMountConf value (%d)", confVal) - } - *g = append(*g, GoferMountConf(confVal)) + *g = append(*g, confVal) } return nil } diff --git a/runsc/boot/gofer_conf_test.go b/runsc/boot/gofer_conf_test.go index f4e00b717..1bc454bc1 100644 --- a/runsc/boot/gofer_conf_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -20,51 +20,103 @@ import ( func TestGoferConf(t *testing.T) { tcs := []struct { - ovl GoferMountConf + cfg GoferMountConf wantOverlay bool wantHostFile bool wantLisafs bool + wantTmpfs bool + wantValid bool }{{ - ovl: VanillaGofer, + cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay}, + // This is not a valid config. + wantValid: false, + }, { + cfg: GoferMountConf{Lower: NoneLower, Upper: MemoryOverlay}, wantOverlay: false, wantHostFile: false, - wantLisafs: true, + wantLisafs: false, + wantTmpfs: true, + wantValid: true, }, { - ovl: MemoryOverlay, - wantOverlay: true, - wantHostFile: false, - wantLisafs: true, - }, { - ovl: SelfOverlay, - wantOverlay: true, - wantHostFile: true, - wantLisafs: true, - }, { - ovl: AnonOverlay, - wantOverlay: true, - wantHostFile: true, - wantLisafs: true, - }, { - ovl: SelfTmpfs, + cfg: GoferMountConf{Lower: NoneLower, Upper: SelfOverlay}, wantOverlay: false, wantHostFile: true, wantLisafs: false, + wantTmpfs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: NoneLower, Upper: AnonOverlay}, + wantOverlay: false, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: NoOverlay}, + wantOverlay: false, + wantHostFile: false, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay}, + wantOverlay: true, + wantHostFile: false, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: SelfOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: AnonOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax}, + // This is not a valid config. + wantValid: false, }} for _, tc := range tcs { - if got := tc.ovl.ShouldUseOverlayfs(); got != tc.wantOverlay { - t.Errorf("gofer conf = %d, ShouldUseOverlayfs() = %t, want = %t", tc.ovl, got, tc.wantOverlay) + if got := tc.cfg.valid(); got != tc.wantValid { + t.Errorf("gofer conf = %+v, valid() = %t, want = %t", tc.cfg, got, tc.wantValid) } - if got := tc.ovl.IsFilestorePresent(); got != tc.wantHostFile { - t.Errorf("gofer conf = %d, IsFilestorePresent() = %t, want = %t", tc.ovl, got, tc.wantHostFile) + if !tc.wantValid { + // Skip the following tests, if this is not a valid config. + continue } - if got := tc.ovl.ShouldUseLisafs(); got != tc.wantLisafs { - t.Errorf("gofer conf = %d, ShouldUseLisafs() = %t, want = %t", tc.ovl, got, tc.wantLisafs) + if got := tc.cfg.ShouldUseOverlayfs(); got != tc.wantOverlay { + t.Errorf("gofer conf = %+v, ShouldUseOverlayfs() = %t, want = %t", tc.cfg, got, tc.wantOverlay) + } + if got := tc.cfg.IsFilestorePresent(); got != tc.wantHostFile { + t.Errorf("gofer conf = %+v, IsFilestorePresent() = %t, want = %t", tc.cfg, got, tc.wantHostFile) + } + if got := tc.cfg.ShouldUseLisafs(); got != tc.wantLisafs { + t.Errorf("gofer conf = %+v, ShouldUseLisafs() = %t, want = %t", tc.cfg, got, tc.wantLisafs) + } + if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs { + t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs) } } } func TestGoferConfFlags(t *testing.T) { - want := GoferMountConfFlags{VanillaGofer, MemoryOverlay, SelfOverlay, AnonOverlay, SelfTmpfs} + want := GoferMountConfFlags{ + {Lower: NoneLower, Upper: MemoryOverlay}, + {Lower: NoneLower, Upper: SelfOverlay}, + {Lower: NoneLower, Upper: AnonOverlay}, + {Lower: Lisafs, Upper: NoOverlay}, + {Lower: Lisafs, Upper: MemoryOverlay}, + {Lower: Lisafs, Upper: SelfOverlay}, + {Lower: Lisafs, Upper: AnonOverlay}, + } var got GoferMountConfFlags got.Set(want.String()) if len(got) != len(want) { diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index 4abb336ce..552e13654 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -140,7 +140,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error ControllerFD: fd, GoferFDs: []int{sandEnd}, StdioFDs: stdio, - GoferMountConfs: []GoferMountConf{VanillaGofer}, + GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}}, PodInitConfigFD: -1, ExecFD: -1, } diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index aca0ed12d..6094e0d46 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -447,33 +447,43 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre // createMountNamespace creates the container's root mount and namespace. func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) { ioFD := c.goferFDs.remove() - data := goferMountData(ioFD, conf.FileAccess, conf) - - // We can't check for overlayfs here because sandbox is chroot'ed and gofer - // can only send mount options for specs.Mounts (specs.Root is missing - // Options field). So assume root is always on top of overlayfs. - data = append(data, "overlayfs_stale_read") - - // Configure the gofer dentry cache size. - gofer.SetDentryCacheSize(conf.DCache) - - log.Infof("Mounting root with gofer, ioFD: %d", ioFD) - opts := &vfs.MountOptions{ - ReadOnly: c.root.Readonly, - GetFilesystemOptions: vfs.GetFilesystemOptions{ - InternalMount: true, - Data: strings.Join(data, ","), - InternalData: gofer.InternalFilesystemOptions{ - UniqueID: "/", - }, - }, - } - - fsName := gofer.Name rootfsConf := c.goferMountConfs[0] - if rootfsConf == SelfTmpfs { - panic("SelfTmpfs is not possible for rootfs") + + var ( + fsName string + opts *vfs.MountOptions + ) + switch { + case rootfsConf.ShouldUseLisafs(): + fsName = gofer.Name + + data := goferMountData(ioFD, conf.FileAccess, conf) + + // We can't check for overlayfs here because sandbox is chroot'ed and gofer + // can only send mount options for specs.Mounts (specs.Root is missing + // Options field). So assume root is always on top of overlayfs. + data = append(data, "overlayfs_stale_read") + + // Configure the gofer dentry cache size. + gofer.SetDentryCacheSize(conf.DCache) + + opts = &vfs.MountOptions{ + ReadOnly: c.root.Readonly, + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalMount: true, + Data: strings.Join(data, ","), + InternalData: gofer.InternalFilesystemOptions{ + UniqueID: "/", + }, + }, + } + + default: + return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf) } + + log.Infof("Mounting root with %s, ioFD: %d", fsName, ioFD) + if rootfsConf.ShouldUseOverlayfs() { log.Infof("Adding overlay on top of root") var ( @@ -608,7 +618,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co } // We need to hide the filestore from the containerized application. - if mountConf == SelfOverlay { + if mountConf.IsSelfBacked() { if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{ Root: upperRootVD, Start: upperRootVD, @@ -721,7 +731,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) { if info.goferMountConf.IsFilestorePresent() { info.filestoreFD = c.goferFilestoreFDs.removeAsFD() } - if info.goferMountConf == SelfTmpfs { + if info.goferMountConf.ShouldUseTmpfs() { specutils.ChangeMountType(info.mount, tmpfs.Name) } goferMntIdx++ diff --git a/runsc/cli/cli_test.go b/runsc/cli/cli_test.go index 1b1a74641..b45dde51e 100644 --- a/runsc/cli/cli_test.go +++ b/runsc/cli/cli_test.go @@ -31,6 +31,7 @@ var fakeFlagValues = [...]string{ "2h45m", "1:1,2:2", "0 0 1,100000 100000 65536", + "lisafs:self,lisafs:none", } func dupFlag(t *testing.T, cmd subcommands.Command, flagName string) *flag.Flag { diff --git a/runsc/container/container.go b/runsc/container/container.go index 4025ef9d0..71f6cbdcc 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -957,18 +957,18 @@ func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc strin switch hint.Mount.Type { case "tmpfs": // Create self-backed tmpfs. - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfTmpfs) + return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.NoneLower, Upper: boot.SelfOverlay}) default: - return nil, boot.VanillaGofer, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type) + return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type) } } switch { case !shouldOverlay: - return nil, boot.VanillaGofer, nil + return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.NoOverlay}, nil case ovlConf.IsBackedByMemory(): - return nil, boot.MemoryOverlay, nil + return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil case ovlConf.IsBackedBySelf(): - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfOverlay) + return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.SelfOverlay}) default: return c.createGoferFilestoreInDir(ovlConf) } @@ -977,11 +977,11 @@ func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc strin func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.MountHint, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { mountSrcInfo, err := os.Stat(mountSrc) if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) } if !mountSrcInfo.IsDir() { log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) - return nil, boot.MemoryOverlay, nil + return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil } // Create the self filestore file. createFlags := unix.O_RDWR | unix.O_CREAT | unix.O_CLOEXEC @@ -998,9 +998,9 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount // same sandbox, and is not shared, then the overlay option doesn't work // correctly. Because each overlay mount is independent and changes to // one are not visible to the other. - return nil, boot.VanillaGofer, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath) + return nil, boot.GoferMountConf{}, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath) } - return nil, boot.VanillaGofer, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) } log.Debugf("Created filestore file at %q for mount source %q", filestorePath, mountSrc) // Filestore in self should be a named path because it needs to be @@ -1015,10 +1015,10 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File filestoreDir := ovlConf.HostFileDir() fileInfo, err := os.Stat(filestoreDir) if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) } if !fileInfo.IsDir() { - return nil, boot.VanillaGofer, fmt.Errorf("overlay2 flag should specify an existing directory") + return nil, boot.GoferMountConf{}, fmt.Errorf("overlay2 flag should specify an existing directory") } // Create an unnamed temporary file in filestore directory which will be // deleted when the last FD on it is closed. We don't use O_TMPFILE because @@ -1027,13 +1027,13 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File // This file will be deleted when the container exits. filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-filestore-") if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) } if err := unix.Unlink(filestoreFile.Name()); err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) } log.Debugf("Created an unnamed filestore file at %q", filestoreDir) - return filestoreFile, boot.AnonOverlay, nil + return filestoreFile, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.AnonOverlay}, nil } // saveLocked saves the container metadata to a file. From 45220a5188eb1505d203db82751b88494220e5e6 Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Thu, 2 Nov 2023 22:11:37 +0800 Subject: [PATCH 2/3] runsc: add the new OverlayMedium type for overlay config This patch refactors the overlay medium from string to a new type OverlayMedium. Now all overlay medium related methods, such as validation and extracting anon directory, are implemented on this type. This is a preparation for adding the EROFS rootfs support. There is no functional change intended. Co-authored-by: Ayush Ranjan Signed-off-by: Tiwei Bie --- runsc/config/config.go | 124 +++++++++++++++++++++-------------- runsc/config/config_test.go | 2 +- runsc/container/BUILD | 1 + runsc/container/container.go | 79 +++++++++++++--------- 4 files changed, 125 insertions(+), 81 deletions(-) diff --git a/runsc/config/config.go b/runsc/config/config.go index 7f97ef15c..37e51a09c 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -737,17 +737,73 @@ func (g HostFifo) AllowOpen() bool { return g&HostFifoOpen != 0 } +// OverlayMedium describes how overlay medium is configured. +type OverlayMedium string + +const ( + // NoOverlay indicates that no overlay will be applied. + NoOverlay = OverlayMedium("") + + // MemoryOverlay indicates that the overlay is backed by app memory. + MemoryOverlay = OverlayMedium("memory") + + // SelfOverlay indicates that the overlaid mount is backed by itself. + SelfOverlay = OverlayMedium("self") + + // AnonOverlayPrefix is the prefix that users should specify in the + // config for the anonymous overlay. + AnonOverlayPrefix = "dir=" +) + +// String returns a human-readable string representing the overlay medium config. +func (m OverlayMedium) String() string { + return string(m) +} + +// Set sets the value. Set(String()) should be idempotent. +func (m *OverlayMedium) Set(v string) error { + switch OverlayMedium(v) { + case NoOverlay, MemoryOverlay, SelfOverlay: // OK + default: + if !strings.HasPrefix(v, AnonOverlayPrefix) { + return fmt.Errorf("unexpected medium: %q", v) + } + if hostFileDir := strings.TrimPrefix(v, AnonOverlayPrefix); !filepath.IsAbs(hostFileDir) { + return fmt.Errorf("overlay host file directory should be an absolute path, got %q", hostFileDir) + } + } + *m = OverlayMedium(v) + return nil +} + +// IsBackedByAnon indicates whether the overlaid mount is backed by a host file +// in an anonymous directory. +func (m OverlayMedium) IsBackedByAnon() bool { + return strings.HasPrefix(string(m), AnonOverlayPrefix) +} + +// HostFileDir indicates the directory in which the overlay-backing host file +// should be created. +// +// Precondition: m.IsBackedByAnon(). +func (m OverlayMedium) HostFileDir() string { + if !m.IsBackedByAnon() { + panic(fmt.Sprintf("anonymous overlay medium = %q does not have %v prefix", m, AnonOverlayPrefix)) + } + return strings.TrimPrefix(string(m), AnonOverlayPrefix) +} + // Overlay2 holds the configuration for setting up overlay filesystems for the // container. type Overlay2 struct { rootMount bool subMounts bool - medium string + medium OverlayMedium } func defaultOverlay2() *Overlay2 { // Rootfs overlay is enabled by default and backed by a file in rootfs itself. - return &Overlay2{rootMount: true, subMounts: false, medium: "self"} + return &Overlay2{rootMount: true, subMounts: false, medium: SelfOverlay} } // Set implements flag.Value. Set(String()) should be idempotent. @@ -755,7 +811,7 @@ func (o *Overlay2) Set(v string) error { if v == "none" { o.rootMount = false o.subMounts = false - o.medium = "" + o.medium = NoOverlay return nil } vs := strings.Split(v, ":") @@ -773,18 +829,7 @@ func (o *Overlay2) Set(v string) error { return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount) } - o.medium = vs[1] - switch o.medium { - case "memory", "self": // OK - default: - if !strings.HasPrefix(o.medium, "dir=") { - return fmt.Errorf("unexpected medium specifier for --overlay2: %q", o.medium) - } - if hostFileDir := strings.TrimPrefix(o.medium, "dir="); !filepath.IsAbs(hostFileDir) { - return fmt.Errorf("overlay host file directory should be an absolute path, got %q", hostFileDir) - } - } - return nil + return o.medium.Set(vs[1]) } // Get implements flag.Value. @@ -806,47 +851,26 @@ func (o Overlay2) String() string { default: panic("invalid state of subMounts = true and rootMount = false") } - - return res + ":" + o.medium + return res + ":" + o.medium.String() } // Enabled returns true if the overlay option is enabled for any mounts. func (o *Overlay2) Enabled() bool { - return o.rootMount || o.subMounts + return o.medium != NoOverlay } -// RootEnabled returns true if the overlay is enabled for the root mount. -func (o *Overlay2) RootEnabled() bool { - return o.rootMount -} - -// SubMountEnabled returns true if the overlay is enabled for submounts. -func (o *Overlay2) SubMountEnabled() bool { - return o.subMounts -} - -// IsBackedByMemory indicates whether the overlay is backed by app memory. -func (o *Overlay2) IsBackedByMemory() bool { - return o.Enabled() && o.medium == "memory" -} - -// IsBackedBySelf indicates whether the overlaid mounts are backed by -// themselves. -func (o *Overlay2) IsBackedBySelf() bool { - return o.Enabled() && o.medium == "self" -} - -// HostFileDir indicates the directory in which the overlay-backing host file -// should be created. -// -// Precondition: o.IsBackedByHostFile() && !o.IsBackedBySelf(). -func (o *Overlay2) HostFileDir() string { - if !strings.HasPrefix(o.medium, "dir=") { - panic(fmt.Sprintf("Overlay2.Medium = %q does not have dir= prefix when overlay is backed by a host file", o.medium)) +// RootOverlayMedium returns the overlay medium config of the root mount. +func (o *Overlay2) RootOverlayMedium() OverlayMedium { + if !o.rootMount { + return NoOverlay } - hostFileDir := strings.TrimPrefix(o.medium, "dir=") - if !filepath.IsAbs(hostFileDir) { - panic(fmt.Sprintf("overlay host file directory should be an absolute path, got %q", hostFileDir)) + return o.medium +} + +// SubMountOverlayMedium returns the overlay medium config of submounts. +func (o *Overlay2) SubMountOverlayMedium() OverlayMedium { + if !o.subMounts { + return NoOverlay } - return hostFileDir + return o.medium } diff --git a/runsc/config/config_test.go b/runsc/config/config_test.go index 5261fb154..d89b4dbbe 100644 --- a/runsc/config/config_test.go +++ b/runsc/config/config_test.go @@ -224,7 +224,7 @@ func TestInvalidFlags(t *testing.T) { { name: "overlay2", value: "root:/tmp", - error: "unexpected medium specifier for --overlay2: \"/tmp\"", + error: "unexpected medium: \"/tmp\"", }, { name: "overlay2", diff --git a/runsc/container/BUILD b/runsc/container/BUILD index eb93d976a..0d2bf7969 100644 --- a/runsc/container/BUILD +++ b/runsc/container/BUILD @@ -22,6 +22,7 @@ go_library( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/pgalloc", "//pkg/sighandling", "//pkg/state/statefile", diff --git a/runsc/container/container.go b/runsc/container/container.go index 71f6cbdcc..6c7f6610c 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -37,6 +37,7 @@ import ( "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sighandling" "gvisor.dev/gvisor/pkg/state/statefile" @@ -917,8 +918,12 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b var goferConfs []boot.GoferMountConf // Handle root mount first. - shouldOverlay := ovlConf.RootEnabled() && !c.Spec.Root.Readonly - filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Root.Path, shouldOverlay, nil /* hint */) + overlayMedium := ovlConf.RootOverlayMedium() + mountType := boot.Bind + if c.Spec.Root.Readonly { + overlayMedium = config.NoOverlay + } + filestore, goferConf, err := c.createGoferFilestore(overlayMedium, c.Spec.Root.Path, mountType, false /* isShared */) if err != nil { return nil, nil, err } @@ -932,9 +937,22 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b if !specutils.IsGoferMount(c.Spec.Mounts[i]) { continue } - hint := mountHints.FindMount(c.Spec.Mounts[i].Source) - shouldOverlay := ovlConf.SubMountEnabled() && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) - filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Mounts[i].Source, shouldOverlay, hint) + overlayMedium = ovlConf.SubMountOverlayMedium() + mountType = boot.Bind + isShared := false + if specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) { + overlayMedium = config.NoOverlay + } + if hint := mountHints.FindMount(c.Spec.Mounts[i].Source); hint != nil { + // Note that we want overlayMedium=self even if this is a read-only mount so that + // the shared mount is created correctly. Future containers may mount this writably. + overlayMedium = config.SelfOverlay + if !specutils.IsGoferMount(hint.Mount) { + mountType = hint.Mount.Type + } + isShared = hint.ShouldShareMount() + } + filestore, goferConf, err := c.createGoferFilestore(overlayMedium, c.Spec.Mounts[i].Source, mountType, isShared) if err != nil { return nil, nil, err } @@ -951,41 +969,43 @@ func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *b return goferFilestores, goferConfs, nil } -func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.GoferMountConf, error) { - // MountHint information takes precedence over shouldOverlay. - if hint != nil && !specutils.IsGoferMount(hint.Mount) { - switch hint.Mount.Type { - case "tmpfs": - // Create self-backed tmpfs. - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.NoneLower, Upper: boot.SelfOverlay}) - default: - return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type) - } - } - switch { - case !shouldOverlay: - return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.NoOverlay}, nil - case ovlConf.IsBackedByMemory(): - return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil - case ovlConf.IsBackedBySelf(): - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.SelfOverlay}) +func (c *Container) createGoferFilestore(overlayMedium config.OverlayMedium, mountSrc string, mountType string, isShared bool) (*os.File, boot.GoferMountConf, error) { + var lower boot.GoferMountConfLowerType + switch mountType { + case boot.Bind: + lower = boot.Lisafs + case tmpfs.Name: + lower = boot.NoneLower default: - return c.createGoferFilestoreInDir(ovlConf) + return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", mountType) + } + switch overlayMedium { + case config.NoOverlay: + return nil, boot.GoferMountConf{Lower: lower, Upper: boot.NoOverlay}, nil + case config.MemoryOverlay: + return nil, boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay}, nil + case config.SelfOverlay: + return c.createGoferFilestoreInSelf(mountSrc, isShared, boot.GoferMountConf{Lower: lower, Upper: boot.SelfOverlay}) + default: + if overlayMedium.IsBackedByAnon() { + return c.createGoferFilestoreInDir(overlayMedium.HostFileDir(), boot.GoferMountConf{Lower: lower, Upper: boot.AnonOverlay}) + } + return nil, boot.GoferMountConf{}, fmt.Errorf("unexpected overlay medium %q", overlayMedium) } } -func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.MountHint, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { +func (c *Container) createGoferFilestoreInSelf(mountSrc string, isShared bool, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { mountSrcInfo, err := os.Stat(mountSrc) if err != nil { return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) } if !mountSrcInfo.IsDir() { log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) - return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil + return nil, boot.GoferMountConf{Lower: successConf.Lower, Upper: boot.MemoryOverlay}, nil } // Create the self filestore file. createFlags := unix.O_RDWR | unix.O_CREAT | unix.O_CLOEXEC - if !(hint != nil && hint.ShouldShareMount()) { + if !isShared { // Allow shared mounts to reuse existing filestore. A previous shared user // may have already set up the filestore. createFlags |= unix.O_EXCL @@ -1011,8 +1031,7 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount return os.NewFile(uintptr(filestoreFD), filestorePath), successConf, nil } -func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File, boot.GoferMountConf, error) { - filestoreDir := ovlConf.HostFileDir() +func (c *Container) createGoferFilestoreInDir(filestoreDir string, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { fileInfo, err := os.Stat(filestoreDir) if err != nil { return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) @@ -1033,7 +1052,7 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File return nil, boot.GoferMountConf{}, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) } log.Debugf("Created an unnamed filestore file at %q", filestoreDir) - return filestoreFile, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.AnonOverlay}, nil + return filestoreFile, successConf, nil } // saveLocked saves the container metadata to a file. From bff11508c7f49ca17afac591eb198d3737cf5e15 Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Wed, 4 Oct 2023 21:40:52 +0800 Subject: [PATCH 3/3] runsc: add the EROFS rootfs support This patch adds the EROFS rootfs support. No gofer process will be created for the container, when there is no need to pass through host files into a container via the gofer process. Annotations for rootfs are also introduced to provide extra information, including the mount source, mount type and overlay config. Additionally, busybox-static is added to the default image and will be used to build the EROFS rootfs images during the test. Updates #8956 Signed-off-by: Tiwei Bie --- images/default/Dockerfile | 2 +- pkg/sentry/fsimpl/erofs/save_restore.go | 4 +- runsc/boot/BUILD | 1 + runsc/boot/gofer_conf.go | 14 + runsc/boot/gofer_conf_test.go | 47 +++ runsc/boot/loader.go | 6 +- runsc/boot/mount_hints.go | 76 +++- runsc/boot/mount_hints_test.go | 103 ++++++ runsc/boot/vfs.go | 13 + runsc/cmd/boot.go | 25 +- runsc/cmd/gofer.go | 57 +-- runsc/container/BUILD | 2 + runsc/container/container.go | 111 ++++-- runsc/container/container_test.go | 473 ++++++++++++++++-------- runsc/sandbox/sandbox.go | 6 +- 15 files changed, 721 insertions(+), 219 deletions(-) diff --git a/images/default/Dockerfile b/images/default/Dockerfile index 128847421..d9af8639f 100644 --- a/images/default/Dockerfile +++ b/images/default/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \ apt-transport-https ca-certificates gnupg-agent \ software-properties-common \ pkg-config libffi-dev patch diffutils libssl-dev iptables kmod \ - clang crossbuild-essential-amd64 erofs-utils + clang crossbuild-essential-amd64 erofs-utils busybox-static # Install Docker client for the website build. RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - diff --git a/pkg/sentry/fsimpl/erofs/save_restore.go b/pkg/sentry/fsimpl/erofs/save_restore.go index a346e5b36..422485bd7 100644 --- a/pkg/sentry/fsimpl/erofs/save_restore.go +++ b/pkg/sentry/fsimpl/erofs/save_restore.go @@ -50,7 +50,9 @@ func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRest if got, want := newImage.SuperBlock(), fs.image.SuperBlock(); got != want { return fmt.Errorf("superblock mismatch detected on restore, got %+v, expected %+v", got, want) } - fs.image = newImage + // We need to update the image in place, as there are other pointers + // pointing to this image as well. + *fs.image = *newImage return nil } diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 3ffdf1d91..97eb48b39 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -148,6 +148,7 @@ go_test( "//pkg/cpuid", "//pkg/fspath", "//pkg/log", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/kernel/auth", "//pkg/sentry/seccheck", "//pkg/sentry/vfs", diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go index f3487346f..9d94cb8f4 100644 --- a/runsc/boot/gofer_conf.go +++ b/runsc/boot/gofer_conf.go @@ -17,6 +17,8 @@ package boot import ( "fmt" "strings" + + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" ) // GoferMountConfUpperType describes how upper layer is configured for the gofer mount. @@ -85,6 +87,9 @@ const ( // Lisafs indicates that this gofer mount has a LISAFS lower layer. Lisafs + // Erofs indicates that this gofer mount has an EROFS lower layer. + Erofs + // LowerMax indicates the number of the valid lower layer types. LowerMax ) @@ -96,6 +101,8 @@ func (l GoferMountConfLowerType) String() string { return "none" case Lisafs: return "lisafs" + case Erofs: + return erofs.Name } panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l)) } @@ -107,6 +114,8 @@ func (l *GoferMountConfLowerType) Set(v string) error { *l = NoneLower case "lisafs": *l = Lisafs + case erofs.Name: + *l = Erofs default: return fmt.Errorf("invalid gofer mount config lower layer type: %s", v) } @@ -168,6 +177,11 @@ func (g GoferMountConf) ShouldUseLisafs() bool { return g.Lower == Lisafs } +// ShouldUseErofs returns true if an EROFS should be applied. +func (g GoferMountConf) ShouldUseErofs() bool { + return g.Lower == Erofs +} + // valid returns true if this is a valid gofer mount config. func (g GoferMountConf) valid() bool { return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay) diff --git a/runsc/boot/gofer_conf_test.go b/runsc/boot/gofer_conf_test.go index 1bc454bc1..49f1f7ab2 100644 --- a/runsc/boot/gofer_conf_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -25,6 +25,7 @@ func TestGoferConf(t *testing.T) { wantHostFile bool wantLisafs bool wantTmpfs bool + wantErofs bool wantValid bool }{{ cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay}, @@ -36,6 +37,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: false, wantLisafs: false, wantTmpfs: true, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: NoneLower, Upper: SelfOverlay}, @@ -43,6 +45,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: false, wantTmpfs: true, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: NoneLower, Upper: AnonOverlay}, @@ -50,6 +53,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: false, wantTmpfs: true, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: NoOverlay}, @@ -57,6 +61,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: false, wantLisafs: true, wantTmpfs: false, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay}, @@ -64,6 +69,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: false, wantLisafs: true, wantTmpfs: false, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: SelfOverlay}, @@ -71,6 +77,7 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: true, wantTmpfs: false, + wantErofs: false, wantValid: true, }, { cfg: GoferMountConf{Lower: Lisafs, Upper: AnonOverlay}, @@ -78,6 +85,39 @@ func TestGoferConf(t *testing.T) { wantHostFile: true, wantLisafs: true, wantTmpfs: false, + wantErofs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: NoOverlay}, + wantOverlay: false, + wantHostFile: false, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: MemoryOverlay}, + wantOverlay: true, + wantHostFile: false, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: SelfOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Erofs, Upper: AnonOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: false, + wantErofs: true, wantValid: true, }, { cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax}, @@ -104,6 +144,9 @@ func TestGoferConf(t *testing.T) { if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs { t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs) } + if got := tc.cfg.ShouldUseErofs(); got != tc.wantErofs { + t.Errorf("gofer conf = %+v, ShouldUseErofs() = %t, want = %t", tc.cfg, got, tc.wantErofs) + } } } @@ -116,6 +159,10 @@ func TestGoferConfFlags(t *testing.T) { {Lower: Lisafs, Upper: MemoryOverlay}, {Lower: Lisafs, Upper: SelfOverlay}, {Lower: Lisafs, Upper: AnonOverlay}, + {Lower: Erofs, Upper: NoOverlay}, + {Lower: Erofs, Upper: MemoryOverlay}, + {Lower: Erofs, Upper: SelfOverlay}, + {Lower: Erofs, Upper: AnonOverlay}, } var got GoferMountConfFlags got.Set(want.String()) diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index e680b02d5..db7a54ecf 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -951,7 +951,11 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne if len(info.goferFDs) < 1 { return nil, nil, fmt.Errorf("rootfs gofer FD not found") } - l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) + // TODO(ayushranjan): The gofer monitor should be started as long as the gofer + // process exists, even if the root mount is not backed by lisafs. + if info.goferMountConfs[0].ShouldUseLisafs() { + l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) + } // We can share l.sharedMounts with containerMounter since l.mu is locked. // Hence, mntr must only be used within this function (while l.mu is locked). diff --git a/runsc/boot/mount_hints.go b/runsc/boot/mount_hints.go index abe903ffc..c8e428cdb 100644 --- a/runsc/boot/mount_hints.go +++ b/runsc/boot/mount_hints.go @@ -16,17 +16,24 @@ package boot import ( "fmt" + "path/filepath" "strings" specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/specutils" ) -// MountPrefix is the annotation prefix for mount hints. -const MountPrefix = "dev.gvisor.spec.mount." +const ( + // MountPrefix is the annotation prefix for mount hints applied at the pod level. + MountPrefix = "dev.gvisor.spec.mount." + + // RootfsPrefix is the annotation prefix for rootfs hint applied at the container level. + RootfsPrefix = "dev.gvisor.spec.rootfs." +) // ShareType indicates who can access/mutate the volume contents. type ShareType int @@ -219,3 +226,68 @@ func (p *PodMountHints) FindMount(mountSrc string) *MountHint { } return nil } + +// RootfsHint represents extra information about rootfs that are provided via +// annotations. They can provide mount source, mount type and overlay config. +type RootfsHint struct { + Mount specs.Mount + Overlay config.OverlayMedium +} + +func (r *RootfsHint) setSource(val string) error { + if !filepath.IsAbs(val) { + return fmt.Errorf("source should be an absolute path, got %q", val) + } + r.Mount.Source = val + return nil +} + +func (r *RootfsHint) setType(val string) error { + switch val { + case erofs.Name, Bind: + r.Mount.Type = val + default: + return fmt.Errorf("invalid type %q", val) + } + return nil +} + +func (r *RootfsHint) setField(key, val string) error { + switch key { + case "source": + return r.setSource(val) + case "type": + return r.setType(val) + case "overlay": + return r.Overlay.Set(val) + default: + return fmt.Errorf("invalid rootfs annotation: %s=%s", key, val) + } +} + +// NewRootfsHint instantiates RootfsHint using spec. +func NewRootfsHint(spec *specs.Spec) (*RootfsHint, error) { + var hint *RootfsHint + for k, v := range spec.Annotations { + // Look for 'dev.gvisor.spec.rootfs' annotations and parse them. + if !strings.HasPrefix(k, RootfsPrefix) { + continue + } + // Remove the prefix. + k = k[len(RootfsPrefix):] + if hint == nil { + hint = &RootfsHint{} + } + if err := hint.setField(k, v); err != nil { + return nil, fmt.Errorf("invalid rootfs annotation (key = %q, value = %q): %v", k, v, err) + } + } + // Validate the parsed hint. + if hint != nil { + log.Infof("Rootfs annotations found, source: %q, type: %q, overlay: %q", hint.Mount.Source, hint.Mount.Type, hint.Overlay) + if len(hint.Mount.Source) == 0 || len(hint.Mount.Type) == 0 { + return nil, fmt.Errorf("rootfs annotations missing required field(s): %+v", hint) + } + } + return hint, nil +} diff --git a/runsc/boot/mount_hints_test.go b/runsc/boot/mount_hints_test.go index 682e03f36..775db0334 100644 --- a/runsc/boot/mount_hints_test.go +++ b/runsc/boot/mount_hints_test.go @@ -20,6 +20,8 @@ import ( "testing" specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" + "gvisor.dev/gvisor/runsc/config" ) func TestPodMountHintsHappy(t *testing.T) { @@ -247,3 +249,104 @@ func TestHintsCheckCompatible(t *testing.T) { }) } } + +// TestRootfsHintHappy tests that valid rootfs annotations can be parsed correctly. +func TestRootfsHintHappy(t *testing.T) { + const imagePath = "/tmp/rootfs.img" + spec := &specs.Spec{ + Annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + } + hint, err := NewRootfsHint(spec) + if err != nil { + t.Fatalf("NewRootfsHint failed: %v", err) + } + + // Check that fields were set correctly. + if hint.Mount.Source != imagePath { + t.Errorf("rootfs source, want: %q, got: %q", imagePath, hint.Mount.Source) + } + if hint.Mount.Type != erofs.Name { + t.Errorf("rootfs type, want: %q, got: %q", erofs.Name, hint.Mount.Type) + } + if hint.Overlay != config.MemoryOverlay { + t.Errorf("rootfs overlay, want: %q, got: %q", config.MemoryOverlay, hint.Overlay) + } +} + +// TestRootfsHintErrors tests that proper errors will be returned when parsing +// invalid rootfs annotations. +func TestRootfsHintErrors(t *testing.T) { + const imagePath = "/tmp/rootfs.img" + for _, tst := range []struct { + name string + annotations map[string]string + error string + }{ + { + name: "invalid source", + annotations: map[string]string{ + RootfsPrefix + "source": "invalid", + RootfsPrefix + "type": erofs.Name, + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid type", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": "invalid", + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid overlay", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": "invalid", + }, + error: "invalid rootfs annotation", + }, + { + name: "invalid key", + annotations: map[string]string{ + RootfsPrefix + "invalid": "invalid", + RootfsPrefix + "source": imagePath, + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "invalid rootfs annotation", + }, + { + name: "missing source", + annotations: map[string]string{ + RootfsPrefix + "type": erofs.Name, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "rootfs annotations missing required field", + }, + { + name: "missing type", + annotations: map[string]string{ + RootfsPrefix + "source": imagePath, + RootfsPrefix + "overlay": config.MemoryOverlay.String(), + }, + error: "rootfs annotations missing required field", + }, + } { + t.Run(tst.name, func(t *testing.T) { + spec := &specs.Spec{Annotations: tst.annotations} + hint, err := NewRootfsHint(spec) + if err == nil || !strings.Contains(err.Error(), tst.error) { + t.Errorf("NewRootfsHint invalid error, want: .*%s.*, got: %v", tst.error, err) + } + if hint != nil { + t.Errorf("NewRootfsHint must return nil on failure: %+v", hint) + } + }) + } +} diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 6094e0d46..8a9f6d984 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -478,6 +478,19 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi }, } + case rootfsConf.ShouldUseErofs(): + fsName = erofs.Name + opts = &vfs.MountOptions{ + ReadOnly: c.root.Readonly, + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalMount: true, + Data: fmt.Sprintf("ifd=%d", ioFD), + InternalData: erofs.InternalFilesystemOptions{ + UniqueID: "/", + }, + }, + } + default: return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf) } diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 6691b0c10..ffa92f76f 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -125,7 +125,9 @@ type Boot struct { // mountsFD is the file descriptor to read list of mounts after they have // been resolved (direct paths, no symlinks). They are resolved outside the - // sandbox (e.g. gofer) and sent through this FD. + // sandbox (e.g. gofer) and sent through this FD. When mountsFD is not + // provided, there is no cleaning required for mounts and the mounts in + // the spec can be used as is. mountsFD int podInitConfigFD int @@ -194,7 +196,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process") f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file") - f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") + f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.") f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.") @@ -202,7 +204,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") - f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") + f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.") f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.") f.Var(&b.nvidiaDevMinors, "nvidia-dev-minors", "list of device minors for Nvidia GPU devices exposed to the sandbox.") @@ -378,15 +380,18 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } - // Read resolved mount list and replace the original one from the spec. - mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file") - cleanMounts, err := specutils.ReadMounts(mountsFile) - if err != nil { + // When mountsFD is not provided, there is no cleaning required. + if b.mountsFD >= 0 { + // Read resolved mount list and replace the original one from the spec. + mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file") + cleanMounts, err := specutils.ReadMounts(mountsFile) + if err != nil { + mountsFile.Close() + util.Fatalf("Error reading mounts file: %v", err) + } mountsFile.Close() - util.Fatalf("Error reading mounts file: %v", err) + spec.Mounts = cleanMounts } - mountsFile.Close() - spec.Mounts = cleanMounts if conf.DirectFS { // sandbox should run with a umask of 0, because we want to preserve file diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index aed4bc4f7..333f513b3 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -115,8 +115,8 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) { f.BoolVar(&g.setUpRoot, "setup-root", true, "if true, set up an empty root for the process") // Open FDs that are donated to the gofer. - f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec") - f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured") + f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. Follows the same order as --gofer-mount-confs. FDs are only donated if the mount is backed by lisafs.") + f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured. They must follow this order: root first, then mounts as defined in the spec.") f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).") @@ -267,16 +267,20 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom DonateMountPointFD: conf.DirectFS, }) - // Start with root mount, then add any other additional mount as needed. - cfgs = append(cfgs, connectionConfig{ - sock: newSocket(g.ioFDs[0]), - mountPath: "/", // fsgofer process is always chroot()ed. So serve root. - readonly: spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs(), - }) - log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly) + ioFDs := g.ioFDs + rootfsConf := g.mountConfs[0] + if rootfsConf.ShouldUseLisafs() { + // Start with root mount, then add any other additional mount as needed. + cfgs = append(cfgs, connectionConfig{ + sock: newSocket(ioFDs[0]), + mountPath: "/", // fsgofer process is always chroot()ed. So serve root. + readonly: spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs(), + }) + log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, ioFDs[0], cfgs[0].readonly) + ioFDs = ioFDs[1:] + } mountIdx := 1 // first one is the root - submountIoFDs := g.ioFDs[1:] for _, m := range spec.Mounts { if !specutils.IsGoferMount(m) { continue @@ -290,11 +294,11 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom util.Fatalf("mount destination must be absolute: %q", m.Destination) } - if len(submountIoFDs) == 0 { + if len(ioFDs) == 0 { util.Fatalf("no FD found for mount. Did you forget --io-fd? FDs: %d, Mount: %+v", len(g.ioFDs), m) } - ioFD := submountIoFDs[0] - submountIoFDs = submountIoFDs[1:] + ioFD := ioFDs[0] + ioFDs = ioFDs[1:] readonly := specutils.IsReadonlyMount(m.Options) || mountConf.ShouldUseOverlayfs() cfgs = append(cfgs, connectionConfig{ sock: newSocket(ioFD), @@ -304,7 +308,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly) } - if len(submountIoFDs) > 0 { + if len(ioFDs) > 0 { util.Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", len(cfgs), len(g.ioFDs)) } @@ -392,17 +396,20 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { procPath = "/proc/proc" } - // Mount root path followed by submounts. - if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil { - return fmt.Errorf("mounting root on root (%q) err: %v", root, err) - } + rootfsConf := g.mountConfs[0] + if rootfsConf.ShouldUseLisafs() { + // Mount root path followed by submounts. + if err := specutils.SafeMount(spec.Root.Path, root, "bind", unix.MS_BIND|unix.MS_REC, "", procPath); err != nil { + return fmt.Errorf("mounting root on root (%q) err: %v", root, err) + } - flags := uint32(unix.MS_SLAVE | unix.MS_REC) - if spec.Linux != nil && spec.Linux.RootfsPropagation != "" { - flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation}) - } - if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil { - return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err) + flags := uint32(unix.MS_SLAVE | unix.MS_REC) + if spec.Linux != nil && spec.Linux.RootfsPropagation != "" { + flags = specutils.PropOptionsToFlags([]string{spec.Linux.RootfsPropagation}) + } + if err := specutils.SafeMount("", root, "", uintptr(flags), "", procPath); err != nil { + return fmt.Errorf("mounting root (%q) with flags: %#x, err: %v", root, flags, err) + } } // Replace the current spec, with the clean spec with symlinks resolved. @@ -423,7 +430,7 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { } // Check if root needs to be remounted as readonly. - if spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs() { + if rootfsConf.ShouldUseLisafs() && (spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs()) { // If root is a mount point but not read-only, we can change mount options // to make it read-only for extra safety. // unix.MS_NOSUID and unix.MS_NODEV are included here not only diff --git a/runsc/container/BUILD b/runsc/container/BUILD index 0d2bf7969..0e129eb4e 100644 --- a/runsc/container/BUILD +++ b/runsc/container/BUILD @@ -22,6 +22,7 @@ go_library( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/pgalloc", "//pkg/sighandling", @@ -75,6 +76,7 @@ go_test( "//pkg/cleanup", "//pkg/log", "//pkg/sentry/control", + "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", "//pkg/sentry/limits", diff --git a/runsc/container/container.go b/runsc/container/container.go index 6c7f6610c..044280d08 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -37,6 +37,7 @@ import ( "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sighandling" @@ -288,16 +289,23 @@ func New(conf *config.Config, args Args) (*Container, error) { if err != nil { return nil, fmt.Errorf("error creating pod mount hints: %w", err) } - goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints) + rootfsHint, err := boot.NewRootfsHint(args.Spec) + if err != nil { + return nil, fmt.Errorf("error creating rootfs hint: %w", err) + } + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints, rootfsHint) if err != nil { return nil, err } + if !goferConfs[0].ShouldUseLisafs() && conf.NVProxyDocker { + return nil, fmt.Errorf("--nvproxy-docker cannot be used together with non-lisafs backed root mount") + } c.GoferMountConfs = goferConfs if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil { return nil, err } if err := runInCgroup(containerCgroup, func() error { - ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached) + ioFiles, specFile, err := c.createGoferProcess(args.Spec, conf, args.BundleDir, args.Attached, rootfsHint) if err != nil { return fmt.Errorf("cannot create gofer process: %w", err) } @@ -450,7 +458,11 @@ func (c *Container) Start(conf *config.Config) error { return err } } else { - goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints) + rootfsHint, err := boot.NewRootfsHint(c.Spec) + if err != nil { + return fmt.Errorf("error creating rootfs hint: %w", err) + } + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints, rootfsHint) if err != nil { return err } @@ -459,12 +471,14 @@ func (c *Container) Start(conf *config.Config) error { // the start (and all their children processes). if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error { // Create the gofer process. - goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false) + goferFiles, mountsFile, err := c.createGoferProcess(c.Spec, conf, c.BundleDir, false, rootfsHint) if err != nil { return err } defer func() { - _ = mountsFile.Close() + if mountsFile != nil { + _ = mountsFile.Close() + } for _, f := range goferFiles { _ = f.Close() } @@ -473,11 +487,13 @@ func (c *Container) Start(conf *config.Config) error { } }() - cleanMounts, err := specutils.ReadMounts(mountsFile) - if err != nil { - return fmt.Errorf("reading mounts file: %v", err) + if mountsFile != nil { + cleanMounts, err := specutils.ReadMounts(mountsFile) + if err != nil { + return fmt.Errorf("reading mounts file: %v", err) + } + c.Spec.Mounts = cleanMounts } - c.Spec.Mounts = cleanMounts // Setup stdios if the container is not using terminal. Otherwise TTY was // already setup in create. @@ -913,13 +929,19 @@ func (c *Container) forEachSelfMount(fn func(mountSrc string)) { // createGoferFilestores creates the regular files that will back the // tmpfs/overlayfs mounts that will overlay some gofer mounts. It also returns // information about how each gofer mount is configured. -func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.GoferMountConf, error) { +func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints, rootfsHint *boot.RootfsHint) ([]*os.File, []boot.GoferMountConf, error) { var goferFilestores []*os.File var goferConfs []boot.GoferMountConf // Handle root mount first. overlayMedium := ovlConf.RootOverlayMedium() mountType := boot.Bind + if rootfsHint != nil { + overlayMedium = rootfsHint.Overlay + if !specutils.IsGoferMount(rootfsHint.Mount) { + mountType = rootfsHint.Mount.Type + } + } if c.Spec.Root.Readonly { overlayMedium = config.NoOverlay } @@ -976,6 +998,8 @@ func (c *Container) createGoferFilestore(overlayMedium config.OverlayMedium, mou lower = boot.Lisafs case tmpfs.Name: lower = boot.NoneLower + case erofs.Name: + lower = boot.Erofs default: return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", mountType) } @@ -1148,7 +1172,33 @@ func (c *Container) waitForStopped() error { return backoff.Retry(op, b) } -func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool) ([]*os.File, *os.File, error) { +// shouldSpawnGofer indicates whether the gofer process should be spawned. +func shouldSpawnGofer(goferConfs []boot.GoferMountConf) bool { + for _, cfg := range goferConfs { + if cfg.ShouldUseLisafs() { + return true + } + } + return false +} + +// createGoferProcess returns an IO file list and a mounts file on success. +// The IO file list consists of image files and/or socket files to connect to +// a gofer endpoint for the mount points using Gofers. The mounts file is the +// file to read list of mounts after they have been resolved (direct paths, +// no symlinks), and will be nil if there is no cleaning required for mounts. +func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool, rootfsHint *boot.RootfsHint) ([]*os.File, *os.File, error) { + if !shouldSpawnGofer(c.GoferMountConfs) { + if !c.GoferMountConfs[0].ShouldUseErofs() { + panic("goferless mode is only possible with EROFS rootfs") + } + ioFile, err := os.Open(rootfsHint.Mount.Source) + if err != nil { + return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + } + return []*os.File{ioFile}, nil, nil + } + donations := donation.Agency{} defer donations.Close() @@ -1209,24 +1259,37 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu } donations.DonateAndClose("mounts-fd", mountsGofer) - // Count the number of mounts using lisafs. - lisafsCount := 0 + // Count the number of mounts that needs an IO file. + ioFileCount := 0 for _, cfg := range c.GoferMountConfs { - if cfg.ShouldUseLisafs() { - lisafsCount++ + if cfg.ShouldUseLisafs() || cfg.ShouldUseErofs() { + ioFileCount++ } } - sandEnds := make([]*os.File, 0, lisafsCount) - for i := 0; i < lisafsCount; i++ { - fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) - if err != nil { - return nil, nil, err - } - sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) + sandEnds := make([]*os.File, 0, ioFileCount) + for i, cfg := range c.GoferMountConfs { + switch { + case cfg.ShouldUseLisafs(): + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) - goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") - donations.DonateAndClose("io-fds", goferEnd) + goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") + donations.DonateAndClose("io-fds", goferEnd) + + case cfg.ShouldUseErofs(): + if i > 0 { + return nil, nil, fmt.Errorf("EROFS lower layer is only supported for root mount") + } + if f, err := os.Open(rootfsHint.Mount.Source); err != nil { + return nil, nil, fmt.Errorf("opening rootfs image %q: %v", rootfsHint.Mount.Source, err) + } else { + sandEnds = append(sandEnds, f) + } + } } if attached { diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 4aa46d8b2..bef659996 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -39,12 +39,14 @@ import ( "gvisor.dev/gvisor/pkg/bits" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/cgroup" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/flag" @@ -1013,167 +1015,174 @@ func TestKillPid(t *testing.T) { } } -// TestCheckpointRestore creates a container that continuously writes successive +// testCheckpointRestore creates a container that continuously writes successive // integers to a file. To test checkpoint and restore functionality, the // container is checkpointed and the last number printed to the file is // recorded. Then, it is restored in two new containers and the first number // printed from these containers is checked. Both should be the next consecutive // number after the last number from the checkpointed container. +func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript func(string) *specs.Spec) { + dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("ioutil.TempDir failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + outputPath := filepath.Join(dir, "output") + outputFile, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile.Close() + + script := fmt.Sprintf("i=0; while true; do echo $i >> %q; sleep 1; i=$((i+1)); done", outputPath) + spec := newSpecWithScript(script) + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Create and start the container. + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(conf); err != nil { + t.Fatalf("error starting container: %v", err) + } + + // Set the image path, which is where the checkpoint image will be saved. + imagePath := filepath.Join(dir, "test-image-file") + + // Create the image file and open for writing. + file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + t.Fatalf("error opening new file at imagePath: %v", err) + } + defer file.Close() + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + // Checkpoint running container; save state into new file. + if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + t.Fatalf("error checkpointing container to empty file: %v", err) + } + defer os.RemoveAll(imagePath) + + lastNum, err := readOutputNum(outputPath, -1) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile2, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile2.Close() + + // Restore into a new container with different ID (e.g. clone). Keep the + // initial container running to ensure no conflict with it. + args2 := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont2, err := New(conf, args2) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont2.Destroy() + + if err := cont2.Restore(conf, imagePath); err != nil { + t.Fatalf("error restoring container: %v", err) + } + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile2); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) + } + cont2.Destroy() + cont2 = nil + + // Restore into a container using the same ID (e.g. save/resume). It requires + // the original container to cease to exist because they share the same identity. + cont.Destroy() + cont = nil + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile3, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile3.Close() + + cont3, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont3.Destroy() + + if err := cont3.Restore(conf, imagePath); err != nil { + t.Fatalf("error restoring container: %v", err) + } + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile3); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum2, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum2 { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) + } + cont3.Destroy() +} + +// TestCheckpointRestore does the checkpoint/restore test on each platform. func TestCheckpointRestore(t *testing.T) { // Skip overlay because test requires writing to host file. for name, conf := range configs(t, true /* noOverlay */) { t.Run(name, func(t *testing.T) { - dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test") - if err != nil { - t.Fatalf("ioutil.TempDir failed: %v", err) - } - defer os.RemoveAll(dir) - if err := os.Chmod(dir, 0777); err != nil { - t.Fatalf("error chmoding file: %q, %v", dir, err) - } - - outputPath := filepath.Join(dir, "output") - outputFile, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile.Close() - - script := fmt.Sprintf("for ((i=0; ;i++)); do echo $i >> %q; sleep 1; done", outputPath) - spec := testutil.NewSpecWithArgs("bash", "-c", script) - _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) - if err != nil { - t.Fatalf("error setting up container: %v", err) - } - defer cleanup() - - // Create and start the container. - args := Args{ - ID: testutil.RandomContainerID(), - Spec: spec, - BundleDir: bundleDir, - } - cont, err := New(conf, args) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont.Destroy() - if err := cont.Start(conf); err != nil { - t.Fatalf("error starting container: %v", err) - } - - // Set the image path, which is where the checkpoint image will be saved. - imagePath := filepath.Join(dir, "test-image-file") - - // Create the image file and open for writing. - file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - t.Fatalf("error opening new file at imagePath: %v", err) - } - defer file.Close() - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { - t.Fatalf("error checkpointing container to empty file: %v", err) - } - defer os.RemoveAll(imagePath) - - lastNum, err := readOutputNum(outputPath, -1) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Delete and recreate file before restoring. - if err := os.Remove(outputPath); err != nil { - t.Fatalf("error removing file") - } - outputFile2, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile2.Close() - - // Restore into a new container with different ID (e.g. clone). Keep the - // initial container running to ensure no conflict with it. - args2 := Args{ - ID: testutil.RandomContainerID(), - Spec: spec, - BundleDir: bundleDir, - } - cont2, err := New(conf, args2) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont2.Destroy() - - if err := cont2.Restore(conf, imagePath); err != nil { - t.Fatalf("error restoring container: %v", err) - } - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile2); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - firstNum, err := readOutputNum(outputPath, 0) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Check that lastNum is one less than firstNum and that the container - // picks up from where it left off. - if lastNum+1 != firstNum { - t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) - } - cont2.Destroy() - cont2 = nil - - // Restore into a container using the same ID (e.g. save/resume). It requires - // the original container to cease to exist because they share the same identity. - cont.Destroy() - cont = nil - - // Delete and recreate file before restoring. - if err := os.Remove(outputPath); err != nil { - t.Fatalf("error removing file") - } - outputFile3, err := createWriteableOutputFile(outputPath) - if err != nil { - t.Fatalf("error creating output file: %v", err) - } - defer outputFile3.Close() - - cont3, err := New(conf, args) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - defer cont3.Destroy() - - if err := cont3.Restore(conf, imagePath); err != nil { - t.Fatalf("error restoring container: %v", err) - } - - // Wait until application has ran. - if err := waitForFileNotEmpty(outputFile3); err != nil { - t.Fatalf("Failed to wait for output file: %v", err) - } - - firstNum2, err := readOutputNum(outputPath, 0) - if err != nil { - t.Fatalf("error with outputFile: %v", err) - } - - // Check that lastNum is one less than firstNum and that the container - // picks up from where it left off. - if lastNum+1 != firstNum2 { - t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) - } - cont3.Destroy() + testCheckpointRestore(t, conf, func(script string) *specs.Spec { + return testutil.NewSpecWithArgs("bash", "-c", script) + }) }) } } @@ -3178,3 +3187,163 @@ find $dir -type l -o -type f | sort | xargs cat | md5sum`), 0755); err != nil { } } } + +// createRootfsEROFS creates a rootfs directory and an EROFS rootfs image in +// the directory dir. +func createRootfsEROFS(dir string) (string, string, error) { + mkfs, err := exec.LookPath("mkfs.erofs") + if err != nil { + return "", "", fmt.Errorf("mkfs.erofs is not available: %v", err) + } + + busybox, err := exec.LookPath("busybox") + if err != nil { + return "", "", fmt.Errorf("busybox is not available: %v", err) + } + + // Create a rootfs directory with busybox in root. + rootfsDir := filepath.Join(dir, "rootfs") + if err := os.Mkdir(rootfsDir, 0755); err != nil { + return "", "", fmt.Errorf("os.Mkdir() failed: %v", err) + } + if err := testutil.Copy(busybox, filepath.Join(rootfsDir, "busybox")); err != nil { + return "", "", fmt.Errorf("failed to copy busybox: %v", err) + } + + // Handcraft the following mount points that the sentry mounts need, because EROFS + // does not support creating synthetic directories yet and we may not want to use + // overlay in some tests. + for _, dir := range []string{"dev", "proc", "sys", "tmp"} { + if err := os.Mkdir(filepath.Join(rootfsDir, dir), 0755); err != nil { + return "", "", fmt.Errorf("os.Mkdir() failed: %v", err) + } + } + + // Build the EROFS rootfs image. + rootfsImage := filepath.Join(dir, "rootfs.img") + cmd := fmt.Sprintf("%s -E noinline_data %s %s", mkfs, rootfsImage, rootfsDir) + if out, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { + return "", "", fmt.Errorf("exec: sh -c %q, err: %v, out: %s", cmd, err, out) + } + + return rootfsDir, rootfsImage, nil +} + +// TestRootfsEROFS starts a container using an EROFS image as the rootfs and checks that +// the rootfs in container is an EROFS. +func TestRootfsEROFS(t *testing.T) { + // Skip this test if mkfs.erofs or busybox are not available. + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skipf("mkfs.erofs is not available: %v", err) + } + if _, err := exec.LookPath("busybox"); err != nil { + t.Skipf("busybox is not available: %v", err) + } + + testDir, err := ioutil.TempDir(testutil.TmpDir(), "erofs_rootfs_test_") + if err != nil { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + rootfsDir, rootfsImage, err := createRootfsEROFS(testDir) + if err != nil { + t.Fatalf("failed to create EROFS rootfs image: %v", err) + } + + // Create the spec and set the EROFS rootfs annotations. + spec := testutil.NewSpecWithArgs("/busybox", "grep", "/ / ro - erofs", "/proc/self/mountinfo") + spec.Root.Path = rootfsDir + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.RootfsPrefix+"type"] = erofs.Name + spec.Annotations[boot.RootfsPrefix+"source"] = rootfsImage + // Disable the overlay, as we want to be sure that rootfs will always be + // shown as EROFS in mountinfo. + spec.Annotations[boot.RootfsPrefix+"overlay"] = config.NoOverlay.String() + + conf := testutil.TestConfig(t) + + for _, mounts := range [][]specs.Mount{ + // Case 1: EROFS rootfs without any other gofer mount. + nil, + + // Case 2: EROFS rootfs with a LISAFS backed gofer mount. + []specs.Mount{ + { + Type: "bind", + Destination: "/tmp", + Source: "/tmp", + }, + }, + } { + spec.Mounts = mounts + + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Create and start the container. + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + Attached: true, + } + ws, err := Run(conf, args) + if err != nil { + t.Fatalf("error running container: %v", err) + } + if ws.ExitStatus() != 0 { + t.Errorf("got exit status %v want %v", ws.ExitStatus(), 0) + } + } +} + +// TestCheckpointRestoreEROFS does the checkpoint/restore test on each platform using +// an EROFS image as the rootfs. +func TestCheckpointRestoreEROFS(t *testing.T) { + // Skip this test if mkfs.erofs or busybox are not available. + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skipf("mkfs.erofs is not available: %v", err) + } + if _, err := exec.LookPath("busybox"); err != nil { + t.Skipf("busybox is not available: %v", err) + } + + testDir, err := ioutil.TempDir(testutil.TmpDir(), "erofs_checkpoint_restore_test_") + if err != nil { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + rootfsDir, rootfsImage, err := createRootfsEROFS(testDir) + if err != nil { + t.Fatalf("failed to create EROFS rootfs image: %v", err) + } + + // Skip overlay because test requires writing to host file. + for name, conf := range configs(t, true /* noOverlay */) { + t.Run(name, func(t *testing.T) { + testCheckpointRestore(t, conf, func(script string) *specs.Spec { + spec := testutil.NewSpecWithArgs("/busybox", "sh", "-c", script) + spec.Root = &specs.Root{ + Path: rootfsDir, + Readonly: false, + } + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.RootfsPrefix+"type"] = erofs.Name + spec.Annotations[boot.RootfsPrefix+"source"] = rootfsImage + // EROFS does not support creating synthetic directories yet, so let's add + // a writeable and savable overlay for rootfs, which allows the sentry to + // create the mount point for the bind mount of the temporary directory shared + // between host and test container. + spec.Annotations[boot.RootfsPrefix+"overlay"] = config.MemoryOverlay.String() + return spec + }) + }) + } +} diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 8b4071a79..44fef8cad 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -223,9 +223,9 @@ type Args struct { // UserLog is the filename to send user-visible logs to. It may be empty. UserLog string - // IOFiles is the list of files that connect to a gofer endpoint for the - // mounts points using Gofers. They must be in the same order as mounts - // appear in the spec. + // IOFiles is the list of image files and/or socket files that connect to + // a gofer endpoint for the mount points using Gofers. They must be in the + // same order as mounts appear in the spec. IOFiles []*os.File // GoferFilestoreFiles are the regular files that will back the overlayfs or