diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go index 49177accb..f3487346f 100644 --- a/runsc/boot/gofer_conf.go +++ b/runsc/boot/gofer_conf.go @@ -16,52 +16,161 @@ package boot import ( "fmt" - "strconv" "strings" ) -// GoferMountConf describes how a gofer mount is configured in the sandbox. -type GoferMountConf int +// GoferMountConfUpperType describes how upper layer is configured for the gofer mount. +type GoferMountConfUpperType byte const ( - // VanillaGofer indicates that this gofer mount has no special configuration. - VanillaGofer GoferMountConf = iota + // NoOverlay indicates that this gofer mount has no upper layer. In this case, + // this gofer mount must have a lower layer (i.e. lower != NoneLower). + NoOverlay GoferMountConfUpperType = iota - // MemoryOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by application memory. + // MemoryOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by application memory. MemoryOverlay - // SelfOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by a host file in the mount's source directory. + // SelfOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by a host file in the mount's source directory. SelfOverlay - // AnonOverlay indicates that this gofer mount should be overlaid with an - // overlayfs backed by a host file in an anonymous directory. + // AnonOverlay indicates that this gofer mount should be overlaid with a + // tmpfs backed by a host file in an anonymous directory. AnonOverlay - // SelfTmpfs indicates that this gofer mount should be overlaid with a tmpfs - // mount backed by a host file in the mount's source directory. - SelfTmpfs + // UpperMax indicates the number of the valid upper layer types. + UpperMax ) +// String returns a human-readable string representing the upper layer type. +func (u GoferMountConfUpperType) String() string { + switch u { + case NoOverlay: + return "none" + case MemoryOverlay: + return "memory" + case SelfOverlay: + return "self" + case AnonOverlay: + return "anon" + } + panic(fmt.Sprintf("Invalid gofer mount config upper layer type: %d", u)) +} + +// Set sets the value. Set(String()) should be idempotent. +func (u *GoferMountConfUpperType) Set(v string) error { + switch v { + case "none": + *u = NoOverlay + case "memory": + *u = MemoryOverlay + case "self": + *u = SelfOverlay + case "anon": + *u = AnonOverlay + default: + return fmt.Errorf("invalid gofer mount config upper layer type: %s", v) + } + return nil +} + +// GoferMountConfLowerType describes how lower layer is configured for the gofer mount. +type GoferMountConfLowerType byte + +const ( + // NoneLower indicates that this gofer mount has no lower layer. + NoneLower GoferMountConfLowerType = iota + + // Lisafs indicates that this gofer mount has a LISAFS lower layer. + Lisafs + + // LowerMax indicates the number of the valid lower layer types. + LowerMax +) + +// String returns a human-readable string representing the lower layer type. +func (l GoferMountConfLowerType) String() string { + switch l { + case NoneLower: + return "none" + case Lisafs: + return "lisafs" + } + panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l)) +} + +// Set sets the value. Set(String()) should be idempotent. +func (l *GoferMountConfLowerType) Set(v string) error { + switch v { + case "none": + *l = NoneLower + case "lisafs": + *l = Lisafs + default: + return fmt.Errorf("invalid gofer mount config lower layer type: %s", v) + } + return nil +} + +// GoferMountConf describes how a gofer mount is configured in the sandbox. +type GoferMountConf struct { + Upper GoferMountConfUpperType `json:"upper"` + Lower GoferMountConfLowerType `json:"lower"` +} + +// String returns a human-readable string representing the gofer mount config. +func (g GoferMountConf) String() string { + return fmt.Sprintf("%s:%s", g.Lower, g.Upper) +} + +// Set sets the value. Set(String()) should be idempotent. +func (g *GoferMountConf) Set(v string) error { + parts := strings.Split(v, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid gofer mount config format: %q", v) + } + if err := g.Lower.Set(parts[0]); err != nil { + return err + } + if err := g.Upper.Set(parts[1]); err != nil { + return err + } + if !g.valid() { + return fmt.Errorf("invalid gofer mount config: %+v", g) + } + return nil +} + // IsFilestorePresent returns true if a filestore file was associated with this. func (g GoferMountConf) IsFilestorePresent() bool { - return g == SelfOverlay || g == AnonOverlay || g == SelfTmpfs + return g.Upper == SelfOverlay || g.Upper == AnonOverlay } // IsSelfBacked returns true if this mount is backed by a filestore in itself. func (g GoferMountConf) IsSelfBacked() bool { - return g == SelfOverlay || g == SelfTmpfs + return g.Upper == SelfOverlay } // ShouldUseOverlayfs returns true if an overlayfs should be applied. func (g GoferMountConf) ShouldUseOverlayfs() bool { - return g == MemoryOverlay || g == SelfOverlay || g == AnonOverlay + return g.Lower != NoneLower && g.Upper != NoOverlay +} + +// ShouldUseTmpfs returns true if a tmpfs should be applied. +func (g GoferMountConf) ShouldUseTmpfs() bool { + // g.valid() implies that g.Upper != NoOverlay. + return g.Lower == NoneLower } // ShouldUseLisafs returns true if a lisafs client/server should be set up. func (g GoferMountConf) ShouldUseLisafs() bool { - return g == VanillaGofer || g.ShouldUseOverlayfs() + return g.Lower == Lisafs +} + +// valid returns true if this is a valid gofer mount config. +func (g GoferMountConf) valid() bool { + return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay) } // GoferMountConfFlags can be used with GoferMountConf flags that appear @@ -70,11 +179,11 @@ type GoferMountConfFlags []GoferMountConf // String implements flag.Value. func (g *GoferMountConfFlags) String() string { - confVals := make([]string, 0, len(*g)) + confs := make([]string, 0, len(*g)) for _, confVal := range *g { - confVals = append(confVals, strconv.Itoa(int(confVal))) + confs = append(confs, confVal.String()) } - return strings.Join(confVals, ",") + return strings.Join(confs, ",") } // Get implements flag.Value. @@ -92,14 +201,11 @@ func (g *GoferMountConfFlags) GetArray() []GoferMountConf { func (g *GoferMountConfFlags) Set(s string) error { confs := strings.Split(s, ",") for _, conf := range confs { - confVal, err := strconv.Atoi(conf) - if err != nil { - return fmt.Errorf("invalid GoferMountConf value (%d): %v", confVal, err) + var confVal GoferMountConf + if err := confVal.Set(conf); err != nil { + return fmt.Errorf("invalid GoferMountConf value (%s): %v", conf, err) } - if confVal > int(SelfTmpfs) { - return fmt.Errorf("invalid GoferMountConf value (%d)", confVal) - } - *g = append(*g, GoferMountConf(confVal)) + *g = append(*g, confVal) } return nil } diff --git a/runsc/boot/gofer_conf_test.go b/runsc/boot/gofer_conf_test.go index f4e00b717..1bc454bc1 100644 --- a/runsc/boot/gofer_conf_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -20,51 +20,103 @@ import ( func TestGoferConf(t *testing.T) { tcs := []struct { - ovl GoferMountConf + cfg GoferMountConf wantOverlay bool wantHostFile bool wantLisafs bool + wantTmpfs bool + wantValid bool }{{ - ovl: VanillaGofer, + cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay}, + // This is not a valid config. + wantValid: false, + }, { + cfg: GoferMountConf{Lower: NoneLower, Upper: MemoryOverlay}, wantOverlay: false, wantHostFile: false, - wantLisafs: true, + wantLisafs: false, + wantTmpfs: true, + wantValid: true, }, { - ovl: MemoryOverlay, - wantOverlay: true, - wantHostFile: false, - wantLisafs: true, - }, { - ovl: SelfOverlay, - wantOverlay: true, - wantHostFile: true, - wantLisafs: true, - }, { - ovl: AnonOverlay, - wantOverlay: true, - wantHostFile: true, - wantLisafs: true, - }, { - ovl: SelfTmpfs, + cfg: GoferMountConf{Lower: NoneLower, Upper: SelfOverlay}, wantOverlay: false, wantHostFile: true, wantLisafs: false, + wantTmpfs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: NoneLower, Upper: AnonOverlay}, + wantOverlay: false, + wantHostFile: true, + wantLisafs: false, + wantTmpfs: true, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: NoOverlay}, + wantOverlay: false, + wantHostFile: false, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay}, + wantOverlay: true, + wantHostFile: false, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: SelfOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: Lisafs, Upper: AnonOverlay}, + wantOverlay: true, + wantHostFile: true, + wantLisafs: true, + wantTmpfs: false, + wantValid: true, + }, { + cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax}, + // This is not a valid config. + wantValid: false, }} for _, tc := range tcs { - if got := tc.ovl.ShouldUseOverlayfs(); got != tc.wantOverlay { - t.Errorf("gofer conf = %d, ShouldUseOverlayfs() = %t, want = %t", tc.ovl, got, tc.wantOverlay) + if got := tc.cfg.valid(); got != tc.wantValid { + t.Errorf("gofer conf = %+v, valid() = %t, want = %t", tc.cfg, got, tc.wantValid) } - if got := tc.ovl.IsFilestorePresent(); got != tc.wantHostFile { - t.Errorf("gofer conf = %d, IsFilestorePresent() = %t, want = %t", tc.ovl, got, tc.wantHostFile) + if !tc.wantValid { + // Skip the following tests, if this is not a valid config. + continue } - if got := tc.ovl.ShouldUseLisafs(); got != tc.wantLisafs { - t.Errorf("gofer conf = %d, ShouldUseLisafs() = %t, want = %t", tc.ovl, got, tc.wantLisafs) + if got := tc.cfg.ShouldUseOverlayfs(); got != tc.wantOverlay { + t.Errorf("gofer conf = %+v, ShouldUseOverlayfs() = %t, want = %t", tc.cfg, got, tc.wantOverlay) + } + if got := tc.cfg.IsFilestorePresent(); got != tc.wantHostFile { + t.Errorf("gofer conf = %+v, IsFilestorePresent() = %t, want = %t", tc.cfg, got, tc.wantHostFile) + } + if got := tc.cfg.ShouldUseLisafs(); got != tc.wantLisafs { + t.Errorf("gofer conf = %+v, ShouldUseLisafs() = %t, want = %t", tc.cfg, got, tc.wantLisafs) + } + if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs { + t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs) } } } func TestGoferConfFlags(t *testing.T) { - want := GoferMountConfFlags{VanillaGofer, MemoryOverlay, SelfOverlay, AnonOverlay, SelfTmpfs} + want := GoferMountConfFlags{ + {Lower: NoneLower, Upper: MemoryOverlay}, + {Lower: NoneLower, Upper: SelfOverlay}, + {Lower: NoneLower, Upper: AnonOverlay}, + {Lower: Lisafs, Upper: NoOverlay}, + {Lower: Lisafs, Upper: MemoryOverlay}, + {Lower: Lisafs, Upper: SelfOverlay}, + {Lower: Lisafs, Upper: AnonOverlay}, + } var got GoferMountConfFlags got.Set(want.String()) if len(got) != len(want) { diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index 4abb336ce..552e13654 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -140,7 +140,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error ControllerFD: fd, GoferFDs: []int{sandEnd}, StdioFDs: stdio, - GoferMountConfs: []GoferMountConf{VanillaGofer}, + GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}}, PodInitConfigFD: -1, ExecFD: -1, } diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index aca0ed12d..6094e0d46 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -447,33 +447,43 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre // createMountNamespace creates the container's root mount and namespace. func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) { ioFD := c.goferFDs.remove() - data := goferMountData(ioFD, conf.FileAccess, conf) - - // We can't check for overlayfs here because sandbox is chroot'ed and gofer - // can only send mount options for specs.Mounts (specs.Root is missing - // Options field). So assume root is always on top of overlayfs. - data = append(data, "overlayfs_stale_read") - - // Configure the gofer dentry cache size. - gofer.SetDentryCacheSize(conf.DCache) - - log.Infof("Mounting root with gofer, ioFD: %d", ioFD) - opts := &vfs.MountOptions{ - ReadOnly: c.root.Readonly, - GetFilesystemOptions: vfs.GetFilesystemOptions{ - InternalMount: true, - Data: strings.Join(data, ","), - InternalData: gofer.InternalFilesystemOptions{ - UniqueID: "/", - }, - }, - } - - fsName := gofer.Name rootfsConf := c.goferMountConfs[0] - if rootfsConf == SelfTmpfs { - panic("SelfTmpfs is not possible for rootfs") + + var ( + fsName string + opts *vfs.MountOptions + ) + switch { + case rootfsConf.ShouldUseLisafs(): + fsName = gofer.Name + + data := goferMountData(ioFD, conf.FileAccess, conf) + + // We can't check for overlayfs here because sandbox is chroot'ed and gofer + // can only send mount options for specs.Mounts (specs.Root is missing + // Options field). So assume root is always on top of overlayfs. + data = append(data, "overlayfs_stale_read") + + // Configure the gofer dentry cache size. + gofer.SetDentryCacheSize(conf.DCache) + + opts = &vfs.MountOptions{ + ReadOnly: c.root.Readonly, + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalMount: true, + Data: strings.Join(data, ","), + InternalData: gofer.InternalFilesystemOptions{ + UniqueID: "/", + }, + }, + } + + default: + return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf) } + + log.Infof("Mounting root with %s, ioFD: %d", fsName, ioFD) + if rootfsConf.ShouldUseOverlayfs() { log.Infof("Adding overlay on top of root") var ( @@ -608,7 +618,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co } // We need to hide the filestore from the containerized application. - if mountConf == SelfOverlay { + if mountConf.IsSelfBacked() { if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{ Root: upperRootVD, Start: upperRootVD, @@ -721,7 +731,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) { if info.goferMountConf.IsFilestorePresent() { info.filestoreFD = c.goferFilestoreFDs.removeAsFD() } - if info.goferMountConf == SelfTmpfs { + if info.goferMountConf.ShouldUseTmpfs() { specutils.ChangeMountType(info.mount, tmpfs.Name) } goferMntIdx++ diff --git a/runsc/cli/cli_test.go b/runsc/cli/cli_test.go index 1b1a74641..b45dde51e 100644 --- a/runsc/cli/cli_test.go +++ b/runsc/cli/cli_test.go @@ -31,6 +31,7 @@ var fakeFlagValues = [...]string{ "2h45m", "1:1,2:2", "0 0 1,100000 100000 65536", + "lisafs:self,lisafs:none", } func dupFlag(t *testing.T, cmd subcommands.Command, flagName string) *flag.Flag { diff --git a/runsc/container/container.go b/runsc/container/container.go index 4025ef9d0..71f6cbdcc 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -957,18 +957,18 @@ func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc strin switch hint.Mount.Type { case "tmpfs": // Create self-backed tmpfs. - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfTmpfs) + return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.NoneLower, Upper: boot.SelfOverlay}) default: - return nil, boot.VanillaGofer, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type) + return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type) } } switch { case !shouldOverlay: - return nil, boot.VanillaGofer, nil + return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.NoOverlay}, nil case ovlConf.IsBackedByMemory(): - return nil, boot.MemoryOverlay, nil + return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil case ovlConf.IsBackedBySelf(): - return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfOverlay) + return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.SelfOverlay}) default: return c.createGoferFilestoreInDir(ovlConf) } @@ -977,11 +977,11 @@ func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc strin func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.MountHint, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) { mountSrcInfo, err := os.Stat(mountSrc) if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) } if !mountSrcInfo.IsDir() { log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) - return nil, boot.MemoryOverlay, nil + return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil } // Create the self filestore file. createFlags := unix.O_RDWR | unix.O_CREAT | unix.O_CLOEXEC @@ -998,9 +998,9 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount // same sandbox, and is not shared, then the overlay option doesn't work // correctly. Because each overlay mount is independent and changes to // one are not visible to the other. - return nil, boot.VanillaGofer, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath) + return nil, boot.GoferMountConf{}, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath) } - return nil, boot.VanillaGofer, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) } log.Debugf("Created filestore file at %q for mount source %q", filestorePath, mountSrc) // Filestore in self should be a named path because it needs to be @@ -1015,10 +1015,10 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File filestoreDir := ovlConf.HostFileDir() fileInfo, err := os.Stat(filestoreDir) if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) } if !fileInfo.IsDir() { - return nil, boot.VanillaGofer, fmt.Errorf("overlay2 flag should specify an existing directory") + return nil, boot.GoferMountConf{}, fmt.Errorf("overlay2 flag should specify an existing directory") } // Create an unnamed temporary file in filestore directory which will be // deleted when the last FD on it is closed. We don't use O_TMPFILE because @@ -1027,13 +1027,13 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File // This file will be deleted when the container exits. filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-filestore-") if err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) } if err := unix.Unlink(filestoreFile.Name()); err != nil { - return nil, boot.VanillaGofer, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) + return nil, boot.GoferMountConf{}, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) } log.Debugf("Created an unnamed filestore file at %q", filestoreDir) - return filestoreFile, boot.AnonOverlay, nil + return filestoreFile, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.AnonOverlay}, nil } // saveLocked saves the container metadata to a file.