runsc: decouple GoferMountConf to two layers

This patch decouples GoferMountConf to two layers to allow us to
configure all combinations of a gofer mount in a succinct way:

- Upper layer config: none, memory, self, anon. The upper layer
  is always tmpfs. It describes the backend for tmpfs.
- Lower layer config: none, lisafs. It describes the backend for
  the filesystem which actually holds the image contents.

The old SelfTmpfs will be represented as "upper=self,lower=none",
MemoryOverlay will be "upper=memory,lower=lisafs", SelfOverlay
will be "upper=self,lower=lisafs", and so on. Thanks to @ayushr2
for the suggestion on how to better decouple this.

This is a preparation for adding the EROFS rootfs support. There
is no functional change intended.

Signed-off-by: Tiwei Bie <tiwei.btw@antgroup.com>
This commit is contained in:
Tiwei Bie
2023-11-05 09:02:22 +08:00
parent 42b69d0151
commit da2b10e207
6 changed files with 265 additions and 96 deletions
+134 -28
View File
@@ -16,52 +16,161 @@ package boot
import (
"fmt"
"strconv"
"strings"
)
// GoferMountConf describes how a gofer mount is configured in the sandbox.
type GoferMountConf int
// GoferMountConfUpperType describes how upper layer is configured for the gofer mount.
type GoferMountConfUpperType byte
const (
// VanillaGofer indicates that this gofer mount has no special configuration.
VanillaGofer GoferMountConf = iota
// NoOverlay indicates that this gofer mount has no upper layer. In this case,
// this gofer mount must have a lower layer (i.e. lower != NoneLower).
NoOverlay GoferMountConfUpperType = iota
// MemoryOverlay indicates that this gofer mount should be overlaid with an
// overlayfs backed by application memory.
// MemoryOverlay indicates that this gofer mount should be overlaid with a
// tmpfs backed by application memory.
MemoryOverlay
// SelfOverlay indicates that this gofer mount should be overlaid with an
// overlayfs backed by a host file in the mount's source directory.
// SelfOverlay indicates that this gofer mount should be overlaid with a
// tmpfs backed by a host file in the mount's source directory.
SelfOverlay
// AnonOverlay indicates that this gofer mount should be overlaid with an
// overlayfs backed by a host file in an anonymous directory.
// AnonOverlay indicates that this gofer mount should be overlaid with a
// tmpfs backed by a host file in an anonymous directory.
AnonOverlay
// SelfTmpfs indicates that this gofer mount should be overlaid with a tmpfs
// mount backed by a host file in the mount's source directory.
SelfTmpfs
// UpperMax indicates the number of the valid upper layer types.
UpperMax
)
// String returns a human-readable string representing the upper layer type.
func (u GoferMountConfUpperType) String() string {
switch u {
case NoOverlay:
return "none"
case MemoryOverlay:
return "memory"
case SelfOverlay:
return "self"
case AnonOverlay:
return "anon"
}
panic(fmt.Sprintf("Invalid gofer mount config upper layer type: %d", u))
}
// Set sets the value. Set(String()) should be idempotent.
func (u *GoferMountConfUpperType) Set(v string) error {
switch v {
case "none":
*u = NoOverlay
case "memory":
*u = MemoryOverlay
case "self":
*u = SelfOverlay
case "anon":
*u = AnonOverlay
default:
return fmt.Errorf("invalid gofer mount config upper layer type: %s", v)
}
return nil
}
// GoferMountConfLowerType describes how lower layer is configured for the gofer mount.
type GoferMountConfLowerType byte
const (
// NoneLower indicates that this gofer mount has no lower layer.
NoneLower GoferMountConfLowerType = iota
// Lisafs indicates that this gofer mount has a LISAFS lower layer.
Lisafs
// LowerMax indicates the number of the valid lower layer types.
LowerMax
)
// String returns a human-readable string representing the lower layer type.
func (l GoferMountConfLowerType) String() string {
switch l {
case NoneLower:
return "none"
case Lisafs:
return "lisafs"
}
panic(fmt.Sprintf("Invalid gofer mount config lower layer type: %d", l))
}
// Set sets the value. Set(String()) should be idempotent.
func (l *GoferMountConfLowerType) Set(v string) error {
switch v {
case "none":
*l = NoneLower
case "lisafs":
*l = Lisafs
default:
return fmt.Errorf("invalid gofer mount config lower layer type: %s", v)
}
return nil
}
// GoferMountConf describes how a gofer mount is configured in the sandbox.
type GoferMountConf struct {
Upper GoferMountConfUpperType `json:"upper"`
Lower GoferMountConfLowerType `json:"lower"`
}
// String returns a human-readable string representing the gofer mount config.
func (g GoferMountConf) String() string {
return fmt.Sprintf("%s:%s", g.Lower, g.Upper)
}
// Set sets the value. Set(String()) should be idempotent.
func (g *GoferMountConf) Set(v string) error {
parts := strings.Split(v, ":")
if len(parts) != 2 {
return fmt.Errorf("invalid gofer mount config format: %q", v)
}
if err := g.Lower.Set(parts[0]); err != nil {
return err
}
if err := g.Upper.Set(parts[1]); err != nil {
return err
}
if !g.valid() {
return fmt.Errorf("invalid gofer mount config: %+v", g)
}
return nil
}
// IsFilestorePresent returns true if a filestore file was associated with this.
func (g GoferMountConf) IsFilestorePresent() bool {
return g == SelfOverlay || g == AnonOverlay || g == SelfTmpfs
return g.Upper == SelfOverlay || g.Upper == AnonOverlay
}
// IsSelfBacked returns true if this mount is backed by a filestore in itself.
func (g GoferMountConf) IsSelfBacked() bool {
return g == SelfOverlay || g == SelfTmpfs
return g.Upper == SelfOverlay
}
// ShouldUseOverlayfs returns true if an overlayfs should be applied.
func (g GoferMountConf) ShouldUseOverlayfs() bool {
return g == MemoryOverlay || g == SelfOverlay || g == AnonOverlay
return g.Lower != NoneLower && g.Upper != NoOverlay
}
// ShouldUseTmpfs returns true if a tmpfs should be applied.
func (g GoferMountConf) ShouldUseTmpfs() bool {
// g.valid() implies that g.Upper != NoOverlay.
return g.Lower == NoneLower
}
// ShouldUseLisafs returns true if a lisafs client/server should be set up.
func (g GoferMountConf) ShouldUseLisafs() bool {
return g == VanillaGofer || g.ShouldUseOverlayfs()
return g.Lower == Lisafs
}
// valid returns true if this is a valid gofer mount config.
func (g GoferMountConf) valid() bool {
return g.Lower < LowerMax && g.Upper < UpperMax && (g.Lower != NoneLower || g.Upper != NoOverlay)
}
// GoferMountConfFlags can be used with GoferMountConf flags that appear
@@ -70,11 +179,11 @@ type GoferMountConfFlags []GoferMountConf
// String implements flag.Value.
func (g *GoferMountConfFlags) String() string {
confVals := make([]string, 0, len(*g))
confs := make([]string, 0, len(*g))
for _, confVal := range *g {
confVals = append(confVals, strconv.Itoa(int(confVal)))
confs = append(confs, confVal.String())
}
return strings.Join(confVals, ",")
return strings.Join(confs, ",")
}
// Get implements flag.Value.
@@ -92,14 +201,11 @@ func (g *GoferMountConfFlags) GetArray() []GoferMountConf {
func (g *GoferMountConfFlags) Set(s string) error {
confs := strings.Split(s, ",")
for _, conf := range confs {
confVal, err := strconv.Atoi(conf)
if err != nil {
return fmt.Errorf("invalid GoferMountConf value (%d): %v", confVal, err)
var confVal GoferMountConf
if err := confVal.Set(conf); err != nil {
return fmt.Errorf("invalid GoferMountConf value (%s): %v", conf, err)
}
if confVal > int(SelfTmpfs) {
return fmt.Errorf("invalid GoferMountConf value (%d)", confVal)
}
*g = append(*g, GoferMountConf(confVal))
*g = append(*g, confVal)
}
return nil
}
+78 -26
View File
@@ -20,51 +20,103 @@ import (
func TestGoferConf(t *testing.T) {
tcs := []struct {
ovl GoferMountConf
cfg GoferMountConf
wantOverlay bool
wantHostFile bool
wantLisafs bool
wantTmpfs bool
wantValid bool
}{{
ovl: VanillaGofer,
cfg: GoferMountConf{Lower: NoneLower, Upper: NoOverlay},
// This is not a valid config.
wantValid: false,
}, {
cfg: GoferMountConf{Lower: NoneLower, Upper: MemoryOverlay},
wantOverlay: false,
wantHostFile: false,
wantLisafs: true,
wantLisafs: false,
wantTmpfs: true,
wantValid: true,
}, {
ovl: MemoryOverlay,
wantOverlay: true,
wantHostFile: false,
wantLisafs: true,
}, {
ovl: SelfOverlay,
wantOverlay: true,
wantHostFile: true,
wantLisafs: true,
}, {
ovl: AnonOverlay,
wantOverlay: true,
wantHostFile: true,
wantLisafs: true,
}, {
ovl: SelfTmpfs,
cfg: GoferMountConf{Lower: NoneLower, Upper: SelfOverlay},
wantOverlay: false,
wantHostFile: true,
wantLisafs: false,
wantTmpfs: true,
wantValid: true,
}, {
cfg: GoferMountConf{Lower: NoneLower, Upper: AnonOverlay},
wantOverlay: false,
wantHostFile: true,
wantLisafs: false,
wantTmpfs: true,
wantValid: true,
}, {
cfg: GoferMountConf{Lower: Lisafs, Upper: NoOverlay},
wantOverlay: false,
wantHostFile: false,
wantLisafs: true,
wantTmpfs: false,
wantValid: true,
}, {
cfg: GoferMountConf{Lower: Lisafs, Upper: MemoryOverlay},
wantOverlay: true,
wantHostFile: false,
wantLisafs: true,
wantTmpfs: false,
wantValid: true,
}, {
cfg: GoferMountConf{Lower: Lisafs, Upper: SelfOverlay},
wantOverlay: true,
wantHostFile: true,
wantLisafs: true,
wantTmpfs: false,
wantValid: true,
}, {
cfg: GoferMountConf{Lower: Lisafs, Upper: AnonOverlay},
wantOverlay: true,
wantHostFile: true,
wantLisafs: true,
wantTmpfs: false,
wantValid: true,
}, {
cfg: GoferMountConf{Lower: LowerMax, Upper: UpperMax},
// This is not a valid config.
wantValid: false,
}}
for _, tc := range tcs {
if got := tc.ovl.ShouldUseOverlayfs(); got != tc.wantOverlay {
t.Errorf("gofer conf = %d, ShouldUseOverlayfs() = %t, want = %t", tc.ovl, got, tc.wantOverlay)
if got := tc.cfg.valid(); got != tc.wantValid {
t.Errorf("gofer conf = %+v, valid() = %t, want = %t", tc.cfg, got, tc.wantValid)
}
if got := tc.ovl.IsFilestorePresent(); got != tc.wantHostFile {
t.Errorf("gofer conf = %d, IsFilestorePresent() = %t, want = %t", tc.ovl, got, tc.wantHostFile)
if !tc.wantValid {
// Skip the following tests, if this is not a valid config.
continue
}
if got := tc.ovl.ShouldUseLisafs(); got != tc.wantLisafs {
t.Errorf("gofer conf = %d, ShouldUseLisafs() = %t, want = %t", tc.ovl, got, tc.wantLisafs)
if got := tc.cfg.ShouldUseOverlayfs(); got != tc.wantOverlay {
t.Errorf("gofer conf = %+v, ShouldUseOverlayfs() = %t, want = %t", tc.cfg, got, tc.wantOverlay)
}
if got := tc.cfg.IsFilestorePresent(); got != tc.wantHostFile {
t.Errorf("gofer conf = %+v, IsFilestorePresent() = %t, want = %t", tc.cfg, got, tc.wantHostFile)
}
if got := tc.cfg.ShouldUseLisafs(); got != tc.wantLisafs {
t.Errorf("gofer conf = %+v, ShouldUseLisafs() = %t, want = %t", tc.cfg, got, tc.wantLisafs)
}
if got := tc.cfg.ShouldUseTmpfs(); got != tc.wantTmpfs {
t.Errorf("gofer conf = %+v, ShouldUseTmpfs() = %t, want = %t", tc.cfg, got, tc.wantTmpfs)
}
}
}
func TestGoferConfFlags(t *testing.T) {
want := GoferMountConfFlags{VanillaGofer, MemoryOverlay, SelfOverlay, AnonOverlay, SelfTmpfs}
want := GoferMountConfFlags{
{Lower: NoneLower, Upper: MemoryOverlay},
{Lower: NoneLower, Upper: SelfOverlay},
{Lower: NoneLower, Upper: AnonOverlay},
{Lower: Lisafs, Upper: NoOverlay},
{Lower: Lisafs, Upper: MemoryOverlay},
{Lower: Lisafs, Upper: SelfOverlay},
{Lower: Lisafs, Upper: AnonOverlay},
}
var got GoferMountConfFlags
got.Set(want.String())
if len(got) != len(want) {
+1 -1
View File
@@ -140,7 +140,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error
ControllerFD: fd,
GoferFDs: []int{sandEnd},
StdioFDs: stdio,
GoferMountConfs: []GoferMountConf{VanillaGofer},
GoferMountConfs: []GoferMountConf{{Lower: Lisafs, Upper: NoOverlay}},
PodInitConfigFD: -1,
ExecFD: -1,
}
+37 -27
View File
@@ -447,33 +447,43 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre
// createMountNamespace creates the container's root mount and namespace.
func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) {
ioFD := c.goferFDs.remove()
data := goferMountData(ioFD, conf.FileAccess, conf)
// We can't check for overlayfs here because sandbox is chroot'ed and gofer
// can only send mount options for specs.Mounts (specs.Root is missing
// Options field). So assume root is always on top of overlayfs.
data = append(data, "overlayfs_stale_read")
// Configure the gofer dentry cache size.
gofer.SetDentryCacheSize(conf.DCache)
log.Infof("Mounting root with gofer, ioFD: %d", ioFD)
opts := &vfs.MountOptions{
ReadOnly: c.root.Readonly,
GetFilesystemOptions: vfs.GetFilesystemOptions{
InternalMount: true,
Data: strings.Join(data, ","),
InternalData: gofer.InternalFilesystemOptions{
UniqueID: "/",
},
},
}
fsName := gofer.Name
rootfsConf := c.goferMountConfs[0]
if rootfsConf == SelfTmpfs {
panic("SelfTmpfs is not possible for rootfs")
var (
fsName string
opts *vfs.MountOptions
)
switch {
case rootfsConf.ShouldUseLisafs():
fsName = gofer.Name
data := goferMountData(ioFD, conf.FileAccess, conf)
// We can't check for overlayfs here because sandbox is chroot'ed and gofer
// can only send mount options for specs.Mounts (specs.Root is missing
// Options field). So assume root is always on top of overlayfs.
data = append(data, "overlayfs_stale_read")
// Configure the gofer dentry cache size.
gofer.SetDentryCacheSize(conf.DCache)
opts = &vfs.MountOptions{
ReadOnly: c.root.Readonly,
GetFilesystemOptions: vfs.GetFilesystemOptions{
InternalMount: true,
Data: strings.Join(data, ","),
InternalData: gofer.InternalFilesystemOptions{
UniqueID: "/",
},
},
}
default:
return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf)
}
log.Infof("Mounting root with %s, ioFD: %d", fsName, ioFD)
if rootfsConf.ShouldUseOverlayfs() {
log.Infof("Adding overlay on top of root")
var (
@@ -608,7 +618,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
}
// We need to hide the filestore from the containerized application.
if mountConf == SelfOverlay {
if mountConf.IsSelfBacked() {
if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{
Root: upperRootVD,
Start: upperRootVD,
@@ -721,7 +731,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
if info.goferMountConf.IsFilestorePresent() {
info.filestoreFD = c.goferFilestoreFDs.removeAsFD()
}
if info.goferMountConf == SelfTmpfs {
if info.goferMountConf.ShouldUseTmpfs() {
specutils.ChangeMountType(info.mount, tmpfs.Name)
}
goferMntIdx++
+1
View File
@@ -31,6 +31,7 @@ var fakeFlagValues = [...]string{
"2h45m",
"1:1,2:2",
"0 0 1,100000 100000 65536",
"lisafs:self,lisafs:none",
}
func dupFlag(t *testing.T, cmd subcommands.Command, flagName string) *flag.Flag {
+14 -14
View File
@@ -957,18 +957,18 @@ func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc strin
switch hint.Mount.Type {
case "tmpfs":
// Create self-backed tmpfs.
return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfTmpfs)
return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.NoneLower, Upper: boot.SelfOverlay})
default:
return nil, boot.VanillaGofer, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type)
return nil, boot.GoferMountConf{}, fmt.Errorf("unsupported mount type %q in mount hint", hint.Mount.Type)
}
}
switch {
case !shouldOverlay:
return nil, boot.VanillaGofer, nil
return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.NoOverlay}, nil
case ovlConf.IsBackedByMemory():
return nil, boot.MemoryOverlay, nil
return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil
case ovlConf.IsBackedBySelf():
return c.createGoferFilestoreInSelf(mountSrc, hint, boot.SelfOverlay)
return c.createGoferFilestoreInSelf(mountSrc, hint, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.SelfOverlay})
default:
return c.createGoferFilestoreInDir(ovlConf)
}
@@ -977,11 +977,11 @@ func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc strin
func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.MountHint, successConf boot.GoferMountConf) (*os.File, boot.GoferMountConf, error) {
mountSrcInfo, err := os.Stat(mountSrc)
if err != nil {
return nil, boot.VanillaGofer, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err)
return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err)
}
if !mountSrcInfo.IsDir() {
log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc)
return nil, boot.MemoryOverlay, nil
return nil, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.MemoryOverlay}, nil
}
// Create the self filestore file.
createFlags := unix.O_RDWR | unix.O_CREAT | unix.O_CLOEXEC
@@ -998,9 +998,9 @@ func (c *Container) createGoferFilestoreInSelf(mountSrc string, hint *boot.Mount
// same sandbox, and is not shared, then the overlay option doesn't work
// correctly. Because each overlay mount is independent and changes to
// one are not visible to the other.
return nil, boot.VanillaGofer, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath)
return nil, boot.GoferMountConf{}, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not supported with overlay optimizations", mountSrc, filestorePath)
}
return nil, boot.VanillaGofer, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err)
return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err)
}
log.Debugf("Created filestore file at %q for mount source %q", filestorePath, mountSrc)
// Filestore in self should be a named path because it needs to be
@@ -1015,10 +1015,10 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File
filestoreDir := ovlConf.HostFileDir()
fileInfo, err := os.Stat(filestoreDir)
if err != nil {
return nil, boot.VanillaGofer, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err)
return nil, boot.GoferMountConf{}, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err)
}
if !fileInfo.IsDir() {
return nil, boot.VanillaGofer, fmt.Errorf("overlay2 flag should specify an existing directory")
return nil, boot.GoferMountConf{}, fmt.Errorf("overlay2 flag should specify an existing directory")
}
// Create an unnamed temporary file in filestore directory which will be
// deleted when the last FD on it is closed. We don't use O_TMPFILE because
@@ -1027,13 +1027,13 @@ func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File
// This file will be deleted when the container exits.
filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-filestore-")
if err != nil {
return nil, boot.VanillaGofer, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err)
return nil, boot.GoferMountConf{}, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err)
}
if err := unix.Unlink(filestoreFile.Name()); err != nil {
return nil, boot.VanillaGofer, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err)
return nil, boot.GoferMountConf{}, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err)
}
log.Debugf("Created an unnamed filestore file at %q", filestoreDir)
return filestoreFile, boot.AnonOverlay, nil
return filestoreFile, boot.GoferMountConf{Lower: boot.Lisafs, Upper: boot.AnonOverlay}, nil
}
// saveLocked saves the container metadata to a file.