diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 2654913b9..f2bc904f4 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -14,12 +14,12 @@ go_library( "controller.go", "debug.go", "events.go", + "gofer_conf.go", "limits.go", "loader.go", "mount_hints.go", "network.go", "nvidia.go", - "overlay.go", "seccheck.go", "strace.go", "vfs.go", @@ -135,9 +135,9 @@ go_test( size = "small", srcs = [ "compat_test.go", + "gofer_conf_test.go", "loader_test.go", "mount_hints_test.go", - "overlay_test.go", "vfs_test.go", ], library = ":boot", diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index b755f5c53..abc55425e 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -273,18 +273,17 @@ type StartArgs struct { // CID is the ID of the container to start. CID string - // NumOverlayFilestoreFDs is the number of overlay filestore FDs donated. - // Optionally configured with the overlay2 flag. - NumOverlayFilestoreFDs int + // NumGoferFilestoreFDs is the number of gofer filestore FDs donated. + NumGoferFilestoreFDs int - // OverlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in Spec.Mounts (in the same order). - OverlayMediums []OverlayMedium + // GoferMountConfs contains information about how the gofer mounts have been + // configured. The first entry is for rootfs and the following entries are + // for bind mounts in Spec.Mounts (in the same order). + GoferMountConfs []GoferMountConf // FilePayload contains, in order: // * stdin, stdout, and stderr (optional: if terminal is disabled). - // * file descriptors to overlay-backing host files (optional: for overlay2). + // * file descriptors to gofer-backing host files (optional). // * file descriptors to connect to gofer to serve the root filesystem. urpc.FilePayload } @@ -306,7 +305,7 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro return errors.New("start argument missing container ID") } expectedFDs := 1 // At least one FD for the root filesystem. - expectedFDs += args.NumOverlayFilestoreFDs + expectedFDs += args.NumGoferFilestoreFDs if !args.Spec.Process.Terminal { expectedFDs += 3 } @@ -335,15 +334,15 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro } }() - var overlayFilestoreFDs []*fd.FD - for i := 0; i < args.NumOverlayFilestoreFDs; i++ { - overlayFilestoreFD, err := fd.NewFromFile(goferFiles[i]) + var goferFilestoreFDs []*fd.FD + for i := 0; i < args.NumGoferFilestoreFDs; i++ { + goferFilestoreFD, err := fd.NewFromFile(goferFiles[i]) if err != nil { - return fmt.Errorf("error dup'ing overlay filestore file: %w", err) + return fmt.Errorf("error dup'ing gofer filestore file: %w", err) } - overlayFilestoreFDs = append(overlayFilestoreFDs, overlayFilestoreFD) + goferFilestoreFDs = append(goferFilestoreFDs, goferFilestoreFD) } - goferFiles = goferFiles[args.NumOverlayFilestoreFDs:] + goferFiles = goferFiles[args.NumGoferFilestoreFDs:] goferFDs, err := fd.NewFromFiles(goferFiles) if err != nil { @@ -355,7 +354,7 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro } }() - if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, overlayFilestoreFDs, args.OverlayMediums); err != nil { + if err := cm.l.startSubcontainer(args.Spec, args.Conf, args.CID, stdios, goferFDs, goferFilestoreFDs, args.GoferMountConfs); err != nil { log.Debugf("containerManager.StartSubcontainer failed, cid: %s, args: %+v, err: %v", args.CID, args, err) return err } diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go new file mode 100644 index 000000000..4801bf30a --- /dev/null +++ b/runsc/boot/gofer_conf.go @@ -0,0 +1,96 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boot + +import ( + "fmt" + "strconv" + "strings" +) + +// GoferMountConf describes how a gofer mount is configured in the sandbox. +type GoferMountConf int + +const ( + // VanillaGofer indicates that this gofer mount has no special configuration. + VanillaGofer GoferMountConf = iota + + // MemoryOverlay indicates that this gofer mount should be overlaid with an + // overlayfs backed by application memory. + MemoryOverlay + + // SelfOverlay indicates that this gofer mount should be overlaid with an + // overlayfs backed by a host file in the mount's source directory. + SelfOverlay + + // AnonOverlay indicates that this gofer mount should be overlaid with an + // overlayfs backed by a host file in an anonymous directory. + AnonOverlay +) + +// IsFilestorePresent returns true if a filestore file was associated with this. +func (g GoferMountConf) IsFilestorePresent() bool { + return g == SelfOverlay || g == AnonOverlay +} + +// IsSelfBacked returns true if this mount is backed by a filestore in itself. +func (g GoferMountConf) IsSelfBacked() bool { + return g == SelfOverlay +} + +// ShouldUseOverlayfs returns true if an overlayfs should be applied. +func (g GoferMountConf) ShouldUseOverlayfs() bool { + return g == MemoryOverlay || g == SelfOverlay || g == AnonOverlay +} + +// GoferMountConfFlags can be used with GoferMountConf flags that appear +// multiple times. +type GoferMountConfFlags []GoferMountConf + +// String implements flag.Value. +func (g *GoferMountConfFlags) String() string { + confVals := make([]string, 0, len(*g)) + for _, confVal := range *g { + confVals = append(confVals, strconv.Itoa(int(confVal))) + } + return strings.Join(confVals, ",") +} + +// Get implements flag.Value. +func (g *GoferMountConfFlags) Get() any { + return g +} + +// GetArray returns an array of mappings. +func (g *GoferMountConfFlags) GetArray() []GoferMountConf { + return *g +} + +// Set implements flag.Value and appends a gofer configuration from the command +// line to the configs array. Set(String()) should be idempotent. +func (g *GoferMountConfFlags) Set(s string) error { + confs := strings.Split(s, ",") + for _, conf := range confs { + confVal, err := strconv.Atoi(conf) + if err != nil { + return fmt.Errorf("invalid GoferMountConf value (%d): %v", confVal, err) + } + if confVal > int(AnonOverlay) { + return fmt.Errorf("invalid GoferMountConf value (%d)", confVal) + } + *g = append(*g, GoferMountConf(confVal)) + } + return nil +} diff --git a/runsc/boot/overlay_test.go b/runsc/boot/gofer_conf_test.go similarity index 50% rename from runsc/boot/overlay_test.go rename to runsc/boot/gofer_conf_test.go index 96b53e8d0..269278d36 100644 --- a/runsc/boot/overlay_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -18,48 +18,48 @@ import ( "testing" ) -func TestOverlayMedium(t *testing.T) { +func TestGoferConf(t *testing.T) { tcs := []struct { - ovl OverlayMedium - wantEnabled bool + ovl GoferMountConf + wantOverlay bool wantHostFile bool }{{ - ovl: NoOverlay, - wantEnabled: false, + ovl: VanillaGofer, + wantOverlay: false, wantHostFile: false, }, { - ovl: MemoryMedium, - wantEnabled: true, + ovl: MemoryOverlay, + wantOverlay: true, wantHostFile: false, }, { - ovl: SelfMedium, - wantEnabled: true, + ovl: SelfOverlay, + wantOverlay: true, wantHostFile: true, }, { - ovl: AnonDirMedium, - wantEnabled: true, + ovl: AnonOverlay, + wantOverlay: true, wantHostFile: true, }} for _, tc := range tcs { - if got := tc.ovl.IsEnabled(); got != tc.wantEnabled { - t.Errorf("overlay medium = %d, IsEnabled() = %t, want = %t", tc.ovl, got, tc.wantEnabled) + if got := tc.ovl.ShouldUseOverlayfs(); got != tc.wantOverlay { + t.Errorf("gofer conf = %d, ShouldUseOverlayfs() = %t, want = %t", tc.ovl, got, tc.wantOverlay) } - if got := tc.ovl.IsBackedByHostFile(); got != tc.wantHostFile { - t.Errorf("overlay medium = %d, IsBackedByHostFile() = %t, want = %t", tc.ovl, got, tc.wantHostFile) + if got := tc.ovl.IsFilestorePresent(); got != tc.wantHostFile { + t.Errorf("gofer conf = %d, IsFilestorePresent() = %t, want = %t", tc.ovl, got, tc.wantHostFile) } } } -func TestOverlayMediumFlags(t *testing.T) { - want := OverlayMediumFlags{MemoryMedium, SelfMedium, AnonDirMedium, NoOverlay} - var got OverlayMediumFlags +func TestGoferConfFlags(t *testing.T) { + want := GoferMountConfFlags{VanillaGofer, MemoryOverlay, SelfOverlay, AnonOverlay} + var got GoferMountConfFlags got.Set(want.String()) if len(got) != len(want) { - t.Fatalf("overlay medium flags is incorrect length: want = %d, got = %d", len(want), len(got)) + t.Fatalf("gofer conf flags is incorrect length: want = %d, got = %d", len(want), len(got)) } for i := range want { if want[i] != got[i] { - t.Errorf("overlay medium is incorrect: want = %d, got = %d", want[i], got[i]) + t.Errorf("gofer conf is incorrect: want = %d, got = %d", want[i], got[i]) } } } diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 23ac4eaf3..6cb3cd282 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -112,14 +112,14 @@ type containerInfo struct { // goferFDs are the FDs that attach the sandbox to the gofers. goferFDs []*fd.FD - // overlayFilestoreFDs are the FDs to the regular files that will back the - // tmpfs upper mount in the overlay mounts. - overlayFilestoreFDs []*fd.FD + // goferFilestoreFDs are FDs to the regular files that will back the tmpfs or + // overlayfs mount for certain gofer mounts. + goferFilestoreFDs []*fd.FD - // overlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in spec.Mounts (in the same order). - overlayMediums []OverlayMedium + // goferMountConfs contains information about how the gofer mounts have been + // configured. The first entry is for rootfs and the following entries are + // for bind mounts in Spec.Mounts (in the same order). + goferMountConfs []GoferMountConf // nvidiaUVMDevMajor is the device major number used for nvidia-uvm. nvidiaUVMDevMajor uint32 @@ -254,13 +254,13 @@ type Args struct { PassFDs []FDMapping // ExecFD is the host file descriptor used for program execution. ExecFD int - // OverlayFilestoreFDs are the FDs to the regular files that will back the - // tmpfs upper mount in the overlay mounts. - OverlayFilestoreFDs []int - // OverlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in Spec.Mounts (in the same order). - OverlayMediums []OverlayMedium + // GoferFilestoreFDs are FDs to the regular files that will back the tmpfs or + // overlayfs mount for certain gofer mounts. + GoferFilestoreFDs []int + // GoferMountConfs contains information about how the gofer mounts have been + // configured. The first entry is for rootfs and the following entries are + // for bind mounts in Spec.Mounts (in the same order). + GoferMountConfs []GoferMountConf // NumCPU is the number of CPUs to create inside the sandbox. NumCPU int // TotalMem is the initial amount of total memory to report back to the @@ -312,9 +312,9 @@ func New(args Args) (*Loader, error) { kernel.IOUringEnabled = args.Conf.IOUring info := containerInfo{ - conf: args.Conf, - spec: args.Spec, - overlayMediums: args.OverlayMediums, + conf: args.Conf, + spec: args.Spec, + goferMountConfs: args.GoferMountConfs, } // Make host FDs stable between invocations. Host FDs must map to the exact @@ -342,8 +342,8 @@ func New(args Args) (*Loader, error) { for _, goferFD := range args.GoferFDs { info.goferFDs = append(info.goferFDs, fd.New(goferFD)) } - for _, overlayFD := range args.OverlayFilestoreFDs { - info.overlayFilestoreFDs = append(info.overlayFilestoreFDs, fd.New(overlayFD)) + for _, filestoreFD := range args.GoferFilestoreFDs { + info.goferFilestoreFDs = append(info.goferFilestoreFDs, fd.New(filestoreFD)) } if args.ExecFD >= 0 { @@ -794,7 +794,7 @@ func (l *Loader) createSubcontainer(cid string, tty *fd.FD) error { // startSubcontainer starts a child container. It returns the thread group ID of // the newly created process. Used FDs are either closed or released. It's safe // for the caller to close any remaining files upon return. -func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, overlayFilestoreFDs []*fd.FD, overlayMediums []OverlayMedium) error { +func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, goferMountConfs []GoferMountConf) error { // Create capabilities. caps, err := specutils.Capabilities(conf.EnableRaw, spec.Process.Capabilities) if err != nil { @@ -847,12 +847,12 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st } info := &containerInfo{ - conf: conf, - spec: spec, - goferFDs: goferFDs, - overlayFilestoreFDs: overlayFilestoreFDs, - overlayMediums: overlayMediums, - nvidiaUVMDevMajor: l.nvidiaUVMDevMajor, + conf: conf, + spec: spec, + goferFDs: goferFDs, + goferFilestoreFDs: goferFilestoreFDs, + goferMountConfs: goferMountConfs, + nvidiaUVMDevMajor: l.nvidiaUVMDevMajor, } info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns) if err != nil { diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index e462357c8..304b1c156 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -140,7 +140,7 @@ func createLoader(conf *config.Config, spec *specs.Spec) (*Loader, func(), error ControllerFD: fd, GoferFDs: []int{sandEnd}, StdioFDs: stdio, - OverlayMediums: []OverlayMedium{NoOverlay}, + GoferMountConfs: []GoferMountConf{VanillaGofer}, PodInitConfigFD: -1, ExecFD: -1, } diff --git a/runsc/boot/overlay.go b/runsc/boot/overlay.go deleted file mode 100644 index f885bdff8..000000000 --- a/runsc/boot/overlay.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2023 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package boot - -import ( - "fmt" - "strconv" - "strings" -) - -// OverlayMedium describes the medium that will be used to back the -// overlay mount's upper layer. -type OverlayMedium int - -const ( - // NoOverlay indicates that this mount should not be overlaid. - NoOverlay OverlayMedium = iota - - // MemoryMedium indicates that this mount should be overlaid with an - // upper layer backed by application memory. - MemoryMedium - - // SelfMedium indicates that this mount should be overlaid with an upper - // layer backed by a host file in the mount's source directory. - SelfMedium - - // AnonDirMedium indicates that this mount should be overlaid with an upper - // layer backed by a host file in an anonymous directory. - AnonDirMedium -) - -// IsBackedByHostFile returns true if the overlay is backed by a host file. -func (o *OverlayMedium) IsBackedByHostFile() bool { - return *o == SelfMedium || *o == AnonDirMedium -} - -// IsEnabled returns true if an overlay is applied. -func (o *OverlayMedium) IsEnabled() bool { - return *o != NoOverlay -} - -// OverlayMediumFlags can be used with OverlayMedium flags that appear -// multiple times. -type OverlayMediumFlags []OverlayMedium - -// String implements flag.Value. -func (o *OverlayMediumFlags) String() string { - mediumVals := make([]string, 0, len(*o)) - for _, medium := range *o { - mediumVals = append(mediumVals, strconv.Itoa(int(medium))) - } - return strings.Join(mediumVals, ",") -} - -// Get implements flag.Value. -func (o *OverlayMediumFlags) Get() any { - return o -} - -// GetArray returns an array of mappings. -func (o *OverlayMediumFlags) GetArray() []OverlayMedium { - return *o -} - -// Set implements flag.Value and appends an overlay medium from the command -// line to the mediums array. Set(String()) should be idempotent. -func (o *OverlayMediumFlags) Set(s string) error { - mediums := strings.Split(s, ",") - for _, medium := range mediums { - mediumVal, err := strconv.Atoi(medium) - if err != nil { - return fmt.Errorf("invalid OverlayMedium value (%d): %v", mediumVal, err) - } - if mediumVal > int(AnonDirMedium) { - return fmt.Errorf("invalid OverlayMedium value (%d)", mediumVal) - } - *o = append(*o, OverlayMedium(mediumVal)) - } - return nil -} diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 9a2ebfcd0..84b157bf5 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -62,22 +62,21 @@ const ( Nonefs = "none" ) -// SelfOverlayFilestorePrefix is the prefix in the file name of the -// self overlay filestore file. -const SelfOverlayFilestorePrefix = ".gvisor.overlay.img." +// SelfFilestorePrefix is the prefix of the self filestore file name. +const SelfFilestorePrefix = ".gvisor.filestore." -// SelfOverlayFilestorePath returns the path at which the self overlay -// filestore file is stored for a given mount. -func SelfOverlayFilestorePath(mountSrc, sandboxID string) string { +// SelfFilestorePath returns the path at which the self filestore file is +// stored for a given mount. +func SelfFilestorePath(mountSrc, sandboxID string) string { // We will place the filestore file in a gVisor specific hidden file inside - // the mount being overlay-ed itself. The same volume can be overlay-ed by + // the mount being overlaid itself. The same volume can be overlaid by // multiple sandboxes. So make the filestore file unique to a sandbox by // suffixing the sandbox ID. - return path.Join(mountSrc, selfOverlayFilestoreName(sandboxID)) + return path.Join(mountSrc, selfFilestoreName(sandboxID)) } -func selfOverlayFilestoreName(sandboxID string) string { - return SelfOverlayFilestorePrefix + sandboxID +func selfFilestoreName(sandboxID string) string { + return SelfFilestorePrefix + sandboxID } // tmpfs has some extra supported options that we must pass through. @@ -378,17 +377,17 @@ type containerMounter struct { // that may be freely modified without affecting the original spec. mounts []specs.Mount - // fds is the list of FDs to be dispensed for mounts that require it. - fds fdDispenser + // goferFDs is the list of FDs to be dispensed for gofer mounts. + goferFDs fdDispenser - // overlayFilestoreFDs are the FDs to the regular files that will back the - // tmpfs upper mount in the overlay mounts. - overlayFilestoreFDs fdDispenser + // goferFilestoreFDs are FDs to the regular files that will back the tmpfs or + // overlayfs mount for certain gofer mounts. + goferFilestoreFDs fdDispenser - // overlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in `mounts` slice above (in the same order). - overlayMediums []OverlayMedium + // goferMountConfs contains information about how the gofer mounts have been + // configured. The first entry is for rootfs and the following entries are + // for bind mounts in Spec.Mounts (in the same order). + goferMountConfs []GoferMountConf k *kernel.Kernel @@ -409,22 +408,22 @@ type containerMounter struct { func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountHints, sharedMounts map[string]*vfs.Mount, productName string, sandboxID string) *containerMounter { return &containerMounter{ - root: info.spec.Root, - mounts: compileMounts(info.spec, info.conf), - fds: fdDispenser{fds: info.goferFDs}, - overlayFilestoreFDs: fdDispenser{fds: info.overlayFilestoreFDs}, - overlayMediums: info.overlayMediums, - k: k, - hints: hints, - sharedMounts: sharedMounts, - productName: productName, - sandboxID: sandboxID, + root: info.spec.Root, + mounts: compileMounts(info.spec, info.conf), + goferFDs: fdDispenser{fds: info.goferFDs}, + goferFilestoreFDs: fdDispenser{fds: info.goferFilestoreFDs}, + goferMountConfs: info.goferMountConfs, + k: k, + hints: hints, + sharedMounts: sharedMounts, + productName: productName, + sandboxID: sandboxID, } } func (c *containerMounter) checkDispenser() error { - if !c.fds.empty() { - return fmt.Errorf("not all gofer FDs were consumed, remaining: %v", c.fds) + if !c.goferFDs.empty() { + return fmt.Errorf("not all gofer FDs were consumed, remaining: %v", c.goferFDs) } return nil } @@ -470,7 +469,7 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre // createMountNamespace creates the container's root mount and namespace. func (c *containerMounter) createMountNamespace(ctx context.Context, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) { - ioFD := c.fds.remove() + ioFD := c.goferFDs.remove() data := goferMountData(ioFD, conf.FileAccess, conf) // We can't check for overlayfs here because sandbox is chroot'ed and gofer @@ -494,17 +493,18 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi } fsName := gofer.Name - if c.overlayMediums[0].IsEnabled() { + rootfsConf := c.goferMountConfs[0] + if rootfsConf.ShouldUseOverlayfs() { log.Infof("Adding overlay on top of root") var ( - err error - cleanup func() - overlayFilestore *fd.FD + err error + cleanup func() + filestoreFD *fd.FD ) - if c.overlayMediums[0].IsBackedByHostFile() { - overlayFilestore = c.overlayFilestoreFDs.removeAsFD() + if rootfsConf.IsFilestorePresent() { + filestoreFD = c.goferFilestoreFDs.removeAsFD() } - opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, overlayFilestore, c.overlayMediums[0]) + opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, filestoreFD, rootfsConf) if err != nil { return nil, fmt.Errorf("mounting root with overlay: %w", err) } @@ -545,7 +545,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi // layer using tmpfs, and return overlay mount options. "cleanup" must be called // after the options have been used to mount the overlay, to release refs on // lower and upper mounts. -func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, filestoreFD *fd.FD, medium OverlayMedium) (*vfs.MountOptions, func(), error) { +func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, filestoreFD *fd.FD, mountConf GoferMountConf) (*vfs.MountOptions, func(), error) { // First copy options from lower layer to upper layer and overlay. Clear // filesystem specific options. upperOpts := *lowerOpts @@ -628,11 +628,11 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co } // We need to hide the filestore from the containerized application. - if medium == SelfMedium { + if mountConf == SelfOverlay { if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{ Root: upperRootVD, Start: upperRootVD, - Path: fspath.Parse(selfOverlayFilestoreName(c.sandboxID)), + Path: fspath.Parse(selfFilestoreName(c.sandboxID)), }); err != nil { return nil, nil, fmt.Errorf("failed to create whiteout to hide self overlay filestore: %w", err) } @@ -714,15 +714,15 @@ func (c *containerMounter) mountSubmounts(ctx context.Context, conf *config.Conf } type mountInfo struct { - mount *specs.Mount - fd int - hint *MountHint - overlayMedium OverlayMedium - overlayFilestoreFD *fd.FD + mount *specs.Mount + goferFD int + hint *MountHint + goferMountConf GoferMountConf + filestoreFD *fd.FD } func newNonGoferMountInfo(mount *specs.Mount) *mountInfo { - return &mountInfo{mount: mount, fd: -1} + return &mountInfo{mount: mount, goferFD: -1} } func (c *containerMounter) prepareMounts() ([]mountInfo, error) { @@ -738,16 +738,15 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) { // Only bind mounts use host FDs; see // containerMounter.getMountNameAndOptions. info := mountInfo{ - mount: m, - fd: -1, - hint: c.hints.FindMount(m.Source), - overlayMedium: NoOverlay, + mount: m, + goferFD: -1, + hint: c.hints.FindMount(m.Source), } if specutils.IsGoferMount(*m) { - info.fd = c.fds.remove() - info.overlayMedium = c.overlayMediums[goferMntIdx] - if info.overlayMedium.IsBackedByHostFile() { - info.overlayFilestoreFD = c.overlayFilestoreFDs.removeAsFD() + info.goferFD = c.goferFDs.remove() + info.goferMountConf = c.goferMountConfs[goferMntIdx] + if info.goferMountConf.IsFilestorePresent() { + info.filestoreFD = c.goferFilestoreFDs.removeAsFD() } goferMntIdx++ } @@ -779,10 +778,10 @@ func (c *containerMounter) mountSubmount(ctx context.Context, conf *config.Confi return nil, fmt.Errorf("creating mount point %q: %w", submount.mount.Destination, err) } - if submount.overlayMedium.IsEnabled() { + if submount.goferMountConf.ShouldUseOverlayfs() { log.Infof("Adding overlay on top of mount %q", submount.mount.Destination) var cleanup func() - opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, submount.overlayFilestoreFD, submount.overlayMedium) + opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, submount.filestoreFD, submount.goferMountConf) if err != nil { return nil, fmt.Errorf("mounting volume with overlay at %q: %w", submount.mount.Destination, err) } @@ -838,11 +837,11 @@ func getMountNameAndOptions(conf *config.Config, m *mountInfo, productName strin case Bind: fsName = gofer.Name - if m.fd < 0 { + if m.goferFD < 0 { // Check that an FD was provided to fails fast. return "", nil, fmt.Errorf("gofer mount requires a connection FD") } - data = goferMountData(m.fd, getMountAccessType(conf, m.hint), conf) + data = goferMountData(m.goferFD, getMountAccessType(conf, m.hint), conf) internalData = gofer.InternalFilesystemOptions{ UniqueID: m.mount.Destination, } @@ -1076,15 +1075,15 @@ func (c *containerMounter) makeMountPoint(ctx context.Context, creds *auth.Crede // state used by restore defined by conf. func (c *containerMounter) configureRestore(ctx context.Context) (context.Context, error) { fdmap := make(map[string]int) - fdmap["/"] = c.fds.remove() + fdmap["/"] = c.goferFDs.remove() mounts, err := c.prepareMounts() if err != nil { return ctx, err } for i := range c.mounts { submount := &mounts[i] - if submount.fd >= 0 { - fdmap[submount.mount.Destination] = submount.fd + if submount.goferFD >= 0 { + fdmap[submount.mount.Destination] = submount.goferFD } } return context.WithValue(ctx, gofer.CtxRestoreServerFDMap, fdmap), nil diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index ec4fef661..ae218a58a 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -81,14 +81,14 @@ type Boot struct { // ioFDs is the list of FDs used to connect to FS gofers. ioFDs intFlags - // overlayFilestoreFDs are FDs to the regular files that will back the tmpfs - // upper mount in the overlay mounts. - overlayFilestoreFDs intFlags + // goferFilestoreFDs are FDs to the regular files that will back the tmpfs or + // overlayfs mount for certain gofer mounts. + goferFilestoreFDs intFlags - // overlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in Spec.Mounts (in the same order). - overlayMediums boot.OverlayMediumFlags + // goferMountConfs contains information about how the gofer mounts have been + // configured. The first entry is for rootfs and the following entries are + // for bind mounts in Spec.Mounts (in the same order). + goferMountConfs boot.GoferMountConfFlags // stdioFDs are the fds for stdin, stdout, and stderr. They must be // provided in that order. @@ -198,8 +198,8 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.") f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.") - f.Var(&b.overlayFilestoreFDs, "overlay-filestore-fds", "FDs to the regular files that will back the tmpfs upper mount in the overlay mounts.") - f.Var(&b.overlayMediums, "overlay-mediums", "information about how the gofer mounts have been overlaid.") + f.Var(&b.goferFilestoreFDs, "gofer-filestore-fds", "FDs to the regular files that will back the overlayfs or tmpfs mount if a gofer mount is to be overlaid.") + f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") @@ -414,25 +414,25 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma // Create the loader. bootArgs := boot.Args{ - ID: f.Arg(0), - Spec: spec, - Conf: conf, - ControllerFD: b.controllerFD, - Device: os.NewFile(uintptr(b.deviceFD), "platform device"), - GoferFDs: b.ioFDs.GetArray(), - StdioFDs: b.stdioFDs.GetArray(), - PassFDs: b.passFDs.GetArray(), - ExecFD: b.execFD, - OverlayFilestoreFDs: b.overlayFilestoreFDs.GetArray(), - OverlayMediums: b.overlayMediums.GetArray(), - NumCPU: b.cpuNum, - TotalMem: b.totalMem, - TotalHostMem: b.totalHostMem, - UserLogFD: b.userLogFD, - ProductName: b.productName, - PodInitConfigFD: b.podInitConfigFD, - SinkFDs: b.sinkFDs.GetArray(), - ProfileOpts: b.profileFDs.ToOpts(), + ID: f.Arg(0), + Spec: spec, + Conf: conf, + ControllerFD: b.controllerFD, + Device: os.NewFile(uintptr(b.deviceFD), "platform device"), + GoferFDs: b.ioFDs.GetArray(), + StdioFDs: b.stdioFDs.GetArray(), + PassFDs: b.passFDs.GetArray(), + ExecFD: b.execFD, + GoferFilestoreFDs: b.goferFilestoreFDs.GetArray(), + GoferMountConfs: b.goferMountConfs.GetArray(), + NumCPU: b.cpuNum, + TotalMem: b.totalMem, + TotalHostMem: b.totalHostMem, + UserLogFD: b.userLogFD, + ProductName: b.productName, + PodInitConfigFD: b.podInitConfigFD, + SinkFDs: b.sinkFDs.GetArray(), + ProfileOpts: b.profileFDs.ToOpts(), } l, err := boot.New(bootArgs) if err != nil { diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index cfa09e9ea..bdf497079 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -80,11 +80,11 @@ type goferSyncFDs struct { // Gofer implements subcommands.Command for the "gofer" command, which starts a // filesystem gofer. This command should not be called directly. type Gofer struct { - bundleDir string - ioFDs intFlags - applyCaps bool - setUpRoot bool - overlayMediums boot.OverlayMediumFlags + bundleDir string + ioFDs intFlags + applyCaps bool + setUpRoot bool + mountConfs boot.GoferMountConfFlags specFD int mountsFD int @@ -116,7 +116,7 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) { // Open FDs that are donated to the gofer. f.Var(&g.ioFDs, "io-fds", "list of FDs to connect gofer servers. They must follow this order: root first, then mounts as defined in the spec") - f.Var(&g.overlayMediums, "overlay-mediums", "information about how the gofer mounts have been overlaid.") + f.Var(&g.mountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured") f.IntVar(&g.specFD, "spec-fd", -1, "required fd with the container spec") f.IntVar(&g.mountsFD, "mounts-fd", -1, "mountsFD is the file descriptor to write list of mounts after they have been resolved (direct paths, no symlinks).") @@ -271,7 +271,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom cfgs = append(cfgs, connectionConfig{ sock: newSocket(g.ioFDs[0]), mountPath: "/", // fsgofer process is always chroot()ed. So serve root. - readonly: spec.Root.Readonly || g.overlayMediums[0].IsEnabled(), + readonly: spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs(), }) log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], cfgs[0].readonly) @@ -291,7 +291,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string) subcom cfgs = append(cfgs, connectionConfig{ sock: newSocket(g.ioFDs[mountIdx]), mountPath: m.Destination, - readonly: specutils.IsReadonlyMount(m.Options) || g.overlayMediums[mountIdx].IsEnabled(), + readonly: specutils.IsReadonlyMount(m.Options) || g.mountConfs[mountIdx].ShouldUseOverlayfs(), }) log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, g.ioFDs[mountIdx], cfgs[mountIdx].readonly) @@ -418,7 +418,7 @@ func (g *Gofer) setupRootFS(spec *specs.Spec, conf *config.Config) error { } // Check if root needs to be remounted as readonly. - if spec.Root.Readonly || g.overlayMediums[0].IsEnabled() { + if spec.Root.Readonly || g.mountConfs[0].ShouldUseOverlayfs() { // If root is a mount point but not read-only, we can change mount options // to make it read-only for extra safety. // unix.MS_NOSUID and unix.MS_NODEV are included here not only @@ -459,7 +459,7 @@ func (g *Gofer) setupMounts(conf *config.Config, mounts []specs.Mount, root, pro } flags := specutils.OptionsToFlags(m.Options) | unix.MS_BIND - if g.overlayMediums[goferMntIdx].IsEnabled() { + if g.mountConfs[goferMntIdx].ShouldUseOverlayfs() { // Force mount read-only if writes are not going to be sent to it. flags |= unix.MS_RDONLY } diff --git a/runsc/container/container.go b/runsc/container/container.go index e50381e43..8819e69da 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -134,10 +134,10 @@ type Container struct { // processes. Saver StateFile `json:"saver"` - // OverlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in Spec.Mounts (in the same order). - OverlayMediums boot.OverlayMediumFlags `json:"overlayMediums"` + // GoferMountConfs contains information about how the gofer mounts have been + // overlaid (with tmpfs or overlayfs). The first entry is for rootfs and the + // following entries are for bind mounts in Spec.Mounts (in the same order). + GoferMountConfs boot.GoferMountConfFlags `json:"goferMountConfs"` // // Fields below this line are not saved in the state file and will not @@ -287,11 +287,11 @@ func New(conf *config.Config, args Args) (*Container, error) { if err != nil { return nil, fmt.Errorf("error creating pod mount hints: %w", err) } - overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores(conf.GetOverlay2(), mountHints) + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), mountHints) if err != nil { return nil, err } - c.OverlayMediums = overlayMediums + c.GoferMountConfs = goferConfs if err := nvProxyPreGoferHostSetup(args.Spec, conf); err != nil { return nil, err } @@ -304,20 +304,20 @@ func New(conf *config.Config, args Args) (*Container, error) { // Start a new sandbox for this container. Any errors after this point // must destroy the container. sandArgs := &sandbox.Args{ - ID: sandboxID, - Spec: args.Spec, - BundleDir: args.BundleDir, - ConsoleSocket: args.ConsoleSocket, - UserLog: args.UserLog, - IOFiles: ioFiles, - MountsFile: specFile, - Cgroup: containerCgroup, - Attached: args.Attached, - OverlayFilestoreFiles: overlayFilestoreFiles, - OverlayMediums: overlayMediums, - MountHints: mountHints, - PassFiles: args.PassFiles, - ExecFile: args.ExecFile, + ID: sandboxID, + Spec: args.Spec, + BundleDir: args.BundleDir, + ConsoleSocket: args.ConsoleSocket, + UserLog: args.UserLog, + IOFiles: ioFiles, + MountsFile: specFile, + Cgroup: containerCgroup, + Attached: args.Attached, + GoferFilestoreFiles: goferFilestores, + GoferMountConfs: goferConfs, + MountHints: mountHints, + PassFiles: args.PassFiles, + ExecFile: args.ExecFile, } if specutils.GPUFunctionalityRequested(args.Spec, conf) { // Expose all Nvidia devices in /dev/, because we don't know what @@ -449,11 +449,11 @@ func (c *Container) Start(conf *config.Config) error { return err } } else { - overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores(conf.GetOverlay2(), c.Sandbox.MountHints) + goferFilestores, goferConfs, err := c.createGoferFilestores(conf.GetOverlay2(), c.Sandbox.MountHints) if err != nil { return err } - c.OverlayMediums = overlayMediums + c.GoferMountConfs = goferConfs // Join cgroup to start gofer process to ensure it's part of the cgroup from // the start (and all their children processes). if err := runInCgroup(c.Sandbox.CgroupJSON.Cgroup, func() error { @@ -482,7 +482,7 @@ func (c *Container) Start(conf *config.Config) error { stdios = []*os.File{os.Stdin, os.Stdout, os.Stderr} } - return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, overlayFilestoreFiles, overlayMediums) + return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, goferConfs) }); err != nil { return err } @@ -814,8 +814,8 @@ func (c *Container) Destroy() error { errs = append(errs, err.Error()) } - // Clean up overlay filestore files created in their respective mounts. - c.forEachSelfOverlay(func(mountSrc string) { + // Clean up self-backed filestore files created in their respective mounts. + c.forEachSelfMount(func(mountSrc string) { if sb != nil { if hint := sb.MountHints.FindMount(mountSrc); hint != nil && hint.ShouldShareMount() { // Don't delete filestore file for shared mounts. The sandbox owns a @@ -824,7 +824,7 @@ func (c *Container) Destroy() error { return } } - filestorePath := boot.SelfOverlayFilestorePath(mountSrc, c.sandboxID()) + filestorePath := boot.SelfFilestorePath(mountSrc, c.sandboxID()) if err := os.Remove(filestorePath); err != nil { err = fmt.Errorf("failed to delete filestore file %q: %v", filestorePath, err) log.Warningf("%v", err) @@ -840,7 +840,7 @@ func (c *Container) Destroy() error { } // Assume this is a self-backed shared mount and try to delete the // filestore. Subsequently ignore the ENOENT if the assumption is wrong. - filestorePath := boot.SelfOverlayFilestorePath(hint.Mount.Source, c.sandboxID()) + filestorePath := boot.SelfFilestorePath(hint.Mount.Source, c.sandboxID()) if err := os.Remove(filestorePath); err != nil && !os.IsNotExist(err) { err = fmt.Errorf("failed to delete shared filestore file %q: %v", filestorePath, err) log.Warningf("%v", err) @@ -886,12 +886,12 @@ func (c *Container) sandboxID() string { return c.Saver.ID.SandboxID } -func (c *Container) forEachSelfOverlay(fn func(mountSrc string)) { - if c.OverlayMediums == nil { +func (c *Container) forEachSelfMount(fn func(mountSrc string)) { + if c.GoferMountConfs == nil { // Container not started? Skip. return } - if c.OverlayMediums[0] == boot.SelfMedium { + if c.GoferMountConfs[0].IsSelfBacked() { fn(c.Spec.Root.Path) } goferMntIdx := 1 // First index is for rootfs. @@ -899,30 +899,30 @@ func (c *Container) forEachSelfOverlay(fn func(mountSrc string)) { if !specutils.IsGoferMount(c.Spec.Mounts[i]) { continue } - if c.OverlayMediums[goferMntIdx] == boot.SelfMedium { + if c.GoferMountConfs[goferMntIdx].IsSelfBacked() { fn(c.Spec.Mounts[i].Source) } goferMntIdx++ } } -// createOverlayFilestores creates the regular files that will back the tmpfs -// upper mount for overlay mounts. It also returns information about the -// overlay medium used for each bind mount. -func (c *Container) createOverlayFilestores(conf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.OverlayMedium, error) { - var filestoreFiles []*os.File - var overlayMediums []boot.OverlayMedium +// createGoferFilestores creates the regular files that will back the +// tmpfs/overlayfs mounts that will overlay some gofer mounts. It also returns +// information about how each gofer mount is configured. +func (c *Container) createGoferFilestores(ovlConf config.Overlay2, mountHints *boot.PodMountHints) ([]*os.File, []boot.GoferMountConf, error) { + var goferFilestores []*os.File + var goferConfs []boot.GoferMountConf // Handle root mount first. - shouldOverlay := conf.RootEnabled() && !c.Spec.Root.Readonly - filestore, medium, err := c.createOverlayFilestore(conf, c.Spec.Root.Path, shouldOverlay, nil /* hint */) + shouldOverlay := ovlConf.RootEnabled() && !c.Spec.Root.Readonly + filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Root.Path, shouldOverlay, nil /* hint */) if err != nil { return nil, nil, err } if filestore != nil { - filestoreFiles = append(filestoreFiles, filestore) + goferFilestores = append(goferFilestores, filestore) } - overlayMediums = append(overlayMediums, medium) + goferConfs = append(goferConfs, goferConf) // Handle bind mounts. for i := range c.Spec.Mounts { @@ -930,52 +930,52 @@ func (c *Container) createOverlayFilestores(conf config.Overlay2, mountHints *bo continue } hint := mountHints.FindMount(c.Spec.Mounts[i].Source) - shouldOverlay := conf.SubMountEnabled() && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) - filestore, medium, err := c.createOverlayFilestore(conf, c.Spec.Mounts[i].Source, shouldOverlay, hint) + shouldOverlay := ovlConf.SubMountEnabled() && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) + filestore, goferConf, err := c.createGoferFilestore(ovlConf, c.Spec.Mounts[i].Source, shouldOverlay, hint) if err != nil { return nil, nil, err } if filestore != nil { - filestoreFiles = append(filestoreFiles, filestore) + goferFilestores = append(goferFilestores, filestore) } - overlayMediums = append(overlayMediums, medium) + goferConfs = append(goferConfs, goferConf) } - for _, filestore := range filestoreFiles { + for _, filestore := range goferFilestores { // Perform this work around outside the sandbox. The sandbox may already be // running with seccomp filters that do not allow this. pgalloc.IMAWorkAroundForMemFile(filestore.Fd()) } - return filestoreFiles, overlayMediums, nil + return goferFilestores, goferConfs, nil } -func (c *Container) createOverlayFilestore(conf config.Overlay2, mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.OverlayMedium, error) { +func (c *Container) createGoferFilestore(ovlConf config.Overlay2, mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.GoferMountConf, error) { if hint != nil && hint.ShouldOverlay() { // MountHint information takes precedence over shouldOverlay. return c.createOverlayFilestoreInSelf(mountSrc) } switch { case !shouldOverlay: - return nil, boot.NoOverlay, nil - case conf.IsBackedByMemory(): - return nil, boot.MemoryMedium, nil - case conf.IsBackedBySelf(): + return nil, boot.VanillaGofer, nil + case ovlConf.IsBackedByMemory(): + return nil, boot.MemoryOverlay, nil + case ovlConf.IsBackedBySelf(): return c.createOverlayFilestoreInSelf(mountSrc) default: - return c.createOverlayFilestoreInDir(conf) + return c.createGoferFilestoreInDir(ovlConf) } } -func (c *Container) createOverlayFilestoreInSelf(mountSrc string) (*os.File, boot.OverlayMedium, error) { +func (c *Container) createOverlayFilestoreInSelf(mountSrc string) (*os.File, boot.GoferMountConf, error) { mountSrcInfo, err := os.Stat(mountSrc) if err != nil { - return nil, boot.NoOverlay, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) + return nil, boot.VanillaGofer, fmt.Errorf("failed to stat mount %q to see if it were a directory: %v", mountSrc, err) } if !mountSrcInfo.IsDir() { - log.Warningf("overlay2 self medium is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) - return nil, boot.MemoryMedium, nil + log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) + return nil, boot.MemoryOverlay, nil } - // Create the self overlay filestore file. - filestorePath := boot.SelfOverlayFilestorePath(mountSrc, c.sandboxID()) + // Create the self filestore file. + filestorePath := boot.SelfFilestorePath(mountSrc, c.sandboxID()) filestoreFD, err := unix.Open(filestorePath, unix.O_RDWR|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC, 0666) if err != nil { if err == unix.EEXIST { @@ -985,42 +985,42 @@ func (c *Container) createOverlayFilestoreInSelf(mountSrc string) (*os.File, boo // visible to the other. Given "overlay on repeated submounts" is // already broken, we don't support such a scenario with the self // medium. The filestore file will already exist for such a case. - return nil, boot.NoOverlay, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not suppported with self medium", mountSrc, filestorePath) + return nil, boot.VanillaGofer, fmt.Errorf("%q mount source already has a filestore file at %q; repeated submounts are not suppported with self medium", mountSrc, filestorePath) } - return nil, boot.NoOverlay, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) + return nil, boot.VanillaGofer, fmt.Errorf("failed to create filestore file inside %q: %v", mountSrc, err) } - log.Debugf("Created overlay filestore file at %q for mount source %q", filestorePath, mountSrc) + log.Debugf("Created filestore file at %q for mount source %q", filestorePath, mountSrc) // Filestore in self should be a named path because it needs to be // discoverable via path traversal so that k8s can scan the filesystem // and apply any limits appropriately (like local ephemeral storage // limits). So don't delete it. These files will be unlinked when the // container is destroyed. This makes self medium appropriate for k8s. - return os.NewFile(uintptr(filestoreFD), filestorePath), boot.SelfMedium, nil + return os.NewFile(uintptr(filestoreFD), filestorePath), boot.SelfOverlay, nil } -func (c *Container) createOverlayFilestoreInDir(conf config.Overlay2) (*os.File, boot.OverlayMedium, error) { - filestoreDir := conf.HostFileDir() +func (c *Container) createGoferFilestoreInDir(ovlConf config.Overlay2) (*os.File, boot.GoferMountConf, error) { + filestoreDir := ovlConf.HostFileDir() fileInfo, err := os.Stat(filestoreDir) if err != nil { - return nil, boot.NoOverlay, fmt.Errorf("failed to stat overlay filestore directory %q: %v", filestoreDir, err) + return nil, boot.VanillaGofer, fmt.Errorf("failed to stat filestore directory %q: %v", filestoreDir, err) } if !fileInfo.IsDir() { - return nil, boot.NoOverlay, fmt.Errorf("overlay2 flag should specify an existing directory") + return nil, boot.VanillaGofer, fmt.Errorf("overlay2 flag should specify an existing directory") } // Create an unnamed temporary file in filestore directory which will be // deleted when the last FD on it is closed. We don't use O_TMPFILE because // it is not supported on all filesystems. So we simulate it by creating a // named file and then immediately unlinking it while keeping an FD on it. // This file will be deleted when the container exits. - filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-overlay-filestore-") + filestoreFile, err := os.CreateTemp(filestoreDir, "runsc-filestore-") if err != nil { - return nil, boot.NoOverlay, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) + return nil, boot.VanillaGofer, fmt.Errorf("failed to create a temporary file inside %q: %v", filestoreDir, err) } if err := unix.Unlink(filestoreFile.Name()); err != nil { - return nil, boot.NoOverlay, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) + return nil, boot.VanillaGofer, fmt.Errorf("failed to unlink temporary file %q: %v", filestoreFile.Name(), err) } - log.Debugf("Created an unnamed overlay filestore file at %q", filestoreDir) - return filestoreFile, boot.AnonDirMedium, nil + log.Debugf("Created an unnamed filestore file at %q", filestoreDir) + return filestoreFile, boot.AnonOverlay, nil } // saveLocked saves the container metadata to a file. @@ -1155,7 +1155,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu nextFD := donations.Transfer(cmd, 3) cmd.Args = append(cmd.Args, "gofer", "--bundle", bundleDir) - cmd.Args = append(cmd.Args, "--overlay-mediums="+c.OverlayMediums.String()) + cmd.Args = append(cmd.Args, "--gofer-mount-confs="+c.GoferMountConfs.String()) // Open the spec file to donate to the sandbox. specFile, err := specutils.OpenSpec(bundleDir) diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index fea678181..ffcae7804 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -3109,7 +3109,7 @@ func TestOverlayByMountAnnotation(t *testing.T) { } // Check that the filestore file is created and is not empty. - filestoreFile := boot.SelfOverlayFilestorePath(subMount, cont.Sandbox.ID) + filestoreFile := boot.SelfFilestorePath(subMount, cont.Sandbox.ID) var stat unix.Stat_t if err := unix.Stat(filestoreFile, &stat); err != nil { t.Fatalf("unix.Stat(%q) failed for submount filestore: %v", filestoreFile, err) diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index d8cc7e612..05ee4b1ec 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -2391,7 +2391,7 @@ func TestMultiContainerOverlayLeaks(t *testing.T) { } // Stat filestoreFile to see its usage. It should have been cleaned up. - filestoreFile := boot.SelfOverlayFilestorePath(s.Root.Path, sandboxID) + filestoreFile := boot.SelfFilestorePath(s.Root.Path, sandboxID) var stat unix.Stat_t if err := unix.Stat(filestoreFile, &stat); err != nil { t.Errorf("unix.Stat(%q) failed for rootfs filestore: %v", filestoreFile, err) diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 734ca3836..8b4071a79 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -228,14 +228,14 @@ type Args struct { // appear in the spec. IOFiles []*os.File - // OverlayFilestoreFiles are the regular files that will back the tmpfs upper - // mount in the overlay mounts. - OverlayFilestoreFiles []*os.File + // GoferFilestoreFiles are the regular files that will back the overlayfs or + // tmpfs mount if a gofer mount is to be overlaid. + GoferFilestoreFiles []*os.File - // OverlayMediums contains information about how the gofer mounts have been - // overlaid. The first entry is for rootfs and the following entries are for - // bind mounts in Spec.Mounts (in the same order). - OverlayMediums boot.OverlayMediumFlags + // GoferMountConfs contains information about how the gofer mounts have been + // configured. The first entry is for rootfs and the following entries are + // for bind mounts in Spec.Mounts (in the same order). + GoferMountConfs boot.GoferMountConfFlags // MountHints provides extra information about containers mounts that apply // to the entire pod. @@ -403,7 +403,7 @@ func (s *Sandbox) StartRoot(conf *config.Config) error { } // StartSubcontainer starts running a sub-container inside the sandbox. -func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, overlayFilestoreFiles []*os.File, overlayMediums []boot.OverlayMedium) error { +func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, goferConfs []boot.GoferMountConf) error { log.Debugf("Start sub-container %q in sandbox %q, PID: %d", cid, s.ID, s.Pid.load()) if err := s.configureStdios(conf, stdios); err != nil { @@ -413,22 +413,21 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s // The payload contains (in this specific order): // * stdin/stdout/stderr (optional: only present when not using TTY) - // * The subcontainer's overlay filestore files (optional: only present when - // host file backed overlay is configured) + // * The subcontainer's gofer filestore files (optional) // * Gofer files. payload := urpc.FilePayload{} payload.Files = append(payload.Files, stdios...) - payload.Files = append(payload.Files, overlayFilestoreFiles...) + payload.Files = append(payload.Files, goferFilestores...) payload.Files = append(payload.Files, goferFiles...) // Start running the container. args := boot.StartArgs{ - Spec: spec, - Conf: conf, - CID: cid, - NumOverlayFilestoreFDs: len(overlayFilestoreFiles), - OverlayMediums: overlayMediums, - FilePayload: payload, + Spec: spec, + Conf: conf, + CID: cid, + NumGoferFilestoreFDs: len(goferFilestores), + GoferMountConfs: goferConfs, + FilePayload: payload, } if err := s.call(boot.ContMgrStartSubcontainer, &args, nil); err != nil { return fmt.Errorf("starting sub-container %v: %w", spec.Process.Args, err) @@ -734,7 +733,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn // If there is a gofer, sends all socket ends to the sandbox. donations.DonateAndClose("io-fds", args.IOFiles...) - donations.DonateAndClose("overlay-filestore-fds", args.OverlayFilestoreFiles...) + donations.DonateAndClose("gofer-filestore-fds", args.GoferFilestoreFiles...) donations.DonateAndClose("mounts-fd", args.MountsFile) donations.Donate("start-sync-fd", startSyncFile) if err := donations.OpenAndDonate("user-log-fd", args.UserLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { @@ -762,8 +761,8 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn cmd.Args = append(cmd.Args, "--nvidia-dev-minors="+args.NvidiaDevMinors.String()) } - // Pass overlay mediums. - cmd.Args = append(cmd.Args, "--overlay-mediums="+args.OverlayMediums.String()) + // Pass gofer mount configs. + cmd.Args = append(cmd.Args, "--gofer-mount-confs="+args.GoferMountConfs.String()) // Create a socket for the control server and donate it to the sandbox. controlSocketPath, sockFD, err := createControlSocket(conf.RootDir, s.ID) diff --git a/test/e2e/integration_runtime_test.go b/test/e2e/integration_runtime_test.go index b101a4af7..dd1985246 100644 --- a/test/e2e/integration_runtime_test.go +++ b/test/e2e/integration_runtime_test.go @@ -208,10 +208,10 @@ func TestOverlayRootfsWhiteout(t *testing.T) { opts := dockerutil.RunOpts{ Image: "basic/ubuntu", } - if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("ls -al / | grep %q || true", boot.SelfOverlayFilestorePrefix)); err != nil { + if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("ls -al / | grep %q || true", boot.SelfFilestorePrefix)); err != nil { t.Fatalf("docker run failed: %s, %v", got, err) } else if got != "" { - t.Errorf("root directory contains a file/directory whose name contains %q: output = %q", boot.SelfOverlayFilestorePrefix, got) + t.Errorf("root directory contains a file/directory whose name contains %q: output = %q", boot.SelfFilestorePrefix, got) } }