From 09459b203a532c24fbb76cc88484d533356b8b91 Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Sun, 5 Feb 2023 19:33:33 -0800 Subject: [PATCH] Hide root overlay filestore from container using whiteout. PiperOrigin-RevId: 507355437 --- g3doc/user_guide/filesystem.md | 10 +++---- pkg/sentry/fsimpl/overlay/filesystem.go | 19 +++++++------ runsc/boot/controller.go | 2 +- runsc/boot/loader.go | 2 +- runsc/boot/loader_test.go | 2 +- runsc/boot/vfs.go | 36 ++++++++++++++++++++----- test/e2e/BUILD | 1 + test/e2e/integration_runtime_test.go | 21 +++++++++++++-- 8 files changed, 69 insertions(+), 24 deletions(-) diff --git a/g3doc/user_guide/filesystem.md b/g3doc/user_guide/filesystem.md index bae4e92e0..9b424587d 100644 --- a/g3doc/user_guide/filesystem.md +++ b/g3doc/user_guide/filesystem.md @@ -48,11 +48,11 @@ layer (tmpfs) be backed by a host file, so all file data is stored on disk. The newer `--overlay2` flag allows you to achieve these. You can specify `--overlay2=root:self` in `runtimeArgs`. The overlay backing host file will be -created in the container's root filesystem. Placing the host file in the -container's root filesystem is important because k8s scans the container's root -filesystem from the host to enforce local ephemeral storage limits. You can also -place the overlay host file in another directory using -`--overlay2=root:/path/dir`. +created in the container's root filesystem. This file will be hidden from the +containerized application. Placing the host file in the container's root +filesystem is important because k8s scans the container's root filesystem from +the host to enforce local ephemeral storage limits. You can also place the +overlay host file in another directory using `--overlay2=root:/path/dir`. ## Shared root filesystem diff --git a/pkg/sentry/fsimpl/overlay/filesystem.go b/pkg/sentry/fsimpl/overlay/filesystem.go index 68ca780a9..771ce6f94 100644 --- a/pkg/sentry/fsimpl/overlay/filesystem.go +++ b/pkg/sentry/fsimpl/overlay/filesystem.go @@ -562,16 +562,19 @@ func (fs *filesystem) doCreateAt(ctx context.Context, rp *vfs.ResolvingPath, ct return nil } +// CreateWhiteout creates a whiteout at pop. Whiteouts are created with +// character devices with device ID = 0. +// // Preconditions: pop's parent directory has been copied up. -func (fs *filesystem) createWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, pop *vfs.PathOperation) error { - return vfsObj.MknodAt(ctx, fs.creds, pop, &vfs.MknodOptions{ +func CreateWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, pop *vfs.PathOperation) error { + return vfsObj.MknodAt(ctx, creds, pop, &vfs.MknodOptions{ Mode: linux.S_IFCHR, // permissions == include/linux/fs.h:WHITEOUT_MODE == 0 // DevMajor == DevMinor == 0, from include/linux/fs.h:WHITEOUT_DEV }) } func (fs *filesystem) cleanupRecreateWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, pop *vfs.PathOperation) { - if err := fs.createWhiteout(ctx, vfsObj, pop); err != nil { + if err := CreateWhiteout(ctx, vfsObj, fs.creds, pop); err != nil { panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to recreate whiteout after failed file creation: %v", err)) } } @@ -1237,7 +1240,7 @@ func (fs *filesystem) RenameAt(ctx context.Context, rp *vfs.ResolvingPath, oldPa if !whiteoutUpper { continue } - if err := fs.createWhiteout(ctx, vfsObj, &vfs.PathOperation{ + if err := CreateWhiteout(ctx, vfsObj, fs.creds, &vfs.PathOperation{ Root: replaced.upperVD, Start: replaced.upperVD, Path: fspath.Parse(whiteoutName), @@ -1320,7 +1323,7 @@ func (fs *filesystem) RenameAt(ctx context.Context, rp *vfs.ResolvingPath, oldPa newParent.children[newName] = renamed oldParent.dirents = nil - if err := fs.createWhiteout(ctx, vfsObj, &oldpop); err != nil { + if err := CreateWhiteout(ctx, vfsObj, fs.creds, &oldpop); err != nil { panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to create whiteout at origin after RenameAt: %v", err)) } if renamed.isDir() { @@ -1410,7 +1413,7 @@ func (fs *filesystem) RmdirAt(ctx context.Context, rp *vfs.ResolvingPath) error if !whiteoutUpper { continue } - if err := fs.createWhiteout(ctx, vfsObj, &vfs.PathOperation{ + if err := CreateWhiteout(ctx, vfsObj, fs.creds, &vfs.PathOperation{ Root: child.upperVD, Start: child.upperVD, Path: fspath.Parse(whiteoutName), @@ -1441,7 +1444,7 @@ func (fs *filesystem) RmdirAt(ctx context.Context, rp *vfs.ResolvingPath) error return err } } - if err := fs.createWhiteout(ctx, vfsObj, &pop); err != nil { + if err := CreateWhiteout(ctx, vfsObj, fs.creds, &pop); err != nil { vfsObj.AbortDeleteDentry(&child.vfsd) if child.upperVD.Ok() { // Don't attempt to recover from this: the original directory is @@ -1655,7 +1658,7 @@ func (fs *filesystem) UnlinkAt(ctx context.Context, rp *vfs.ResolvingPath) error return err } } - if err := fs.createWhiteout(ctx, vfsObj, &pop); err != nil { + if err := CreateWhiteout(ctx, vfsObj, fs.creds, &pop); err != nil { vfsObj.AbortDeleteDentry(&child.vfsd) if childLayer == lookupLayerUpper { panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to create whiteout after unlinking upper layer file during UnlinkAt: %v", err)) diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index 32ffcc983..43d56bed4 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -438,7 +438,7 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { // Set up the restore environment. ctx := k.SupervisorContext() - mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.productName) + mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.productName, o.SandboxID) ctx, err = mntr.configureRestore(ctx) if err != nil { return fmt.Errorf("configuring filesystem restore: %v", err) diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 780cf036b..72a7b1017 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -822,7 +822,7 @@ func (l *Loader) createContainerProcess(root bool, cid string, info *containerIn } l.startGoferMonitor(cid, int32(info.goferFDs[0].FD())) - mntr := newContainerMounter(info, l.k, l.mountHints, l.productName) + mntr := newContainerMounter(info, l.k, l.mountHints, l.productName, cid) if root { if err := mntr.processHints(info.conf, info.procArgs.Credentials); err != nil { return nil, nil, err diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index 3939dd5e1..fe0633ce3 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -417,7 +417,7 @@ func TestCreateMountNamespace(t *testing.T) { defer l.Destroy() defer loaderCleanup() - mntr := newContainerMounter(&l.root, l.k, l.mountHints, "") + mntr := newContainerMounter(&l.root, l.k, l.mountHints, "", l.sandboxID) if err := mntr.processHints(l.root.conf, l.root.procArgs.Credentials); err != nil { t.Fatalf("failed process hints: %v", err) } diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 3d1f2e2f0..f63e177ac 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -58,6 +58,10 @@ const ( Nonefs = "none" ) +// SelfOverlayFilestoreDirPrefix is the prefix in the directory name of the +// self overlay filestore directory. +const SelfOverlayFilestoreDirPrefix = ".gvisor.overlay.img." + // SelfOverlayFilestoreDir returns the directory path in which self overlay filestore // files are stored for a given mount. func SelfOverlayFilestoreDir(mountSrc, cid string) string { @@ -65,7 +69,11 @@ func SelfOverlayFilestoreDir(mountSrc, cid string) string { // the mount being overlayed itself. The same volume can be overlay-ed by // multiple containers. So make the filestore directory unique to container // by suffixing the container ID. - return path.Join(mountSrc, ".gvisor.overlay.img."+cid) + return path.Join(mountSrc, selfOverlayFilestoreDirName(cid)) +} + +func selfOverlayFilestoreDirName(cid string) string { + return SelfOverlayFilestoreDirPrefix + cid } // tmpfs has some extra supported options that we must pass through. @@ -341,9 +349,12 @@ type containerMounter struct { // productName is the value to show in // /sys/devices/virtual/dmi/id/product_name. productName string + + // cid is the container ID for the container. + cid string } -func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string) *containerMounter { +func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string, cid string) *containerMounter { return &containerMounter{ root: info.spec.Root, mounts: compileMounts(info.spec, info.conf), @@ -352,6 +363,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountH k: k, hints: hints, productName: productName, + cid: cid, } } @@ -457,8 +469,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi return mns, nil } -func useOverlayFilestoreFD(conf *config.Config, isDir bool) bool { - overlay2 := conf.GetOverlay2() +func useOverlayFilestoreFD(overlay2 config.Overlay2, isDir bool) bool { if !overlay2.IsBackedByHostFile() { return false } @@ -473,7 +484,7 @@ func useOverlayFilestoreFD(conf *config.Config, isDir bool) bool { // layer using tmpfs, and return overlay mount options. "cleanup" must be called // after the options have been used to mount the overlay, to release refs on // lower and upper mounts. -func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(conf *config.Config, isDir bool) bool) (*vfs.MountOptions, func(), error) { +func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(overlay2 config.Overlay2, isDir bool) bool) (*vfs.MountOptions, func(), error) { // First copy options from lower layer to upper layer and overlay. Clear // filesystem specific options. upperOpts := *lowerOpts @@ -514,7 +525,8 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co tmpfsOpts := tmpfs.FilesystemOpts{ RootFileType: uint16(rootType), } - if useFilestoreFD != nil && useFilestoreFD(conf, rootType == linux.S_IFDIR) { + overlay2 := conf.GetOverlay2() + if useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) { tmpfsOpts.FilestoreFD = c.overlayFilestoreFDs.removeAsFD() } upperOpts.GetFilesystemOptions.InternalData = tmpfsOpts @@ -554,6 +566,18 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co } } + // If host filestore is being used and it is backed by self, then we need to + // hide the filestore from the containerized application. + if overlay2.IsBackedBySelf() && useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) { + if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{ + Root: upperRootVD, + Start: upperRootVD, + Path: fspath.Parse(selfOverlayFilestoreDirName(c.cid)), + }); err != nil { + return nil, nil, fmt.Errorf("failed to create whiteout to hide self overlay filestore: %w", err) + } + } + // Propagate the lower layer's root's owner, group, and mode to the upper // layer's root for consistency with VFS1. err = c.k.VFS().SetStatAt(ctx, creds, &vfs.PathOperation{ diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 53c6219e0..d8db4d407 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -42,6 +42,7 @@ go_test( deps = [ "//pkg/test/dockerutil", "//pkg/test/testutil", + "//runsc/boot", "@com_github_docker_docker//api/types/mount:go_default_library", "@org_golang_x_sys//unix:go_default_library", ], diff --git a/test/e2e/integration_runtime_test.go b/test/e2e/integration_runtime_test.go index b08e5db36..28157c384 100644 --- a/test/e2e/integration_runtime_test.go +++ b/test/e2e/integration_runtime_test.go @@ -39,6 +39,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/runsc/boot" ) const ( @@ -186,8 +187,7 @@ func TestOverlayNameTooLong(t *testing.T) { defer d.CleanUp(ctx) opts := dockerutil.RunOpts{ - Image: "basic/integrationtest", - WorkDir: "/root", + Image: "basic/ubuntu", } longName := strings.Repeat("a", unix.NAME_MAX+1) if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("stat %s || true", longName)); err != nil { @@ -234,3 +234,20 @@ func TestMultipleOverlayMounts(t *testing.T) { t.Errorf("overlay not applied to both bind mounts, %q file exists", filePath) } } + +// Tests that the overlay backing host file inside the container's rootfs is +// hidden from the application. +func TestOverlayRootfsWhiteout(t *testing.T) { + ctx := context.Background() + d := dockerutil.MakeContainerWithRuntime(ctx, t, "-overlay") + defer d.CleanUp(ctx) + + opts := dockerutil.RunOpts{ + Image: "basic/ubuntu", + } + if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("ls -al / | grep %q || true", boot.SelfOverlayFilestoreDirPrefix)); err != nil { + t.Fatalf("docker run failed: %s, %v", got, err) + } else if got != "" { + t.Errorf("root directory contains a file/directory whose name contains %q: output = %q", boot.SelfOverlayFilestoreDirPrefix, got) + } +}