Hide root overlay filestore from container using whiteout.

PiperOrigin-RevId: 507355437

g3doc/user_guide/filesystem.md

@@ -48,11 +48,11 @@ layer (tmpfs) be backed by a host file, so all file data is stored on disk.
 
 The newer `--overlay2` flag allows you to achieve these. You can specify
 `--overlay2=root:self` in `runtimeArgs`. The overlay backing host file will be
-created in the container's root filesystem. Placing the host file in the
-container's root filesystem is important because k8s scans the container's root
-filesystem from the host to enforce local ephemeral storage limits. You can also
-place the overlay host file in another directory using
-`--overlay2=root:/path/dir`.
+created in the container's root filesystem. This file will be hidden from the
+containerized application. Placing the host file in the container's root
+filesystem is important because k8s scans the container's root filesystem from
+the host to enforce local ephemeral storage limits. You can also place the
+overlay host file in another directory using `--overlay2=root:/path/dir`.
 
 ## Shared root filesystem
 

pkg/sentry/fsimpl/overlay/filesystem.go

@@ -562,16 +562,19 @@ func (fs *filesystem) doCreateAt(ctx context.Context, rp *vfs.ResolvingPath, ct
 	return nil
 }
 
+// CreateWhiteout creates a whiteout at pop. Whiteouts are created with
+// character devices with device ID = 0.
+//
 // Preconditions: pop's parent directory has been copied up.
-func (fs *filesystem) createWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, pop *vfs.PathOperation) error {
-	return vfsObj.MknodAt(ctx, fs.creds, pop, &vfs.MknodOptions{
+func CreateWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, pop *vfs.PathOperation) error {
+	return vfsObj.MknodAt(ctx, creds, pop, &vfs.MknodOptions{
 		Mode: linux.S_IFCHR, // permissions == include/linux/fs.h:WHITEOUT_MODE == 0
 		// DevMajor == DevMinor == 0, from include/linux/fs.h:WHITEOUT_DEV
 	})
 }
 
 func (fs *filesystem) cleanupRecreateWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, pop *vfs.PathOperation) {
-	if err := fs.createWhiteout(ctx, vfsObj, pop); err != nil {
+	if err := CreateWhiteout(ctx, vfsObj, fs.creds, pop); err != nil {
 		panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to recreate whiteout after failed file creation: %v", err))
 	}
 }
@@ -1237,7 +1240,7 @@ func (fs *filesystem) RenameAt(ctx context.Context, rp *vfs.ResolvingPath, oldPa
 			if !whiteoutUpper {
 				continue
 			}
-			if err := fs.createWhiteout(ctx, vfsObj, &vfs.PathOperation{
+			if err := CreateWhiteout(ctx, vfsObj, fs.creds, &vfs.PathOperation{
 				Root:  replaced.upperVD,
 				Start: replaced.upperVD,
 				Path:  fspath.Parse(whiteoutName),
@@ -1320,7 +1323,7 @@ func (fs *filesystem) RenameAt(ctx context.Context, rp *vfs.ResolvingPath, oldPa
 	newParent.children[newName] = renamed
 	oldParent.dirents = nil
 
-	if err := fs.createWhiteout(ctx, vfsObj, &oldpop); err != nil {
+	if err := CreateWhiteout(ctx, vfsObj, fs.creds, &oldpop); err != nil {
 		panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to create whiteout at origin after RenameAt: %v", err))
 	}
 	if renamed.isDir() {
@@ -1410,7 +1413,7 @@ func (fs *filesystem) RmdirAt(ctx context.Context, rp *vfs.ResolvingPath) error
 				if !whiteoutUpper {
 					continue
 				}
-				if err := fs.createWhiteout(ctx, vfsObj, &vfs.PathOperation{
+				if err := CreateWhiteout(ctx, vfsObj, fs.creds, &vfs.PathOperation{
 					Root:  child.upperVD,
 					Start: child.upperVD,
 					Path:  fspath.Parse(whiteoutName),
@@ -1441,7 +1444,7 @@ func (fs *filesystem) RmdirAt(ctx context.Context, rp *vfs.ResolvingPath) error
 			return err
 		}
 	}
-	if err := fs.createWhiteout(ctx, vfsObj, &pop); err != nil {
+	if err := CreateWhiteout(ctx, vfsObj, fs.creds, &pop); err != nil {
 		vfsObj.AbortDeleteDentry(&child.vfsd)
 		if child.upperVD.Ok() {
 			// Don't attempt to recover from this: the original directory is
@@ -1655,7 +1658,7 @@ func (fs *filesystem) UnlinkAt(ctx context.Context, rp *vfs.ResolvingPath) error
 			return err
 		}
 	}
-	if err := fs.createWhiteout(ctx, vfsObj, &pop); err != nil {
+	if err := CreateWhiteout(ctx, vfsObj, fs.creds, &pop); err != nil {
 		vfsObj.AbortDeleteDentry(&child.vfsd)
 		if childLayer == lookupLayerUpper {
 			panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to create whiteout after unlinking upper layer file during UnlinkAt: %v", err))

runsc/boot/controller.go

@@ -438,7 +438,7 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
 
 	// Set up the restore environment.
 	ctx := k.SupervisorContext()
-	mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.productName)
+	mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.productName, o.SandboxID)
 	ctx, err = mntr.configureRestore(ctx)
 	if err != nil {
 		return fmt.Errorf("configuring filesystem restore: %v", err)

runsc/boot/loader.go

@@ -822,7 +822,7 @@ func (l *Loader) createContainerProcess(root bool, cid string, info *containerIn
 	}
 	l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
 
-	mntr := newContainerMounter(info, l.k, l.mountHints, l.productName)
+	mntr := newContainerMounter(info, l.k, l.mountHints, l.productName, cid)
 	if root {
 		if err := mntr.processHints(info.conf, info.procArgs.Credentials); err != nil {
 			return nil, nil, err

runsc/boot/loader_test.go

@@ -417,7 +417,7 @@ func TestCreateMountNamespace(t *testing.T) {
 			defer l.Destroy()
 			defer loaderCleanup()
 
-			mntr := newContainerMounter(&l.root, l.k, l.mountHints, "")
+			mntr := newContainerMounter(&l.root, l.k, l.mountHints, "", l.sandboxID)
 			if err := mntr.processHints(l.root.conf, l.root.procArgs.Credentials); err != nil {
 				t.Fatalf("failed process hints: %v", err)
 			}

runsc/boot/vfs.go

@@ -58,6 +58,10 @@ const (
 	Nonefs = "none"
 )
 
+// SelfOverlayFilestoreDirPrefix is the prefix in the directory name of the
+// self overlay filestore directory.
+const SelfOverlayFilestoreDirPrefix = ".gvisor.overlay.img."
+
 // SelfOverlayFilestoreDir returns the directory path in which self overlay filestore
 // files are stored for a given mount.
 func SelfOverlayFilestoreDir(mountSrc, cid string) string {
@@ -65,7 +69,11 @@ func SelfOverlayFilestoreDir(mountSrc, cid string) string {
 	// the mount being overlayed itself. The same volume can be overlay-ed by
 	// multiple containers. So make the filestore directory unique to container
 	// by suffixing the container ID.
-	return path.Join(mountSrc, ".gvisor.overlay.img."+cid)
+	return path.Join(mountSrc, selfOverlayFilestoreDirName(cid))
+}
+
+func selfOverlayFilestoreDirName(cid string) string {
+	return SelfOverlayFilestoreDirPrefix + cid
 }
 
 // tmpfs has some extra supported options that we must pass through.
@@ -341,9 +349,12 @@ type containerMounter struct {
 	// productName is the value to show in
 	// /sys/devices/virtual/dmi/id/product_name.
 	productName string
+
+	// cid is the container ID for the container.
+	cid string
 }
 
-func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string) *containerMounter {
+func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string, cid string) *containerMounter {
 	return &containerMounter{
 		root:                info.spec.Root,
 		mounts:              compileMounts(info.spec, info.conf),
@@ -352,6 +363,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountH
 		k:                   k,
 		hints:               hints,
 		productName:         productName,
+		cid:                 cid,
 	}
 }
 
@@ -457,8 +469,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	return mns, nil
 }
 
-func useOverlayFilestoreFD(conf *config.Config, isDir bool) bool {
-	overlay2 := conf.GetOverlay2()
+func useOverlayFilestoreFD(overlay2 config.Overlay2, isDir bool) bool {
 	if !overlay2.IsBackedByHostFile() {
 		return false
 	}
@@ -473,7 +484,7 @@ func useOverlayFilestoreFD(conf *config.Config, isDir bool) bool {
 // layer using tmpfs, and return overlay mount options. "cleanup" must be called
 // after the options have been used to mount the overlay, to release refs on
 // lower and upper mounts.
-func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(conf *config.Config, isDir bool) bool) (*vfs.MountOptions, func(), error) {
+func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(overlay2 config.Overlay2, isDir bool) bool) (*vfs.MountOptions, func(), error) {
 	// First copy options from lower layer to upper layer and overlay. Clear
 	// filesystem specific options.
 	upperOpts := *lowerOpts
@@ -514,7 +525,8 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 	tmpfsOpts := tmpfs.FilesystemOpts{
 		RootFileType: uint16(rootType),
 	}
-	if useFilestoreFD != nil && useFilestoreFD(conf, rootType == linux.S_IFDIR) {
+	overlay2 := conf.GetOverlay2()
+	if useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) {
 		tmpfsOpts.FilestoreFD = c.overlayFilestoreFDs.removeAsFD()
 	}
 	upperOpts.GetFilesystemOptions.InternalData = tmpfsOpts
@@ -554,6 +566,18 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 		}
 	}
 
+	// If host filestore is being used and it is backed by self, then we need to
+	// hide the filestore from the containerized application.
+	if overlay2.IsBackedBySelf() && useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) {
+		if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{
+			Root:  upperRootVD,
+			Start: upperRootVD,
+			Path:  fspath.Parse(selfOverlayFilestoreDirName(c.cid)),
+		}); err != nil {
+			return nil, nil, fmt.Errorf("failed to create whiteout to hide self overlay filestore: %w", err)
+		}
+	}
+
 	// Propagate the lower layer's root's owner, group, and mode to the upper
 	// layer's root for consistency with VFS1.
 	err = c.k.VFS().SetStatAt(ctx, creds, &vfs.PathOperation{

test/e2e/BUILD

@@ -42,6 +42,7 @@ go_test(
     deps = [
         "//pkg/test/dockerutil",
         "//pkg/test/testutil",
+        "//runsc/boot",
         "@com_github_docker_docker//api/types/mount:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",
     ],

test/e2e/integration_runtime_test.go

@@ -39,6 +39,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
+	"gvisor.dev/gvisor/runsc/boot"
 )
 
 const (
@@ -186,8 +187,7 @@ func TestOverlayNameTooLong(t *testing.T) {
 	defer d.CleanUp(ctx)
 
 	opts := dockerutil.RunOpts{
-		Image:   "basic/integrationtest",
-		WorkDir: "/root",
+		Image: "basic/ubuntu",
 	}
 	longName := strings.Repeat("a", unix.NAME_MAX+1)
 	if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("stat %s || true", longName)); err != nil {
@@ -234,3 +234,20 @@ func TestMultipleOverlayMounts(t *testing.T) {
 		t.Errorf("overlay not applied to both bind mounts, %q file exists", filePath)
 	}
 }
+
+// Tests that the overlay backing host file inside the container's rootfs is
+// hidden from the application.
+func TestOverlayRootfsWhiteout(t *testing.T) {
+	ctx := context.Background()
+	d := dockerutil.MakeContainerWithRuntime(ctx, t, "-overlay")
+	defer d.CleanUp(ctx)
+
+	opts := dockerutil.RunOpts{
+		Image: "basic/ubuntu",
+	}
+	if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("ls -al / | grep %q || true", boot.SelfOverlayFilestoreDirPrefix)); err != nil {
+		t.Fatalf("docker run failed: %s, %v", got, err)
+	} else if got != "" {
+		t.Errorf("root directory contains a file/directory whose name contains %q: output = %q", boot.SelfOverlayFilestoreDirPrefix, got)
+	}
+}