Create separate overlay filestore for subcontainers.

Earlier we were using the same filestore for all containers in the
sandbox. This was based on the assumption that all containers in a
k8s pod have the same runsc configuration. We get rid of our
dependence on that assumption. Subcontainers can independently use
overlay configurations.

This is needed because in k8s, different ephemeral storage limits can
be enforced on containers in the same pod. This behavior is required
to get limit enforcement to work correctly on a per container basis.

This change also moves the ownership of overlay filestore to tmpfs.
Now that each container has its own overlay-backing MemoryFile, it will
be much harder to manage lifecycle of these MemoryFiles as containers
are created and destroyed.

Instead, move ownership to tmpfs mount. When the overlay upper layer tmpfs
is unmounted, it will destroy the MemoryFile.

We also extract out IMA work around logic from NewMemoryFile() and perform
it outside the sandbox. This is done because now tmpfs is creating the
MemoryFile. By then the sandbox is already running with seccomp filters
installed that do not allow certain mmap(2) calls that the said work around
uses. We hit the same issue for subcontainers. When they are started,
the sandbox is already running with seccomp filters. Added a new
MemoryFileOpts field for this which defaults to old behavior.

PiperOrigin-RevId: 505892085

pkg/sentry/fsimpl/tmpfs/BUILD

@@ -92,6 +92,7 @@ go_library(
         "//pkg/atomicbitops",
         "//pkg/context",
         "//pkg/errors/linuxerr",
+        "//pkg/fd",
         "//pkg/fspath",
         "//pkg/hostarch",
         "//pkg/log",

pkg/sentry/fsimpl/tmpfs/regular_file.go

@@ -42,9 +42,6 @@ import (
 type regularFile struct {
 	inode inode
 
-	// memFile is a platform.File used to allocate pages to this regularFile.
-	memFile *pgalloc.MemoryFile `state:"nosave"`
-
 	// memoryUsageKind is the memory accounting category under which pages backing
 	// this regularFile's contents are accounted.
 	memoryUsageKind usage.MemoryKind
@@ -93,7 +90,6 @@ type regularFile struct {
 
 func (fs *filesystem) newRegularFile(kuid auth.KUID, kgid auth.KGID, mode linux.FileMode, parentDir *directory) *inode {
 	file := &regularFile{
-		memFile:         fs.mfp.MemoryFile(),
 		memoryUsageKind: fs.usage,
 		seals:           linux.F_SEAL_SEAL,
 	}
@@ -224,7 +220,7 @@ func (rf *regularFile) truncateLocked(newSize uint64) (bool, error) {
 	// We are now guaranteed that there are no translations of truncated pages,
 	// and can remove them.
 	rf.dataMu.Lock()
-	decPages := rf.data.Truncate(newSize, rf.memFile)
+	decPages := rf.data.Truncate(newSize, rf.inode.fs.mf)
 	rf.dataMu.Unlock()
 	rf.inode.fs.unaccountPages(decPages)
 	return true, nil
@@ -311,7 +307,7 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap.
 		}
 		optional = required
 	}
-	pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
+	pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.inode.fs.mf, rf.memoryUsageKind, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
 		// Newly-allocated pages are zeroed, so we don't need to do anything.
 		return dsts.NumBytes(), nil
 	})
@@ -325,7 +321,7 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap.
 		segMR := seg.Range().Intersect(optional)
 		ts = append(ts, memmap.Translation{
 			Source: segMR,
-			File:   rf.memFile,
+			File:   rf.inode.fs.mf,
 			Offset: seg.FileRangeOf(segMR).Start,
 			Perms:  hostarch.AnyAccess,
 		})
@@ -392,7 +388,7 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint
 	// Pass populate = true here despite the fact that we don't touch these pages
 	// both for consistency with the expected behavior of fallocate(2) and in
 	// expectation of a future write to them.
-	pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, true /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
+	pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.inode.fs.mf, f.memoryUsageKind, true /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
 		// Newly-allocated pages are zeroed, so we don't need to do anything.
 		return dsts.NumBytes(), nil
 	})
@@ -606,7 +602,7 @@ func (rw *regularFileReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, er
 		switch {
 		case seg.Ok():
 			// Get internal mappings.
-			ims, err := rw.file.memFile.MapInternal(seg.FileRangeOf(seg.Range().Intersect(mr)), hostarch.Read)
+			ims, err := rw.file.inode.fs.mf.MapInternal(seg.FileRangeOf(seg.Range().Intersect(mr)), hostarch.Read)
 			if err != nil {
 				return done, err
 			}
@@ -700,7 +696,7 @@ func (rw *regularFileReadWriter) WriteFromBlocks(srcs safemem.BlockSeq) (uint64,
 		switch {
 		case seg.Ok():
 			// Get internal mappings.
-			ims, err := rw.file.memFile.MapInternal(seg.FileRangeOf(seg.Range().Intersect(mr)), hostarch.Write)
+			ims, err := rw.file.inode.fs.mf.MapInternal(seg.FileRangeOf(seg.Range().Intersect(mr)), hostarch.Write)
 			if err != nil {
 				retErr = err
 				goto exitLoop
@@ -733,7 +729,7 @@ func (rw *regularFileReadWriter) WriteFromBlocks(srcs safemem.BlockSeq) (uint64,
 				goto exitLoop
 			}
 			gapMR.End = gapMR.Start + (hostarch.PageSize * pagesReserved)
-			fr, err := rw.file.memFile.Allocate(gapMR.Length(), pgalloc.AllocOpts{Kind: rw.file.memoryUsageKind})
+			fr, err := rw.file.inode.fs.mf.Allocate(gapMR.Length(), pgalloc.AllocOpts{Kind: rw.file.memoryUsageKind})
 			if err != nil {
 				retErr = err
 				rw.file.inode.fs.unaccountPages(pagesReserved)

pkg/sentry/fsimpl/tmpfs/save_restore.go

@@ -15,6 +15,10 @@
 package tmpfs
 
 // afterLoad is called by stateify.
-func (rf *regularFile) afterLoad() {
-	rf.memFile = rf.inode.fs.mfp.MemoryFile()
+func (fs *filesystem) afterLoad() {
+	if fs.privateMF {
+		// TODO(b/241832602): Add S/R support.
+		panic("S/R not supported for private memory files")
+	}
+	fs.mf = fs.mfp.MemoryFile()
 }

pkg/sentry/fsimpl/tmpfs/tmpfs.go

@@ -38,6 +38,7 @@ import (
 	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
+	"gvisor.dev/gvisor/pkg/fd"
 	"gvisor.dev/gvisor/pkg/hostarch"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/time"
@@ -61,8 +62,16 @@ type FilesystemType struct{}
 type filesystem struct {
 	vfsfs vfs.Filesystem
 
-	// mfp is used to allocate memory that stores regular file contents. mfp is
-	// immutable.
+	// mf is used to allocate memory that stores regular file contents. mf is
+	// immutable, except it may to changed during restore.
+	mf *pgalloc.MemoryFile `state:"nosave"`
+
+	// privateMF indicates whether mf is private to this tmpfs mount. If so,
+	// tmpfs takes ownership of mf. privateMF is immutable.
+	privateMF bool
+
+	// mfp is used to provide mf, when privateMF == false. This is required to
+	// re-provide mf on restore. mfp is immutable.
 	mfp pgalloc.MemoryFileProvider
 
 	// clock is a realtime clock used to set timestamps in file operations.
@@ -128,9 +137,9 @@ type FilesystemOpts struct {
 	// MaxFilenameLen is the maximum filename length allowed by the tmpfs.
 	MaxFilenameLen int
 
-	// Filestore is the MemoryFile that will be used to store file data. If this
-	// is nil, then MemoryFileProviderFromContext() is used.
-	Filestore *pgalloc.MemoryFile
+	// FilestoreFD is the FD for the memory file that will be used to store file
+	// data. If this is nil, then MemoryFileProviderFromContext() is used.
+	FilestoreFD *fd.FD
 }
 
 // GetFilesystem implements vfs.FilesystemType.GetFilesystem.
@@ -139,6 +148,8 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 	if mfp == nil {
 		panic("MemoryFileProviderFromContext returned nil")
 	}
+	mf := mfp.MemoryFile()
+	privateMF := false
 
 	rootFileType := uint16(linux.S_IFDIR)
 	newFSType := vfs.FilesystemType(&fstype)
@@ -150,8 +161,20 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 		if tmpfsOpts.FilesystemType != nil {
 			newFSType = tmpfsOpts.FilesystemType
 		}
-		if tmpfsOpts.Filestore != nil {
-			mfp = tmpfsOpts.Filestore
+		if tmpfsOpts.FilestoreFD != nil {
+			// DecommitOnDestroy because tmpfsOpts.FilestoreFD may be backed by a
+			// host filesystem based file, which needs to be decommited on destroy.
+			// DisableIMAWorkAround because sentry's seccomp filters don't allow the
+			// mmap(2) syscalls that this work around uses. User of this feature is
+			// expected to have performed the work around outside the sandbox.
+			mfOpts := pgalloc.MemoryFileOpts{DecommitOnDestroy: true, DisableIMAWorkAround: true}
+			var err error
+			mf, err = pgalloc.NewMemoryFile(tmpfsOpts.FilestoreFD.ReleaseToFile("overlay-filestore"), mfOpts)
+			if err != nil {
+				ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: pgalloc.NewMemoryFile failed: %v", err)
+				return nil, nil, err
+			}
+			privateMF = true
 		}
 	}
 
@@ -243,6 +266,8 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 		memUsage = *tmpfsOpts.Usage
 	}
 	fs := filesystem{
+		mf:             mf,
+		privateMF:      privateMF,
 		mfp:            mfp,
 		clock:          clock,
 		devMinor:       devMinor,
@@ -285,6 +310,9 @@ func (fs *filesystem) Release(ctx context.Context) {
 		fs.root.releaseChildrenLocked(ctx)
 	}
 	fs.mu.Unlock()
+	if fs.privateMF {
+		fs.mf.Destroy()
+	}
 }
 
 // releaseChildrenLocked is called on the mount point by filesystem.Release() to
@@ -533,7 +561,7 @@ func (i *inode) decRef(ctx context.Context) {
 			// Release memory used by regFile to store data. Since regFile is
 			// no longer usable, we don't need to grab any locks or update any
 			// metadata.
-			pagesDec := impl.data.DropAll(impl.memFile)
+			pagesDec := impl.data.DropAll(i.fs.mf)
 			impl.inode.fs.unaccountPages(pagesDec)
 		}