mirror of
https://github.com/netbirdio/gvisor.git
synced 2026-05-22 17:12:49 -07:00
Only populate pages for tmpfs-backed MemoryFile in tmpfs Allocate().
Pre-populating pages on fallocate(2) when the MemoryFile is backed by a disk
is wasteful.
It is currently wasteful because the mlock+munlock mechanism used in
pgalloc.tryPopulate() to populate pages does not work with some disk-backed
filesystems like ext4. We end up faulting read-only pages, which need to
be re-faulted on write anyways.
Later we will update pgalloc.tryPopulate() to use madvice(MADV_POPULATE_WRITE).
It wouldn't work in that case too because even though it does fault writable
pages, they are written back to disk before the application can write to them.
And so they will be re-faulted on write. This would actually worsen the
performance because now we have the extra cost of writeback.
Hence, avoid populating pages for disk-backed MemoryFile on fallocate(2).
Instead we just commit the pages to satisfy fallocate(2) semantics.
When the application writes to these file offsets, the pages will fault
normally.
To allow this distinction between "committing" and "populating" pages in
addition to allocating them, the pgalloc API has been extended by introducing
`pgalloc.AllocationMode` enum. It allows the caller to select if pages should
be only-allocated, allocated+committed or allocated+committed+populated.
Here are the benchmarking results:
```
goos: linux
goarch: amd64
cpu: Intel(R) Xeon(R) CPU @ 2.80GHz
│ /tmp/benchout.runsc │ /tmp/benchout.runsc-cl1 │
│ sec/op │ sec/op vs base │
BuildABSL/page_cache.clean/filesystem.bindfs-4 100.73 ± 12% 99.72 ± 58% ~ (p=0.805 n=7)
BuildGRPC/page_cache.clean/filesystem.bindfs-4 703.5 ± 1% 690.9 ± 1% -1.78% (p=0.004 n=7)
Fio/operation.write/blockSize.4K/filesystem.bindfs-4 2.008m ± 0% 2.009m ± 25% ~ (p=0.805 n=7)
Fio/operation.write/blockSize.64K/filesystem.bindfs-4 1.507m ± 0% 1.507m ± 0% ~ (p=0.318 n=7)
Fio/operation.write/blockSize.1024K/filesystem.bindfs-4 1.507m ± 0% 1.507m ± 0% ~ (p=1.000 n=7)
Fio/operation.write/blockSize.4K/filesystem.tmpfs-4 2.510m ± 0% 2.509m ± 0% ~ (p=0.805 n=7)
Fio/operation.write/blockSize.64K/filesystem.tmpfs-4 2.009m ± 0% 2.009m ± 0% ~ (p=0.805 n=7)
Fio/operation.write/blockSize.1024K/filesystem.tmpfs-4 2.009m ± 0% 2.008m ± 0% ~ (p=0.097 n=7)
Fio/operation.write/blockSize.4K/filesystem.rootfs-4 3.010m ± 17% 2.510m ± 0% ~ (p=0.053 n=7)
Fio/operation.write/blockSize.64K/filesystem.rootfs-4 2.009m ± 25% 2.008m ± 0% ~ (p=1.000 n=7)
Fio/operation.write/blockSize.1024K/filesystem.rootfs-4 2.009m ± 0% 2.008m ± 0% ~ (p=0.259 n=7)
RubySpecTest/page_cache.clean/filesystem.bindfs-4 63.15 ± 2% 61.65 ± 2% -2.38% (p=0.011 n=7)
geomean 34.15m 33.48m -1.94%
│ /tmp/benchout.runsc │ /tmp/benchout.runsc-cl1 │
│ bandwidth.bytes_per_second │ bandwidth.bytes_per_second vs base │
Fio/operation.write/blockSize.4K/filesystem.bindfs-4 939.6M ± 2% 944.7M ± 4% ~ (p=0.737 n=7)
Fio/operation.write/blockSize.64K/filesystem.bindfs-4 2.937G ± 5% 2.873G ± 2% ~ (p=0.216 n=7)
Fio/operation.write/blockSize.1024K/filesystem.bindfs-4 3.139G ± 7% 3.112G ± 6% ~ (p=0.879 n=7)
Fio/operation.write/blockSize.4K/filesystem.tmpfs-4 1.355G ± 3% 1.382G ± 5% ~ (p=0.090 n=7)
Fio/operation.write/blockSize.64K/filesystem.tmpfs-4 3.913G ± 10% 3.785G ± 10% ~ (p=0.779 n=7)
Fio/operation.write/blockSize.1024K/filesystem.tmpfs-4 3.692G ± 11% 3.855G ± 6% ~ (p=1.000 n=7)
Fio/operation.write/blockSize.4K/filesystem.rootfs-4 788.4M ± 2% 628.6M ± 1% -20.26% (p=0.001 n=7)
Fio/operation.write/blockSize.64K/filesystem.rootfs-4 1346.1M ± 5% 940.4M ± 6% -30.13% (p=0.001 n=7)
Fio/operation.write/blockSize.1024K/filesystem.rootfs-4 1.646G ± 5% 1.173G ± 3% -28.75% (p=0.001 n=7)
geomean 1.882G 1.699G -9.72%
│ /tmp/benchout.runsc │ /tmp/benchout.runsc-cl1 │
│ io_ops.ops_per_second │ io_ops.ops_per_second vs base │
Fio/operation.write/blockSize.4K/filesystem.bindfs-4 229.4k ± 2% 230.6k ± 4% ~ (p=0.737 n=7)
Fio/operation.write/blockSize.64K/filesystem.bindfs-4 44.82k ± 5% 43.84k ± 2% ~ (p=0.216 n=7)
Fio/operation.write/blockSize.1024K/filesystem.bindfs-4 2.994k ± 7% 2.967k ± 6% ~ (p=0.879 n=7)
Fio/operation.write/blockSize.4K/filesystem.tmpfs-4 330.7k ± 3% 337.3k ± 5% ~ (p=0.090 n=7)
Fio/operation.write/blockSize.64K/filesystem.tmpfs-4 59.70k ± 10% 57.76k ± 10% ~ (p=0.779 n=7)
Fio/operation.write/blockSize.1024K/filesystem.tmpfs-4 3.521k ± 11% 3.676k ± 6% ~ (p=1.000 n=7)
Fio/operation.write/blockSize.4K/filesystem.rootfs-4 192.5k ± 2% 153.5k ± 1% -20.26% (p=0.001 n=7)
Fio/operation.write/blockSize.64K/filesystem.rootfs-4 20.54k ± 5% 14.35k ± 6% -30.13% (p=0.001 n=7)
Fio/operation.write/blockSize.1024K/filesystem.rootfs-4 1.570k ± 5% 1.119k ± 3% -28.73% (p=0.001 n=7)
geomean 28.72k 25.93k -9.72%
│ /tmp/benchout.runsc │ /tmp/benchout.runsc-cl1 │
│ load.sec │ load.sec vs base │
RubySpecTest/page_cache.clean/filesystem.bindfs-4 9.430 ± 7% 9.150 ± 7% -2.97% (p=0.026 n=7)
```
The broader filesystem benchmarks (ABSL, gRPC, Ruby) all improved with this
change. Improvement is ~2-3% reduction in absolute time to run these
workloads. However, the FIO microbenchmark is a little more complicated.
The latency metric shows improvement (for blockSize=4K & filesystem=rootfs).
It is stable for other blockSize values. But FIO bandwidth seems to have
regressed. We think this is an accounting gimmick. We suspect that sec/op
includes the app fallocate() and bandwidth does not. Since this change moves
ext4_mpage_readpages() from app fallocate() to the page faults that occur
during app write(), bandwidth appears to have regressed.
Given that the broader filesystem benchmarks look good and that this is more
consistent with Linux (app fallocate() now only commits disk space rather than
also materializing memory pages), this change should be an improvement.
PiperOrigin-RevId: 542331863
This commit is contained in:
Ayush Ranjan
gVisor bot
@@ -439,7 +439,7 @@ func (rw *dentryReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error)
|
||||
End: gapEnd,
|
||||
}
|
||||
optMR := gap.Range()
|
||||
_, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt)
|
||||
_, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, pgalloc.AllocateAndWritePopulate, h.readToBlocksAt)
|
||||
mf.MarkEvictable(rw.d, pgalloc.EvictableRange{optMR.Start, optMR.End})
|
||||
seg, gap = rw.d.cache.Find(rw.off)
|
||||
if !seg.Ok() {
|
||||
@@ -798,7 +798,7 @@ func (d *dentry) Translate(ctx context.Context, required, optional memmap.Mappab
|
||||
|
||||
mf := d.fs.mfp.MemoryFile()
|
||||
h := d.readHandle()
|
||||
_, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt)
|
||||
_, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, pgalloc.AllocateAndWritePopulate, h.readToBlocksAt)
|
||||
|
||||
var ts []memmap.Translation
|
||||
var translatedEnd uint64
|
||||
|
||||
@@ -307,7 +307,7 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap.
|
||||
}
|
||||
optional = required
|
||||
}
|
||||
pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.inode.fs.mf, rf.memoryUsageKind, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
|
||||
pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.inode.fs.mf, rf.memoryUsageKind, pgalloc.AllocateOnly, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
|
||||
// Newly-allocated pages are zeroed, so we don't need to do anything.
|
||||
return dsts.NumBytes(), nil
|
||||
})
|
||||
@@ -385,10 +385,9 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint
|
||||
if !f.inode.fs.accountPages(pagesToFill) {
|
||||
return linuxerr.ENOSPC
|
||||
}
|
||||
// Pass populate = true here despite the fact that we don't touch these pages
|
||||
// both for consistency with the expected behavior of fallocate(2) and in
|
||||
// expectation of a future write to them.
|
||||
pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.inode.fs.mf, f.memoryUsageKind, true /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
|
||||
// Given our definitions in pgalloc, fallocate(2) semantics imply that pages
|
||||
// in the MemoryFile must be committed, in addition to being allocated.
|
||||
pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.inode.fs.mf, f.memoryUsageKind, pgalloc.AllocateAndCommit, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
|
||||
// Newly-allocated pages are zeroed, so we don't need to do anything.
|
||||
return dsts.NumBytes(), nil
|
||||
})
|
||||
|
||||
@@ -162,12 +162,18 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
|
||||
newFSType = tmpfsOpts.FilesystemType
|
||||
}
|
||||
if tmpfsOpts.FilestoreFD != nil {
|
||||
// DecommitOnDestroy because tmpfsOpts.FilestoreFD may be backed by a
|
||||
// host filesystem based file, which needs to be decommited on destroy.
|
||||
// DisableIMAWorkAround because sentry's seccomp filters don't allow the
|
||||
// mmap(2) syscalls that this work around uses. User of this feature is
|
||||
// expected to have performed the work around outside the sandbox.
|
||||
mfOpts := pgalloc.MemoryFileOpts{DecommitOnDestroy: true, DisableIMAWorkAround: true}
|
||||
mfOpts := pgalloc.MemoryFileOpts{
|
||||
// tmpfsOpts.FilestoreFD may be backed by a file on disk (not memfd),
|
||||
// which needs to be decommited on destroy to release disk space.
|
||||
DecommitOnDestroy: true,
|
||||
// sentry's seccomp filters don't allow the mmap(2) syscalls that
|
||||
// pgalloc.IMAWorkAroundForMemFile() uses. Users of tmpfsOpts.FilestoreFD
|
||||
// are expected to have performed the work around outside the sandbox.
|
||||
DisableIMAWorkAround: true,
|
||||
// Custom filestore FDs are usually backed by files on disk. Ideally we
|
||||
// would confirm with fstatfs(2) but that is prohibited by seccomp.
|
||||
DiskBackedFile: true,
|
||||
}
|
||||
var err error
|
||||
mf, err = pgalloc.NewMemoryFile(tmpfsOpts.FilestoreFD.ReleaseToFile("overlay-filestore"), mfOpts)
|
||||
if err != nil {
|
||||
|
||||
@@ -111,7 +111,7 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui
|
||||
// - required.Length() > 0.
|
||||
// - optional.IsSupersetOf(required).
|
||||
// - required and optional must be page-aligned.
|
||||
func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, populate bool, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) {
|
||||
func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, allocMode pgalloc.AllocationMode, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) {
|
||||
gap := frs.LowerBoundGap(required.Start)
|
||||
var pagesAlloced uint64
|
||||
for gap.Ok() && gap.Start() < required.End {
|
||||
@@ -122,7 +122,7 @@ func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.Map
|
||||
gr := gap.Range().Intersect(optional)
|
||||
|
||||
// Read data into the gap.
|
||||
fr, err := mf.AllocateAndFill(gr.Length(), kind, populate, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) {
|
||||
fr, err := mf.AllocateAndFill(gr.Length(), kind, allocMode, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) {
|
||||
var done uint64
|
||||
for !dsts.IsEmpty() {
|
||||
n, err := func() (uint64, error) {
|
||||
|
||||
@@ -373,7 +373,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
|
||||
return pstart, pseg.PrevGap(), err
|
||||
}
|
||||
// Copy contents.
|
||||
fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, true /* populate */, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)})
|
||||
fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, pgalloc.AllocateAndWritePopulate, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)})
|
||||
if _, ok := err.(safecopy.BusError); ok {
|
||||
// If we got SIGBUS during the copy, deliver SIGBUS to
|
||||
// userspace (instead of SIGSEGV) if we're breaking
|
||||
|
||||
@@ -209,6 +209,9 @@ type MemoryFileOpts struct {
|
||||
// If DisableIMAWorkAround is true, NewMemoryFile will not call
|
||||
// IMAWorkAroundForMemFile().
|
||||
DisableIMAWorkAround bool
|
||||
|
||||
// DiskBackedFile indicates that the MemoryFile is backed by a file on disk.
|
||||
DiskBackedFile bool
|
||||
}
|
||||
|
||||
// DelayedEvictionType is the type of MemoryFileOpts.DelayedEviction.
|
||||
@@ -580,35 +583,77 @@ func findAvailableRangeBottomUp(usage *usageSet, length, alignment uint64) (memm
|
||||
panic(fmt.Sprintf("NextLargeEnoughGap didn't return a gap at the end, length: %d", length))
|
||||
}
|
||||
|
||||
// AllocationMode provides a way to inform the pgalloc API how to allocate
|
||||
// memory and pages on the host.
|
||||
// A page will exist in one of the following incremental states:
|
||||
// 1. Allocated: A page is allocated if it was returned by Allocate() and its
|
||||
// reference count hasn't dropped to 0 since then.
|
||||
// 2. Committed: As described in MemoryFile documentation above, a page is
|
||||
// committed if the host kernel is spending resources to store its
|
||||
// contents. A committed page is implicitly allocated.
|
||||
// 3. Populated: A page is populated for reading/writing in a page table
|
||||
// hierarchy if it has a page table entry that permits reading/writing
|
||||
// respectively. A populated page is implicitly committed, since the page
|
||||
// table entry needs a physical page to point to, but not vice versa.
|
||||
type AllocationMode int
|
||||
|
||||
const (
|
||||
// AllocateOnly indicates that pages need to only be allocated.
|
||||
AllocateOnly AllocationMode = iota
|
||||
// AllocateAndCommit indicates that pages need to be committed, in addition
|
||||
// to being allocated.
|
||||
AllocateAndCommit
|
||||
// AllocateAndWritePopulate indicates that writable pages should ideally be
|
||||
// populated in the page table, in addition to being allocated. This is a
|
||||
// suggestion, not a requirement.
|
||||
AllocateAndWritePopulate
|
||||
)
|
||||
|
||||
// AllocateAndFill allocates memory of the given kind and fills it by calling
|
||||
// r.ReadToBlocks() repeatedly until either length bytes are read or a non-nil
|
||||
// error is returned. It returns the memory filled by r, truncated down to the
|
||||
// nearest page. If this is shorter than length bytes due to an error returned
|
||||
// by r.ReadToBlocks(), it returns that error.
|
||||
//
|
||||
// If populate is true, AllocateAndFill will attempt to pre-fault pages in bulk
|
||||
// in the safemem.BlockSeq passed to r. Callers that will fill the allocated
|
||||
// memory by writing to it in the sentry should pass populate = true to avoid
|
||||
// faulting page-by-page. Callers that will fill the allocated memory by
|
||||
// invoking host system calls should pass populate = false.
|
||||
// allocMode allows the callers to select how the pages are allocated in the
|
||||
// MemoryFile. Callers that will fill the allocated memory by writing to it
|
||||
// should pass AllocateAndWritePopulate to avoid faulting page-by-page. Callers
|
||||
// that will fill the allocated memory by invoking host system calls should
|
||||
// pass AllocateOnly. Note that the mode may be upgraded in certain scenarios
|
||||
// for performance. See implementation for more details.
|
||||
//
|
||||
// Preconditions:
|
||||
// - length > 0.
|
||||
// - length must be page-aligned.
|
||||
func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, populate bool, r safemem.Reader) (memmap.FileRange, error) {
|
||||
func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, allocMode AllocationMode, r safemem.Reader) (memmap.FileRange, error) {
|
||||
if !f.opts.DiskBackedFile && allocMode == AllocateAndCommit {
|
||||
// Upgrade to AllocateAndWritePopulate for memory(shmem)-backed files. We
|
||||
// take a more aggressive approach in populating pages for memory-backed
|
||||
// MemoryFiles. shmem pages are subject to swap rather than disk writeback.
|
||||
// They are not likely to be swapped before they are written to. Hence it
|
||||
// is beneficial to populate (in addition to commit) shmem pages to avoid
|
||||
// faulting page-by-page when these pages are written to in the future.
|
||||
allocMode = AllocateAndWritePopulate
|
||||
}
|
||||
fr, err := f.Allocate(length, AllocOpts{Kind: kind})
|
||||
if err != nil {
|
||||
return memmap.FileRange{}, err
|
||||
}
|
||||
if allocMode == AllocateAndCommit {
|
||||
if err := f.commitFile(fr); err != nil {
|
||||
f.DecRef(fr)
|
||||
return memmap.FileRange{}, err
|
||||
}
|
||||
}
|
||||
dsts, err := f.MapInternal(fr, hostarch.Write)
|
||||
if err != nil {
|
||||
f.DecRef(fr)
|
||||
return memmap.FileRange{}, err
|
||||
}
|
||||
if populate && canPopulate() {
|
||||
if allocMode == AllocateAndWritePopulate && canPopulate() {
|
||||
rem := dsts
|
||||
for {
|
||||
if !tryPopulate(rem.Head()) {
|
||||
if !f.tryPopulate(rem.Head()) {
|
||||
break
|
||||
}
|
||||
rem = rem.Tail()
|
||||
@@ -634,7 +679,11 @@ func canPopulate() bool {
|
||||
return mlockDisabled.Load() == 0
|
||||
}
|
||||
|
||||
func tryPopulate(b safemem.Block) bool {
|
||||
func (f *MemoryFile) tryPopulate(b safemem.Block) bool {
|
||||
// For disk-backed MemoryFiles, mlock+munlock will populate pages read-only.
|
||||
if mlockDisabled.Load() != 0 || f.opts.DiskBackedFile {
|
||||
return false
|
||||
}
|
||||
// Call mlock to populate pages, then munlock to cancel the mlock (but keep
|
||||
// the pages populated). Only do so for hugepage-aligned address ranges to
|
||||
// ensure that splitting the VMA in mlock doesn't split any existing
|
||||
@@ -705,6 +754,16 @@ func (f *MemoryFile) manuallyZero(fr memmap.FileRange) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (f *MemoryFile) commitFile(fr memmap.FileRange) error {
|
||||
// "The default operation (i.e., mode is zero) of fallocate() allocates the
|
||||
// disk space within the range specified by offset and len." - fallocate(2)
|
||||
return unix.Fallocate(
|
||||
int(f.file.Fd()),
|
||||
0, // mode
|
||||
int64(fr.Start),
|
||||
int64(fr.Length()))
|
||||
}
|
||||
|
||||
func (f *MemoryFile) decommitFile(fr memmap.FileRange) error {
|
||||
// "After a successful call, subsequent reads from this range will
|
||||
// return zeroes. The FALLOC_FL_PUNCH_HOLE flag must be ORed with
|
||||
|
||||
Reference in New Issue
Block a user