From 1f101ba738bf38cb5bc9b021d18c5023081c6c4f Mon Sep 17 00:00:00 2001 From: Jamie Liu Date: Thu, 6 Oct 2022 18:19:52 -0700 Subject: [PATCH] Add populate parameter to pgalloc.MemoryFile.AllocateAndFill(). This targets the issue raised by #7977, but uses mlock(2) to force mm_populate() of a precise address range, and also enables the use of the same feature in other places where it makes sense (when breaking copy-on-write, and when filling gofer page caches). On the benchmark given in #7977, using KVM platform: ``` docker exec -ti $id fio --output-format=json --ioengine=sync --name=write --size=10G --blocksize=4K --filename=/data/data.test --iodepth=4 --rw=write ``` Before this CL: ``` "write" : { "io_bytes" : 10737418240, "io_kbytes" : 10485760, "bw" : 889981, "iops" : 222495.331862, "runtime" : 11782, "total_ios" : 2621440, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000 }, "clat_ns" : { "min" : 3678, "max" : 183826, "mean" : 4253.962175, "stddev" : 1003.818651, "percentile" : { "1.000000" : 3792, "5.000000" : 3824, "10.000000" : 3856, "20.000000" : 3888, "30.000000" : 3920, "40.000000" : 3952, "50.000000" : 3984, "60.000000" : 4016, "70.000000" : 4080, "80.000000" : 4576, "90.000000" : 5024, "95.000000" : 5344, "99.000000" : 6752, "99.500000" : 7840, "99.900000" : 16512, "99.950000" : 19840, "99.990000" : 28544, "0.00" : 0, "0.00" : 0, "0.00" : 0 } }, "lat_ns" : { "min" : 3714, "max" : 183863, "mean" : 4294.114721, "stddev" : 1008.663442 }, "bw_min" : 760015, "bw_max" : 922560, "bw_agg" : 100.000000, "bw_mean" : 892699.913043, "bw_dev" : 41761.998527, "bw_samples" : 23, "iops_min" : 190003, "iops_max" : 230640, "iops_mean" : 223174.826087, "iops_stddev" : 10440.656065, "iops_samples" : 23 }, ``` After this CL: ``` "write" : { "io_bytes" : 10737418240, "io_kbytes" : 10485760, "bw" : 1354049, "iops" : 338512.396694, "runtime" : 7744, "total_ios" : 2621440, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000 }, "clat_ns" : { "min" : 2471, "max" : 125666, "mean" : 2732.605543, "stddev" : 702.327985, "percentile" : { "1.000000" : 2512, "5.000000" : 2544, "10.000000" : 2576, "20.000000" : 2576, "30.000000" : 2576, "40.000000" : 2608, "50.000000" : 2608, "60.000000" : 2640, "70.000000" : 2672, "80.000000" : 2768, "90.000000" : 3056, "95.000000" : 3088, "99.000000" : 3888, "99.500000" : 4576, "99.900000" : 13504, "99.950000" : 18048, "99.990000" : 25984, "0.00" : 0, "0.00" : 0, "0.00" : 0 } }, "lat_ns" : { "min" : 2514, "max" : 125707, "mean" : 2778.332321, "stddev" : 708.893090 }, "bw_min" : 1173397, "bw_max" : 1402299, "bw_agg" : 99.894475, "bw_mean" : 1352620.133333, "bw_dev" : 74302.901710, "bw_samples" : 15, "iops_min" : 293349, "iops_max" : 350574, "iops_mean" : 338154.866667, "iops_stddev" : 18575.487826, "iops_samples" : 15 }, ``` Simple process creation benchmark to exercise copy-on-write breaking `docker run --runtime=runsc debian /bin/bash -c 'time for i in {1..10000}; do /bin/true; done;'` using KVM platform: Before this CL: ``` real 1m5.794s user 0m7.680s sys 1m37.050s ``` After this CL: ``` real 0m57.837s user 0m7.310s sys 1m30.500s ``` PiperOrigin-RevId: 479461293 --- pkg/sentry/fs/fsutil/file_range_set.go | 7 +-- pkg/sentry/fs/fsutil/inode_cached.go | 4 +- pkg/sentry/fs/tmpfs/inode_file.go | 2 +- pkg/sentry/fsimpl/gofer/regular_file.go | 4 +- pkg/sentry/fsimpl/tmpfs/regular_file.go | 7 ++- pkg/sentry/mm/pma.go | 2 +- pkg/sentry/pgalloc/pgalloc.go | 58 ++++++++++++++++++++++++- runsc/boot/filter/config.go | 2 + runsc/cli/main.go | 17 ++++++++ 9 files changed, 91 insertions(+), 12 deletions(-) diff --git a/pkg/sentry/fs/fsutil/file_range_set.go b/pkg/sentry/fs/fsutil/file_range_set.go index 79ff740e8..3f1bdcd83 100644 --- a/pkg/sentry/fs/fsutil/file_range_set.go +++ b/pkg/sentry/fs/fsutil/file_range_set.go @@ -98,7 +98,8 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui // bytes have been read.) EOF is handled consistently with the requirements of // mmap(2): bytes after EOF on the same page are zeroed; pages after EOF are // invalid. fileSize is an upper bound on the file's size; bytes after fileSize -// will be zeroed without calling readAt. +// will be zeroed without calling readAt. populate has the same meaning as the +// pgalloc.MemoryFile.AllocateAndFill() argument of the same name. // // Fill may read offsets outside of required, but will never read offsets // outside of optional. It returns a non-nil error if any error occurs, even @@ -110,7 +111,7 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui // - required.Length() > 0. // - optional.IsSupersetOf(required). // - required and optional must be page-aligned. -func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) { +func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, populate bool, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) { gap := frs.LowerBoundGap(required.Start) var pagesAlloced uint64 for gap.Ok() && gap.Start() < required.End { @@ -121,7 +122,7 @@ func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.Map gr := gap.Range().Intersect(optional) // Read data into the gap. - fr, err := mf.AllocateAndFill(gr.Length(), kind, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) { + fr, err := mf.AllocateAndFill(gr.Length(), kind, populate, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) { var done uint64 for !dsts.IsEmpty() { n, err := func() (uint64, error) { diff --git a/pkg/sentry/fs/fsutil/inode_cached.go b/pkg/sentry/fs/fsutil/inode_cached.go index 737ccf04e..fb2750b55 100644 --- a/pkg/sentry/fs/fsutil/inode_cached.go +++ b/pkg/sentry/fs/fsutil/inode_cached.go @@ -658,7 +658,7 @@ func (rw *inodeReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) { End: fs.OffsetPageEnd(int64(gapMR.End)), } optMR := gap.Range() - _, err := rw.c.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), uint64(rw.c.attr.Size), mem, usage.PageCache, rw.c.backingFile.ReadToBlocksAt) + _, err := rw.c.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), uint64(rw.c.attr.Size), mem, usage.PageCache, false /* populate */, rw.c.backingFile.ReadToBlocksAt) mem.MarkEvictable(rw.c, pgalloc.EvictableRange{optMR.Start, optMR.End}) seg, gap = rw.c.cache.Find(uint64(rw.offset)) if !seg.Ok() { @@ -891,7 +891,7 @@ func (c *CachingInodeOperations) Translate(ctx context.Context, required, option } mf := c.mfp.MemoryFile() - _, cerr := c.cache.Fill(ctx, required, maxFillRange(required, optional), uint64(c.attr.Size), mf, usage.PageCache, c.backingFile.ReadToBlocksAt) + _, cerr := c.cache.Fill(ctx, required, maxFillRange(required, optional), uint64(c.attr.Size), mf, usage.PageCache, false /* populate */, c.backingFile.ReadToBlocksAt) var ts []memmap.Translation var translatedEnd uint64 diff --git a/pkg/sentry/fs/tmpfs/inode_file.go b/pkg/sentry/fs/tmpfs/inode_file.go index 0d223bfbb..4c3493754 100644 --- a/pkg/sentry/fs/tmpfs/inode_file.go +++ b/pkg/sentry/fs/tmpfs/inode_file.go @@ -614,7 +614,7 @@ func (f *fileInodeOperations) Translate(ctx context.Context, required, optional } mf := f.kernel.MemoryFile() - _, cerr := f.data.Fill(ctx, required, optional, uint64(f.attr.Size), mf, f.memUsage, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { + _, cerr := f.data.Fill(ctx, required, optional, uint64(f.attr.Size), mf, f.memUsage, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { // Newly-allocated pages are zeroed, so we don't need to do anything. return dsts.NumBytes(), nil }) diff --git a/pkg/sentry/fsimpl/gofer/regular_file.go b/pkg/sentry/fsimpl/gofer/regular_file.go index d2e6d6959..8ffb0b92a 100644 --- a/pkg/sentry/fsimpl/gofer/regular_file.go +++ b/pkg/sentry/fsimpl/gofer/regular_file.go @@ -467,7 +467,7 @@ func (rw *dentryReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) End: gapEnd, } optMR := gap.Range() - _, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, h.readToBlocksAt) + _, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt) mf.MarkEvictable(rw.d, pgalloc.EvictableRange{optMR.Start, optMR.End}) seg, gap = rw.d.cache.Find(rw.off) if !seg.Ok() { @@ -826,7 +826,7 @@ func (d *dentry) Translate(ctx context.Context, required, optional memmap.Mappab mf := d.fs.mfp.MemoryFile() h := d.readHandleLocked() - _, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, h.readToBlocksAt) + _, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt) var ts []memmap.Translation var translatedEnd uint64 diff --git a/pkg/sentry/fsimpl/tmpfs/regular_file.go b/pkg/sentry/fsimpl/tmpfs/regular_file.go index af843f41a..556025086 100644 --- a/pkg/sentry/fsimpl/tmpfs/regular_file.go +++ b/pkg/sentry/fsimpl/tmpfs/regular_file.go @@ -315,7 +315,7 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap. optional = required } } - pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { + pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { // Newly-allocated pages are zeroed, so we don't need to do anything. return dsts.NumBytes(), nil }) @@ -397,7 +397,10 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint return linuxerr.ENOSPC } } - pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { + // Pass populate = true here despite the fact that we don't touch these pages + // both for consistency with the expected behavior of fallocate(2) and in + // expectation of a future write to them. + pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, true /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { // Newly-allocated pages are zeroed, so we don't need to do anything. return dsts.NumBytes(), nil }) diff --git a/pkg/sentry/mm/pma.go b/pkg/sentry/mm/pma.go index 468e11932..40f04f525 100644 --- a/pkg/sentry/mm/pma.go +++ b/pkg/sentry/mm/pma.go @@ -373,7 +373,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter return pstart, pseg.PrevGap(), err } // Copy contents. - fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)}) + fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, true /* populate */, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)}) if _, ok := err.(safecopy.BusError); ok { // If we got SIGBUS during the copy, deliver SIGBUS to // userspace (instead of SIGSEGV) if we're breaking diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index 252535f54..146784ce9 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -30,6 +30,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" @@ -571,10 +572,16 @@ func findAvailableRangeBottomUp(usage *usageSet, length, alignment uint64) (memm // nearest page. If this is shorter than length bytes due to an error returned // by r.ReadToBlocks(), it returns that error. // +// If populate is true, AllocateAndFill will attempt to pre-fault pages in bulk +// in the safemem.BlockSeq passed to r. Callers that will fill the allocated +// memory by writing to it in the sentry should pass populate = true to avoid +// faulting page-by-page. Callers that will fill the allocated memory by +// invoking host system calls should pass populate = false. +// // Preconditions: // - length > 0. // - length must be page-aligned. -func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r safemem.Reader) (memmap.FileRange, error) { +func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, populate bool, r safemem.Reader) (memmap.FileRange, error) { fr, err := f.Allocate(length, AllocOpts{Kind: kind}) if err != nil { return memmap.FileRange{}, err @@ -584,6 +591,18 @@ func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r saf f.DecRef(fr) return memmap.FileRange{}, err } + if populate && canPopulate() { + rem := dsts + for { + if !tryPopulate(rem.Head()) { + break + } + rem = rem.Tail() + if rem.IsEmpty() { + break + } + } + } n, err := safemem.ReadFullToBlocks(r, dsts) un := uint64(hostarch.Addr(n).RoundDown()) if un < length { @@ -595,6 +614,43 @@ func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r saf return fr, err } +var mlockDisabled atomicbitops.Uint32 + +func canPopulate() bool { + return mlockDisabled.Load() == 0 +} + +func tryPopulate(b safemem.Block) bool { + // Call mlock to populate pages, then munlock to cancel the mlock (but keep + // the pages populated). Only do so for hugepage-aligned address ranges to + // ensure that splitting the VMA in mlock doesn't split any existing + // hugepages. This assumes that two host syscalls, plus the MM overhead of + // mlock + munlock, is faster on average than trapping for + // HugePageSize/PageSize small page faults. + start, ok := hostarch.Addr(b.Addr()).HugeRoundUp() + if !ok { + return true + } + end := hostarch.Addr(b.Addr() + uintptr(b.Len())).HugeRoundDown() + if start >= end { + return true + } + _, _, errno := unix.Syscall(unix.SYS_MLOCK, uintptr(start), uintptr(end-start), 0) + unix.RawSyscall(unix.SYS_MUNLOCK, uintptr(start), uintptr(end-start), 0) + if errno != 0 { + if errno == unix.ENOMEM || errno == unix.EPERM { + // These errors are expected from hitting non-zero RLIMIT_MEMLOCK, or + // hitting zero RLIMIT_MEMLOCK without CAP_IPC_LOCK, respectively. + log.Infof("Disabling pgalloc.MemoryFile.AllocateAndFill pre-population: mlock failed: %s", errno) + } else { + log.Warningf("Disabling pgalloc.MemoryFile.AllocateAndFill pre-population: mlock failed: %s", errno) + } + mlockDisabled.Store(1) + return false + } + return true +} + // fallocate(2) modes, defined in Linux's include/uapi/linux/falloc.h. const ( _FALLOC_FL_KEEP_SIZE = 1 diff --git a/runsc/boot/filter/config.go b/runsc/boot/filter/config.go index fe09a4bc1..b47ae032a 100644 --- a/runsc/boot/filter/config.go +++ b/runsc/boot/filter/config.go @@ -188,6 +188,7 @@ var allowedSyscalls = seccomp.SyscallRules{ }, }, unix.SYS_MINCORE: {}, + unix.SYS_MLOCK: {}, unix.SYS_MMAP: []seccomp.Rule{ { seccomp.MatchAny{}, @@ -233,6 +234,7 @@ var allowedSyscalls = seccomp.SyscallRules{ }, }, unix.SYS_MPROTECT: {}, + unix.SYS_MUNLOCK: {}, unix.SYS_MUNMAP: {}, unix.SYS_NANOSLEEP: {}, unix.SYS_PPOLL: {}, diff --git a/runsc/cli/main.go b/runsc/cli/main.go index af62a343e..3aefb32f1 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -238,6 +238,23 @@ func Main(version string) { signal.Ignore(unix.SIGTERM) } + // pgalloc.MemoryFile (which provides application memory) sometimes briefly + // mlock(2)s ranges of memory in order to fault in a large number of pages at + // a time. Try to make RLIMIT_MEMLOCK unlimited so that it can do so. runsc + // expects to run in a memory cgroup that limits its memory usage as + // required. + var rlim unix.Rlimit + if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlim); err != nil { + log.Warningf("Failed to get RLIMIT_MEMLOCK: %v", err) + } else if rlim.Cur != unix.RLIM_INFINITY || rlim.Max != unix.RLIM_INFINITY { + rlim.Cur = unix.RLIM_INFINITY + rlim.Max = unix.RLIM_INFINITY + if err := unix.Setrlimit(unix.RLIMIT_MEMLOCK, &rlim); err != nil { + // We may not have CAP_SYS_RESOURCE, so this failure may be expected. + log.Infof("Failed to set RLIMIT_MEMLOCK: %v", err) + } + } + // Call the subcommand and pass in the configuration. var ws unix.WaitStatus subcmdCode := subcommands.Execute(context.Background(), conf, &ws)