diff --git a/pkg/sentry/fs/fsutil/file_range_set.go b/pkg/sentry/fs/fsutil/file_range_set.go index 79ff740e8..3f1bdcd83 100644 --- a/pkg/sentry/fs/fsutil/file_range_set.go +++ b/pkg/sentry/fs/fsutil/file_range_set.go @@ -98,7 +98,8 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui // bytes have been read.) EOF is handled consistently with the requirements of // mmap(2): bytes after EOF on the same page are zeroed; pages after EOF are // invalid. fileSize is an upper bound on the file's size; bytes after fileSize -// will be zeroed without calling readAt. +// will be zeroed without calling readAt. populate has the same meaning as the +// pgalloc.MemoryFile.AllocateAndFill() argument of the same name. // // Fill may read offsets outside of required, but will never read offsets // outside of optional. It returns a non-nil error if any error occurs, even @@ -110,7 +111,7 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui // - required.Length() > 0. // - optional.IsSupersetOf(required). // - required and optional must be page-aligned. -func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) { +func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, populate bool, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) { gap := frs.LowerBoundGap(required.Start) var pagesAlloced uint64 for gap.Ok() && gap.Start() < required.End { @@ -121,7 +122,7 @@ func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.Map gr := gap.Range().Intersect(optional) // Read data into the gap. - fr, err := mf.AllocateAndFill(gr.Length(), kind, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) { + fr, err := mf.AllocateAndFill(gr.Length(), kind, populate, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) { var done uint64 for !dsts.IsEmpty() { n, err := func() (uint64, error) { diff --git a/pkg/sentry/fs/fsutil/inode_cached.go b/pkg/sentry/fs/fsutil/inode_cached.go index 737ccf04e..fb2750b55 100644 --- a/pkg/sentry/fs/fsutil/inode_cached.go +++ b/pkg/sentry/fs/fsutil/inode_cached.go @@ -658,7 +658,7 @@ func (rw *inodeReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) { End: fs.OffsetPageEnd(int64(gapMR.End)), } optMR := gap.Range() - _, err := rw.c.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), uint64(rw.c.attr.Size), mem, usage.PageCache, rw.c.backingFile.ReadToBlocksAt) + _, err := rw.c.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), uint64(rw.c.attr.Size), mem, usage.PageCache, false /* populate */, rw.c.backingFile.ReadToBlocksAt) mem.MarkEvictable(rw.c, pgalloc.EvictableRange{optMR.Start, optMR.End}) seg, gap = rw.c.cache.Find(uint64(rw.offset)) if !seg.Ok() { @@ -891,7 +891,7 @@ func (c *CachingInodeOperations) Translate(ctx context.Context, required, option } mf := c.mfp.MemoryFile() - _, cerr := c.cache.Fill(ctx, required, maxFillRange(required, optional), uint64(c.attr.Size), mf, usage.PageCache, c.backingFile.ReadToBlocksAt) + _, cerr := c.cache.Fill(ctx, required, maxFillRange(required, optional), uint64(c.attr.Size), mf, usage.PageCache, false /* populate */, c.backingFile.ReadToBlocksAt) var ts []memmap.Translation var translatedEnd uint64 diff --git a/pkg/sentry/fs/tmpfs/inode_file.go b/pkg/sentry/fs/tmpfs/inode_file.go index 0d223bfbb..4c3493754 100644 --- a/pkg/sentry/fs/tmpfs/inode_file.go +++ b/pkg/sentry/fs/tmpfs/inode_file.go @@ -614,7 +614,7 @@ func (f *fileInodeOperations) Translate(ctx context.Context, required, optional } mf := f.kernel.MemoryFile() - _, cerr := f.data.Fill(ctx, required, optional, uint64(f.attr.Size), mf, f.memUsage, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { + _, cerr := f.data.Fill(ctx, required, optional, uint64(f.attr.Size), mf, f.memUsage, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { // Newly-allocated pages are zeroed, so we don't need to do anything. return dsts.NumBytes(), nil }) diff --git a/pkg/sentry/fsimpl/gofer/regular_file.go b/pkg/sentry/fsimpl/gofer/regular_file.go index d2e6d6959..8ffb0b92a 100644 --- a/pkg/sentry/fsimpl/gofer/regular_file.go +++ b/pkg/sentry/fsimpl/gofer/regular_file.go @@ -467,7 +467,7 @@ func (rw *dentryReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) End: gapEnd, } optMR := gap.Range() - _, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, h.readToBlocksAt) + _, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt) mf.MarkEvictable(rw.d, pgalloc.EvictableRange{optMR.Start, optMR.End}) seg, gap = rw.d.cache.Find(rw.off) if !seg.Ok() { @@ -826,7 +826,7 @@ func (d *dentry) Translate(ctx context.Context, required, optional memmap.Mappab mf := d.fs.mfp.MemoryFile() h := d.readHandleLocked() - _, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, h.readToBlocksAt) + _, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt) var ts []memmap.Translation var translatedEnd uint64 diff --git a/pkg/sentry/fsimpl/tmpfs/regular_file.go b/pkg/sentry/fsimpl/tmpfs/regular_file.go index af843f41a..556025086 100644 --- a/pkg/sentry/fsimpl/tmpfs/regular_file.go +++ b/pkg/sentry/fsimpl/tmpfs/regular_file.go @@ -315,7 +315,7 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap. optional = required } } - pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { + pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { // Newly-allocated pages are zeroed, so we don't need to do anything. return dsts.NumBytes(), nil }) @@ -397,7 +397,10 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint return linuxerr.ENOSPC } } - pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { + // Pass populate = true here despite the fact that we don't touch these pages + // both for consistency with the expected behavior of fallocate(2) and in + // expectation of a future write to them. + pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, true /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) { // Newly-allocated pages are zeroed, so we don't need to do anything. return dsts.NumBytes(), nil }) diff --git a/pkg/sentry/mm/pma.go b/pkg/sentry/mm/pma.go index 468e11932..40f04f525 100644 --- a/pkg/sentry/mm/pma.go +++ b/pkg/sentry/mm/pma.go @@ -373,7 +373,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter return pstart, pseg.PrevGap(), err } // Copy contents. - fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)}) + fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, true /* populate */, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)}) if _, ok := err.(safecopy.BusError); ok { // If we got SIGBUS during the copy, deliver SIGBUS to // userspace (instead of SIGSEGV) if we're breaking diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index 252535f54..146784ce9 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -30,6 +30,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" @@ -571,10 +572,16 @@ func findAvailableRangeBottomUp(usage *usageSet, length, alignment uint64) (memm // nearest page. If this is shorter than length bytes due to an error returned // by r.ReadToBlocks(), it returns that error. // +// If populate is true, AllocateAndFill will attempt to pre-fault pages in bulk +// in the safemem.BlockSeq passed to r. Callers that will fill the allocated +// memory by writing to it in the sentry should pass populate = true to avoid +// faulting page-by-page. Callers that will fill the allocated memory by +// invoking host system calls should pass populate = false. +// // Preconditions: // - length > 0. // - length must be page-aligned. -func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r safemem.Reader) (memmap.FileRange, error) { +func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, populate bool, r safemem.Reader) (memmap.FileRange, error) { fr, err := f.Allocate(length, AllocOpts{Kind: kind}) if err != nil { return memmap.FileRange{}, err @@ -584,6 +591,18 @@ func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r saf f.DecRef(fr) return memmap.FileRange{}, err } + if populate && canPopulate() { + rem := dsts + for { + if !tryPopulate(rem.Head()) { + break + } + rem = rem.Tail() + if rem.IsEmpty() { + break + } + } + } n, err := safemem.ReadFullToBlocks(r, dsts) un := uint64(hostarch.Addr(n).RoundDown()) if un < length { @@ -595,6 +614,43 @@ func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r saf return fr, err } +var mlockDisabled atomicbitops.Uint32 + +func canPopulate() bool { + return mlockDisabled.Load() == 0 +} + +func tryPopulate(b safemem.Block) bool { + // Call mlock to populate pages, then munlock to cancel the mlock (but keep + // the pages populated). Only do so for hugepage-aligned address ranges to + // ensure that splitting the VMA in mlock doesn't split any existing + // hugepages. This assumes that two host syscalls, plus the MM overhead of + // mlock + munlock, is faster on average than trapping for + // HugePageSize/PageSize small page faults. + start, ok := hostarch.Addr(b.Addr()).HugeRoundUp() + if !ok { + return true + } + end := hostarch.Addr(b.Addr() + uintptr(b.Len())).HugeRoundDown() + if start >= end { + return true + } + _, _, errno := unix.Syscall(unix.SYS_MLOCK, uintptr(start), uintptr(end-start), 0) + unix.RawSyscall(unix.SYS_MUNLOCK, uintptr(start), uintptr(end-start), 0) + if errno != 0 { + if errno == unix.ENOMEM || errno == unix.EPERM { + // These errors are expected from hitting non-zero RLIMIT_MEMLOCK, or + // hitting zero RLIMIT_MEMLOCK without CAP_IPC_LOCK, respectively. + log.Infof("Disabling pgalloc.MemoryFile.AllocateAndFill pre-population: mlock failed: %s", errno) + } else { + log.Warningf("Disabling pgalloc.MemoryFile.AllocateAndFill pre-population: mlock failed: %s", errno) + } + mlockDisabled.Store(1) + return false + } + return true +} + // fallocate(2) modes, defined in Linux's include/uapi/linux/falloc.h. const ( _FALLOC_FL_KEEP_SIZE = 1 diff --git a/runsc/boot/filter/config.go b/runsc/boot/filter/config.go index fe09a4bc1..b47ae032a 100644 --- a/runsc/boot/filter/config.go +++ b/runsc/boot/filter/config.go @@ -188,6 +188,7 @@ var allowedSyscalls = seccomp.SyscallRules{ }, }, unix.SYS_MINCORE: {}, + unix.SYS_MLOCK: {}, unix.SYS_MMAP: []seccomp.Rule{ { seccomp.MatchAny{}, @@ -233,6 +234,7 @@ var allowedSyscalls = seccomp.SyscallRules{ }, }, unix.SYS_MPROTECT: {}, + unix.SYS_MUNLOCK: {}, unix.SYS_MUNMAP: {}, unix.SYS_NANOSLEEP: {}, unix.SYS_PPOLL: {}, diff --git a/runsc/cli/main.go b/runsc/cli/main.go index af62a343e..3aefb32f1 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -238,6 +238,23 @@ func Main(version string) { signal.Ignore(unix.SIGTERM) } + // pgalloc.MemoryFile (which provides application memory) sometimes briefly + // mlock(2)s ranges of memory in order to fault in a large number of pages at + // a time. Try to make RLIMIT_MEMLOCK unlimited so that it can do so. runsc + // expects to run in a memory cgroup that limits its memory usage as + // required. + var rlim unix.Rlimit + if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlim); err != nil { + log.Warningf("Failed to get RLIMIT_MEMLOCK: %v", err) + } else if rlim.Cur != unix.RLIM_INFINITY || rlim.Max != unix.RLIM_INFINITY { + rlim.Cur = unix.RLIM_INFINITY + rlim.Max = unix.RLIM_INFINITY + if err := unix.Setrlimit(unix.RLIMIT_MEMLOCK, &rlim); err != nil { + // We may not have CAP_SYS_RESOURCE, so this failure may be expected. + log.Infof("Failed to set RLIMIT_MEMLOCK: %v", err) + } + } + // Call the subcommand and pass in the configuration. var ws unix.WaitStatus subcmdCode := subcommands.Execute(context.Background(), conf, &ws)