Add populate parameter to pgalloc.MemoryFile.AllocateAndFill().

This targets the issue raised by #7977, but uses mlock(2) to force mm_populate() of a precise address range, and also enables the use of the same feature in other places where it makes sense (when breaking copy-on-write, and when filling gofer page caches). On the benchmark given in #7977, using KVM platform: ``` docker exec -ti $id fio --output-format=json --ioengine=sync --name=write --size=10G --blocksize=4K --filename=/data/data.test --iodepth=4 --rw=write ``` Before this CL: ``` "write" : { "io_bytes" : 10737418240, "io_kbytes" : 10485760, "bw" : 889981, "iops" : 222495.331862, "runtime" : 11782, "total_ios" : 2621440, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000 }, "clat_ns" : { "min" : 3678, "max" : 183826, "mean" : 4253.962175, "stddev" : 1003.818651, "percentile" : { "1.000000" : 3792, "5.000000" : 3824, "10.000000" : 3856, "20.000000" : 3888, "30.000000" : 3920, "40.000000" : 3952, "50.000000" : 3984, "60.000000" : 4016, "70.000000" : 4080, "80.000000" : 4576, "90.000000" : 5024, "95.000000" : 5344, "99.000000" : 6752, "99.500000" : 7840, "99.900000" : 16512, "99.950000" : 19840, "99.990000" : 28544, "0.00" : 0, "0.00" : 0, "0.00" : 0 } }, "lat_ns" : { "min" : 3714, "max" : 183863, "mean" : 4294.114721, "stddev" : 1008.663442 }, "bw_min" : 760015, "bw_max" : 922560, "bw_agg" : 100.000000, "bw_mean" : 892699.913043, "bw_dev" : 41761.998527, "bw_samples" : 23, "iops_min" : 190003, "iops_max" : 230640, "iops_mean" : 223174.826087, "iops_stddev" : 10440.656065, "iops_samples" : 23 }, ``` After this CL: ``` "write" : { "io_bytes" : 10737418240, "io_kbytes" : 10485760, "bw" : 1354049, "iops" : 338512.396694, "runtime" : 7744, "total_ios" : 2621440, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000 }, "clat_ns" : { "min" : 2471, "max" : 125666, "mean" : 2732.605543, "stddev" : 702.327985, "percentile" : { "1.000000" : 2512, "5.000000" : 2544, "10.000000" : 2576, "20.000000" : 2576, "30.000000" : 2576, "40.000000" : 2608, "50.000000" : 2608, "60.000000" : 2640, "70.000000" : 2672, "80.000000" : 2768, "90.000000" : 3056, "95.000000" : 3088, "99.000000" : 3888, "99.500000" : 4576, "99.900000" : 13504, "99.950000" : 18048, "99.990000" : 25984, "0.00" : 0, "0.00" : 0, "0.00" : 0 } }, "lat_ns" : { "min" : 2514, "max" : 125707, "mean" : 2778.332321, "stddev" : 708.893090 }, "bw_min" : 1173397, "bw_max" : 1402299, "bw_agg" : 99.894475, "bw_mean" : 1352620.133333, "bw_dev" : 74302.901710, "bw_samples" : 15, "iops_min" : 293349, "iops_max" : 350574, "iops_mean" : 338154.866667, "iops_stddev" : 18575.487826, "iops_samples" : 15 }, ``` Simple process creation benchmark to exercise copy-on-write breaking `docker run --runtime=runsc debian /bin/bash -c 'time for i in {1..10000}; do /bin/true; done;'` using KVM platform: Before this CL: ``` real 1m5.794s user 0m7.680s sys 1m37.050s ``` After this CL: ``` real 0m57.837s user 0m7.310s sys 1m30.500s ``` PiperOrigin-RevId: 479461293
2026-05-22 17:12:49 -07:00 · 2022-10-06 18:22:39 -07:00
parent 4eca206fa4
commit 1f101ba738
9 changed files with 91 additions and 12 deletions
@@ -98,7 +98,8 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui
 // bytes have been read.) EOF is handled consistently with the requirements of
 // mmap(2): bytes after EOF on the same page are zeroed; pages after EOF are
 // invalid. fileSize is an upper bound on the file's size; bytes after fileSize
-// will be zeroed without calling readAt.
+// will be zeroed without calling readAt. populate has the same meaning as the
+// pgalloc.MemoryFile.AllocateAndFill() argument of the same name.
 //
 // Fill may read offsets outside of required, but will never read offsets
 // outside of optional. It returns a non-nil error if any error occurs, even
@@ -110,7 +111,7 @@ func (frs *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) ui
 //   - required.Length() > 0.
 //   - optional.IsSupersetOf(required).
 //   - required and optional must be page-aligned.
-func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) {
+func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, populate bool, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) {
 	gap := frs.LowerBoundGap(required.Start)
 	var pagesAlloced uint64
 	for gap.Ok() && gap.Start() < required.End {
@@ -121,7 +122,7 @@ func (frs *FileRangeSet) Fill(ctx context.Context, required, optional memmap.Map
 		gr := gap.Range().Intersect(optional)

 		// Read data into the gap.
-		fr, err := mf.AllocateAndFill(gr.Length(), kind, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) {
+		fr, err := mf.AllocateAndFill(gr.Length(), kind, populate, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) {
 			var done uint64
 			for !dsts.IsEmpty() {
 				n, err := func() (uint64, error) {
@@ -658,7 +658,7 @@ func (rw *inodeReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) {
 					End:   fs.OffsetPageEnd(int64(gapMR.End)),
 				}
 				optMR := gap.Range()
-				_, err := rw.c.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), uint64(rw.c.attr.Size), mem, usage.PageCache, rw.c.backingFile.ReadToBlocksAt)
+				_, err := rw.c.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), uint64(rw.c.attr.Size), mem, usage.PageCache, false /* populate */, rw.c.backingFile.ReadToBlocksAt)
 				mem.MarkEvictable(rw.c, pgalloc.EvictableRange{optMR.Start, optMR.End})
 				seg, gap = rw.c.cache.Find(uint64(rw.offset))
 				if !seg.Ok() {
@@ -891,7 +891,7 @@ func (c *CachingInodeOperations) Translate(ctx context.Context, required, option
 	}

 	mf := c.mfp.MemoryFile()
-	_, cerr := c.cache.Fill(ctx, required, maxFillRange(required, optional), uint64(c.attr.Size), mf, usage.PageCache, c.backingFile.ReadToBlocksAt)
+	_, cerr := c.cache.Fill(ctx, required, maxFillRange(required, optional), uint64(c.attr.Size), mf, usage.PageCache, false /* populate */, c.backingFile.ReadToBlocksAt)

 	var ts []memmap.Translation
 	var translatedEnd uint64
@@ -614,7 +614,7 @@ func (f *fileInodeOperations) Translate(ctx context.Context, required, optional
 	}

 	mf := f.kernel.MemoryFile()
-	_, cerr := f.data.Fill(ctx, required, optional, uint64(f.attr.Size), mf, f.memUsage, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
+	_, cerr := f.data.Fill(ctx, required, optional, uint64(f.attr.Size), mf, f.memUsage, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
 		// Newly-allocated pages are zeroed, so we don't need to do anything.
 		return dsts.NumBytes(), nil
 	})
@@ -467,7 +467,7 @@ func (rw *dentryReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error)
 					End:   gapEnd,
 				}
 				optMR := gap.Range()
-				_, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, h.readToBlocksAt)
+				_, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt)
 				mf.MarkEvictable(rw.d, pgalloc.EvictableRange{optMR.Start, optMR.End})
 				seg, gap = rw.d.cache.Find(rw.off)
 				if !seg.Ok() {
@@ -826,7 +826,7 @@ func (d *dentry) Translate(ctx context.Context, required, optional memmap.Mappab

 	mf := d.fs.mfp.MemoryFile()
 	h := d.readHandleLocked()
-	_, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, h.readToBlocksAt)
+	_, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, true /* populate */, h.readToBlocksAt)

 	var ts []memmap.Translation
 	var translatedEnd uint64
@@ -315,7 +315,7 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap.
 			optional = required
 		}
 	}
-	pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
+	pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.memFile, rf.memoryUsageKind, false /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
 		// Newly-allocated pages are zeroed, so we don't need to do anything.
 		return dsts.NumBytes(), nil
 	})
@@ -397,7 +397,10 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint
 			return linuxerr.ENOSPC
 		}
 	}
-	pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
+	// Pass populate = true here despite the fact that we don't touch these pages
+	// both for consistency with the expected behavior of fallocate(2) and in
+	// expectation of a future write to them.
+	pagesAlloced, err := f.data.Fill(ctx, required, required, newSize, f.memFile, f.memoryUsageKind, true /* populate */, func(_ context.Context, dsts safemem.BlockSeq, _ uint64) (uint64, error) {
 		// Newly-allocated pages are zeroed, so we don't need to do anything.
 		return dsts.NumBytes(), nil
 	})
@@ -373,7 +373,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
 						return pstart, pseg.PrevGap(), err
 					}
 					// Copy contents.
-					fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)})
+					fr, err := mf.AllocateAndFill(uint64(copyAR.Length()), usage.Anonymous, true /* populate */, &safemem.BlockSeqReader{mm.internalMappingsLocked(pseg, copyAR)})
 					if _, ok := err.(safecopy.BusError); ok {
 						// If we got SIGBUS during the copy, deliver SIGBUS to
 						// userspace (instead of SIGSEGV) if we're breaking
@@ -30,6 +30,7 @@ import (

 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/abi/linux"
+	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
 	"gvisor.dev/gvisor/pkg/hostarch"
@@ -571,10 +572,16 @@ func findAvailableRangeBottomUp(usage *usageSet, length, alignment uint64) (memm
 // nearest page. If this is shorter than length bytes due to an error returned
 // by r.ReadToBlocks(), it returns that error.
 //
+// If populate is true, AllocateAndFill will attempt to pre-fault pages in bulk
+// in the safemem.BlockSeq passed to r. Callers that will fill the allocated
+// memory by writing to it in the sentry should pass populate = true to avoid
+// faulting page-by-page. Callers that will fill the allocated memory by
+// invoking host system calls should pass populate = false.
+//
 // Preconditions:
 //   - length > 0.
 //   - length must be page-aligned.
-func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r safemem.Reader) (memmap.FileRange, error) {
+func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, populate bool, r safemem.Reader) (memmap.FileRange, error) {
 	fr, err := f.Allocate(length, AllocOpts{Kind: kind})
 	if err != nil {
 		return memmap.FileRange{}, err
@@ -584,6 +591,18 @@ func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r saf
 		f.DecRef(fr)
 		return memmap.FileRange{}, err
 	}
+	if populate && canPopulate() {
+		rem := dsts
+		for {
+			if !tryPopulate(rem.Head()) {
+				break
+			}
+			rem = rem.Tail()
+			if rem.IsEmpty() {
+				break
+			}
+		}
+	}
 	n, err := safemem.ReadFullToBlocks(r, dsts)
 	un := uint64(hostarch.Addr(n).RoundDown())
 	if un < length {
@@ -595,6 +614,43 @@ func (f *MemoryFile) AllocateAndFill(length uint64, kind usage.MemoryKind, r saf
 	return fr, err
 }

+var mlockDisabled atomicbitops.Uint32
+
+func canPopulate() bool {
+	return mlockDisabled.Load() == 0
+}
+
+func tryPopulate(b safemem.Block) bool {
+	// Call mlock to populate pages, then munlock to cancel the mlock (but keep
+	// the pages populated). Only do so for hugepage-aligned address ranges to
+	// ensure that splitting the VMA in mlock doesn't split any existing
+	// hugepages. This assumes that two host syscalls, plus the MM overhead of
+	// mlock + munlock, is faster on average than trapping for
+	// HugePageSize/PageSize small page faults.
+	start, ok := hostarch.Addr(b.Addr()).HugeRoundUp()
+	if !ok {
+		return true
+	}
+	end := hostarch.Addr(b.Addr() + uintptr(b.Len())).HugeRoundDown()
+	if start >= end {
+		return true
+	}
+	_, _, errno := unix.Syscall(unix.SYS_MLOCK, uintptr(start), uintptr(end-start), 0)
+	unix.RawSyscall(unix.SYS_MUNLOCK, uintptr(start), uintptr(end-start), 0)
+	if errno != 0 {
+		if errno == unix.ENOMEM || errno == unix.EPERM {
+			// These errors are expected from hitting non-zero RLIMIT_MEMLOCK, or
+			// hitting zero RLIMIT_MEMLOCK without CAP_IPC_LOCK, respectively.
+			log.Infof("Disabling pgalloc.MemoryFile.AllocateAndFill pre-population: mlock failed: %s", errno)
+		} else {
+			log.Warningf("Disabling pgalloc.MemoryFile.AllocateAndFill pre-population: mlock failed: %s", errno)
+		}
+		mlockDisabled.Store(1)
+		return false
+	}
+	return true
+}
+
 // fallocate(2) modes, defined in Linux's include/uapi/linux/falloc.h.
 const (
 	_FALLOC_FL_KEEP_SIZE  = 1
@@ -188,6 +188,7 @@ var allowedSyscalls = seccomp.SyscallRules{
 		},
 	},
 	unix.SYS_MINCORE: {},
+	unix.SYS_MLOCK:   {},
 	unix.SYS_MMAP: []seccomp.Rule{
 		{
 			seccomp.MatchAny{},
@@ -233,6 +234,7 @@ var allowedSyscalls = seccomp.SyscallRules{
 		},
 	},
 	unix.SYS_MPROTECT:  {},
+	unix.SYS_MUNLOCK:   {},
 	unix.SYS_MUNMAP:    {},
 	unix.SYS_NANOSLEEP: {},
 	unix.SYS_PPOLL:     {},
@@ -238,6 +238,23 @@ func Main(version string) {
 		signal.Ignore(unix.SIGTERM)
 	}

+	// pgalloc.MemoryFile (which provides application memory) sometimes briefly
+	// mlock(2)s ranges of memory in order to fault in a large number of pages at
+	// a time. Try to make RLIMIT_MEMLOCK unlimited so that it can do so. runsc
+	// expects to run in a memory cgroup that limits its memory usage as
+	// required.
+	var rlim unix.Rlimit
+	if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlim); err != nil {
+		log.Warningf("Failed to get RLIMIT_MEMLOCK: %v", err)
+	} else if rlim.Cur != unix.RLIM_INFINITY || rlim.Max != unix.RLIM_INFINITY {
+		rlim.Cur = unix.RLIM_INFINITY
+		rlim.Max = unix.RLIM_INFINITY
+		if err := unix.Setrlimit(unix.RLIMIT_MEMLOCK, &rlim); err != nil {
+			// We may not have CAP_SYS_RESOURCE, so this failure may be expected.
+			log.Infof("Failed to set RLIMIT_MEMLOCK: %v", err)
+		}
+	}
+
 	// Call the subcommand and pass in the configuration.
 	var ws unix.WaitStatus
 	subcmdCode := subcommands.Execute(context.Background(), conf, &ws)