From b042aeefb78c269513c091579c31c0539c05c11e Mon Sep 17 00:00:00 2001 From: Jamie Liu Date: Fri, 10 Nov 2023 11:25:33 -0800 Subject: [PATCH] Cache pgalloc.MemoryFile in mm.MemoryManager. This just saves an interface method call and indirection through kernel.Kernel for a frequently-used resource. PiperOrigin-RevId: 581318869 --- pkg/sentry/mm/lifecycle.go | 2 ++ pkg/sentry/mm/mm.go | 5 +++++ pkg/sentry/mm/pma.go | 16 +++++++--------- pkg/sentry/mm/save_restore.go | 7 +++---- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/sentry/mm/lifecycle.go b/pkg/sentry/mm/lifecycle.go index 4ed7d38f3..bd91ed379 100644 --- a/pkg/sentry/mm/lifecycle.go +++ b/pkg/sentry/mm/lifecycle.go @@ -32,6 +32,7 @@ func NewMemoryManager(p platform.Platform, mfp pgalloc.MemoryFileProvider, sleep return &MemoryManager{ p: p, mfp: mfp, + mf: mfp.MemoryFile(), haveASIO: p.SupportsAddressSpaceIO(), privateRefs: &privateRefs{}, users: atomicbitops.FromInt32(1), @@ -75,6 +76,7 @@ func (mm *MemoryManager) Fork(ctx context.Context) (*MemoryManager, error) { mm2 := &MemoryManager{ p: mm.p, mfp: mm.mfp, + mf: mm.mf, haveASIO: mm.haveASIO, layout: mm.layout, privateRefs: mm.privateRefs, diff --git a/pkg/sentry/mm/mm.go b/pkg/sentry/mm/mm.go index c84727d6c..3fb02e225 100644 --- a/pkg/sentry/mm/mm.go +++ b/pkg/sentry/mm/mm.go @@ -61,6 +61,11 @@ type MemoryManager struct { p platform.Platform mfp pgalloc.MemoryFileProvider + // mf is the cached result of mfp.MemoryFile(). + // + // mf is immutable. + mf *pgalloc.MemoryFile `state:"nosave"` + // haveASIO is the cached result of p.SupportsAddressSpaceIO(). Aside from // eliminating an indirect call in the hot I/O path, this makes // MemoryManager.asioEnabled() a leaf function, allowing it to be inlined. diff --git a/pkg/sentry/mm/pma.go b/pkg/sentry/mm/pma.go index c0f16698c..3dcd38017 100644 --- a/pkg/sentry/mm/pma.go +++ b/pkg/sentry/mm/pma.go @@ -215,7 +215,6 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter } atomic.StoreUintptr(&vma.lastFault, uintptr(ar.Start)) - mf := mm.mfp.MemoryFile() // Limit the range we allocate to ar, aligned to privateAllocUnit. maskAR := privateAligned(ar) didUnmapAS := false @@ -241,7 +240,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter if vma.mappable == nil { // Private anonymous mappings get pmas by allocating. allocAR := optAR.Intersect(maskAR) - fr, err := mf.Allocate(uint64(allocAR.Length()), opts) + fr, err := mm.mf.Allocate(uint64(allocAR.Length()), opts) if err != nil { return pstart, pgap, err } @@ -252,9 +251,9 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter } mm.addRSSLocked(allocAR) mm.incPrivateRef(fr) - mf.IncRef(fr, memCgID) + mm.mf.IncRef(fr, memCgID) pseg, pgap = mm.pmas.Insert(pgap, allocAR, pma{ - file: mf, + file: mm.mf, off: fr.Start, translatePerms: hostarch.AnyAccess, effectivePerms: vma.effectivePerms, @@ -374,7 +373,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter return pstart, pseg.PrevGap(), err } // Copy contents. - fr, err := mf.Allocate(uint64(copyAR.Length()), pgalloc.AllocOpts{ + fr, err := mm.mf.Allocate(uint64(copyAR.Length()), pgalloc.AllocOpts{ Kind: usage.Anonymous, Mode: pgalloc.AllocateAndWritePopulate, MemCgID: memCgID, @@ -410,8 +409,8 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter } oldpma.file.DecRef(pseg.fileRange()) mm.incPrivateRef(fr) - mf.IncRef(fr, memCgID) - oldpma.file = mf + mm.mf.IncRef(fr, memCgID) + oldpma.file = mm.mf oldpma.off = fr.Start oldpma.translatePerms = hostarch.AnyAccess oldpma.effectivePerms = vma.effectivePerms @@ -975,9 +974,8 @@ func (mm *MemoryManager) decPrivateRef(fr memmap.FileRange) { refSet.MergeAdjacent(fr) mm.privateRefs.mu.Unlock() - mf := mm.mfp.MemoryFile() for _, fr := range freed { - mf.DecRef(fr) + mm.mf.DecRef(fr) } } diff --git a/pkg/sentry/mm/save_restore.go b/pkg/sentry/mm/save_restore.go index fee9d03a5..a85fccb70 100644 --- a/pkg/sentry/mm/save_restore.go +++ b/pkg/sentry/mm/save_restore.go @@ -37,9 +37,8 @@ func (mm *MemoryManager) InvalidateUnsavable(ctx context.Context) error { // beforeSave is invoked by stateify. func (mm *MemoryManager) beforeSave() { - mf := mm.mfp.MemoryFile() for pseg := mm.pmas.FirstSegment(); pseg.Ok(); pseg = pseg.NextSegment() { - if pma := pseg.ValuePtr(); pma.file != mf { + if pma := pseg.ValuePtr(); pma.file != mm.mf { // InvalidateUnsavable should have caused all such pmas to be // invalidated. panic(fmt.Sprintf("Can't save pma %#v with non-MemoryFile of type %T:\n%s", pseg.Range(), pma.file, mm)) @@ -49,10 +48,10 @@ func (mm *MemoryManager) beforeSave() { // afterLoad is invoked by stateify. func (mm *MemoryManager) afterLoad() { + mm.mf = mm.mfp.MemoryFile() mm.haveASIO = mm.p.SupportsAddressSpaceIO() - mf := mm.mfp.MemoryFile() for pseg := mm.pmas.FirstSegment(); pseg.Ok(); pseg = pseg.NextSegment() { - pseg.ValuePtr().file = mf + pseg.ValuePtr().file = mm.mf } }