diff --git a/g3doc/user_guide/production.md b/g3doc/user_guide/production.md
index 8468ad696..0dad701fa 100644
--- a/g3doc/user_guide/production.md
+++ b/g3doc/user_guide/production.md
@@ -140,6 +140,15 @@ Passthrough to use the host's (Linux's) network stack, rather than gVisor's own.
Configure Networking
»
+### Optimizing MM performance {#configure-mm}
+
+gVisor will make transparent huge pages (THP) available to applications if
+provided by the host Linux kernel. Linux disables this feature by default; to
+enable it, write "advise" to the file
+`/sys/kernel/mm/transparent_hugepage/shmem_enabled`. Performance effects of THP
+vary by workload and platform; KVM platform performance in particular can
+benefit greatly from enabling THP.
+
[Istio]: https://istio.io/
[Istio overhead]: https://istio.io/latest/docs/ops/deployment/performance-and-scalability/
[Security Model]: /docs/architecture_guide/security/
diff --git a/nogo.yaml b/nogo.yaml
index d0722439b..bd96f2194 100644
--- a/nogo.yaml
+++ b/nogo.yaml
@@ -190,8 +190,10 @@ analyzers:
- pkg/gohacks/noescape_unsafe.go # Special case.
- pkg/ring0/pagetables/allocator_unsafe.go # Special case.
- pkg/sentry/fsutil/host_file_mapper_unsafe.go # Special case.
+ - pkg/sentry/pgalloc/pgalloc_unsafe.go # Special case.
- pkg/sentry/platform/kvm/bluepill_unsafe.go # Special case.
- pkg/sentry/platform/kvm/machine_unsafe.go # Special case.
+ - pkg/sentry/platform/pgalloc/pgalloc_unsafe.go # Special case.
- pkg/sentry/platform/systrap/stub_unsafe.go # Special case.
- pkg/sentry/platform/systrap/syscall_thread_unsafe.go # Special case.
- pkg/sentry/platform/systrap/sysmsg_thread_unsafe.go # Special case.
diff --git a/pkg/sentry/contexttest/contexttest.go b/pkg/sentry/contexttest/contexttest.go
index 4ed00774f..aba96d5ce 100644
--- a/pkg/sentry/contexttest/contexttest.go
+++ b/pkg/sentry/contexttest/contexttest.go
@@ -44,7 +44,9 @@ func Context(tb testing.TB) context.Context {
tb.Fatalf("error creating application memory file: %v", err)
}
memfile := os.NewFile(uintptr(memfd), memfileName)
- mf, err := pgalloc.NewMemoryFile(memfile, pgalloc.MemoryFileOpts{})
+ mf, err := pgalloc.NewMemoryFile(memfile, pgalloc.MemoryFileOpts{
+ DisableMemoryAccounting: true,
+ })
if err != nil {
memfile.Close()
tb.Fatalf("error creating pgalloc.MemoryFile: %v", err)
diff --git a/pkg/sentry/fsimpl/gofer/regular_file.go b/pkg/sentry/fsimpl/gofer/regular_file.go
index f23540696..cd2d2e3a1 100644
--- a/pkg/sentry/fsimpl/gofer/regular_file.go
+++ b/pkg/sentry/fsimpl/gofer/regular_file.go
@@ -374,6 +374,7 @@ func (rw *dentryReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error)
}
// Otherwise read from/through the cache.
+ memCgID := pgalloc.MemoryCgroupIDFromContext(rw.ctx)
mf := rw.d.fs.mf
fillCache := mf.ShouldCacheEvictable()
var dataMuUnlock func()
@@ -435,7 +436,11 @@ func (rw *dentryReadWriter) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error)
End: gapEnd,
}
optMR := gap.Range()
- _, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, usage.PageCache, pgalloc.AllocateAndWritePopulate, h.readToBlocksAt)
+ _, err := rw.d.cache.Fill(rw.ctx, reqMR, maxFillRange(reqMR, optMR), rw.d.size.Load(), mf, pgalloc.AllocOpts{
+ Kind: usage.PageCache,
+ MemCgID: memCgID,
+ Mode: pgalloc.AllocateAndWritePopulate,
+ }, h.readToBlocksAt)
mf.MarkEvictable(rw.d, pgalloc.EvictableRange{optMR.Start, optMR.End})
seg, gap = rw.d.cache.Find(rw.off)
if !seg.Ok() {
@@ -773,6 +778,7 @@ func (d *dentry) Translate(ctx context.Context, required, optional memmap.Mappab
}, nil
}
+ memCgID := pgalloc.MemoryCgroupIDFromContext(ctx)
d.dataMu.Lock()
// Constrain translations to d.size (rounded up) to prevent translation to
@@ -794,7 +800,11 @@ func (d *dentry) Translate(ctx context.Context, required, optional memmap.Mappab
mf := d.fs.mf
h := d.readHandle()
- _, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, usage.PageCache, pgalloc.AllocateAndWritePopulate, h.readToBlocksAt)
+ _, cerr := d.cache.Fill(ctx, required, maxFillRange(required, optional), d.size.Load(), mf, pgalloc.AllocOpts{
+ Kind: usage.PageCache,
+ MemCgID: memCgID,
+ Mode: pgalloc.AllocateAndWritePopulate,
+ }, h.readToBlocksAt)
var ts []memmap.Translation
var translatedEnd uint64
diff --git a/pkg/sentry/fsimpl/testutil/BUILD b/pkg/sentry/fsimpl/testutil/BUILD
index d0cf0c7ce..e1089017c 100644
--- a/pkg/sentry/fsimpl/testutil/BUILD
+++ b/pkg/sentry/fsimpl/testutil/BUILD
@@ -32,6 +32,7 @@ go_library(
"//pkg/sentry/platform/ptrace",
"//pkg/sentry/seccheck",
"//pkg/sentry/time",
+ "//pkg/sentry/usage",
"//pkg/sentry/vfs",
"//pkg/sync",
"//pkg/usermem",
diff --git a/pkg/sentry/fsimpl/testutil/kernel.go b/pkg/sentry/fsimpl/testutil/kernel.go
index 3d5516a66..19f53f2f1 100644
--- a/pkg/sentry/fsimpl/testutil/kernel.go
+++ b/pkg/sentry/fsimpl/testutil/kernel.go
@@ -36,6 +36,7 @@ import (
"gvisor.dev/gvisor/pkg/sentry/platform"
"gvisor.dev/gvisor/pkg/sentry/seccheck"
"gvisor.dev/gvisor/pkg/sentry/time"
+ "gvisor.dev/gvisor/pkg/sentry/usage"
"gvisor.dev/gvisor/pkg/sentry/vfs"
// Platforms are pluggable.
@@ -53,6 +54,10 @@ func Boot() (*kernel.Kernel, error) {
cpuid.Initialize()
seccheck.Initialize()
+ if err := usage.Init(); err != nil {
+ return nil, fmt.Errorf("setting up memory accounting: %v", err)
+ }
+
platformCtr, err := platform.Lookup(*platformFlag)
if err != nil {
return nil, fmt.Errorf("platform not found: %v", err)
diff --git a/pkg/sentry/fsimpl/tmpfs/regular_file.go b/pkg/sentry/fsimpl/tmpfs/regular_file.go
index 86c18fdd0..b524820a9 100644
--- a/pkg/sentry/fsimpl/tmpfs/regular_file.go
+++ b/pkg/sentry/fsimpl/tmpfs/regular_file.go
@@ -280,6 +280,8 @@ func (rf *regularFile) CopyMapping(ctx context.Context, ms memmap.MappingSpace,
// Translate implements memmap.Mappable.Translate.
func (rf *regularFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) {
+ memCgID := pgalloc.MemoryCgroupIDFromContext(ctx)
+
rf.dataMu.Lock()
defer rf.dataMu.Unlock()
@@ -308,7 +310,10 @@ func (rf *regularFile) Translate(ctx context.Context, required, optional memmap.
}
optional = required
}
- pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.inode.fs.mf, rf.memoryUsageKind, pgalloc.AllocateOnly, nil /* r */)
+ pagesAlloced, cerr := rf.data.Fill(ctx, required, optional, rf.size.RacyLoad(), rf.inode.fs.mf, pgalloc.AllocOpts{
+ Kind: rf.memoryUsageKind,
+ MemCgID: memCgID,
+ }, nil)
// rf.data.Fill() may fail mid-way. We still want to account any pages that
// were allocated, irrespective of an error.
rf.inode.fs.adjustPageAcct(pagesToFill, pagesAlloced)
@@ -360,6 +365,8 @@ func (fd *regularFileFD) Release(context.Context) {
// Allocate implements vfs.FileDescriptionImpl.Allocate.
func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint64) error {
f := fd.inode().impl.(*regularFile)
+ memCgID := pgalloc.MemoryCgroupIDFromContext(ctx)
+
// To be consistent with Linux, inode.mu must be locked throughout.
f.inode.mu.Lock()
defer f.inode.mu.Unlock()
@@ -383,7 +390,7 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint
newSize = curPgEnd
}
required := memmap.MappableRange{Start: curPgStart, End: curPgEnd}
- if err := f.allocateLocked(ctx, mode, newSize, required); err != nil {
+ if err := f.allocateLocked(ctx, mode, newSize, required, memCgID); err != nil {
return err
}
// This loop can take a long time to process, so periodically check for
@@ -401,7 +408,7 @@ func (fd *regularFileFD) Allocate(ctx context.Context, mode, offset, length uint
// - rf.inode.mu is locked.
// - required must be page-aligned.
// - required.Start < newSize <= required.End.
-func (rf *regularFile) allocateLocked(ctx context.Context, mode, newSize uint64, required memmap.MappableRange) error {
+func (rf *regularFile) allocateLocked(ctx context.Context, mode, newSize uint64, required memmap.MappableRange, memCgID uint32) error {
rf.dataMu.Lock()
defer rf.dataMu.Unlock()
@@ -427,7 +434,11 @@ func (rf *regularFile) allocateLocked(ctx context.Context, mode, newSize uint64,
// faulting page-by-page when these pages are written to in the future.
allocMode = pgalloc.AllocateAndWritePopulate
}
- pagesAlloced, err := rf.data.Fill(ctx, required, required, newSize, rf.inode.fs.mf, rf.memoryUsageKind, allocMode, nil /* r */)
+ pagesAlloced, err := rf.data.Fill(ctx, required, required, newSize, rf.inode.fs.mf, pgalloc.AllocOpts{
+ Kind: rf.memoryUsageKind,
+ MemCgID: memCgID,
+ Mode: allocMode,
+ }, nil /* r */)
// f.data.Fill() may fail mid-way. We still want to account any pages that
// were allocated, irrespective of an error.
rf.inode.fs.adjustPageAcct(pagesToFill, pagesAlloced)
@@ -464,6 +475,8 @@ func (fd *regularFileFD) PRead(ctx context.Context, dst usermem.IOSequence, offs
return 0, nil
}
f := fd.inode().impl.(*regularFile)
+ // memCgID can be 0 here because regularFileReadWriter.ReadToBlocks() never
+ // allocates from pgalloc.
rw := getRegularFileReadWriter(f, offset, 0)
n, err := dst.CopyOutFrom(ctx, rw)
putRegularFileReadWriter(rw)
@@ -770,12 +783,12 @@ func (rw *regularFileReadWriter) WriteFromBlocks(srcs safemem.BlockSeq) (uint64,
// prepopulating disk-backed pages deteriorates performance as it fails
// to eliminate future page faults and we also additionally incur
// useless disk writebacks.
- allocMode = pgalloc.AllocateOnly
+ allocMode = pgalloc.AllocateCallerIndirectCommit
}
fr, err := rw.file.inode.fs.mf.Allocate(gapMR.Length(), pgalloc.AllocOpts{
Kind: rw.file.memoryUsageKind,
- Mode: allocMode,
MemCgID: rw.memCgID,
+ Mode: allocMode,
})
if err != nil {
retErr = err
diff --git a/pkg/sentry/fsutil/file_range_set.go b/pkg/sentry/fsutil/file_range_set.go
index 41477b4a0..c79a834d2 100644
--- a/pkg/sentry/fsutil/file_range_set.go
+++ b/pkg/sentry/fsutil/file_range_set.go
@@ -24,7 +24,6 @@ import (
"gvisor.dev/gvisor/pkg/safemem"
"gvisor.dev/gvisor/pkg/sentry/memmap"
"gvisor.dev/gvisor/pkg/sentry/pgalloc"
- "gvisor.dev/gvisor/pkg/sentry/usage"
)
// FileRangeSet maps offsets into a memmap.Mappable to offsets into a
@@ -92,14 +91,14 @@ func (s *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) uint
}
// Fill attempts to ensure that all memmap.Mappable offsets in required are
-// mapped to a memmap.File offset, by allocating from mf with the given
-// memory usage kind and invoking readAt to store data into memory. (If readAt
-// returns a successful partial read, Fill will call it repeatedly until all
-// bytes have been read.) EOF is handled consistently with the requirements of
-// mmap(2): bytes after EOF on the same page are zeroed; pages after EOF are
-// invalid. fileSize is an upper bound on the file's size; bytes after fileSize
-// will be zeroed without calling readAt. populate has the same meaning as the
-// pgalloc.MemoryFile.AllocateAndFill() argument of the same name.
+// mapped to a memmap.File offset, by allocating from mf with the given options
+// and invoking readAt to store data into memory. (If readAt is not nil,
+// opts.ReaderFunc will be overridden. If readAt returns a successful partial
+// read, Fill will call it repeatedly until all bytes have been read.) EOF is
+// handled consistently with the requirements of mmap(2): bytes after EOF on
+// the same page are zeroed; pages after EOF are invalid. fileSize is an upper
+// bound on the file's size; bytes after fileSize will be zeroed without
+// calling readAt.
//
// Fill may read offsets outside of required, but will never read offsets
// outside of optional. It returns a non-nil error if any error occurs, even
@@ -111,10 +110,9 @@ func (s *FileRangeSet) PagesToFill(required, optional memmap.MappableRange) uint
// - required.Length() > 0.
// - optional.IsSupersetOf(required).
// - required and optional must be page-aligned.
-func (s *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, kind usage.MemoryKind, allocMode pgalloc.AllocationMode, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) {
+func (s *FileRangeSet) Fill(ctx context.Context, required, optional memmap.MappableRange, fileSize uint64, mf *pgalloc.MemoryFile, opts pgalloc.AllocOpts, readAt func(ctx context.Context, dsts safemem.BlockSeq, offset uint64) (uint64, error)) (uint64, error) {
gap := s.LowerBoundGap(required.Start)
var pagesAlloced uint64
- memCgID := pgalloc.MemoryCgroupIDFromContext(ctx)
for gap.Ok() && gap.Start() < required.End {
if gap.Range().Length() == 0 {
gap = gap.NextGap()
@@ -123,11 +121,6 @@ func (s *FileRangeSet) Fill(ctx context.Context, required, optional memmap.Mappa
gr := gap.Range().Intersect(optional)
// Read data into the gap.
- opts := pgalloc.AllocOpts{
- Kind: kind,
- Mode: allocMode,
- MemCgID: memCgID,
- }
if readAt != nil {
opts.ReaderFunc = func(dsts safemem.BlockSeq) (uint64, error) {
var done uint64
diff --git a/pkg/sentry/hostmm/hostmm.go b/pkg/sentry/hostmm/hostmm.go
index 5df06a60f..b64b0aaeb 100644
--- a/pkg/sentry/hostmm/hostmm.go
+++ b/pkg/sentry/hostmm/hostmm.go
@@ -20,11 +20,31 @@ import (
"fmt"
"os"
"path"
+ "regexp"
"gvisor.dev/gvisor/pkg/eventfd"
"gvisor.dev/gvisor/pkg/log"
)
+// GetTransparentHugepageEnum returns the currently selected option for
+// whichever of
+// /sys/kernel/mm/transparent_hugepage/{enabled,shmem_enabled,defrag} is
+// specified by filename. (Only the basename is required, not the full path.)
+func GetTransparentHugepageEnum(filename string) (string, error) {
+ pathname := path.Join("/sys/kernel/mm/transparent_hugepage/", filename)
+ data, err := os.ReadFile(pathname)
+ if err != nil {
+ return "", err
+ }
+ // In these files, the selected option is highlighted by square brackets.
+ m := regexp.MustCompile(`\[.*\]`).Find(data)
+ if m == nil {
+ return "", fmt.Errorf("failed to parse %s: %q", pathname, data)
+ }
+ // Remove the square brackets.
+ return string(m[1 : len(m)-1]), nil
+}
+
// NotifyCurrentMemcgPressureCallback requests that f is called whenever the
// calling process' memory cgroup indicates memory pressure of the given level,
// as specified by Linux's Documentation/cgroup-v1/memory.txt.
diff --git a/pkg/sentry/kernel/syscalls.go b/pkg/sentry/kernel/syscalls.go
index 0c1d62bc1..680b15c68 100644
--- a/pkg/sentry/kernel/syscalls.go
+++ b/pkg/sentry/kernel/syscalls.go
@@ -496,7 +496,10 @@ type SyscallInfo struct {
// IncrementUnimplementedSyscallCounter increments the "unimplemented syscall" metric for the given
// syscall number.
// A syscall table must have been initialized prior to calling this function.
-// +checkescape:all
+//
+// FIXME(gvisor.dev/issue/10556): checkescape can't distinguish between this
+// file and files named syscalls.go in other directories, resulting in false
+// positives, so this function cannot be +checkescape:all.
//
//go:nosplit
func IncrementUnimplementedSyscallCounter(sysno uintptr) {
diff --git a/pkg/sentry/memmap/memmap.go b/pkg/sentry/memmap/memmap.go
index cb57079d2..091419d6b 100644
--- a/pkg/sentry/memmap/memmap.go
+++ b/pkg/sentry/memmap/memmap.go
@@ -356,6 +356,10 @@ type MMapOpts struct {
// downward on guard page faults.
GrowsDown bool
+ // Stack is equivalent to MAP_STACK, which has no mandatory semantics in
+ // Linux.
+ Stack bool
+
PlatformEffect MMapPlatformEffect
// MLockMode specifies the memory locking behavior of the mapping.
diff --git a/pkg/sentry/mm/io.go b/pkg/sentry/mm/io.go
index 0ac1469b1..601af1e29 100644
--- a/pkg/sentry/mm/io.go
+++ b/pkg/sentry/mm/io.go
@@ -499,7 +499,7 @@ func (mm *MemoryManager) handleASIOFault(ctx context.Context, addr hostarch.Addr
// Ensure that we have usable pmas.
mm.activeMu.Lock()
- pseg, pend, err := mm.getPMAsLocked(ctx, vseg, ar, at)
+ pseg, pend, err := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */)
mm.mappingMu.RUnlock()
if pendaddr := pend.Start(); pendaddr < ar.End {
if pendaddr <= ar.Start {
@@ -553,7 +553,7 @@ func (mm *MemoryManager) withInternalMappings(ctx context.Context, ar hostarch.A
// Ensure that we have usable pmas.
mm.activeMu.Lock()
- pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at)
+ pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */)
mm.mappingMu.RUnlock()
if pendaddr := pend.Start(); pendaddr < ar.End {
if pendaddr <= ar.Start {
@@ -627,7 +627,7 @@ func (mm *MemoryManager) withVecInternalMappings(ctx context.Context, ars hostar
// Ensure that we have usable pmas.
mm.activeMu.Lock()
- pars, perr := mm.getVecPMAsLocked(ctx, vars, at)
+ pars, perr := mm.getVecPMAsLocked(ctx, vars, at, true /* callerIndirectCommit */)
mm.mappingMu.RUnlock()
if pars.NumBytes() == 0 {
mm.activeMu.Unlock()
diff --git a/pkg/sentry/mm/mm.go b/pkg/sentry/mm/mm.go
index dc1431889..e87d65bd2 100644
--- a/pkg/sentry/mm/mm.go
+++ b/pkg/sentry/mm/mm.go
@@ -294,6 +294,9 @@ type vma struct {
// metag, none of which we currently support.
growsDown bool `state:"manual"`
+ // isStack is true if this is a MAP_STACK mapping.
+ isStack bool `state:"manual"`
+
// dontfork is the MADV_DONTFORK setting for this vma configured by madvise().
dontfork bool
@@ -330,6 +333,7 @@ func (v *vma) copy() vma {
maxPerms: v.maxPerms,
private: v.private,
growsDown: v.growsDown,
+ isStack: v.isStack,
dontfork: v.dontfork,
mlockMode: v.mlockMode,
numaPolicy: v.numaPolicy,
@@ -380,6 +384,13 @@ type pma struct {
// corresponding vma's memmap.Mappable.Translate.
private bool
+ // If huge is true, this pma was returned by a call to MemoryFile.Allocate()
+ // with AllocOpts.Hugepage = true. Note that due to pma splitting, pma may
+ // no longer be hugepage-aligned.
+ //
+ // Invariant: If huge == true, then private == true.
+ huge bool
+
// If internalMappings is not empty, it is the cached return value of
// file.MapInternal for the memmap.FileRange mapped by this pma.
internalMappings safemem.BlockSeq `state:"nosave"`
diff --git a/pkg/sentry/mm/pma.go b/pkg/sentry/mm/pma.go
index 00ea047ea..8946445d4 100644
--- a/pkg/sentry/mm/pma.go
+++ b/pkg/sentry/mm/pma.go
@@ -94,6 +94,10 @@ func (mm *MemoryManager) existingVecPMAsLocked(ars hostarch.AddrRangeSeq, at hos
//
// - An error that is non-nil if pmas exist for only a subset of ar.
//
+// If callerIndirectCommit is true, the caller of getPMAsLocked will shortly
+// commit all pages in ar without using the caller's page tables, in the same
+// sense as pgalloc.AllocateCallerIndirectCommit.
+//
// Preconditions:
// - mm.mappingMu must be locked.
// - mm.activeMu must be locked for writing.
@@ -101,7 +105,7 @@ func (mm *MemoryManager) existingVecPMAsLocked(ars hostarch.AddrRangeSeq, at hos
// - vseg.Range().Contains(ar.Start).
// - vmas must exist for all addresses in ar, and support accesses of type at
// (i.e. permission checks must have been performed against vmas).
-func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType) (pmaIterator, pmaGapIterator, error) {
+func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, callerIndirectCommit bool) (pmaIterator, pmaGapIterator, error) {
if checkInvariants {
if !ar.WellFormed() || ar.Length() == 0 {
panic(fmt.Sprintf("invalid ar: %v", ar))
@@ -123,7 +127,7 @@ func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar
}
ar = hostarch.AddrRange{ar.Start.RoundDown(), end}
- pstart, pend, perr := mm.getPMAsInternalLocked(ctx, vseg, ar, at)
+ pstart, pend, perr := mm.getPMAsInternalLocked(ctx, vseg, ar, at, callerIndirectCommit)
if pend.Start() <= ar.Start {
return pmaIterator{}, pend, perr
}
@@ -148,7 +152,7 @@ func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar
// - mm.activeMu must be locked for writing.
// - vmas must exist for all addresses in ars, and support accesses of type at
// (i.e. permission checks must have been performed against vmas).
-func (mm *MemoryManager) getVecPMAsLocked(ctx context.Context, ars hostarch.AddrRangeSeq, at hostarch.AccessType) (hostarch.AddrRangeSeq, error) {
+func (mm *MemoryManager) getVecPMAsLocked(ctx context.Context, ars hostarch.AddrRangeSeq, at hostarch.AccessType, callerIndirectCommit bool) (hostarch.AddrRangeSeq, error) {
for arsit := ars; !arsit.IsEmpty(); arsit = arsit.Tail() {
ar := arsit.Head()
if ar.Length() == 0 {
@@ -169,7 +173,7 @@ func (mm *MemoryManager) getVecPMAsLocked(ctx context.Context, ars hostarch.Addr
}
ar = hostarch.AddrRange{ar.Start.RoundDown(), end}
- _, pend, perr := mm.getPMAsInternalLocked(ctx, mm.vmas.FindSegment(ar.Start), ar, at)
+ _, pend, perr := mm.getPMAsInternalLocked(ctx, mm.vmas.FindSegment(ar.Start), ar, at, callerIndirectCommit)
if perr != nil {
return truncatedAddrRangeSeq(ars, arsit, pend.Start()), perr
}
@@ -193,7 +197,7 @@ func (mm *MemoryManager) getVecPMAsLocked(ctx context.Context, ars hostarch.Addr
// - getPMAsInternalLocked additionally requires that ar is page-aligned.
// getPMAsInternalLocked is an implementation helper for getPMAsLocked and
// getVecPMAsLocked; other clients should call one of those instead.
-func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType) (pmaIterator, pmaGapIterator, error) {
+func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, callerIndirectCommit bool) (pmaIterator, pmaGapIterator, error) {
if checkInvariants {
if !ar.WellFormed() || ar.Length() == 0 || !ar.IsPageAligned() {
panic(fmt.Sprintf("invalid ar: %v", ar))
@@ -214,18 +218,18 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
mm.unmapASLocked(unmapAR)
}()
- memCgID := pgalloc.MemoryCgroupIDFromContext(ctx)
- opts := pgalloc.AllocOpts{Kind: usage.Anonymous, Dir: pgalloc.BottomUp, MemCgID: memCgID}
vma := vseg.ValuePtr()
+ memCgID := pgalloc.MemoryCgroupIDFromContext(ctx)
+ allocDir := pgalloc.BottomUp
if uintptr(ar.Start) < atomic.LoadUintptr(&vma.lastFault) {
// Detect cases where memory is accessed downwards and change memory file
// allocation order to increase the chances that pages are coalesced.
- opts.Dir = pgalloc.TopDown
+ allocDir = pgalloc.TopDown
}
atomic.StoreUintptr(&vma.lastFault, uintptr(ar.Start))
- // Limit the range we allocate to ar, aligned to privateAllocUnit.
- maskAR := privateAligned(ar)
+ // Limit the range we allocate to ar, aligned to hugepage boundaries.
+ hugeMaskAR := hugepageAligned(ar)
// The range in which we iterate vmas and pmas is still limited to ar, to
// ensure that we don't allocate or COW-break a pma we don't need.
pseg, pgap := mm.pmas.Find(ar.Start)
@@ -247,8 +251,32 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
}
if vma.mappable == nil {
// Private anonymous mappings get pmas by allocating.
- allocAR := optAR.Intersect(maskAR)
- fr, err := mm.mf.Allocate(uint64(allocAR.Length()), opts)
+ // The allocated range is limited to ar, expanded to
+ // hugepage alignment. This is done even if the allocation
+ // will not be hugepage-backed, in an attempt to reduce
+ // application page faults (that trap into the sentry) by
+ // creating AddressSpace mappings in advance.
+ allocAR := optAR.Intersect(hugeMaskAR)
+ // Don't back stacks with huge pages due to low utilization
+ // and because they're often fragmented by copy-on-write.
+ huge := mm.mf.HugepagesEnabled() && allocAR.IsHugePageAligned() && !vma.growsDown && !vma.isStack
+ allocOpts := pgalloc.AllocOpts{
+ Kind: usage.Anonymous,
+ MemCgID: memCgID,
+ Mode: pgalloc.AllocateUncommitted,
+ Huge: huge,
+ Dir: allocDir,
+ }
+ // If the allocation is hugepage-backed and
+ // callerIndirectCommit is true, the caller will commit every
+ // allocated huge page. If the allocation is not
+ // hugepage-backed, the caller won't commit every allocated
+ // page since hugeMaskAR is ar expanded to huge alignment,
+ // unless only one page in optAR falls into the huge page.
+ if callerIndirectCommit && (huge || allocAR.Length() == hostarch.PageSize) {
+ allocOpts.Mode = pgalloc.AllocateCallerIndirectCommit
+ }
+ fr, err := mm.mf.Allocate(uint64(allocAR.Length()), allocOpts)
if err != nil {
return pstart, pgap, err
}
@@ -268,6 +296,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
// only reference, the new pma does not need
// copy-on-write.
private: true,
+ huge: huge,
}).NextNonEmpty()
pstart = pmaIterator{} // iterators invalidated
} else {
@@ -341,7 +370,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
}
}
var copyAR hostarch.AddrRange
- if vma := vseg.ValuePtr(); vma.effectivePerms.Execute {
+ if vma.effectivePerms.Execute {
// The majority of copy-on-write breaks on executable
// pages come from:
//
@@ -355,7 +384,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
// to benefit from copying nearby pages, so if the vma
// is executable, only copy the pages required.
copyAR = pseg.Range().Intersect(ar)
- } else if vma.growsDown {
+ } else if vma.growsDown || vma.isStack {
// In most cases, the new process will not use most of
// its stack before exiting or invoking execve(); it is
// especially unlikely to return very far down its call
@@ -372,18 +401,23 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
}
copyAR = pseg.Range().Intersect(stackMaskAR)
} else {
- copyAR = pseg.Range().Intersect(maskAR)
+ // Hugepage-align the range to be copied, for the same
+ // reasons as for private anonymous allocations.
+ copyAR = pseg.Range().Intersect(hugeMaskAR)
}
// Get internal mappings from the pma to copy from.
if err := pseg.getInternalMappingsLocked(); err != nil {
return pstart, pseg.PrevGap(), err
}
// Copy contents.
+ huge := mm.mf.HugepagesEnabled() && copyAR.IsHugePageAligned()
reader := safemem.BlockSeqReader{Blocks: mm.internalMappingsLocked(pseg, copyAR)}
fr, err := mm.mf.Allocate(uint64(copyAR.Length()), pgalloc.AllocOpts{
Kind: usage.Anonymous,
- Mode: pgalloc.AllocateAndWritePopulate,
MemCgID: memCgID,
+ Mode: pgalloc.AllocateAndWritePopulate,
+ Huge: huge,
+ Dir: allocDir,
ReaderFunc: reader.ReadToBlocks,
})
if _, ok := err.(safecopy.BusError); ok {
@@ -413,6 +447,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
oldpma.maxPerms = vma.maxPerms
oldpma.needCOW = false
oldpma.private = true
+ oldpma.huge = huge
oldpma.internalMappings = safemem.BlockSeq{}
// Try to merge the pma with its neighbors.
if prev := pseg.PrevSegment(); prev.Ok() {
@@ -518,21 +553,9 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter
}
}
-const (
- // When memory is allocated for a private pma, align the allocated address
- // range to a privateAllocUnit boundary when possible. Larger values of
- // privateAllocUnit may reduce page faults by allowing fewer, larger pmas
- // to be mapped, but may result in larger amounts of wasted memory in the
- // presence of fragmentation. privateAllocUnit must be a power-of-2
- // multiple of hostarch.PageSize.
- privateAllocUnit = hostarch.HugePageSize
-
- privateAllocMask = privateAllocUnit - 1
-)
-
-func privateAligned(ar hostarch.AddrRange) hostarch.AddrRange {
- aligned := hostarch.AddrRange{ar.Start &^ privateAllocMask, ar.End}
- if end := (ar.End + privateAllocMask) &^ privateAllocMask; end >= ar.End {
+func hugepageAligned(ar hostarch.AddrRange) hostarch.AddrRange {
+ aligned := hostarch.AddrRange{ar.Start.HugeRoundDown(), ar.End}
+ if end, ok := ar.End.HugeRoundUp(); ok {
aligned.End = end
}
if checkInvariants {
@@ -684,7 +707,7 @@ func (mm *MemoryManager) Pin(ctx context.Context, ar hostarch.AddrRange, at host
// Ensure that we have usable pmas.
mm.activeMu.Lock()
- pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at)
+ pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at, false /* callerIndirectCommit */)
mm.mappingMu.RUnlock()
if pendaddr := pend.Start(); pendaddr < ar.End {
if pendaddr <= ar.Start {
@@ -900,7 +923,8 @@ func (pmaSetFunctions) Merge(ar1 hostarch.AddrRange, pma1 pma, ar2 hostarch.Addr
pma1.effectivePerms != pma2.effectivePerms ||
pma1.maxPerms != pma2.maxPerms ||
pma1.needCOW != pma2.needCOW ||
- pma1.private != pma2.private {
+ pma1.private != pma2.private ||
+ pma1.huge != pma2.huge {
return pma{}, false
}
diff --git a/pkg/sentry/mm/save_restore.go b/pkg/sentry/mm/save_restore.go
index 6f2e0ceb7..6d1c38ec9 100644
--- a/pkg/sentry/mm/save_restore.go
+++ b/pkg/sentry/mm/save_restore.go
@@ -60,6 +60,7 @@ const (
vmaMaxPermsExecute
vmaPrivate
vmaGrowsDown
+ vmaIsStack
)
func (v *vma) saveRealPerms() int {
@@ -97,6 +98,9 @@ func (v *vma) saveRealPerms() int {
if v.growsDown {
b |= vmaGrowsDown
}
+ if v.isStack {
+ b |= vmaIsStack
+ }
return b
}
@@ -134,6 +138,9 @@ func (v *vma) loadRealPerms(_ goContext.Context, b int) {
if b&vmaGrowsDown > 0 {
v.growsDown = true
}
+ if b&vmaIsStack > 0 {
+ v.isStack = true
+ }
}
func (p *pma) saveFile() string {
diff --git a/pkg/sentry/mm/syscalls.go b/pkg/sentry/mm/syscalls.go
index 1e84a7631..fc6bac91f 100644
--- a/pkg/sentry/mm/syscalls.go
+++ b/pkg/sentry/mm/syscalls.go
@@ -53,7 +53,7 @@ func (mm *MemoryManager) HandleUserFault(ctx context.Context, addr hostarch.Addr
// Ensure that we have a usable pma.
mm.activeMu.Lock()
- pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, at)
+ pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */)
mm.mappingMu.RUnlock()
if err != nil {
mm.activeMu.Unlock()
@@ -135,7 +135,7 @@ func (mm *MemoryManager) MMap(ctx context.Context, opts memmap.MMapOpts) (hostar
// Get pmas and map as requested.
mm.populateVMAAndUnlock(ctx, vseg, ar, opts.PlatformEffect)
- case opts.Mappable == nil && length <= privateAllocUnit:
+ case opts.Mappable == nil && length <= hostarch.HugePageSize:
// NOTE(b/63077076, b/63360184): Get pmas and map eagerly in the hope
// that doing so will save on future page faults. We only do this for
// anonymous mappings, since otherwise the cost of
@@ -179,7 +179,7 @@ func (mm *MemoryManager) populateVMA(ctx context.Context, vseg vmaIterator, ar h
}
// Ensure that we have usable pmas.
- pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess)
+ pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess, platformEffect == memmap.PlatformEffectCommit)
if err != nil {
// mm/util.c:vm_mmap_pgoff() ignores the error, if any, from
// mm/gup.c:mm_populate(). If it matters, we'll get it again when
@@ -226,7 +226,7 @@ func (mm *MemoryManager) populateVMAAndUnlock(ctx context.Context, vseg vmaItera
// mm.mappingMu doesn't need to be write-locked for getPMAsLocked, and it
// isn't needed at all for mapASLocked.
mm.mappingMu.DowngradeLock()
- pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess)
+ pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess, platformEffect == memmap.PlatformEffectCommit)
mm.mappingMu.RUnlock()
if err != nil {
mm.activeMu.Unlock()
@@ -449,6 +449,7 @@ func (mm *MemoryManager) MRemap(ctx context.Context, oldAddr hostarch.Addr, oldS
MaxPerms: vma.maxPerms,
Private: vma.private,
GrowsDown: vma.growsDown,
+ Stack: vma.isStack,
MLockMode: vma.mlockMode,
Hint: vma.hint,
}, droppedIDs)
@@ -890,7 +891,7 @@ func (mm *MemoryManager) MLock(ctx context.Context, addr hostarch.Addr, length u
mm.mappingMu.RUnlock()
return linuxerr.ENOMEM
}
- _, _, err := mm.getPMAsLocked(ctx, vseg, vseg.Range().Intersect(ar), hostarch.NoAccess)
+ _, _, err := mm.getPMAsLocked(ctx, vseg, vseg.Range().Intersect(ar), hostarch.NoAccess, true /* callerIndirectCommit */)
if err != nil {
mm.activeMu.Unlock()
mm.mappingMu.RUnlock()
@@ -985,7 +986,7 @@ func (mm *MemoryManager) MLockAll(ctx context.Context, opts MLockAllOpts) error
mm.mappingMu.DowngradeLock()
for vseg := mm.vmas.FirstSegment(); vseg.Ok(); vseg = vseg.NextSegment() {
if vseg.ValuePtr().effectivePerms.Any() {
- mm.getPMAsLocked(ctx, vseg, vseg.Range(), hostarch.NoAccess)
+ mm.getPMAsLocked(ctx, vseg, vseg.Range(), hostarch.NoAccess, true /* callerIndirectCommit */)
}
}
@@ -1096,11 +1097,24 @@ func (mm *MemoryManager) Decommit(addr hostarch.Addr, length uint64) error {
defer mm.activeMu.Unlock()
// This is invalidateLocked(invalidatePrivate=true, invalidateShared=true),
- // with the additional wrinkle that we must refuse to invalidate pmas under
- // mlocked vmas.
- var didUnmapAS bool
+ // but:
+ //
+ // - We must refuse to invalidate pmas under mlocked vmas.
+ //
+ // - If at least one byte in ar is not covered by a vma, decommit the rest
+ // but return ENOMEM.
+ //
+ // - If we would invalidate only part of a huge page that we own (is not
+ // copy-on-write), use MemoryFile.Decommit() instead to keep the allocated
+ // huge page intact for future use.
+ didUnmapAS := false
pseg := mm.pmas.LowerBoundSegment(ar.Start)
- for vseg := mm.vmas.LowerBoundSegment(ar.Start); vseg.Ok() && vseg.Start() < ar.End; vseg = vseg.NextSegment() {
+ vseg := mm.vmas.LowerBoundSegment(ar.Start)
+ if !vseg.Ok() {
+ return linuxerr.ENOMEM
+ }
+ hadvgap := ar.Start < vseg.Start()
+ for vseg.Ok() && vseg.Start() < ar.End {
vma := vseg.ValuePtr()
if vma.mlockMode != memmap.MLockNone {
return linuxerr.EINVAL
@@ -1114,8 +1128,62 @@ func (mm *MemoryManager) Decommit(addr hostarch.Addr, length uint64) error {
}
}
for pseg.Ok() && pseg.Start() < vsegAR.End {
- pseg = mm.pmas.Isolate(pseg, vsegAR)
pma := pseg.ValuePtr()
+ if pma.huge && !mm.isPMACopyOnWriteLocked(vseg, pseg) {
+ psegAR := pseg.Range().Intersect(vsegAR)
+ if !psegAR.IsHugePageAligned() {
+ firstHugeStart := psegAR.Start.HugeRoundDown()
+ firstHugeEnd := firstHugeStart + hostarch.HugePageSize
+ lastWholeHugeEnd := psegAR.End.HugeRoundDown()
+ if firstHugeStart != psegAR.Start {
+ // psegAR.Start is not hugepage-aligned.
+ if psegAR.End <= firstHugeEnd {
+ // All of psegAR falls within a single huge page.
+ mm.mf.Decommit(pseg.fileRangeOf(psegAR))
+ pseg = pseg.NextSegment()
+ continue
+ }
+ if firstHugeEnd == lastWholeHugeEnd && lastWholeHugeEnd != psegAR.End {
+ // All of psegAR falls within two huge pages, and
+ // psegAR.End is also not hugepage-aligned. The
+ // logic below would handle this correctly, but
+ // would make two separate calls to
+ // MemoryFile.Decommit() for the first and last
+ // huge pages respectively.
+ mm.mf.Decommit(pseg.fileRangeOf(psegAR))
+ pseg = pseg.NextSegment()
+ continue
+ }
+ mm.mf.Decommit(pseg.fileRangeOf(hostarch.AddrRange{psegAR.Start, firstHugeEnd}))
+ psegAR.Start = firstHugeEnd
+ }
+ // Drop whole huge pages between psegAR.Start (which after the above
+ // is either firstHugeStart or firstHugeEnd) and lastWholeHugeEnd
+ // normally.
+ if psegAR.Start < lastWholeHugeEnd {
+ pseg = mm.pmas.Isolate(pseg, hostarch.AddrRange{psegAR.Start, lastWholeHugeEnd})
+ pma = pseg.ValuePtr()
+ if !didUnmapAS {
+ // Unmap all of ar, not just pseg.Range(), to minimize host
+ // syscalls. AddressSpace mappings must be removed before
+ // pma.file.DecRef().
+ mm.unmapASLocked(ar)
+ didUnmapAS = true
+ }
+ pma.file.DecRef(pseg.fileRange())
+ mm.removeRSSLocked(pseg.Range())
+ pseg = mm.pmas.Remove(pseg).NextSegment()
+ }
+ if lastWholeHugeEnd != psegAR.End {
+ // psegAR.End is not hugepage-aligned.
+ mm.mf.Decommit(pseg.fileRangeOf(hostarch.AddrRange{lastWholeHugeEnd, psegAR.End}))
+ pseg = pseg.NextSegment()
+ }
+ continue
+ }
+ }
+ pseg = mm.pmas.Isolate(pseg, vsegAR)
+ pma = pseg.ValuePtr()
if !didUnmapAS {
// Unmap all of ar, not just pseg.Range(), to minimize host
// syscalls. AddressSpace mappings must be removed before
@@ -1127,13 +1195,21 @@ func (mm *MemoryManager) Decommit(addr hostarch.Addr, length uint64) error {
mm.removeRSSLocked(pseg.Range())
pseg = mm.pmas.Remove(pseg).NextSegment()
}
+ if ar.End <= vseg.End() {
+ break
+ }
+ vgap := vseg.NextGap()
+ if !vgap.IsEmpty() {
+ hadvgap = true
+ }
+ vseg = vgap.NextSegment()
}
// "If there are some parts of the specified address space that are not
// mapped, the Linux version of madvise() ignores them and applies the call
// to the rest (but returns ENOMEM from the system call, as it should)." -
// madvise(2)
- if mm.vmas.SpanRange(ar) != ar.Length() {
+ if hadvgap {
return linuxerr.ENOMEM
}
return nil
diff --git a/pkg/sentry/mm/vma.go b/pkg/sentry/mm/vma.go
index 7971ea90f..13ee8f40d 100644
--- a/pkg/sentry/mm/vma.go
+++ b/pkg/sentry/mm/vma.go
@@ -44,10 +44,13 @@ func (mm *MemoryManager) createVMALocked(ctx context.Context, opts memmap.MMapOp
// Find a usable range.
addr, err := mm.findAvailableLocked(opts.Length, findAvailableOpts{
- Addr: opts.Addr,
- Fixed: opts.Fixed,
- Unmap: opts.Unmap,
- Map32Bit: opts.Map32Bit,
+ Addr: opts.Addr,
+ Fixed: opts.Fixed,
+ GrowsDown: opts.GrowsDown,
+ Stack: opts.Stack,
+ Private: opts.Private,
+ Unmap: opts.Unmap,
+ Map32Bit: opts.Map32Bit,
})
if err != nil {
// Can't force without opts.Unmap and opts.Fixed.
@@ -119,6 +122,7 @@ func (mm *MemoryManager) createVMALocked(ctx context.Context, opts memmap.MMapOp
maxPerms: opts.MaxPerms,
private: opts.Private,
growsDown: opts.GrowsDown,
+ isStack: opts.Stack,
mlockMode: opts.MLockMode,
numaPolicy: linux.MPOL_DEFAULT,
id: opts.MappingIdentity,
@@ -144,10 +148,13 @@ type findAvailableOpts struct {
//
// - Unmap allows existing guard pages in the returned range.
- Addr hostarch.Addr
- Fixed bool
- Unmap bool
- Map32Bit bool
+ Addr hostarch.Addr
+ Fixed bool
+ GrowsDown bool
+ Stack bool
+ Private bool
+ Unmap bool
+ Map32Bit bool
}
// map32Start/End are the bounds to which MAP_32BIT mappings are constrained,
@@ -187,9 +194,10 @@ func (mm *MemoryManager) findAvailableLocked(length uint64, opts findAvailableOp
return 0, linuxerr.ENOMEM
}
- // Prefer hugepage alignment if a hugepage or more is requested.
+ // Prefer hugepage alignment if a hugepage or more is requested and the vma
+ // will actually be eligible for hugepages.
alignment := uint64(hostarch.PageSize)
- if length >= hostarch.HugePageSize {
+ if length >= hostarch.HugePageSize && opts.Private && !opts.GrowsDown && !opts.Stack {
alignment = hostarch.HugePageSize
}
@@ -465,6 +473,7 @@ func (vmaSetFunctions) Merge(ar1 hostarch.AddrRange, vma1 vma, ar2 hostarch.Addr
vma1.maxPerms != vma2.maxPerms ||
vma1.private != vma2.private ||
vma1.growsDown != vma2.growsDown ||
+ vma1.isStack != vma2.isStack ||
vma1.mlockMode != vma2.mlockMode ||
vma1.numaPolicy != vma2.numaPolicy ||
vma1.numaNodemask != vma2.numaNodemask ||
diff --git a/pkg/sentry/pgalloc/BUILD b/pkg/sentry/pgalloc/BUILD
index bca426dba..a0abd7683 100644
--- a/pkg/sentry/pgalloc/BUILD
+++ b/pkg/sentry/pgalloc/BUILD
@@ -7,13 +7,6 @@ package(
licenses = ["notice"],
)
-declare_mutex(
- name = "memory_file_mutex",
- out = "memory_file_mutex.go",
- package = "pgalloc",
- prefix = "memoryFile",
-)
-
declare_mutex(
name = "mappings_mutex",
out = "mappings_mutex.go",
@@ -21,6 +14,13 @@ declare_mutex(
prefix = "mappings",
)
+declare_mutex(
+ name = "memory_file_mutex",
+ out = "memory_file_mutex.go",
+ package = "pgalloc",
+ prefix = "memoryFile",
+)
+
go_template_instance(
name = "evictable_range",
out = "evictable_range.go",
@@ -47,8 +47,8 @@ go_template_instance(
)
go_template_instance(
- name = "usage_set",
- out = "usage_set.go",
+ name = "memacct_set",
+ out = "memacct_set.go",
consts = {
"minDegree": "10",
"trackGaps": "1",
@@ -57,33 +57,55 @@ go_template_instance(
"memmap": "gvisor.dev/gvisor/pkg/sentry/memmap",
},
package = "pgalloc",
- prefix = "usage",
+ prefix = "memAcct",
template = "//pkg/segment:generic_set",
types = {
"Key": "uint64",
"Range": "memmap.FileRange",
- "Value": "usageInfo",
- "Functions": "usageSetFunctions",
+ "Value": "memAcctInfo",
+ "Functions": "memAcctSetFunctions",
},
)
go_template_instance(
- name = "reclaim_set",
- out = "reclaim_set.go",
+ name = "unfree_set",
+ out = "unfree_set.go",
consts = {
"minDegree": "10",
+ "trackGaps": "1",
},
imports = {
"memmap": "gvisor.dev/gvisor/pkg/sentry/memmap",
},
package = "pgalloc",
- prefix = "reclaim",
+ prefix = "unfree",
template = "//pkg/segment:generic_set",
types = {
"Key": "uint64",
"Range": "memmap.FileRange",
- "Value": "reclaimSetValue",
- "Functions": "reclaimSetFunctions",
+ "Value": "unfreeInfo",
+ "Functions": "unfreeSetFunctions",
+ },
+)
+
+go_template_instance(
+ name = "unwaste_set",
+ out = "unwaste_set.go",
+ consts = {
+ "minDegree": "10",
+ "trackGaps": "1",
+ },
+ imports = {
+ "memmap": "gvisor.dev/gvisor/pkg/sentry/memmap",
+ },
+ package = "pgalloc",
+ prefix = "unwaste",
+ template = "//pkg/segment:generic_set",
+ types = {
+ "Key": "uint64",
+ "Range": "memmap.FileRange",
+ "Value": "unwasteInfo",
+ "Functions": "unwasteSetFunctions",
},
)
@@ -94,12 +116,13 @@ go_library(
"evictable_range.go",
"evictable_range_set.go",
"mappings_mutex.go",
+ "memacct_set.go",
"memory_file_mutex.go",
"pgalloc.go",
"pgalloc_unsafe.go",
- "reclaim_set.go",
"save_restore.go",
- "usage_set.go",
+ "unfree_set.go",
+ "unwaste_set.go",
],
visibility = ["//pkg/sentry:internal"],
deps = [
@@ -109,7 +132,6 @@ go_library(
"//pkg/errors/linuxerr",
"//pkg/hostarch",
"//pkg/log",
- "//pkg/memutil",
"//pkg/safemem",
"//pkg/sentry/arch",
"//pkg/sentry/hostmm",
@@ -130,5 +152,8 @@ go_test(
size = "small",
srcs = ["pgalloc_test.go"],
library = ":pgalloc",
- deps = ["//pkg/hostarch"],
+ deps = [
+ "//pkg/hostarch",
+ "//pkg/sentry/memmap",
+ ],
)
diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go
index 8ce1c9f9d..919f661c6 100644
--- a/pkg/sentry/pgalloc/pgalloc.go
+++ b/pkg/sentry/pgalloc/pgalloc.go
@@ -12,19 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Package pgalloc contains the page allocator subsystem, which manages memory
-// that may be mapped into application address spaces.
-//
-// Lock order:
-//
-// pgalloc.MemoryFile.mu
-// pgalloc.MemoryFile.mappingsMu
+// Package pgalloc contains the page allocator subsystem, which provides
+// allocatable memory that may be mapped into application address spaces.
package pgalloc
import (
"fmt"
"math"
"os"
+ "strings"
"sync/atomic"
"time"
@@ -42,136 +38,126 @@ import (
"gvisor.dev/gvisor/pkg/sync"
)
-// Direction describes how to allocate offsets from MemoryFile.
-type Direction int
-
-const (
- // BottomUp allocates offsets in increasing offsets.
- BottomUp Direction = iota
- // TopDown allocates offsets in decreasing offsets.
- TopDown
-)
-
-// String implements fmt.Stringer.
-func (d Direction) String() string {
- switch d {
- case BottomUp:
- return "up"
- case TopDown:
- return "down"
- }
- panic(fmt.Sprintf("invalid direction: %d", d))
-}
+const pagesPerHugePage = hostarch.HugePageSize / hostarch.PageSize
// MemoryFile is a memmap.File whose pages may be allocated to arbitrary
// users.
type MemoryFile struct {
memmap.NoBufferedIOFallback
- // opts holds options passed to NewMemoryFile. opts is immutable.
- opts MemoryFileOpts
-
- // MemoryFile owns a single backing file, which is modeled as follows:
+ // MemoryFile owns a single backing file. Each page in the backing file is
+ // considered "committed" or "uncommitted". A page is committed if the host
+ // kernel is spending resources to store its contents and uncommitted
+ // otherwise. This definition includes pages that the host kernel has
+ // swapped. This is intentional; it means that committed pages can only
+ // become uncommitted as a result of MemoryFile's actions, such that page
+ // commitment does not change even if host kernel swapping behavior changes.
//
- // Each page in the file can be committed or uncommitted. A page is
- // committed if the host kernel is spending resources to store its contents
- // and uncommitted otherwise. This definition includes pages that the host
- // kernel has swapped; this is intentional, to ensure that accounting does
- // not change even if host kernel swapping behavior changes, and that
- // memory used by pseudo-swap mechanisms like zswap is still accounted.
+ // Each page in the MemoryFile is in one of the following logical states,
+ // protected by mu:
//
- // The initial contents of uncommitted pages are implicitly zero bytes. A
- // read or write to the contents of an uncommitted page causes it to be
- // committed. This is the only event that can cause a uncommitted page to
- // be committed.
+ // - Void: Pages beyond the backing file's current size cannot store data.
+ // Void pages are uncommitted. Extending the file's size transitions pages
+ // between the old and new sizes from void to free.
//
- // fallocate(FALLOC_FL_PUNCH_HOLE) (MemoryFile.Decommit) causes committed
- // pages to be uncommitted. This is the only event that can cause a
- // committed page to be uncommitted.
+ // - Free: Free pages are immediately allocatable. Free pages are
+ // uncommitted, and implicitly zeroed. Free pages become used when they are
+ // allocated.
//
- // Memory accounting is based on identifying the set of committed pages.
- // Since we do not have direct access to the MMU, tracking reads and writes
- // to uncommitted pages to detect commitment would introduce additional
- // page faults, which would be prohibitively expensive. Instead, we query
- // the host kernel to determine which pages are committed.
-
- // file is the backing file. The file pointer is immutable.
- file *os.File
+ // - Used: Used pages have been allocated and currently have a non-zero
+ // reference count. Used pages may transition from uncommitted to committed
+ // outside of MemoryFile's control, but can only transition from committed
+ // to uncommitted via MemoryFile.Decommit(). The content of used pages is
+ // unknown. Used pages become waste when their reference count becomes
+ // zero.
+ //
+ // - Waste: Waste pages have no users, but cannot be immediately
+ // reallocated since their commitment state and content is unknown. Waste
+ // pages may be uncommitted or committed, but cannot transition between the
+ // two. MemoryFile's releaser goroutine transitions pages from waste to
+ // releasing. Allocations that may return committed pages can transition
+ // pages from waste to used (referred to as "recycling").
+ //
+ // - Releasing: Releasing pages are waste pages that the releaser goroutine
+ // has removed from waste-tracking, making them ineligible for recycling.
+ // The releaser decommits releasing pages without holding mu, then
+ // transitions them back to free or sub-released with mu locked.
+ //
+ // - Sub-release: Sub-released pages are released small pages within a
+ // huge-page-backed allocation where the containing huge page as a whole
+ // has not yet been released, which can arise because references are still
+ // counted at page granularity within huge-page-backed ranges. Sub-released
+ // pages cannot be used for allocations until release of the whole
+ // containing huge page causes it to transition it to free. We assume that
+ // sub-released pages are uncommitted; this isn't necessarily true (see
+ // discussion of khugepaged elsewhere in this file), but the assumption is
+ // consistent with legacy behavior.
mu memoryFileMutex
- // usage maps each page in the file to metadata for that page. Pages for
- // which no segment exists in usage are both unallocated (not in use) and
- // uncommitted.
+ // unwasteSmall and unwasteHuge track waste ranges backed by small/huge pages
+ // respectively. Both sets are "inverted"; segments exist for all ranges that
+ // are *not* waste, allowing use of segment.Set gap-tracking to efficiently
+ // find ranges for both release and recycling allocations.
//
- // Since usage stores usageInfo objects by value, clients should usually
- // use usageIterator.ValuePtr() instead of usageIterator.Value() to get a
- // pointer to the usageInfo rather than a copy.
- //
- // usage must be kept maximally merged (that is, there should never be two
- // adjacent segments with the same values). At least markReclaimed depends
- // on this property.
- //
- // usage is protected by mu.
- usage usageSet
+ // unwasteSmall and unwasteHuge are protected by mu.
+ unwasteSmall unwasteSet
+ unwasteHuge unwasteSet
- // The UpdateUsage function scans all segments with knownCommitted set
- // to false, sees which pages are committed and creates corresponding
- // segments with knownCommitted set to true.
+ // haveWaste is true if there may be at least one waste page in the
+ // MemoryFile.
//
- // In order to avoid unnecessary scans, usageExpected tracks the total
- // file blocks expected. This is used to elide the scan when this
- // matches the underlying file blocks.
- //
- // To track swapped pages, usageSwapped tracks the discrepancy between
- // what is observed in core and what is reported by the file. When
- // usageSwapped is non-zero, a sweep will be performed at least every
- // second. The start of the last sweep is recorded in usageLast.
- //
- // All usage attributes are all protected by mu.
- usageExpected uint64
- usageSwapped uint64
- usageLast time.Time
+ // haveWaste is protected by mu.
+ haveWaste bool
- // fileSize is the size of the backing memory file in bytes. fileSize is
- // always a power-of-two multiple of chunkSize.
- //
- // fileSize is protected by mu.
- fileSize int64
-
- // Pages from the backing file are mapped into the local address space on
- // the granularity of large pieces called chunks. mappings is a []uintptr
- // that stores, for each chunk, the start address of a mapping of that
- // chunk in the current process' address space, or 0 if no such mapping
- // exists. Once a chunk is mapped, it is never remapped or unmapped until
- // the MemoryFile is destroyed.
- //
- // Mutating the mappings slice or its contents requires both holding
- // mappingsMu and using atomic memory operations. (The slice is mutated
- // whenever the file is expanded. Per the above, the only permitted
- // mutation of the slice's contents is the assignment of a mapping to a
- // chunk that was previously unmapped.) Reading the slice or its contents
- // only requires *either* holding mappingsMu or using atomic memory
- // operations. This allows MemoryFile.MapInternal to avoid locking in the
- // common case where chunk mappings already exist.
- mappingsMu mappingsMutex
- mappings atomic.Pointer[[]uintptr]
-
- // destroyed is set by Destroy to instruct the reclaimer goroutine to
- // release resources and exit. destroyed is protected by mu.
- destroyed bool
-
- // reclaimable is true if usage may contain reclaimable pages. reclaimable
- // is protected by mu.
- reclaimable bool
-
- // reclaim is the collection of regions for reclaim. reclaim is protected
- // by mu.
- reclaim reclaimSet
-
- // reclaimCond is signaled (with mu locked) when reclaimable or destroyed
+ // releaseCond is signaled (with mu locked) when haveWaste or destroyed
// transitions from false to true.
- reclaimCond sync.Cond
+ releaseCond sync.Cond
+
+ // unfreeSmall and unfreeHuge track information for non-free ranges backed
+ // by small/huge pages respectively. Each unfreeSet also contains segments
+ // representing chunks that are backed by a different page size. Gaps in
+ // the sets therefore represent free ranges backed by small/huge pages,
+ // allowing use of segment.Set gap-tracking to efficiently find free ranges
+ // for allocation.
+ //
+ // unfreeSmall and unfreeHuge are protected by mu.
+ unfreeSmall unfreeSet
+ unfreeHuge unfreeSet
+
+ // subreleased maps hugepage-aligned file offsets to the number of
+ // sub-released small pages within the hugepage beginning at that offset.
+ // subreleased is protected by mu.
+ subreleased map[uint64]uint64
+
+ // These fields are used for memory accounting.
+ //
+ // Memory accounting is based on identifying the set of committed pages.
+ // Since we do not have direct access to application page tables (on most
+ // platforms), tracking application accesses to uncommitted pages to detect
+ // commitment would introduce additional page faults, which would be
+ // prohibitively expensive. Instead, we query the host kernel to determine
+ // which pages are committed.
+ //
+ // memAcct tracks memory accounting state, including commitment status, for
+ // each page. Non-empty gaps in memAcct represent pages known to be
+ // uncommitted (void, free, and sub-released pages).
+ //
+ // knownCommittedBytes is the number of bytes in the file known to be
+ // committed, i.e. the span of all segments in memAcct for which
+ // knownCommitted is true.
+ //
+ // commitSeq is a sequence counter used to detect races between scans for
+ // committed pages and concurrent decommitment.
+ //
+ // nextCommitScan is the next time at which UpdateUsage() may scan the
+ // backing file for commitment information.
+ //
+ // All of these fields are protected by mu.
+ memAcct memAcctSet
+ knownCommittedBytes uint64
+ commitSeq uint64
+ nextCommitScan time.Time
// evictable maps EvictableMemoryUsers to eviction state.
//
@@ -181,95 +167,120 @@ type MemoryFile struct {
// evictionWG counts the number of goroutines currently performing evictions.
evictionWG sync.WaitGroup
+ // opts holds options passed to NewMemoryFile. opts is immutable.
+ opts MemoryFileOpts
+
+ // savable is true if this MemoryFile will be saved via SaveTo() during
+ // the kernel's SaveTo operation. savable is protected by mu.
+ savable bool
+
+ // destroyed is set by Destroy to instruct the releaser goroutine to
+ // release all MemoryFile resources and exit. destroyed is protected by mu.
+ destroyed bool
+
// stopNotifyPressure stops memory cgroup pressure level
// notifications used to drive eviction. stopNotifyPressure is
// immutable.
stopNotifyPressure func()
- // savable is true if this MemoryFile will be saved via SaveTo() during
- // the kernel's SaveTo operation. savable is protected by mu.
- savable bool
+ // file is the backing file. The file pointer is immutable.
+ file *os.File
+
+ // chunks holds metadata for each usable chunk in the backing file.
+ //
+ // chunks is at the end of MemoryFile in hopes of placing it on a relatively
+ // quiet cache line, since MapInternal() is by far the hottest path through
+ // pgalloc.
+ //
+ // chunks is protected by mu. chunks slices are immutable.
+ chunks atomic.Pointer[[]chunkInfo]
}
-// MemoryFileOpts provides options to NewMemoryFile.
-type MemoryFileOpts struct {
- // DelayedEviction controls the extent to which the MemoryFile may delay
- // eviction of evictable allocations.
- DelayedEviction DelayedEvictionType
-
- // If UseHostMemcgPressure is true, use host memory cgroup pressure level
- // notifications to determine when eviction is necessary. This option has
- // no effect unless DelayedEviction is DelayedEvictionEnabled.
- UseHostMemcgPressure bool
-
- // DecommitOnDestroy indicates whether the entire host file should be
- // decommitted on destruction. This is appropriate for host filesystem based
- // files that need to be explicitly cleaned up to release disk space.
- DecommitOnDestroy bool
-
- // If ManualZeroing is true, MemoryFile must not assume that new pages
- // obtained from the host are zero-filled, such that MemoryFile must manually
- // zero newly-allocated pages.
- ManualZeroing bool
-
- // If DisableIMAWorkAround is true, NewMemoryFile will not call
- // IMAWorkAroundForMemFile().
- DisableIMAWorkAround bool
-
- // DiskBackedFile indicates that the MemoryFile is backed by a file on disk.
- DiskBackedFile bool
-
- // RestoreID is an opaque string used to reassociate the MemoryFile with its
- // replacement during restore.
- RestoreID string
-}
-
-// DelayedEvictionType is the type of MemoryFileOpts.DelayedEviction.
-type DelayedEvictionType int
-
const (
- // DelayedEvictionDefault has unspecified behavior.
- DelayedEvictionDefault DelayedEvictionType = iota
-
- // DelayedEvictionDisabled requires that evictable allocations are evicted
- // as soon as possible.
- DelayedEvictionDisabled
-
- // DelayedEvictionEnabled requests that the MemoryFile delay eviction of
- // evictable allocations until doing so is considered necessary to avoid
- // performance degradation due to host memory pressure, or OOM kills.
- //
- // As of this writing, the behavior of DelayedEvictionEnabled depends on
- // whether or not MemoryFileOpts.UseHostMemcgPressure is enabled:
- //
- // - If UseHostMemcgPressure is true, evictions are delayed until memory
- // pressure is indicated.
- //
- // - Otherwise, evictions are only delayed until the reclaimer goroutine
- // is out of work (pages to reclaim).
- DelayedEvictionEnabled
-
- // DelayedEvictionManual requires that evictable allocations are only
- // evicted when MemoryFile.StartEvictions() is called. This is extremely
- // dangerous outside of tests.
- DelayedEvictionManual
+ chunkShift = 30
+ chunkSize = 1 << chunkShift // 1 GB
+ chunkMask = chunkSize - 1
+ maxChunks = math.MaxInt64 / chunkSize // because file size is int64
)
-// usageInfo tracks usage information.
+// chunkInfo is the value type of MemoryFile.chunks.
//
// +stateify savable
-type usageInfo struct {
- // kind is the usage kind.
+type chunkInfo struct {
+ // mapping is the start address of a mapping of the chunk.
+ //
+ // mapping is immutable.
+ mapping uintptr `state:"nosave"`
+
+ // huge is true if this chunk is expected to be hugepage-backed and false if
+ // this chunk is expected to be smallpage-backed.
+ //
+ // huge is immutable.
+ huge bool
+}
+
+func (f *MemoryFile) chunksLoad() []chunkInfo {
+ return *f.chunks.Load()
+}
+
+// forEachChunk invokes fn on a sequence of chunks that collectively span all
+// bytes in fr. In each call, chunkFR is the subset of fr that falls within
+// chunk. If any call to f returns false, forEachChunk stops iteration and
+// returns.
+func (f *MemoryFile) forEachChunk(fr memmap.FileRange, fn func(chunk *chunkInfo, chunkFR memmap.FileRange) bool) {
+ chunks := f.chunksLoad()
+ chunkStart := fr.Start &^ chunkMask
+ i := int(fr.Start / chunkSize)
+ for chunkStart < fr.End {
+ chunkEnd := chunkStart + chunkSize
+ if !fn(&chunks[i], fr.Intersect(memmap.FileRange{chunkStart, chunkEnd})) {
+ return
+ }
+ chunkStart = chunkEnd
+ i++
+ }
+}
+
+// unwasteInfo is the value type of MemoryFile.unwasteSmall/Huge.
+//
+// +stateify savable
+type unwasteInfo struct{}
+
+// unfreeInfo is the value type of MemoryFile.unfreeSmall/Huge.
+//
+// +stateify savable
+type unfreeInfo struct {
+ // refs is the per-page reference count. refs is non-zero for used pages,
+ // and zero for void, waste, releasing, and sub-released pages, as well as
+ // pages backed by a different page size.
+ refs uint64
+}
+
+// memAcctInfo is the value type of MemoryFile.memAcct.
+//
+// +stateify savable
+type memAcctInfo struct {
+ // kind is the memory accounting type. kind is allocation-dependent for
+ // used pages, and usage.System for void, waste, releasing, and
+ // sub-released pages.
kind usage.MemoryKind
- // knownCommitted is true if the tracked region is definitely committed.
- // (If it is false, the tracked region may or may not be committed.)
+ // memCgID is the memory cgroup ID to which represented pages are accounted.
+ memCgID uint32
+
+ // knownCommitted is true if represented pages are definitely committed.
+ // (If knownCommitted is false, represented pages may or may not be
+ // committed; pages that are definitely not committed are represented by
+ // gaps in MemoryFile.memAcct.)
knownCommitted bool
- refs uint64
+ // If true, represented pages are waste or releasing pages.
+ wasteOrReleasing bool
- // memCgID is the memory cgroup id to which this page is committed.
- memCgID uint32
+ // If knownCommitted is false, commitSeq was the value of
+ // MemoryFile.commitSeq when knownCommitted last transitioned to false.
+ // Otherwise, commitSeq is 0.
+ commitSeq uint64
}
// An EvictableMemoryUser represents a user of MemoryFile-allocated memory that
@@ -312,13 +323,82 @@ type evictableMemoryUserInfo struct {
evicting bool
}
-const (
- chunkShift = 30
- chunkSize = 1 << chunkShift // 1 GB
- chunkMask = chunkSize - 1
+// MemoryFileOpts provides options to NewMemoryFile.
+type MemoryFileOpts struct {
+ // DelayedEviction controls the extent to which the MemoryFile may delay
+ // eviction of evictable allocations.
+ DelayedEviction DelayedEvictionType
- // maxPage is the highest 64-bit page.
- maxPage = math.MaxUint64 &^ (hostarch.PageSize - 1)
+ // If UseHostMemcgPressure is true, use host memory cgroup pressure level
+ // notifications to determine when eviction is necessary. This option has
+ // no effect unless DelayedEviction is DelayedEvictionEnabled.
+ UseHostMemcgPressure bool
+
+ // DecommitOnDestroy indicates whether the entire host file should be
+ // decommitted on destruction. This is appropriate for host filesystem based
+ // files that need to be explicitly cleaned up to release disk space.
+ DecommitOnDestroy bool
+
+ // If DisableIMAWorkAround is true, NewMemoryFile will not call
+ // IMAWorkAroundForMemFile().
+ DisableIMAWorkAround bool
+
+ // DiskBackedFile indicates that the MemoryFile is backed by a file on disk.
+ DiskBackedFile bool
+
+ // RestoreID is an opaque string used to reassociate the MemoryFile with its
+ // replacement during restore.
+ RestoreID string
+
+ // If ExpectHugepages is true, MemoryFile will expect that the host will
+ // attempt to back AllocOpts.Huge == true allocations with huge pages. If
+ // ExpectHugepages is false, MemoryFile will expect that the host will back
+ // all allocations with small pages.
+ ExpectHugepages bool
+
+ // If AdviseHugepage is true, MemoryFile will request that the host back
+ // AllocOpts.Huge == true allocations with huge pages using MADV_HUGEPAGE.
+ AdviseHugepage bool
+
+ // If AdviseNoHugepage is true, MemoryFile will request that the host back
+ // AllocOpts.Huge == false allocations with small pages using
+ // MADV_NOHUGEPAGE.
+ AdviseNoHugepage bool
+
+ // If DisableMemoryAccounting is true, memory usage observed by the
+ // MemoryFile will not be reported in usage.MemoryAccounting.
+ DisableMemoryAccounting bool
+}
+
+// DelayedEvictionType is the type of MemoryFileOpts.DelayedEviction.
+type DelayedEvictionType uint8
+
+const (
+ // DelayedEvictionDefault has unspecified behavior.
+ DelayedEvictionDefault DelayedEvictionType = iota
+
+ // DelayedEvictionDisabled requires that evictable allocations are evicted
+ // as soon as possible.
+ DelayedEvictionDisabled
+
+ // DelayedEvictionEnabled requests that the MemoryFile delay eviction of
+ // evictable allocations until doing so is considered necessary to avoid
+ // performance degradation due to host memory pressure, or OOM kills.
+ //
+ // As of this writing, the behavior of DelayedEvictionEnabled depends on
+ // whether or not MemoryFileOpts.UseHostMemcgPressure is enabled:
+ //
+ // - If UseHostMemcgPressure is true, evictions are delayed until memory
+ // pressure is indicated.
+ //
+ // - Otherwise, evictions are only delayed until the releaser goroutine is
+ // out of work (pages to release).
+ DelayedEvictionEnabled
+
+ // DelayedEvictionManual requires that evictable allocations are only
+ // evicted when MemoryFile.StartEvictions() is called. This is extremely
+ // dangerous outside of tests.
+ DelayedEvictionManual
)
// NewMemoryFile creates a MemoryFile backed by the given file. If
@@ -341,12 +421,10 @@ func NewMemoryFile(file *os.File, opts MemoryFileOpts) (*MemoryFile, error) {
return nil, err
}
f := &MemoryFile{
- opts: opts,
- file: file,
- evictable: make(map[EvictableMemoryUser]*evictableMemoryUserInfo),
+ opts: opts,
+ file: file,
}
- f.mappings.Store(&[]uintptr{})
- f.reclaimCond.L = &f.mu
+ f.initFields()
if f.opts.DelayedEviction == DelayedEvictionEnabled && f.opts.UseHostMemcgPressure {
stop, err := hostmm.NotifyCurrentMemcgPressureCallback(func() {
@@ -363,7 +441,7 @@ func NewMemoryFile(file *os.File, opts MemoryFileOpts) (*MemoryFile, error) {
f.stopNotifyPressure = stop
}
- go f.runReclaim() // S/R-SAFE: f.mu
+ go f.releaserMain() // S/R-SAFE: f.mu
if !opts.DisableIMAWorkAround {
IMAWorkAroundForMemFile(file.Fd())
@@ -371,6 +449,20 @@ func NewMemoryFile(file *os.File, opts MemoryFileOpts) (*MemoryFile, error) {
return f, nil
}
+func (f *MemoryFile) initFields() {
+ // Initially, all pages are void.
+ fullFR := memmap.FileRange{0, math.MaxUint64}
+ f.unwasteSmall.InsertRange(fullFR, unwasteInfo{})
+ f.unwasteHuge.InsertRange(fullFR, unwasteInfo{})
+ f.releaseCond.L = &f.mu
+ f.unfreeSmall.InsertRange(fullFR, unfreeInfo{})
+ f.unfreeHuge.InsertRange(fullFR, unfreeInfo{})
+ f.subreleased = make(map[uint64]uint64)
+ f.evictable = make(map[EvictableMemoryUser]*evictableMemoryUserInfo)
+ chunks := []chunkInfo(nil)
+ f.chunks.Store(&chunks)
+}
+
// IMAWorkAroundForMemFile works around IMA by immediately creating a temporary
// PROT_EXEC mapping, while the backing file is still small. IMA will ignore
// any future mappings.
@@ -415,50 +507,56 @@ func (f *MemoryFile) Destroy() {
f.mu.Lock()
defer f.mu.Unlock()
f.destroyed = true
- f.reclaimCond.Signal()
+ f.releaseCond.Signal()
}
-// AllocationMode provides a way to inform the pgalloc API how to allocate
-// memory and pages on the host.
-// A page will exist in one of the following incremental states:
-// 1. Allocated: A page is allocated if it was returned by Allocate() and its
-// reference count hasn't dropped to 0 since then.
-// 2. Committed: As described in MemoryFile documentation above, a page is
-// committed if the host kernel is spending resources to store its
-// contents. A committed page is implicitly allocated.
-// 3. Populated: A page is populated for reading/writing in a page table
-// hierarchy if it has a page table entry that permits reading/writing
-// respectively. A populated page is implicitly committed, since the page
-// table entry needs a physical page to point to, but not vice versa.
-type AllocationMode int
+// Preconditions: f.mu must be locked.
+func (f *MemoryFile) releaserDestroyLocked() {
+ if !f.destroyed {
+ panic("destroyed is no longer set")
+ }
-const (
- // AllocateOnly indicates that pages need to only be allocated.
- AllocateOnly AllocationMode = iota
- // AllocateAndCommit indicates that pages need to be committed, in addition
- // to being allocated.
- AllocateAndCommit
- // AllocateAndWritePopulate indicates that writable pages should ideally be
- // populated in the page table, in addition to being allocated. This is a
- // suggestion, not a requirement.
- AllocateAndWritePopulate
-)
+ if f.opts.DecommitOnDestroy {
+ if chunks := f.chunksLoad(); len(chunks) != 0 {
+ if err := f.decommitFile(memmap.FileRange{0, uint64(len(chunks)) * chunkSize}); err != nil {
+ panic(fmt.Sprintf("failed to decommit entire memory file during destruction: %v", err))
+ }
+ }
+ }
+
+ f.file.Close()
+ // Ensure that any attempts to use f.file.Fd() fail instead of getting a fd
+ // that has possibly been reassigned.
+ f.file = nil
+ chunks := f.chunksLoad()
+ for i := range chunks {
+ chunk := &chunks[i]
+ _, _, errno := unix.Syscall(unix.SYS_MUNMAP, chunk.mapping, chunkSize, 0)
+ if errno != 0 {
+ log.Warningf("Failed to unmap mapping %#x for MemoryFile chunk %d: %v", chunk.mapping, i, errno)
+ }
+ chunk.mapping = 0
+ }
+}
// AllocOpts are options used in MemoryFile.Allocate.
type AllocOpts struct {
- // Kind is the memory kind to be used for accounting.
+ // Kind is the allocation's memory accounting type.
Kind usage.MemoryKind
- // Dir indicates the direction in which offsets are allocated.
- Dir Direction
+
// MemCgID is the memory cgroup ID and the zero value indicates that
// the memory will not be accounted to any cgroup.
MemCgID uint32
- // Mode allows the callers to select how the pages are allocated in the
- // MemoryFile. Callers that will fill the allocated memory by writing to it
- // should pass AllocateAndWritePopulate to avoid faulting page-by-page. Callers
- // that will fill the allocated memory by invoking host system calls should
- // pass AllocateOnly.
+
+ // Mode controls the commitment status of returned pages.
Mode AllocationMode
+
+ // If Huge is true, the allocation should be hugepage-backed if possible.
+ Huge bool
+
+ // Dir indicates the direction in which offsets are allocated.
+ Dir Direction
+
// If ReaderFunc is provided, the allocated memory is filled by calling it
// repeatedly until either length bytes are read or a non-nil error is
// returned. It returns the allocated memory, truncated down to the nearest
@@ -467,48 +565,174 @@ type AllocOpts struct {
ReaderFunc safemem.ReaderFunc
}
-// Allocate returns a range of initially-zeroed pages of the given length with
-// the given accounting kind and a single reference held by the caller. When
-// the last reference on an allocated page is released, ownership of the page
-// is returned to the MemoryFile, allowing it to be returned by a future call
-// to Allocate.
-//
-// Preconditions: length must be page-aligned and non-zero.
-func (f *MemoryFile) Allocate(length uint64, opts AllocOpts) (memmap.FileRange, error) {
- fr, err := f.allocate(length, &opts)
- if err != nil {
- return memmap.FileRange{}, err
+// Direction is the type of AllocOpts.Dir.
+type Direction uint8
+
+const (
+ // BottomUp allocates offsets in increasing offsets.
+ BottomUp Direction = iota
+ // TopDown allocates offsets in decreasing offsets.
+ TopDown
+)
+
+// String implements fmt.Stringer.
+func (d Direction) String() string {
+ switch d {
+ case BottomUp:
+ return "up"
+ case TopDown:
+ return "down"
}
+ panic(fmt.Sprintf("invalid direction: %d", d))
+}
+
+// AllocationMode is the type of AllocOpts.Mode.
+type AllocationMode int
+
+const (
+ // AllocateUncommitted indicates that MemoryFile.Allocate() must return
+ // uncommitted pages.
+ AllocateUncommitted AllocationMode = iota
+
+ // AllocateCallerIndirectCommit indicates that the caller of
+ // MemoryFile.Allocate() intends to commit all allocated pages, without
+ // using our page tables. Thus, Allocate() may return committed or
+ // uncommitted pages.
+ AllocateCallerIndirectCommit
+
+ // AllocateAndCommit indicates that MemoryFile.Allocate() must return
+ // committed pages.
+ AllocateAndCommit
+
+ // AllocateAndWritePopulate indicates that the caller of
+ // MemoryFile.Allocate() intends to commit all allocated pages, using our
+ // page tables. Thus, Allocate() may return committed or uncommitted pages,
+ // and should pre-populate page table entries permitting writing for
+ // mappings of those pages returned by MapInternal().
+ AllocateAndWritePopulate
+)
+
+// allocState holds the state of a call to MemoryFile.Allocate().
+type allocState struct {
+ length uint64
+ opts AllocOpts
+ willCommit bool // either us or our caller
+ recycled bool
+ huge bool
+}
+
+// Allocate returns a range of initially-zeroed pages of the given length, with
+// a single reference on each page held by the caller. When the last reference
+// on an allocated page is released, ownership of the page is returned to the
+// MemoryFile, allowing it to be returned by a future call to Allocate.
+//
+// Preconditions:
+// - length > 0.
+// - length must be page-aligned.
+// - If opts.Hugepage == true, length must be hugepage-aligned.
+func (f *MemoryFile) Allocate(length uint64, opts AllocOpts) (memmap.FileRange, error) {
+ if length == 0 || !hostarch.IsPageAligned(length) || (opts.Huge && !hostarch.IsHugePageAligned(length)) {
+ panic(fmt.Sprintf("invalid allocation length: %#x", length))
+ }
+
+ alloc := allocState{
+ length: length,
+ opts: opts,
+ willCommit: opts.Mode != AllocateUncommitted,
+ huge: opts.Huge && f.opts.ExpectHugepages,
+ }
+
+ fr, err := f.findAllocatableAndMarkUsed(&alloc)
+ if err != nil {
+ return fr, err
+ }
+
var dsts safemem.BlockSeq
- switch opts.Mode {
- case AllocateOnly: // Allocation is handled above. Nothing more to do.
- case AllocateAndCommit:
- if err := f.commitFile(fr); err != nil {
- f.DecRef(fr)
- return memmap.FileRange{}, err
+ if alloc.willCommit {
+ needHugeTouch := false
+ if alloc.recycled {
+ // We will need writable page table entries in our address space to
+ // zero these pages.
+ alloc.opts.Mode = AllocateAndWritePopulate
+ } else if alloc.opts.Mode != AllocateAndWritePopulate && ((alloc.huge && f.opts.AdviseHugepage) || (!alloc.huge && f.opts.AdviseNoHugepage)) {
+ // If Mode is AllocateCallerIndirectCommit and we do nothing, the
+ // first access to the allocation may be by the application,
+ // through a platform.AddressSpace, which may not have
+ // MADV_HUGEPAGE (=> vma flag VM_HUGEPAGE) set. Consequently,
+ // shmem_fault() => shmem_get_folio_gfp() will commit a small page.
+ //
+ // If Mode is AllocateAndCommit and we do nothing, the first access
+ // to the allocation is via fallocate(2), which has the same
+ // problem: shmem_fallocate() => shmem_get_folio() =>
+ // shmem_get_folio_gfp(vma=NULL).
+ //
+ // khugepaged may eventually collapse the containing
+ // hugepage-aligned region into a huge page when it scans our
+ // mapping (khugepaged_scan_mm_slot() => khugepaged_scan_file()),
+ // but this depends on khugepaged_max_ptes_none, and in addition to
+ // the latency and overhead of doing so, this will incur another
+ // round of page faults.
+ //
+ // If write-populating through our mappings succeeds, then it will
+ // avoid this problem. Otherwise, we need to touch each huge page
+ // through our mappings.
+ //
+ // An analogous problem applies if MADV_NOHUGEPAGE is required
+ // rather than MADV_HUGEPAGE; MADV_NOHUGEPAGE is only enabled if
+ // the file defaults to huge pages, so populating or touching
+ // through our mappings is needed to ensure that the allocation is
+ // small-page-backed. In this case, we only need to force
+ // commitment of one small page per huge page to prevent future
+ // page faults within the huge page from faulting a huge page,
+ // though there's nothing we can do about khugepaged.
+ alloc.opts.Mode = AllocateAndWritePopulate
+ needHugeTouch = true
}
- case AllocateAndWritePopulate:
- dsts, err = f.MapInternal(fr, hostarch.Write)
- if err != nil {
- f.DecRef(fr)
- return memmap.FileRange{}, err
- }
- if canPopulate() {
- rem := dsts
- for {
- if !tryPopulate(rem.Head()) {
- break
- }
- rem = rem.Tail()
- if rem.IsEmpty() {
- break
+
+ switch alloc.opts.Mode {
+ case AllocateUncommitted, AllocateCallerIndirectCommit:
+ // Nothing for us to do.
+ case AllocateAndCommit:
+ if err := f.commitFile(fr); err != nil {
+ f.DecRef(fr)
+ return memmap.FileRange{}, err
+ }
+ case AllocateAndWritePopulate:
+ dsts, err = f.MapInternal(fr, hostarch.Write)
+ if err != nil {
+ f.DecRef(fr)
+ return memmap.FileRange{}, err
+ }
+ if canPopulate() {
+ rem := dsts
+ for {
+ if !tryPopulate(rem.Head()) {
+ break
+ }
+ rem = rem.Tail()
+ if rem.IsEmpty() {
+ needHugeTouch = false
+ break
+ }
}
}
+ if alloc.recycled {
+ // The contents of recycled waste pages are initially unknown, so we
+ // need to zero them.
+ f.manuallyZero(fr)
+ } else if needHugeTouch {
+ // We only need to touch a single byte in each huge page.
+ f.forEachMappingSlice(fr, func(bs []byte) {
+ for i := 0; i < len(bs); i += hostarch.HugePageSize {
+ bs[i] = 0
+ }
+ })
+ }
+ default:
+ panic(fmt.Sprintf("unknown AllocOpts.Mode %d", alloc.opts.Mode))
}
- default:
- panic(fmt.Sprintf("unknown allocation mode: %d", opts.Mode))
}
- if opts.ReaderFunc != nil {
+ if alloc.opts.ReaderFunc != nil {
if dsts.IsEmpty() {
dsts, err = f.MapInternal(fr, hostarch.Write)
if err != nil {
@@ -516,7 +740,7 @@ func (f *MemoryFile) Allocate(length uint64, opts AllocOpts) (memmap.FileRange,
return memmap.FileRange{}, err
}
}
- n, err := safemem.ReadFullToBlocks(opts.ReaderFunc, dsts)
+ n, err := safemem.ReadFullToBlocks(alloc.opts.ReaderFunc, dsts)
un := uint64(hostarch.Addr(n).RoundDown())
if un < length {
// Free unused memory and update fr to contain only the memory that is
@@ -528,161 +752,223 @@ func (f *MemoryFile) Allocate(length uint64, opts AllocOpts) (memmap.FileRange,
return fr, err
}
}
+
return fr, nil
}
-func (f *MemoryFile) allocate(length uint64, opts *AllocOpts) (memmap.FileRange, error) {
- if length == 0 || length%hostarch.PageSize != 0 {
- panic(fmt.Sprintf("invalid allocation length: %#x", length))
+func (f *MemoryFile) findAllocatableAndMarkUsed(alloc *allocState) (fr memmap.FileRange, err error) {
+ unwaste := &f.unwasteSmall
+ unfree := &f.unfreeSmall
+ if alloc.huge {
+ unwaste = &f.unwasteHuge
+ unfree = &f.unfreeHuge
}
f.mu.Lock()
defer f.mu.Unlock()
- // Align hugepage-and-larger allocations on hugepage boundaries to try
- // to take advantage of hugetmpfs.
- alignment := uint64(hostarch.PageSize)
- if length >= hostarch.HugePageSize {
- alignment = hostarch.HugePageSize
- }
-
- // Find a range in the underlying file.
- fr, ok := f.findAvailableRange(length, alignment, opts.Dir)
- if !ok {
- return memmap.FileRange{}, linuxerr.ENOMEM
- }
-
- // Expand the file if needed.
- if int64(fr.End) > f.fileSize {
- // Round the new file size up to be chunk-aligned.
- newFileSize := (int64(fr.End) + chunkMask) &^ chunkMask
- if err := f.file.Truncate(newFileSize); err != nil {
- return memmap.FileRange{}, err
+ if alloc.willCommit {
+ // Try to recycle waste pages, since this avoids the overhead of
+ // decommitting and then committing them again.
+ var uwgap unwasteGapIterator
+ if alloc.opts.Dir == BottomUp {
+ uwgap = unwaste.FirstLargeEnoughGap(alloc.length)
+ } else {
+ uwgap = unwaste.LastLargeEnoughGap(alloc.length)
}
- f.fileSize = newFileSize
- f.mappingsMu.Lock()
- oldMappings := *f.mappings.Load()
- newMappings := make([]uintptr, newFileSize>>chunkShift)
- copy(newMappings, oldMappings)
- f.mappings.Store(&newMappings)
- f.mappingsMu.Unlock()
- }
-
- if f.opts.ManualZeroing {
- if err := f.manuallyZero(fr); err != nil {
- return memmap.FileRange{}, err
+ if uwgap.Ok() {
+ alloc.recycled = true
+ if alloc.opts.Dir == BottomUp {
+ fr = memmap.FileRange{
+ Start: uwgap.Start(),
+ End: uwgap.Start() + alloc.length,
+ }
+ } else {
+ fr = memmap.FileRange{
+ Start: uwgap.End() - alloc.length,
+ End: uwgap.End(),
+ }
+ }
+ unwaste.Insert(uwgap, fr, unwasteInfo{})
+ // Update reference count for these pages from 0 to 1.
+ unfree.MutateFullRange(fr, func(ufseg unfreeIterator) bool {
+ uf := ufseg.ValuePtr()
+ if uf.refs != 0 {
+ panic(fmt.Sprintf("waste pages %v have unexpected refcount %d during recycling of %v\n%s", ufseg.Range(), uf.refs, fr, f.stringLocked()))
+ }
+ uf.refs = 1
+ return true
+ })
+ // These pages should all be unknown-commitment or known-committed;
+ // mark them unknown-commitment, for consistency with non-recycling
+ // allocations (below).
+ f.memAcct.MutateFullRange(fr, func(maseg memAcctIterator) bool {
+ ma := maseg.ValuePtr()
+ malen := maseg.Range().Length()
+ if ma.knownCommitted {
+ if ma.kind != usage.System {
+ panic(fmt.Sprintf("waste pages %v have unexpected kind %v\n%s", maseg.Range(), ma.kind, f.stringLocked()))
+ }
+ ma.knownCommitted = false
+ ma.commitSeq = 0
+ f.knownCommittedBytes -= malen
+ if !f.opts.DisableMemoryAccounting {
+ usage.MemoryAccounting.Dec(malen, usage.System, ma.memCgID)
+ }
+ }
+ ma.kind = alloc.opts.Kind
+ ma.memCgID = alloc.opts.MemCgID
+ ma.wasteOrReleasing = false
+ return true
+ })
+ return
}
}
- // Mark selected pages as in use.
- f.usage.InsertRange(fr, usageInfo{
- kind: opts.Kind,
- refs: 1,
- memCgID: opts.MemCgID,
+
+ // No suitable waste pages or we can't use them.
+retryFree:
+ // Try to allocate free pages from existing chunks.
+ var ufgap unfreeGapIterator
+ if alloc.opts.Dir == BottomUp {
+ ufgap = unfree.FirstLargeEnoughGap(alloc.length)
+ } else {
+ ufgap = unfree.LastLargeEnoughGap(alloc.length)
+ }
+ if !ufgap.Ok() {
+ // Extend the file to create more chunks.
+ err = f.extendChunksLocked(alloc)
+ if err != nil {
+ return
+ }
+ // Retry the allocation using new chunks.
+ goto retryFree
+ }
+ if alloc.opts.Dir == BottomUp {
+ fr = memmap.FileRange{
+ Start: ufgap.Start(),
+ End: ufgap.Start() + alloc.length,
+ }
+ } else {
+ fr = memmap.FileRange{
+ Start: ufgap.End() - alloc.length,
+ End: ufgap.End(),
+ }
+ }
+ unfree.Insert(ufgap, fr, unfreeInfo{refs: 1})
+ // These pages should all be known-decommitted; mark them
+ // unknown-commitment, since they can be concurrently committed by the
+ // allocation's users at any time until deallocation.
+ //
+ // If alloc.willCommit is true, we expect these pages to become committed
+ // in the near future; mark them unknown-commitment anyway, since marking
+ // them committed prematurely makes them more likely to be saved even if
+ // zeroed, unless SaveOpts.ExcludeCommittedZeroPages is enabled.
+ f.memAcct.InsertRange(fr, memAcctInfo{
+ kind: alloc.opts.Kind,
+ memCgID: alloc.opts.MemCgID,
+ knownCommitted: false,
+ commitSeq: f.commitSeq,
+ })
+ return
+}
+
+// Preconditions: f.mu must be locked.
+func (f *MemoryFile) extendChunksLocked(alloc *allocState) error {
+ unfree := &f.unfreeSmall
+ if alloc.huge {
+ unfree = &f.unfreeHuge
+ }
+
+ oldChunks := f.chunksLoad()
+ oldNrChunks := uint64(len(oldChunks))
+ oldFileSize := oldNrChunks * chunkSize
+
+ // Determine how many chunks we need to satisfy alloc.
+ tail := uint64(0)
+ if oldNrChunks != 0 {
+ if lastChunk := oldChunks[oldNrChunks-1]; lastChunk.huge == alloc.huge {
+ // We can use free pages at the end of the current last chunk.
+ if ufgap := unfree.FindGap(oldFileSize - 1); ufgap.Ok() {
+ tail = ufgap.Range().Length()
+ }
+ }
+ }
+ incNrChunks := (alloc.length + chunkMask - tail) / chunkSize
+ incFileSize := incNrChunks * chunkSize
+ newNrChunks := oldNrChunks + incNrChunks
+ if newNrChunks > maxChunks || newNrChunks < oldNrChunks /* overflow */ {
+ return linuxerr.ENOMEM
+ }
+ newFileSize := newNrChunks * chunkSize
+
+ // Extend the backing file and obtain mappings for the new chunks. If the
+ // backing file is memory-backed, and THP is enabled, Linux will align our
+ // mapping to a hugepage boundary; see
+ // mm/shmem.c:shmem_get_unmapped_area().
+ //
+ // In tests, f.file may be nil.
+ var mapStart uintptr
+ if f.file != nil {
+ if err := f.file.Truncate(int64(newFileSize)); err != nil {
+ return err
+ }
+ m, _, errno := unix.Syscall6(
+ unix.SYS_MMAP,
+ 0,
+ uintptr(incFileSize),
+ unix.PROT_READ|unix.PROT_WRITE,
+ unix.MAP_SHARED,
+ f.file.Fd(),
+ uintptr(oldFileSize))
+ if errno != 0 {
+ return errno
+ }
+ mapStart = m
+ f.madviseChunkMapping(mapStart, uintptr(incFileSize), alloc.huge)
+ }
+
+ // Update chunk state.
+ newChunks := make([]chunkInfo, newNrChunks, newNrChunks)
+ copy(newChunks, oldChunks)
+ m := mapStart
+ for i := oldNrChunks; i < newNrChunks; i++ {
+ newChunks[i].huge = alloc.huge
+ if f.file != nil {
+ newChunks[i].mapping = m
+ m += chunkSize
+ }
+ }
+ f.chunks.Store(&newChunks)
+
+ // Mark void pages free.
+ unfree.RemoveFullRange(memmap.FileRange{
+ Start: oldNrChunks * chunkSize,
+ End: newNrChunks * chunkSize,
})
- return fr, nil
+ return nil
}
-// findAvailableRange returns an available range in the usageSet.
-//
-// Note that scanning for available slots takes place from end first backwards,
-// then forwards. This heuristic has important consequence for how sequential
-// mappings can be merged in the host VMAs, given that addresses for both
-// application and sentry mappings are allocated top-down (from higher to
-// lower addresses). The file is also grown exponentially in order to create
-// space for mappings to be allocated downwards.
-//
-// Precondition: alignment must be a power of 2.
-func (f *MemoryFile) findAvailableRange(length, alignment uint64, dir Direction) (memmap.FileRange, bool) {
- if dir == BottomUp {
- return findAvailableRangeBottomUp(&f.usage, length, alignment)
- }
- return findAvailableRangeTopDown(&f.usage, f.fileSize, length, alignment)
-}
-
-func findAvailableRangeTopDown(usage *usageSet, fileSize int64, length, alignment uint64) (memmap.FileRange, bool) {
- alignmentMask := alignment - 1
-
- // Search for space in existing gaps, starting at the current end of the
- // file and working backward.
- lastGap := usage.LastGap()
- gap := lastGap
- for {
- end := gap.End()
- if end > uint64(fileSize) {
- end = uint64(fileSize)
- }
-
- // Try to allocate from the end of this gap, with the start of the
- // allocated range aligned down to alignment.
- unalignedStart := end - length
- if unalignedStart > end {
- // Negative overflow: this and all preceding gaps are too small to
- // accommodate length.
- break
- }
- if start := unalignedStart &^ alignmentMask; start >= gap.Start() {
- return memmap.FileRange{start, start + length}, true
- }
-
- gap = gap.PrevLargeEnoughGap(length)
- if !gap.Ok() {
- break
- }
- }
-
- // Check that it's possible to fit this allocation at the end of a file of any size.
- min := lastGap.Start()
- min = (min + alignmentMask) &^ alignmentMask
- if min+length < min {
- // Overflow: allocation would exceed the range of uint64.
- return memmap.FileRange{}, false
- }
-
- // Determine the minimum file size required to fit this allocation at its end.
- for {
- newFileSize := 2 * fileSize
- if newFileSize <= fileSize {
- if fileSize != 0 {
- // Overflow: allocation would exceed the range of int64.
- return memmap.FileRange{}, false
+func (f *MemoryFile) madviseChunkMapping(addr, len uintptr, huge bool) {
+ if huge {
+ if f.opts.AdviseHugepage {
+ _, _, errno := unix.Syscall(unix.SYS_MADVISE, addr, len, unix.MADV_HUGEPAGE)
+ if errno != 0 {
+ // Log this failure but continue.
+ log.Warningf("madvise(%#x, %d, MADV_HUGEPAGE) failed: %s", addr, len, errno)
}
- newFileSize = chunkSize
}
- fileSize = newFileSize
-
- unalignedStart := uint64(fileSize) - length
- if unalignedStart > uint64(fileSize) {
- // Negative overflow: fileSize is still inadequate.
- continue
- }
- if start := unalignedStart &^ alignmentMask; start >= min {
- return memmap.FileRange{start, start + length}, true
+ } else {
+ if f.opts.AdviseNoHugepage {
+ _, _, errno := unix.Syscall(unix.SYS_MADVISE, addr, len, unix.MADV_NOHUGEPAGE)
+ if errno != 0 {
+ // Log this failure but continue.
+ log.Warningf("madvise(%#x, %d, MADV_NOHUGEPAGE) failed: %s", addr, len, errno)
+ }
}
}
}
-func findAvailableRangeBottomUp(usage *usageSet, length, alignment uint64) (memmap.FileRange, bool) {
- alignmentMask := alignment - 1
- for gap := usage.FirstGap(); gap.Ok(); gap = gap.NextLargeEnoughGap(length) {
- // Align the start address and check if allocation still fits in the gap.
- start := (gap.Start() + alignmentMask) &^ alignmentMask
-
- // File offsets are int64s. Since length must be strictly positive, end
- // cannot legitimately be 0.
- end := start + length
- if end < start || int64(end) <= 0 {
- return memmap.FileRange{}, false
- }
- if end <= gap.End() {
- return memmap.FileRange{start, end}, true
- }
- }
-
- // NextLargeEnoughGap should have returned a gap at the end.
- panic(fmt.Sprintf("NextLargeEnoughGap didn't return a gap at the end, length: %d", length))
-}
-
var mlockDisabled atomicbitops.Uint32
var madvPopulateWriteDisabled atomicbitops.Uint32
@@ -694,19 +980,13 @@ func tryPopulateMadv(b safemem.Block) bool {
if madvPopulateWriteDisabled.Load() != 0 {
return false
}
- start, ok := hostarch.Addr(b.Addr()).RoundUp()
- if !ok {
- return true
- }
- end := hostarch.Addr(b.Addr() + uintptr(b.Len())).RoundDown()
- bLen := end - start
// Only call madvise(MADV_POPULATE_WRITE) if >=2 pages are being populated.
// 1 syscall overhead >= 1 page fault overhead. This is because syscalls are
// susceptible to additional overheads like seccomp-bpf filters and auditing.
- if start >= end || bLen <= hostarch.PageSize {
+ if b.Len() <= hostarch.PageSize {
return true
}
- _, _, errno := unix.RawSyscall(unix.SYS_MADVISE, uintptr(start), uintptr(bLen), unix.MADV_POPULATE_WRITE)
+ _, _, errno := unix.Syscall(unix.SYS_MADVISE, b.Addr(), uintptr(b.Len()), unix.MADV_POPULATE_WRITE)
if errno != 0 {
if errno == unix.EINVAL {
// EINVAL is expected if MADV_POPULATE_WRITE is not supported (Linux <5.14).
@@ -779,41 +1059,35 @@ func tryPopulate(b safemem.Block) bool {
return tryPopulateMlock(b)
}
-// fallocate(2) modes, defined in Linux's include/uapi/linux/falloc.h.
-const (
- _FALLOC_FL_KEEP_SIZE = 1
- _FALLOC_FL_PUNCH_HOLE = 2
-)
-
-// Decommit releases resources associated with maintaining the contents of the
-// given pages. If Decommit succeeds, future accesses of the decommitted pages
-// will read zeroes.
+// Decommit uncommits the given pages, causing them to become zeroed.
//
-// Preconditions: fr.Length() > 0.
-func (f *MemoryFile) Decommit(fr memmap.FileRange) error {
+// Preconditions:
+// - fr.Start and fr.End must be page-aligned.
+// - fr.Length() > 0.
+// - At least one reference must be held on all pages in fr.
+func (f *MemoryFile) Decommit(fr memmap.FileRange) {
if !fr.WellFormed() || fr.Length() == 0 || fr.Start%hostarch.PageSize != 0 || fr.End%hostarch.PageSize != 0 {
panic(fmt.Sprintf("invalid range: %v", fr))
}
- if f.opts.ManualZeroing {
- // FALLOC_FL_PUNCH_HOLE may not zero pages if ManualZeroing is in
- // effect.
- if err := f.manuallyZero(fr); err != nil {
- return err
- }
- } else {
- if err := f.decommitFile(fr); err != nil {
- return err
- }
- }
+ f.decommitOrManuallyZero(fr)
- f.markDecommitted(fr)
- return nil
-}
-
-func (f *MemoryFile) manuallyZero(fr memmap.FileRange) error {
- return f.forEachMappingSlice(fr, func(bs []byte) {
- clear(bs)
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.memAcct.MutateFullRange(fr, func(maseg memAcctIterator) bool {
+ ma := maseg.ValuePtr()
+ if ma.knownCommitted {
+ ma.knownCommitted = false
+ malen := maseg.Range().Length()
+ f.knownCommittedBytes -= malen
+ if !f.opts.DisableMemoryAccounting {
+ usage.MemoryAccounting.Dec(malen, ma.kind, ma.memCgID)
+ }
+ }
+ // Update commitSeq to invalidate any observations made by
+ // concurrent calls to f.updateUsageLocked().
+ ma.commitSeq = f.commitSeq
+ return true
})
}
@@ -833,30 +1107,26 @@ func (f *MemoryFile) decommitFile(fr memmap.FileRange) error {
// FALLOC_FL_KEEP_SIZE in mode ..." - fallocate(2)
return unix.Fallocate(
int(f.file.Fd()),
- _FALLOC_FL_PUNCH_HOLE|_FALLOC_FL_KEEP_SIZE,
+ unix.FALLOC_FL_PUNCH_HOLE|unix.FALLOC_FL_KEEP_SIZE,
int64(fr.Start),
int64(fr.Length()))
}
-func (f *MemoryFile) markDecommitted(fr memmap.FileRange) {
- f.mu.Lock()
- defer f.mu.Unlock()
- // Since we're changing the knownCommitted attribute, we need to merge
- // across the entire range to ensure that the usage tree is minimal.
- f.usage.MutateFullRange(fr, func(seg usageIterator) bool {
- val := seg.ValuePtr()
- if val.knownCommitted {
- // Drop the usageExpected appropriately.
- amount := seg.Range().Length()
- usage.MemoryAccounting.Dec(amount, val.kind, val.memCgID)
- f.usageExpected -= amount
- val.knownCommitted = false
- }
- val.memCgID = 0
- return true
+func (f *MemoryFile) manuallyZero(fr memmap.FileRange) {
+ f.forEachMappingSlice(fr, func(bs []byte) {
+ clear(bs)
})
}
+func (f *MemoryFile) decommitOrManuallyZero(fr memmap.FileRange) {
+ if err := f.decommitFile(fr); err != nil {
+ log.Warningf("Failed to decommit %v: %v", fr, err)
+ // Zero the pages manually. This won't reduce memory usage, but at
+ // least ensures that the pages will be zeroed when reallocated.
+ f.manuallyZero(fr)
+ }
+}
+
// HasUniqueRef returns true if all pages in the given range have exactly one
// reference. A return value of false is inherently racy, but if the caller
// holds a reference on the given range and is preventing other goroutines from
@@ -864,67 +1134,278 @@ func (f *MemoryFile) markDecommitted(fr memmap.FileRange) {
//
// Preconditions: At least one reference must be held on all pages in fr.
func (f *MemoryFile) HasUniqueRef(fr memmap.FileRange) bool {
+ hasUniqueRef := true
f.mu.Lock()
defer f.mu.Unlock()
- hasUniqueRef := true
- f.usage.VisitFullRange(fr, func(seg usageIterator) bool {
- if seg.ValuePtr().refs != 1 {
- hasUniqueRef = false
- return false
+ f.forEachChunk(fr, func(chunk *chunkInfo, chunkFR memmap.FileRange) bool {
+ unfree := &f.unfreeSmall
+ if chunk.huge {
+ unfree = &f.unfreeHuge
}
- return true
+ unfree.VisitFullRange(fr, func(ufseg unfreeIterator) bool {
+ if ufseg.ValuePtr().refs != 1 {
+ hasUniqueRef = false
+ return false
+ }
+ return true
+ })
+ return hasUniqueRef
})
return hasUniqueRef
}
// IncRef implements memmap.File.IncRef.
func (f *MemoryFile) IncRef(fr memmap.FileRange, memCgID uint32) {
- if !fr.WellFormed() || fr.Length() == 0 || fr.Start%hostarch.PageSize != 0 || fr.End%hostarch.PageSize != 0 {
+ if !fr.WellFormed() || fr.Length() == 0 || !hostarch.IsPageAligned(fr.Start) || !hostarch.IsPageAligned(fr.End) {
panic(fmt.Sprintf("invalid range: %v", fr))
}
f.mu.Lock()
defer f.mu.Unlock()
- f.usage.MutateFullRange(fr, func(seg usageIterator) bool {
- seg.ValuePtr().refs++
+ f.forEachChunk(fr, func(chunk *chunkInfo, chunkFR memmap.FileRange) bool {
+ unfree := &f.unfreeSmall
+ if chunk.huge {
+ unfree = &f.unfreeHuge
+ }
+ unfree.MutateFullRange(chunkFR, func(ufseg unfreeIterator) bool {
+ uf := ufseg.ValuePtr()
+ if uf.refs <= 0 {
+ panic(fmt.Sprintf("IncRef(%v) called with %d references on pages %v", fr, uf.refs, ufseg.Range()))
+ }
+ uf.refs++
+ return true
+ })
return true
})
}
// DecRef implements memmap.File.DecRef.
func (f *MemoryFile) DecRef(fr memmap.FileRange) {
- if !fr.WellFormed() || fr.Length() == 0 || fr.Start%hostarch.PageSize != 0 || fr.End%hostarch.PageSize != 0 {
+ if !fr.WellFormed() || fr.Length() == 0 || !hostarch.IsPageAligned(fr.Start) || !hostarch.IsPageAligned(fr.End) {
panic(fmt.Sprintf("invalid range: %v", fr))
}
- var freed bool
-
f.mu.Lock()
defer f.mu.Unlock()
- f.usage.MutateFullRange(fr, func(seg usageIterator) bool {
- val := seg.ValuePtr()
- if val.refs == 0 {
- panic(fmt.Sprintf("DecRef(%v): 0 existing references on %v:\n%v", fr, seg.Range(), &f.usage))
+ haveWaste := false
+ f.forEachChunk(fr, func(chunk *chunkInfo, chunkFR memmap.FileRange) bool {
+ unwaste := &f.unwasteSmall
+ unfree := &f.unfreeSmall
+ if chunk.huge {
+ unwaste = &f.unwasteHuge
+ unfree = &f.unfreeHuge
}
- val.refs--
- if val.refs == 0 {
- f.reclaim.InsertRange(seg.Range(), reclaimSetValue{})
- freed = true
- // Reclassify memory as System, until it's freed by the reclaim
- // goroutine.
- if val.knownCommitted {
- usage.MemoryAccounting.Move(seg.Range().Length(), usage.System, val.kind, val.memCgID)
+ unfree.MutateFullRange(chunkFR, func(ufseg unfreeIterator) bool {
+ uf := ufseg.ValuePtr()
+ if uf.refs <= 0 {
+ panic(fmt.Sprintf("DecRef(%v) called with %d references on pages %v", fr, uf.refs, ufseg.Range()))
}
- val.kind = usage.System
- }
+ uf.refs--
+ if uf.refs == 0 {
+ // Mark these pages as waste.
+ wasteFR := ufseg.Range()
+ unwaste.RemoveFullRange(wasteFR)
+ haveWaste = true
+ // Reclassify waste memory as System until it's recycled or
+ // released.
+ f.memAcct.MutateFullRange(wasteFR, func(maseg memAcctIterator) bool {
+ ma := maseg.ValuePtr()
+ if !f.opts.DisableMemoryAccounting && ma.knownCommitted {
+ usage.MemoryAccounting.Move(maseg.Range().Length(), usage.System, ma.kind, ma.memCgID)
+ }
+ ma.kind = usage.System
+ ma.wasteOrReleasing = true
+ return true
+ })
+ }
+ return true
+ })
return true
})
- if freed {
- f.reclaimable = true
- f.reclaimCond.Signal()
+ // Wake the releaser if we marked any pages as waste. Leave this until just
+ // before unlocking f.mu.
+ if haveWaste && !f.haveWaste {
+ f.haveWaste = true
+ f.releaseCond.Signal()
+ }
+}
+
+// releaserMain implements the releaser goroutine.
+func (f *MemoryFile) releaserMain() {
+ f.mu.Lock()
+MainLoop:
+ for {
+ for {
+ if f.destroyed {
+ f.releaserDestroyLocked()
+ f.mu.Unlock()
+ // This must be called without holding f.mu to avoid circular lock
+ // ordering.
+ if f.stopNotifyPressure != nil {
+ f.stopNotifyPressure()
+ }
+ return
+ }
+ if f.haveWaste {
+ break
+ }
+ if f.opts.DelayedEviction == DelayedEvictionEnabled && !f.opts.UseHostMemcgPressure {
+ // No work to do. Evict any pending evictable allocations to
+ // get more waste pages before going to sleep.
+ f.startEvictionsLocked()
+ }
+ f.releaseCond.Wait() // releases f.mu while waiting
+ }
+ // Huge pages are relatively rare and expensive due to fragmentation
+ // and the cost of compaction. Fragmentation is expected to increase
+ // over time. Most allocations are done upwards, with the main
+ // exception being thread stacks. So we expect lower offsets to weakly
+ // correlate with older allocations, which are more likely to actually
+ // be hugepage-backed. Thus, release from unwasteSmall before
+ // unwasteHuge, and higher offsets before lower ones.
+ for i, unwaste := range []*unwasteSet{&f.unwasteSmall, &f.unwasteHuge} {
+ if uwgap := unwaste.LastLargeEnoughGap(1); uwgap.Ok() {
+ fr := uwgap.Range()
+ // Linux serializes fallocate()s on shmem files, so limit the amount we
+ // release at once to avoid starving Decommit().
+ const maxReleasingBytes = 128 << 20 // 128 MB
+ if fr.Length() > maxReleasingBytes {
+ fr.Start = fr.End - maxReleasingBytes
+ }
+ unwaste.Insert(uwgap, fr, unwasteInfo{})
+ f.releaseLocked(fr, i == 1)
+ continue MainLoop
+ }
+ }
+ f.haveWaste = false
+ }
+}
+
+// Preconditions: f.mu must be locked; it may be unlocked and reacquired.
+func (f *MemoryFile) releaseLocked(fr memmap.FileRange, huge bool) {
+ defer func() {
+ maseg := f.memAcct.LowerBoundSegmentSplitBefore(fr.Start)
+ for maseg.Ok() && maseg.Start() < fr.End {
+ maseg = f.memAcct.SplitAfter(maseg, fr.End)
+ ma := maseg.ValuePtr()
+ if ma.kind != usage.System {
+ panic(fmt.Sprintf("waste pages %v have unexpected kind %v\n%s", maseg.Range(), ma.kind, f.stringLocked()))
+ }
+ if ma.knownCommitted {
+ malen := maseg.Range().Length()
+ f.knownCommittedBytes -= malen
+ if !f.opts.DisableMemoryAccounting {
+ usage.MemoryAccounting.Dec(malen, ma.kind, ma.memCgID)
+ }
+ }
+ maseg = f.memAcct.Remove(maseg).NextSegment()
+ }
+ }()
+
+ if !huge {
+ // Decommit the range being released, then mark the released range as
+ // freed.
+ f.mu.Unlock()
+ f.decommitOrManuallyZero(fr)
+ f.mu.Lock()
+ f.unfreeSmall.RemoveFullRange(fr)
+ return
+ }
+
+ // Handle huge pages and sub-release.
+
+ firstHugeStart := hostarch.HugePageRoundDown(fr.Start)
+ lastHugeStart := hostarch.HugePageRoundDown(fr.End - 1)
+ firstHugeEnd := firstHugeStart + hostarch.HugePageSize
+ lastHugeEnd := lastHugeStart + hostarch.HugePageSize
+ if firstHugeStart == lastHugeStart {
+ // All of fr falls within a single huge page.
+ oldSubrel := f.subreleased[firstHugeStart]
+ incSubrel := fr.Length() / hostarch.PageSize
+ newSubrel := oldSubrel + incSubrel
+ if newSubrel == pagesPerHugePage {
+ // Free this huge page.
+ //
+ // When a small page within a hugepage-backed allocation is
+ // individually deallocated (becomes waste), we decommit it to
+ // reduce memory usage (and for consistency with legacy behavior).
+ // This requires the host to split the containing huge page, if one
+ // exists. khugepaged may later re-assemble the containing huge
+ // page, implicitly re-committing previously-decommitted small
+ // pages as a result.
+ //
+ // Thus: When a huge page is freed, ensure that the whole huge page
+ // is decommitted rather than just the final small page(s), to
+ // ensure that we leave behind an uncommitted hugepage-sized range
+ // with no re-committed small pages.
+ if oldSubrel != 0 {
+ delete(f.subreleased, firstHugeStart)
+ }
+ hugeFR := memmap.FileRange{firstHugeStart, firstHugeEnd}
+ f.mu.Unlock()
+ f.decommitOrManuallyZero(hugeFR)
+ f.mu.Lock()
+ f.unfreeHuge.RemoveFullRange(hugeFR)
+ } else {
+ f.subreleased[firstHugeStart] = newSubrel
+ f.mu.Unlock()
+ f.decommitOrManuallyZero(fr)
+ f.mu.Lock()
+ }
+ return
+ }
+
+ // fr spans at least two huge pages. Resolve sub-release in the first and
+ // last huge pages; any huge pages in between are decommitted/freed in
+ // full.
+ var (
+ decommitFR memmap.FileRange
+ freeFR memmap.FileRange
+ )
+ if fr.Start == firstHugeStart {
+ decommitFR.Start = firstHugeStart
+ freeFR.Start = firstHugeStart
+ } else {
+ oldSubrel := f.subreleased[firstHugeStart]
+ incSubrel := (firstHugeEnd - fr.Start) / hostarch.PageSize
+ newSubrel := oldSubrel + incSubrel
+ if newSubrel == pagesPerHugePage {
+ if oldSubrel != 0 {
+ delete(f.subreleased, firstHugeStart)
+ }
+ decommitFR.Start = firstHugeStart
+ freeFR.Start = firstHugeStart
+ } else {
+ decommitFR.Start = fr.Start
+ freeFR.Start = firstHugeEnd
+ }
+ }
+ if fr.End == lastHugeEnd {
+ decommitFR.End = lastHugeEnd
+ freeFR.End = lastHugeEnd
+ } else {
+ oldSubrel := f.subreleased[lastHugeStart]
+ incSubrel := (fr.End - lastHugeStart) / hostarch.PageSize
+ newSubrel := oldSubrel + incSubrel
+ if newSubrel == pagesPerHugePage {
+ if oldSubrel != 0 {
+ delete(f.subreleased, lastHugeStart)
+ }
+ decommitFR.End = lastHugeEnd
+ freeFR.End = lastHugeEnd
+ } else {
+ decommitFR.End = fr.End
+ freeFR.End = lastHugeStart
+ }
+ }
+ f.mu.Unlock()
+ f.decommitOrManuallyZero(decommitFR)
+ f.mu.Lock()
+ if freeFR.Length() != 0 {
+ f.unfreeHuge.RemoveFullRange(freeFR)
}
}
@@ -937,72 +1418,29 @@ func (f *MemoryFile) MapInternal(fr memmap.FileRange, at hostarch.AccessType) (s
return safemem.BlockSeq{}, linuxerr.EACCES
}
- chunks := ((fr.End + chunkMask) >> chunkShift) - (fr.Start >> chunkShift)
+ chunks := ((fr.End + chunkMask) / chunkSize) - (fr.Start / chunkSize)
if chunks == 1 {
// Avoid an unnecessary slice allocation.
var seq safemem.BlockSeq
- err := f.forEachMappingSlice(fr, func(bs []byte) {
+ f.forEachMappingSlice(fr, func(bs []byte) {
seq = safemem.BlockSeqOf(safemem.BlockFromSafeSlice(bs))
})
- return seq, err
+ return seq, nil
}
blocks := make([]safemem.Block, 0, chunks)
- err := f.forEachMappingSlice(fr, func(bs []byte) {
+ f.forEachMappingSlice(fr, func(bs []byte) {
blocks = append(blocks, safemem.BlockFromSafeSlice(bs))
})
- return safemem.BlockSeqFromSlice(blocks), err
+ return safemem.BlockSeqFromSlice(blocks), nil
}
// forEachMappingSlice invokes fn on a sequence of byte slices that
// collectively map all bytes in fr.
-func (f *MemoryFile) forEachMappingSlice(fr memmap.FileRange, fn func([]byte)) error {
- mappings := *f.mappings.Load()
- for chunkStart := fr.Start &^ chunkMask; chunkStart < fr.End; chunkStart += chunkSize {
- chunk := int(chunkStart >> chunkShift)
- m := atomic.LoadUintptr(&mappings[chunk])
- if m == 0 {
- var err error
- mappings, m, err = f.getChunkMapping(chunk)
- if err != nil {
- return err
- }
- }
- startOff := uint64(0)
- if chunkStart < fr.Start {
- startOff = fr.Start - chunkStart
- }
- endOff := uint64(chunkSize)
- if chunkStart+chunkSize > fr.End {
- endOff = fr.End - chunkStart
- }
- fn(unsafeSlice(m, chunkSize)[startOff:endOff])
- }
- return nil
-}
-
-func (f *MemoryFile) getChunkMapping(chunk int) ([]uintptr, uintptr, error) {
- f.mappingsMu.Lock()
- defer f.mappingsMu.Unlock()
- // Another thread may have replaced f.mappings altogether due to file
- // expansion.
- mappings := *f.mappings.Load()
- // Another thread may have already mapped the chunk.
- if m := mappings[chunk]; m != 0 {
- return mappings, m, nil
- }
- m, _, errno := unix.Syscall6(
- unix.SYS_MMAP,
- 0,
- chunkSize,
- unix.PROT_READ|unix.PROT_WRITE,
- unix.MAP_SHARED,
- f.file.Fd(),
- uintptr(chunk< 0 {
- if err := f.decommitFile(memmap.FileRange{Start: 0, End: uint64(f.fileSize)}); err != nil {
- f.mu.Unlock()
- panic(fmt.Sprintf("failed to decommit entire memory file during destruction: %v", err))
+ fmt.Fprintf(&b, "unfreeSmall:\n%s", &f.unfreeSmall)
+ if f.opts.ExpectHugepages {
+ fmt.Fprintf(&b, "unfreeHuge:\n%s", &f.unfreeHuge)
+ fmt.Fprintf(&b, "subreleased:\n")
+ for off, pgs := range f.subreleased {
+ fmt.Fprintf(&b, "- %#x: %d\n", off, pgs)
}
}
- f.file.Close()
- // Ensure that any attempts to use f.file.Fd() fail instead of getting a fd
- // that has possibly been reassigned.
- f.file = nil
- f.mappingsMu.Lock()
- defer f.mappingsMu.Unlock()
- mappings := *f.mappings.Load()
- for i, m := range mappings {
- if m != 0 {
- _, _, errno := unix.Syscall(unix.SYS_MUNMAP, m, chunkSize, 0)
- if errno != 0 {
- log.Warningf("Failed to unmap mapping %#x for MemoryFile chunk %d: %v", m, i, errno)
- }
- }
- }
- // Similarly, invalidate f.mappings
- f.mappings.Store(nil)
- f.mu.Unlock()
-
- // This must be called without holding f.mu to avoid circular lock
- // ordering.
- if f.stopNotifyPressure != nil {
- f.stopNotifyPressure()
- }
-}
-
-// findReclaimable finds memory that has been marked for reclaim.
-//
-// Note that there returned range will be removed from tracking. It
-// must be reclaimed (removed from f.usage) at this point.
-func (f *MemoryFile) findReclaimable() (memmap.FileRange, bool) {
- f.mu.Lock()
- defer f.mu.Unlock()
- for {
- for {
- if f.destroyed {
- return memmap.FileRange{}, false
- }
- if f.reclaimable {
- break
- }
- if f.opts.DelayedEviction == DelayedEvictionEnabled && !f.opts.UseHostMemcgPressure {
- // No work to do. Evict any pending evictable allocations to
- // get more reclaimable pages before going to sleep.
- f.startEvictionsLocked()
- }
- f.reclaimCond.Wait()
- }
- // Most allocations are done upwards, with exceptions being stacks and some
- // allocators that allocate top-down. Reclaim preserves this order to
- // minimize the cost of the search.
- if seg := f.reclaim.FirstSegment(); seg.Ok() {
- fr := seg.Range()
- f.reclaim.Remove(seg)
- return fr, true
- }
- // Nothing is reclaimable.
- f.reclaimable = false
- }
-}
-
-func (f *MemoryFile) markReclaimed(fr memmap.FileRange) {
- f.mu.Lock()
- defer f.mu.Unlock()
- seg := f.usage.FindSegment(fr.Start)
- // All of fr should be mapped to a single uncommitted reclaimable
- // segment accounted to System.
- if !seg.Ok() {
- panic(fmt.Sprintf("reclaimed pages %v include unreferenced pages:\n%v", fr, &f.usage))
- }
- if !seg.Range().IsSupersetOf(fr) {
- panic(fmt.Sprintf("reclaimed pages %v are not entirely contained in segment %v with state %v:\n%v", fr, seg.Range(), seg.Value(), &f.usage))
- }
- if got, want := seg.Value(), (usageInfo{
- kind: usage.System,
- knownCommitted: false,
- refs: 0,
- memCgID: 0,
- }); got != want {
- panic(fmt.Sprintf("reclaimed pages %v in segment %v has incorrect state %v, wanted %v:\n%v", fr, seg.Range(), got, want, &f.usage))
- }
- // Deallocate reclaimed pages. Even though all of seg is reclaimable,
- // the caller of markReclaimed may not have decommitted it, so we can
- // only mark fr as reclaimed.
- f.usage.Remove(f.usage.Isolate(seg, fr))
+ fmt.Fprintf(&b, "memAcct:\n%s", &f.memAcct)
+ return b.String()
}
// StartEvictions requests that f evict all evictable allocations. It does not
@@ -1569,24 +1859,66 @@ func (f *MemoryFile) WaitForEvictions() {
f.evictionWG.Wait()
}
-type usageSetFunctions struct{}
+type unwasteSetFunctions struct{}
-func (usageSetFunctions) MinKey() uint64 {
+func (unwasteSetFunctions) MinKey() uint64 {
return 0
}
-func (usageSetFunctions) MaxKey() uint64 {
+func (unwasteSetFunctions) MaxKey() uint64 {
return math.MaxUint64
}
-func (usageSetFunctions) ClearValue(val *usageInfo) {
+func (unwasteSetFunctions) ClearValue(val *unwasteInfo) {
}
-func (usageSetFunctions) Merge(_ memmap.FileRange, val1 usageInfo, _ memmap.FileRange, val2 usageInfo) (usageInfo, bool) {
+func (unwasteSetFunctions) Merge(_ memmap.FileRange, val1 unwasteInfo, _ memmap.FileRange, val2 unwasteInfo) (unwasteInfo, bool) {
return val1, val1 == val2
}
-func (usageSetFunctions) Split(_ memmap.FileRange, val usageInfo, _ uint64) (usageInfo, usageInfo) {
+func (unwasteSetFunctions) Split(_ memmap.FileRange, val unwasteInfo, _ uint64) (unwasteInfo, unwasteInfo) {
+ return val, val
+}
+
+type unfreeSetFunctions struct{}
+
+func (unfreeSetFunctions) MinKey() uint64 {
+ return 0
+}
+
+func (unfreeSetFunctions) MaxKey() uint64 {
+ return math.MaxUint64
+}
+
+func (unfreeSetFunctions) ClearValue(val *unfreeInfo) {
+}
+
+func (unfreeSetFunctions) Merge(_ memmap.FileRange, val1 unfreeInfo, _ memmap.FileRange, val2 unfreeInfo) (unfreeInfo, bool) {
+ return val1, val1 == val2
+}
+
+func (unfreeSetFunctions) Split(_ memmap.FileRange, val unfreeInfo, _ uint64) (unfreeInfo, unfreeInfo) {
+ return val, val
+}
+
+type memAcctSetFunctions struct{}
+
+func (memAcctSetFunctions) MinKey() uint64 {
+ return 0
+}
+
+func (memAcctSetFunctions) MaxKey() uint64 {
+ return math.MaxUint64
+}
+
+func (memAcctSetFunctions) ClearValue(val *memAcctInfo) {
+}
+
+func (memAcctSetFunctions) Merge(_ memmap.FileRange, val1 memAcctInfo, _ memmap.FileRange, val2 memAcctInfo) (memAcctInfo, bool) {
+ return val1, val1 == val2
+}
+
+func (memAcctSetFunctions) Split(_ memmap.FileRange, val memAcctInfo, _ uint64) (memAcctInfo, memAcctInfo) {
return val, val
}
@@ -1613,27 +1945,3 @@ func (evictableRangeSetFunctions) Merge(_ EvictableRange, _ evictableRangeSetVal
func (evictableRangeSetFunctions) Split(_ EvictableRange, _ evictableRangeSetValue, _ uint64) (evictableRangeSetValue, evictableRangeSetValue) {
return evictableRangeSetValue{}, evictableRangeSetValue{}
}
-
-// reclaimSetValue is the value type of reclaimSet.
-type reclaimSetValue struct{}
-
-type reclaimSetFunctions struct{}
-
-func (reclaimSetFunctions) MinKey() uint64 {
- return 0
-}
-
-func (reclaimSetFunctions) MaxKey() uint64 {
- return math.MaxUint64
-}
-
-func (reclaimSetFunctions) ClearValue(val *reclaimSetValue) {
-}
-
-func (reclaimSetFunctions) Merge(_ memmap.FileRange, _ reclaimSetValue, _ memmap.FileRange, _ reclaimSetValue) (reclaimSetValue, bool) {
- return reclaimSetValue{}, true
-}
-
-func (reclaimSetFunctions) Split(_ memmap.FileRange, _ reclaimSetValue, _ uint64) (reclaimSetValue, reclaimSetValue) {
- return reclaimSetValue{}, reclaimSetValue{}
-}
diff --git a/pkg/sentry/pgalloc/pgalloc_test.go b/pkg/sentry/pgalloc/pgalloc_test.go
index 0cd88e27c..603f97108 100644
--- a/pkg/sentry/pgalloc/pgalloc_test.go
+++ b/pkg/sentry/pgalloc/pgalloc_test.go
@@ -15,338 +15,570 @@
package pgalloc
import (
- "fmt"
"testing"
"gvisor.dev/gvisor/pkg/hostarch"
+ "gvisor.dev/gvisor/pkg/sentry/memmap"
)
const (
page = hostarch.PageSize
hugepage = hostarch.HugePageSize
- topPage = (1 << 63) - page
)
-func TestFindUnallocatedRange(t *testing.T) {
+// existingSegment represents a range of pages in a test MemoryFile that is not
+// void or free.
+type existingSegment struct {
+ start uint64
+ end uint64
+ state int
+}
+
+// Possible values for existingSegment.state:
+const (
+ existingUnspecified = iota
+ existingUsed
+ existingWaste
+ existingReleasing // or sub-releasing
+)
+
+func TestFindAllocatable(t *testing.T) {
for _, test := range []struct {
- name string
- usage []usageFlatSegment
- fileSize int64
- length uint64
- alignment uint64
- direction Direction
- want uint64
- expectFail bool
+ name string
+ // Initial state:
+ chunkHuge []bool
+ existing []existingSegment
+ // Allocation parameters:
+ length uint64
+ huge bool
+ recycle bool
+ dir Direction
+ // Expected outcome:
+ want uint64
}{
{
- name: "Initial allocation succeeds",
- length: page,
- alignment: page,
- direction: BottomUp,
- want: 0,
+ name: "initial small allocation, bottom-up",
+ length: page,
+ want: 0,
},
{
- name: "Initial allocation succeeds",
- length: page,
- alignment: page,
- direction: TopDown,
- want: chunkSize - page, // Grows by chunkSize, allocate down.
+ name: "initial small allocation, top-down",
+ length: page,
+ dir: TopDown,
+ want: chunkSize - page,
},
{
- name: "Allocation begins at start of file",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- },
- length: page,
- alignment: page,
- direction: BottomUp,
- want: 0,
+ name: "initial small allocation, multiple pages, top-down",
+ length: 2 * page,
+ dir: TopDown,
+ want: chunkSize - 2*page,
},
{
- name: "Allocation finds empty space at start of file",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- },
- fileSize: 2 * page,
- length: page,
- alignment: page,
- direction: TopDown,
+ name: "initial small allocation, recycling enabled, bottom-up",
+ length: page,
+ recycle: true,
+ want: 0,
},
{
- name: "Allocation finds empty space at end of file",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- },
- fileSize: 2 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: page,
+ name: "initial huge allocation, bottom-up",
+ length: hugepage,
+ huge: true,
+ want: 0,
},
{
- name: "In-use frames are not allocatable",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {page, 2 * page, usageInfo{refs: 2}},
- },
- length: page,
- alignment: page,
- direction: BottomUp,
- want: 2 * page,
+ name: "initial huge allocation, top-down",
+ length: hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - hugepage,
},
{
- name: "In-use frames are not allocatable",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {page, 2 * page, usageInfo{refs: 2}},
- },
- fileSize: 2 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: 3 * page, // Double fileSize, allocate top-down.
+ name: "initial huge allocation, multiple pages, top-down",
+ length: 2 * hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - 2*hugepage,
},
{
- name: "Reclaimable frames are not allocatable",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {page, 2 * page, usageInfo{refs: 0}},
- {2 * page, 3 * page, usageInfo{refs: 1}},
- },
- length: page,
- alignment: page,
- direction: BottomUp,
- want: 3 * page,
+ name: "initial huge allocation, recycling enabled, bottom-up",
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ want: 0,
},
{
- name: "Reclaimable frames are not allocatable",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {page, 2 * page, usageInfo{refs: 0}},
- {2 * page, 3 * page, usageInfo{refs: 1}},
- },
- fileSize: 3 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: 5 * page, // Double fileSize, grow down.
- },
- {
- name: "Gaps between in-use frames are allocatable",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {2 * page, 3 * page, usageInfo{refs: 1}},
- },
- length: page,
- alignment: page,
- direction: BottomUp,
- want: page,
- },
- {
- name: "Gaps between in-use frames are allocatable",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {2 * page, 3 * page, usageInfo{refs: 1}},
- },
- fileSize: 3 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: page,
- },
- {
- name: "Inadequately-sized gaps are rejected",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {2 * page, 3 * page, usageInfo{refs: 1}},
- },
- length: 2 * page,
- alignment: page,
- direction: BottomUp,
- want: 3 * page,
- },
- {
- name: "Inadequately-sized gaps are rejected",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- {2 * page, 3 * page, usageInfo{refs: 1}},
- },
- fileSize: 3 * page,
- length: 2 * page,
- alignment: page,
- direction: TopDown,
- want: 4 * page, // Double fileSize, grow down.
- },
- {
- name: "Alignment is honored at end of file",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- // Hugepage-sized gap here that shouldn't be allocated from
- // since it's incorrectly aligned.
- {hugepage + page, hugepage + 2*page, usageInfo{refs: 1}},
- },
+ name: "huge allocation uses huge pages in new chunk",
+ chunkHuge: []bool{false},
length: hugepage,
- alignment: hugepage,
- direction: BottomUp,
- want: 2 * hugepage,
+ huge: true,
+ want: chunkSize,
},
{
- name: "Alignment is honored at end of file",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- // Hugepage-sized gap here that shouldn't be allocated from
- // since it's incorrectly aligned.
- {hugepage + page, hugepage + 2*page, usageInfo{refs: 1}},
- },
- fileSize: hugepage + 2*page,
+ name: "huge allocation uses huge pages in existing chunk",
+ chunkHuge: []bool{false, true},
length: hugepage,
- alignment: hugepage,
- direction: TopDown,
- want: 3 * hugepage, // Double fileSize until alignment is satisfied, grow down.
+ huge: true,
+ want: chunkSize,
},
{
- name: "Alignment is honored before end of file",
- usage: []usageFlatSegment{
- {0, page, usageInfo{refs: 1}},
- // Page will need to be shifted down from top.
- {2*hugepage + page, 2*hugepage + 2*page, usageInfo{refs: 1}},
- },
- fileSize: 2*hugepage + 2*page,
+ name: "hugepage-sized non-huge allocation uses small pages in new chunk",
+ chunkHuge: []bool{true},
length: hugepage,
- alignment: hugepage,
- direction: TopDown,
- want: hugepage,
+ want: chunkSize,
},
{
- name: "Allocation doubles file size more than once if necessary",
- fileSize: page,
- length: 4 * page,
- alignment: page,
- direction: BottomUp,
- want: 0,
+ name: "hugepage-sized non-huge allocation uses small pages in existing chunk",
+ chunkHuge: []bool{true, false},
+ length: hugepage,
+ want: chunkSize,
},
{
- name: "Allocation doubles file size more than once if necessary",
- fileSize: page,
- length: 4 * page,
- alignment: page,
- direction: TopDown,
- want: 0,
- },
- {
- name: "Allocations are compact if possible",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- {3 * page, 4 * page, usageInfo{refs: 2}},
+ name: "bottom-up small allocation begins at start of file",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {page, 2 * page, existingUsed},
},
- fileSize: 4 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: 2 * page,
+ length: page,
+ want: 0,
},
{
- name: "Top-down allocation within one gap",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- {4 * page, 5 * page, usageInfo{refs: 2}},
- {7 * page, 8 * page, usageInfo{refs: 1}},
+ name: "top-down small allocation begins at end of last chunk",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - 2*page, chunkSize - page, existingUsed},
},
- fileSize: 8 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: 6 * page,
+ length: page,
+ dir: TopDown,
+ want: chunkSize - page,
},
{
- name: "Top-down allocation between multiple gaps",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- {3 * page, 4 * page, usageInfo{refs: 2}},
- {5 * page, 6 * page, usageInfo{refs: 1}},
+ name: "bottom-up huge allocation begins at start of file",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {hugepage, 2 * hugepage, existingUsed},
},
- fileSize: 6 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: 4 * page,
+ length: hugepage,
+ huge: true,
+ want: 0,
},
{
- name: "Top-down allocation with large top gap",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- {3 * page, 4 * page, usageInfo{refs: 2}},
+ name: "top-down huge allocation begins at end of last chunk",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - 2*hugepage, chunkSize - hugepage, existingUsed},
},
- fileSize: 8 * page,
- length: page,
- alignment: page,
- direction: TopDown,
- want: 7 * page,
+ length: hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - hugepage,
},
{
- name: "Gaps found with possible overflow",
- usage: []usageFlatSegment{
- {page, 2 * page, usageInfo{refs: 1}},
- {topPage - page, topPage, usageInfo{refs: 1}},
+ name: "bottom-up small allocation can extend multiple chunks",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize/2 - page, chunkSize / 2, existingUsed},
},
- fileSize: topPage,
- length: page,
- alignment: page,
- direction: TopDown,
- want: topPage - 2*page,
+ length: 2*chunkSize + page,
+ want: chunkSize / 2,
},
{
- name: "Overflow detected",
- usage: []usageFlatSegment{
- {page, topPage, usageInfo{refs: 1}},
+ name: "top-down small allocation can extend multiple chunks",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize/2 - page, chunkSize / 2, existingUsed},
},
- fileSize: topPage,
- length: 2 * page,
- alignment: page,
- direction: BottomUp,
- expectFail: true,
+ length: 2*chunkSize + page,
+ dir: TopDown,
+ want: chunkSize - page,
},
{
- name: "Overflow detected",
- usage: []usageFlatSegment{
- {page, topPage, usageInfo{refs: 1}},
+ name: "bottom-up huge allocation can extend multiple chunks",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize/2 - hugepage, chunkSize / 2, existingUsed},
},
- fileSize: topPage,
- length: 2 * page,
- alignment: page,
- direction: TopDown,
- expectFail: true,
+ length: 2*chunkSize + hugepage,
+ huge: true,
+ want: chunkSize / 2,
},
{
- name: "start may be in the middle of segment",
- usage: []usageFlatSegment{
- {0, 2 * page, usageInfo{refs: 1}},
- {3 * page, 4 * page, usageInfo{refs: 2}},
+ name: "top-down huge allocation can extend multiple chunks",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize/2 - hugepage, chunkSize / 2, existingUsed},
},
- length: page,
- alignment: page,
- direction: BottomUp,
- want: 2 * page,
+ length: 2*chunkSize + hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - hugepage,
+ },
+ {
+ name: "bottom-up small allocation finds first free gap",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingUsed},
+ {2 * page, 3 * page, existingUsed},
+ },
+ length: page,
+ want: page,
+ },
+ {
+ name: "top-down small allocation finds last free gap",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingUsed},
+ {chunkSize - 3*page, chunkSize - 2*page, existingUsed},
+ },
+ length: page,
+ dir: TopDown,
+ want: chunkSize - 2*page,
+ },
+ {
+ name: "bottom-up huge allocation finds first free gap",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingUsed},
+ {2 * hugepage, 3 * hugepage, existingUsed},
+ },
+ length: hugepage,
+ huge: true,
+ want: hugepage,
+ },
+ {
+ name: "top-down huge allocation finds last free gap",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingUsed},
+ {chunkSize - 3*hugepage, chunkSize - 2*hugepage, existingUsed},
+ },
+ length: hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - 2*hugepage,
+ },
+ {
+ name: "bottom-up small allocation skips undersized free gap",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingUsed},
+ {2 * page, 3 * page, existingUsed},
+ },
+ length: 2 * page,
+ want: 3 * page,
+ },
+ {
+ name: "top-down small allocation skips undersized free gap",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingUsed},
+ {chunkSize - 3*page, chunkSize - 2*page, existingUsed},
+ },
+ length: 2 * page,
+ dir: TopDown,
+ want: chunkSize - 5*page,
+ },
+ {
+ name: "bottom-up huge allocation skips undersized free gap",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingUsed},
+ {2 * hugepage, 3 * hugepage, existingUsed},
+ },
+ length: 2 * hugepage,
+ huge: true,
+ want: 3 * hugepage,
+ },
+ {
+ name: "top-down huge allocation skips undersized free gap",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingUsed},
+ {chunkSize - 3*hugepage, chunkSize - 2*hugepage, existingUsed},
+ },
+ length: 2 * hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - 5*hugepage,
+ },
+ {
+ name: "recycling bottom-up small allocation skips used pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingUsed},
+ },
+ length: page,
+ recycle: true,
+ want: page,
+ },
+ {
+ name: "recycling top-down small allocation skips used pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingUsed},
+ },
+ length: page,
+ recycle: true,
+ dir: TopDown,
+ want: chunkSize - 2*page,
+ },
+ {
+ name: "recycling bottom-up huge allocation skips used pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingUsed},
+ },
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ want: hugepage,
+ },
+ {
+ name: "recycling top-down huge allocation skips used pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingUsed},
+ },
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ dir: TopDown,
+ want: chunkSize - 2*hugepage,
+ },
+ {
+ name: "non-recycling bottom-up small allocation skips waste pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingWaste},
+ },
+ length: page,
+ want: page,
+ },
+ {
+ name: "non-recycling top-down small allocation skips waste pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingWaste},
+ },
+ length: page,
+ dir: TopDown,
+ want: chunkSize - 2*page,
+ },
+ {
+ name: "non-recycling bottom-up huge allocation skips waste pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingWaste},
+ },
+ length: hugepage,
+ huge: true,
+ want: hugepage,
+ },
+ {
+ name: "non-recycling top-down huge allocation skips waste pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingWaste},
+ },
+ length: hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - 2*hugepage,
+ },
+ {
+ name: "recycling bottom-up small allocation recycles waste pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingWaste},
+ },
+ length: page,
+ recycle: true,
+ want: 0,
+ },
+ {
+ name: "recycling top-down small allocation recycles waste pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingWaste},
+ },
+ length: page,
+ recycle: true,
+ dir: TopDown,
+ want: chunkSize - page,
+ },
+ {
+ name: "recycling bottom-up huge allocation recycles waste pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingWaste},
+ },
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ want: 0,
+ },
+ {
+ name: "recycling top-down huge allocation recycles waste pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingWaste},
+ },
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ dir: TopDown,
+ want: chunkSize - hugepage,
+ },
+ {
+ name: "non-recycling bottom-up small allocation skips releasing pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingReleasing},
+ },
+ length: page,
+ want: page,
+ },
+ {
+ name: "non-recycling top-down small allocation skips releasing pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingReleasing},
+ },
+ length: page,
+ dir: TopDown,
+ want: chunkSize - 2*page,
+ },
+ {
+ name: "non-recycling bottom-up huge allocation skips releasing pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingReleasing},
+ },
+ length: hugepage,
+ huge: true,
+ want: hugepage,
+ },
+ {
+ name: "non-recycling top-down huge allocation skips releasing pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingReleasing},
+ },
+ length: hugepage,
+ huge: true,
+ dir: TopDown,
+ want: chunkSize - 2*hugepage,
+ },
+ {
+ name: "recycling bottom-up small allocation skips releasing pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {0, page, existingReleasing},
+ },
+ length: page,
+ recycle: true,
+ want: page,
+ },
+ {
+ name: "recycling top-down small allocation skips releasing pages",
+ chunkHuge: []bool{false},
+ existing: []existingSegment{
+ {chunkSize - page, chunkSize, existingReleasing},
+ },
+ length: page,
+ recycle: true,
+ dir: TopDown,
+ want: chunkSize - 2*page,
+ },
+ {
+ name: "recycling bottom-up huge allocation skips releasing pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {0, hugepage, existingReleasing},
+ },
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ want: hugepage,
+ },
+ {
+ name: "recycling top-down huge allocation skips releasing pages",
+ chunkHuge: []bool{true},
+ existing: []existingSegment{
+ {chunkSize - hugepage, chunkSize, existingReleasing},
+ },
+ length: hugepage,
+ huge: true,
+ recycle: true,
+ dir: TopDown,
+ want: chunkSize - 2*hugepage,
},
} {
- name := fmt.Sprintf("%s (%v)", test.name, test.direction)
- t.Run(name, func(t *testing.T) {
- f := MemoryFile{fileSize: test.fileSize}
- if err := f.usage.ImportSlice(test.usage); err != nil {
- t.Fatalf("Failed to initialize usage from %v: %v", test.usage, err)
+ t.Run(test.name, func(t *testing.T) {
+ // Build the fake MemoryFile.
+ f := &MemoryFile{
+ opts: MemoryFileOpts{
+ ExpectHugepages: true,
+ DisableMemoryAccounting: true,
+ },
}
- if fr, ok := f.findAvailableRange(test.length, test.alignment, test.direction); ok {
- if test.expectFail {
- t.Fatalf("findAvailableRange(%v, %x, %x, %x, %v): got: %x, want: fail", test.usage, test.fileSize, test.length, test.alignment, test.direction, fr.Start)
+ f.initFields()
+ chunks := make([]chunkInfo, len(test.chunkHuge))
+ for i, huge := range test.chunkHuge {
+ chunks[i].huge = huge
+ chunkFR := memmap.FileRange{uint64(i) * chunkSize, uint64(i+1) * chunkSize}
+ if huge {
+ f.unfreeHuge.RemoveRange(chunkFR)
+ } else {
+ f.unfreeSmall.RemoveRange(chunkFR)
}
- if fr.Start != test.want {
- t.Errorf("findAvailableRange(%v, %x, %x, %x, %v): got: start=%x, want: %x", test.usage, test.fileSize, test.length, test.alignment, test.direction, fr.Start, test.want)
- }
- if fr.End != test.want+test.length {
- t.Errorf("findAvailableRange(%v, %x, %x, %x, %v): got: end=%x, want: %x", test.usage, test.fileSize, test.length, test.alignment, test.direction, fr.End, test.want+test.length)
- }
- } else if !test.expectFail {
- t.Fatalf("findAvailableRange(%v, %x, %x, %x, %v): failed, want: %x", test.usage, test.fileSize, test.length, test.alignment, test.direction, test.want)
+ }
+ f.chunks.Store(&chunks)
+ for _, es := range test.existing {
+ f.forEachChunk(memmap.FileRange{es.start, es.end}, func(chunk *chunkInfo, chunkFR memmap.FileRange) bool {
+ unwaste, unfree := &f.unwasteSmall, &f.unfreeSmall
+ if chunk.huge {
+ unwaste, unfree = &f.unwasteHuge, &f.unfreeHuge
+ }
+ switch es.state {
+ case existingUsed:
+ unfree.InsertRange(chunkFR, unfreeInfo{refs: 1})
+ case existingWaste:
+ unfree.InsertRange(chunkFR, unfreeInfo{refs: 0})
+ unwaste.RemoveRange(chunkFR)
+ case existingReleasing:
+ unfree.InsertRange(chunkFR, unfreeInfo{refs: 0})
+ default:
+ t.Fatalf("existingSegment %+v has unknown state", es)
+ }
+ f.memAcct.InsertRange(chunkFR, memAcctInfo{
+ wasteOrReleasing: es.state != existingUsed,
+ })
+ return true
+ })
+ }
+
+ // Perform the test allocation.
+ alloc := allocState{
+ length: test.length,
+ opts: AllocOpts{
+ Huge: test.huge,
+ Dir: test.dir,
+ },
+ huge: test.huge,
+ }
+ if test.recycle {
+ alloc.opts.Mode = AllocateCallerIndirectCommit
+ alloc.willCommit = true
+ }
+ fr, err := f.findAllocatableAndMarkUsed(&alloc)
+ if err != nil {
+ t.Fatalf("findAllocatableAndMarkUsed(%+v): failed: %v, want: %#x\n%v", alloc, err, test.want, f)
+ }
+ if fr.Start != test.want {
+ t.Errorf("findAllocatableAndMarkUsed(%+v): got: start=%#x, want: %#x\n%v", alloc, fr.Start, test.want, f)
+ }
+ if wantEnd := test.want + test.length; fr.End != wantEnd {
+ t.Errorf("findAllocatableAndMarkUsed(%+v): got: end=%#x, want: %#x\n%v", alloc, fr.End, wantEnd, f)
}
})
}
diff --git a/pkg/sentry/pgalloc/pgalloc_unsafe.go b/pkg/sentry/pgalloc/pgalloc_unsafe.go
index 59d4fdef3..73a1cf140 100644
--- a/pkg/sentry/pgalloc/pgalloc_unsafe.go
+++ b/pkg/sentry/pgalloc/pgalloc_unsafe.go
@@ -15,18 +15,15 @@
package pgalloc
import (
- "reflect"
"unsafe"
"golang.org/x/sys/unix"
+ "gvisor.dev/gvisor/pkg/sentry/memmap"
)
-func unsafeSlice(addr uintptr, length int) (slice []byte) {
- sh := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- sh.Data = addr
- sh.Len = length
- sh.Cap = length
- return
+// Preconditions: The FileRange represented by c is a superset of fr.
+func (c *chunkInfo) sliceAt(fr memmap.FileRange) []byte {
+ return unsafe.Slice((*byte)(unsafe.Pointer(c.mapping+uintptr(fr.Start&chunkMask))), fr.Length())
}
func mincore(s []byte, buf []byte, off uint64, wasCommitted bool) error {
diff --git a/pkg/sentry/pgalloc/save_restore.go b/pkg/sentry/pgalloc/save_restore.go
index 3051dbcfd..60be2b231 100644
--- a/pkg/sentry/pgalloc/save_restore.go
+++ b/pkg/sentry/pgalloc/save_restore.go
@@ -21,6 +21,7 @@ import (
"io"
"runtime"
+ "golang.org/x/sys/unix"
"gvisor.dev/gvisor/pkg/atomicbitops"
"gvisor.dev/gvisor/pkg/hostarch"
"gvisor.dev/gvisor/pkg/log"
@@ -48,11 +49,10 @@ type SaveOpts struct {
// SaveTo writes f's state to the given stream.
func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer, opts SaveOpts) error {
- // Wait for reclaim.
+ // Wait for memory release.
f.mu.Lock()
defer f.mu.Unlock()
- for f.reclaimable {
- f.reclaimCond.Signal()
+ for f.haveWaste {
f.mu.Unlock()
runtime.Gosched()
f.mu.Lock()
@@ -63,8 +63,8 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer, opts
panic(fmt.Sprintf("evictions still pending for %d users; call StartEvictions and WaitForEvictions before SaveTo", len(f.evictable)))
}
- // Ensure that all pages that contain non-zero bytes have knownCommitted
- // set, since we only store knownCommitted pages below.
+ // Ensure that all pages that contain non-zero bytes are marked
+ // known-committed, since we only store known-committed pages below.
zeroPage := make([]byte, hostarch.PageSize)
var (
decommitWarnOnce sync.Once
@@ -110,7 +110,7 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer, opts
decommitPendingFR = memmap.FileRange{}
}
}
- err := f.updateUsageLocked(0, nil, opts.ExcludeCommittedZeroPages, func(bs []byte, committed []byte, off uint64, wasCommitted bool) error {
+ err := f.updateUsageLocked(nil, opts.ExcludeCommittedZeroPages, func(bs []byte, committed []byte, off uint64, wasCommitted bool) error {
scanTotal += uint64(len(bs))
for pgoff := 0; pgoff < len(bs); pgoff += hostarch.PageSize {
i := pgoff / hostarch.PageSize
@@ -138,25 +138,46 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer, opts
log.Debugf("MemoryFile.SaveTo: scanned %d bytes, decommitted %d bytes in %d syscalls", scanTotal, decommitTotal, decommitCount)
// Save metadata.
- if _, err := state.Save(ctx, w, &f.fileSize); err != nil {
+ if _, err := state.Save(ctx, w, &f.unwasteSmall); err != nil {
return err
}
- if _, err := state.Save(ctx, w, &f.usage); err != nil {
+ if _, err := state.Save(ctx, w, &f.unwasteHuge); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, &f.unfreeSmall); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, &f.unfreeHuge); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, &f.subreleased); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, &f.memAcct); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, &f.knownCommittedBytes); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, &f.commitSeq); err != nil {
+ return err
+ }
+ if _, err := state.Save(ctx, w, f.chunks.Load()); err != nil {
return err
}
// Dump out committed pages.
- for seg := f.usage.FirstSegment(); seg.Ok(); seg = seg.NextSegment() {
- if !seg.Value().knownCommitted {
+ for maseg := f.memAcct.FirstSegment(); maseg.Ok(); maseg = maseg.NextSegment() {
+ if !maseg.ValuePtr().knownCommitted {
continue
}
// Write a header to distinguish from objects.
- if err := state.WriteHeader(w, uint64(seg.Range().Length()), false); err != nil {
+ if err := state.WriteHeader(w, uint64(maseg.Range().Length()), false); err != nil {
return err
}
// Write out data.
var ioErr error
- err := f.forEachMappingSlice(seg.Range(), func(s []byte) {
+ f.forEachMappingSlice(maseg.Range(), func(s []byte) {
if ioErr != nil {
return
}
@@ -165,9 +186,6 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer, opts
if ioErr != nil {
return ioErr
}
- if err != nil {
- return err
- }
}
return nil
@@ -194,45 +212,88 @@ func (f *MemoryFile) RestoreID() string {
// LoadFrom loads MemoryFile state from the given stream.
func (f *MemoryFile) LoadFrom(ctx context.Context, r io.Reader, pr *statefile.AsyncReader) error {
- // Load metadata.
- if _, err := state.Load(ctx, r, &f.fileSize); err != nil {
- return err
- }
- if err := f.file.Truncate(f.fileSize); err != nil {
- return err
- }
- newMappings := make([]uintptr, f.fileSize>>chunkShift)
- f.mappings.Store(&newMappings)
- if _, err := state.Load(ctx, r, &f.usage); err != nil {
- return err
- }
+ // Clear sets since non-empty sets will panic if loaded into.
+ f.unwasteSmall.RemoveAll()
+ f.unwasteHuge.RemoveAll()
+ f.unfreeSmall.RemoveAll()
+ f.unfreeHuge.RemoveAll()
+ f.memAcct.RemoveAll()
- // Try to map committed chunks concurrently: For any given chunk, either
- // this loop or the following one will mmap the chunk first and cache it in
- // f.mappings for the other, but this loop is likely to run ahead of the
- // other since it doesn't do any work between mmaps. The rest of this
- // function doesn't mutate f.usage, so it's safe to iterate concurrently.
- mapperDone := make(chan struct{})
- mapperCanceled := atomicbitops.FromInt32(0)
- go func() { // S/R-SAFE: see comment
- defer func() { close(mapperDone) }()
- for seg := f.usage.FirstSegment(); seg.Ok(); seg = seg.NextSegment() {
- if mapperCanceled.Load() != 0 {
- return
- }
- if seg.Value().knownCommitted {
- f.forEachMappingSlice(seg.Range(), func(s []byte) {})
- }
+ // Load metadata.
+ if _, err := state.Load(ctx, r, &f.unwasteSmall); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.unwasteHuge); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.unfreeSmall); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.unfreeHuge); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.subreleased); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.memAcct); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.knownCommittedBytes); err != nil {
+ return err
+ }
+ if _, err := state.Load(ctx, r, &f.commitSeq); err != nil {
+ return err
+ }
+ var chunks []chunkInfo
+ if _, err := state.Load(ctx, r, &chunks); err != nil {
+ return err
+ }
+ f.chunks.Store(&chunks)
+ if err := f.file.Truncate(int64(len(chunks)) * chunkSize); err != nil {
+ return err
+ }
+ // Obtain chunk mappings, then madvise them concurrently with loading data.
+ var (
+ madviseEnd atomicbitops.Uint64
+ madviseChan = make(chan struct{}, 1)
+ madviseWG sync.WaitGroup
+ )
+ if len(chunks) != 0 {
+ m, _, errno := unix.Syscall6(
+ unix.SYS_MMAP,
+ 0,
+ uintptr(len(chunks)*chunkSize),
+ unix.PROT_READ|unix.PROT_WRITE,
+ unix.MAP_SHARED,
+ f.file.Fd(),
+ 0)
+ if errno != 0 {
+ return fmt.Errorf("failed to mmap MemoryFile: %w", errno)
}
- }()
- defer func() {
- mapperCanceled.Store(1)
- <-mapperDone
- }()
+ for i := range chunks {
+ chunk := &chunks[i]
+ chunk.mapping = m
+ m += chunkSize
+ }
+ madviseWG.Add(1)
+ go func() {
+ defer madviseWG.Done()
+ for i := range chunks {
+ chunk := &chunks[i]
+ f.madviseChunkMapping(chunk.mapping, chunkSize, chunk.huge)
+ madviseEnd.Add(chunkSize)
+ select {
+ case madviseChan <- struct{}{}:
+ default:
+ }
+ }
+ }()
+ }
+ defer madviseWG.Wait()
// Load committed pages.
- for seg := f.usage.FirstSegment(); seg.Ok(); seg = seg.NextSegment() {
- if !seg.Value().knownCommitted {
+ for maseg := f.memAcct.FirstSegment(); maseg.Ok(); maseg = maseg.NextSegment() {
+ if !maseg.ValuePtr().knownCommitted {
continue
}
// Verify header.
@@ -244,13 +305,17 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r io.Reader, pr *statefile.As
// Not expected.
return fmt.Errorf("unexpected object")
}
- if expected := uint64(seg.Range().Length()); length != expected {
+ if expected := uint64(maseg.Range().Length()); length != expected {
// Size mismatch.
return fmt.Errorf("mismatched segment: expected %d, got %d", expected, length)
}
+ // Wait for all chunks spanned by this segment to be madvised.
+ for madviseEnd.Load() < maseg.End() {
+ <-madviseChan
+ }
// Read data.
var ioErr error
- err = f.forEachMappingSlice(seg.Range(), func(s []byte) {
+ f.forEachMappingSlice(maseg.Range(), func(s []byte) {
if ioErr != nil {
return
}
@@ -263,16 +328,14 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r io.Reader, pr *statefile.As
if ioErr != nil {
return ioErr
}
- if err != nil {
- return err
- }
// Update accounting for restored pages. We need to do this here since
// these segments are marked as "known committed", and will be skipped
// over on accounting scans.
- amount := seg.Range().Length()
- usage.MemoryAccounting.Inc(amount, seg.Value().kind, seg.Value().memCgID)
- f.usageExpected += amount
+ if !f.opts.DisableMemoryAccounting {
+ amount := maseg.Range().Length()
+ usage.MemoryAccounting.Inc(amount, maseg.ValuePtr().kind, maseg.ValuePtr().memCgID)
+ }
}
return nil
diff --git a/pkg/sentry/syscalls/linux/sys_mmap.go b/pkg/sentry/syscalls/linux/sys_mmap.go
index 2da71232f..de0879d9f 100644
--- a/pkg/sentry/syscalls/linux/sys_mmap.go
+++ b/pkg/sentry/syscalls/linux/sys_mmap.go
@@ -67,6 +67,7 @@ func Mmap(t *kernel.Task, sysno uintptr, args arch.SyscallArguments) (uintptr, *
},
MaxPerms: hostarch.AnyAccess,
GrowsDown: linux.MAP_GROWSDOWN&flags != 0,
+ Stack: linux.MAP_STACK&flags != 0,
}
if linux.MAP_POPULATE&flags != 0 {
opts.PlatformEffect = memmap.PlatformEffectCommit
diff --git a/pkg/sentry/usage/memory.go b/pkg/sentry/usage/memory.go
index d03274057..4793df7b2 100644
--- a/pkg/sentry/usage/memory.go
+++ b/pkg/sentry/usage/memory.go
@@ -22,6 +22,7 @@ import (
"gvisor.dev/gvisor/pkg/atomicbitops"
"gvisor.dev/gvisor/pkg/bits"
"gvisor.dev/gvisor/pkg/memutil"
+ "gvisor.dev/gvisor/pkg/sync"
)
// MemoryKind represents a type of memory used by the application.
@@ -199,32 +200,42 @@ type MemoryLocked struct {
MemCgIDToMemStats map[uint32]*memoryStats
}
+var (
+ initOnce sync.Once
+ initErr error
+)
+
// Init initializes global 'MemoryAccounting'.
func Init() error {
- const name = "memory-usage"
- fd, err := memutil.CreateMemFD(name, 0)
- if err != nil {
- return fmt.Errorf("error creating usage file: %v", err)
- }
- file := os.NewFile(uintptr(fd), name)
- if err := file.Truncate(int64(RTMemoryStatsSize)); err != nil {
- return fmt.Errorf("error truncating usage file: %v", err)
- }
- // Note: We rely on the returned page being initially zeroed. This will
- // always be the case for a newly mapped page from /dev/shm. If we obtain
- // the shared memory through some other means in the future, we may have to
- // explicitly zero the page.
- mmap, err := memutil.MapFile(0, RTMemoryStatsSize, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED, file.Fd(), 0)
- if err != nil {
- return fmt.Errorf("error mapping usage file: %v", err)
- }
+ initOnce.Do(func() {
+ initErr = func() error {
+ const name = "memory-usage"
+ fd, err := memutil.CreateMemFD(name, 0)
+ if err != nil {
+ return fmt.Errorf("error creating usage file: %v", err)
+ }
+ file := os.NewFile(uintptr(fd), name)
+ if err := file.Truncate(int64(RTMemoryStatsSize)); err != nil {
+ return fmt.Errorf("error truncating usage file: %v", err)
+ }
+ // Note: We rely on the returned page being initially zeroed. This will
+ // always be the case for a newly mapped page from /dev/shm. If we obtain
+ // the shared memory through some other means in the future, we may have to
+ // explicitly zero the page.
+ mmap, err := memutil.MapFile(0, RTMemoryStatsSize, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED, file.Fd(), 0)
+ if err != nil {
+ return fmt.Errorf("error mapping usage file: %v", err)
+ }
- MemoryAccounting = &MemoryLocked{
- File: file,
- RTMemoryStats: RTMemoryStatsPointer(mmap),
- MemCgIDToMemStats: make(map[uint32]*memoryStats),
- }
- return nil
+ MemoryAccounting = &MemoryLocked{
+ File: file,
+ RTMemoryStats: RTMemoryStatsPointer(mmap),
+ MemCgIDToMemStats: make(map[uint32]*memoryStats),
+ }
+ return nil
+ }()
+ })
+ return initErr
}
// MemoryAccounting is the global memory stats.
diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go
index 26e9b1f9a..fe0760b33 100644
--- a/runsc/boot/loader.go
+++ b/runsc/boot/loader.go
@@ -204,6 +204,10 @@ type Loader struct {
// /sys/devices/virtual/dmi/id/product_name.
productName string
+ // hostShmemHuge is the host's value of
+ // /sys/kernel/mm/transparent_hugepage/shmem_enabled.
+ hostShmemHuge string
+
// mu guards the fields below.
mu sync.Mutex
@@ -340,6 +344,10 @@ type Args struct {
// NvidiaDriverVersion is the NVIDIA driver ABI version to use for
// communicating with NVIDIA devices on the host.
NvidiaDriverVersion string
+ // HostShmemHuge is the host's value of
+ // /sys/kernel/mm/transparent_hugepage/shmem_enabled, or empty if this is
+ // unknown.
+ HostShmemHuge string
SaveFDs []*fd.FD
}
@@ -409,6 +417,7 @@ func New(args Args) (*Loader, error) {
sharedMounts: make(map[string]*vfs.Mount),
stopProfiling: stopProfiling,
productName: args.ProductName,
+ hostShmemHuge: args.HostShmemHuge,
containerIDs: map[string]string{},
saveFDs: args.SaveFDs,
}
@@ -476,7 +485,7 @@ func New(args Args) (*Loader, error) {
l.k = &kernel.Kernel{Platform: p}
// Create memory file.
- mf, err := createMemoryFile()
+ mf, err := createMemoryFile(args.Conf.AppHugePages, args.HostShmemHuge)
if err != nil {
return nil, fmt.Errorf("creating memory file: %w", err)
}
@@ -738,17 +747,45 @@ func createPlatform(conf *config.Config, deviceFile *fd.FD) (platform.Platform,
return p.New(deviceFile)
}
-func createMemoryFile() (*pgalloc.MemoryFile, error) {
+func createMemoryFile(appHugePages bool, hostShmemHuge string) (*pgalloc.MemoryFile, error) {
const memfileName = "runsc-memory"
memfd, err := memutil.CreateMemFD(memfileName, 0)
if err != nil {
return nil, fmt.Errorf("error creating memfd: %w", err)
}
memfile := os.NewFile(uintptr(memfd), memfileName)
- // We can't enable pgalloc.MemoryFileOpts.UseHostMemcgPressure even if
- // there are memory cgroups specified, because at this point we're already
- // in a mount namespace in which the relevant cgroupfs is not visible.
- mf, err := pgalloc.NewMemoryFile(memfile, pgalloc.MemoryFileOpts{})
+
+ mfopts := pgalloc.MemoryFileOpts{
+ // We can't enable pgalloc.MemoryFileOpts.UseHostMemcgPressure even if
+ // there are memory cgroups specified, because at this point we're already
+ // in a mount namespace in which the relevant cgroupfs is not visible.
+ }
+ if appHugePages {
+ switch hostShmemHuge {
+ case "":
+ log.Infof("Disabling application huge pages: host shmem_huge is unknown")
+ case "never", "deny":
+ log.Infof("Disabling application huge pages: host shmem_huge is %q", hostShmemHuge)
+ case "advise":
+ log.Infof("Enabling application huge pages: host shmem_huge is %q", hostShmemHuge)
+ mfopts.ExpectHugepages = true
+ mfopts.AdviseHugepage = true
+ case "always", "within_size":
+ log.Infof("Enabling application huge pages: host shmem_huge is %q", hostShmemHuge)
+ // In these cases, memfds will default to using huge pages, and we have to
+ // explicitly ask for small pages.
+ mfopts.ExpectHugepages = true
+ mfopts.AdviseNoHugepage = true
+ case "force":
+ log.Infof("Enabling application huge pages: host shmem_huge is %q", hostShmemHuge)
+ // The kernel will ignore MADV_NOHUGEPAGE, so don't bother.
+ mfopts.ExpectHugepages = true
+ default:
+ log.Infof("Disabling application huge pages: host shmem_huge is unknown value %q", hostShmemHuge)
+ }
+ }
+
+ mf, err := pgalloc.NewMemoryFile(memfile, mfopts)
if err != nil {
_ = memfile.Close()
return nil, fmt.Errorf("error creating pgalloc.MemoryFile: %w", err)
diff --git a/runsc/boot/restore.go b/runsc/boot/restore.go
index 88737d2ff..c7031d63f 100644
--- a/runsc/boot/restore.go
+++ b/runsc/boot/restore.go
@@ -165,7 +165,7 @@ func (r *restorer) restore(l *Loader) error {
Platform: p,
}
- mf, err := createMemoryFile()
+ mf, err := createMemoryFile(l.root.conf.AppHugePages, l.hostShmemHuge)
if err != nil {
return fmt.Errorf("creating memory file: %v", err)
}
diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD
index 32354e1d6..f2bcb84df 100644
--- a/runsc/cmd/BUILD
+++ b/runsc/cmd/BUILD
@@ -90,6 +90,7 @@ go_library(
"//pkg/ring0",
"//pkg/sentry/control",
"//pkg/sentry/devices/tpuproxy",
+ "//pkg/sentry/hostmm",
"//pkg/sentry/kernel",
"//pkg/sentry/kernel/auth",
"//pkg/sentry/pgalloc",
diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go
index d0a52c317..2d4716c7c 100644
--- a/runsc/cmd/boot.go
+++ b/runsc/cmd/boot.go
@@ -36,6 +36,7 @@ import (
"gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/metric"
"gvisor.dev/gvisor/pkg/ring0"
+ "gvisor.dev/gvisor/pkg/sentry/hostmm"
"gvisor.dev/gvisor/pkg/sentry/platform"
"gvisor.dev/gvisor/runsc/boot"
"gvisor.dev/gvisor/runsc/cmd/util"
@@ -152,6 +153,9 @@ type Boot struct {
// /sys/devices/virtual/dmi/id/product_name.
productName string
+ // Value of /sys/kernel/mm/transparent_hugepage/shmem_enabled on the host.
+ hostShmemHuge string
+
// FDs for profile data.
profileFDs profile.FDArgs
@@ -204,6 +208,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
f.BoolVar(&b.attached, "attached", false, "if attached is true, kills the sandbox process when the parent process terminates")
f.StringVar(&b.productName, "product-name", "", "value to show in /sys/devices/virtual/dmi/id/product_name")
f.StringVar(&b.nvidiaDriverVersion, "nvidia-driver-version", "", "Nvidia driver version on the host")
+ f.StringVar(&b.hostShmemHuge, "host-shmem-huge", "", "value of /sys/kernel/mm/transparent_hugepage/shmem_enabled on the host")
// Open FDs that are donated to the sandbox.
f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec")
@@ -249,8 +254,9 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
ring0.InitDefault()
argOverride := make(map[string]string)
+
+ // Do these before chroot takes effect, otherwise we can't read /sys.
if len(b.productName) == 0 {
- // Do this before chroot takes effect, otherwise we can't read /sys.
if product, err := ioutil.ReadFile("/sys/devices/virtual/dmi/id/product_name"); err != nil {
log.Warningf("Not setting product_name: %v", err)
} else {
@@ -259,6 +265,16 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
argOverride["product-name"] = b.productName
}
}
+ if conf.AppHugePages && len(b.hostShmemHuge) == 0 {
+ hostShmemHuge, err := hostmm.GetTransparentHugepageEnum("shmem_enabled")
+ if err != nil {
+ log.Warningf("Failed to infer --host-shmem-huge: %v", err)
+ } else {
+ b.hostShmemHuge = hostShmemHuge
+ log.Infof("Setting host-shmem-huge: %q", b.hostShmemHuge)
+ argOverride["host-shmem-huge"] = b.hostShmemHuge
+ }
+ }
if b.attached {
// Ensure this process is killed after parent process terminates when
@@ -456,6 +472,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
SinkFDs: b.sinkFDs.GetArray(),
ProfileOpts: b.profileFDs.ToOpts(),
NvidiaDriverVersion: b.nvidiaDriverVersion,
+ HostShmemHuge: b.hostShmemHuge,
SaveFDs: b.saveFDs.GetFDs(),
}
l, err := boot.New(bootArgs)
diff --git a/runsc/config/config.go b/runsc/config/config.go
index 8b5751696..0b5bbe5ad 100644
--- a/runsc/config/config.go
+++ b/runsc/config/config.go
@@ -308,6 +308,9 @@ type Config struct {
// exists, but is mostly idle. Not supported in rootless mode.
DirectFS bool `flag:"directfs"`
+ // AppHugePages enables support for application huge pages.
+ AppHugePages bool `flag:"app-huge-pages"`
+
// NVProxy enables support for Nvidia GPUs.
NVProxy bool `flag:"nvproxy"`
diff --git a/runsc/config/flags.go b/runsc/config/flags.go
index bc4c5c850..2fd7a8fdc 100644
--- a/runsc/config/flags.go
+++ b/runsc/config/flags.go
@@ -90,6 +90,9 @@ func RegisterFlags(flagSet *flag.FlagSet) {
flagSet.Bool("enable-core-tags", false, "enables core tagging. Requires host linux kernel >= 5.14.")
flagSet.String("pod-init-config", "", "path to configuration file with additional steps to take during pod creation.")
+ // Flags that control sandbox runtime behavior: MM related.
+ flagSet.Bool("app-huge-pages", true, "enable use of huge pages for application memory; requires /sys/kernel/mm/transparent_hugepage/shmem_enabled = advise")
+
// Flags that control sandbox runtime behavior: FS related.
flagSet.Var(fileAccessTypePtr(FileAccessExclusive), "file-access", "specifies which filesystem validation to use for the root mount: exclusive (default), shared.")
flagSet.Var(fileAccessTypePtr(FileAccessShared), "file-access-mounts", "specifies which filesystem validation to use for volumes other than the root mount: shared (default), exclusive.")
diff --git a/test/syscalls/linux/madvise.cc b/test/syscalls/linux/madvise.cc
index 6e714b12c..c2ce706b1 100644
--- a/test/syscalls/linux/madvise.cc
+++ b/test/syscalls/linux/madvise.cc
@@ -40,7 +40,7 @@ namespace {
void ExpectAllMappingBytes(Mapping const& m, char c) {
auto const v = m.view();
- for (size_t i = 0; i < kPageSize; i++) {
+ for (size_t i = 0; i < v.size(); i++) {
ASSERT_EQ(v[i], c) << "at offset " << i;
}
}
@@ -49,7 +49,7 @@ void ExpectAllMappingBytes(Mapping const& m, char c) {
// helpful failure messages.
void CheckAllMappingBytes(Mapping const& m, char c) {
auto const v = m.view();
- for (size_t i = 0; i < kPageSize; i++) {
+ for (size_t i = 0; i < v.size(); i++) {
TEST_CHECK_MSG(v[i] == c, "mapping contains wrong value");
}
}
@@ -136,6 +136,96 @@ TEST(MadviseDontneedTest, IgnoresPermissions) {
EXPECT_THAT(madvise(m.ptr(), m.len(), MADV_DONTNEED), SyscallSucceeds());
}
+class MadviseDontneedHugePageSubrangeTest : public ::testing::Test {
+ protected:
+ static constexpr char kDataValue = 9;
+
+ void TearDown() override {
+ if (data_start_) {
+ free(data_start_);
+ }
+ }
+
+ PosixError GetAndFillHugePages(size_t length) {
+ void* memptr;
+ if (int ret = posix_memalign(&memptr, kHugePageSize, length); ret != 0) {
+ return PosixError(ret, "posix_memalign failed");
+ }
+ data_start_ = static_cast(memptr);
+ data_len_ = length;
+ memset(data_start_, kDataValue, data_len_);
+ return PosixError();
+ }
+
+ void TestMadvDontneedSubrange(size_t start_off, size_t end_off) {
+ TEST_CHECK(start_off <= data_len_);
+ TEST_CHECK(end_off <= data_len_);
+ TEST_CHECK(start_off <= end_off);
+ char* const inner_start = data_start_ + start_off;
+ size_t const inner_len = end_off - start_off;
+ ASSERT_THAT(madvise(inner_start, inner_len, MADV_DONTNEED),
+ SyscallSucceeds());
+ auto v = absl::string_view(data_start_, start_off);
+ for (size_t i = 0; i < v.size(); i++) {
+ ASSERT_EQ(v[i], kDataValue)
+ << "at offset " << i << " of range before MADV_DONTNEED";
+ }
+ v = absl::string_view(inner_start, inner_len);
+ for (size_t i = 0; i < v.size(); i++) {
+ ASSERT_EQ(v[i], 0) << "at offset " << i
+ << " of range under MADV_DONTNEED";
+ }
+ v = absl::string_view(data_start_ + end_off, data_len_ - end_off);
+ for (size_t i = 0; i < v.size(); i++) {
+ ASSERT_EQ(v[i], kDataValue)
+ << "at offset " << i << " of range after MADV_DONTNEED";
+ }
+ }
+
+ char* data_start_ = nullptr;
+ size_t data_len_ = 0;
+};
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, OneHugePageWhole) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize));
+ TestMadvDontneedSubrange(0, kHugePageSize);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, OneHugePagePartial) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize));
+ TestMadvDontneedSubrange(kHugePageSize / 4, kHugePageSize * 3 / 4);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, TwoHugePagesPartialStart) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize * 2));
+ TestMadvDontneedSubrange(kHugePageSize / 2, kHugePageSize);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, TwoHugePagesPartialEnd) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize * 2));
+ TestMadvDontneedSubrange(kHugePageSize, kHugePageSize * 3 / 2);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, TwoHugePagesPartialStartAndEnd) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize * 2));
+ TestMadvDontneedSubrange(kHugePageSize / 2, kHugePageSize * 3 / 2);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, ThreeHugePagesPartialStart) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize * 3));
+ TestMadvDontneedSubrange(kHugePageSize / 2, kHugePageSize * 2);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, ThreeHugePagesPartialEnd) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize * 3));
+ TestMadvDontneedSubrange(kHugePageSize, kHugePageSize * 5 / 2);
+}
+
+TEST_F(MadviseDontneedHugePageSubrangeTest, ThreeHugePagesPartialStartAndEnd) {
+ ASSERT_NO_ERRNO(GetAndFillHugePages(kHugePageSize * 3));
+ TestMadvDontneedSubrange(kHugePageSize / 2, kHugePageSize * 5 / 2);
+}
+
TEST(MadviseDontforkTest, AddressLength) {
auto m =
ASSERT_NO_ERRNO_AND_VALUE(MmapAnon(kPageSize, PROT_NONE, MAP_PRIVATE));
diff --git a/test/util/test_util.h b/test/util/test_util.h
index 7421c0e0c..1ace57301 100644
--- a/test/util/test_util.h
+++ b/test/util/test_util.h
@@ -252,6 +252,10 @@ PosixErrorOr GetKernelVersion();
static const size_t kPageSize = sysconf(_SC_PAGESIZE);
+#if defined(__x86_64__) || defined(__aarch64__)
+inline constexpr size_t kHugePageSize = 1 << 21;
+#endif
+
enum class CPUVendor { kIntel, kAMD, kUnknownVendor };
CPUVendor GetCPUVendor();