pgalloc: add SaveOpts.ExcludeCommittedZeroPages

This option, enabled via `runsc checkpoint --exclude-committed-zero-pages`,
instructs `pgalloc.MemoryFile.SaveTo()` to also exclude definitely-committed
zero pages from checkpointing (in addition to possibly-committed zero pages,
which are always scanned for and excluded). This is useful when the application
being checkpointed is known to have a large number of committed zero pages:
pages that (1) have been touched by application memory accesses, a syscall such
as read(), or page pinning by e.g. a driver, and (2) have not been subsequently
released by the application to the operating system by e.g. munmap() or
madvise(MADV_DONTNEED) (+ page unpinning if necessary), and (3) are filled with
zero bytes.

Minor changes:

- In `MemoryFile.updateUsageLocked()`, pass file offset to `checkCommitted` so
  that `MemoryFile.SaveTo()`'s `checkCommitted` can use `FALLOC_FL_PUNCH_HOLE`
  to decommit pages rather than `MADV_REMOVE` (which translates addresses to
  file offsets and then invokes `FALLOC_FL_PUNCH_HOLE`).

- In `MemoryFile.SaveTo()`, buffer up to a hugepage worth of pages to decommit
  rather than decommitting one page per syscall.

- Increment `MemoryFile.usageExpected` in `MemoryFile.LoadFrom()`, such that
  the first following call to `MemoryFile.UpdateUsage()` might skip the call
  to `MemoryFile.updateUsageLocked()` (if memory usage hasn't changed since
  loading).

PiperOrigin-RevId: 632370455

pkg/sentry/control/BUILD

@@ -51,6 +51,7 @@ go_library(
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/kernel/time",
         "//pkg/sentry/limits",
+        "//pkg/sentry/pgalloc",
         "//pkg/sentry/state",
         "//pkg/sentry/strace",
         "//pkg/sentry/usage",

pkg/sentry/control/state.go

@@ -21,6 +21,7 @@ import (
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/state"
 	"gvisor.dev/gvisor/pkg/sentry/watchdog"
 	"gvisor.dev/gvisor/pkg/urpc"
@@ -44,6 +45,9 @@ type SaveOpts struct {
 	// Metadata is the set of metadata to prepend to the state file.
 	Metadata map[string]string `json:"metadata"`
 
+	// MemoryFileSaveOpts is passed to calls to pgalloc.MemoryFile.SaveTo().
+	MemoryFileSaveOpts pgalloc.SaveOpts
+
 	// HavePagesFile indicates whether the pages file and its corresponding
 	// metadata file is provided.
 	HavePagesFile bool `json:"have_pages_file"`
@@ -76,9 +80,10 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error {
 	}
 	defer stateFile.Close()
 	saveOpts := state.SaveOpts{
-		Destination: stateFile,
-		Key:         o.Key,
-		Metadata:    o.Metadata,
+		Destination:        stateFile,
+		Key:                o.Key,
+		Metadata:           o.Metadata,
+		MemoryFileSaveOpts: o.MemoryFileSaveOpts,
 		Callback: func(err error) {
 			if err == nil {
 				log.Infof("Save succeeded: exiting...")

pkg/sentry/kernel/kernel.go

@@ -534,7 +534,11 @@ type privateMemoryFileMetadata struct {
 	owners []string
 }
 
-func savePrivateMFs(ctx context.Context, w io.Writer, pw io.Writer, mfsToSave map[string]*pgalloc.MemoryFile) error {
+func savePrivateMFs(ctx context.Context, w io.Writer, pw io.Writer, mfsToSave map[string]*pgalloc.MemoryFile, mfOpts pgalloc.SaveOpts) error {
+	// mfOpts.ExcludeCommittedZeroPages is expected to reflect application
+	// memory usage behavior, but not necessarily usage of private MemoryFiles.
+	mfOpts.ExcludeCommittedZeroPages = false
+
 	var meta privateMemoryFileMetadata
 	// Generate the order in which private memory files are saved.
 	for fsID := range mfsToSave {
@@ -546,7 +550,7 @@ func savePrivateMFs(ctx context.Context, w io.Writer, pw io.Writer, mfsToSave ma
 	}
 	// Followed by the private memory files in order.
 	for _, fsID := range meta.owners {
-		if err := mfsToSave[fsID].SaveTo(ctx, w, pw); err != nil {
+		if err := mfsToSave[fsID].SaveTo(ctx, w, pw, mfOpts); err != nil {
 			return err
 		}
 	}
@@ -580,7 +584,7 @@ func loadPrivateMFs(ctx context.Context, r io.Reader, pr *statefile.AsyncReader)
 // SaveTo saves the state of k to w.
 //
 // Preconditions: The kernel must be paused throughout the call to SaveTo.
-func (k *Kernel) SaveTo(ctx context.Context, w io.Writer, pagesMetadata, pagesFile *fd.FD) error {
+func (k *Kernel) SaveTo(ctx context.Context, w io.Writer, pagesMetadata, pagesFile *fd.FD, mfOpts pgalloc.SaveOpts) error {
 	saveStart := time.Now()
 
 	// Do not allow other Kernel methods to affect it while it's being saved.
@@ -656,10 +660,10 @@ func (k *Kernel) SaveTo(ctx context.Context, w io.Writer, pagesMetadata, pagesFi
 	if pagesFile != nil {
 		pw = pagesFile
 	}
-	if err := k.mf.SaveTo(ctx, pmw, pw); err != nil {
+	if err := k.mf.SaveTo(ctx, pmw, pw, mfOpts); err != nil {
 		return err
 	}
-	if err := savePrivateMFs(ctx, pmw, pw, mfsToSave); err != nil {
+	if err := savePrivateMFs(ctx, pmw, pw, mfsToSave, mfOpts); err != nil {
 		return err
 	}
 	log.Infof("Memory files save took [%s].", time.Since(memoryStart))

pkg/sentry/pgalloc/pgalloc.go

@@ -272,18 +272,6 @@ type usageInfo struct {
 	memCgID uint32
 }
 
-// canCommit returns true if the tracked region can be committed.
-func (u *usageInfo) canCommit() bool {
-	// refs must be greater than 0 because we assume that reclaimable pages
-	// (that aren't already known to be committed) are not committed. This
-	// isn't necessarily true, even after the reclaimer does Decommit(),
-	// because the kernel may subsequently back the hugepage-sized region
-	// containing the decommitted page with a hugepage. However, it's
-	// consistent with our treatment of unallocated pages, which have the same
-	// property.
-	return !u.knownCommitted && u.refs != 0
-}
-
 // An EvictableMemoryUser represents a user of MemoryFile-allocated memory that
 // may be asked to deallocate that memory in the presence of memory pressure.
 type EvictableMemoryUser interface {
@@ -1141,20 +1129,29 @@ func (f *MemoryFile) UpdateUsage(memCgIDs map[uint32]struct{}) error {
 	if memCgIDs == nil {
 		f.usageLast = time.Now()
 	}
-	err = f.updateUsageLocked(currentUsage, memCgIDs, mincore)
+	err = f.updateUsageLocked(currentUsage, memCgIDs, false /* alsoScanCommitted */, mincore)
 	log.Debugf("UpdateUsage: currentUsage=%d, usageExpected=%d, usageSwapped=%d.",
 		currentUsage, f.usageExpected, f.usageSwapped)
 	log.Debugf("UpdateUsage: took %v.", time.Since(f.usageLast))
 	return err
 }
 
-// updateUsageLocked attempts to detect commitment of previous-uncommitted
-// pages by invoking checkCommitted, which is a function that, for each page i
-// in bs, sets committed[i] to 1 if the page is committed and 0 otherwise.
+// updateUsageLocked attempts to detect commitment of previously-uncommitted
+// pages by invoking checkCommitted, and updates memory accounting to reflect
+// newly-committed pages. If alsoScanCommitted is true, updateUsageLocked also
+// attempts to detect decommitment of previously-committed pages; this is only
+// used by save/restore, which optionally temporarily treats zeroed pages as
+// decommitted in order to skip saving them.
+//
+// For each page i in bs, checkCommitted must set committed[i] to 1 if the page
+// is committed and 0 otherwise. off is the offset at which bs begins.
+// wasCommitted is true if the page was known-committed before the call to
+// checkCommitted and false otherwise; wasCommitted can only be true if
+// alsoScanCommitted is true.
 //
 // Precondition: f.mu must be held; it may be unlocked and reacquired.
 // +checklocks:f.mu
-func (f *MemoryFile) updateUsageLocked(currentUsage uint64, memCgIDs map[uint32]struct{}, checkCommitted func(bs []byte, committed []byte) error) error {
+func (f *MemoryFile) updateUsageLocked(currentUsage uint64, memCgIDs map[uint32]struct{}, alsoScanCommitted bool, checkCommitted func(bs []byte, committed []byte, off uint64, wasCommitted bool) error) error {
 	// Track if anything changed to elide the merge. In the common case, we
 	// expect all segments to be committed and no merge to occur.
 	changedAny := false
@@ -1193,7 +1190,19 @@ func (f *MemoryFile) updateUsageLocked(currentUsage uint64, memCgIDs map[uint32]
 	// Iterate over all usage data. There will only be usage segments
 	// present when there is an associated reference.
 	for seg := f.usage.FirstSegment(); seg.Ok(); {
-		if !seg.ValuePtr().canCommit() {
+		if seg.ValuePtr().refs == 0 {
+			// We assume that reclaimable pages (that aren't already known to
+			// be committed) are not committed. This isn't necessarily true,
+			// even after the reclaimer does Decommit(), because the kernel may
+			// subsequently back the hugepage-sized region containing the
+			// decommitted page with a hugepage. However, it's consistent with
+			// our treatment of unallocated pages, which have the same
+			// property.
+			seg = seg.NextSegment()
+			continue
+		}
+		wasCommitted := seg.ValuePtr().knownCommitted
+		if !alsoScanCommitted && wasCommitted {
 			seg = seg.NextSegment()
 			continue
 		}
@@ -1233,48 +1242,60 @@ func (f *MemoryFile) updateUsageLocked(currentUsage uint64, memCgIDs map[uint32]
 				// by f.UpdateUsage() might take a really long time. So unlock f.mu
 				// while checkCommitted runs.
 				f.mu.Unlock() // +checklocksforce
-				err := checkCommitted(s, buf)
+				err := checkCommitted(s, buf, r.Start, wasCommitted)
 				f.mu.Lock()
 				if err != nil {
 					checkErr = err
 					return
 				}
 
-				// Scan each page and switch out segments.
+				// Scan each page and switch out segments. If wasCommitted is
+				// false, then we are marking ranges that are now committed;
+				// otherwise, we are marking ranges that are now uncommitted.
+				unchangedVal := byte(0)
+				if wasCommitted {
+					unchangedVal = 1
+				}
 				seg := f.usage.LowerBoundSegment(r.Start)
 				for i := 0; i < bufLen; {
-					if buf[i]&0x1 == 0 {
+					if buf[i]&0x1 == unchangedVal {
 						i++
 						continue
 					}
-					// Scan to the end of this committed range.
+					// Scan to the end of this changed range.
 					j := i + 1
 					for ; j < bufLen; j++ {
-						if buf[j]&0x1 == 0 {
+						if buf[j]&0x1 == unchangedVal {
 							break
 						}
 					}
-					committedFR := memmap.FileRange{
+					changedFR := memmap.FileRange{
 						Start: r.Start + uint64(i*hostarch.PageSize),
 						End:   r.Start + uint64(j*hostarch.PageSize),
 					}
-					// Advance seg to committedFR.Start.
-					for seg.Ok() && seg.End() < committedFR.Start {
+					// Advance seg to changedFR.Start.
+					for seg.Ok() && seg.End() <= changedFR.Start {
 						seg = seg.NextSegment()
 					}
-					// Mark pages overlapping committedFR as committed.
-					for seg.Ok() && seg.Start() < committedFR.End {
-						if seg.ValuePtr().canCommit() {
-							seg = f.usage.Isolate(seg, committedFR)
-							seg.ValuePtr().knownCommitted = true
+					// Mark pages overlapping changedFR as committed or
+					// decommitted.
+					for seg.Ok() && seg.Start() < changedFR.End {
+						if seg.ValuePtr().refs != 0 && seg.ValuePtr().knownCommitted == wasCommitted {
+							seg = f.usage.Isolate(seg, changedFR)
+							seg.ValuePtr().knownCommitted = !wasCommitted
 							amount := seg.Range().Length()
-							usage.MemoryAccounting.Inc(amount, seg.ValuePtr().kind, seg.ValuePtr().memCgID)
-							f.usageExpected += amount
+							if wasCommitted {
+								usage.MemoryAccounting.Dec(amount, seg.ValuePtr().kind, seg.ValuePtr().memCgID)
+								f.usageExpected -= amount
+							} else {
+								usage.MemoryAccounting.Inc(amount, seg.ValuePtr().kind, seg.ValuePtr().memCgID)
+								f.usageExpected += amount
+							}
 							changedAny = true
 						}
 						seg = seg.NextSegment()
 					}
-					// Continue scanning for committed pages.
+					// Continue scanning for changed pages.
 					i = j + 1
 				}
 

pkg/sentry/pgalloc/pgalloc_unsafe.go

@@ -29,7 +29,7 @@ func unsafeSlice(addr uintptr, length int) (slice []byte) {
 	return
 }
 
-func mincore(s []byte, buf []byte) error {
+func mincore(s []byte, buf []byte, off uint64, wasCommitted bool) error {
 	if _, _, errno := unix.RawSyscall(
 		unix.SYS_MINCORE,
 		uintptr(unsafe.Pointer(&s[0])),

pkg/sentry/pgalloc/save_restore.go

@@ -21,17 +21,33 @@ import (
 	"io"
 	"runtime"
 
-	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/hostarch"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/sentry/memmap"
 	"gvisor.dev/gvisor/pkg/sentry/usage"
 	"gvisor.dev/gvisor/pkg/state"
 	"gvisor.dev/gvisor/pkg/state/statefile"
+	"gvisor.dev/gvisor/pkg/sync"
 )
 
+// SaveOpts provides options to MemoryFile.SaveTo().
+type SaveOpts struct {
+	// If ExcludeCommittedZeroPages is true, SaveTo() will scan both committed
+	// and possibly-committed pages to find zero pages, whose contents are
+	// saved implicitly rather than explicitly to reduce checkpoint size. If
+	// ExcludeCommittedZeroPages is false, SaveTo() will scan only
+	// possibly-committed pages to find zero pages.
+	//
+	// Enabling ExcludeCommittedZeroPages will usually increase the time taken
+	// by SaveTo() (due to the larger number of pages that must be scanned),
+	// but may instead improve SaveTo() and LoadFrom() time, and checkpoint
+	// size, if the application has many committed zero pages.
+	ExcludeCommittedZeroPages bool
+}
+
 // SaveTo writes f's state to the given stream.
-func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer) error {
+func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer, opts SaveOpts) error {
 	// Wait for reclaim.
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -47,10 +63,55 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer) erro
 		panic(fmt.Sprintf("evictions still pending for %d users; call StartEvictions and WaitForEvictions before SaveTo", len(f.evictable)))
 	}
 
-	// Ensure that all pages that contain data have knownCommitted set, since
-	// we only store knownCommitted pages below.
+	// Ensure that all pages that contain non-zero bytes have knownCommitted
+	// set, since we only store knownCommitted pages below.
 	zeroPage := make([]byte, hostarch.PageSize)
-	err := f.updateUsageLocked(0, nil, func(bs []byte, committed []byte) error {
+	var (
+		decommitWarnOnce  sync.Once
+		decommitPendingFR memmap.FileRange
+		scanTotal         uint64
+		decommitTotal     uint64
+		decommitCount     uint64
+	)
+	decommitNow := func(fr memmap.FileRange) {
+		decommitTotal += fr.Length()
+		decommitCount++
+		if err := f.decommitFile(fr); err != nil {
+			// This doesn't impact the correctness of saved memory, it just
+			// means that we're incrementally more likely to OOM. Complain, but
+			// don't abort saving.
+			decommitWarnOnce.Do(func() {
+				log.Warningf("Decommitting MemoryFile offsets %v while saving failed: %v", fr, err)
+			})
+		}
+	}
+	decommitAddPage := func(off uint64) {
+		// Invariants:
+		// (1) All of decommitPendingFR lies within a single huge page.
+		// (2) decommitPendingFR.End is hugepage-aligned iff
+		// decommitPendingFR.Length() == 0.
+		end := off + hostarch.PageSize
+		if decommitPendingFR.End == off {
+			// Merge with the existing range. By invariants, the page {off,
+			// end} must be within the same huge page as the rest of
+			// decommitPendingFR.
+			decommitPendingFR.End = end
+		} else {
+			// Decommit the existing range and start a new one.
+			if decommitPendingFR.Length() != 0 {
+				decommitNow(decommitPendingFR)
+			}
+			decommitPendingFR = memmap.FileRange{off, end}
+		}
+		// Maintain invariants by decommitting if we've reached the end of the
+		// containing huge page.
+		if hostarch.IsHugePageAligned(end) {
+			decommitNow(decommitPendingFR)
+			decommitPendingFR = memmap.FileRange{}
+		}
+	}
+	err := f.updateUsageLocked(0, nil, opts.ExcludeCommittedZeroPages, func(bs []byte, committed []byte, off uint64, wasCommitted bool) error {
+		scanTotal += uint64(len(bs))
 		for pgoff := 0; pgoff < len(bs); pgoff += hostarch.PageSize {
 			i := pgoff / hostarch.PageSize
 			pg := bs[pgoff : pgoff+hostarch.PageSize]
@@ -59,25 +120,22 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w io.Writer, pw io.Writer) erro
 				continue
 			}
 			committed[i] = 0
-			// Reading the page caused it to be committed; decommit it to
-			// reduce memory usage.
-			//
-			// "MADV_REMOVE [...] Free up a given range of pages and its
-			// associated backing store. This is equivalent to punching a hole
-			// in the corresponding byte range of the backing store (see
-			// fallocate(2))." - madvise(2)
-			if err := unix.Madvise(pg, unix.MADV_REMOVE); err != nil {
-				// This doesn't impact the correctness of saved memory, it
-				// just means that we're incrementally more likely to OOM.
-				// Complain, but don't abort saving.
-				log.Warningf("Decommitting page %p while saving failed: %v", pg, err)
+			if !wasCommitted {
+				// Reading the page may have caused it to be committed;
+				// decommit it to reduce memory usage.
+				decommitAddPage(off + uint64(pgoff))
 			}
 		}
 		return nil
 	})
+	if decommitPendingFR.Length() != 0 {
+		decommitNow(decommitPendingFR)
+		decommitPendingFR = memmap.FileRange{}
+	}
 	if err != nil {
 		return err
 	}
+	log.Debugf("MemoryFile.SaveTo: scanned %d bytes, decommitted %d bytes in %d syscalls", scanTotal, decommitTotal, decommitCount)
 
 	// Save metadata.
 	if _, err := state.Save(ctx, w, &f.fileSize); err != nil {
@@ -212,7 +270,9 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r io.Reader, pr *statefile.As
 		// Update accounting for restored pages. We need to do this here since
 		// these segments are marked as "known committed", and will be skipped
 		// over on accounting scans.
-		usage.MemoryAccounting.Inc(seg.End()-seg.Start(), seg.Value().kind, seg.Value().memCgID)
+		amount := seg.Range().Length()
+		usage.MemoryAccounting.Inc(amount, seg.Value().kind, seg.Value().memCgID)
+		f.usageExpected += amount
 	}
 
 	return nil

pkg/sentry/state/BUILD

@@ -21,6 +21,7 @@ go_library(
         "//pkg/log",
         "//pkg/sentry/inet",
         "//pkg/sentry/kernel",
+        "//pkg/sentry/pgalloc",
         "//pkg/sentry/time",
         "//pkg/sentry/vfs",
         "//pkg/sentry/watchdog",

pkg/sentry/state/state.go

@@ -25,6 +25,7 @@ import (
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/inet"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/time"
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 	"gvisor.dev/gvisor/pkg/sentry/watchdog"
@@ -63,6 +64,9 @@ type SaveOpts struct {
 	// Metadata is save metadata.
 	Metadata map[string]string
 
+	// MemoryFileSaveOpts is passed to calls to pgalloc.MemoryFile.SaveTo().
+	MemoryFileSaveOpts pgalloc.SaveOpts
+
 	// Callback is called prior to unpause, with any save error.
 	Callback func(err error)
 
@@ -95,7 +99,7 @@ func (opts SaveOpts) Save(ctx context.Context, k *kernel.Kernel, w *watchdog.Wat
 		err = ErrStateFile{err}
 	} else {
 		// Save the kernel.
-		err = k.SaveTo(ctx, wc, opts.PagesMetadata, opts.PagesFile)
+		err = k.SaveTo(ctx, wc, opts.PagesMetadata, opts.PagesFile, opts.MemoryFileSaveOpts)
 
 		// ENOSPC is a state file error. This error can only come from
 		// writing the state file, and not from fs.FileOperations.Fsync

runsc/cmd/BUILD

@@ -92,6 +92,7 @@ go_library(
         "//pkg/sentry/devices/tpuproxy",
         "//pkg/sentry/kernel",
         "//pkg/sentry/kernel/auth",
+        "//pkg/sentry/pgalloc",
         "//pkg/sentry/platform",
         "//pkg/state/pretty",
         "//pkg/state/statefile",

runsc/cmd/checkpoint.go

@@ -20,6 +20,7 @@ import (
 	"os"
 
 	"github.com/google/subcommands"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/state/statefile"
 	"gvisor.dev/gvisor/runsc/cmd/util"
 	"gvisor.dev/gvisor/runsc/config"
@@ -29,9 +30,10 @@ import (
 
 // Checkpoint implements subcommands.Command for the "checkpoint" command.
 type Checkpoint struct {
-	imagePath    string
-	leaveRunning bool
-	compression  CheckpointCompression
+	imagePath                 string
+	leaveRunning              bool
+	compression               CheckpointCompression
+	excludeCommittedZeroPages bool
 }
 
 // Name implements subcommands.Command.Name.
@@ -55,6 +57,7 @@ func (c *Checkpoint) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&c.imagePath, "image-path", "", "directory path to saved container image")
 	f.BoolVar(&c.leaveRunning, "leave-running", false, "restart the container after checkpointing")
 	f.Var(newCheckpointCompressionValue(statefile.CompressionLevelDefault, &c.compression), "compression", "compress checkpoint image on disk. Values: none|flate-best-speed.")
+	f.BoolVar(&c.excludeCommittedZeroPages, "exclude-committed-zero-pages", false, "exclude committed zero-filled pages from checkpoint")
 
 	// Unimplemented flags necessary for compatibility with docker.
 	var wp string
@@ -84,14 +87,19 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su
 		util.Fatalf("making directories at path provided: %v", err)
 	}
 
-	sOpts := statefile.Options{Compression: c.compression.Level()}
+	sOpts := statefile.Options{
+		Compression: c.compression.Level(),
+	}
+	mfOpts := pgalloc.SaveOpts{
+		ExcludeCommittedZeroPages: c.excludeCommittedZeroPages,
+	}
 
 	if c.leaveRunning {
 		// Do not destroy the sandbox after saving.
 		sOpts.Resume = true
 	}
 
-	if err := cont.Checkpoint(c.imagePath, sOpts); err != nil {
+	if err := cont.Checkpoint(c.imagePath, sOpts, mfOpts); err != nil {
 		util.Fatalf("checkpoint failed: %v", err)
 	}
 

runsc/container/BUILD

@@ -80,6 +80,7 @@ go_test(
         "//pkg/sentry/kernel",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/limits",
+        "//pkg/sentry/pgalloc",
         "//pkg/sentry/platform",
         "//pkg/sentry/seccheck",
         "//pkg/sentry/seccheck/points:points_go_proto",

runsc/container/container.go

@@ -702,12 +702,12 @@ func (c *Container) ForwardSignals(pid int32, fgProcess bool) func() {
 
 // Checkpoint sends the checkpoint call to the container.
 // The statefile will be written to f, the file at the specified image-path.
-func (c *Container) Checkpoint(imagePath string, options statefile.Options) error {
+func (c *Container) Checkpoint(imagePath string, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error {
 	log.Debugf("Checkpoint container, cid: %s", c.ID)
 	if err := c.requireStatus("checkpoint", Created, Running, Paused); err != nil {
 		return err
 	}
-	return c.Sandbox.Checkpoint(c.ID, imagePath, options)
+	return c.Sandbox.Checkpoint(c.ID, imagePath, sfOpts, mfOpts)
 }
 
 // Pause suspends the container and its kernel.

runsc/container/container_test.go

@@ -42,6 +42,7 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
 	"gvisor.dev/gvisor/pkg/state/statefile"
 	"gvisor.dev/gvisor/pkg/sync"
@@ -1069,7 +1070,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, compression statef
 	}
 
 	// Checkpoint running container; save state into new file.
-	if err := cont.Checkpoint(dir, statefile.Options{Compression: compression}); err != nil {
+	if err := cont.Checkpoint(dir, statefile.Options{Compression: compression}, pgalloc.SaveOpts{}); err != nil {
 		t.Fatalf("error checkpointing container to empty file: %v", err)
 	}
 
@@ -1251,7 +1252,7 @@ func TestCheckpointRestoreExecKilled(t *testing.T) {
 	}
 
 	// Checkpoint running container.
-	if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+	if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil {
 		t.Fatalf("error checkpointing container: %v", err)
 	}
 	cont.Destroy()
@@ -1345,7 +1346,7 @@ func TestUnixDomainSockets(t *testing.T) {
 			}
 
 			// Checkpoint running container; save state into new file.
-			if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelDefault}); err != nil {
+			if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelDefault}, pgalloc.SaveOpts{}); err != nil {
 				t.Fatalf("error checkpointing container to empty file: %v", err)
 			}
 

runsc/container/multi_container_test.go

@@ -32,6 +32,7 @@ import (
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/state/statefile"
 	"gvisor.dev/gvisor/pkg/sync"
 	"gvisor.dev/gvisor/pkg/test/testutil"
@@ -2772,7 +2773,7 @@ func TestMultiContainerCheckpointRestore(t *testing.T) {
 			}
 
 			// Checkpoint root container; save state into new file.
-			if err := conts[0].Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+			if err := conts[0].Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil {
 				t.Fatalf("error checkpointing container to empty file: %v", err)
 			}
 			defer os.RemoveAll(dir)

runsc/sandbox/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//pkg/sentry/control",
         "//pkg/sentry/devices/nvproxy",
         "//pkg/sentry/fsimpl/erofs",
+        "//pkg/sentry/pgalloc",
         "//pkg/sentry/platform",
         "//pkg/sentry/seccheck",
         "//pkg/state/statefile",

runsc/sandbox/sandbox.go

@@ -48,6 +48,7 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
 	"gvisor.dev/gvisor/pkg/sentry/seccheck"
 	"gvisor.dev/gvisor/pkg/state/statefile"
@@ -1324,8 +1325,8 @@ func (s *Sandbox) SignalProcess(cid string, pid int32, sig unix.Signal, fgProces
 
 // Checkpoint sends the checkpoint call for a container in the sandbox.
 // The statefile will be written to f.
-func (s *Sandbox) Checkpoint(cid string, imagePath string, options statefile.Options) error {
-	log.Debugf("Checkpoint sandbox %q, options %+v", s.ID, options)
+func (s *Sandbox) Checkpoint(cid string, imagePath string, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error {
+	log.Debugf("Checkpoint sandbox %q, statefile options %+v, MemoryFile options %+v", s.ID, sfOpts, mfOpts)
 
 	stateFilePath := filepath.Join(imagePath, boot.CheckpointStateFileName)
 	sf, err := os.OpenFile(stateFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
@@ -1335,17 +1336,18 @@ func (s *Sandbox) Checkpoint(cid string, imagePath string, options statefile.Opt
 	defer sf.Close()
 
 	opt := control.SaveOpts{
-		Metadata: options.WriteToMetadata(map[string]string{}),
+		Metadata:           sfOpts.WriteToMetadata(map[string]string{}),
+		MemoryFileSaveOpts: mfOpts,
 		FilePayload: urpc.FilePayload{
 			Files: []*os.File{sf},
 		},
-		Resume: options.Resume,
+		Resume: sfOpts.Resume,
 	}
 
 	// When there is no compression, MemoryFile contents are page-aligned.
 	// It is beneficial to store them separately so certain optimizations can be
 	// applied during restore. See Restore().
-	if options.Compression == statefile.CompressionLevelNone {
+	if sfOpts.Compression == statefile.CompressionLevelNone {
 		pagesFilePath := filepath.Join(imagePath, boot.CheckpointPagesFileName)
 		// TODO(b/327603247): Implement optional async O_DIRECT write.
 		pf, err := os.OpenFile(pagesFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)