Create separate pages.img checkpoint file when compression=none.

PiperOrigin-RevId: 624210535

g3doc/user_guide/checkpoint_restore.md

@@ -15,9 +15,8 @@ runsc run <container id>
 ```
 
 To checkpoint the container, the `--image-path` flag must be provided. This is
-the directory path within which the checkpoint state-file will be created. The
-file will be called `checkpoint.img` and necessary directories will be created
-if they do not yet exist.
+the directory path within which the checkpoint related files will be created.
+All necessary directories will be created if they do not yet exist.
 
 > Note: Two checkpoints cannot be saved to the same directory; every image-path
 > provided must be unique.
@@ -41,9 +40,10 @@ their processes after committing a checkpoint.)
 runsc checkpoint --image-path=<path> --leave-running <container id>
 ```
 
-To restore, provide the image path to the `checkpoint.img` file created during
-the checkpoint. Because containers stop by default after checkpointing, restore
-needs to happen in a new container (restore is a command which parallels start).
+To restore, provide the image path to the directory containing all the files
+created during the checkpoint. Because containers stop by default after
+checkpointing, restore needs to happen in a new container (restore is a command
+which parallels start).
 
 ```bash
 runsc create <container id>

pkg/sentry/control/state.go

@@ -16,6 +16,7 @@ package control
 
 import (
 	"errors"
+	"fmt"
 
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/log"
@@ -43,7 +44,12 @@ type SaveOpts struct {
 	// Metadata is the set of metadata to prepend to the state file.
 	Metadata map[string]string `json:"metadata"`
 
-	// FilePayload contains the destination for the state.
+	// HavePagesFile indicates whether the checkpoint pages file is provided.
+	HavePagesFile bool `json:"have_pages_file"`
+
+	// FilePayload contains the following:
+	// 1. checkpoint state file.
+	// 2. optional checkpoint pages file.
 	urpc.FilePayload
 
 	// Resume indicates if the sandbox process should continue running
@@ -53,11 +59,13 @@ type SaveOpts struct {
 
 // Save saves the running system.
 func (s *State) Save(o *SaveOpts, _ *struct{}) error {
-	// Create an output stream.
-	if len(o.FilePayload.Files) != 1 {
-		return ErrInvalidFiles
+	wantFiles := 1
+	if o.HavePagesFile {
+		wantFiles++
+	}
+	if gotFiles := len(o.FilePayload.Files); gotFiles != wantFiles {
+		return fmt.Errorf("got %d files, wanted %d", gotFiles, wantFiles)
 	}
-	defer o.FilePayload.Files[0].Close()
 
 	// Save to the first provided stream.
 	saveOpts := state.SaveOpts{
@@ -77,5 +85,10 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error {
 			}
 		},
 	}
+	defer o.FilePayload.Files[0].Close()
+	if o.HavePagesFile {
+		saveOpts.PagesFile = o.FilePayload.Files[1]
+		defer saveOpts.PagesFile.Close()
+	}
 	return saveOpts.Save(s.Kernel.SupervisorContext(), s.Kernel, s.Watchdog)
 }

pkg/sentry/kernel/kernel.go

@@ -34,6 +34,8 @@ package kernel
 import (
 	"errors"
 	"fmt"
+	"io"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -524,7 +526,7 @@ type privateMemoryFileMetadata struct {
 	owners []string
 }
 
-func savePrivateMFs(ctx context.Context, w wire.Writer, mfsToSave map[string]*pgalloc.MemoryFile) error {
+func savePrivateMFs(ctx context.Context, w wire.Writer, pw io.Writer, mfsToSave map[string]*pgalloc.MemoryFile) error {
 	var meta privateMemoryFileMetadata
 	// Generate the order in which private memory files are saved.
 	for fsID := range mfsToSave {
@@ -536,14 +538,14 @@ func savePrivateMFs(ctx context.Context, w wire.Writer, mfsToSave map[string]*pg
 	}
 	// Followed by the private memory files in order.
 	for _, fsID := range meta.owners {
-		if err := mfsToSave[fsID].SaveTo(ctx, w); err != nil {
+		if err := mfsToSave[fsID].SaveTo(ctx, w, pw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func loadPrivateMFs(ctx context.Context, r wire.Reader) error {
+func loadPrivateMFs(ctx context.Context, r wire.Reader, pr io.Reader) error {
 	// Load the metadata.
 	var meta privateMemoryFileMetadata
 	if _, err := state.Load(ctx, r, &meta); err != nil {
@@ -560,7 +562,7 @@ func loadPrivateMFs(ctx context.Context, r wire.Reader) error {
 		if !ok {
 			return fmt.Errorf("saved memory file for %q was not configured on restore", fsID)
 		}
-		if err := mf.LoadFrom(ctx, r); err != nil {
+		if err := mf.LoadFrom(ctx, r, pr); err != nil {
 			return err
 		}
 	}
@@ -570,7 +572,7 @@ func loadPrivateMFs(ctx context.Context, r wire.Reader) error {
 // SaveTo saves the state of k to w.
 //
 // Preconditions: The kernel must be paused throughout the call to SaveTo.
-func (k *Kernel) SaveTo(ctx context.Context, w wire.Writer) error {
+func (k *Kernel) SaveTo(ctx context.Context, w wire.Writer, pagesFile *os.File) error {
 	saveStart := time.Now()
 
 	// Do not allow other Kernel methods to affect it while it's being saved.
@@ -638,10 +640,14 @@ func (k *Kernel) SaveTo(ctx context.Context, w wire.Writer) error {
 
 	// Save the memory files' state.
 	memoryStart := time.Now()
-	if err := k.mf.SaveTo(ctx, w); err != nil {
+	pw := io.Writer(w)
+	if pagesFile != nil {
+		pw = pagesFile
+	}
+	if err := k.mf.SaveTo(ctx, w, pw); err != nil {
 		return err
 	}
-	if err := savePrivateMFs(ctx, w, mfsToSave); err != nil {
+	if err := savePrivateMFs(ctx, w, pw, mfsToSave); err != nil {
 		return err
 	}
 	log.Infof("Memory files save took [%s].", time.Since(memoryStart))
@@ -677,7 +683,7 @@ func (k *Kernel) invalidateUnsavableMappings(ctx context.Context) error {
 }
 
 // LoadFrom returns a new Kernel loaded from args.
-func (k *Kernel) LoadFrom(ctx context.Context, r wire.Reader, timeReady chan struct{}, net inet.Stack, clocks sentrytime.Clocks, vfsOpts *vfs.CompleteRestoreOptions) error {
+func (k *Kernel) LoadFrom(ctx context.Context, r wire.Reader, pagesFile *os.File, timeReady chan struct{}, net inet.Stack, clocks sentrytime.Clocks, vfsOpts *vfs.CompleteRestoreOptions) error {
 	loadStart := time.Now()
 
 	k.runningTasksCond.L = &k.runningTasksMu
@@ -719,10 +725,14 @@ func (k *Kernel) LoadFrom(ctx context.Context, r wire.Reader, timeReady chan str
 
 	// Load the memory files' state.
 	memoryStart := time.Now()
-	if err := k.mf.LoadFrom(ctx, r); err != nil {
+	pr := io.Reader(r)
+	if pagesFile != nil {
+		pr = pagesFile
+	}
+	if err := k.mf.LoadFrom(ctx, r, pr); err != nil {
 		return err
 	}
-	if err := loadPrivateMFs(ctx, r); err != nil {
+	if err := loadPrivateMFs(ctx, r, pr); err != nil {
 		return err
 	}
 	log.Infof("Memory files load took [%s].", time.Since(memoryStart))

pkg/sentry/pgalloc/save_restore.go

@@ -31,7 +31,7 @@ import (
 )
 
 // SaveTo writes f's state to the given stream.
-func (f *MemoryFile) SaveTo(ctx context.Context, w wire.Writer) error {
+func (f *MemoryFile) SaveTo(ctx context.Context, w wire.Writer, pw io.Writer) error {
 	// Wait for reclaim.
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -102,7 +102,7 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w wire.Writer) error {
 			if ioErr != nil {
 				return
 			}
-			_, ioErr = w.Write(s)
+			_, ioErr = pw.Write(s)
 		})
 		if ioErr != nil {
 			return ioErr
@@ -135,7 +135,7 @@ func (f *MemoryFile) RestoreID() string {
 }
 
 // LoadFrom loads MemoryFile state from the given stream.
-func (f *MemoryFile) LoadFrom(ctx context.Context, r wire.Reader) error {
+func (f *MemoryFile) LoadFrom(ctx context.Context, r wire.Reader, pr io.Reader) error {
 	// Load metadata.
 	if _, err := state.Load(ctx, r, &f.fileSize); err != nil {
 		return err
@@ -196,7 +196,7 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r wire.Reader) error {
 			if ioErr != nil {
 				return
 			}
-			_, ioErr = io.ReadFull(r, s)
+			_, ioErr = io.ReadFull(pr, s)
 		})
 		if ioErr != nil {
 			return ioErr

pkg/sentry/state/state.go

@@ -18,6 +18,7 @@ package state
 import (
 	"fmt"
 	"io"
+	"os"
 
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
@@ -48,6 +49,10 @@ type SaveOpts struct {
 	// Destination is the save target.
 	Destination io.Writer
 
+	// PagesFile is the file in which all MemoryFile pages are stored if
+	// PagesFile is non-nil.
+	PagesFile *os.File
+
 	// Key is used for state integrity check.
 	Key []byte
 
@@ -86,7 +91,7 @@ func (opts SaveOpts) Save(ctx context.Context, k *kernel.Kernel, w *watchdog.Wat
 		err = ErrStateFile{err}
 	} else {
 		// Save the kernel.
-		err = k.SaveTo(ctx, wc)
+		err = k.SaveTo(ctx, wc, opts.PagesFile)
 
 		// ENOSPC is a state file error. This error can only come from
 		// writing the state file, and not from fs.FileOperations.Fsync
@@ -108,6 +113,10 @@ type LoadOpts struct {
 	// Destination is the load source.
 	Source io.Reader
 
+	// PagesFile is the file in which all MemoryFile pages are stored if
+	// PagesFile is non-nil.
+	PagesFile *os.File
+
 	// Key is used for state integrity check.
 	Key []byte
 }
@@ -123,5 +132,5 @@ func (opts LoadOpts) Load(ctx context.Context, k *kernel.Kernel, timeReady chan
 	previousMetadata = m
 
 	// Restore the Kernel object graph.
-	return k.LoadFrom(ctx, r, timeReady, n, clocks, vfsOpts)
+	return k.LoadFrom(ctx, r, opts.PagesFile, timeReady, n, clocks, vfsOpts)
 }

runsc/boot/controller.go

@@ -443,12 +443,13 @@ func (cm *containerManager) PortForward(opts *PortForwardOpts, _ *struct{}) erro
 
 // RestoreOpts contains options related to restoring a container's file system.
 type RestoreOpts struct {
-	// FilePayload contains the state file to be restored, followed by the
-	// platform device file if necessary.
+	// FilePayload contains the state file to be restored, followed in order by:
+	// 1. checkpoint state file.
+	// 2. optional checkpoint pages file.
+	// 3. optional platform device file.
 	urpc.FilePayload
-
-	// SandboxID contains the ID of the sandbox.
-	SandboxID string
+	HavePagesFile  bool
+	HaveDeviceFile bool
 }
 
 // Restore loads a container from a statefile.
@@ -458,29 +459,40 @@ type RestoreOpts struct {
 func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
 	log.Debugf("containerManager.Restore")
 
+	if len(o.Files) == 0 {
+		return fmt.Errorf("at least one file must be passed to Restore")
+	}
+	fileIdx := 0
+
 	r := restorer{container: &cm.l.root}
-	switch numFiles := len(o.Files); numFiles {
-	case 2:
+	r.stateFile = o.Files[fileIdx]
+	fileIdx++
+	defer r.stateFile.Close()
+	if info, err := r.stateFile.Stat(); err != nil {
+		return err
+	} else if info.Size() == 0 {
+		return fmt.Errorf("statefile cannot be empty")
+	}
+
+	if o.HavePagesFile {
+		r.pagesFile = o.Files[fileIdx]
+		fileIdx++
+		defer r.pagesFile.Close()
+	}
+
+	if o.HaveDeviceFile {
 		// The device file is donated to the platform.
 		// Can't take ownership away from os.File. dup them to get a new FD.
-		fd, err := unix.Dup(int(o.Files[1].Fd()))
+		fd, err := unix.Dup(int(o.Files[fileIdx].Fd()))
 		if err != nil {
 			return fmt.Errorf("failed to dup file: %v", err)
 		}
 		r.deviceFile = os.NewFile(uintptr(fd), "platform device")
-		fallthrough
-	case 1:
-		r.stateFile = o.Files[0]
-		if info, err := r.stateFile.Stat(); err != nil {
-			return err
-		} else if info.Size() == 0 {
-			return fmt.Errorf("file cannot be empty")
-		}
+		fileIdx++
+	}
 
-	case 0:
-		return fmt.Errorf("at least one file must be passed to Restore")
-	default:
-		return fmt.Errorf("at most two files may be passed to Restore")
+	if fileIdx < len(o.Files) {
+		return fmt.Errorf("more files passed to Restore than expected")
 	}
 
 	// Pause the kernel while we build a new one.

runsc/boot/restore.go

@@ -35,9 +35,19 @@ import (
 	"gvisor.dev/gvisor/runsc/boot/pprof"
 )
 
+const (
+	// CheckpointStateFileName is the file within the given image-path's
+	// directory which contains the container's saved state.
+	CheckpointStateFileName = "checkpoint.img"
+	// CheckpointPagesFileName is the file within the given image-path's
+	// directory containing the container's MemoryFile pages.
+	CheckpointPagesFileName = "pages.img"
+)
+
 type restorer struct {
 	container  *containerInfo
 	stateFile  *os.File
+	pagesFile  *os.File
 	deviceFile *os.File
 }
 
@@ -135,7 +145,7 @@ func (r *restorer) restore(l *Loader) error {
 	ctx = context.WithValue(ctx, pgalloc.CtxMemoryFileMap, mfmap)
 
 	// Load the state.
-	loadOpts := state.LoadOpts{Source: r.stateFile}
+	loadOpts := state.LoadOpts{Source: r.stateFile, PagesFile: r.pagesFile}
 	if err := loadOpts.Load(ctx, l.k, nil, netns.Stack(), time.NewCalibratedClocks(), &vfs.CompleteRestoreOptions{}); err != nil {
 		return err
 	}

runsc/cmd/boot.go

@@ -462,7 +462,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 	}
 
 	if conf.TestOnlyAutosaveImagePath != "" {
-		fName := filepath.Join(conf.TestOnlyAutosaveImagePath, checkpointFileName)
+		fName := filepath.Join(conf.TestOnlyAutosaveImagePath, boot.CheckpointStateFileName)
 		f, err := os.OpenFile(fName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
 		if err != nil {
 			util.Fatalf("error in creating state file %v", err)

runsc/cmd/checkpoint.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/google/subcommands"
 	"gvisor.dev/gvisor/pkg/state/statefile"
@@ -28,9 +27,6 @@ import (
 	"gvisor.dev/gvisor/runsc/flag"
 )
 
-// File containing the container's saved image/state within the given image-path's directory.
-const checkpointFileName = "checkpoint.img"
-
 // Checkpoint implements subcommands.Command for the "checkpoint" command.
 type Checkpoint struct {
 	imagePath    string
@@ -88,15 +84,6 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su
 		util.Fatalf("making directories at path provided: %v", err)
 	}
 
-	fullImagePath := filepath.Join(c.imagePath, checkpointFileName)
-
-	// Create the image file and open for writing.
-	file, err := os.OpenFile(fullImagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
-	if err != nil {
-		util.Fatalf("os.OpenFile(%q) failed: %v", fullImagePath, err)
-	}
-	defer file.Close()
-
 	sOpts := statefile.Options{Compression: c.compression.Level()}
 
 	if c.leaveRunning {
@@ -104,7 +91,7 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su
 		sOpts.Resume = true
 	}
 
-	if err := cont.Checkpoint(file, sOpts); err != nil {
+	if err := cont.Checkpoint(c.imagePath, sOpts); err != nil {
 		util.Fatalf("checkpoint failed: %v", err)
 	}
 

runsc/cmd/restore.go

@@ -17,7 +17,6 @@ package cmd
 import (
 	"context"
 	"os"
-	"path/filepath"
 
 	"github.com/google/subcommands"
 	"golang.org/x/sys/unix"
@@ -99,8 +98,6 @@ func (r *Restore) Execute(_ context.Context, f *flag.FlagSet, args ...any) subco
 	var cu cleanup.Cleanup
 	defer cu.Clean()
 
-	conf.RestoreFile = filepath.Join(r.imagePath, checkpointFileName)
-
 	runArgs := container.Args{
 		ID:            id,
 		Spec:          nil,
@@ -140,8 +137,8 @@ func (r *Restore) Execute(_ context.Context, f *flag.FlagSet, args ...any) subco
 		runArgs.Spec = c.Spec
 	}
 
-	log.Debugf("Restore: %v", conf.RestoreFile)
-	if err := c.Restore(conf, conf.RestoreFile); err != nil {
+	log.Debugf("Restore: %v", r.imagePath)
+	if err := c.Restore(conf, r.imagePath); err != nil {
 		return util.Errorf("starting container: %v", err)
 	}
 

runsc/config/config.go

@@ -234,9 +234,6 @@ type Config struct {
 	// for the duration of the container execution.
 	TraceFile string `flag:"trace"`
 
-	// RestoreFile is the path to the saved container image.
-	RestoreFile string
-
 	// NumNetworkChannels controls the number of AF_PACKET sockets that map
 	// to the same underlying network device. This allows netstack to better
 	// scale for high throughput use cases.

runsc/container/container.go

@@ -525,7 +525,7 @@ func (c *Container) Start(conf *config.Config) error {
 
 // Restore takes a container and replaces its kernel and file system
 // to restore a container from its state file.
-func (c *Container) Restore(conf *config.Config, restoreFile string) error {
+func (c *Container) Restore(conf *config.Config, imagePath string) error {
 	log.Debugf("Restore container, cid: %s", c.ID)
 	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
@@ -542,7 +542,7 @@ func (c *Container) Restore(conf *config.Config, restoreFile string) error {
 		log.Warningf("StartContainer hook skipped because running inside container namespace is not supported")
 	}
 
-	if err := c.Sandbox.Restore(conf, c.ID, restoreFile); err != nil {
+	if err := c.Sandbox.Restore(conf, c.ID, imagePath); err != nil {
 		return err
 	}
 	c.changeStatus(Running)
@@ -563,15 +563,8 @@ func Run(conf *config.Config, args Args) (unix.WaitStatus, error) {
 	})
 	defer cu.Clean()
 
-	if conf.RestoreFile != "" {
-		log.Debugf("Restore: %v", conf.RestoreFile)
-		if err := c.Restore(conf, conf.RestoreFile); err != nil {
-			return 0, fmt.Errorf("starting container: %v", err)
-		}
-	} else {
-		if err := c.Start(conf); err != nil {
-			return 0, fmt.Errorf("starting container: %v", err)
-		}
+	if err := c.Start(conf); err != nil {
+		return 0, fmt.Errorf("starting container: %v", err)
 	}
 
 	// If we allocate a terminal, forward signals to the sandbox process.
@@ -721,12 +714,12 @@ func (c *Container) ForwardSignals(pid int32, fgProcess bool) func() {
 
 // Checkpoint sends the checkpoint call to the container.
 // The statefile will be written to f, the file at the specified image-path.
-func (c *Container) Checkpoint(f *os.File, options statefile.Options) error {
+func (c *Container) Checkpoint(imagePath string, options statefile.Options) error {
 	log.Debugf("Checkpoint container, cid: %s", c.ID)
 	if err := c.requireStatus("checkpoint", Created, Running, Paused); err != nil {
 		return err
 	}
-	return c.Sandbox.Checkpoint(c.ID, f, options)
+	return c.Sandbox.Checkpoint(c.ID, imagePath, options)
 }
 
 // Pause suspends the container and its kernel.

runsc/container/container_test.go

@@ -1063,26 +1063,15 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript
 		t.Fatalf("error starting container: %v", err)
 	}
 
-	// Set the image path, which is where the checkpoint image will be saved.
-	imagePath := filepath.Join(dir, "test-image-file")
-
-	// Create the image file and open for writing.
-	file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
-	if err != nil {
-		t.Fatalf("error opening new file at imagePath: %v", err)
-	}
-	defer file.Close()
-
 	// Wait until application has ran.
 	if err := waitForFileNotEmpty(outputFile); err != nil {
 		t.Fatalf("Failed to wait for output file: %v", err)
 	}
 
 	// Checkpoint running container; save state into new file.
-	if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+	if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
 		t.Fatalf("error checkpointing container to empty file: %v", err)
 	}
-	defer os.RemoveAll(imagePath)
 
 	lastNum, err := readOutputNum(outputPath, -1)
 	if err != nil {
@@ -1112,7 +1101,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript
 	}
 	defer cont2.Destroy()
 
-	if err := cont2.Restore(conf, imagePath); err != nil {
+	if err := cont2.Restore(conf, dir); err != nil {
 		t.Fatalf("error restoring container: %v", err)
 	}
 
@@ -1155,7 +1144,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript
 	}
 	defer cont3.Destroy()
 
-	if err := cont3.Restore(conf, imagePath); err != nil {
+	if err := cont3.Restore(conf, dir); err != nil {
 		t.Fatalf("error restoring container: %v", err)
 	}
 
@@ -1253,16 +1242,8 @@ func TestCheckpointRestoreExecKilled(t *testing.T) {
 		t.Fatalf("error chmoding file: %q, %v", dir, err)
 	}
 
-	// Create the image file and open for writing.
-	checkpointPath := filepath.Join(dir, "test-image-file")
-	checkpointFile, err := os.OpenFile(checkpointPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
-	if err != nil {
-		t.Fatalf("error opening new file at imagePath: %v", err)
-	}
-	defer checkpointFile.Close()
-
-	// Checkpoint running container; save state into new file.
-	if err := cont.Checkpoint(checkpointFile, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+	// Checkpoint running container.
+	if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
 		t.Fatalf("error checkpointing container: %v", err)
 	}
 	cont.Destroy()
@@ -1274,7 +1255,7 @@ func TestCheckpointRestoreExecKilled(t *testing.T) {
 	}
 	defer cont2.Destroy()
 
-	if err := cont2.Restore(conf, checkpointPath); err != nil {
+	if err := cont2.Restore(conf, dir); err != nil {
 		t.Fatalf("error restoring container: %v", err)
 	}
 
@@ -1350,24 +1331,13 @@ func TestUnixDomainSockets(t *testing.T) {
 				t.Fatalf("error starting container: %v", err)
 			}
 
-			// Set the image path, the location where the checkpoint image will be saved.
-			imagePath := filepath.Join(dir, "test-image-file")
-
-			// Create the image file and open for writing.
-			file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
-			if err != nil {
-				t.Fatalf("error opening new file at imagePath: %v", err)
-			}
-			defer file.Close()
-			defer os.RemoveAll(imagePath)
-
 			// Wait until application has ran.
 			if err := waitForFileNotEmpty(outputFile); err != nil {
 				t.Fatalf("Failed to wait for output file: %v", err)
 			}
 
 			// Checkpoint running container; save state into new file.
-			if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+			if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
 				t.Fatalf("error checkpointing container to empty file: %v", err)
 			}
 
@@ -1399,7 +1369,7 @@ func TestUnixDomainSockets(t *testing.T) {
 			}
 			defer contRestore.Destroy()
 
-			if err := contRestore.Restore(conf, imagePath); err != nil {
+			if err := contRestore.Restore(conf, dir); err != nil {
 				t.Fatalf("error restoring container: %v", err)
 			}
 

runsc/sandbox/sandbox.go

@@ -24,6 +24,7 @@ import (
 	"math"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -441,20 +442,30 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s
 }
 
 // Restore sends the restore call for a container in the sandbox.
-func (s *Sandbox) Restore(conf *config.Config, cid string, filename string) error {
+func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string) error {
 	log.Debugf("Restore sandbox %q", s.ID)
 
-	rf, err := os.Open(filename)
+	stateFileName := path.Join(imagePath, boot.CheckpointStateFileName)
+	sf, err := os.Open(stateFileName)
 	if err != nil {
-		return fmt.Errorf("opening restore file %q failed: %v", filename, err)
+		return fmt.Errorf("opening state file %q failed: %v", stateFileName, err)
 	}
-	defer rf.Close()
+	defer sf.Close()
 
 	opt := boot.RestoreOpts{
 		FilePayload: urpc.FilePayload{
-			Files: []*os.File{rf},
+			Files: []*os.File{sf},
 		},
-		SandboxID: s.ID,
+	}
+
+	// If the image file exists, we must pass it in.
+	pagesFileName := path.Join(imagePath, boot.CheckpointPagesFileName)
+	if pf, err := os.Open(pagesFileName); err == nil {
+		defer pf.Close()
+		opt.HavePagesFile = true
+		opt.FilePayload.Files = append(opt.FilePayload.Files, pf)
+	} else if !os.IsNotExist(err) {
+		return fmt.Errorf("opening restore image file %q failed: %v", pagesFileName, err)
 	}
 
 	// If the platform needs a device FD we must pass it in.
@@ -462,6 +473,7 @@ func (s *Sandbox) Restore(conf *config.Config, cid string, filename string) erro
 		return err
 	} else if deviceFile != nil {
 		defer deviceFile.Close()
+		opt.HaveDeviceFile = true
 		opt.FilePayload.Files = append(opt.FilePayload.Files, deviceFile)
 	}
 
@@ -1256,16 +1268,38 @@ func (s *Sandbox) SignalProcess(cid string, pid int32, sig unix.Signal, fgProces
 
 // Checkpoint sends the checkpoint call for a container in the sandbox.
 // The statefile will be written to f.
-func (s *Sandbox) Checkpoint(cid string, f *os.File, options statefile.Options) error {
+func (s *Sandbox) Checkpoint(cid string, imagePath string, options statefile.Options) error {
 	log.Debugf("Checkpoint sandbox %q, options %+v", s.ID, options)
+
+	stateFilePath := filepath.Join(imagePath, boot.CheckpointStateFileName)
+	sf, err := os.OpenFile(stateFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("creating checkpoint state file %q: %w", stateFilePath, err)
+	}
+	defer sf.Close()
+
 	opt := control.SaveOpts{
 		Metadata: options.WriteToMetadata(map[string]string{}),
 		FilePayload: urpc.FilePayload{
-			Files: []*os.File{f},
+			Files: []*os.File{sf},
 		},
 		Resume: options.Resume,
 	}
 
+	// When there is no compression, MemoryFile contents are page-aligned.
+	// It is beneficial to store them separately so certain optimizations can be
+	// applied during restore.
+	if options.Compression == statefile.CompressionLevelNone {
+		pagesFilePath := filepath.Join(imagePath, boot.CheckpointPagesFileName)
+		pf, err := os.OpenFile(pagesFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
+		if err != nil {
+			return fmt.Errorf("creating checkpoint pages file %q: %w", pagesFilePath, err)
+		}
+		defer pf.Close()
+		opt.FilePayload.Files = append(opt.FilePayload.Files, pf)
+		opt.HavePagesFile = true
+	}
+
 	if err := s.call(boot.ContMgrCheckpoint, &opt, nil); err != nil {
 		return fmt.Errorf("checkpointing container %q: %w", cid, err)
 	}

test/runner/main.go

@@ -277,12 +277,6 @@ func prepareSave(args []string, undeclaredOutputsDir string, dirs []string, inde
 	if err != nil {
 		return args, dirs, fmt.Errorf("failed to create state file directory: %v", err)
 	}
-	// Create the state/checkpoint file.
-	fName := filepath.Join(dir, checkpointFile)
-	_, err = os.Create(fName)
-	if err != nil {
-		return args, dirs, fmt.Errorf("failed to create state file: %v", err)
-	}
 	// Pass the directory path of the state file to the sandbox.
 	args = append(args, "-TESTONLY-autosave-image-path", dir)
 	dirs = append(dirs, dir)