From 5e9207a966e5bd2b65f30539e6a95c1908749c85 Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Fri, 12 Apr 2024 09:53:47 -0700 Subject: [PATCH] Create separate pages.img checkpoint file when compression=none. PiperOrigin-RevId: 624210535 --- g3doc/user_guide/checkpoint_restore.md | 12 +++--- pkg/sentry/control/state.go | 23 +++++++++--- pkg/sentry/kernel/kernel.go | 30 ++++++++++----- pkg/sentry/pgalloc/save_restore.go | 8 ++-- pkg/sentry/state/state.go | 13 ++++++- runsc/boot/controller.go | 52 ++++++++++++++++---------- runsc/boot/restore.go | 12 +++++- runsc/cmd/boot.go | 2 +- runsc/cmd/checkpoint.go | 15 +------- runsc/cmd/restore.go | 7 +--- runsc/config/config.go | 3 -- runsc/container/container.go | 19 +++------- runsc/container/container_test.go | 46 ++++------------------- runsc/sandbox/sandbox.go | 50 +++++++++++++++++++++---- test/runner/main.go | 6 --- 15 files changed, 162 insertions(+), 136 deletions(-) diff --git a/g3doc/user_guide/checkpoint_restore.md b/g3doc/user_guide/checkpoint_restore.md index 245470aaa..cb4f12c21 100644 --- a/g3doc/user_guide/checkpoint_restore.md +++ b/g3doc/user_guide/checkpoint_restore.md @@ -15,9 +15,8 @@ runsc run ``` To checkpoint the container, the `--image-path` flag must be provided. This is -the directory path within which the checkpoint state-file will be created. The -file will be called `checkpoint.img` and necessary directories will be created -if they do not yet exist. +the directory path within which the checkpoint related files will be created. +All necessary directories will be created if they do not yet exist. > Note: Two checkpoints cannot be saved to the same directory; every image-path > provided must be unique. @@ -41,9 +40,10 @@ their processes after committing a checkpoint.) runsc checkpoint --image-path= --leave-running ``` -To restore, provide the image path to the `checkpoint.img` file created during -the checkpoint. Because containers stop by default after checkpointing, restore -needs to happen in a new container (restore is a command which parallels start). +To restore, provide the image path to the directory containing all the files +created during the checkpoint. Because containers stop by default after +checkpointing, restore needs to happen in a new container (restore is a command +which parallels start). ```bash runsc create diff --git a/pkg/sentry/control/state.go b/pkg/sentry/control/state.go index 7610648fb..51619ae27 100644 --- a/pkg/sentry/control/state.go +++ b/pkg/sentry/control/state.go @@ -16,6 +16,7 @@ package control import ( "errors" + "fmt" "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/log" @@ -43,7 +44,12 @@ type SaveOpts struct { // Metadata is the set of metadata to prepend to the state file. Metadata map[string]string `json:"metadata"` - // FilePayload contains the destination for the state. + // HavePagesFile indicates whether the checkpoint pages file is provided. + HavePagesFile bool `json:"have_pages_file"` + + // FilePayload contains the following: + // 1. checkpoint state file. + // 2. optional checkpoint pages file. urpc.FilePayload // Resume indicates if the sandbox process should continue running @@ -53,11 +59,13 @@ type SaveOpts struct { // Save saves the running system. func (s *State) Save(o *SaveOpts, _ *struct{}) error { - // Create an output stream. - if len(o.FilePayload.Files) != 1 { - return ErrInvalidFiles + wantFiles := 1 + if o.HavePagesFile { + wantFiles++ + } + if gotFiles := len(o.FilePayload.Files); gotFiles != wantFiles { + return fmt.Errorf("got %d files, wanted %d", gotFiles, wantFiles) } - defer o.FilePayload.Files[0].Close() // Save to the first provided stream. saveOpts := state.SaveOpts{ @@ -77,5 +85,10 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error { } }, } + defer o.FilePayload.Files[0].Close() + if o.HavePagesFile { + saveOpts.PagesFile = o.FilePayload.Files[1] + defer saveOpts.PagesFile.Close() + } return saveOpts.Save(s.Kernel.SupervisorContext(), s.Kernel, s.Watchdog) } diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index 23454d50d..c662b26e3 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -34,6 +34,8 @@ package kernel import ( "errors" "fmt" + "io" + "os" "path/filepath" "time" @@ -524,7 +526,7 @@ type privateMemoryFileMetadata struct { owners []string } -func savePrivateMFs(ctx context.Context, w wire.Writer, mfsToSave map[string]*pgalloc.MemoryFile) error { +func savePrivateMFs(ctx context.Context, w wire.Writer, pw io.Writer, mfsToSave map[string]*pgalloc.MemoryFile) error { var meta privateMemoryFileMetadata // Generate the order in which private memory files are saved. for fsID := range mfsToSave { @@ -536,14 +538,14 @@ func savePrivateMFs(ctx context.Context, w wire.Writer, mfsToSave map[string]*pg } // Followed by the private memory files in order. for _, fsID := range meta.owners { - if err := mfsToSave[fsID].SaveTo(ctx, w); err != nil { + if err := mfsToSave[fsID].SaveTo(ctx, w, pw); err != nil { return err } } return nil } -func loadPrivateMFs(ctx context.Context, r wire.Reader) error { +func loadPrivateMFs(ctx context.Context, r wire.Reader, pr io.Reader) error { // Load the metadata. var meta privateMemoryFileMetadata if _, err := state.Load(ctx, r, &meta); err != nil { @@ -560,7 +562,7 @@ func loadPrivateMFs(ctx context.Context, r wire.Reader) error { if !ok { return fmt.Errorf("saved memory file for %q was not configured on restore", fsID) } - if err := mf.LoadFrom(ctx, r); err != nil { + if err := mf.LoadFrom(ctx, r, pr); err != nil { return err } } @@ -570,7 +572,7 @@ func loadPrivateMFs(ctx context.Context, r wire.Reader) error { // SaveTo saves the state of k to w. // // Preconditions: The kernel must be paused throughout the call to SaveTo. -func (k *Kernel) SaveTo(ctx context.Context, w wire.Writer) error { +func (k *Kernel) SaveTo(ctx context.Context, w wire.Writer, pagesFile *os.File) error { saveStart := time.Now() // Do not allow other Kernel methods to affect it while it's being saved. @@ -638,10 +640,14 @@ func (k *Kernel) SaveTo(ctx context.Context, w wire.Writer) error { // Save the memory files' state. memoryStart := time.Now() - if err := k.mf.SaveTo(ctx, w); err != nil { + pw := io.Writer(w) + if pagesFile != nil { + pw = pagesFile + } + if err := k.mf.SaveTo(ctx, w, pw); err != nil { return err } - if err := savePrivateMFs(ctx, w, mfsToSave); err != nil { + if err := savePrivateMFs(ctx, w, pw, mfsToSave); err != nil { return err } log.Infof("Memory files save took [%s].", time.Since(memoryStart)) @@ -677,7 +683,7 @@ func (k *Kernel) invalidateUnsavableMappings(ctx context.Context) error { } // LoadFrom returns a new Kernel loaded from args. -func (k *Kernel) LoadFrom(ctx context.Context, r wire.Reader, timeReady chan struct{}, net inet.Stack, clocks sentrytime.Clocks, vfsOpts *vfs.CompleteRestoreOptions) error { +func (k *Kernel) LoadFrom(ctx context.Context, r wire.Reader, pagesFile *os.File, timeReady chan struct{}, net inet.Stack, clocks sentrytime.Clocks, vfsOpts *vfs.CompleteRestoreOptions) error { loadStart := time.Now() k.runningTasksCond.L = &k.runningTasksMu @@ -719,10 +725,14 @@ func (k *Kernel) LoadFrom(ctx context.Context, r wire.Reader, timeReady chan str // Load the memory files' state. memoryStart := time.Now() - if err := k.mf.LoadFrom(ctx, r); err != nil { + pr := io.Reader(r) + if pagesFile != nil { + pr = pagesFile + } + if err := k.mf.LoadFrom(ctx, r, pr); err != nil { return err } - if err := loadPrivateMFs(ctx, r); err != nil { + if err := loadPrivateMFs(ctx, r, pr); err != nil { return err } log.Infof("Memory files load took [%s].", time.Since(memoryStart)) diff --git a/pkg/sentry/pgalloc/save_restore.go b/pkg/sentry/pgalloc/save_restore.go index c9b3878b6..d0909dc2d 100644 --- a/pkg/sentry/pgalloc/save_restore.go +++ b/pkg/sentry/pgalloc/save_restore.go @@ -31,7 +31,7 @@ import ( ) // SaveTo writes f's state to the given stream. -func (f *MemoryFile) SaveTo(ctx context.Context, w wire.Writer) error { +func (f *MemoryFile) SaveTo(ctx context.Context, w wire.Writer, pw io.Writer) error { // Wait for reclaim. f.mu.Lock() defer f.mu.Unlock() @@ -102,7 +102,7 @@ func (f *MemoryFile) SaveTo(ctx context.Context, w wire.Writer) error { if ioErr != nil { return } - _, ioErr = w.Write(s) + _, ioErr = pw.Write(s) }) if ioErr != nil { return ioErr @@ -135,7 +135,7 @@ func (f *MemoryFile) RestoreID() string { } // LoadFrom loads MemoryFile state from the given stream. -func (f *MemoryFile) LoadFrom(ctx context.Context, r wire.Reader) error { +func (f *MemoryFile) LoadFrom(ctx context.Context, r wire.Reader, pr io.Reader) error { // Load metadata. if _, err := state.Load(ctx, r, &f.fileSize); err != nil { return err @@ -196,7 +196,7 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r wire.Reader) error { if ioErr != nil { return } - _, ioErr = io.ReadFull(r, s) + _, ioErr = io.ReadFull(pr, s) }) if ioErr != nil { return ioErr diff --git a/pkg/sentry/state/state.go b/pkg/sentry/state/state.go index b2baad6ef..85b6afae1 100644 --- a/pkg/sentry/state/state.go +++ b/pkg/sentry/state/state.go @@ -18,6 +18,7 @@ package state import ( "fmt" "io" + "os" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" @@ -48,6 +49,10 @@ type SaveOpts struct { // Destination is the save target. Destination io.Writer + // PagesFile is the file in which all MemoryFile pages are stored if + // PagesFile is non-nil. + PagesFile *os.File + // Key is used for state integrity check. Key []byte @@ -86,7 +91,7 @@ func (opts SaveOpts) Save(ctx context.Context, k *kernel.Kernel, w *watchdog.Wat err = ErrStateFile{err} } else { // Save the kernel. - err = k.SaveTo(ctx, wc) + err = k.SaveTo(ctx, wc, opts.PagesFile) // ENOSPC is a state file error. This error can only come from // writing the state file, and not from fs.FileOperations.Fsync @@ -108,6 +113,10 @@ type LoadOpts struct { // Destination is the load source. Source io.Reader + // PagesFile is the file in which all MemoryFile pages are stored if + // PagesFile is non-nil. + PagesFile *os.File + // Key is used for state integrity check. Key []byte } @@ -123,5 +132,5 @@ func (opts LoadOpts) Load(ctx context.Context, k *kernel.Kernel, timeReady chan previousMetadata = m // Restore the Kernel object graph. - return k.LoadFrom(ctx, r, timeReady, n, clocks, vfsOpts) + return k.LoadFrom(ctx, r, opts.PagesFile, timeReady, n, clocks, vfsOpts) } diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index ca5e8bf1b..bd587a268 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -443,12 +443,13 @@ func (cm *containerManager) PortForward(opts *PortForwardOpts, _ *struct{}) erro // RestoreOpts contains options related to restoring a container's file system. type RestoreOpts struct { - // FilePayload contains the state file to be restored, followed by the - // platform device file if necessary. + // FilePayload contains the state file to be restored, followed in order by: + // 1. checkpoint state file. + // 2. optional checkpoint pages file. + // 3. optional platform device file. urpc.FilePayload - - // SandboxID contains the ID of the sandbox. - SandboxID string + HavePagesFile bool + HaveDeviceFile bool } // Restore loads a container from a statefile. @@ -458,29 +459,40 @@ type RestoreOpts struct { func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { log.Debugf("containerManager.Restore") + if len(o.Files) == 0 { + return fmt.Errorf("at least one file must be passed to Restore") + } + fileIdx := 0 + r := restorer{container: &cm.l.root} - switch numFiles := len(o.Files); numFiles { - case 2: + r.stateFile = o.Files[fileIdx] + fileIdx++ + defer r.stateFile.Close() + if info, err := r.stateFile.Stat(); err != nil { + return err + } else if info.Size() == 0 { + return fmt.Errorf("statefile cannot be empty") + } + + if o.HavePagesFile { + r.pagesFile = o.Files[fileIdx] + fileIdx++ + defer r.pagesFile.Close() + } + + if o.HaveDeviceFile { // The device file is donated to the platform. // Can't take ownership away from os.File. dup them to get a new FD. - fd, err := unix.Dup(int(o.Files[1].Fd())) + fd, err := unix.Dup(int(o.Files[fileIdx].Fd())) if err != nil { return fmt.Errorf("failed to dup file: %v", err) } r.deviceFile = os.NewFile(uintptr(fd), "platform device") - fallthrough - case 1: - r.stateFile = o.Files[0] - if info, err := r.stateFile.Stat(); err != nil { - return err - } else if info.Size() == 0 { - return fmt.Errorf("file cannot be empty") - } + fileIdx++ + } - case 0: - return fmt.Errorf("at least one file must be passed to Restore") - default: - return fmt.Errorf("at most two files may be passed to Restore") + if fileIdx < len(o.Files) { + return fmt.Errorf("more files passed to Restore than expected") } // Pause the kernel while we build a new one. diff --git a/runsc/boot/restore.go b/runsc/boot/restore.go index e74623fb1..3ff1c7443 100644 --- a/runsc/boot/restore.go +++ b/runsc/boot/restore.go @@ -35,9 +35,19 @@ import ( "gvisor.dev/gvisor/runsc/boot/pprof" ) +const ( + // CheckpointStateFileName is the file within the given image-path's + // directory which contains the container's saved state. + CheckpointStateFileName = "checkpoint.img" + // CheckpointPagesFileName is the file within the given image-path's + // directory containing the container's MemoryFile pages. + CheckpointPagesFileName = "pages.img" +) + type restorer struct { container *containerInfo stateFile *os.File + pagesFile *os.File deviceFile *os.File } @@ -135,7 +145,7 @@ func (r *restorer) restore(l *Loader) error { ctx = context.WithValue(ctx, pgalloc.CtxMemoryFileMap, mfmap) // Load the state. - loadOpts := state.LoadOpts{Source: r.stateFile} + loadOpts := state.LoadOpts{Source: r.stateFile, PagesFile: r.pagesFile} if err := loadOpts.Load(ctx, l.k, nil, netns.Stack(), time.NewCalibratedClocks(), &vfs.CompleteRestoreOptions{}); err != nil { return err } diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 5b171fa18..6c2ea6b31 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -462,7 +462,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } if conf.TestOnlyAutosaveImagePath != "" { - fName := filepath.Join(conf.TestOnlyAutosaveImagePath, checkpointFileName) + fName := filepath.Join(conf.TestOnlyAutosaveImagePath, boot.CheckpointStateFileName) f, err := os.OpenFile(fName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { util.Fatalf("error in creating state file %v", err) diff --git a/runsc/cmd/checkpoint.go b/runsc/cmd/checkpoint.go index 32f68384c..c25e4bdca 100644 --- a/runsc/cmd/checkpoint.go +++ b/runsc/cmd/checkpoint.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "os" - "path/filepath" "github.com/google/subcommands" "gvisor.dev/gvisor/pkg/state/statefile" @@ -28,9 +27,6 @@ import ( "gvisor.dev/gvisor/runsc/flag" ) -// File containing the container's saved image/state within the given image-path's directory. -const checkpointFileName = "checkpoint.img" - // Checkpoint implements subcommands.Command for the "checkpoint" command. type Checkpoint struct { imagePath string @@ -88,15 +84,6 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su util.Fatalf("making directories at path provided: %v", err) } - fullImagePath := filepath.Join(c.imagePath, checkpointFileName) - - // Create the image file and open for writing. - file, err := os.OpenFile(fullImagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - util.Fatalf("os.OpenFile(%q) failed: %v", fullImagePath, err) - } - defer file.Close() - sOpts := statefile.Options{Compression: c.compression.Level()} if c.leaveRunning { @@ -104,7 +91,7 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su sOpts.Resume = true } - if err := cont.Checkpoint(file, sOpts); err != nil { + if err := cont.Checkpoint(c.imagePath, sOpts); err != nil { util.Fatalf("checkpoint failed: %v", err) } diff --git a/runsc/cmd/restore.go b/runsc/cmd/restore.go index 475f8e63a..5ee11b480 100644 --- a/runsc/cmd/restore.go +++ b/runsc/cmd/restore.go @@ -17,7 +17,6 @@ package cmd import ( "context" "os" - "path/filepath" "github.com/google/subcommands" "golang.org/x/sys/unix" @@ -99,8 +98,6 @@ func (r *Restore) Execute(_ context.Context, f *flag.FlagSet, args ...any) subco var cu cleanup.Cleanup defer cu.Clean() - conf.RestoreFile = filepath.Join(r.imagePath, checkpointFileName) - runArgs := container.Args{ ID: id, Spec: nil, @@ -140,8 +137,8 @@ func (r *Restore) Execute(_ context.Context, f *flag.FlagSet, args ...any) subco runArgs.Spec = c.Spec } - log.Debugf("Restore: %v", conf.RestoreFile) - if err := c.Restore(conf, conf.RestoreFile); err != nil { + log.Debugf("Restore: %v", r.imagePath) + if err := c.Restore(conf, r.imagePath); err != nil { return util.Errorf("starting container: %v", err) } diff --git a/runsc/config/config.go b/runsc/config/config.go index 0d76dfb53..acd6c70b8 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -234,9 +234,6 @@ type Config struct { // for the duration of the container execution. TraceFile string `flag:"trace"` - // RestoreFile is the path to the saved container image. - RestoreFile string - // NumNetworkChannels controls the number of AF_PACKET sockets that map // to the same underlying network device. This allows netstack to better // scale for high throughput use cases. diff --git a/runsc/container/container.go b/runsc/container/container.go index 01f330116..a81dc9f13 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -525,7 +525,7 @@ func (c *Container) Start(conf *config.Config) error { // Restore takes a container and replaces its kernel and file system // to restore a container from its state file. -func (c *Container) Restore(conf *config.Config, restoreFile string) error { +func (c *Container) Restore(conf *config.Config, imagePath string) error { log.Debugf("Restore container, cid: %s", c.ID) if err := c.Saver.lock(BlockAcquire); err != nil { return err @@ -542,7 +542,7 @@ func (c *Container) Restore(conf *config.Config, restoreFile string) error { log.Warningf("StartContainer hook skipped because running inside container namespace is not supported") } - if err := c.Sandbox.Restore(conf, c.ID, restoreFile); err != nil { + if err := c.Sandbox.Restore(conf, c.ID, imagePath); err != nil { return err } c.changeStatus(Running) @@ -563,15 +563,8 @@ func Run(conf *config.Config, args Args) (unix.WaitStatus, error) { }) defer cu.Clean() - if conf.RestoreFile != "" { - log.Debugf("Restore: %v", conf.RestoreFile) - if err := c.Restore(conf, conf.RestoreFile); err != nil { - return 0, fmt.Errorf("starting container: %v", err) - } - } else { - if err := c.Start(conf); err != nil { - return 0, fmt.Errorf("starting container: %v", err) - } + if err := c.Start(conf); err != nil { + return 0, fmt.Errorf("starting container: %v", err) } // If we allocate a terminal, forward signals to the sandbox process. @@ -721,12 +714,12 @@ func (c *Container) ForwardSignals(pid int32, fgProcess bool) func() { // Checkpoint sends the checkpoint call to the container. // The statefile will be written to f, the file at the specified image-path. -func (c *Container) Checkpoint(f *os.File, options statefile.Options) error { +func (c *Container) Checkpoint(imagePath string, options statefile.Options) error { log.Debugf("Checkpoint container, cid: %s", c.ID) if err := c.requireStatus("checkpoint", Created, Running, Paused); err != nil { return err } - return c.Sandbox.Checkpoint(c.ID, f, options) + return c.Sandbox.Checkpoint(c.ID, imagePath, options) } // Pause suspends the container and its kernel. diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index adbe5fa5d..61bf71a44 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -1063,26 +1063,15 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript t.Fatalf("error starting container: %v", err) } - // Set the image path, which is where the checkpoint image will be saved. - imagePath := filepath.Join(dir, "test-image-file") - - // Create the image file and open for writing. - file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - t.Fatalf("error opening new file at imagePath: %v", err) - } - defer file.Close() - // Wait until application has ran. if err := waitForFileNotEmpty(outputFile); err != nil { t.Fatalf("Failed to wait for output file: %v", err) } // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } - defer os.RemoveAll(imagePath) lastNum, err := readOutputNum(outputPath, -1) if err != nil { @@ -1112,7 +1101,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript } defer cont2.Destroy() - if err := cont2.Restore(conf, imagePath); err != nil { + if err := cont2.Restore(conf, dir); err != nil { t.Fatalf("error restoring container: %v", err) } @@ -1155,7 +1144,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, newSpecWithScript } defer cont3.Destroy() - if err := cont3.Restore(conf, imagePath); err != nil { + if err := cont3.Restore(conf, dir); err != nil { t.Fatalf("error restoring container: %v", err) } @@ -1253,16 +1242,8 @@ func TestCheckpointRestoreExecKilled(t *testing.T) { t.Fatalf("error chmoding file: %q, %v", dir, err) } - // Create the image file and open for writing. - checkpointPath := filepath.Join(dir, "test-image-file") - checkpointFile, err := os.OpenFile(checkpointPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - t.Fatalf("error opening new file at imagePath: %v", err) - } - defer checkpointFile.Close() - - // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(checkpointFile, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + // Checkpoint running container. + if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { t.Fatalf("error checkpointing container: %v", err) } cont.Destroy() @@ -1274,7 +1255,7 @@ func TestCheckpointRestoreExecKilled(t *testing.T) { } defer cont2.Destroy() - if err := cont2.Restore(conf, checkpointPath); err != nil { + if err := cont2.Restore(conf, dir); err != nil { t.Fatalf("error restoring container: %v", err) } @@ -1350,24 +1331,13 @@ func TestUnixDomainSockets(t *testing.T) { t.Fatalf("error starting container: %v", err) } - // Set the image path, the location where the checkpoint image will be saved. - imagePath := filepath.Join(dir, "test-image-file") - - // Create the image file and open for writing. - file, err := os.OpenFile(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - t.Fatalf("error opening new file at imagePath: %v", err) - } - defer file.Close() - defer os.RemoveAll(imagePath) - // Wait until application has ran. if err := waitForFileNotEmpty(outputFile); err != nil { t.Fatalf("Failed to wait for output file: %v", err) } // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(file, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } @@ -1399,7 +1369,7 @@ func TestUnixDomainSockets(t *testing.T) { } defer contRestore.Destroy() - if err := contRestore.Restore(conf, imagePath); err != nil { + if err := contRestore.Restore(conf, dir); err != nil { t.Fatalf("error restoring container: %v", err) } diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index d175187c7..dcf0e2ac7 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -24,6 +24,7 @@ import ( "math" "os" "os/exec" + "path" "path/filepath" "strconv" "strings" @@ -441,20 +442,30 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s } // Restore sends the restore call for a container in the sandbox. -func (s *Sandbox) Restore(conf *config.Config, cid string, filename string) error { +func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string) error { log.Debugf("Restore sandbox %q", s.ID) - rf, err := os.Open(filename) + stateFileName := path.Join(imagePath, boot.CheckpointStateFileName) + sf, err := os.Open(stateFileName) if err != nil { - return fmt.Errorf("opening restore file %q failed: %v", filename, err) + return fmt.Errorf("opening state file %q failed: %v", stateFileName, err) } - defer rf.Close() + defer sf.Close() opt := boot.RestoreOpts{ FilePayload: urpc.FilePayload{ - Files: []*os.File{rf}, + Files: []*os.File{sf}, }, - SandboxID: s.ID, + } + + // If the image file exists, we must pass it in. + pagesFileName := path.Join(imagePath, boot.CheckpointPagesFileName) + if pf, err := os.Open(pagesFileName); err == nil { + defer pf.Close() + opt.HavePagesFile = true + opt.FilePayload.Files = append(opt.FilePayload.Files, pf) + } else if !os.IsNotExist(err) { + return fmt.Errorf("opening restore image file %q failed: %v", pagesFileName, err) } // If the platform needs a device FD we must pass it in. @@ -462,6 +473,7 @@ func (s *Sandbox) Restore(conf *config.Config, cid string, filename string) erro return err } else if deviceFile != nil { defer deviceFile.Close() + opt.HaveDeviceFile = true opt.FilePayload.Files = append(opt.FilePayload.Files, deviceFile) } @@ -1256,16 +1268,38 @@ func (s *Sandbox) SignalProcess(cid string, pid int32, sig unix.Signal, fgProces // Checkpoint sends the checkpoint call for a container in the sandbox. // The statefile will be written to f. -func (s *Sandbox) Checkpoint(cid string, f *os.File, options statefile.Options) error { +func (s *Sandbox) Checkpoint(cid string, imagePath string, options statefile.Options) error { log.Debugf("Checkpoint sandbox %q, options %+v", s.ID, options) + + stateFilePath := filepath.Join(imagePath, boot.CheckpointStateFileName) + sf, err := os.OpenFile(stateFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + return fmt.Errorf("creating checkpoint state file %q: %w", stateFilePath, err) + } + defer sf.Close() + opt := control.SaveOpts{ Metadata: options.WriteToMetadata(map[string]string{}), FilePayload: urpc.FilePayload{ - Files: []*os.File{f}, + Files: []*os.File{sf}, }, Resume: options.Resume, } + // When there is no compression, MemoryFile contents are page-aligned. + // It is beneficial to store them separately so certain optimizations can be + // applied during restore. + if options.Compression == statefile.CompressionLevelNone { + pagesFilePath := filepath.Join(imagePath, boot.CheckpointPagesFileName) + pf, err := os.OpenFile(pagesFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + return fmt.Errorf("creating checkpoint pages file %q: %w", pagesFilePath, err) + } + defer pf.Close() + opt.FilePayload.Files = append(opt.FilePayload.Files, pf) + opt.HavePagesFile = true + } + if err := s.call(boot.ContMgrCheckpoint, &opt, nil); err != nil { return fmt.Errorf("checkpointing container %q: %w", cid, err) } diff --git a/test/runner/main.go b/test/runner/main.go index 3415d3d4e..4ead8f65f 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -277,12 +277,6 @@ func prepareSave(args []string, undeclaredOutputsDir string, dirs []string, inde if err != nil { return args, dirs, fmt.Errorf("failed to create state file directory: %v", err) } - // Create the state/checkpoint file. - fName := filepath.Join(dir, checkpointFile) - _, err = os.Create(fName) - if err != nil { - return args, dirs, fmt.Errorf("failed to create state file: %v", err) - } // Pass the directory path of the state file to the sandbox. args = append(args, "-TESTONLY-autosave-image-path", dir) dirs = append(dirs, dir)