diff --git a/runsc/cmd/checkpoint.go b/runsc/cmd/checkpoint.go index 92d943544..6d28f6e45 100644 --- a/runsc/cmd/checkpoint.go +++ b/runsc/cmd/checkpoint.go @@ -34,6 +34,13 @@ type Checkpoint struct { leaveRunning bool compression CheckpointCompression excludeCommittedZeroPages bool + + // direct indicates whether O_DIRECT should be used for writing the + // checkpoint pages file. It bypasses the kernel page cache. It is beneficial + // if the checkpoint files are not expected to be read again on this host. + // For example, if the checkpoint files will be stored on a network block + // device, which will be detached after the checkpoint is done. + direct bool } // Name implements subcommands.Command.Name. @@ -58,6 +65,7 @@ func (c *Checkpoint) SetFlags(f *flag.FlagSet) { f.BoolVar(&c.leaveRunning, "leave-running", false, "restart the container after checkpointing") f.Var(newCheckpointCompressionValue(statefile.CompressionLevelDefault, &c.compression), "compression", "compress checkpoint image on disk. Values: none|flate-best-speed.") f.BoolVar(&c.excludeCommittedZeroPages, "exclude-committed-zero-pages", false, "exclude committed zero-filled pages from checkpoint") + f.BoolVar(&c.direct, "direct", false, "use O_DIRECT for writing checkpoint pages file") // Unimplemented flags necessary for compatibility with docker. var wp string @@ -99,7 +107,7 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su sOpts.Resume = true } - if err := cont.Checkpoint(c.imagePath, sOpts, mfOpts); err != nil { + if err := cont.Checkpoint(c.imagePath, c.direct, sOpts, mfOpts); err != nil { util.Fatalf("checkpoint failed: %v", err) } diff --git a/runsc/container/container.go b/runsc/container/container.go index 430745d78..32d0022a5 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -702,12 +702,12 @@ func (c *Container) ForwardSignals(pid int32, fgProcess bool) func() { // Checkpoint sends the checkpoint call to the container. // The statefile will be written to f, the file at the specified image-path. -func (c *Container) Checkpoint(imagePath string, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { +func (c *Container) Checkpoint(imagePath string, direct bool, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { log.Debugf("Checkpoint container, cid: %s", c.ID) if err := c.requireStatus("checkpoint", Created, Running, Paused); err != nil { return err } - return c.Sandbox.Checkpoint(c.ID, imagePath, sfOpts, mfOpts) + return c.Sandbox.Checkpoint(c.ID, imagePath, direct, sfOpts, mfOpts) } // Pause suspends the container and its kernel. diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 275f3f7e2..bdebd8c0f 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -1071,7 +1071,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, compression statef } // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(dir, statefile.Options{Compression: compression}, pgalloc.SaveOpts{}); err != nil { + if err := cont.Checkpoint(dir, false /* direct */, statefile.Options{Compression: compression}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } @@ -1252,7 +1252,7 @@ func TestCheckpointRestoreExecKilled(t *testing.T) { } // Checkpoint running container. - if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { + if err := cont.Checkpoint(dir, false /* direct */, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container: %v", err) } cont.Destroy() @@ -1346,7 +1346,7 @@ func TestUnixDomainSockets(t *testing.T) { } // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelDefault}, pgalloc.SaveOpts{}); err != nil { + if err := cont.Checkpoint(dir, false /* direct */, statefile.Options{Compression: statefile.CompressionLevelDefault}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index 4c9154f6b..ddeab1096 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -131,7 +131,7 @@ func restoreContainers(conf *config.Config, specs []*specs.Spec, ids []string, i cu.Add(func() { cont.Destroy() }) containers = append(containers, cont) - if err := cont.Restore(conf, imagePath, false); err != nil { + if err := cont.Restore(conf, imagePath, false /* direct */); err != nil { return nil, nil, fmt.Errorf("error restoring container: %v", err) } @@ -2747,7 +2747,7 @@ func TestMultiContainerCheckpointRestore(t *testing.T) { } // Checkpoint root container; save state into new file. - if err := conts[0].Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { + if err := conts[0].Checkpoint(dir, false /* direct */, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } defer os.RemoveAll(dir) diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 83b207723..2788ca518 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -1325,7 +1325,7 @@ func (s *Sandbox) SignalProcess(cid string, pid int32, sig unix.Signal, fgProces // Checkpoint sends the checkpoint call for a container in the sandbox. // The statefile will be written to f. -func (s *Sandbox) Checkpoint(cid string, imagePath string, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { +func (s *Sandbox) Checkpoint(cid string, imagePath string, direct bool, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { log.Debugf("Checkpoint sandbox %q, statefile options %+v, MemoryFile options %+v", s.ID, sfOpts, mfOpts) stateFilePath := filepath.Join(imagePath, boot.CheckpointStateFileName) @@ -1349,8 +1349,12 @@ func (s *Sandbox) Checkpoint(cid string, imagePath string, sfOpts statefile.Opti // applied during restore. See Restore(). if sfOpts.Compression == statefile.CompressionLevelNone { pagesFilePath := filepath.Join(imagePath, boot.CheckpointPagesFileName) - // TODO(b/327603247): Implement optional async O_DIRECT write. - pf, err := os.OpenFile(pagesFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + pagesWriteFlags := os.O_CREATE | os.O_EXCL | os.O_RDWR + if direct { + // The writes will be page-aligned, so it can be opened with O_DIRECT. + pagesWriteFlags |= syscall.O_DIRECT + } + pf, err := os.OpenFile(pagesFilePath, pagesWriteFlags, 0644) if err != nil { return fmt.Errorf("creating checkpoint pages file %q: %w", pagesFilePath, err) }