From 032b103ec7ca742837e0cb17058456bb33d7089b Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Sat, 18 May 2024 21:51:52 -0700 Subject: [PATCH] Add checkpoint --direct flag to allow pages file to be written using O_DIRECT. Compliments the `restore --direct` flag added in f895b63b04fa ("Use O_DIRECT to read pages.img.") It helps bypass the kernel page cache. It is beneficial if the checkpoint files are not expected to be read again on this host. For example, if the checkpoint files will be stored on a network block device, which will be detached after the checkpoint is done. PiperOrigin-RevId: 635155071 --- runsc/cmd/checkpoint.go | 10 +++++++++- runsc/container/container.go | 4 ++-- runsc/container/container_test.go | 6 +++--- runsc/container/multi_container_test.go | 4 ++-- runsc/sandbox/sandbox.go | 10 +++++++--- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/runsc/cmd/checkpoint.go b/runsc/cmd/checkpoint.go index 92d943544..6d28f6e45 100644 --- a/runsc/cmd/checkpoint.go +++ b/runsc/cmd/checkpoint.go @@ -34,6 +34,13 @@ type Checkpoint struct { leaveRunning bool compression CheckpointCompression excludeCommittedZeroPages bool + + // direct indicates whether O_DIRECT should be used for writing the + // checkpoint pages file. It bypasses the kernel page cache. It is beneficial + // if the checkpoint files are not expected to be read again on this host. + // For example, if the checkpoint files will be stored on a network block + // device, which will be detached after the checkpoint is done. + direct bool } // Name implements subcommands.Command.Name. @@ -58,6 +65,7 @@ func (c *Checkpoint) SetFlags(f *flag.FlagSet) { f.BoolVar(&c.leaveRunning, "leave-running", false, "restart the container after checkpointing") f.Var(newCheckpointCompressionValue(statefile.CompressionLevelDefault, &c.compression), "compression", "compress checkpoint image on disk. Values: none|flate-best-speed.") f.BoolVar(&c.excludeCommittedZeroPages, "exclude-committed-zero-pages", false, "exclude committed zero-filled pages from checkpoint") + f.BoolVar(&c.direct, "direct", false, "use O_DIRECT for writing checkpoint pages file") // Unimplemented flags necessary for compatibility with docker. var wp string @@ -99,7 +107,7 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su sOpts.Resume = true } - if err := cont.Checkpoint(c.imagePath, sOpts, mfOpts); err != nil { + if err := cont.Checkpoint(c.imagePath, c.direct, sOpts, mfOpts); err != nil { util.Fatalf("checkpoint failed: %v", err) } diff --git a/runsc/container/container.go b/runsc/container/container.go index 430745d78..32d0022a5 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -702,12 +702,12 @@ func (c *Container) ForwardSignals(pid int32, fgProcess bool) func() { // Checkpoint sends the checkpoint call to the container. // The statefile will be written to f, the file at the specified image-path. -func (c *Container) Checkpoint(imagePath string, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { +func (c *Container) Checkpoint(imagePath string, direct bool, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { log.Debugf("Checkpoint container, cid: %s", c.ID) if err := c.requireStatus("checkpoint", Created, Running, Paused); err != nil { return err } - return c.Sandbox.Checkpoint(c.ID, imagePath, sfOpts, mfOpts) + return c.Sandbox.Checkpoint(c.ID, imagePath, direct, sfOpts, mfOpts) } // Pause suspends the container and its kernel. diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 275f3f7e2..bdebd8c0f 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -1071,7 +1071,7 @@ func testCheckpointRestore(t *testing.T, conf *config.Config, compression statef } // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(dir, statefile.Options{Compression: compression}, pgalloc.SaveOpts{}); err != nil { + if err := cont.Checkpoint(dir, false /* direct */, statefile.Options{Compression: compression}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } @@ -1252,7 +1252,7 @@ func TestCheckpointRestoreExecKilled(t *testing.T) { } // Checkpoint running container. - if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { + if err := cont.Checkpoint(dir, false /* direct */, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container: %v", err) } cont.Destroy() @@ -1346,7 +1346,7 @@ func TestUnixDomainSockets(t *testing.T) { } // Checkpoint running container; save state into new file. - if err := cont.Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelDefault}, pgalloc.SaveOpts{}); err != nil { + if err := cont.Checkpoint(dir, false /* direct */, statefile.Options{Compression: statefile.CompressionLevelDefault}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index 4c9154f6b..ddeab1096 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -131,7 +131,7 @@ func restoreContainers(conf *config.Config, specs []*specs.Spec, ids []string, i cu.Add(func() { cont.Destroy() }) containers = append(containers, cont) - if err := cont.Restore(conf, imagePath, false); err != nil { + if err := cont.Restore(conf, imagePath, false /* direct */); err != nil { return nil, nil, fmt.Errorf("error restoring container: %v", err) } @@ -2747,7 +2747,7 @@ func TestMultiContainerCheckpointRestore(t *testing.T) { } // Checkpoint root container; save state into new file. - if err := conts[0].Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { + if err := conts[0].Checkpoint(dir, false /* direct */, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}, pgalloc.SaveOpts{}); err != nil { t.Fatalf("error checkpointing container to empty file: %v", err) } defer os.RemoveAll(dir) diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 83b207723..2788ca518 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -1325,7 +1325,7 @@ func (s *Sandbox) SignalProcess(cid string, pid int32, sig unix.Signal, fgProces // Checkpoint sends the checkpoint call for a container in the sandbox. // The statefile will be written to f. -func (s *Sandbox) Checkpoint(cid string, imagePath string, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { +func (s *Sandbox) Checkpoint(cid string, imagePath string, direct bool, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { log.Debugf("Checkpoint sandbox %q, statefile options %+v, MemoryFile options %+v", s.ID, sfOpts, mfOpts) stateFilePath := filepath.Join(imagePath, boot.CheckpointStateFileName) @@ -1349,8 +1349,12 @@ func (s *Sandbox) Checkpoint(cid string, imagePath string, sfOpts statefile.Opti // applied during restore. See Restore(). if sfOpts.Compression == statefile.CompressionLevelNone { pagesFilePath := filepath.Join(imagePath, boot.CheckpointPagesFileName) - // TODO(b/327603247): Implement optional async O_DIRECT write. - pf, err := os.OpenFile(pagesFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + pagesWriteFlags := os.O_CREATE | os.O_EXCL | os.O_RDWR + if direct { + // The writes will be page-aligned, so it can be opened with O_DIRECT. + pagesWriteFlags |= syscall.O_DIRECT + } + pf, err := os.OpenFile(pagesFilePath, pagesWriteFlags, 0644) if err != nil { return fmt.Errorf("creating checkpoint pages file %q: %w", pagesFilePath, err) }