From 87d8df37c71eb60f44d833400445830166fa0ad0 Mon Sep 17 00:00:00 2001 From: Nayana Bidari Date: Tue, 9 Apr 2024 14:31:12 -0700 Subject: [PATCH] Enable save/checkpoint resume with runsc checkpoint command. Enables save resume with checkpoint command. Previously when --leave-running was set, the sandbox was destroyed after the checkpoint and restored with the same id. With this change the sandbox will not be destroyed and resumes running after the checkpoint. PiperOrigin-RevId: 623282685 --- pkg/sentry/control/state.go | 8 ++- pkg/state/statefile/statefile.go | 4 ++ pkg/test/dockerutil/container.go | 5 ++ runsc/cmd/checkpoint.go | 66 +++----------------- runsc/sandbox/sandbox.go | 1 + test/e2e/integration_test.go | 102 +++++++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 59 deletions(-) diff --git a/pkg/sentry/control/state.go b/pkg/sentry/control/state.go index 4c83b8e8e..7610648fb 100644 --- a/pkg/sentry/control/state.go +++ b/pkg/sentry/control/state.go @@ -45,6 +45,10 @@ type SaveOpts struct { // FilePayload contains the destination for the state. urpc.FilePayload + + // Resume indicates if the sandbox process should continue running + // after checkpointing. + Resume bool } // Save saves the running system. @@ -68,7 +72,9 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error { log.Warningf("Save failed: exiting...") s.Kernel.SetSaveError(err) } - s.Kernel.Kill(linux.WaitStatusExit(0)) + if !o.Resume { + s.Kernel.Kill(linux.WaitStatusExit(0)) + } }, } return saveOpts.Save(s.Kernel.SupervisorContext(), s.Kernel, s.Watchdog) diff --git a/pkg/state/statefile/statefile.go b/pkg/state/statefile/statefile.go index 93fe63ccc..66d2990bd 100644 --- a/pkg/state/statefile/statefile.go +++ b/pkg/state/statefile/statefile.go @@ -105,6 +105,10 @@ const ( type Options struct { // Compression is an image compression type/level. Compression CompressionLevel + + // Resume indicates if the sandbox process should continue running + // after checkpointing. + Resume bool } // WriteToMetadata save options to the metadata storage. Method returns the diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 077a73057..9230a0140 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -333,6 +333,11 @@ func (c *Container) Restore(ctx context.Context, name string) error { return c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{CheckpointID: name}) } +// CheckpointResume is analogous to 'docker checkpoint'. +func (c *Container) CheckpointResume(ctx context.Context, name string) error { + return c.client.CheckpointCreate(ctx, c.Name, types.CheckpointCreateOptions{CheckpointID: name, Exit: false}) +} + // Logs is analogous 'docker logs'. func (c *Container) Logs(ctx context.Context) (string, error) { var out bytes.Buffer diff --git a/runsc/cmd/checkpoint.go b/runsc/cmd/checkpoint.go index e7590177c..32f68384c 100644 --- a/runsc/cmd/checkpoint.go +++ b/runsc/cmd/checkpoint.go @@ -21,14 +21,11 @@ import ( "path/filepath" "github.com/google/subcommands" - "golang.org/x/sys/unix" - "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/runsc/cmd/util" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/container" "gvisor.dev/gvisor/runsc/flag" - "gvisor.dev/gvisor/runsc/specutils" ) // File containing the container's saved image/state within the given image-path's directory. @@ -77,7 +74,6 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su id := f.Arg(0) conf := args[0].(*config.Config) - waitStatus := args[1].(*unix.WaitStatus) cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{}) if err != nil { @@ -101,63 +97,17 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su } defer file.Close() - if err := cont.Checkpoint(file, statefile.Options{Compression: c.compression.Level()}); err != nil { + sOpts := statefile.Options{Compression: c.compression.Level()} + + if c.leaveRunning { + // Do not destroy the sandbox after saving. + sOpts.Resume = true + } + + if err := cont.Checkpoint(file, sOpts); err != nil { util.Fatalf("checkpoint failed: %v", err) } - if !c.leaveRunning { - return subcommands.ExitSuccess - } - - // TODO(b/110843694): Make it possible to restore into same container. - // For now, we can fake it by destroying the container and making a - // new container with the same ID. This hack does not work with docker - // which uses the container pid to ensure that the restore-container is - // actually the same as the checkpoint-container. By restoring into - // the same container, we will solve the docker incompatibility. - - // Restore into new container with same ID. - bundleDir := cont.BundleDir - if bundleDir == "" { - util.Fatalf("setting bundleDir") - } - - spec, err := specutils.ReadSpec(bundleDir, conf) - if err != nil { - util.Fatalf("reading spec: %v", err) - } - - specutils.LogSpecDebug(spec, conf.OCISeccomp) - - if cont.ConsoleSocket != "" { - log.Warningf("ignoring console socket since it cannot be restored") - } - - if err := cont.Destroy(); err != nil { - util.Fatalf("destroying container: %v", err) - } - - contArgs := container.Args{ - ID: id, - Spec: spec, - BundleDir: bundleDir, - } - cont, err = container.New(conf, contArgs) - if err != nil { - util.Fatalf("restoring container: %v", err) - } - defer cont.Destroy() - - if err := cont.Restore(conf, fullImagePath); err != nil { - util.Fatalf("starting container: %v", err) - } - - ws, err := cont.Wait() - if err != nil { - util.Fatalf("Error waiting for container: %v", err) - } - *waitStatus = ws - return subcommands.ExitSuccess } diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 7a8bb3fbd..07864ccfd 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -1263,6 +1263,7 @@ func (s *Sandbox) Checkpoint(cid string, f *os.File, options statefile.Options) FilePayload: urpc.FilePayload{ Files: []*os.File{f}, }, + Resume: options.Resume, } if err := s.call(boot.ContMgrCheckpoint, &opt, nil); err != nil { diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go index 5b391f847..7bfdbb034 100644 --- a/test/e2e/integration_test.go +++ b/test/e2e/integration_test.go @@ -1110,3 +1110,105 @@ func TestBlockHostUds(t *testing.T) { t.Errorf("err should be non-nil and output should contain %q, but got err = %v and output = %q", want, err, got) } } + +func readLogs(logs string, position int) (int, error) { + if len(logs) == 0 { + return 0, fmt.Errorf("error no content was read") + } + + nums := strings.Split(logs, "\n") + if position >= len(nums) { + return 0, fmt.Errorf("position %v is not within the length of content %v", position, nums) + } + if position == -1 { + // Expectation of newline at the end of last position. + position = len(nums) - 2 + } + num, err := strconv.Atoi(nums[position]) + if err != nil { + return 0, fmt.Errorf("error getting number from file: %v", err) + } + + return num, nil +} + +func checkLogs(logs string, oldPos int) error { + if len(logs) == 0 { + return fmt.Errorf("error no content was read") + } + + nums := strings.Split(logs, "\n") + // Expectation of newline at the end of last position. + if oldPos >= len(nums)-2 { + return fmt.Errorf("oldPos %v is not within the length of content %v", oldPos, nums) + } + for i := oldPos + 1; i < len(nums)-1; i++ { + num, err := strconv.Atoi(nums[i]) + if err != nil { + return fmt.Errorf("error getting number from file: %v", err) + } + if num != oldPos+1 { + return fmt.Errorf("error in save/resume, numbers not in order, previous: %d, next: %d", oldPos, num) + } + oldPos++ + } + return nil +} + +// Checkpoint the container and continue running. +func TestCheckpointResume(t *testing.T) { + if !testutil.IsCheckpointSupported() { + t.Skip("Checkpoint is not supported.") + } + dockerutil.EnsureDockerExperimentalEnabled() + + ctx := context.Background() + d := dockerutil.MakeContainer(ctx, t) + defer d.CleanUp(ctx) + + // Start the container. + if err := d.Spawn(ctx, dockerutil.RunOpts{ + Image: "basic/alpine", + }, "sh", "-c", "i=0; while true; do echo \"$i\"; i=\"$(expr \"$i\" + 1)\"; sleep .01; done"); err != nil { + t.Fatalf("docker run failed: %v", err) + } + + time.Sleep(2 * time.Second) + + // Get the logs before checkpointing. + logs, err := d.Logs(ctx) + if err != nil { + t.Fatalf("docker logs failed: %v", err) + } + + // Get the last position of the logs printed. + pos, err := readLogs(logs, -1) + if err != nil { + t.Fatalf("readLogs failed: %v", err) + } + + // Create a snapshot and continue running. + if err := d.CheckpointResume(ctx, "test"); err != nil { + t.Fatalf("docker checkpoint failed: %v", err) + } + + var newLogs string + // Wait for the container to resume running and print new logs. + if err := testutil.Poll(func() error { + // Get the logs after checkpointing to check if the container resumed. + newLogs, err = d.Logs(ctx) + if err != nil { + t.Fatalf("docker logs failed: %v", err) + } + return nil + }, defaultWait); err != nil { + t.Fatalf("container read logs failed after resume: %v", err) + } + + if err := checkLogs(newLogs, pos); err != nil { + t.Fatalf("checkLogs failed: %v", err) + } + if err := d.Kill(ctx); err != nil { + t.Fatalf("docker kill failed: %v", err) + } +}