diff --git a/pkg/sentry/control/state.go b/pkg/sentry/control/state.go index 4c83b8e8e..7610648fb 100644 --- a/pkg/sentry/control/state.go +++ b/pkg/sentry/control/state.go @@ -45,6 +45,10 @@ type SaveOpts struct { // FilePayload contains the destination for the state. urpc.FilePayload + + // Resume indicates if the sandbox process should continue running + // after checkpointing. + Resume bool } // Save saves the running system. @@ -68,7 +72,9 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error { log.Warningf("Save failed: exiting...") s.Kernel.SetSaveError(err) } - s.Kernel.Kill(linux.WaitStatusExit(0)) + if !o.Resume { + s.Kernel.Kill(linux.WaitStatusExit(0)) + } }, } return saveOpts.Save(s.Kernel.SupervisorContext(), s.Kernel, s.Watchdog) diff --git a/pkg/state/statefile/statefile.go b/pkg/state/statefile/statefile.go index 93fe63ccc..66d2990bd 100644 --- a/pkg/state/statefile/statefile.go +++ b/pkg/state/statefile/statefile.go @@ -105,6 +105,10 @@ const ( type Options struct { // Compression is an image compression type/level. Compression CompressionLevel + + // Resume indicates if the sandbox process should continue running + // after checkpointing. + Resume bool } // WriteToMetadata save options to the metadata storage. Method returns the diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 077a73057..9230a0140 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -333,6 +333,11 @@ func (c *Container) Restore(ctx context.Context, name string) error { return c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{CheckpointID: name}) } +// CheckpointResume is analogous to 'docker checkpoint'. +func (c *Container) CheckpointResume(ctx context.Context, name string) error { + return c.client.CheckpointCreate(ctx, c.Name, types.CheckpointCreateOptions{CheckpointID: name, Exit: false}) +} + // Logs is analogous 'docker logs'. func (c *Container) Logs(ctx context.Context) (string, error) { var out bytes.Buffer diff --git a/runsc/cmd/checkpoint.go b/runsc/cmd/checkpoint.go index e7590177c..32f68384c 100644 --- a/runsc/cmd/checkpoint.go +++ b/runsc/cmd/checkpoint.go @@ -21,14 +21,11 @@ import ( "path/filepath" "github.com/google/subcommands" - "golang.org/x/sys/unix" - "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/runsc/cmd/util" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/container" "gvisor.dev/gvisor/runsc/flag" - "gvisor.dev/gvisor/runsc/specutils" ) // File containing the container's saved image/state within the given image-path's directory. @@ -77,7 +74,6 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su id := f.Arg(0) conf := args[0].(*config.Config) - waitStatus := args[1].(*unix.WaitStatus) cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{}) if err != nil { @@ -101,63 +97,17 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su } defer file.Close() - if err := cont.Checkpoint(file, statefile.Options{Compression: c.compression.Level()}); err != nil { + sOpts := statefile.Options{Compression: c.compression.Level()} + + if c.leaveRunning { + // Do not destroy the sandbox after saving. + sOpts.Resume = true + } + + if err := cont.Checkpoint(file, sOpts); err != nil { util.Fatalf("checkpoint failed: %v", err) } - if !c.leaveRunning { - return subcommands.ExitSuccess - } - - // TODO(b/110843694): Make it possible to restore into same container. - // For now, we can fake it by destroying the container and making a - // new container with the same ID. This hack does not work with docker - // which uses the container pid to ensure that the restore-container is - // actually the same as the checkpoint-container. By restoring into - // the same container, we will solve the docker incompatibility. - - // Restore into new container with same ID. - bundleDir := cont.BundleDir - if bundleDir == "" { - util.Fatalf("setting bundleDir") - } - - spec, err := specutils.ReadSpec(bundleDir, conf) - if err != nil { - util.Fatalf("reading spec: %v", err) - } - - specutils.LogSpecDebug(spec, conf.OCISeccomp) - - if cont.ConsoleSocket != "" { - log.Warningf("ignoring console socket since it cannot be restored") - } - - if err := cont.Destroy(); err != nil { - util.Fatalf("destroying container: %v", err) - } - - contArgs := container.Args{ - ID: id, - Spec: spec, - BundleDir: bundleDir, - } - cont, err = container.New(conf, contArgs) - if err != nil { - util.Fatalf("restoring container: %v", err) - } - defer cont.Destroy() - - if err := cont.Restore(conf, fullImagePath); err != nil { - util.Fatalf("starting container: %v", err) - } - - ws, err := cont.Wait() - if err != nil { - util.Fatalf("Error waiting for container: %v", err) - } - *waitStatus = ws - return subcommands.ExitSuccess } diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 7a8bb3fbd..07864ccfd 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -1263,6 +1263,7 @@ func (s *Sandbox) Checkpoint(cid string, f *os.File, options statefile.Options) FilePayload: urpc.FilePayload{ Files: []*os.File{f}, }, + Resume: options.Resume, } if err := s.call(boot.ContMgrCheckpoint, &opt, nil); err != nil { diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go index 5b391f847..7bfdbb034 100644 --- a/test/e2e/integration_test.go +++ b/test/e2e/integration_test.go @@ -1110,3 +1110,105 @@ func TestBlockHostUds(t *testing.T) { t.Errorf("err should be non-nil and output should contain %q, but got err = %v and output = %q", want, err, got) } } + +func readLogs(logs string, position int) (int, error) { + if len(logs) == 0 { + return 0, fmt.Errorf("error no content was read") + } + + nums := strings.Split(logs, "\n") + if position >= len(nums) { + return 0, fmt.Errorf("position %v is not within the length of content %v", position, nums) + } + if position == -1 { + // Expectation of newline at the end of last position. + position = len(nums) - 2 + } + num, err := strconv.Atoi(nums[position]) + if err != nil { + return 0, fmt.Errorf("error getting number from file: %v", err) + } + + return num, nil +} + +func checkLogs(logs string, oldPos int) error { + if len(logs) == 0 { + return fmt.Errorf("error no content was read") + } + + nums := strings.Split(logs, "\n") + // Expectation of newline at the end of last position. + if oldPos >= len(nums)-2 { + return fmt.Errorf("oldPos %v is not within the length of content %v", oldPos, nums) + } + for i := oldPos + 1; i < len(nums)-1; i++ { + num, err := strconv.Atoi(nums[i]) + if err != nil { + return fmt.Errorf("error getting number from file: %v", err) + } + if num != oldPos+1 { + return fmt.Errorf("error in save/resume, numbers not in order, previous: %d, next: %d", oldPos, num) + } + oldPos++ + } + return nil +} + +// Checkpoint the container and continue running. +func TestCheckpointResume(t *testing.T) { + if !testutil.IsCheckpointSupported() { + t.Skip("Checkpoint is not supported.") + } + dockerutil.EnsureDockerExperimentalEnabled() + + ctx := context.Background() + d := dockerutil.MakeContainer(ctx, t) + defer d.CleanUp(ctx) + + // Start the container. + if err := d.Spawn(ctx, dockerutil.RunOpts{ + Image: "basic/alpine", + }, "sh", "-c", "i=0; while true; do echo \"$i\"; i=\"$(expr \"$i\" + 1)\"; sleep .01; done"); err != nil { + t.Fatalf("docker run failed: %v", err) + } + + time.Sleep(2 * time.Second) + + // Get the logs before checkpointing. + logs, err := d.Logs(ctx) + if err != nil { + t.Fatalf("docker logs failed: %v", err) + } + + // Get the last position of the logs printed. + pos, err := readLogs(logs, -1) + if err != nil { + t.Fatalf("readLogs failed: %v", err) + } + + // Create a snapshot and continue running. + if err := d.CheckpointResume(ctx, "test"); err != nil { + t.Fatalf("docker checkpoint failed: %v", err) + } + + var newLogs string + // Wait for the container to resume running and print new logs. + if err := testutil.Poll(func() error { + // Get the logs after checkpointing to check if the container resumed. + newLogs, err = d.Logs(ctx) + if err != nil { + t.Fatalf("docker logs failed: %v", err) + } + return nil + }, defaultWait); err != nil { + t.Fatalf("container read logs failed after resume: %v", err) + } + + if err := checkLogs(newLogs, pos); err != nil { + t.Fatalf("checkLogs failed: %v", err) + } + if err := d.Kill(ctx); err != nil { + t.Fatalf("docker kill failed: %v", err) + } +}