Enable save/checkpoint resume with runsc checkpoint command.

Enables save resume with checkpoint command. Previously when --leave-running
was set, the sandbox was destroyed after the checkpoint and restored with the
same id. With this change the sandbox will not be destroyed and resumes running
after the checkpoint.

PiperOrigin-RevId: 623282685

pkg/sentry/control/state.go

@@ -45,6 +45,10 @@ type SaveOpts struct {
 
 	// FilePayload contains the destination for the state.
 	urpc.FilePayload
+
+	// Resume indicates if the sandbox process should continue running
+	// after checkpointing.
+	Resume bool
 }
 
 // Save saves the running system.
@@ -68,7 +72,9 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error {
 				log.Warningf("Save failed: exiting...")
 				s.Kernel.SetSaveError(err)
 			}
-			s.Kernel.Kill(linux.WaitStatusExit(0))
+			if !o.Resume {
+				s.Kernel.Kill(linux.WaitStatusExit(0))
+			}
 		},
 	}
 	return saveOpts.Save(s.Kernel.SupervisorContext(), s.Kernel, s.Watchdog)

pkg/state/statefile/statefile.go

@@ -105,6 +105,10 @@ const (
 type Options struct {
 	// Compression is an image compression type/level.
 	Compression CompressionLevel
+
+	// Resume indicates if the sandbox process should continue running
+	// after checkpointing.
+	Resume bool
 }
 
 // WriteToMetadata save options to the metadata storage.  Method returns the

pkg/test/dockerutil/container.go

@@ -333,6 +333,11 @@ func (c *Container) Restore(ctx context.Context, name string) error {
 	return c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{CheckpointID: name})
 }
 
+// CheckpointResume is analogous to 'docker checkpoint'.
+func (c *Container) CheckpointResume(ctx context.Context, name string) error {
+	return c.client.CheckpointCreate(ctx, c.Name, types.CheckpointCreateOptions{CheckpointID: name, Exit: false})
+}
+
 // Logs is analogous 'docker logs'.
 func (c *Container) Logs(ctx context.Context) (string, error) {
 	var out bytes.Buffer

runsc/cmd/checkpoint.go

@@ -21,14 +21,11 @@ import (
 	"path/filepath"
 
 	"github.com/google/subcommands"
-	"golang.org/x/sys/unix"
-	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/state/statefile"
 	"gvisor.dev/gvisor/runsc/cmd/util"
 	"gvisor.dev/gvisor/runsc/config"
 	"gvisor.dev/gvisor/runsc/container"
 	"gvisor.dev/gvisor/runsc/flag"
-	"gvisor.dev/gvisor/runsc/specutils"
 )
 
 // File containing the container's saved image/state within the given image-path's directory.
@@ -77,7 +74,6 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su
 
 	id := f.Arg(0)
 	conf := args[0].(*config.Config)
-	waitStatus := args[1].(*unix.WaitStatus)
 
 	cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{})
 	if err != nil {
@@ -101,63 +97,17 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) su
 	}
 	defer file.Close()
 
-	if err := cont.Checkpoint(file, statefile.Options{Compression: c.compression.Level()}); err != nil {
+	sOpts := statefile.Options{Compression: c.compression.Level()}
+
+	if c.leaveRunning {
+		// Do not destroy the sandbox after saving.
+		sOpts.Resume = true
+	}
+
+	if err := cont.Checkpoint(file, sOpts); err != nil {
 		util.Fatalf("checkpoint failed: %v", err)
 	}
 
-	if !c.leaveRunning {
-		return subcommands.ExitSuccess
-	}
-
-	// TODO(b/110843694): Make it possible to restore into same container.
-	// For now, we can fake it by destroying the container and making a
-	// new container with the same ID. This hack does not work with docker
-	// which uses the container pid to ensure that the restore-container is
-	// actually the same as the checkpoint-container. By restoring into
-	// the same container, we will solve the docker incompatibility.
-
-	// Restore into new container with same ID.
-	bundleDir := cont.BundleDir
-	if bundleDir == "" {
-		util.Fatalf("setting bundleDir")
-	}
-
-	spec, err := specutils.ReadSpec(bundleDir, conf)
-	if err != nil {
-		util.Fatalf("reading spec: %v", err)
-	}
-
-	specutils.LogSpecDebug(spec, conf.OCISeccomp)
-
-	if cont.ConsoleSocket != "" {
-		log.Warningf("ignoring console socket since it cannot be restored")
-	}
-
-	if err := cont.Destroy(); err != nil {
-		util.Fatalf("destroying container: %v", err)
-	}
-
-	contArgs := container.Args{
-		ID:        id,
-		Spec:      spec,
-		BundleDir: bundleDir,
-	}
-	cont, err = container.New(conf, contArgs)
-	if err != nil {
-		util.Fatalf("restoring container: %v", err)
-	}
-	defer cont.Destroy()
-
-	if err := cont.Restore(conf, fullImagePath); err != nil {
-		util.Fatalf("starting container: %v", err)
-	}
-
-	ws, err := cont.Wait()
-	if err != nil {
-		util.Fatalf("Error waiting for container: %v", err)
-	}
-	*waitStatus = ws
-
 	return subcommands.ExitSuccess
 }
 

runsc/sandbox/sandbox.go

@@ -1263,6 +1263,7 @@ func (s *Sandbox) Checkpoint(cid string, f *os.File, options statefile.Options)
 		FilePayload: urpc.FilePayload{
 			Files: []*os.File{f},
 		},
+		Resume: options.Resume,
 	}
 
 	if err := s.call(boot.ContMgrCheckpoint, &opt, nil); err != nil {

test/e2e/integration_test.go

@@ -1110,3 +1110,105 @@ func TestBlockHostUds(t *testing.T) {
 		t.Errorf("err should be non-nil and output should contain %q, but got err = %v and output = %q", want, err, got)
 	}
 }
+
+func readLogs(logs string, position int) (int, error) {
+	if len(logs) == 0 {
+		return 0, fmt.Errorf("error no content was read")
+	}
+
+	nums := strings.Split(logs, "\n")
+	if position >= len(nums) {
+		return 0, fmt.Errorf("position %v is not within the length of content %v", position, nums)
+	}
+	if position == -1 {
+		// Expectation of newline at the end of last position.
+		position = len(nums) - 2
+	}
+	num, err := strconv.Atoi(nums[position])
+	if err != nil {
+		return 0, fmt.Errorf("error getting number from file: %v", err)
+	}
+
+	return num, nil
+}
+
+func checkLogs(logs string, oldPos int) error {
+	if len(logs) == 0 {
+		return fmt.Errorf("error no content was read")
+	}
+
+	nums := strings.Split(logs, "\n")
+	// Expectation of newline at the end of last position.
+	if oldPos >= len(nums)-2 {
+		return fmt.Errorf("oldPos %v is not within the length of content %v", oldPos, nums)
+	}
+	for i := oldPos + 1; i < len(nums)-1; i++ {
+		num, err := strconv.Atoi(nums[i])
+		if err != nil {
+			return fmt.Errorf("error getting number from file: %v", err)
+		}
+		if num != oldPos+1 {
+			return fmt.Errorf("error in save/resume, numbers not in order, previous: %d, next: %d", oldPos, num)
+		}
+		oldPos++
+	}
+	return nil
+}
+
+// Checkpoint the container and continue running.
+func TestCheckpointResume(t *testing.T) {
+	if !testutil.IsCheckpointSupported() {
+		t.Skip("Checkpoint is not supported.")
+	}
+	dockerutil.EnsureDockerExperimentalEnabled()
+
+	ctx := context.Background()
+	d := dockerutil.MakeContainer(ctx, t)
+	defer d.CleanUp(ctx)
+
+	// Start the container.
+	if err := d.Spawn(ctx, dockerutil.RunOpts{
+		Image: "basic/alpine",
+	}, "sh", "-c", "i=0; while true; do echo \"$i\"; i=\"$(expr \"$i\" + 1)\"; sleep .01; done"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
+	}
+
+	time.Sleep(2 * time.Second)
+
+	// Get the logs before checkpointing.
+	logs, err := d.Logs(ctx)
+	if err != nil {
+		t.Fatalf("docker logs failed: %v", err)
+	}
+
+	// Get the last position of the logs printed.
+	pos, err := readLogs(logs, -1)
+	if err != nil {
+		t.Fatalf("readLogs failed: %v", err)
+	}
+
+	// Create a snapshot and continue running.
+	if err := d.CheckpointResume(ctx, "test"); err != nil {
+		t.Fatalf("docker checkpoint failed: %v", err)
+	}
+
+	var newLogs string
+	// Wait for the container to resume running and print new logs.
+	if err := testutil.Poll(func() error {
+		// Get the logs after checkpointing to check if the container resumed.
+		newLogs, err = d.Logs(ctx)
+		if err != nil {
+			t.Fatalf("docker logs failed: %v", err)
+		}
+		return nil
+	}, defaultWait); err != nil {
+		t.Fatalf("container read logs failed after resume: %v", err)
+	}
+
+	if err := checkLogs(newLogs, pos); err != nil {
+		t.Fatalf("checkLogs failed: %v", err)
+	}
+	if err := d.Kill(ctx); err != nil {
+		t.Fatalf("docker kill failed: %v", err)
+	}
+}