Add save-resume variant to syscall tests.

Adds the save-resume variant to all syscall tests. These tests save/checkpoint
the sandbox for every syscall in the test and then resume.

PiperOrigin-RevId: 623601440

.buildkite/pipeline.yaml

@@ -302,6 +302,21 @@ steps:
     parallelism: 10
     agents:
       arch: "arm64"
+  # All Save Resume system call tests.
+  - <<: *common
+    <<: *source_test
+    label: ":muscle: System call save resume tests (AMD64)"
+    command: make BAZEL_OPTIONS=--test_tag_filters=save_resume syscall-tests
+    parallelism: 20
+    agents:
+      arch: "amd64"
+  - <<: *common
+    <<: *source_test
+    label: ":muscle: System call save resume tests (ARM64)"
+    command: make BAZEL_OPTIONS=--test_tag_filters=save_resume syscall-tests
+    parallelism: 10
+    agents:
+      arch: "arm64"
 
   # Integration tests.
   - <<: *common

pkg/sentry/state/state.go

@@ -56,6 +56,9 @@ type SaveOpts struct {
 
 	// Callback is called prior to unpause, with any save error.
 	Callback func(err error)
+
+	// Resume indicates if the statefile is used for save-resume.
+	Resume bool
 }
 
 // Save saves the system state.

runsc/boot/autosave.go

@@ -15,6 +15,7 @@
 package boot
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 
@@ -27,34 +28,63 @@ import (
 	"gvisor.dev/gvisor/pkg/sync"
 )
 
-// EnableAutosave enables auto save restore in syscall tests.
-func EnableAutosave(l *Loader, f *os.File) error {
-	var once sync.Once // Used by target.
-	target := func(k *kernel.Kernel) {
-		once.Do(func() {
-			t, _ := state.CPUTime()
-			log.Infof("Before save CPU usage: %s", t.String())
-			saveOpts := state.SaveOpts{
-				Destination: f,
-				Key:         nil,
-				Callback: func(err error) {
-					t1, _ := state.CPUTime()
-					log.Infof("Save CPU usage: %s", (t1 - t).String())
-					if err == nil {
-						log.Infof("Save succeeded: exiting...")
-						k.SetSaveSuccess(true)
-					} else {
-						log.Warningf("Save failed: exiting... %v", err)
-						k.SetSaveError(err)
-					}
-
-					// Kill the sandbox.
-					k.Kill(linux.WaitStatusExit(0))
-				},
+func getSaveOpts(l *Loader, k *kernel.Kernel, isResume bool) state.SaveOpts {
+	t, _ := state.CPUTime()
+	log.Infof("Before save CPU usage: %s", t.String())
+	saveOpts := state.SaveOpts{
+		Key:    nil,
+		Resume: isResume,
+		Callback: func(err error) {
+			t1, _ := state.CPUTime()
+			log.Infof("Save CPU usage: %s", (t1 - t).String())
+			if err == nil {
+				log.Infof("Save succeeded: exiting...")
+				k.SetSaveSuccess(true)
+			} else {
+				log.Warningf("Save failed: exiting... %v", err)
+				k.SetSaveError(err)
 			}
+
+			if !isResume {
+				// Kill the sandbox.
+				k.Kill(linux.WaitStatusExit(0))
+			}
+		},
+	}
+	return saveOpts
+}
+
+func getTargetForSaveResume(l *Loader) func(k *kernel.Kernel) {
+	return func(k *kernel.Kernel) {
+		saveOpts := getSaveOpts(l, k, true /* isResume */)
+		// Store the state file contents in a buffer for save-resume.
+		// There is no need to verify the state file, we just need the
+		// sandbox to continue running after save.
+		var buf bytes.Buffer
+		saveOpts.Destination = &buf
+		saveOpts.Save(k.SupervisorContext(), k, l.watchdog)
+	}
+}
+
+func getTargetForSaveRestore(l *Loader, f *os.File) func(k *kernel.Kernel) {
+	var once sync.Once
+	return func(k *kernel.Kernel) {
+		once.Do(func() {
+			saveOpts := getSaveOpts(l, k, false /* isResume */)
+			saveOpts.Destination = f
 			saveOpts.Save(k.SupervisorContext(), k, l.watchdog)
 		})
 	}
+}
+
+// EnableAutosave enables auto save restore in syscall tests.
+func EnableAutosave(l *Loader, f *os.File, isResume bool) error {
+	var target func(k *kernel.Kernel)
+	if isResume {
+		target = getTargetForSaveResume(l)
+	} else {
+		target = getTargetForSaveRestore(l, f)
+	}
 
 	for _, table := range kernel.SyscallTables() {
 		sys, ok := strace.Lookup(table.OS, table.Arch)

runsc/cmd/boot.go

@@ -468,7 +468,8 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 			util.Fatalf("error in creating state file %v", err)
 		}
 		defer f.Close()
-		boot.EnableAutosave(l, f)
+
+		boot.EnableAutosave(l, f, conf.TestOnlyAutosaveResume)
 	}
 
 	// Prepare metrics.

runsc/config/config.go

@@ -350,6 +350,9 @@ type Config struct {
 	// TestOnlyAutosaveImagePath if not empty enables auto save for syscall tests
 	// and stores the directory path to the saved state file.
 	TestOnlyAutosaveImagePath string `flag:"TESTONLY-autosave-image-path"`
+
+	// TestOnlyAutosaveResume indicates save resume for syscall tests.
+	TestOnlyAutosaveResume bool `flag:"TESTONLY-autosave-resume"`
 }
 
 func (c *Config) validate() error {

runsc/config/flags.go

@@ -136,6 +136,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.Bool("TESTONLY-allow-packet-endpoint-write", false, "TEST ONLY; do not ever use! Used for tests to allow writes on packet sockets.")
 	flagSet.Bool("TESTONLY-afs-syscall-panic", false, "TEST ONLY; do not ever use! Used for tests exercising gVisor panic reporting.")
 	flagSet.String("TESTONLY-autosave-image-path", "", "TEST ONLY; enable auto save for syscall tests and set path for state file.")
+	flagSet.Bool("TESTONLY-autosave-resume", false, "TEST ONLY; enable auto save and resume for syscall tests and set path for state file.")
 }
 
 // overrideAllowlist lists all flags that can be changed using OCI

test/runner/defs.bzl

@@ -78,6 +78,7 @@ def _syscall_test(
         directfs = False,
         leak_check = False,
         save = False,
+        save_resume = False,
         **kwargs):
     # Prepend "runsc" to non-native platform names.
     full_platform = platform if platform == "native" else "runsc_" + platform
@@ -96,6 +97,8 @@ def _syscall_test(
         name += "_directfs"
     if save:
         name += "_save"
+    if save_resume:
+        name += "_save_resume"
 
     # Apply all tags.
     if tags == None:
@@ -106,10 +109,13 @@ def _syscall_test(
     tags = list(tags)
     tags += [full_platform, "file_" + file_access]
 
-    if save:
+    if save or save_resume:
         tags.append("allsave")
         if platform in save_restore_platforms:
-            tags.append("save_restore")
+            if save:
+                tags.append("save_restore")
+            if save_resume:
+                tags.append("save_resume")
 
     # Hash this target into one of 15 buckets. This can be used to
     # randomly split targets between different workflows.
@@ -162,6 +168,7 @@ def _syscall_test(
         "--directfs=" + str(directfs),
         "--leak-check=" + str(leak_check),
         "--save=" + str(save),
+        "--save-resume=" + str(save_resume),
     ]
 
     # Trace points are platform agnostic, so enable them for ptrace only.
@@ -201,6 +208,7 @@ def syscall_test_variants(
         container = None,
         tags = None,
         save = False,
+        save_resume = False,
         size = "medium",
         timeout = None,
         **kwargs):
@@ -226,6 +234,7 @@ def syscall_test_variants(
       save: save restore test.
       size: test size.
       timeout: timeout for the test.
+      save_resume: save resume test.
       **kwargs: additional test arguments.
     """
     for platform, platform_tags in all_platforms():
@@ -246,6 +255,7 @@ def syscall_test_variants(
             one_sandbox = one_sandbox,
             leak_check = leak_check,
             save = save,
+            save_resume = save_resume,
             size = size,
             timeout = timeout,
             **kwargs
@@ -267,13 +277,14 @@ def syscall_test_variants(
             overlay = True,
             leak_check = leak_check,
             save = save,
+            save_resume = save_resume,
             size = size,
             timeout = timeout,
             **kwargs
         )
 
     # TODO(b/192114729): hostinet is not supported with S/R.
-    if add_hostinet and not save:
+    if add_hostinet and not (save or save_resume):
         _syscall_test(
             test = test,
             platform = default_platform,
@@ -289,6 +300,7 @@ def syscall_test_variants(
             one_sandbox = one_sandbox,
             leak_check = leak_check,
             save = save,
+            save_resume = save_resume,
             size = size,
             timeout = timeout,
             **kwargs
@@ -310,6 +322,7 @@ def syscall_test_variants(
             file_access = "shared",
             leak_check = leak_check,
             save = save,
+            save_resume = save_resume,
             size = size,
             timeout = timeout,
             **kwargs
@@ -329,6 +342,7 @@ def syscall_test_variants(
             one_sandbox = one_sandbox,
             leak_check = leak_check,
             save = save,
+            save_resume = save_resume,
             size = size,
             timeout = timeout,
             **kwargs
@@ -373,7 +387,7 @@ def syscall_test(
       container: Run the test in a container. If None, determined from other information.
       tags: starting test tags.
       leak_check: enables leak check.
-      save: save restore test.
+      save: enables save/restore and save/resume test variants.
       size: test size.
       **kwargs: additional test arguments.
     """
@@ -414,11 +428,12 @@ def syscall_test(
         container,
         tags,
         False,  # save, generate all tests without save variant.
+        False,  # save_resume, generate all tests without save_resume variant.
         size,
         **kwargs
     )
 
-    # Add save variant to all other variants generated above.
+    # Add save and save_resume variants to all other variants generated above.
     if save:
         # Disable go sanitizers for save tests.
         tags.append("nogotsan")
@@ -440,6 +455,32 @@ def syscall_test(
             container,
             tags,
             True,  # save, generate all tests with save variant.
+            False,  # save_resume, generate all tests without save_resume variant.
+            "large",  # size, use size as large by default for all S/R tests.
+            "long",  # timeout, use long timeout for S/R tests.
+            **kwargs
+        )
+
+        # Add save resume variant to all other variants generated above.
+        syscall_test_variants(
+            test,
+            use_tmpfs,
+            add_fusefs,
+            add_overlay,
+            add_host_uds,
+            add_host_connector,
+            add_host_fifo,
+            add_hostinet,
+            add_directfs,
+            one_sandbox,
+            iouring,
+            allow_native,
+            leak_check,
+            debug,
+            container,
+            tags,
+            False,  # save, generate all tests without save variant.
+            True,  # save_resume, generate all tests with save_resume variant.
             "large",  # size, use size as large by default for all S/R tests.
             "long",  # timeout, use long timeout for S/R tests.
             **kwargs

test/runner/main.go

@@ -70,6 +70,7 @@ var (
 	leakCheck        = flag.Bool("leak-check", false, "check for reference leaks")
 	waitForPid       = flag.Duration("delay-for-debugger", 0, "Print out the sandbox PID and wait for the specified duration to start the test. This is useful for attaching a debugger to the runsc-sandbox process.")
 	save             = flag.Bool("save", false, "enables save restore")
+	saveResume       = flag.Bool("save-resume", false, "enables save resume")
 )
 
 const (
@@ -405,14 +406,17 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error {
 		args = append(args, "-log=/dev/null")
 
 		// Create the state file.
-		if *save {
+		if *save || *saveResume {
 			saveArgs = args
 			args, dirs, err = prepareSave(args, undeclaredOutputsDir, dirs, 0)
 			if err != nil {
 				return fmt.Errorf("prepareSave error: %v", err)
 			}
+			if *saveResume {
+				args = append(args, "-TESTONLY-autosave-resume=true")
+			}
 		}
-	} else if *save {
+	} else if *save || *saveResume {
 		// TEST_UNDECLARED_OUTPUTS_DIR directory should be present with S/R to create
 		// the state file.
 		return fmt.Errorf("TEST_UNDECLARED_OUTPUTS_DIR is not set with S/R enabled")
@@ -574,6 +578,14 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error {
 		}
 		// Do not output state files when the test succeeds.
 		removeAll(dirs)
+	} else if *saveResume {
+		err = cmd.Run()
+		if err != nil {
+			printAll(dirs)
+			removeAll(dirs)
+			return fmt.Errorf("run error: %v", err)
+		}
+		removeAll(dirs)
 	} else {
 		err = cmd.Run()
 		if *waitForPid != 0 {
@@ -871,7 +883,7 @@ func runTestCaseRunsc(testBin string, tc *gtest.TestCase, args []string, t *test
 	} else {
 		env = append(env, fuseVar+"=FALSE")
 	}
-	if *save {
+	if *save || *saveResume {
 		env = append(env, saveVar+"=TRUE")
 	} else {
 		env = append(env, saveVar+"=FALSE")