From be1a31aa233a4aa9a798ea07761e4d307ebe718d Mon Sep 17 00:00:00 2001 From: Nayana Bidari Date: Wed, 10 Apr 2024 14:11:06 -0700 Subject: [PATCH] Add save-resume variant to syscall tests. Adds the save-resume variant to all syscall tests. These tests save/checkpoint the sandbox for every syscall in the test and then resume. PiperOrigin-RevId: 623601440 --- .buildkite/pipeline.yaml | 15 ++++++++ pkg/sentry/state/state.go | 3 ++ runsc/boot/autosave.go | 78 +++++++++++++++++++++++++++------------ runsc/cmd/boot.go | 3 +- runsc/config/config.go | 3 ++ runsc/config/flags.go | 1 + test/runner/defs.bzl | 51 ++++++++++++++++++++++--- test/runner/main.go | 18 +++++++-- 8 files changed, 139 insertions(+), 33 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 0dd671a36..b778dd258 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -302,6 +302,21 @@ steps: parallelism: 10 agents: arch: "arm64" + # All Save Resume system call tests. + - <<: *common + <<: *source_test + label: ":muscle: System call save resume tests (AMD64)" + command: make BAZEL_OPTIONS=--test_tag_filters=save_resume syscall-tests + parallelism: 20 + agents: + arch: "amd64" + - <<: *common + <<: *source_test + label: ":muscle: System call save resume tests (ARM64)" + command: make BAZEL_OPTIONS=--test_tag_filters=save_resume syscall-tests + parallelism: 10 + agents: + arch: "arm64" # Integration tests. - <<: *common diff --git a/pkg/sentry/state/state.go b/pkg/sentry/state/state.go index e9d544f3d..b2baad6ef 100644 --- a/pkg/sentry/state/state.go +++ b/pkg/sentry/state/state.go @@ -56,6 +56,9 @@ type SaveOpts struct { // Callback is called prior to unpause, with any save error. Callback func(err error) + + // Resume indicates if the statefile is used for save-resume. + Resume bool } // Save saves the system state. diff --git a/runsc/boot/autosave.go b/runsc/boot/autosave.go index 12bb753ff..16f3dda05 100644 --- a/runsc/boot/autosave.go +++ b/runsc/boot/autosave.go @@ -15,6 +15,7 @@ package boot import ( + "bytes" "fmt" "os" @@ -27,34 +28,63 @@ import ( "gvisor.dev/gvisor/pkg/sync" ) -// EnableAutosave enables auto save restore in syscall tests. -func EnableAutosave(l *Loader, f *os.File) error { - var once sync.Once // Used by target. - target := func(k *kernel.Kernel) { - once.Do(func() { - t, _ := state.CPUTime() - log.Infof("Before save CPU usage: %s", t.String()) - saveOpts := state.SaveOpts{ - Destination: f, - Key: nil, - Callback: func(err error) { - t1, _ := state.CPUTime() - log.Infof("Save CPU usage: %s", (t1 - t).String()) - if err == nil { - log.Infof("Save succeeded: exiting...") - k.SetSaveSuccess(true) - } else { - log.Warningf("Save failed: exiting... %v", err) - k.SetSaveError(err) - } - - // Kill the sandbox. - k.Kill(linux.WaitStatusExit(0)) - }, +func getSaveOpts(l *Loader, k *kernel.Kernel, isResume bool) state.SaveOpts { + t, _ := state.CPUTime() + log.Infof("Before save CPU usage: %s", t.String()) + saveOpts := state.SaveOpts{ + Key: nil, + Resume: isResume, + Callback: func(err error) { + t1, _ := state.CPUTime() + log.Infof("Save CPU usage: %s", (t1 - t).String()) + if err == nil { + log.Infof("Save succeeded: exiting...") + k.SetSaveSuccess(true) + } else { + log.Warningf("Save failed: exiting... %v", err) + k.SetSaveError(err) } + + if !isResume { + // Kill the sandbox. + k.Kill(linux.WaitStatusExit(0)) + } + }, + } + return saveOpts +} + +func getTargetForSaveResume(l *Loader) func(k *kernel.Kernel) { + return func(k *kernel.Kernel) { + saveOpts := getSaveOpts(l, k, true /* isResume */) + // Store the state file contents in a buffer for save-resume. + // There is no need to verify the state file, we just need the + // sandbox to continue running after save. + var buf bytes.Buffer + saveOpts.Destination = &buf + saveOpts.Save(k.SupervisorContext(), k, l.watchdog) + } +} + +func getTargetForSaveRestore(l *Loader, f *os.File) func(k *kernel.Kernel) { + var once sync.Once + return func(k *kernel.Kernel) { + once.Do(func() { + saveOpts := getSaveOpts(l, k, false /* isResume */) + saveOpts.Destination = f saveOpts.Save(k.SupervisorContext(), k, l.watchdog) }) } +} + +// EnableAutosave enables auto save restore in syscall tests. +func EnableAutosave(l *Loader, f *os.File, isResume bool) error { + var target func(k *kernel.Kernel) + if isResume { + target = getTargetForSaveResume(l) + } else { + target = getTargetForSaveRestore(l, f) + } for _, table := range kernel.SyscallTables() { sys, ok := strace.Lookup(table.OS, table.Arch) diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 3ace17120..5b171fa18 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -468,7 +468,8 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma util.Fatalf("error in creating state file %v", err) } defer f.Close() - boot.EnableAutosave(l, f) + + boot.EnableAutosave(l, f, conf.TestOnlyAutosaveResume) } // Prepare metrics. diff --git a/runsc/config/config.go b/runsc/config/config.go index fe56a5ef9..415b8b908 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -350,6 +350,9 @@ type Config struct { // TestOnlyAutosaveImagePath if not empty enables auto save for syscall tests // and stores the directory path to the saved state file. TestOnlyAutosaveImagePath string `flag:"TESTONLY-autosave-image-path"` + + // TestOnlyAutosaveResume indicates save resume for syscall tests. + TestOnlyAutosaveResume bool `flag:"TESTONLY-autosave-resume"` } func (c *Config) validate() error { diff --git a/runsc/config/flags.go b/runsc/config/flags.go index e44d1d30c..db49aaf88 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -136,6 +136,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("TESTONLY-allow-packet-endpoint-write", false, "TEST ONLY; do not ever use! Used for tests to allow writes on packet sockets.") flagSet.Bool("TESTONLY-afs-syscall-panic", false, "TEST ONLY; do not ever use! Used for tests exercising gVisor panic reporting.") flagSet.String("TESTONLY-autosave-image-path", "", "TEST ONLY; enable auto save for syscall tests and set path for state file.") + flagSet.Bool("TESTONLY-autosave-resume", false, "TEST ONLY; enable auto save and resume for syscall tests and set path for state file.") } // overrideAllowlist lists all flags that can be changed using OCI diff --git a/test/runner/defs.bzl b/test/runner/defs.bzl index 3e3b26a27..b147019ea 100644 --- a/test/runner/defs.bzl +++ b/test/runner/defs.bzl @@ -78,6 +78,7 @@ def _syscall_test( directfs = False, leak_check = False, save = False, + save_resume = False, **kwargs): # Prepend "runsc" to non-native platform names. full_platform = platform if platform == "native" else "runsc_" + platform @@ -96,6 +97,8 @@ def _syscall_test( name += "_directfs" if save: name += "_save" + if save_resume: + name += "_save_resume" # Apply all tags. if tags == None: @@ -106,10 +109,13 @@ def _syscall_test( tags = list(tags) tags += [full_platform, "file_" + file_access] - if save: + if save or save_resume: tags.append("allsave") if platform in save_restore_platforms: - tags.append("save_restore") + if save: + tags.append("save_restore") + if save_resume: + tags.append("save_resume") # Hash this target into one of 15 buckets. This can be used to # randomly split targets between different workflows. @@ -162,6 +168,7 @@ def _syscall_test( "--directfs=" + str(directfs), "--leak-check=" + str(leak_check), "--save=" + str(save), + "--save-resume=" + str(save_resume), ] # Trace points are platform agnostic, so enable them for ptrace only. @@ -201,6 +208,7 @@ def syscall_test_variants( container = None, tags = None, save = False, + save_resume = False, size = "medium", timeout = None, **kwargs): @@ -226,6 +234,7 @@ def syscall_test_variants( save: save restore test. size: test size. timeout: timeout for the test. + save_resume: save resume test. **kwargs: additional test arguments. """ for platform, platform_tags in all_platforms(): @@ -246,6 +255,7 @@ def syscall_test_variants( one_sandbox = one_sandbox, leak_check = leak_check, save = save, + save_resume = save_resume, size = size, timeout = timeout, **kwargs @@ -267,13 +277,14 @@ def syscall_test_variants( overlay = True, leak_check = leak_check, save = save, + save_resume = save_resume, size = size, timeout = timeout, **kwargs ) # TODO(b/192114729): hostinet is not supported with S/R. - if add_hostinet and not save: + if add_hostinet and not (save or save_resume): _syscall_test( test = test, platform = default_platform, @@ -289,6 +300,7 @@ def syscall_test_variants( one_sandbox = one_sandbox, leak_check = leak_check, save = save, + save_resume = save_resume, size = size, timeout = timeout, **kwargs @@ -310,6 +322,7 @@ def syscall_test_variants( file_access = "shared", leak_check = leak_check, save = save, + save_resume = save_resume, size = size, timeout = timeout, **kwargs @@ -329,6 +342,7 @@ def syscall_test_variants( one_sandbox = one_sandbox, leak_check = leak_check, save = save, + save_resume = save_resume, size = size, timeout = timeout, **kwargs @@ -373,7 +387,7 @@ def syscall_test( container: Run the test in a container. If None, determined from other information. tags: starting test tags. leak_check: enables leak check. - save: save restore test. + save: enables save/restore and save/resume test variants. size: test size. **kwargs: additional test arguments. """ @@ -414,11 +428,12 @@ def syscall_test( container, tags, False, # save, generate all tests without save variant. + False, # save_resume, generate all tests without save_resume variant. size, **kwargs ) - # Add save variant to all other variants generated above. + # Add save and save_resume variants to all other variants generated above. if save: # Disable go sanitizers for save tests. tags.append("nogotsan") @@ -440,6 +455,32 @@ def syscall_test( container, tags, True, # save, generate all tests with save variant. + False, # save_resume, generate all tests without save_resume variant. + "large", # size, use size as large by default for all S/R tests. + "long", # timeout, use long timeout for S/R tests. + **kwargs + ) + + # Add save resume variant to all other variants generated above. + syscall_test_variants( + test, + use_tmpfs, + add_fusefs, + add_overlay, + add_host_uds, + add_host_connector, + add_host_fifo, + add_hostinet, + add_directfs, + one_sandbox, + iouring, + allow_native, + leak_check, + debug, + container, + tags, + False, # save, generate all tests without save variant. + True, # save_resume, generate all tests with save_resume variant. "large", # size, use size as large by default for all S/R tests. "long", # timeout, use long timeout for S/R tests. **kwargs diff --git a/test/runner/main.go b/test/runner/main.go index 3a1b50025..3415d3d4e 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -70,6 +70,7 @@ var ( leakCheck = flag.Bool("leak-check", false, "check for reference leaks") waitForPid = flag.Duration("delay-for-debugger", 0, "Print out the sandbox PID and wait for the specified duration to start the test. This is useful for attaching a debugger to the runsc-sandbox process.") save = flag.Bool("save", false, "enables save restore") + saveResume = flag.Bool("save-resume", false, "enables save resume") ) const ( @@ -405,14 +406,17 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { args = append(args, "-log=/dev/null") // Create the state file. - if *save { + if *save || *saveResume { saveArgs = args args, dirs, err = prepareSave(args, undeclaredOutputsDir, dirs, 0) if err != nil { return fmt.Errorf("prepareSave error: %v", err) } + if *saveResume { + args = append(args, "-TESTONLY-autosave-resume=true") + } } - } else if *save { + } else if *save || *saveResume { // TEST_UNDECLARED_OUTPUTS_DIR directory should be present with S/R to create // the state file. return fmt.Errorf("TEST_UNDECLARED_OUTPUTS_DIR is not set with S/R enabled") @@ -574,6 +578,14 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { } // Do not output state files when the test succeeds. removeAll(dirs) + } else if *saveResume { + err = cmd.Run() + if err != nil { + printAll(dirs) + removeAll(dirs) + return fmt.Errorf("run error: %v", err) + } + removeAll(dirs) } else { err = cmd.Run() if *waitForPid != 0 { @@ -871,7 +883,7 @@ func runTestCaseRunsc(testBin string, tc *gtest.TestCase, args []string, t *test } else { env = append(env, fuseVar+"=FALSE") } - if *save { + if *save || *saveResume { env = append(env, saveVar+"=TRUE") } else { env = append(env, saveVar+"=FALSE")