From 2a0dff228add57e3e21d61d43a7dee8be683a26e Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Mon, 10 Jun 2024 12:02:54 -0700 Subject: [PATCH] Move auto-save file creation outside of boot `runsc boot` is not guaranteed to file system access to create files. Create save files before the sandbox is created and pass FDs to it, like it's done for other files. PiperOrigin-RevId: 641979089 --- runsc/boot/autosave.go | 19 ++++++--- runsc/cmd/boot.go | 14 +++---- runsc/cmd/cmd.go | 12 +++++- runsc/sandbox/sandbox.go | 89 ++++++++++++++++++++++++++-------------- 4 files changed, 88 insertions(+), 46 deletions(-) diff --git a/runsc/boot/autosave.go b/runsc/boot/autosave.go index 16f3dda05..591ec9c01 100644 --- a/runsc/boot/autosave.go +++ b/runsc/boot/autosave.go @@ -17,9 +17,9 @@ package boot import ( "bytes" "fmt" - "os" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/fd" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/kernel" @@ -66,24 +66,33 @@ func getTargetForSaveResume(l *Loader) func(k *kernel.Kernel) { } } -func getTargetForSaveRestore(l *Loader, f *os.File) func(k *kernel.Kernel) { +func getTargetForSaveRestore(l *Loader, files ...*fd.FD) func(k *kernel.Kernel) { + if len(files) != 1 && len(files) != 3 { + panic(fmt.Sprintf("Unexpected number of files: %v", len(files))) + } + var once sync.Once return func(k *kernel.Kernel) { once.Do(func() { saveOpts := getSaveOpts(l, k, false /* isResume */) - saveOpts.Destination = f + saveOpts.Destination = files[0] + if len(files) == 3 { + saveOpts.PagesMetadata = files[1] + saveOpts.PagesFile = files[2] + } + saveOpts.Save(k.SupervisorContext(), k, l.watchdog) }) } } // EnableAutosave enables auto save restore in syscall tests. -func EnableAutosave(l *Loader, f *os.File, isResume bool) error { +func EnableAutosave(l *Loader, isResume bool, files ...*fd.FD) error { var target func(k *kernel.Kernel) if isResume { target = getTargetForSaveResume(l) } else { - target = getTargetForSaveRestore(l, f) + target = getTargetForSaveRestore(l, files...) } for _, table := range kernel.SyscallTables() { diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 1262ef511..379107697 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -138,6 +138,8 @@ type Boot struct { sinkFDs intFlags + saveFDs intFlags + // pidns is set if the sandbox is in its own pid namespace. pidns bool @@ -219,6 +221,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).") f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.") f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.") + f.Var(&b.saveFDs, "save-fds", "ordered list of file descriptors to be used save checkpoints. Order: kernel state, page metadata, page file") // Profiling flags. b.profileFDs.SetFromFlags(f) @@ -471,15 +474,8 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } - if conf.TestOnlyAutosaveImagePath != "" { - fName := filepath.Join(conf.TestOnlyAutosaveImagePath, boot.CheckpointStateFileName) - f, err := os.OpenFile(fName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - util.Fatalf("error in creating state file %v", err) - } - defer f.Close() - - boot.EnableAutosave(l, f, conf.TestOnlyAutosaveResume) + if len(conf.TestOnlyAutosaveImagePath) != 0 { + boot.EnableAutosave(l, conf.TestOnlyAutosaveResume, b.saveFDs.GetFDs()...) } // Prepare metrics. diff --git a/runsc/cmd/cmd.go b/runsc/cmd/cmd.go index c37220d6c..40d65ee12 100644 --- a/runsc/cmd/cmd.go +++ b/runsc/cmd/cmd.go @@ -24,6 +24,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/fd" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/runsc/specutils" ) @@ -46,11 +47,20 @@ func (i *intFlags) Get() any { return i } -// GetArray returns array of FDs. +// GetArray returns an array of ints representing FDs. func (i *intFlags) GetArray() []int { return *i } +// GetFDs returns an array of *fd.FD. +func (i *intFlags) GetFDs() []*fd.FD { + rv := make([]*fd.FD, 0, len(*i)) + for _, val := range *i { + rv = append(rv, fd.New(val)) + } + return rv +} + // Set implements flag.Value. Set(String()) should be idempotent. func (i *intFlags) Set(s string) error { for _, sFD := range strings.Split(s, ",") { diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 2788ca518..6fee71d59 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -850,6 +850,14 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn } donations.DonateAndClose("sink-fds", args.SinkFiles...) + if len(conf.TestOnlyAutosaveImagePath) != 0 { + files, err := createSaveFiles(conf.TestOnlyAutosaveImagePath, false, statefile.CompressionLevelFlateBestSpeed) + if err != nil { + return fmt.Errorf("failed to create auto save files: %w", err) + } + donations.DonateAndClose("save-fds", files...) + } + gPlatform, err := platform.Lookup(conf.Platform) if err != nil { return fmt.Errorf("cannot look up platform: %w", err) @@ -1328,45 +1336,24 @@ func (s *Sandbox) SignalProcess(cid string, pid int32, sig unix.Signal, fgProces func (s *Sandbox) Checkpoint(cid string, imagePath string, direct bool, sfOpts statefile.Options, mfOpts pgalloc.SaveOpts) error { log.Debugf("Checkpoint sandbox %q, statefile options %+v, MemoryFile options %+v", s.ID, sfOpts, mfOpts) - stateFilePath := filepath.Join(imagePath, boot.CheckpointStateFileName) - sf, err := os.OpenFile(stateFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + files, err := createSaveFiles(imagePath, direct, sfOpts.Compression) if err != nil { - return fmt.Errorf("creating checkpoint state file %q: %w", stateFilePath, err) + return err } - defer sf.Close() + defer func() { + for _, f := range files { + _ = f.Close() + } + }() opt := control.SaveOpts{ Metadata: sfOpts.WriteToMetadata(map[string]string{}), MemoryFileSaveOpts: mfOpts, FilePayload: urpc.FilePayload{ - Files: []*os.File{sf}, + Files: files, }, - Resume: sfOpts.Resume, - } - - // When there is no compression, MemoryFile contents are page-aligned. - // It is beneficial to store them separately so certain optimizations can be - // applied during restore. See Restore(). - if sfOpts.Compression == statefile.CompressionLevelNone { - pagesFilePath := filepath.Join(imagePath, boot.CheckpointPagesFileName) - pagesWriteFlags := os.O_CREATE | os.O_EXCL | os.O_RDWR - if direct { - // The writes will be page-aligned, so it can be opened with O_DIRECT. - pagesWriteFlags |= syscall.O_DIRECT - } - pf, err := os.OpenFile(pagesFilePath, pagesWriteFlags, 0644) - if err != nil { - return fmt.Errorf("creating checkpoint pages file %q: %w", pagesFilePath, err) - } - defer pf.Close() - pagesMetadataFilePath := filepath.Join(imagePath, boot.CheckpointPagesMetadataFileName) - pmf, err := os.OpenFile(pagesMetadataFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err != nil { - return fmt.Errorf("creating checkpoint pages metadata file %q: %w", pagesMetadataFilePath, err) - } - defer pmf.Close() - opt.FilePayload.Files = append(opt.FilePayload.Files, pmf, pf) - opt.HavePagesFile = true + HavePagesFile: len(files) > 1, + Resume: sfOpts.Resume, } if err := s.call(boot.ContMgrCheckpoint, &opt, nil); err != nil { @@ -1375,6 +1362,46 @@ func (s *Sandbox) Checkpoint(cid string, imagePath string, direct bool, sfOpts s return nil } +// createSaveFiles creates the files used by checkpoint to save the state. They are returned in +// the following order: sentry state, page metadata, page file. This is the same order expected by +// RPCs and argument passing to the sandbox. +func createSaveFiles(path string, direct bool, compression statefile.CompressionLevel) ([]*os.File, error) { + var files []*os.File + + stateFilePath := filepath.Join(path, boot.CheckpointStateFileName) + f, err := os.OpenFile(stateFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + return nil, fmt.Errorf("creating checkpoint state file %q: %w", stateFilePath, err) + } + files = append(files, f) + + // When there is no compression, MemoryFile contents are page-aligned. + // It is beneficial to store them separately so certain optimizations can be + // applied during restore. See Restore(). + if compression == statefile.CompressionLevelNone { + pagesMetadataFilePath := filepath.Join(path, boot.CheckpointPagesMetadataFileName) + f, err = os.OpenFile(pagesMetadataFilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + if err != nil { + return nil, fmt.Errorf("creating checkpoint pages metadata file %q: %w", pagesMetadataFilePath, err) + } + files = append(files, f) + + pagesFilePath := filepath.Join(path, boot.CheckpointPagesFileName) + pagesWriteFlags := os.O_CREATE | os.O_EXCL | os.O_RDWR + if direct { + // The writes will be page-aligned, so it can be opened with O_DIRECT. + pagesWriteFlags |= syscall.O_DIRECT + } + f, err := os.OpenFile(pagesFilePath, pagesWriteFlags, 0644) + if err != nil { + return nil, fmt.Errorf("creating checkpoint pages file %q: %w", pagesFilePath, err) + } + files = append(files, f) + } + + return files, nil +} + // Pause sends the pause call for a container in the sandbox. func (s *Sandbox) Pause(cid string) error { log.Debugf("Pause sandbox %q", s.ID)