diff --git a/pkg/sentry/fsimpl/proc/BUILD b/pkg/sentry/fsimpl/proc/BUILD index a89bd8b2c..96cb6f178 100644 --- a/pkg/sentry/fsimpl/proc/BUILD +++ b/pkg/sentry/fsimpl/proc/BUILD @@ -66,6 +66,7 @@ go_library( "fd_dir_inode_refs.go", "fd_info_dir_inode_refs.go", "filesystem.go", + "proc_impl.go", "subtasks.go", "subtasks_inode_refs.go", "task.go", diff --git a/pkg/sentry/fsimpl/proc/filesystem.go b/pkg/sentry/fsimpl/proc/filesystem.go index 5c282b795..798beacf2 100644 --- a/pkg/sentry/fsimpl/proc/filesystem.go +++ b/pkg/sentry/fsimpl/proc/filesystem.go @@ -95,13 +95,14 @@ func (ft FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualF procfs.MaxCachedDentries = maxCachedDentries procfs.VFSFilesystem().Init(vfsObj, &ft, procfs) - var fakeCgroupControllers map[string]string - if opts.InternalData != nil { - data := opts.InternalData.(*InternalData) - fakeCgroupControllers = data.Cgroups + var internalData *InternalData + if opts.InternalData == nil { + internalData = &InternalData{} + } else { + internalData = opts.InternalData.(*InternalData) } - inode := procfs.newTasksInode(ctx, k, pidns, fakeCgroupControllers) + inode := procfs.newTasksInode(ctx, k, pidns, internalData) var dentry kernfs.Dentry dentry.InitRoot(&procfs.Filesystem, inode) return procfs.VFSFilesystem(), dentry.VFSDentry(), nil @@ -157,6 +158,7 @@ func (fs *filesystem) newStaticDir(ctx context.Context, creds *auth.Credentials, // // +stateify savable type InternalData struct { + ExtraInternalData Cgroups map[string]string } diff --git a/pkg/sentry/fsimpl/proc/proc_impl.go b/pkg/sentry/fsimpl/proc/proc_impl.go new file mode 100644 index 000000000..797403b42 --- /dev/null +++ b/pkg/sentry/fsimpl/proc/proc_impl.go @@ -0,0 +1,33 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !false +// +build !false + +package proc + +import ( + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" +) + +// ExtraInternalData is an empty struct that could contain extra data for the procfs. +// +// +stateify savable +type ExtraInternalData struct{} + +func (fs *filesystem) newTasksInodeExtra(context.Context, *auth.Credentials, *InternalData, *kernel.Kernel, map[string]kernfs.Inode) { +} diff --git a/pkg/sentry/fsimpl/proc/tasks.go b/pkg/sentry/fsimpl/proc/tasks.go index 36cc34d65..e4948791c 100644 --- a/pkg/sentry/fsimpl/proc/tasks.go +++ b/pkg/sentry/fsimpl/proc/tasks.go @@ -64,8 +64,9 @@ type tasksInode struct { var _ kernfs.Inode = (*tasksInode)(nil) -func (fs *filesystem) newTasksInode(ctx context.Context, k *kernel.Kernel, pidns *kernel.PIDNamespace, fakeCgroupControllers map[string]string) *tasksInode { +func (fs *filesystem) newTasksInode(ctx context.Context, k *kernel.Kernel, pidns *kernel.PIDNamespace, internalData *InternalData) *tasksInode { root := auth.NewRootCredentials(pidns.UserNamespace()) + contents := map[string]kernfs.Inode{ "cmdline": fs.newInode(ctx, root, 0444, &cmdLineData{}), "cpuinfo": fs.newInode(ctx, root, 0444, newStaticFileSetStat(cpuInfoData(k))), @@ -86,14 +87,16 @@ func (fs *filesystem) newTasksInode(ctx context.Context, k *kernel.Kernel, pidns } // If fakeCgroupControllers are provided, don't create a cgroupfs backed // /proc/cgroup as it will not match the fake controllers. - if len(fakeCgroupControllers) == 0 { + if len(internalData.Cgroups) == 0 { contents["cgroups"] = fs.newInode(ctx, root, 0444, &cgroupsData{}) } + fs.newTasksInodeExtra(ctx, root, internalData, k, contents) + inode := &tasksInode{ pidns: pidns, fs: fs, - fakeCgroupControllers: fakeCgroupControllers, + fakeCgroupControllers: internalData.Cgroups, } inode.InodeAttrs.Init(ctx, root, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), linux.ModeDirectory|0555) inode.InitRefs() diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index daf746b1f..06d18c158 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -361,6 +361,14 @@ type Kernel struct { // additionalCheckpointState stores additional state that needs // to be checkpointed. It's protected by extMu. additionalCheckpointState map[any]any + + // Saver registers someone that knows how to save the kernel. + saver Saver `state:"nosave"` +} + +// Saver is an interface for saving the kernel. +type Saver interface { + SaveAsync(done func()) error } // InitKernelArgs holds arguments to Init. @@ -2086,3 +2094,15 @@ func (k *Kernel) ContainerName(cid string) string { defer k.extMu.Unlock() return k.containerNames[cid] } + +// SetSaver sets the kernel's Saver. +// Thread-compatible. +func (k *Kernel) SetSaver(s Saver) { + k.saver = s +} + +// Saver returns the kernel's Saver. +// Thread-compatible. +func (k *Kernel) Saver() Saver { + return k.saver +} diff --git a/pkg/state/statefile/statefile.go b/pkg/state/statefile/statefile.go index 80b5f0465..34ebd4982 100644 --- a/pkg/state/statefile/statefile.go +++ b/pkg/state/statefile/statefile.go @@ -87,7 +87,8 @@ var ErrMetadataInvalid = fmt.Errorf("metadata invalid, can't start with _") var ErrInvalidFlags = fmt.Errorf("flags set is invalid") const ( - compressionKey = "compression" + // CompressionKey is the key for the compression level in the metadata. + CompressionKey = "compression" ) // CompressionLevel is the image compression level. @@ -102,6 +103,10 @@ const ( CompressionLevelDefault = CompressionLevelFlateBestSpeed ) +func (c CompressionLevel) String() string { + return string(c) +} + // Options is statefile options. type Options struct { // Compression is an image compression type/level. @@ -115,7 +120,7 @@ type Options struct { // WriteToMetadata save options to the metadata storage. Method returns the // reference to the original metadata map to allow to be used in the chain calls. func (o Options) WriteToMetadata(metadata map[string]string) map[string]string { - metadata[compressionKey] = string(o.Compression) + metadata[CompressionKey] = string(o.Compression) return metadata } @@ -126,6 +131,8 @@ func CompressionLevelFromString(val string) (CompressionLevel, error) { return CompressionLevelFlateBestSpeed, nil case string(CompressionLevelNone): return CompressionLevelNone, nil + case "": + return CompressionLevelDefault, nil default: return CompressionLevelNone, ErrInvalidFlags } @@ -136,16 +143,15 @@ func CompressionLevelFromString(val string) (CompressionLevel, error) { // is the "flate-best-speed" state because the default behavior used to be to always // compress. If the parameter is missing it will be set to default. func CompressionLevelFromMetadata(metadata map[string]string) (CompressionLevel, error) { - var err error + compression := CompressionLevelDefault - compression := CompressionLevelFlateBestSpeed - - if val, ok := metadata[compressionKey]; ok { + if val, ok := metadata[CompressionKey]; ok { + var err error if compression, err = CompressionLevelFromString(val); err != nil { return CompressionLevelNone, err } } else { - metadata[compressionKey] = string(compression) + metadata[CompressionKey] = string(compression) } return compression, nil diff --git a/pkg/state/statefile/statefile_test.go b/pkg/state/statefile/statefile_test.go index d2a1c5193..c45029152 100644 --- a/pkg/state/statefile/statefile_test.go +++ b/pkg/state/statefile/statefile_test.go @@ -92,7 +92,7 @@ func TestStatefile(t *testing.T) { c.metadata = map[string]string{} } - c.metadata[compressionKey] = string(compress) + c.metadata[CompressionKey] = string(compress) t.Run(c.name, func(t *testing.T) { for _, key := range [][]byte{nil, integrityKey} { diff --git a/runsc/boot/autosave.go b/runsc/boot/autosave.go index 591ec9c01..7027313ca 100644 --- a/runsc/boot/autosave.go +++ b/runsc/boot/autosave.go @@ -66,7 +66,7 @@ func getTargetForSaveResume(l *Loader) func(k *kernel.Kernel) { } } -func getTargetForSaveRestore(l *Loader, files ...*fd.FD) func(k *kernel.Kernel) { +func getTargetForSaveRestore(l *Loader, files []*fd.FD) func(k *kernel.Kernel) { if len(files) != 1 && len(files) != 3 { panic(fmt.Sprintf("Unexpected number of files: %v", len(files))) } @@ -86,13 +86,13 @@ func getTargetForSaveRestore(l *Loader, files ...*fd.FD) func(k *kernel.Kernel) } } -// EnableAutosave enables auto save restore in syscall tests. -func EnableAutosave(l *Loader, isResume bool, files ...*fd.FD) error { +// enableAutosave enables auto save restore in syscall tests. +func enableAutosave(l *Loader, isResume bool, files []*fd.FD) error { var target func(k *kernel.Kernel) if isResume { target = getTargetForSaveResume(l) } else { - target = getTargetForSaveRestore(l, files...) + target = getTargetForSaveRestore(l, files) } for _, table := range kernel.SyscallTables() { diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 746fbfb47..aba608b36 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -234,6 +234,8 @@ type Loader struct { // // portForwardProxies is guarded by mu. portForwardProxies []*pf.Proxy + + saveFDs []*fd.FD } // execID uniquely identifies a sentry process that is executed in a container. @@ -338,6 +340,8 @@ type Args struct { // NvidiaDriverVersion is the NVIDIA driver ABI version to use for // communicating with NVIDIA devices on the host. NvidiaDriverVersion string + + SaveFDs []*fd.FD } // make sure stdioFDs are always the same on initial start and on restore @@ -406,6 +410,7 @@ func New(args Args) (*Loader, error) { stopProfiling: stopProfiling, productName: args.ProductName, containerIDs: map[string]string{}, + saveFDs: args.SaveFDs, } containerName := l.registerContainerLocked(args.Spec, args.ID) @@ -608,6 +613,14 @@ func New(args Args) (*Loader, error) { return nil, fmt.Errorf("ignore child stop signals failed: %w", err) } + if len(args.Conf.TestOnlyAutosaveImagePath) != 0 { + enableAutosave(l, args.Conf.TestOnlyAutosaveResume, l.saveFDs) + } + + if err := l.initDone(args); err != nil { + return nil, err + } + // Create the control server using the provided FD. // // This must be done *after* we have initialized the kernel since the diff --git a/runsc/boot/restore_impl.go b/runsc/boot/restore_impl.go index 22244b61a..b7436629f 100644 --- a/runsc/boot/restore_impl.go +++ b/runsc/boot/restore_impl.go @@ -18,7 +18,9 @@ package boot import ( + specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/sentry/control" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/proc" "gvisor.dev/gvisor/pkg/sentry/kernel" ) @@ -35,3 +37,11 @@ func postRestoreImpl(*kernel.Kernel) error { func postResumeImpl(*kernel.Kernel) error { return nil } + +func newProcInternalData(*specs.Spec) *proc.InternalData { + return &proc.InternalData{} +} + +func (l *Loader) initDone(args Args) error { + return nil +} diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index ca998a71c..9d073beb3 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -877,12 +877,15 @@ func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, // Find filesystem name and FS specific data field. switch m.mount.Type { - case devpts.Name, dev.Name, proc.Name: + case devpts.Name, dev.Name: // Nothing to do. case Nonefs: fsName = sys.Name + case proc.Name: + internalData = newProcInternalData(spec) + case sys.Name: sysData := &sys.InternalData{EnableTPUProxyPaths: specutils.TPUProxyIsEnabled(spec, conf)} if len(productName) > 0 { diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 379107697..d0a52c317 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -456,6 +456,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma SinkFDs: b.sinkFDs.GetArray(), ProfileOpts: b.profileFDs.ToOpts(), NvidiaDriverVersion: b.nvidiaDriverVersion, + SaveFDs: b.saveFDs.GetFDs(), } l, err := boot.New(bootArgs) if err != nil { @@ -474,10 +475,6 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } - if len(conf.TestOnlyAutosaveImagePath) != 0 { - boot.EnableAutosave(l, conf.TestOnlyAutosaveResume, b.saveFDs.GetFDs()...) - } - // Prepare metrics. // This needs to happen after the kernel is initialized (such that all metrics are registered) // but before the start-sync file is notified, as the parent process needs to query for diff --git a/runsc/sandbox/BUILD b/runsc/sandbox/BUILD index 4b1979584..8a76fa2da 100644 --- a/runsc/sandbox/BUILD +++ b/runsc/sandbox/BUILD @@ -12,6 +12,7 @@ go_library( "network.go", "network_unsafe.go", "sandbox.go", + "sandbox_impl.go", "xdp.go", ], visibility = [ diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 3ea49d158..0724ad338 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -475,10 +475,16 @@ func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, dir return fmt.Errorf("opening restore image file %q failed: %v", pagesMetadataFileName, err) } defer pmf.Close() + opt.HavePagesFile = true opt.FilePayload.Files = append(opt.FilePayload.Files, pmf, pf) + log.Infof("Found page files for sandbox %q. Page metadata: %q, pages: %q", s.ID, pagesMetadataFileName, pagesFileName) + } else if !os.IsNotExist(err) { - return fmt.Errorf("opening restore image file %q failed: %v", pagesFileName, err) + return fmt.Errorf("opening restore pages file %q failed: %v", pagesFileName, err) + + } else { + log.Infof("Using single checkpoint file for sandbox %q", s.ID) } // If the platform needs a device FD we must pass it in. @@ -858,6 +864,10 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn donations.DonateAndClose("save-fds", files...) } + if err := createSandboxProcessExtra(conf, args, &donations); err != nil { + return err + } + gPlatform, err := platform.Lookup(conf.Platform) if err != nil { return fmt.Errorf("cannot look up platform: %w", err) diff --git a/runsc/sandbox/sandbox_impl.go b/runsc/sandbox/sandbox_impl.go new file mode 100644 index 000000000..4ca2402f3 --- /dev/null +++ b/runsc/sandbox/sandbox_impl.go @@ -0,0 +1,27 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !false +// +build !false + +package sandbox + +import ( + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/donation" +) + +func createSandboxProcessExtra(conf *config.Config, args *Args, donations *donation.Agency) error { + return nil +} diff --git a/runsc/specutils/nvidia.go b/runsc/specutils/nvidia.go index 261f1669c..6eda896ea 100644 --- a/runsc/specutils/nvidia.go +++ b/runsc/specutils/nvidia.go @@ -20,7 +20,6 @@ import ( "strings" specs "github.com/opencontainers/runtime-spec/specs-go" - "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/runsc/config" ) @@ -34,15 +33,7 @@ func NVProxyEnabled(spec *specs.Spec, conf *config.Config) bool { if conf.NVProxy { return true } - val, ok := spec.Annotations[AnnotationNVProxy] - if !ok { - return false - } - ret, err := strconv.ParseBool(val) - if err != nil { - log.Warningf("nvproxy annotation set to invalid value %q: %w. Skipping.", val, err) - } - return ret + return AnnotationToBool(spec, AnnotationNVProxy) } // GPUFunctionalityRequested returns true if the container should have access diff --git a/runsc/specutils/specutils.go b/runsc/specutils/specutils.go index b734ecf49..0503e2258 100644 --- a/runsc/specutils/specutils.go +++ b/runsc/specutils/specutils.go @@ -571,15 +571,7 @@ func TPUProxyIsEnabled(spec *specs.Spec, conf *config.Config) bool { if conf.TPUProxy { return true } - val, ok := spec.Annotations[AnnotationTPU] - if !ok { - return false - } - ret, err := strconv.ParseBool(val) - if err != nil { - log.Warningf("tpuproxy annotation set to invalid value %q: %w. Skipping.", val, err) - } - return ret + return AnnotationToBool(spec, AnnotationTPU) } // VFIOFunctionalityRequested returns true if the container should have access @@ -764,3 +756,18 @@ func FaqErrorMsg(anchor, msg string) string { func ContainerName(spec *specs.Spec) string { return spec.Annotations[annotationContainerName] } + +// AnnotationToBool parses the annotation value as a bool. On failure, it logs a warning and +// returns false. +func AnnotationToBool(spec *specs.Spec, annotation string) bool { + val, ok := spec.Annotations[annotation] + if !ok { + return false + } + ret, err := strconv.ParseBool(val) + if err != nil { + log.Warningf("Failed to parse annotation %q=%q as a bool: %v", annotation, val, err) + return false + } + return ret +}