Add a few extension points

PiperOrigin-RevId: 644476039

pkg/sentry/fsimpl/proc/BUILD

@@ -66,6 +66,7 @@ go_library(
         "fd_dir_inode_refs.go",
         "fd_info_dir_inode_refs.go",
         "filesystem.go",
+        "proc_impl.go",
         "subtasks.go",
         "subtasks_inode_refs.go",
         "task.go",

pkg/sentry/fsimpl/proc/filesystem.go

@@ -95,13 +95,14 @@ func (ft FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualF
 	procfs.MaxCachedDentries = maxCachedDentries
 	procfs.VFSFilesystem().Init(vfsObj, &ft, procfs)
 
-	var fakeCgroupControllers map[string]string
-	if opts.InternalData != nil {
-		data := opts.InternalData.(*InternalData)
-		fakeCgroupControllers = data.Cgroups
+	var internalData *InternalData
+	if opts.InternalData == nil {
+		internalData = &InternalData{}
+	} else {
+		internalData = opts.InternalData.(*InternalData)
 	}
 
-	inode := procfs.newTasksInode(ctx, k, pidns, fakeCgroupControllers)
+	inode := procfs.newTasksInode(ctx, k, pidns, internalData)
 	var dentry kernfs.Dentry
 	dentry.InitRoot(&procfs.Filesystem, inode)
 	return procfs.VFSFilesystem(), dentry.VFSDentry(), nil
@@ -157,6 +158,7 @@ func (fs *filesystem) newStaticDir(ctx context.Context, creds *auth.Credentials,
 //
 // +stateify savable
 type InternalData struct {
+	ExtraInternalData
 	Cgroups map[string]string
 }
 

pkg/sentry/fsimpl/proc/proc_impl.go

@@ -0,0 +1,33 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !false
+// +build !false
+
+package proc
+
+import (
+	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs"
+	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
+)
+
+// ExtraInternalData is an empty struct that could contain extra data for the procfs.
+//
+// +stateify savable
+type ExtraInternalData struct{}
+
+func (fs *filesystem) newTasksInodeExtra(context.Context, *auth.Credentials, *InternalData, *kernel.Kernel, map[string]kernfs.Inode) {
+}

pkg/sentry/fsimpl/proc/tasks.go

@@ -64,8 +64,9 @@ type tasksInode struct {
 
 var _ kernfs.Inode = (*tasksInode)(nil)
 
-func (fs *filesystem) newTasksInode(ctx context.Context, k *kernel.Kernel, pidns *kernel.PIDNamespace, fakeCgroupControllers map[string]string) *tasksInode {
+func (fs *filesystem) newTasksInode(ctx context.Context, k *kernel.Kernel, pidns *kernel.PIDNamespace, internalData *InternalData) *tasksInode {
 	root := auth.NewRootCredentials(pidns.UserNamespace())
+
 	contents := map[string]kernfs.Inode{
 		"cmdline":        fs.newInode(ctx, root, 0444, &cmdLineData{}),
 		"cpuinfo":        fs.newInode(ctx, root, 0444, newStaticFileSetStat(cpuInfoData(k))),
@@ -86,14 +87,16 @@ func (fs *filesystem) newTasksInode(ctx context.Context, k *kernel.Kernel, pidns
 	}
 	// If fakeCgroupControllers are provided, don't create a cgroupfs backed
 	// /proc/cgroup as it will not match the fake controllers.
-	if len(fakeCgroupControllers) == 0 {
+	if len(internalData.Cgroups) == 0 {
 		contents["cgroups"] = fs.newInode(ctx, root, 0444, &cgroupsData{})
 	}
 
+	fs.newTasksInodeExtra(ctx, root, internalData, k, contents)
+
 	inode := &tasksInode{
 		pidns:                 pidns,
 		fs:                    fs,
-		fakeCgroupControllers: fakeCgroupControllers,
+		fakeCgroupControllers: internalData.Cgroups,
 	}
 	inode.InodeAttrs.Init(ctx, root, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), linux.ModeDirectory|0555)
 	inode.InitRefs()

pkg/sentry/kernel/kernel.go

@@ -361,6 +361,14 @@ type Kernel struct {
 	// additionalCheckpointState stores additional state that needs
 	// to be checkpointed. It's protected by extMu.
 	additionalCheckpointState map[any]any
+
+	// Saver registers someone that knows how to save the kernel.
+	saver Saver `state:"nosave"`
+}
+
+// Saver is an interface for saving the kernel.
+type Saver interface {
+	SaveAsync(done func()) error
 }
 
 // InitKernelArgs holds arguments to Init.
@@ -2086,3 +2094,15 @@ func (k *Kernel) ContainerName(cid string) string {
 	defer k.extMu.Unlock()
 	return k.containerNames[cid]
 }
+
+// SetSaver sets the kernel's Saver.
+// Thread-compatible.
+func (k *Kernel) SetSaver(s Saver) {
+	k.saver = s
+}
+
+// Saver returns the kernel's Saver.
+// Thread-compatible.
+func (k *Kernel) Saver() Saver {
+	return k.saver
+}

pkg/state/statefile/statefile.go

@@ -87,7 +87,8 @@ var ErrMetadataInvalid = fmt.Errorf("metadata invalid, can't start with _")
 var ErrInvalidFlags = fmt.Errorf("flags set is invalid")
 
 const (
-	compressionKey = "compression"
+	// CompressionKey is the key for the compression level in the metadata.
+	CompressionKey = "compression"
 )
 
 // CompressionLevel is the image compression level.
@@ -102,6 +103,10 @@ const (
 	CompressionLevelDefault = CompressionLevelFlateBestSpeed
 )
 
+func (c CompressionLevel) String() string {
+	return string(c)
+}
+
 // Options is statefile options.
 type Options struct {
 	// Compression is an image compression type/level.
@@ -115,7 +120,7 @@ type Options struct {
 // WriteToMetadata save options to the metadata storage.  Method returns the
 // reference to the original metadata map to allow to be used in the chain calls.
 func (o Options) WriteToMetadata(metadata map[string]string) map[string]string {
-	metadata[compressionKey] = string(o.Compression)
+	metadata[CompressionKey] = string(o.Compression)
 	return metadata
 }
 
@@ -126,6 +131,8 @@ func CompressionLevelFromString(val string) (CompressionLevel, error) {
 		return CompressionLevelFlateBestSpeed, nil
 	case string(CompressionLevelNone):
 		return CompressionLevelNone, nil
+	case "":
+		return CompressionLevelDefault, nil
 	default:
 		return CompressionLevelNone, ErrInvalidFlags
 	}
@@ -136,16 +143,15 @@ func CompressionLevelFromString(val string) (CompressionLevel, error) {
 // is the "flate-best-speed" state because the default behavior used to be to always
 // compress. If the parameter is missing it will be set to default.
 func CompressionLevelFromMetadata(metadata map[string]string) (CompressionLevel, error) {
-	var err error
+	compression := CompressionLevelDefault
 
-	compression := CompressionLevelFlateBestSpeed
-
-	if val, ok := metadata[compressionKey]; ok {
+	if val, ok := metadata[CompressionKey]; ok {
+		var err error
 		if compression, err = CompressionLevelFromString(val); err != nil {
 			return CompressionLevelNone, err
 		}
 	} else {
-		metadata[compressionKey] = string(compression)
+		metadata[CompressionKey] = string(compression)
 	}
 
 	return compression, nil

pkg/state/statefile/statefile_test.go

@@ -92,7 +92,7 @@ func TestStatefile(t *testing.T) {
 					c.metadata = map[string]string{}
 				}
 
-				c.metadata[compressionKey] = string(compress)
+				c.metadata[CompressionKey] = string(compress)
 
 				t.Run(c.name, func(t *testing.T) {
 					for _, key := range [][]byte{nil, integrityKey} {

runsc/boot/autosave.go

@@ -66,7 +66,7 @@ func getTargetForSaveResume(l *Loader) func(k *kernel.Kernel) {
 	}
 }
 
-func getTargetForSaveRestore(l *Loader, files ...*fd.FD) func(k *kernel.Kernel) {
+func getTargetForSaveRestore(l *Loader, files []*fd.FD) func(k *kernel.Kernel) {
 	if len(files) != 1 && len(files) != 3 {
 		panic(fmt.Sprintf("Unexpected number of files: %v", len(files)))
 	}
@@ -86,13 +86,13 @@ func getTargetForSaveRestore(l *Loader, files ...*fd.FD) func(k *kernel.Kernel)
 	}
 }
 
-// EnableAutosave enables auto save restore in syscall tests.
-func EnableAutosave(l *Loader, isResume bool, files ...*fd.FD) error {
+// enableAutosave enables auto save restore in syscall tests.
+func enableAutosave(l *Loader, isResume bool, files []*fd.FD) error {
 	var target func(k *kernel.Kernel)
 	if isResume {
 		target = getTargetForSaveResume(l)
 	} else {
-		target = getTargetForSaveRestore(l, files...)
+		target = getTargetForSaveRestore(l, files)
 	}
 
 	for _, table := range kernel.SyscallTables() {

runsc/boot/loader.go

@@ -234,6 +234,8 @@ type Loader struct {
 	//
 	// portForwardProxies is guarded by mu.
 	portForwardProxies []*pf.Proxy
+
+	saveFDs []*fd.FD
 }
 
 // execID uniquely identifies a sentry process that is executed in a container.
@@ -338,6 +340,8 @@ type Args struct {
 	// NvidiaDriverVersion is the NVIDIA driver ABI version to use for
 	// communicating with NVIDIA devices on the host.
 	NvidiaDriverVersion string
+
+	SaveFDs []*fd.FD
 }
 
 // make sure stdioFDs are always the same on initial start and on restore
@@ -406,6 +410,7 @@ func New(args Args) (*Loader, error) {
 		stopProfiling: stopProfiling,
 		productName:   args.ProductName,
 		containerIDs:  map[string]string{},
+		saveFDs:       args.SaveFDs,
 	}
 
 	containerName := l.registerContainerLocked(args.Spec, args.ID)
@@ -608,6 +613,14 @@ func New(args Args) (*Loader, error) {
 		return nil, fmt.Errorf("ignore child stop signals failed: %w", err)
 	}
 
+	if len(args.Conf.TestOnlyAutosaveImagePath) != 0 {
+		enableAutosave(l, args.Conf.TestOnlyAutosaveResume, l.saveFDs)
+	}
+
+	if err := l.initDone(args); err != nil {
+		return nil, err
+	}
+
 	// Create the control server using the provided FD.
 	//
 	// This must be done *after* we have initialized the kernel since the

runsc/boot/restore_impl.go

@@ -18,7 +18,9 @@
 package boot
 
 import (
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.dev/gvisor/pkg/sentry/control"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/proc"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
 )
 
@@ -35,3 +37,11 @@ func postRestoreImpl(*kernel.Kernel) error {
 func postResumeImpl(*kernel.Kernel) error {
 	return nil
 }
+
+func newProcInternalData(*specs.Spec) *proc.InternalData {
+	return &proc.InternalData{}
+}
+
+func (l *Loader) initDone(args Args) error {
+	return nil
+}

runsc/boot/vfs.go

@@ -877,12 +877,15 @@ func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo,
 
 	// Find filesystem name and FS specific data field.
 	switch m.mount.Type {
-	case devpts.Name, dev.Name, proc.Name:
+	case devpts.Name, dev.Name:
 		// Nothing to do.
 
 	case Nonefs:
 		fsName = sys.Name
 
+	case proc.Name:
+		internalData = newProcInternalData(spec)
+
 	case sys.Name:
 		sysData := &sys.InternalData{EnableTPUProxyPaths: specutils.TPUProxyIsEnabled(spec, conf)}
 		if len(productName) > 0 {

runsc/cmd/boot.go

@@ -456,6 +456,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		SinkFDs:             b.sinkFDs.GetArray(),
 		ProfileOpts:         b.profileFDs.ToOpts(),
 		NvidiaDriverVersion: b.nvidiaDriverVersion,
+		SaveFDs:             b.saveFDs.GetFDs(),
 	}
 	l, err := boot.New(bootArgs)
 	if err != nil {
@@ -474,10 +475,6 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		}
 	}
 
-	if len(conf.TestOnlyAutosaveImagePath) != 0 {
-		boot.EnableAutosave(l, conf.TestOnlyAutosaveResume, b.saveFDs.GetFDs()...)
-	}
-
 	// Prepare metrics.
 	// This needs to happen after the kernel is initialized (such that all metrics are registered)
 	// but before the start-sync file is notified, as the parent process needs to query for

runsc/sandbox/BUILD

@@ -12,6 +12,7 @@ go_library(
         "network.go",
         "network_unsafe.go",
         "sandbox.go",
+        "sandbox_impl.go",
         "xdp.go",
     ],
     visibility = [

runsc/sandbox/sandbox.go

@@ -475,10 +475,16 @@ func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, dir
 			return fmt.Errorf("opening restore image file %q failed: %v", pagesMetadataFileName, err)
 		}
 		defer pmf.Close()
+
 		opt.HavePagesFile = true
 		opt.FilePayload.Files = append(opt.FilePayload.Files, pmf, pf)
+		log.Infof("Found page files for sandbox %q. Page metadata: %q, pages: %q", s.ID, pagesMetadataFileName, pagesFileName)
+
 	} else if !os.IsNotExist(err) {
-		return fmt.Errorf("opening restore image file %q failed: %v", pagesFileName, err)
+		return fmt.Errorf("opening restore pages file %q failed: %v", pagesFileName, err)
+
+	} else {
+		log.Infof("Using single checkpoint file for sandbox %q", s.ID)
 	}
 
 	// If the platform needs a device FD we must pass it in.
@@ -858,6 +864,10 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 		donations.DonateAndClose("save-fds", files...)
 	}
 
+	if err := createSandboxProcessExtra(conf, args, &donations); err != nil {
+		return err
+	}
+
 	gPlatform, err := platform.Lookup(conf.Platform)
 	if err != nil {
 		return fmt.Errorf("cannot look up platform: %w", err)

runsc/sandbox/sandbox_impl.go

@@ -0,0 +1,27 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !false
+// +build !false
+
+package sandbox
+
+import (
+	"gvisor.dev/gvisor/runsc/config"
+	"gvisor.dev/gvisor/runsc/donation"
+)
+
+func createSandboxProcessExtra(conf *config.Config, args *Args, donations *donation.Agency) error {
+	return nil
+}

runsc/specutils/nvidia.go

@@ -20,7 +20,6 @@ import (
 	"strings"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
-	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/runsc/config"
 )
 
@@ -34,15 +33,7 @@ func NVProxyEnabled(spec *specs.Spec, conf *config.Config) bool {
 	if conf.NVProxy {
 		return true
 	}
-	val, ok := spec.Annotations[AnnotationNVProxy]
-	if !ok {
-		return false
-	}
-	ret, err := strconv.ParseBool(val)
-	if err != nil {
-		log.Warningf("nvproxy annotation set to invalid value %q: %w. Skipping.", val, err)
-	}
-	return ret
+	return AnnotationToBool(spec, AnnotationNVProxy)
 }
 
 // GPUFunctionalityRequested returns true if the container should have access

runsc/specutils/specutils.go

@@ -571,15 +571,7 @@ func TPUProxyIsEnabled(spec *specs.Spec, conf *config.Config) bool {
 	if conf.TPUProxy {
 		return true
 	}
-	val, ok := spec.Annotations[AnnotationTPU]
-	if !ok {
-		return false
-	}
-	ret, err := strconv.ParseBool(val)
-	if err != nil {
-		log.Warningf("tpuproxy annotation set to invalid value %q: %w. Skipping.", val, err)
-	}
-	return ret
+	return AnnotationToBool(spec, AnnotationTPU)
 }
 
 // VFIOFunctionalityRequested returns true if the container should have access
@@ -764,3 +756,18 @@ func FaqErrorMsg(anchor, msg string) string {
 func ContainerName(spec *specs.Spec) string {
 	return spec.Annotations[annotationContainerName]
 }
+
+// AnnotationToBool parses the annotation value as a bool. On failure, it logs a warning and
+// returns false.
+func AnnotationToBool(spec *specs.Spec, annotation string) bool {
+	val, ok := spec.Annotations[annotation]
+	if !ok {
+		return false
+	}
+	ret, err := strconv.ParseBool(val)
+	if err != nil {
+		log.Warningf("Failed to parse annotation %q=%q as a bool: %v", annotation, val, err)
+		return false
+	}
+	return ret
+}