From 396d0beb1d3b496f850f108b967c09a2abe4fc41 Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Wed, 4 Oct 2023 21:40:52 +0800 Subject: [PATCH 1/2] vfs: introduce CtxRestoreFilesystemFDMap This patch introduces CtxRestoreFilesystemFDMap in vfs to replace the CtxRestoreServerFDMap defined in gofer, which is a preparation for adding the checkpoint/restore support in EROFS. It allows the configureRestore() in boot/vfs.go to be agnostic to the filesystems that will be used in the sentry when configuring restore. Signed-off-by: Tiwei Bie --- pkg/sentry/fsimpl/gofer/save_restore.go | 11 +---------- pkg/sentry/vfs/context.go | 5 +++++ runsc/boot/vfs.go | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pkg/sentry/fsimpl/gofer/save_restore.go b/pkg/sentry/fsimpl/gofer/save_restore.go index 1ae976d4a..132a5f1a3 100644 --- a/pkg/sentry/fsimpl/gofer/save_restore.go +++ b/pkg/sentry/fsimpl/gofer/save_restore.go @@ -29,15 +29,6 @@ import ( "gvisor.dev/gvisor/pkg/sentry/vfs" ) -type saveRestoreContextID int - -const ( - // CtxRestoreServerFDMap is a Context.Value key for a map[string]int - // mapping filesystem unique IDs (cf. InternalFilesystemOptions.UniqueID) - // to host FDs. - CtxRestoreServerFDMap saveRestoreContextID = iota -) - // +stateify savable type savedDentryRW struct { read bool @@ -181,7 +172,7 @@ func (d *dentry) loadParent(parent *dentry) { // CompleteRestore implements // vfs.FilesystemImplSaveRestoreExtension.CompleteRestore. func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRestoreOptions) error { - fdmapv := ctx.Value(CtxRestoreServerFDMap) + fdmapv := ctx.Value(vfs.CtxRestoreFilesystemFDMap) if fdmapv == nil { return fmt.Errorf("no server FD map available") } diff --git a/pkg/sentry/vfs/context.go b/pkg/sentry/vfs/context.go index 70a78872c..ff1281023 100644 --- a/pkg/sentry/vfs/context.go +++ b/pkg/sentry/vfs/context.go @@ -27,6 +27,11 @@ const ( // CtxRoot is a Context.Value key for a VFS root. CtxRoot + + // CtxRestoreFilesystemFDMap is a Context.Value key for a map[string]int + // mapping filesystem unique IDs (cf. gofer.InternalFilesystemOptions.UniqueID) + // to host FDs. + CtxRestoreFilesystemFDMap ) // MountNamespaceFromContext returns the MountNamespace used by ctx. If ctx is diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index b1e5f5215..39bc3b435 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -1087,7 +1087,7 @@ func (c *containerMounter) configureRestore(ctx context.Context) (context.Contex fdmap[submount.mount.Destination] = submount.goferFD.Release() } } - return context.WithValue(ctx, gofer.CtxRestoreServerFDMap, fdmap), nil + return context.WithValue(ctx, vfs.CtxRestoreFilesystemFDMap, fdmap), nil } func createDeviceFiles(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry) error { From 78a667cb8a0bd779535cf8523bbd26ac84e5a419 Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Wed, 4 Oct 2023 21:40:52 +0800 Subject: [PATCH 2/2] erofs: add checkpoint/restore support Updates #8956 Signed-off-by: Tiwei Bie --- pkg/erofs/erofs.go | 19 +++++++++++++++ pkg/sentry/fsimpl/erofs/erofs.go | 27 +++++++++++++++++++++ pkg/sentry/fsimpl/erofs/regular_file.go | 10 ++++++++ pkg/sentry/fsimpl/erofs/save_restore.go | 31 ++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index 306df5108..2c26ca029 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -223,6 +223,25 @@ func OpenImage(src *os.File) (*Image, error) { return i, nil } +// UpdateImage updates the underlying image file. This is typically used in checkpoint/restore. +// +// On success, the ownership of src is transferred to Image. +// +// Preconditions: +// - i.src == nil. +// - i.bytes == nil. +func (i *Image) UpdateImage(src *os.File) error { + newImage, err := OpenImage(src) + if err != nil { + return err + } + if newImage.sb != i.sb { + return fmt.Errorf("superblock mismatch detected, got %+v, expected %+v", newImage.sb, i.sb) + } + *i = *newImage + return nil +} + // Close closes the image. func (i *Image) Close() { unix.Munmap(i.bytes) diff --git a/pkg/sentry/fsimpl/erofs/erofs.go b/pkg/sentry/fsimpl/erofs/erofs.go index 1ec9fc4de..24e003cea 100644 --- a/pkg/sentry/fsimpl/erofs/erofs.go +++ b/pkg/sentry/fsimpl/erofs/erofs.go @@ -28,6 +28,7 @@ import ( "gvisor.dev/gvisor/pkg/erofs" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/vfs" ) @@ -51,6 +52,7 @@ type filesystem struct { // Immutable options. mopts string + iopts InternalFilesystemOptions // devMinor is the filesystem's minor device number. devMinor is immutable. devMinor uint32 @@ -70,6 +72,16 @@ type filesystem struct { inodeBuckets []inodeBucket } +// InternalFilesystemOptions may be passed as +// vfs.GetFilesystemOptions.InternalData to FilesystemType.GetFilesystem. +// +// +stateify savable +type InternalFilesystemOptions struct { + // If UniqueID is non-empty, it is an opaque string used to reassociate the + // filesystem with a new image FD during restoration from checkpoint. + UniqueID string +} + // Name implements vfs.FilesystemType.Name. func (FilesystemType) Name() string { return Name @@ -98,6 +110,12 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt } cu.Add(func() { image.Close() }) + iopts, ok := opts.InternalData.(InternalFilesystemOptions) + if opts.InternalData != nil && !ok { + ctx.Warningf("erofs.FilesystemType.GetFilesystem: GetFilesystemOptions.InternalData has type %T, wanted erofs.InternalFilesystemOptions", opts.InternalData) + return nil, nil, linuxerr.EINVAL + } + devMinor, err := vfsObj.GetAnonBlockDevMinor() if err != nil { return nil, nil, err @@ -105,6 +123,7 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt fs := &filesystem{ mopts: opts.Data, + iopts: iopts, image: image, devMinor: devMinor, mf: imageMemmapFile{image: image}, @@ -243,6 +262,14 @@ type inode struct { // +checklocks:dirMu dirents []vfs.Dirent `state:"nosave"` + // mapsMu protects mappings. + mapsMu sync.Mutex `state:"nosave"` + + // mappings tracks the mappings of the file into memmap.MappingSpaces + // if this inode represents a regular file. + // +checklocks:mapsMu + mappings memmap.MappingSet + // locks supports POSIX and BSD style locks. locks vfs.FileLocks diff --git a/pkg/sentry/fsimpl/erofs/regular_file.go b/pkg/sentry/fsimpl/erofs/regular_file.go index a2e541e2f..088dae932 100644 --- a/pkg/sentry/fsimpl/erofs/regular_file.go +++ b/pkg/sentry/fsimpl/erofs/regular_file.go @@ -131,15 +131,22 @@ func (fd *regularFileFD) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpt // AddMapping implements memmap.Mappable.AddMapping. func (i *inode) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error { + i.mapsMu.Lock() + i.mappings.AddMapping(ms, ar, offset, writable) + i.mapsMu.Unlock() return nil } // RemoveMapping implements memmap.Mappable.RemoveMapping. func (i *inode) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) { + i.mapsMu.Lock() + i.mappings.RemoveMapping(ms, ar, offset, writable) + i.mapsMu.Unlock() } // CopyMapping implements memmap.Mappable.CopyMapping. func (i *inode) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error { + i.AddMapping(ctx, ms, dstAR, offset, writable) return nil } @@ -175,6 +182,9 @@ func (i *inode) Translate(ctx context.Context, required, optional memmap.Mappabl // InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable. func (i *inode) InvalidateUnsavable(ctx context.Context) error { + i.mapsMu.Lock() + i.mappings.InvalidateAll(memmap.InvalidateOpts{}) + i.mapsMu.Unlock() return nil } diff --git a/pkg/sentry/fsimpl/erofs/save_restore.go b/pkg/sentry/fsimpl/erofs/save_restore.go index d979881d6..c1a73bf3e 100644 --- a/pkg/sentry/fsimpl/erofs/save_restore.go +++ b/pkg/sentry/fsimpl/erofs/save_restore.go @@ -14,7 +14,36 @@ package erofs -// TODO: support checkpoint/restore. +import ( + "fmt" + "os" + + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// Compile-time assertion that filesystem implements vfs.FilesystemImplSaveRestoreExtension. +var _ = vfs.FilesystemImplSaveRestoreExtension((*filesystem)(nil)) + +// PreprareSave implements vfs.FilesystemImplSaveRestoreExtension.PrepareSave. +func (fs *filesystem) PrepareSave(ctx context.Context) error { + return nil +} + +// CompleteRestore implements +// vfs.FilesystemImplSaveRestoreExtension.CompleteRestore. +func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRestoreOptions) error { + fdmapv := ctx.Value(vfs.CtxRestoreFilesystemFDMap) + if fdmapv == nil { + return fmt.Errorf("no image FD map available") + } + fdmap := fdmapv.(map[string]int) + fd, ok := fdmap[fs.iopts.UniqueID] + if !ok { + return fmt.Errorf("no image FD available for filesystem with unique ID %q", fs.iopts.UniqueID) + } + return fs.image.UpdateImage(os.NewFile(uintptr(fd), "EROFS image file")) +} // saveParent is called by stateify. func (d *dentry) saveParent() *dentry {