diff --git a/pkg/sentry/control/state.go b/pkg/sentry/control/state.go index 51619ae27..e25757953 100644 --- a/pkg/sentry/control/state.go +++ b/pkg/sentry/control/state.go @@ -77,7 +77,7 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error { log.Infof("Save succeeded: exiting...") s.Kernel.SetSaveSuccess(false /* autosave */) } else { - log.Warningf("Save failed: exiting...") + log.Warningf("Save failed: %v", err) s.Kernel.SetSaveError(err) } if !o.Resume { diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index 8c63ef9ee..9eb61dba9 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -351,6 +351,13 @@ type Kernel struct { // devGofers maps container ID to its device gofer client. devGofers map[string]*devutil.GoferClient `state:"nosave"` devGofersMu sync.Mutex `state:"nosave"` + + // containerNames store the container name based on their container ID. + // Names are preserved between save/restore session, while IDs can change. + // + // Mapping: cid -> name. + // It's protected by extMu. + containerNames map[string]string } // InitKernelArgs holds arguments to Init. @@ -457,6 +464,7 @@ func (k *Kernel) Init(args InitKernelArgs) error { args.MaxFDLimit = MaxFdLimit } k.MaxFDLimit.Store(args.MaxFDLimit) + k.containerNames = make(map[string]string) ctx := k.SupervisorContext() if err := k.vfs.Init(ctx); err != nil { @@ -1976,3 +1984,31 @@ func (k *Kernel) cleaupDevGofers() { } k.devGofers = nil } + +// RegisterContainerName registers a container name for a given container ID. +func (k *Kernel) RegisterContainerName(cid, containerName string) { + k.extMu.Lock() + defer k.extMu.Unlock() + k.containerNames[cid] = containerName +} + +// RestoreContainerMapping remaps old container IDs to new ones after a restore. +// containerIDs maps "name -> new container ID". Note that container names remain +// constant between restore sessions. +func (k *Kernel) RestoreContainerMapping(containerIDs map[string]string) { + k.extMu.Lock() + defer k.extMu.Unlock() + + // Delete mapping from old session and replace with new values. + k.containerNames = make(map[string]string) + for name, cid := range containerIDs { + k.containerNames[cid] = name + } +} + +// TaskContainerName returns the container name for a given task. +func (k *Kernel) TaskContainerName(task *Task) string { + k.extMu.Lock() + defer k.extMu.Unlock() + return k.containerNames[task.ContainerID()] +} diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 4c92deff8..b557aa849 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -102,6 +102,7 @@ go_library( "//pkg/sentry/vfs", "//pkg/sentry/watchdog", "//pkg/sighandling", + "//pkg/state/statefile", "//pkg/sync", "//pkg/tcpip", "//pkg/tcpip/link/ethernet", diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index 65ce8d1fd..3e9a82ed5 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -18,6 +18,8 @@ import ( "errors" "fmt" "path" + "strconv" + "sync" gtime "time" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -34,6 +36,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/seccheck" "gvisor.dev/gvisor/pkg/sentry/socket/netstack" "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/pkg/urpc" "gvisor.dev/gvisor/runsc/boot/procfs" "gvisor.dev/gvisor/runsc/config" @@ -66,6 +69,9 @@ const ( // ContMgrRestore restores a container from a statefile. ContMgrRestore = "containerManager.Restore" + // ContMgrRestoreSubcontainer restores a container from a statefile. + ContMgrRestoreSubcontainer = "containerManager.RestoreSubcontainer" + // ContMgrSignal sends a signal to a container. ContMgrSignal = "containerManager.Signal" @@ -209,13 +215,17 @@ type containerManager struct { // be started. startChan chan struct{} - // startResultChan is used to signal when the root container has + // startResultChan is used to signal when the root container has // started. Any errors encountered during startup will be sent to the // channel. A nil value indicates success. startResultChan chan error // l is the loader that creates containers and sandboxes. l *Loader + + // restorer is set when the sandbox in being restored. It stores the state + // of all containers and perform all actions required by restore. + restorer *restorer } // StartRoot will start the root container process. @@ -413,16 +423,7 @@ func (cm *containerManager) ExecuteAsync(args *control.ExecArgs, pid *int32) err // Checkpoint pauses a sandbox and saves its state. func (cm *containerManager) Checkpoint(o *control.SaveOpts, _ *struct{}) error { log.Debugf("containerManager.Checkpoint") - // TODO(gvisor.dev/issues/6243): save/restore not supported w/ hostinet - if cm.l.root.conf.Network == config.NetworkHost { - return errors.New("checkpoint not supported when using hostinet") - } - - state := control.State{ - Kernel: cm.l.k, - Watchdog: cm.l.watchdog, - } - return state.Save(o, nil) + return cm.l.save(o) } // PortForwardOpts contains options for port forwarding to a port in a @@ -480,7 +481,6 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { if err != nil { return err } - defer stateFile.Close() var stat unix.Stat_t if err := unix.Fstat(stateFile.FD(), &stat); err != nil { @@ -490,7 +490,8 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { return fmt.Errorf("statefile cannot be empty") } - r := restorer{container: &cm.l.root, stateFile: stateFile} + cm.restorer = &restorer{restoreDone: cm.onRestoreDone, stateFile: stateFile} + cm.l.restoreWaiters = sync.NewCond(&cm.l.mu) cm.l.state = restoring fileIdx := 1 @@ -499,13 +500,12 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { if err != nil { return err } - defer pagesFile.Close() fileIdx++ - r.pagesFile = pagesFile + cm.restorer.pagesFile = pagesFile } if o.HaveDeviceFile { - r.deviceFile, err = o.ReleaseFD(fileIdx) + cm.restorer.deviceFile, err = o.ReleaseFD(fileIdx) if err != nil { return err } @@ -519,10 +519,121 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { // Pause the kernel while we build a new one. cm.l.k.Pause() - if err := r.restore(cm.l); err != nil { + metadata, err := statefile.MetadataUnsafe(cm.restorer.stateFile) + if err != nil { + return fmt.Errorf("reading metadata from statefile: %w", err) + } + var count int + countStr, ok := metadata["container_count"] + if !ok { + // TODO(gvisor.dev/issue/1956): Add container count with syscall save + // trigger. For now, assume that only a single container exists if metadata + // isn't present. + // + // -return errors.New("container count not present in state file") + count = 1 + } else { + count, err = strconv.Atoi(countStr) + if err != nil { + return fmt.Errorf("invalid container count: %w", err) + } + if count < 1 { + return fmt.Errorf("invalid container count value: %v", count) + } + } + cm.restorer.totalContainers = count + log.Infof("Restoring a total of %d containers", cm.restorer.totalContainers) + + if _, err := unix.Seek(stateFile.FD(), 0, 0); err != nil { + return fmt.Errorf("rewinding state file: %w", err) + } + + return cm.restorer.restoreContainerInfo(cm.l, &cm.l.root) +} + +func (cm *containerManager) onRestoreDone() error { + if err := cm.onStart(); err != nil { return err } - return cm.onStart() + + cm.l.restoreWaiters.Broadcast() + cm.restorer = nil + return nil +} + +func (cm *containerManager) RestoreSubcontainer(args *StartArgs, _ *struct{}) error { + log.Debugf("containerManager.RestoreSubcontainer, cid: %s, args: %+v", args.CID, args) + + if cm.l.state != restoring { + return fmt.Errorf("sandbox is not being restored, cannot restore subcontainer") + } + + // Validate arguments. + if args.Spec == nil { + return errors.New("start arguments missing spec") + } + if args.Conf == nil { + return errors.New("start arguments missing config") + } + if args.CID == "" { + return errors.New("start argument missing container ID") + } + expectedFDs := 1 // At least one FD for the root filesystem. + expectedFDs += args.NumGoferFilestoreFDs + if !args.Spec.Process.Terminal { + expectedFDs += 3 + } + if len(args.Files) < expectedFDs { + return fmt.Errorf("restore arguments must contain at least %d FDs, but only got %d", expectedFDs, len(args.Files)) + } + + // All validation passed, logs the spec for debugging. + specutils.LogSpecDebug(args.Spec, args.Conf.OCISeccomp) + + goferFiles := args.Files + var stdios []*fd.FD + if !args.Spec.Process.Terminal { + // When not using a terminal, stdios come as the first 3 files in the + // payload. + var err error + stdios, err = fd.NewFromFiles(goferFiles[:3]) + if err != nil { + return fmt.Errorf("error dup'ing stdio files: %w", err) + } + goferFiles = goferFiles[3:] + } + + var goferFilestoreFDs []*fd.FD + for i := 0; i < args.NumGoferFilestoreFDs; i++ { + overlayFilestoreFD, err := fd.NewFromFile(goferFiles[i]) + if err != nil { + return fmt.Errorf("error dup'ing overlay filestore file: %w", err) + } + goferFilestoreFDs = append(goferFilestoreFDs, overlayFilestoreFD) + } + goferFiles = goferFiles[args.NumGoferFilestoreFDs:] + + var devGoferFD *fd.FD + if args.IsDevIoFilePresent { + var err error + devGoferFD, err = fd.NewFromFile(goferFiles[0]) + if err != nil { + return fmt.Errorf("error dup'ing dev gofer file: %w", err) + } + goferFiles = goferFiles[1:] + } + + goferFDs, err := fd.NewFromFiles(goferFiles) + if err != nil { + return fmt.Errorf("error dup'ing gofer files: %w", err) + } + + if err := cm.restorer.restoreSubcontainer(args.Spec, args.Conf, cm.l, args.CID, stdios, goferFDs, goferFilestoreFDs, devGoferFD, args.GoferMountConfs); err != nil { + log.Debugf("containerManager.RestoreSubcontainer failed, cid: %s, args: %+v, err: %v", args.CID, args, err) + return err + } + log.Debugf("Container restored, cid: %s", args.CID) + return nil } // Wait waits for the init process in the given container. diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 872b108df..582fd5d45 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -190,6 +190,8 @@ type Loader struct { // restore is set to true if we are restoring a container. restore bool + restoreWaiters *sync.Cond + // sandboxID is the ID for the whole sandbox. sandboxID string @@ -209,6 +211,8 @@ type Loader struct { // sharedMounts holds VFS mounts that may be shared between containers within // the same pod. It is mapped by mount source. + // + // sharedMounts is guarded by mu. sharedMounts map[string]*vfs.Mount // processes maps containers init process and invocation of exec. Root @@ -218,6 +222,13 @@ type Loader struct { // processes is guarded by mu. processes map[execID]*execProcess + // containerIDs store container names and IDs to assist with restore and container + // naming when user didn't provide one. + // + // Mapping: name -> cid. + // processes is guarded by mu. + containerIDs map[string]string + // portForwardProxies is a list of active port forwarding connections. // // portForwardProxies is guarded by mu. @@ -382,9 +393,20 @@ func New(args Args) (*Loader, error) { kernel.IOUringEnabled = args.Conf.IOUring - info := containerInfo{ + eid := execID{cid: args.ID} + l := &Loader{ + sandboxID: args.ID, + processes: map[execID]*execProcess{eid: {}}, + sharedMounts: make(map[string]*vfs.Mount), + stopProfiling: stopProfiling, + productName: args.ProductName, + containerIDs: map[string]string{}, + } + + containerName := l.registerContainerLocked(args.Spec, args.ID) + l.root = containerInfo{ cid: args.ID, - containerName: specutils.ContainerName(args.Spec), + containerName: containerName, conf: args.Conf, spec: args.Spec, goferMountConfs: args.GoferMountConfs, @@ -409,25 +431,25 @@ func New(args Args) (*Loader, error) { if err != nil { return nil, fmt.Errorf("dup3 of stdios failed: %w", err) } - info.stdioFDs = append(info.stdioFDs, fd.New(newfd)) + l.root.stdioFDs = append(l.root.stdioFDs, fd.New(newfd)) _ = unix.Close(stdioFD) newfd++ } for _, goferFD := range args.GoferFDs { - info.goferFDs = append(info.goferFDs, fd.New(goferFD)) + l.root.goferFDs = append(l.root.goferFDs, fd.New(goferFD)) } for _, filestoreFD := range args.GoferFilestoreFDs { - info.goferFilestoreFDs = append(info.goferFilestoreFDs, fd.New(filestoreFD)) + l.root.goferFilestoreFDs = append(l.root.goferFilestoreFDs, fd.New(filestoreFD)) } if args.DevGoferFD >= 0 { - info.devGoferFD = fd.New(args.DevGoferFD) + l.root.devGoferFD = fd.New(args.DevGoferFD) } if args.ExecFD >= 0 { - info.execFD = fd.New(args.ExecFD) + l.root.execFD = fd.New(args.ExecFD) } for _, customFD := range args.PassFDs { - info.passFDs = append(info.passFDs, fdMapping{ + l.root.passFDs = append(l.root.passFDs, fdMapping{ host: fd.New(customFD.Host), guest: customFD.Guest, }) @@ -441,27 +463,25 @@ func New(args Args) (*Loader, error) { if specutils.NVProxyEnabled(args.Spec, args.Conf) && p.OwnsPageTables() { return nil, fmt.Errorf("--nvproxy is incompatible with platform %s: owns page tables", args.Conf.Platform) } - k := &kernel.Kernel{ - Platform: p, - } + l.k = &kernel.Kernel{Platform: p} // Create memory file. mf, err := createMemoryFile() if err != nil { return nil, fmt.Errorf("creating memory file: %w", err) } - k.SetMemoryFile(mf) + l.k.SetMemoryFile(mf) // Create VDSO. // // Pass k as the platform since it is savable, unlike the actual platform. - vdso, err := loader.PrepareVDSO(k.MemoryFile()) + vdso, err := loader.PrepareVDSO(l.k.MemoryFile()) if err != nil { return nil, fmt.Errorf("creating vdso: %w", err) } // Create timekeeper. - tk := kernel.NewTimekeeper(k.MemoryFile(), vdso.ParamPage.FileRange()) + tk := kernel.NewTimekeeper(l.k.MemoryFile(), vdso.ParamPage.FileRange()) tk.SetClocks(time.NewCalibratedClocks()) if err := enableStrace(args.Conf); err != nil { @@ -473,7 +493,7 @@ func New(args Args) (*Loader, error) { return nil, fmt.Errorf("getting root credentials") } // Create root network namespace/stack. - netns, err := newRootNetworkNamespace(args.Conf, tk, k, creds.UserNamespace) + netns, err := newRootNetworkNamespace(args.Conf, tk, l.k, creds.UserNamespace) if err != nil { return nil, fmt.Errorf("creating network: %w", err) } @@ -513,7 +533,7 @@ func New(args Args) (*Loader, error) { } // Initiate the Kernel object, which is required by the Context passed // to createVFS in order to mount (among other things) procfs. - if err = k.Init(kernel.InitKernelArgs{ + if err = l.k.Init(kernel.InitKernelArgs{ FeatureSet: cpuid.HostFeatureSet().Fixed(), Timekeeper: tk, RootUserNamespace: creds.UserNamespace, @@ -528,7 +548,7 @@ func New(args Args) (*Loader, error) { return nil, fmt.Errorf("initializing kernel: %w", err) } - if err := registerFilesystems(k, &info); err != nil { + if err := registerFilesystems(l.k, &l.root); err != nil { return nil, fmt.Errorf("registering filesystems: %w", err) } @@ -544,30 +564,30 @@ func New(args Args) (*Loader, error) { // Create a watchdog. dogOpts := watchdog.DefaultOpts dogOpts.TaskTimeoutAction = args.Conf.WatchdogAction - dog := watchdog.New(k, dogOpts) + l.watchdog = watchdog.New(l.k, dogOpts) - procArgs, err := createProcessArgs(args.ID, args.Spec, args.Conf, creds, k, k.RootPIDNamespace()) + procArgs, err := createProcessArgs(args.ID, args.Spec, args.Conf, creds, l.k, l.k.RootPIDNamespace()) if err != nil { return nil, fmt.Errorf("creating init process for root container: %w", err) } - info.procArgs = procArgs + l.root.procArgs = procArgs if err := initCompatLogs(args.UserLogFD); err != nil { return nil, fmt.Errorf("initializing compat logs: %w", err) } - mountHints, err := NewPodMountHints(args.Spec) + l.mountHints, err = NewPodMountHints(args.Spec) if err != nil { return nil, fmt.Errorf("creating pod mount hints: %w", err) } // Set up host mount that will be used for imported fds. - hostFilesystem, err := host.NewFilesystem(k.VFS()) + hostFilesystem, err := host.NewFilesystem(l.k.VFS()) if err != nil { return nil, fmt.Errorf("failed to create hostfs filesystem: %w", err) } - defer hostFilesystem.DecRef(k.SupervisorContext()) - k.SetHostMount(k.VFS().NewDisconnectedMount(hostFilesystem, nil, &vfs.MountOptions{})) + defer hostFilesystem.DecRef(l.k.SupervisorContext()) + l.k.SetHostMount(l.k.VFS().NewDisconnectedMount(hostFilesystem, nil, &vfs.MountOptions{})) if args.PodInitConfigFD >= 0 { if err := setupSeccheck(args.PodInitConfigFD, args.SinkFDs); err != nil { @@ -575,18 +595,7 @@ func New(args Args) (*Loader, error) { } } - eid := execID{cid: args.ID} - l := &Loader{ - k: k, - watchdog: dog, - sandboxID: args.ID, - processes: map[execID]*execProcess{eid: {}}, - mountHints: mountHints, - sharedMounts: make(map[string]*vfs.Mount), - root: info, - stopProfiling: stopProfiling, - productName: args.ProductName, - } + l.k.RegisterContainerName(args.ID, l.root.containerName) // We don't care about child signals; some platforms can generate a // tremendous number of useless ones (I'm looking at you, ptrace). @@ -922,9 +931,10 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st pidns = l.k.RootPIDNamespace() } + containerName := l.registerContainerLocked(spec, cid) info := &containerInfo{ cid: cid, - containerName: specutils.ContainerName(spec), + containerName: containerName, conf: conf, spec: spec, goferFDs: goferFDs, @@ -990,6 +1000,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st }) } + l.k.RegisterContainerName(cid, info.containerName) l.k.StartProcess(ep.tg) // No more failures from this point on. cu.Release() @@ -1258,9 +1269,26 @@ func (l *Loader) executeAsync(args *control.ExecArgs) (kernel.ThreadID, error) { func (l *Loader) waitContainer(cid string, waitStatus *uint32) error { // Don't defer unlock, as doing so would make it impossible for // multiple clients to wait on the same container. - tg, err := l.threadGroupFromID(execID{cid: cid}) + key := execID{cid: cid} + tg, err := l.threadGroupFromID(key) if err != nil { - return fmt.Errorf("can't wait for container %q: %w", cid, err) + l.mu.Lock() + // Extra handling is needed if the container is restoring. + if l.state != restoring { + l.mu.Unlock() + return err + } + // Container could be restoring, first check if container exists. + if _, err := l.findProcessLocked(key); err != nil { + l.mu.Unlock() + return err + } + log.Infof("Waiting for container being restored, CID: %q", cid) + l.restoreWaiters.Wait() + l.mu.Unlock() + + log.Infof("Restore is completed, trying to wait for container %q again.", cid) + return l.waitContainer(cid, waitStatus) } // If the thread either has already exited or exits during waiting, @@ -1583,9 +1611,9 @@ func (l *Loader) threadGroupFromID(key execID) (*kernel.ThreadGroup, error) { // error if execution ID is invalid or if the container cannot be found (maybe // it has been deleted). Caller must hold 'mu'. func (l *Loader) tryThreadGroupFromIDLocked(key execID) (*kernel.ThreadGroup, error) { - ep := l.processes[key] - if ep == nil { - return nil, fmt.Errorf("container %q not found", key.cid) + ep, err := l.findProcessLocked(key) + if err != nil { + return nil, err } return ep.tg, nil } @@ -1595,9 +1623,9 @@ func (l *Loader) tryThreadGroupFromIDLocked(key execID) (*kernel.ThreadGroup, er // execution ID is invalid or if the container cannot be found (maybe it has // been deleted). Caller must hold 'mu'. func (l *Loader) ttyFromIDLocked(key execID) (*host.TTYFileDescription, error) { - ep := l.processes[key] - if ep == nil { - return nil, fmt.Errorf("container %q not found", key.cid) + ep, err := l.findProcessLocked(key) + if err != nil { + return nil, err } return ep.tty, nil } @@ -1776,6 +1804,33 @@ func (l *Loader) networkStats() ([]*NetworkInterface, error) { return stats, nil } +func (l *Loader) findProcessLocked(key execID) (*execProcess, error) { + ep := l.processes[key] + if ep == nil { + return nil, fmt.Errorf("container %q not found", key.cid) + } + return ep, nil +} + +func (l *Loader) registerContainer(spec *specs.Spec, cid string) string { + l.mu.Lock() + defer l.mu.Unlock() + + return l.registerContainerLocked(spec, cid) +} + +func (l *Loader) registerContainerLocked(spec *specs.Spec, cid string) string { + containerName := specutils.ContainerName(spec) + if len(containerName) == 0 { + // If no name was provided, require containers to be restored in the same order + // they were created. + containerName = "__no_name_" + strconv.Itoa(len(l.containerIDs)) + } + + l.containerIDs[containerName] = cid + return containerName +} + func (l *Loader) containerRuntimeState(cid string) ContainerRuntimeState { l.mu.Lock() defer l.mu.Unlock() diff --git a/runsc/boot/restore.go b/runsc/boot/restore.go index 3a1c15b08..b58e03e0e 100644 --- a/runsc/boot/restore.go +++ b/runsc/boot/restore.go @@ -15,13 +15,19 @@ package boot import ( + "errors" "fmt" "io" + "strconv" + time2 "time" + specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/fd" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/fsimpl/host" "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" @@ -32,8 +38,10 @@ import ( "gvisor.dev/gvisor/pkg/sentry/time" "gvisor.dev/gvisor/pkg/sentry/vfs" "gvisor.dev/gvisor/pkg/sentry/watchdog" + "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/tcpip/stack" "gvisor.dev/gvisor/runsc/boot/pprof" + "gvisor.dev/gvisor/runsc/config" ) const ( @@ -45,11 +53,74 @@ const ( CheckpointPagesFileName = "pages.img" ) +// restorer manages a restore session for a sandbox. It stores information about +// all containers and triggers the full sandbox restore after the last +// container is restored. type restorer struct { - container *containerInfo - stateFile io.Reader - pagesFile *fd.FD + mu sync.Mutex + + // totalContainers is the number of containers expected to be restored in + // the sandbox. Sandbox restore can only happen, after all containers have + // been restored. + totalContainers int + + // containers is the list of containers restored so far. + containers []*containerInfo + + // Files used by restore to rehydrate the state. + stateFile io.ReadCloser + pagesFile *fd.FD + + // deviceFile is the required to start the platform. deviceFile *fd.FD + + // restoreDone is a callback triggered when restore is successful. + restoreDone func() error +} + +func (r *restorer) restoreSubcontainer(spec *specs.Spec, conf *config.Config, l *Loader, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, devGoferFD *fd.FD, goferMountConfs []GoferMountConf) error { + containerName := l.registerContainer(spec, cid) + info := &containerInfo{ + cid: cid, + containerName: containerName, + conf: conf, + spec: spec, + stdioFDs: stdioFDs, + goferFDs: goferFDs, + devGoferFD: devGoferFD, + goferFilestoreFDs: goferFilestoreFDs, + goferMountConfs: goferMountConfs, + } + return r.restoreContainerInfo(l, info) +} + +func (r *restorer) restoreContainerInfo(l *Loader, info *containerInfo) error { + r.mu.Lock() + defer r.mu.Unlock() + + for _, container := range r.containers { + if container.containerName == info.containerName { + return fmt.Errorf("container %q already restored", info.containerName) + } + if container.cid == info.cid { + return fmt.Errorf("container CID %q already belongs to container %q", info.cid, container.containerName) + } + } + + r.containers = append(r.containers, info) + + log.Infof("Restored container %d of %d", len(r.containers), r.totalContainers) + if log.IsLogging(log.Debug) { + for i, fd := range info.stdioFDs { + log.Debugf("Restore app FD: %d host FD: %d", i, fd.FD()) + } + } + + if len(r.containers) == r.totalContainers { + // Trigger the restore if this is the last container. + return r.restore(l) + } + return nil } func createNetworkNamespaceForRestore(l *Loader) (*stack.Stack, *inet.Namespace, error) { @@ -74,6 +145,8 @@ func createNetworkNamespaceForRestore(l *Loader) (*stack.Stack, *inet.Namespace, } func (r *restorer) restore(l *Loader) error { + log.Infof("Starting to restore %d containers", len(r.containers)) + // Create a new root network namespace with the network stack of the // old kernel to preserve the existing network configuration. oldStack, netns, err := createNetworkNamespaceForRestore(l) @@ -126,23 +199,28 @@ func (r *restorer) restore(l *Loader) error { ctx = context.WithValue(ctx, stack.CtxRestoreStack, oldStack) } - // TODO(b/298078576): Need to process hints here probably - mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, l.productName, l.sandboxID) - fdmap := make(map[vfs.RestoreID]int) mfmap := make(map[string]*pgalloc.MemoryFile) - if err := mntr.configureRestore(fdmap, mfmap); err != nil { - return fmt.Errorf("configuring filesystem restore: %v", err) - } - for appFD, fd := range r.container.stdioFDs { - key := host.MakeRestoreID(r.container.containerName, appFD) - fdmap[key] = fd.Release() - } - for _, customFD := range r.container.passFDs { - key := host.MakeRestoreID(r.container.containerName, customFD.guest) - fdmap[key] = customFD.host.FD() + for _, cont := range r.containers { + // TODO(b/298078576): Need to process hints here probably + mntr := newContainerMounter(cont, l.k, l.mountHints, l.sharedMounts, l.productName, cont.cid) + if err = mntr.configureRestore(fdmap, mfmap); err != nil { + return fmt.Errorf("configuring filesystem restore: %v", err) + } + + for i, fd := range cont.stdioFDs { + key := host.MakeRestoreID(cont.containerName, i) + fdmap[key] = fd.Release() + } + for _, customFD := range cont.passFDs { + key := host.MakeRestoreID(cont.containerName, customFD.guest) + fdmap[key] = customFD.host.FD() + } } + + log.Debugf("Restore using fdmap: %v", fdmap) ctx = context.WithValue(ctx, vfs.CtxRestoreFilesystemFDMap, fdmap) + log.Debugf("Restore using mfmap: %v", fdmap) ctx = context.WithValue(ctx, pgalloc.CtxMemoryFileMap, mfmap) // Load the state. @@ -154,6 +232,7 @@ func (r *restorer) restore(l *Loader) error { // Since we have a new kernel we also must make a new watchdog. dogOpts := watchdog.DefaultOpts dogOpts.TaskTimeoutAction = l.root.conf.WatchdogAction + dogOpts.StartupTimeout = 3 * time2.Minute // Give extra time for all containers to restore. dog := watchdog.New(l.k, dogOpts) // Change the loader fields to reflect the changes made when restoring. @@ -161,23 +240,40 @@ func (r *restorer) restore(l *Loader) error { l.root.procArgs = kernel.CreateProcessArgs{} l.restore = true - // Reinitialize the sandbox ID and processes map. Note that it doesn't - // restore the state of multiple containers, nor exec processes. - l.sandboxID = r.container.cid + l.sandboxID = l.root.cid l.mu.Lock() - defer l.mu.Unlock() + cu := cleanup.Make(func() { + l.mu.Unlock() + }) + defer cu.Clean() - // Set new container ID if it has changed. - tasks := l.k.TaskSet().Root.Tasks() - if tasks[0].ContainerID() != l.sandboxID { // There must be at least 1 task. - for _, task := range tasks { - task.RestoreContainerID(l.sandboxID) + // Update all tasks in the system with their respective new container IDs. + for _, task := range l.k.TaskSet().Root.Tasks() { + name := l.k.TaskContainerName(task) + newCid, ok := l.containerIDs[name] + if !ok { + return fmt.Errorf("unable to remap task with CID %q (name: %q). Available names: %v", task.ContainerID(), name, l.containerIDs) + } + task.RestoreContainerID(newCid) + } + + // Rebuild `processes` map with containers' root process from the restored kernel. + for _, tg := range l.k.RootPIDNamespace().ThreadGroups() { + // Find all processes with no parent (root of execution), that were not started + // via a call to `exec`. + if tg.Leader().Parent() == nil && tg.Leader().Origin != kernel.OriginExec { + cid := tg.Leader().ContainerID() + proc := l.processes[execID{cid: cid}] + if proc == nil { + return fmt.Errorf("unable to find container root process with CID %q, processes: %v", cid, l.processes) + } + proc.tg = tg } } // Kill all processes that have been exec'd since they cannot be properly - // restored, since the caller is no longer connected. + // restored -- the caller is no longer connected. for _, tg := range l.k.RootPIDNamespace().ThreadGroups() { if tg.Leader().Origin == kernel.OriginExec { if err := l.k.SendExternalSignalThreadGroup(tg, &linux.SignalInfo{Signo: int32(linux.SIGKILL)}); err != nil { @@ -186,12 +282,39 @@ func (r *restorer) restore(l *Loader) error { } } - eid := execID{cid: l.sandboxID} - l.processes = map[execID]*execProcess{ - eid: { - tg: l.k.GlobalInit(), - }, + l.k.RestoreContainerMapping(l.containerIDs) + + // Release `l.mu` before calling into callbacks. + cu.Clean() + + if err := r.restoreDone(); err != nil { + return err } + r.stateFile.Close() + if r.pagesFile != nil { + r.pagesFile.Close() + } + + log.Infof("Restore successful") return nil } + +func (l *Loader) save(o *control.SaveOpts) error { + // TODO(gvisor.dev/issues/6243): save/restore not supported w/ hostinet + if l.root.conf.Network == config.NetworkHost { + return errors.New("checkpoint not supported when using hostinet") + } + + if o.Metadata == nil { + o.Metadata = make(map[string]string) + } + o.Metadata["container_count"] = strconv.Itoa(l.containerCount()) + + state := control.State{ + Kernel: l.k, + Watchdog: l.watchdog, + } + + return state.Save(o, nil) +} diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index eda9a7742..36e143b64 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -424,7 +424,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountH hints: hints, sharedMounts: sharedMounts, productName: productName, - containerID: info.procArgs.ContainerID, + containerID: info.cid, sandboxID: sandboxID, containerName: info.containerName, } diff --git a/runsc/container/container.go b/runsc/container/container.go index 77cd5e7f7..064c36862 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -423,14 +423,28 @@ func New(conf *config.Config, args Args) (*Container, error) { // Start starts running the containerized process inside the sandbox. func (c *Container) Start(conf *config.Config) error { log.Debugf("Start container, cid: %s", c.ID) + return c.startImpl(conf, "start", c.Sandbox.StartRoot, c.Sandbox.StartSubcontainer) +} +// Restore takes a container and replaces its kernel and file system +// to restore a container from its state file. +func (c *Container) Restore(conf *config.Config, imagePath string, direct bool) error { + log.Debugf("Restore container, cid: %s", c.ID) + + restore := func(conf *config.Config) error { + return c.Sandbox.Restore(conf, c.ID, imagePath, direct) + } + return c.startImpl(conf, "restore", restore, c.Sandbox.RestoreSubcontainer) +} + +func (c *Container) startImpl(conf *config.Config, action string, startRoot func(conf *config.Config) error, startSub func(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, devIOFile *os.File, goferConfs []boot.GoferMountConf) error) error { if err := c.Saver.lock(BlockAcquire); err != nil { return err } unlock := cleanup.Make(c.Saver.UnlockOrDie) defer unlock.Clean() - if err := c.requireStatus("start", Created); err != nil { + if err := c.requireStatus(action, Created); err != nil { return err } @@ -441,7 +455,7 @@ func (c *Container) Start(conf *config.Config) error { } if isRoot(c.Spec) { - if err := c.Sandbox.StartRoot(conf); err != nil { + if err := startRoot(conf); err != nil { return err } } else { @@ -492,7 +506,7 @@ func (c *Container) Start(conf *config.Config) error { stdios = []*os.File{os.Stdin, os.Stdout, os.Stderr} } - return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, devIOFile, goferConfs) + return startSub(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, devIOFile, goferConfs) }); err != nil { return err } @@ -523,32 +537,6 @@ func (c *Container) Start(conf *config.Config) error { return c.adjustGoferOOMScoreAdj() } -// Restore takes a container and replaces its kernel and file system -// to restore a container from its state file. -func (c *Container) Restore(conf *config.Config, imagePath string, direct bool) error { - log.Debugf("Restore container, cid: %s", c.ID) - if err := c.Saver.lock(BlockAcquire); err != nil { - return err - } - defer c.Saver.UnlockOrDie() - - if err := c.requireStatus("restore", Created); err != nil { - return err - } - - // "If any prestart hook fails, the runtime MUST generate an error, - // stop and destroy the container" -OCI spec. - if c.Spec.Hooks != nil && len(c.Spec.Hooks.StartContainer) > 0 { - log.Warningf("StartContainer hook skipped because running inside container namespace is not supported") - } - - if err := c.Sandbox.Restore(conf, c.ID, imagePath, direct); err != nil { - return err - } - c.changeStatus(Running) - return c.saveLocked() -} - // Run is a helper that calls Create + Start + Wait. func Run(conf *config.Config, args Args) (unix.WaitStatus, error) { log.Debugf("Run container, cid: %s, rootDir: %q", args.ID, conf.RootDir) diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index e64469898..a2dbefbd7 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -32,6 +32,7 @@ import ( "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/state/statefile" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/test/testutil" "gvisor.dev/gvisor/runsc/boot" @@ -99,6 +100,44 @@ func startContainers(conf *config.Config, specs []*specs.Spec, ids []string) ([] return containers, cu.Release(), nil } +func restoreContainers(conf *config.Config, specs []*specs.Spec, ids []string, imagePath string) ([]*Container, func(), error) { + if len(conf.RootDir) == 0 { + panic("conf.RootDir not set. Call testutil.SetupRootDir() to set.") + } + + cu := cleanup.Cleanup{} + defer cu.Clean() + + var containers []*Container + for i, spec := range specs { + bundleDir, cleanup, err := testutil.SetupBundleDir(spec) + if err != nil { + return nil, nil, fmt.Errorf("error setting up container: %v", err) + } + cu.Add(cleanup) + + args := Args{ + ID: ids[i], + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + return nil, nil, fmt.Errorf("error creating container: %v", err) + } + cu.Add(func() { cont.Destroy() }) + containers = append(containers, cont) + + if err := cont.Restore(conf, imagePath, false); err != nil { + return nil, nil, fmt.Errorf("error restoring container: %v", err) + } + + time.Sleep(100 * time.Millisecond) + } + + return containers, cu.Release(), nil +} + type execDesc struct { c *Container cmd []string @@ -2682,6 +2721,156 @@ func TestMultiContainerMemoryLeakStress(t *testing.T) { } } +// TestCheckpointRestore tests that checkpoint/restore works +// with multi-containers. +func TestMultiContainerCheckpointRestore(t *testing.T) { + // Skip overlay because test requires writing to host file. + for name, conf := range configs(t, true /* noOverlay */) { + t.Run(name, func(t *testing.T) { + rootDir, cleanup, err := testutil.SetupRootDir() + if err != nil { + t.Fatalf("error creating root dir: %v", err) + } + defer cleanup() + conf.RootDir = rootDir + + dir, err := os.MkdirTemp(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + outputPath := filepath.Join(dir, "output") + outputFile, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile.Close() + + // Create 3 containers. First requires a restore call, second requires a restoreSubcontainer + // that needs to wait, third issues a restoreSubcontainer call that actually restores the + // entire sandbox. + script := fmt.Sprintf("for ((i=0; ;i++)); do echo $i >> %q; sleep 1; done", outputPath) + testSpecs, ids := createSpecs( + []string{"sleep", "100"}, + []string{"bash", "-c", script}, + []string{"sleep", "100"}, + ) + + conts, cleanup, err := startContainers(conf, testSpecs, ids) + if err != nil { + t.Fatalf("error starting containers: %v", err) + } + defer cleanup() + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + // Checkpoint root container; save state into new file. + if err := conts[0].Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + t.Fatalf("error checkpointing container to empty file: %v", err) + } + defer os.RemoveAll(dir) + + lastNum, err := readOutputNum(outputPath, -1) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile2, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile2.Close() + + // Restore into a new container with different ID (e.g. clone). Keep the + // initial container running to ensure no conflict with it. + newIds := make([]string, 0, len(ids)) + for range ids { + newIds = append(newIds, testutil.RandomContainerID()) + } + for _, specs := range testSpecs[1:] { + specs.Annotations[specutils.ContainerdSandboxIDAnnotation] = newIds[0] + } + conts2, cleanup2, err := restoreContainers(conf, testSpecs, newIds, dir) + if err != nil { + t.Fatalf("error restoring containers: %v", err) + } + defer cleanup2() + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile2); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum) + } + + for _, cont := range conts2 { + state := cont.State() + if state.Status != Running { + t.Fatalf("container %v is not running: %v", cont.ID, state.Status) + } + } + + // Restore into a new container with different ID (e.g. clone). It + // requires the original container to cease to exist because they share + // the same identity. + cleanup2() + conts2 = nil + + // Delete and recreate file before restoring. + if err := os.Remove(outputPath); err != nil { + t.Fatalf("error removing file") + } + outputFile3, err := createWriteableOutputFile(outputPath) + if err != nil { + t.Fatalf("error creating output file: %v", err) + } + defer outputFile3.Close() + + _, cleanup3, err := restoreContainers(conf, testSpecs, newIds, dir) + if err != nil { + t.Fatalf("error creating containers: %v", err) + } + defer cleanup3() + + // Wait until application has ran. + if err := waitForFileNotEmpty(outputFile3); err != nil { + t.Fatalf("Failed to wait for output file: %v", err) + } + + firstNum2, err := readOutputNum(outputPath, 0) + if err != nil { + t.Fatalf("error with outputFile: %v", err) + } + + // Check that lastNum is one less than firstNum and that the container + // picks up from where it left off. + if lastNum+1 != firstNum2 { + t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2) + } + }) + } +} + // Tests cgroups are mounted in only containers which have a cgroup mount in // the spec. func TestMultiContainerCgroups(t *testing.T) { diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 70b4fde60..e2aca8b62 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -444,7 +444,7 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s // Restore sends the restore call for a container in the sandbox. func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, direct bool) error { - log.Debugf("Restore sandbox %q", s.ID) + log.Debugf("Restore sandbox %q from path %q", s.ID, imagePath) stateFileName := path.Join(imagePath, boot.CheckpointStateFileName) sf, err := os.Open(stateFileName) @@ -502,6 +502,44 @@ func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, dir return nil } +// RestoreSubcontainer sends the restore call for a sub-container in the sandbox. +func (s *Sandbox) RestoreSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestoreFiles []*os.File, devIOFile *os.File, goferMountConf []boot.GoferMountConf) error { + log.Debugf("Restore sub-container %q in sandbox %q, PID: %d", cid, s.ID, s.Pid.load()) + + if err := s.configureStdios(conf, stdios); err != nil { + return err + } + s.fixPidns(spec) + + // The payload contains (in this specific order): + // * stdin/stdout/stderr (optional: only present when not using TTY) + // * The subcontainer's overlay filestore files (optional: only present when + // host file backed overlay is configured) + // * Gofer files. + payload := urpc.FilePayload{} + payload.Files = append(payload.Files, stdios...) + payload.Files = append(payload.Files, goferFilestoreFiles...) + if devIOFile != nil { + payload.Files = append(payload.Files, devIOFile) + } + payload.Files = append(payload.Files, goferFiles...) + + // Start running the container. + args := boot.StartArgs{ + Spec: spec, + Conf: conf, + CID: cid, + NumGoferFilestoreFDs: len(goferFilestoreFiles), + IsDevIoFilePresent: devIOFile != nil, + GoferMountConfs: goferMountConf, + FilePayload: payload, + } + if err := s.call(boot.ContMgrRestoreSubcontainer, &args, nil); err != nil { + return fmt.Errorf("starting sub-container %v: %w", spec.Process.Args, err) + } + return nil +} + // Processes retrieves the list of processes and associated metadata for a // given container in this sandbox. func (s *Sandbox) Processes(cid string) ([]*control.Process, error) {