Multi-container restore

Checkpoint of any container continues to trigger an entire pod
checkpoint, which includes the state of all containers.

Restore must be done for each of the containers, one at a time.
The actual restore is triggered when the last container is restored.
The set of flags and spec must be the same for the `restore` command
as it was for the `create` commands when the containers were created.
Containers are identified by their names and can be restored in any
order. If containers have no name, they must be stored in the same
order they were created. Container IDs and host FDs are rewired
correctly after restore.

Updates #1956

PiperOrigin-RevId: 629272041

pkg/sentry/control/state.go

@@ -77,7 +77,7 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error {
 				log.Infof("Save succeeded: exiting...")
 				s.Kernel.SetSaveSuccess(false /* autosave */)
 			} else {
-				log.Warningf("Save failed: exiting...")
+				log.Warningf("Save failed: %v", err)
 				s.Kernel.SetSaveError(err)
 			}
 			if !o.Resume {

pkg/sentry/kernel/kernel.go

@@ -351,6 +351,13 @@ type Kernel struct {
 	// devGofers maps container ID to its device gofer client.
 	devGofers   map[string]*devutil.GoferClient `state:"nosave"`
 	devGofersMu sync.Mutex                      `state:"nosave"`
+
+	// containerNames store the container name based on their container ID.
+	// Names are preserved between save/restore session, while IDs can change.
+	//
+	// Mapping: cid -> name.
+	// It's protected by extMu.
+	containerNames map[string]string
 }
 
 // InitKernelArgs holds arguments to Init.
@@ -457,6 +464,7 @@ func (k *Kernel) Init(args InitKernelArgs) error {
 		args.MaxFDLimit = MaxFdLimit
 	}
 	k.MaxFDLimit.Store(args.MaxFDLimit)
+	k.containerNames = make(map[string]string)
 
 	ctx := k.SupervisorContext()
 	if err := k.vfs.Init(ctx); err != nil {
@@ -1976,3 +1984,31 @@ func (k *Kernel) cleaupDevGofers() {
 	}
 	k.devGofers = nil
 }
+
+// RegisterContainerName registers a container name for a given container ID.
+func (k *Kernel) RegisterContainerName(cid, containerName string) {
+	k.extMu.Lock()
+	defer k.extMu.Unlock()
+	k.containerNames[cid] = containerName
+}
+
+// RestoreContainerMapping remaps old container IDs to new ones after a restore.
+// containerIDs maps "name -> new container ID". Note that container names remain
+// constant between restore sessions.
+func (k *Kernel) RestoreContainerMapping(containerIDs map[string]string) {
+	k.extMu.Lock()
+	defer k.extMu.Unlock()
+
+	// Delete mapping from old session and replace with new values.
+	k.containerNames = make(map[string]string)
+	for name, cid := range containerIDs {
+		k.containerNames[cid] = name
+	}
+}
+
+// TaskContainerName returns the container name for a given task.
+func (k *Kernel) TaskContainerName(task *Task) string {
+	k.extMu.Lock()
+	defer k.extMu.Unlock()
+	return k.containerNames[task.ContainerID()]
+}

runsc/boot/BUILD

@@ -102,6 +102,7 @@ go_library(
         "//pkg/sentry/vfs",
         "//pkg/sentry/watchdog",
         "//pkg/sighandling",
+        "//pkg/state/statefile",
         "//pkg/sync",
         "//pkg/tcpip",
         "//pkg/tcpip/link/ethernet",

runsc/boot/controller.go

@@ -18,6 +18,8 @@ import (
 	"errors"
 	"fmt"
 	"path"
+	"strconv"
+	"sync"
 	gtime "time"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -34,6 +36,7 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/seccheck"
 	"gvisor.dev/gvisor/pkg/sentry/socket/netstack"
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
+	"gvisor.dev/gvisor/pkg/state/statefile"
 	"gvisor.dev/gvisor/pkg/urpc"
 	"gvisor.dev/gvisor/runsc/boot/procfs"
 	"gvisor.dev/gvisor/runsc/config"
@@ -66,6 +69,9 @@ const (
 	// ContMgrRestore restores a container from a statefile.
 	ContMgrRestore = "containerManager.Restore"
 
+	// ContMgrRestoreSubcontainer restores a container from a statefile.
+	ContMgrRestoreSubcontainer = "containerManager.RestoreSubcontainer"
+
 	// ContMgrSignal sends a signal to a container.
 	ContMgrSignal = "containerManager.Signal"
 
@@ -209,13 +215,17 @@ type containerManager struct {
 	// be started.
 	startChan chan struct{}
 
-	// startResultChan is used to signal when the root container  has
+	// startResultChan is used to signal when the root container has
 	// started. Any errors encountered during startup will be sent to the
 	// channel. A nil value indicates success.
 	startResultChan chan error
 
 	// l is the loader that creates containers and sandboxes.
 	l *Loader
+
+	// restorer is set when the sandbox in being restored. It stores the state
+	// of all containers and perform all actions required by restore.
+	restorer *restorer
 }
 
 // StartRoot will start the root container process.
@@ -413,16 +423,7 @@ func (cm *containerManager) ExecuteAsync(args *control.ExecArgs, pid *int32) err
 // Checkpoint pauses a sandbox and saves its state.
 func (cm *containerManager) Checkpoint(o *control.SaveOpts, _ *struct{}) error {
 	log.Debugf("containerManager.Checkpoint")
-	// TODO(gvisor.dev/issues/6243): save/restore not supported w/ hostinet
-	if cm.l.root.conf.Network == config.NetworkHost {
-		return errors.New("checkpoint not supported when using hostinet")
-	}
-
-	state := control.State{
-		Kernel:   cm.l.k,
-		Watchdog: cm.l.watchdog,
-	}
-	return state.Save(o, nil)
+	return cm.l.save(o)
 }
 
 // PortForwardOpts contains options for port forwarding to a port in a
@@ -480,7 +481,6 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
 	if err != nil {
 		return err
 	}
-	defer stateFile.Close()
 
 	var stat unix.Stat_t
 	if err := unix.Fstat(stateFile.FD(), &stat); err != nil {
@@ -490,7 +490,8 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
 		return fmt.Errorf("statefile cannot be empty")
 	}
 
-	r := restorer{container: &cm.l.root, stateFile: stateFile}
+	cm.restorer = &restorer{restoreDone: cm.onRestoreDone, stateFile: stateFile}
+	cm.l.restoreWaiters = sync.NewCond(&cm.l.mu)
 	cm.l.state = restoring
 
 	fileIdx := 1
@@ -499,13 +500,12 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
 		if err != nil {
 			return err
 		}
-		defer pagesFile.Close()
 		fileIdx++
-		r.pagesFile = pagesFile
+		cm.restorer.pagesFile = pagesFile
 	}
 
 	if o.HaveDeviceFile {
-		r.deviceFile, err = o.ReleaseFD(fileIdx)
+		cm.restorer.deviceFile, err = o.ReleaseFD(fileIdx)
 		if err != nil {
 			return err
 		}
@@ -519,10 +519,121 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
 	// Pause the kernel while we build a new one.
 	cm.l.k.Pause()
 
-	if err := r.restore(cm.l); err != nil {
+	metadata, err := statefile.MetadataUnsafe(cm.restorer.stateFile)
+	if err != nil {
+		return fmt.Errorf("reading metadata from statefile: %w", err)
+	}
+	var count int
+	countStr, ok := metadata["container_count"]
+	if !ok {
+		// TODO(gvisor.dev/issue/1956): Add container count with syscall save
+		// trigger. For now, assume that only a single container exists if metadata
+		// isn't present.
+		//
+		// -return errors.New("container count not present in state file")
+		count = 1
+	} else {
+		count, err = strconv.Atoi(countStr)
+		if err != nil {
+			return fmt.Errorf("invalid container count: %w", err)
+		}
+		if count < 1 {
+			return fmt.Errorf("invalid container count value: %v", count)
+		}
+	}
+	cm.restorer.totalContainers = count
+	log.Infof("Restoring a total of %d containers", cm.restorer.totalContainers)
+
+	if _, err := unix.Seek(stateFile.FD(), 0, 0); err != nil {
+		return fmt.Errorf("rewinding state file: %w", err)
+	}
+
+	return cm.restorer.restoreContainerInfo(cm.l, &cm.l.root)
+}
+
+func (cm *containerManager) onRestoreDone() error {
+	if err := cm.onStart(); err != nil {
 		return err
 	}
-	return cm.onStart()
+
+	cm.l.restoreWaiters.Broadcast()
+	cm.restorer = nil
+	return nil
+}
+
+func (cm *containerManager) RestoreSubcontainer(args *StartArgs, _ *struct{}) error {
+	log.Debugf("containerManager.RestoreSubcontainer, cid: %s, args: %+v", args.CID, args)
+
+	if cm.l.state != restoring {
+		return fmt.Errorf("sandbox is not being restored, cannot restore subcontainer")
+	}
+
+	// Validate arguments.
+	if args.Spec == nil {
+		return errors.New("start arguments missing spec")
+	}
+	if args.Conf == nil {
+		return errors.New("start arguments missing config")
+	}
+	if args.CID == "" {
+		return errors.New("start argument missing container ID")
+	}
+	expectedFDs := 1 // At least one FD for the root filesystem.
+	expectedFDs += args.NumGoferFilestoreFDs
+	if !args.Spec.Process.Terminal {
+		expectedFDs += 3
+	}
+	if len(args.Files) < expectedFDs {
+		return fmt.Errorf("restore arguments must contain at least %d FDs, but only got %d", expectedFDs, len(args.Files))
+	}
+
+	// All validation passed, logs the spec for debugging.
+	specutils.LogSpecDebug(args.Spec, args.Conf.OCISeccomp)
+
+	goferFiles := args.Files
+	var stdios []*fd.FD
+	if !args.Spec.Process.Terminal {
+		// When not using a terminal, stdios come as the first 3 files in the
+		// payload.
+		var err error
+		stdios, err = fd.NewFromFiles(goferFiles[:3])
+		if err != nil {
+			return fmt.Errorf("error dup'ing stdio files: %w", err)
+		}
+		goferFiles = goferFiles[3:]
+	}
+
+	var goferFilestoreFDs []*fd.FD
+	for i := 0; i < args.NumGoferFilestoreFDs; i++ {
+		overlayFilestoreFD, err := fd.NewFromFile(goferFiles[i])
+		if err != nil {
+			return fmt.Errorf("error dup'ing overlay filestore file: %w", err)
+		}
+		goferFilestoreFDs = append(goferFilestoreFDs, overlayFilestoreFD)
+	}
+	goferFiles = goferFiles[args.NumGoferFilestoreFDs:]
+
+	var devGoferFD *fd.FD
+	if args.IsDevIoFilePresent {
+		var err error
+		devGoferFD, err = fd.NewFromFile(goferFiles[0])
+		if err != nil {
+			return fmt.Errorf("error dup'ing dev gofer file: %w", err)
+		}
+		goferFiles = goferFiles[1:]
+	}
+
+	goferFDs, err := fd.NewFromFiles(goferFiles)
+	if err != nil {
+		return fmt.Errorf("error dup'ing gofer files: %w", err)
+	}
+
+	if err := cm.restorer.restoreSubcontainer(args.Spec, args.Conf, cm.l, args.CID, stdios, goferFDs, goferFilestoreFDs, devGoferFD, args.GoferMountConfs); err != nil {
+		log.Debugf("containerManager.RestoreSubcontainer failed, cid: %s, args: %+v, err: %v", args.CID, args, err)
+		return err
+	}
+	log.Debugf("Container restored, cid: %s", args.CID)
+	return nil
 }
 
 // Wait waits for the init process in the given container.

runsc/boot/loader.go

@@ -190,6 +190,8 @@ type Loader struct {
 	// restore is set to true if we are restoring a container.
 	restore bool
 
+	restoreWaiters *sync.Cond
+
 	// sandboxID is the ID for the whole sandbox.
 	sandboxID string
 
@@ -209,6 +211,8 @@ type Loader struct {
 
 	// sharedMounts holds VFS mounts that may be shared between containers within
 	// the same pod. It is mapped by mount source.
+	//
+	// sharedMounts is guarded by mu.
 	sharedMounts map[string]*vfs.Mount
 
 	// processes maps containers init process and invocation of exec. Root
@@ -218,6 +222,13 @@ type Loader struct {
 	// processes is guarded by mu.
 	processes map[execID]*execProcess
 
+	// containerIDs store container names and IDs to assist with restore and container
+	// naming when user didn't provide one.
+	//
+	// Mapping: name -> cid.
+	// processes is guarded by mu.
+	containerIDs map[string]string
+
 	// portForwardProxies is a list of active port forwarding connections.
 	//
 	// portForwardProxies is guarded by mu.
@@ -382,9 +393,20 @@ func New(args Args) (*Loader, error) {
 
 	kernel.IOUringEnabled = args.Conf.IOUring
 
-	info := containerInfo{
+	eid := execID{cid: args.ID}
+	l := &Loader{
+		sandboxID:     args.ID,
+		processes:     map[execID]*execProcess{eid: {}},
+		sharedMounts:  make(map[string]*vfs.Mount),
+		stopProfiling: stopProfiling,
+		productName:   args.ProductName,
+		containerIDs:  map[string]string{},
+	}
+
+	containerName := l.registerContainerLocked(args.Spec, args.ID)
+	l.root = containerInfo{
 		cid:                 args.ID,
-		containerName:       specutils.ContainerName(args.Spec),
+		containerName:       containerName,
 		conf:                args.Conf,
 		spec:                args.Spec,
 		goferMountConfs:     args.GoferMountConfs,
@@ -409,25 +431,25 @@ func New(args Args) (*Loader, error) {
 		if err != nil {
 			return nil, fmt.Errorf("dup3 of stdios failed: %w", err)
 		}
-		info.stdioFDs = append(info.stdioFDs, fd.New(newfd))
+		l.root.stdioFDs = append(l.root.stdioFDs, fd.New(newfd))
 		_ = unix.Close(stdioFD)
 		newfd++
 	}
 	for _, goferFD := range args.GoferFDs {
-		info.goferFDs = append(info.goferFDs, fd.New(goferFD))
+		l.root.goferFDs = append(l.root.goferFDs, fd.New(goferFD))
 	}
 	for _, filestoreFD := range args.GoferFilestoreFDs {
-		info.goferFilestoreFDs = append(info.goferFilestoreFDs, fd.New(filestoreFD))
+		l.root.goferFilestoreFDs = append(l.root.goferFilestoreFDs, fd.New(filestoreFD))
 	}
 	if args.DevGoferFD >= 0 {
-		info.devGoferFD = fd.New(args.DevGoferFD)
+		l.root.devGoferFD = fd.New(args.DevGoferFD)
 	}
 	if args.ExecFD >= 0 {
-		info.execFD = fd.New(args.ExecFD)
+		l.root.execFD = fd.New(args.ExecFD)
 	}
 
 	for _, customFD := range args.PassFDs {
-		info.passFDs = append(info.passFDs, fdMapping{
+		l.root.passFDs = append(l.root.passFDs, fdMapping{
 			host:  fd.New(customFD.Host),
 			guest: customFD.Guest,
 		})
@@ -441,27 +463,25 @@ func New(args Args) (*Loader, error) {
 	if specutils.NVProxyEnabled(args.Spec, args.Conf) && p.OwnsPageTables() {
 		return nil, fmt.Errorf("--nvproxy is incompatible with platform %s: owns page tables", args.Conf.Platform)
 	}
-	k := &kernel.Kernel{
-		Platform: p,
-	}
+	l.k = &kernel.Kernel{Platform: p}
 
 	// Create memory file.
 	mf, err := createMemoryFile()
 	if err != nil {
 		return nil, fmt.Errorf("creating memory file: %w", err)
 	}
-	k.SetMemoryFile(mf)
+	l.k.SetMemoryFile(mf)
 
 	// Create VDSO.
 	//
 	// Pass k as the platform since it is savable, unlike the actual platform.
-	vdso, err := loader.PrepareVDSO(k.MemoryFile())
+	vdso, err := loader.PrepareVDSO(l.k.MemoryFile())
 	if err != nil {
 		return nil, fmt.Errorf("creating vdso: %w", err)
 	}
 
 	// Create timekeeper.
-	tk := kernel.NewTimekeeper(k.MemoryFile(), vdso.ParamPage.FileRange())
+	tk := kernel.NewTimekeeper(l.k.MemoryFile(), vdso.ParamPage.FileRange())
 	tk.SetClocks(time.NewCalibratedClocks())
 
 	if err := enableStrace(args.Conf); err != nil {
@@ -473,7 +493,7 @@ func New(args Args) (*Loader, error) {
 		return nil, fmt.Errorf("getting root credentials")
 	}
 	// Create root network namespace/stack.
-	netns, err := newRootNetworkNamespace(args.Conf, tk, k, creds.UserNamespace)
+	netns, err := newRootNetworkNamespace(args.Conf, tk, l.k, creds.UserNamespace)
 	if err != nil {
 		return nil, fmt.Errorf("creating network: %w", err)
 	}
@@ -513,7 +533,7 @@ func New(args Args) (*Loader, error) {
 	}
 	// Initiate the Kernel object, which is required by the Context passed
 	// to createVFS in order to mount (among other things) procfs.
-	if err = k.Init(kernel.InitKernelArgs{
+	if err = l.k.Init(kernel.InitKernelArgs{
 		FeatureSet:           cpuid.HostFeatureSet().Fixed(),
 		Timekeeper:           tk,
 		RootUserNamespace:    creds.UserNamespace,
@@ -528,7 +548,7 @@ func New(args Args) (*Loader, error) {
 		return nil, fmt.Errorf("initializing kernel: %w", err)
 	}
 
-	if err := registerFilesystems(k, &info); err != nil {
+	if err := registerFilesystems(l.k, &l.root); err != nil {
 		return nil, fmt.Errorf("registering filesystems: %w", err)
 	}
 
@@ -544,30 +564,30 @@ func New(args Args) (*Loader, error) {
 	// Create a watchdog.
 	dogOpts := watchdog.DefaultOpts
 	dogOpts.TaskTimeoutAction = args.Conf.WatchdogAction
-	dog := watchdog.New(k, dogOpts)
+	l.watchdog = watchdog.New(l.k, dogOpts)
 
-	procArgs, err := createProcessArgs(args.ID, args.Spec, args.Conf, creds, k, k.RootPIDNamespace())
+	procArgs, err := createProcessArgs(args.ID, args.Spec, args.Conf, creds, l.k, l.k.RootPIDNamespace())
 	if err != nil {
 		return nil, fmt.Errorf("creating init process for root container: %w", err)
 	}
-	info.procArgs = procArgs
+	l.root.procArgs = procArgs
 
 	if err := initCompatLogs(args.UserLogFD); err != nil {
 		return nil, fmt.Errorf("initializing compat logs: %w", err)
 	}
 
-	mountHints, err := NewPodMountHints(args.Spec)
+	l.mountHints, err = NewPodMountHints(args.Spec)
 	if err != nil {
 		return nil, fmt.Errorf("creating pod mount hints: %w", err)
 	}
 
 	// Set up host mount that will be used for imported fds.
-	hostFilesystem, err := host.NewFilesystem(k.VFS())
+	hostFilesystem, err := host.NewFilesystem(l.k.VFS())
 	if err != nil {
 		return nil, fmt.Errorf("failed to create hostfs filesystem: %w", err)
 	}
-	defer hostFilesystem.DecRef(k.SupervisorContext())
-	k.SetHostMount(k.VFS().NewDisconnectedMount(hostFilesystem, nil, &vfs.MountOptions{}))
+	defer hostFilesystem.DecRef(l.k.SupervisorContext())
+	l.k.SetHostMount(l.k.VFS().NewDisconnectedMount(hostFilesystem, nil, &vfs.MountOptions{}))
 
 	if args.PodInitConfigFD >= 0 {
 		if err := setupSeccheck(args.PodInitConfigFD, args.SinkFDs); err != nil {
@@ -575,18 +595,7 @@ func New(args Args) (*Loader, error) {
 		}
 	}
 
-	eid := execID{cid: args.ID}
-	l := &Loader{
-		k:             k,
-		watchdog:      dog,
-		sandboxID:     args.ID,
-		processes:     map[execID]*execProcess{eid: {}},
-		mountHints:    mountHints,
-		sharedMounts:  make(map[string]*vfs.Mount),
-		root:          info,
-		stopProfiling: stopProfiling,
-		productName:   args.ProductName,
-	}
+	l.k.RegisterContainerName(args.ID, l.root.containerName)
 
 	// We don't care about child signals; some platforms can generate a
 	// tremendous number of useless ones (I'm looking at you, ptrace).
@@ -922,9 +931,10 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 		pidns = l.k.RootPIDNamespace()
 	}
 
+	containerName := l.registerContainerLocked(spec, cid)
 	info := &containerInfo{
 		cid:                 cid,
-		containerName:       specutils.ContainerName(spec),
+		containerName:       containerName,
 		conf:                conf,
 		spec:                spec,
 		goferFDs:            goferFDs,
@@ -990,6 +1000,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st
 		})
 	}
 
+	l.k.RegisterContainerName(cid, info.containerName)
 	l.k.StartProcess(ep.tg)
 	// No more failures from this point on.
 	cu.Release()
@@ -1258,9 +1269,26 @@ func (l *Loader) executeAsync(args *control.ExecArgs) (kernel.ThreadID, error) {
 func (l *Loader) waitContainer(cid string, waitStatus *uint32) error {
 	// Don't defer unlock, as doing so would make it impossible for
 	// multiple clients to wait on the same container.
-	tg, err := l.threadGroupFromID(execID{cid: cid})
+	key := execID{cid: cid}
+	tg, err := l.threadGroupFromID(key)
 	if err != nil {
-		return fmt.Errorf("can't wait for container %q: %w", cid, err)
+		l.mu.Lock()
+		// Extra handling is needed if the container is restoring.
+		if l.state != restoring {
+			l.mu.Unlock()
+			return err
+		}
+		// Container could be restoring, first check if container exists.
+		if _, err := l.findProcessLocked(key); err != nil {
+			l.mu.Unlock()
+			return err
+		}
+		log.Infof("Waiting for container being restored, CID: %q", cid)
+		l.restoreWaiters.Wait()
+		l.mu.Unlock()
+
+		log.Infof("Restore is completed, trying to wait for container %q again.", cid)
+		return l.waitContainer(cid, waitStatus)
 	}
 
 	// If the thread either has already exited or exits during waiting,
@@ -1583,9 +1611,9 @@ func (l *Loader) threadGroupFromID(key execID) (*kernel.ThreadGroup, error) {
 // error if execution ID is invalid or if the container cannot be found (maybe
 // it has been deleted). Caller must hold 'mu'.
 func (l *Loader) tryThreadGroupFromIDLocked(key execID) (*kernel.ThreadGroup, error) {
-	ep := l.processes[key]
-	if ep == nil {
-		return nil, fmt.Errorf("container %q not found", key.cid)
+	ep, err := l.findProcessLocked(key)
+	if err != nil {
+		return nil, err
 	}
 	return ep.tg, nil
 }
@@ -1595,9 +1623,9 @@ func (l *Loader) tryThreadGroupFromIDLocked(key execID) (*kernel.ThreadGroup, er
 // execution ID is invalid or if the container cannot be found (maybe it has
 // been deleted). Caller must hold 'mu'.
 func (l *Loader) ttyFromIDLocked(key execID) (*host.TTYFileDescription, error) {
-	ep := l.processes[key]
-	if ep == nil {
-		return nil, fmt.Errorf("container %q not found", key.cid)
+	ep, err := l.findProcessLocked(key)
+	if err != nil {
+		return nil, err
 	}
 	return ep.tty, nil
 }
@@ -1776,6 +1804,33 @@ func (l *Loader) networkStats() ([]*NetworkInterface, error) {
 	return stats, nil
 }
 
+func (l *Loader) findProcessLocked(key execID) (*execProcess, error) {
+	ep := l.processes[key]
+	if ep == nil {
+		return nil, fmt.Errorf("container %q not found", key.cid)
+	}
+	return ep, nil
+}
+
+func (l *Loader) registerContainer(spec *specs.Spec, cid string) string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	return l.registerContainerLocked(spec, cid)
+}
+
+func (l *Loader) registerContainerLocked(spec *specs.Spec, cid string) string {
+	containerName := specutils.ContainerName(spec)
+	if len(containerName) == 0 {
+		// If no name was provided, require containers to be restored in the same order
+		// they were created.
+		containerName = "__no_name_" + strconv.Itoa(len(l.containerIDs))
+	}
+
+	l.containerIDs[containerName] = cid
+	return containerName
+}
+
 func (l *Loader) containerRuntimeState(cid string) ContainerRuntimeState {
 	l.mu.Lock()
 	defer l.mu.Unlock()

runsc/boot/restore.go

@@ -15,13 +15,19 @@
 package boot
 
 import (
+	"errors"
 	"fmt"
 	"io"
+	"strconv"
+	time2 "time"
 
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.dev/gvisor/pkg/abi/linux"
+	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/fd"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/host"
 	"gvisor.dev/gvisor/pkg/sentry/inet"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
@@ -32,8 +38,10 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/time"
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 	"gvisor.dev/gvisor/pkg/sentry/watchdog"
+	"gvisor.dev/gvisor/pkg/sync"
 	"gvisor.dev/gvisor/pkg/tcpip/stack"
 	"gvisor.dev/gvisor/runsc/boot/pprof"
+	"gvisor.dev/gvisor/runsc/config"
 )
 
 const (
@@ -45,11 +53,74 @@ const (
 	CheckpointPagesFileName = "pages.img"
 )
 
+// restorer manages a restore session for a sandbox. It stores information about
+// all containers and triggers the full sandbox restore after the last
+// container is restored.
 type restorer struct {
-	container  *containerInfo
-	stateFile  io.Reader
-	pagesFile  *fd.FD
+	mu sync.Mutex
+
+	// totalContainers is the number of containers expected to be restored in
+	// the sandbox. Sandbox restore can only happen, after all containers have
+	// been restored.
+	totalContainers int
+
+	// containers is the list of containers restored so far.
+	containers []*containerInfo
+
+	// Files used by restore to rehydrate the state.
+	stateFile io.ReadCloser
+	pagesFile *fd.FD
+
+	// deviceFile is the required to start the platform.
 	deviceFile *fd.FD
+
+	// restoreDone is a callback triggered when restore is successful.
+	restoreDone func() error
+}
+
+func (r *restorer) restoreSubcontainer(spec *specs.Spec, conf *config.Config, l *Loader, cid string, stdioFDs, goferFDs, goferFilestoreFDs []*fd.FD, devGoferFD *fd.FD, goferMountConfs []GoferMountConf) error {
+	containerName := l.registerContainer(spec, cid)
+	info := &containerInfo{
+		cid:               cid,
+		containerName:     containerName,
+		conf:              conf,
+		spec:              spec,
+		stdioFDs:          stdioFDs,
+		goferFDs:          goferFDs,
+		devGoferFD:        devGoferFD,
+		goferFilestoreFDs: goferFilestoreFDs,
+		goferMountConfs:   goferMountConfs,
+	}
+	return r.restoreContainerInfo(l, info)
+}
+
+func (r *restorer) restoreContainerInfo(l *Loader, info *containerInfo) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	for _, container := range r.containers {
+		if container.containerName == info.containerName {
+			return fmt.Errorf("container %q already restored", info.containerName)
+		}
+		if container.cid == info.cid {
+			return fmt.Errorf("container CID %q already belongs to container %q", info.cid, container.containerName)
+		}
+	}
+
+	r.containers = append(r.containers, info)
+
+	log.Infof("Restored container %d of %d", len(r.containers), r.totalContainers)
+	if log.IsLogging(log.Debug) {
+		for i, fd := range info.stdioFDs {
+			log.Debugf("Restore app FD: %d host FD: %d", i, fd.FD())
+		}
+	}
+
+	if len(r.containers) == r.totalContainers {
+		// Trigger the restore if this is the last container.
+		return r.restore(l)
+	}
+	return nil
 }
 
 func createNetworkNamespaceForRestore(l *Loader) (*stack.Stack, *inet.Namespace, error) {
@@ -74,6 +145,8 @@ func createNetworkNamespaceForRestore(l *Loader) (*stack.Stack, *inet.Namespace,
 }
 
 func (r *restorer) restore(l *Loader) error {
+	log.Infof("Starting to restore %d containers", len(r.containers))
+
 	// Create a new root network namespace with the network stack of the
 	// old kernel to preserve the existing network configuration.
 	oldStack, netns, err := createNetworkNamespaceForRestore(l)
@@ -126,23 +199,28 @@ func (r *restorer) restore(l *Loader) error {
 		ctx = context.WithValue(ctx, stack.CtxRestoreStack, oldStack)
 	}
 
-	// TODO(b/298078576): Need to process hints here probably
-	mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, l.productName, l.sandboxID)
-
 	fdmap := make(map[vfs.RestoreID]int)
 	mfmap := make(map[string]*pgalloc.MemoryFile)
-	if err := mntr.configureRestore(fdmap, mfmap); err != nil {
-		return fmt.Errorf("configuring filesystem restore: %v", err)
-	}
-	for appFD, fd := range r.container.stdioFDs {
-		key := host.MakeRestoreID(r.container.containerName, appFD)
-		fdmap[key] = fd.Release()
-	}
-	for _, customFD := range r.container.passFDs {
-		key := host.MakeRestoreID(r.container.containerName, customFD.guest)
-		fdmap[key] = customFD.host.FD()
+	for _, cont := range r.containers {
+		// TODO(b/298078576): Need to process hints here probably
+		mntr := newContainerMounter(cont, l.k, l.mountHints, l.sharedMounts, l.productName, cont.cid)
+		if err = mntr.configureRestore(fdmap, mfmap); err != nil {
+			return fmt.Errorf("configuring filesystem restore: %v", err)
+		}
+
+		for i, fd := range cont.stdioFDs {
+			key := host.MakeRestoreID(cont.containerName, i)
+			fdmap[key] = fd.Release()
+		}
+		for _, customFD := range cont.passFDs {
+			key := host.MakeRestoreID(cont.containerName, customFD.guest)
+			fdmap[key] = customFD.host.FD()
+		}
 	}
+
+	log.Debugf("Restore using fdmap: %v", fdmap)
 	ctx = context.WithValue(ctx, vfs.CtxRestoreFilesystemFDMap, fdmap)
+	log.Debugf("Restore using mfmap: %v", fdmap)
 	ctx = context.WithValue(ctx, pgalloc.CtxMemoryFileMap, mfmap)
 
 	// Load the state.
@@ -154,6 +232,7 @@ func (r *restorer) restore(l *Loader) error {
 	// Since we have a new kernel we also must make a new watchdog.
 	dogOpts := watchdog.DefaultOpts
 	dogOpts.TaskTimeoutAction = l.root.conf.WatchdogAction
+	dogOpts.StartupTimeout = 3 * time2.Minute // Give extra time for all containers to restore.
 	dog := watchdog.New(l.k, dogOpts)
 
 	// Change the loader fields to reflect the changes made when restoring.
@@ -161,23 +240,40 @@ func (r *restorer) restore(l *Loader) error {
 	l.root.procArgs = kernel.CreateProcessArgs{}
 	l.restore = true
 
-	// Reinitialize the sandbox ID and processes map. Note that it doesn't
-	// restore the state of multiple containers, nor exec processes.
-	l.sandboxID = r.container.cid
+	l.sandboxID = l.root.cid
 
 	l.mu.Lock()
-	defer l.mu.Unlock()
+	cu := cleanup.Make(func() {
+		l.mu.Unlock()
+	})
+	defer cu.Clean()
 
-	// Set new container ID if it has changed.
-	tasks := l.k.TaskSet().Root.Tasks()
-	if tasks[0].ContainerID() != l.sandboxID { // There must be at least 1 task.
-		for _, task := range tasks {
-			task.RestoreContainerID(l.sandboxID)
+	// Update all tasks in the system with their respective new container IDs.
+	for _, task := range l.k.TaskSet().Root.Tasks() {
+		name := l.k.TaskContainerName(task)
+		newCid, ok := l.containerIDs[name]
+		if !ok {
+			return fmt.Errorf("unable to remap task with CID %q (name: %q). Available names: %v", task.ContainerID(), name, l.containerIDs)
+		}
+		task.RestoreContainerID(newCid)
+	}
+
+	// Rebuild `processes` map with containers' root process from the restored kernel.
+	for _, tg := range l.k.RootPIDNamespace().ThreadGroups() {
+		// Find all processes with no parent (root of execution), that were not started
+		// via a call to `exec`.
+		if tg.Leader().Parent() == nil && tg.Leader().Origin != kernel.OriginExec {
+			cid := tg.Leader().ContainerID()
+			proc := l.processes[execID{cid: cid}]
+			if proc == nil {
+				return fmt.Errorf("unable to find container root process with CID %q, processes: %v", cid, l.processes)
+			}
+			proc.tg = tg
 		}
 	}
 
 	// Kill all processes that have been exec'd since they cannot be properly
-	// restored, since the caller is no longer connected.
+	// restored -- the caller is no longer connected.
 	for _, tg := range l.k.RootPIDNamespace().ThreadGroups() {
 		if tg.Leader().Origin == kernel.OriginExec {
 			if err := l.k.SendExternalSignalThreadGroup(tg, &linux.SignalInfo{Signo: int32(linux.SIGKILL)}); err != nil {
@@ -186,12 +282,39 @@ func (r *restorer) restore(l *Loader) error {
 		}
 	}
 
-	eid := execID{cid: l.sandboxID}
-	l.processes = map[execID]*execProcess{
-		eid: {
-			tg: l.k.GlobalInit(),
-		},
+	l.k.RestoreContainerMapping(l.containerIDs)
+
+	// Release `l.mu` before calling into callbacks.
+	cu.Clean()
+
+	if err := r.restoreDone(); err != nil {
+		return err
 	}
 
+	r.stateFile.Close()
+	if r.pagesFile != nil {
+		r.pagesFile.Close()
+	}
+
+	log.Infof("Restore successful")
 	return nil
 }
+
+func (l *Loader) save(o *control.SaveOpts) error {
+	// TODO(gvisor.dev/issues/6243): save/restore not supported w/ hostinet
+	if l.root.conf.Network == config.NetworkHost {
+		return errors.New("checkpoint not supported when using hostinet")
+	}
+
+	if o.Metadata == nil {
+		o.Metadata = make(map[string]string)
+	}
+	o.Metadata["container_count"] = strconv.Itoa(l.containerCount())
+
+	state := control.State{
+		Kernel:   l.k,
+		Watchdog: l.watchdog,
+	}
+
+	return state.Save(o, nil)
+}

runsc/boot/vfs.go

@@ -424,7 +424,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountH
 		hints:             hints,
 		sharedMounts:      sharedMounts,
 		productName:       productName,
-		containerID:       info.procArgs.ContainerID,
+		containerID:       info.cid,
 		sandboxID:         sandboxID,
 		containerName:     info.containerName,
 	}

runsc/container/container.go

@@ -423,14 +423,28 @@ func New(conf *config.Config, args Args) (*Container, error) {
 // Start starts running the containerized process inside the sandbox.
 func (c *Container) Start(conf *config.Config) error {
 	log.Debugf("Start container, cid: %s", c.ID)
+	return c.startImpl(conf, "start", c.Sandbox.StartRoot, c.Sandbox.StartSubcontainer)
+}
 
+// Restore takes a container and replaces its kernel and file system
+// to restore a container from its state file.
+func (c *Container) Restore(conf *config.Config, imagePath string, direct bool) error {
+	log.Debugf("Restore container, cid: %s", c.ID)
+
+	restore := func(conf *config.Config) error {
+		return c.Sandbox.Restore(conf, c.ID, imagePath, direct)
+	}
+	return c.startImpl(conf, "restore", restore, c.Sandbox.RestoreSubcontainer)
+}
+
+func (c *Container) startImpl(conf *config.Config, action string, startRoot func(conf *config.Config) error, startSub func(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestores []*os.File, devIOFile *os.File, goferConfs []boot.GoferMountConf) error) error {
 	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
 	}
 	unlock := cleanup.Make(c.Saver.UnlockOrDie)
 	defer unlock.Clean()
 
-	if err := c.requireStatus("start", Created); err != nil {
+	if err := c.requireStatus(action, Created); err != nil {
 		return err
 	}
 
@@ -441,7 +455,7 @@ func (c *Container) Start(conf *config.Config) error {
 	}
 
 	if isRoot(c.Spec) {
-		if err := c.Sandbox.StartRoot(conf); err != nil {
+		if err := startRoot(conf); err != nil {
 			return err
 		}
 	} else {
@@ -492,7 +506,7 @@ func (c *Container) Start(conf *config.Config) error {
 				stdios = []*os.File{os.Stdin, os.Stdout, os.Stderr}
 			}
 
-			return c.Sandbox.StartSubcontainer(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, devIOFile, goferConfs)
+			return startSub(c.Spec, conf, c.ID, stdios, goferFiles, goferFilestores, devIOFile, goferConfs)
 		}); err != nil {
 			return err
 		}
@@ -523,32 +537,6 @@ func (c *Container) Start(conf *config.Config) error {
 	return c.adjustGoferOOMScoreAdj()
 }
 
-// Restore takes a container and replaces its kernel and file system
-// to restore a container from its state file.
-func (c *Container) Restore(conf *config.Config, imagePath string, direct bool) error {
-	log.Debugf("Restore container, cid: %s", c.ID)
-	if err := c.Saver.lock(BlockAcquire); err != nil {
-		return err
-	}
-	defer c.Saver.UnlockOrDie()
-
-	if err := c.requireStatus("restore", Created); err != nil {
-		return err
-	}
-
-	// "If any prestart hook fails, the runtime MUST generate an error,
-	// stop and destroy the container" -OCI spec.
-	if c.Spec.Hooks != nil && len(c.Spec.Hooks.StartContainer) > 0 {
-		log.Warningf("StartContainer hook skipped because running inside container namespace is not supported")
-	}
-
-	if err := c.Sandbox.Restore(conf, c.ID, imagePath, direct); err != nil {
-		return err
-	}
-	c.changeStatus(Running)
-	return c.saveLocked()
-}
-
 // Run is a helper that calls Create + Start + Wait.
 func Run(conf *config.Config, args Args) (unix.WaitStatus, error) {
 	log.Debugf("Run container, cid: %s, rootDir: %q", args.ID, conf.RootDir)

runsc/container/multi_container_test.go

@@ -32,6 +32,7 @@ import (
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/state/statefile"
 	"gvisor.dev/gvisor/pkg/sync"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 	"gvisor.dev/gvisor/runsc/boot"
@@ -99,6 +100,44 @@ func startContainers(conf *config.Config, specs []*specs.Spec, ids []string) ([]
 	return containers, cu.Release(), nil
 }
 
+func restoreContainers(conf *config.Config, specs []*specs.Spec, ids []string, imagePath string) ([]*Container, func(), error) {
+	if len(conf.RootDir) == 0 {
+		panic("conf.RootDir not set. Call testutil.SetupRootDir() to set.")
+	}
+
+	cu := cleanup.Cleanup{}
+	defer cu.Clean()
+
+	var containers []*Container
+	for i, spec := range specs {
+		bundleDir, cleanup, err := testutil.SetupBundleDir(spec)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error setting up container: %v", err)
+		}
+		cu.Add(cleanup)
+
+		args := Args{
+			ID:        ids[i],
+			Spec:      spec,
+			BundleDir: bundleDir,
+		}
+		cont, err := New(conf, args)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error creating container: %v", err)
+		}
+		cu.Add(func() { cont.Destroy() })
+		containers = append(containers, cont)
+
+		if err := cont.Restore(conf, imagePath, false); err != nil {
+			return nil, nil, fmt.Errorf("error restoring container: %v", err)
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	return containers, cu.Release(), nil
+}
+
 type execDesc struct {
 	c    *Container
 	cmd  []string
@@ -2682,6 +2721,156 @@ func TestMultiContainerMemoryLeakStress(t *testing.T) {
 	}
 }
 
+// TestCheckpointRestore tests that checkpoint/restore works
+// with multi-containers.
+func TestMultiContainerCheckpointRestore(t *testing.T) {
+	// Skip overlay because test requires writing to host file.
+	for name, conf := range configs(t, true /* noOverlay */) {
+		t.Run(name, func(t *testing.T) {
+			rootDir, cleanup, err := testutil.SetupRootDir()
+			if err != nil {
+				t.Fatalf("error creating root dir: %v", err)
+			}
+			defer cleanup()
+			conf.RootDir = rootDir
+
+			dir, err := os.MkdirTemp(testutil.TmpDir(), "checkpoint-test")
+			if err != nil {
+				t.Fatalf("os.MkdirTemp() failed: %v", err)
+			}
+			defer os.RemoveAll(dir)
+			if err := os.Chmod(dir, 0777); err != nil {
+				t.Fatalf("error chmoding file: %q, %v", dir, err)
+			}
+
+			outputPath := filepath.Join(dir, "output")
+			outputFile, err := createWriteableOutputFile(outputPath)
+			if err != nil {
+				t.Fatalf("error creating output file: %v", err)
+			}
+			defer outputFile.Close()
+
+			// Create 3 containers. First requires a restore call, second requires a restoreSubcontainer
+			// that needs to wait, third issues a restoreSubcontainer call that actually restores the
+			// entire sandbox.
+			script := fmt.Sprintf("for ((i=0; ;i++)); do echo $i >> %q; sleep 1; done", outputPath)
+			testSpecs, ids := createSpecs(
+				[]string{"sleep", "100"},
+				[]string{"bash", "-c", script},
+				[]string{"sleep", "100"},
+			)
+
+			conts, cleanup, err := startContainers(conf, testSpecs, ids)
+			if err != nil {
+				t.Fatalf("error starting containers: %v", err)
+			}
+			defer cleanup()
+
+			// Wait until application has ran.
+			if err := waitForFileNotEmpty(outputFile); err != nil {
+				t.Fatalf("Failed to wait for output file: %v", err)
+			}
+
+			// Checkpoint root container; save state into new file.
+			if err := conts[0].Checkpoint(dir, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+				t.Fatalf("error checkpointing container to empty file: %v", err)
+			}
+			defer os.RemoveAll(dir)
+
+			lastNum, err := readOutputNum(outputPath, -1)
+			if err != nil {
+				t.Fatalf("error with outputFile: %v", err)
+			}
+
+			// Delete and recreate file before restoring.
+			if err := os.Remove(outputPath); err != nil {
+				t.Fatalf("error removing file")
+			}
+			outputFile2, err := createWriteableOutputFile(outputPath)
+			if err != nil {
+				t.Fatalf("error creating output file: %v", err)
+			}
+			defer outputFile2.Close()
+
+			// Restore into a new container with different ID (e.g. clone). Keep the
+			// initial container running to ensure no conflict with it.
+			newIds := make([]string, 0, len(ids))
+			for range ids {
+				newIds = append(newIds, testutil.RandomContainerID())
+			}
+			for _, specs := range testSpecs[1:] {
+				specs.Annotations[specutils.ContainerdSandboxIDAnnotation] = newIds[0]
+			}
+			conts2, cleanup2, err := restoreContainers(conf, testSpecs, newIds, dir)
+			if err != nil {
+				t.Fatalf("error restoring containers: %v", err)
+			}
+			defer cleanup2()
+
+			// Wait until application has ran.
+			if err := waitForFileNotEmpty(outputFile2); err != nil {
+				t.Fatalf("Failed to wait for output file: %v", err)
+			}
+
+			firstNum, err := readOutputNum(outputPath, 0)
+			if err != nil {
+				t.Fatalf("error with outputFile: %v", err)
+			}
+
+			// Check that lastNum is one less than firstNum and that the container
+			// picks up from where it left off.
+			if lastNum+1 != firstNum {
+				t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum)
+			}
+
+			for _, cont := range conts2 {
+				state := cont.State()
+				if state.Status != Running {
+					t.Fatalf("container %v is not running: %v", cont.ID, state.Status)
+				}
+			}
+
+			// Restore into a new container with different ID (e.g. clone). It
+			// requires the original container to cease to exist because they share
+			// the same identity.
+			cleanup2()
+			conts2 = nil
+
+			// Delete and recreate file before restoring.
+			if err := os.Remove(outputPath); err != nil {
+				t.Fatalf("error removing file")
+			}
+			outputFile3, err := createWriteableOutputFile(outputPath)
+			if err != nil {
+				t.Fatalf("error creating output file: %v", err)
+			}
+			defer outputFile3.Close()
+
+			_, cleanup3, err := restoreContainers(conf, testSpecs, newIds, dir)
+			if err != nil {
+				t.Fatalf("error creating containers: %v", err)
+			}
+			defer cleanup3()
+
+			// Wait until application has ran.
+			if err := waitForFileNotEmpty(outputFile3); err != nil {
+				t.Fatalf("Failed to wait for output file: %v", err)
+			}
+
+			firstNum2, err := readOutputNum(outputPath, 0)
+			if err != nil {
+				t.Fatalf("error with outputFile: %v", err)
+			}
+
+			// Check that lastNum is one less than firstNum and that the container
+			// picks up from where it left off.
+			if lastNum+1 != firstNum2 {
+				t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2)
+			}
+		})
+	}
+}
+
 // Tests cgroups are mounted in only containers which have a cgroup mount in
 // the spec.
 func TestMultiContainerCgroups(t *testing.T) {

runsc/sandbox/sandbox.go

@@ -444,7 +444,7 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s
 
 // Restore sends the restore call for a container in the sandbox.
 func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, direct bool) error {
-	log.Debugf("Restore sandbox %q", s.ID)
+	log.Debugf("Restore sandbox %q from path %q", s.ID, imagePath)
 
 	stateFileName := path.Join(imagePath, boot.CheckpointStateFileName)
 	sf, err := os.Open(stateFileName)
@@ -502,6 +502,44 @@ func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, dir
 	return nil
 }
 
+// RestoreSubcontainer sends the restore call for a sub-container in the sandbox.
+func (s *Sandbox) RestoreSubcontainer(spec *specs.Spec, conf *config.Config, cid string, stdios, goferFiles, goferFilestoreFiles []*os.File, devIOFile *os.File, goferMountConf []boot.GoferMountConf) error {
+	log.Debugf("Restore sub-container %q in sandbox %q, PID: %d", cid, s.ID, s.Pid.load())
+
+	if err := s.configureStdios(conf, stdios); err != nil {
+		return err
+	}
+	s.fixPidns(spec)
+
+	// The payload contains (in this specific order):
+	// * stdin/stdout/stderr (optional: only present when not using TTY)
+	// * The subcontainer's overlay filestore files (optional: only present when
+	//   host file backed overlay is configured)
+	// * Gofer files.
+	payload := urpc.FilePayload{}
+	payload.Files = append(payload.Files, stdios...)
+	payload.Files = append(payload.Files, goferFilestoreFiles...)
+	if devIOFile != nil {
+		payload.Files = append(payload.Files, devIOFile)
+	}
+	payload.Files = append(payload.Files, goferFiles...)
+
+	// Start running the container.
+	args := boot.StartArgs{
+		Spec:                 spec,
+		Conf:                 conf,
+		CID:                  cid,
+		NumGoferFilestoreFDs: len(goferFilestoreFiles),
+		IsDevIoFilePresent:   devIOFile != nil,
+		GoferMountConfs:      goferMountConf,
+		FilePayload:          payload,
+	}
+	if err := s.call(boot.ContMgrRestoreSubcontainer, &args, nil); err != nil {
+		return fmt.Errorf("starting sub-container %v: %w", spec.Process.Args, err)
+	}
+	return nil
+}
+
 // Processes retrieves the list of processes and associated metadata for a
 // given container in this sandbox.
 func (s *Sandbox) Processes(cid string) ([]*control.Process, error) {