From 18a5701716eb102bce14a06622fab9dc5e764b44 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Tue, 21 Nov 2023 16:59:39 -0800 Subject: [PATCH] Introduce the restorer Move the restore related code to make it easier to add more features in the future. PiperOrigin-RevId: 584465292 --- runsc/boot/BUILD | 1 + runsc/boot/controller.go | 100 ++++----------------------------- runsc/boot/loader.go | 24 ++++---- runsc/boot/restore.go | 118 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 100 deletions(-) create mode 100644 runsc/boot/restore.go diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 162fa952b..e2ffa65e0 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -19,6 +19,7 @@ go_library( "loader.go", "mount_hints.go", "network.go", + "restore.go", "seccheck.go", "strace.go", "vfs.go", diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index cd80b4bb8..ca5e8bf1b 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -34,13 +34,8 @@ import ( "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/seccheck" "gvisor.dev/gvisor/pkg/sentry/socket/netstack" - "gvisor.dev/gvisor/pkg/sentry/state" - "gvisor.dev/gvisor/pkg/sentry/time" "gvisor.dev/gvisor/pkg/sentry/vfs" - "gvisor.dev/gvisor/pkg/sentry/watchdog" - "gvisor.dev/gvisor/pkg/tcpip/stack" "gvisor.dev/gvisor/pkg/urpc" - "gvisor.dev/gvisor/runsc/boot/pprof" "gvisor.dev/gvisor/runsc/boot/procfs" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/specutils" @@ -463,7 +458,7 @@ type RestoreOpts struct { func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { log.Debugf("containerManager.Restore") - var specFile, deviceFile *os.File + r := restorer{container: &cm.l.root} switch numFiles := len(o.Files); numFiles { case 2: // The device file is donated to the platform. @@ -472,10 +467,16 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { if err != nil { return fmt.Errorf("failed to dup file: %v", err) } - deviceFile = os.NewFile(uintptr(fd), "platform device") + r.deviceFile = os.NewFile(uintptr(fd), "platform device") fallthrough case 1: - specFile = o.Files[0] + r.stateFile = o.Files[0] + if info, err := r.stateFile.Stat(); err != nil { + return err + } else if info.Size() == 0 { + return fmt.Errorf("file cannot be empty") + } + case 0: return fmt.Errorf("at least one file must be passed to Restore") default: @@ -485,90 +486,9 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error { // Pause the kernel while we build a new one. cm.l.k.Pause() - p, err := createPlatform(cm.l.root.conf, deviceFile) - if err != nil { - return fmt.Errorf("creating platform: %v", err) - } - k := &kernel.Kernel{ - Platform: p, - } - mf, err := createMemoryFile() - if err != nil { - return fmt.Errorf("creating memory file: %v", err) - } - k.SetMemoryFile(mf) - networkStack := cm.l.k.RootNetworkNamespace().Stack() - cm.l.k = k - - // Set up the restore environment. - ctx := k.SupervisorContext() - // TODO(b/298078576): Need to process hints here probably - mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.sharedMounts, cm.l.productName, o.SandboxID) - ctx, err = mntr.configureRestore(ctx) - if err != nil { - return fmt.Errorf("configuring filesystem restore: %v", err) - } - - // Prepare to load from the state file. - if eps, ok := networkStack.(*netstack.Stack); ok { - stack.StackFromEnv = eps.Stack // FIXME(b/36201077) - } - info, err := specFile.Stat() - if err != nil { + if err := r.restore(cm.l); err != nil { return err } - if info.Size() == 0 { - return fmt.Errorf("file cannot be empty") - } - - if cm.l.root.conf.ProfileEnable { - // pprof.Initialize opens /proc/self/maps, so has to be called before - // installing seccomp filters. - pprof.Initialize() - } - - // Seccomp filters have to be applied before parsing the state file. - if err := cm.l.installSeccompFilters(); err != nil { - return err - } - - // Load the state. - loadOpts := state.LoadOpts{Source: specFile} - if err := loadOpts.Load(ctx, k, nil, networkStack, time.NewCalibratedClocks(), &vfs.CompleteRestoreOptions{}); err != nil { - return err - } - - // Since we have a new kernel we also must make a new watchdog. - dogOpts := watchdog.DefaultOpts - dogOpts.TaskTimeoutAction = cm.l.root.conf.WatchdogAction - dog := watchdog.New(k, dogOpts) - - // Change the loader fields to reflect the changes made when restoring. - cm.l.k = k - cm.l.watchdog = dog - cm.l.root.procArgs = kernel.CreateProcessArgs{} - cm.l.restore = true - - // Reinitialize the sandbox ID and processes map. Note that it doesn't - // restore the state of multiple containers, nor exec processes. - cm.l.sandboxID = o.SandboxID - cm.l.mu.Lock() - - // Set new container ID if it has changed. - tasks := cm.l.k.TaskSet().Root.Tasks() - if tasks[0].ContainerID() != o.SandboxID { // There must be at least 1 task. - for _, task := range tasks { - task.RestoreContainerID(o.SandboxID) - } - } - - eid := execID{cid: o.SandboxID} - cm.l.processes = map[execID]*execProcess{ - eid: { - tg: cm.l.k.GlobalInit(), - }, - } - cm.l.mu.Unlock() // Tell the root container to start and wait for the result. cm.startChan <- struct{}{} diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 68680ee62..e3f3e9cfe 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -93,6 +93,8 @@ import ( ) type containerInfo struct { + cid string + conf *config.Config // spec is the base configuration for the root container. @@ -321,6 +323,7 @@ func New(args Args) (*Loader, error) { kernel.IOUringEnabled = args.Conf.IOUring info := containerInfo{ + cid: args.ID, conf: args.Conf, spec: args.Spec, goferMountConfs: args.GoferMountConfs, @@ -752,7 +755,7 @@ func (l *Loader) run() error { tg *kernel.ThreadGroup err error ) - tg, ep.tty, err = l.createContainerProcess(l.sandboxID, &l.root) + tg, ep.tty, err = l.createContainerProcess(&l.root) if err != nil { return err } @@ -879,6 +882,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st } info := &containerInfo{ + cid: cid, conf: conf, spec: spec, goferFDs: goferFDs, @@ -918,7 +922,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st }) } - ep.tg, ep.tty, err = l.createContainerProcess(cid, info) + ep.tg, ep.tty, err = l.createContainerProcess(info) if err != nil { return err } @@ -950,7 +954,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st } // +checklocks:l.mu -func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kernel.ThreadGroup, *host.TTYFileDescription, error) { +func (l *Loader) createContainerProcess(info *containerInfo) (*kernel.ThreadGroup, *host.TTYFileDescription, error) { // Create the FD map, which will set stdin, stdout, and stderr. ctx := info.procArgs.NewContext(l.k) fdTable, ttyFile, err := createFDTable(ctx, info.spec.Process.Terminal, info.stdioFDs, info.passFDs, info.spec.Process.User) @@ -985,7 +989,7 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne if len(info.goferFDs) < 1 { return nil, nil, fmt.Errorf("rootfs gofer FD not found") } - l.startGoferMonitor(cid, info) + l.startGoferMonitor(info) // We can share l.sharedMounts with containerMounter since l.mu is locked. // Hence, mntr must only be used within this function (while l.mu is locked). @@ -1046,7 +1050,7 @@ func (l *Loader) createContainerProcess(cid string, info *containerInfo) (*kerne // startGoferMonitor runs a goroutine to monitor gofer's health. It polls on // the gofer FD looking for disconnects, and kills the container processes if // the gofer connection disconnects. -func (l *Loader) startGoferMonitor(cid string, info *containerInfo) { +func (l *Loader) startGoferMonitor(info *containerInfo) { // We need to pick a suitable gofer connection that is expected to be alive // for the entire container lifecycle. Only the following can be used: // 1. Rootfs gofer connection @@ -1064,7 +1068,7 @@ func (l *Loader) startGoferMonitor(cid string, info *containerInfo) { return } go func() { - log.Debugf("Monitoring gofer health for container %q", cid) + log.Debugf("Monitoring gofer health for container %q", info.cid) events := []unix.PollFd{ { Fd: int32(goferFD), @@ -1085,10 +1089,10 @@ func (l *Loader) startGoferMonitor(cid string, info *containerInfo) { // The gofer could have been stopped due to a normal container shutdown. // Check if the container has not stopped yet. - if tg, _ := l.tryThreadGroupFromIDLocked(execID{cid: cid}); tg != nil { - log.Infof("Gofer socket disconnected, killing container %q", cid) - if err := l.signalAllProcesses(cid, int32(linux.SIGKILL)); err != nil { - log.Warningf("Error killing container %q after gofer stopped: %s", cid, err) + if tg, _ := l.tryThreadGroupFromIDLocked(execID{cid: info.cid}); tg != nil { + log.Infof("Gofer socket disconnected, killing container %q", info.cid) + if err := l.signalAllProcesses(info.cid, int32(linux.SIGKILL)); err != nil { + log.Warningf("Error killing container %q after gofer stopped: %s", info.cid, err) } } }() diff --git a/runsc/boot/restore.go b/runsc/boot/restore.go new file mode 100644 index 000000000..96e8f8470 --- /dev/null +++ b/runsc/boot/restore.go @@ -0,0 +1,118 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boot + +import ( + "fmt" + "os" + + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/socket/netstack" + "gvisor.dev/gvisor/pkg/sentry/state" + "gvisor.dev/gvisor/pkg/sentry/time" + "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/sentry/watchdog" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/runsc/boot/pprof" +) + +type restorer struct { + container *containerInfo + stateFile *os.File + deviceFile *os.File +} + +func (r *restorer) restore(l *Loader) error { + // Save the current network stack to slap on top of the one that was restored. + curNetwork := l.k.RootNetworkNamespace().Stack() + if eps, ok := curNetwork.(*netstack.Stack); ok { + stack.StackFromEnv = eps.Stack // FIXME(b/36201077) + } + + p, err := createPlatform(l.root.conf, r.deviceFile) + if err != nil { + return fmt.Errorf("creating platform: %v", err) + } + // Replace the old kernel with a new one that will be restored into. + l.k = &kernel.Kernel{ + Platform: p, + } + + mf, err := createMemoryFile() + if err != nil { + return fmt.Errorf("creating memory file: %v", err) + } + l.k.SetMemoryFile(mf) + + // Set up the restore environment. + ctx := l.k.SupervisorContext() + // TODO(b/298078576): Need to process hints here probably + mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, l.productName, r.container.cid) + ctx, err = mntr.configureRestore(ctx) + if err != nil { + return fmt.Errorf("configuring filesystem restore: %v", err) + } + + if l.root.conf.ProfileEnable { + // pprof.Initialize opens /proc/self/maps, so has to be called before + // installing seccomp filters. + pprof.Initialize() + } + + // Seccomp filters have to be applied before parsing the state file. + if err := l.installSeccompFilters(); err != nil { + return err + } + + // Load the state. + loadOpts := state.LoadOpts{Source: r.stateFile} + if err := loadOpts.Load(ctx, l.k, nil, curNetwork, time.NewCalibratedClocks(), &vfs.CompleteRestoreOptions{}); err != nil { + return err + } + + // Since we have a new kernel we also must make a new watchdog. + dogOpts := watchdog.DefaultOpts + dogOpts.TaskTimeoutAction = l.root.conf.WatchdogAction + dog := watchdog.New(l.k, dogOpts) + + // Change the loader fields to reflect the changes made when restoring. + l.watchdog = dog + l.root.procArgs = kernel.CreateProcessArgs{} + l.restore = true + + // Reinitialize the sandbox ID and processes map. Note that it doesn't + // restore the state of multiple containers, nor exec processes. + l.sandboxID = r.container.cid + + l.mu.Lock() + defer l.mu.Unlock() + + // Set new container ID if it has changed. + tasks := l.k.TaskSet().Root.Tasks() + if tasks[0].ContainerID() != l.sandboxID { // There must be at least 1 task. + for _, task := range tasks { + task.RestoreContainerID(l.sandboxID) + } + } + + eid := execID{cid: l.sandboxID} + l.processes = map[execID]*execProcess{ + eid: { + tg: l.k.GlobalInit(), + }, + } + + return nil +}