Track exec'ed processes and kill them after restore

Processes that are exec'ed into a container cannot be properly
restored because the caller is no longer present. This change
tracks processes that are exec'ed and kill them upon restore.

Updates #1956

PiperOrigin-RevId: 623644184

pkg/sentry/control/proc.go

@@ -213,6 +213,7 @@ func (proc *Proc) execAsync(args *ExecArgs) (*kernel.ThreadGroup, kernel.ThreadI
 		IPCNamespace:         proc.Kernel.RootIPCNamespace(),
 		ContainerID:          args.ContainerID,
 		PIDNamespace:         pidns,
+		Origin:               kernel.OriginExec,
 	}
 	if initArgs.MountNamespace != nil {
 		// initArgs must hold a reference on MountNamespace, which will

pkg/sentry/kernel/kernel.go

@@ -831,6 +831,9 @@ type CreateProcessArgs struct {
 
 	// InitialCgroups are the cgroups the container is initialized to.
 	InitialCgroups map[Cgroup]struct{}
+
+	// Origin indicates how the task was first created.
+	Origin TaskOrigin
 }
 
 // NewContext returns a context.Context that represents the task that will be
@@ -1049,6 +1052,7 @@ func (k *Kernel) CreateProcess(args CreateProcessArgs) (*ThreadGroup, ThreadID,
 		ContainerID:      args.ContainerID,
 		InitialCgroups:   args.InitialCgroups,
 		UserCounters:     k.GetUserCounters(args.Credentials.RealKUID),
+		Origin:           args.Origin,
 		// A task with no parent starts out with no session keyring.
 		SessionKeyring: nil,
 	}

pkg/sentry/kernel/task.go

@@ -36,6 +36,17 @@ import (
 	"gvisor.dev/gvisor/pkg/waiter"
 )
 
+// TaskOrigin indicates how the task was initially created.
+type TaskOrigin int
+
+const (
+	// OriginUnknown indicates that task creation source is not known (or not important).
+	OriginUnknown TaskOrigin = iota
+
+	// OriginExec indicates that task was created due to an exec request inside a container.
+	OriginExec
+)
+
 // Task represents a thread of execution in the untrusted app.  It
 // includes registers and any thread-specific state that you would
 // normally expect.
@@ -596,6 +607,9 @@ type Task struct {
 	//
 	// +checklocks:mu
 	sessionKeyring *auth.Key
+
+	// Origin is the origin of the task.
+	Origin TaskOrigin
 }
 
 // Task related metrics

pkg/sentry/kernel/task_clone.go

@@ -265,6 +265,7 @@ func (t *Task) Clone(args *linux.CloneArgs) (ThreadID, *SyscallControl, error) {
 		ContainerID:      t.ContainerID(),
 		UserCounters:     uc,
 		SessionKeyring:   sessionKeyring,
+		Origin:           t.Origin,
 	}
 	if args.Flags&linux.CLONE_THREAD == 0 {
 		cfg.Parent = t

pkg/sentry/kernel/task_start.go

@@ -103,6 +103,8 @@ type TaskConfig struct {
 	// SessionKeyring is the session keyring associated with the parent task.
 	// It may be nil.
 	SessionKeyring *auth.Key
+
+	Origin TaskOrigin
 }
 
 // NewTask creates a new task defined by cfg.
@@ -172,6 +174,7 @@ func (ts *TaskSet) newTask(ctx context.Context, cfg *TaskConfig) (*Task, error)
 		cgroups:        make(map[Cgroup]struct{}),
 		userCounters:   cfg.UserCounters,
 		sessionKeyring: cfg.SessionKeyring,
+		Origin:         cfg.Origin,
 	}
 	t.netns = cfg.NetworkNamespace
 	t.creds.Store(cfg.Credentials)

runsc/boot/restore.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"os"
 
+	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/host"
 	"gvisor.dev/gvisor/pkg/sentry/inet"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
@@ -163,6 +165,16 @@ func (r *restorer) restore(l *Loader) error {
 		}
 	}
 
+	// Kill all processes that have been exec'd since they cannot be properly
+	// restored, since the caller is no longer connected.
+	for _, tg := range l.k.RootPIDNamespace().ThreadGroups() {
+		if tg.Leader().Origin == kernel.OriginExec {
+			if err := l.k.SendExternalSignalThreadGroup(tg, &linux.SignalInfo{Signo: int32(linux.SIGKILL)}); err != nil {
+				log.Warningf("Failed to kill exec process after restore: %v", err)
+			}
+		}
+	}
+
 	eid := execID{cid: l.sandboxID}
 	l.processes = map[execID]*execProcess{
 		eid: {

runsc/container/container_test.go

@@ -1189,6 +1189,105 @@ func TestCheckpointRestore(t *testing.T) {
 	}
 }
 
+// TestCheckpointRestoreExecKilled checks that exec'd processes are killed
+// after the container is restored.
+func TestCheckpointRestoreExecKilled(t *testing.T) {
+	spec := testutil.NewSpecWithArgs("/bin/sleep", "10000")
+	conf := testutil.TestConfig(t)
+	_, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf)
+	if err != nil {
+		t.Fatalf("error setting up container: %v", err)
+	}
+	defer cleanup()
+
+	// Create and start the container.
+	args := Args{
+		ID:        testutil.RandomContainerID(),
+		Spec:      spec,
+		BundleDir: bundleDir,
+	}
+	cont, err := New(conf, args)
+	if err != nil {
+		t.Fatalf("error creating container: %v", err)
+	}
+	defer cont.Destroy()
+	if err := cont.Start(conf); err != nil {
+		t.Fatalf("error starting container: %v", err)
+	}
+
+	execArgs := &control.ExecArgs{
+		Filename: "/bin/sleep",
+		Argv:     []string{"/bin/sleep", "10000"},
+	}
+	pid1, err := cont.Execute(conf, execArgs)
+	if err != nil {
+		t.Fatalf("error executing in container: %v", err)
+	}
+	pid2, err := cont.Execute(conf, execArgs)
+	if err != nil {
+		t.Fatalf("error executing in container: %v", err)
+	}
+
+	// Since both share the same process name, ensure that the exec'd process
+	// has a different PID than the init process.
+	if pid1 == 1 || pid2 == 1 {
+		t.Fatalf("exec'd PID cannot be 1")
+	}
+	// Wait until the init process and exec'd processes are present.
+	expectedPL := []*control.Process{
+		newProcessBuilder().Cmd("sleep").PID(1).Process(),
+		newProcessBuilder().Cmd("sleep").PID(kernel.ThreadID(pid1)).Process(),
+		newProcessBuilder().Cmd("sleep").PID(kernel.ThreadID(pid2)).Process(),
+	}
+	if err := waitForProcessList(cont, expectedPL); err != nil {
+		t.Fatalf("Failed to kill exec'ed process, err: %v", err)
+	}
+
+	// Set the image path, which is where the checkpoint image will be saved.
+	dir, err := ioutil.TempDir(testutil.TmpDir(), "checkpoint-test")
+	if err != nil {
+		t.Fatalf("ioutil.TempDir failed: %v", err)
+	}
+	defer os.RemoveAll(dir)
+	if err := os.Chmod(dir, 0777); err != nil {
+		t.Fatalf("error chmoding file: %q, %v", dir, err)
+	}
+
+	// Create the image file and open for writing.
+	checkpointPath := filepath.Join(dir, "test-image-file")
+	checkpointFile, err := os.OpenFile(checkpointPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatalf("error opening new file at imagePath: %v", err)
+	}
+	defer checkpointFile.Close()
+
+	// Checkpoint running container; save state into new file.
+	if err := cont.Checkpoint(checkpointFile, statefile.Options{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil {
+		t.Fatalf("error checkpointing container: %v", err)
+	}
+	cont.Destroy()
+	cont = nil
+
+	cont2, err := New(conf, args)
+	if err != nil {
+		t.Fatalf("error creating container: %v", err)
+	}
+	defer cont2.Destroy()
+
+	if err := cont2.Restore(conf, checkpointPath); err != nil {
+		t.Fatalf("error restoring container: %v", err)
+	}
+
+	// Check that only the init process is present and the exec'ed
+	// processes were killed.
+	expectedPL = []*control.Process{
+		newProcessBuilder().Cmd("sleep").PID(1).Process(),
+	}
+	if err := waitForProcessList(cont2, expectedPL); err != nil {
+		t.Fatalf("Failed to kill exec'ed process, err: %v", err)
+	}
+}
+
 // TestUnixDomainSockets checks that Checkpoint/Restore works in cases
 // with filesystem Unix Domain Socket use.
 func TestUnixDomainSockets(t *testing.T) {