From 761bda09a55eed621673e9742fe5c5747fa42677 Mon Sep 17 00:00:00 2001 From: "B. Blechschmidt" Date: Fri, 31 Mar 2023 23:31:28 +0200 Subject: [PATCH] Add support for execution via host file descriptor This commit adds support for program execution via a host file descriptor. To make use of this feature, the host file descriptor must be provided to the --exec-fd argument. For example, exec 3= 0 { + info.execFD = fd.New(args.ExecFD) + } + for _, customFD := range args.PassFDs { info.passFDs = append(info.passFDs, fdMapping{ host: fd.New(customFD.Host), @@ -860,6 +870,26 @@ func (l *Loader) createContainerProcess(root bool, cid string, info *containerIn // ours either way. info.procArgs.FDTable = fdTable + if info.execFD != nil { + if info.procArgs.Filename != "" { + return nil, nil, fmt.Errorf("process must either be started from a file or a filename, not both") + } + file, err := host.NewFD(ctx, l.k.HostMount(), info.execFD.FD(), &host.NewFDOptions{ + Readonly: true, + Savable: true, + VirtualOwner: true, + UID: auth.KUID(info.spec.Process.User.UID), + GID: auth.KGID(info.spec.Process.User.GID), + }) + if err != nil { + return nil, nil, err + } + defer file.DecRef(ctx) + info.execFD.Release() + + info.procArgs.File = file + } + // Gofer FDs must be ordered and the first FD is always the rootfs. if len(info.goferFDs) < 1 { return nil, nil, fmt.Errorf("rootfs gofer FD not found") @@ -1430,6 +1460,9 @@ func createFDTable(ctx context.Context, console bool, stdioFDs []*fd.FD, passFDs // Create the entries for the host files that were passed to our app. for _, customFD := range passFDs { + if customFD.guest < 0 { + return nil, nil, fmt.Errorf("guest file descriptors must be 0 or greater") + } fdMap[customFD.guest] = customFD.host } diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 4c59859ca..c0a8c38f8 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -179,6 +179,10 @@ func setupContainerVFS(ctx context.Context, conf *config.Config, mntr *container } procArgs.MountNamespace = mns + // We are executing a file directly. Do not resolve the executable path. + if procArgs.File != nil { + return nil + } // Resolve the executable path from working dir and environment. resolved, err := user.ResolveExecutablePath(ctx, procArgs) if err != nil { diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 140ee5b64..da53104a0 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -89,6 +89,9 @@ type Boot struct { // passFDs are mappings of user-supplied host to guest file descriptors. passFDs fdMappings + // execFD is the host file descriptor used for program execution. + execFD int + // applyCaps determines if capabilities defined in the spec should be applied // to the process. applyCaps bool @@ -172,6 +175,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.Var(&b.ioFDs, "io-fds", "list of FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec") f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order") f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.") + f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.") f.Var(&b.overlayFilestoreFDs, "overlay-filestore-fds", "FDs to the regular files that will back the tmpfs upper mount in the overlay mounts.") f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.") f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup") @@ -390,6 +394,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma GoferFDs: b.ioFDs.GetArray(), StdioFDs: b.stdioFDs.GetArray(), PassFDs: b.passFDs.GetArray(), + ExecFD: b.execFD, OverlayFilestoreFDs: b.overlayFilestoreFDs.GetArray(), NumCPU: b.cpuNum, TotalMem: b.totalMem, diff --git a/runsc/cmd/exec.go b/runsc/cmd/exec.go index 6da5e96a5..de268ae0a 100644 --- a/runsc/cmd/exec.go +++ b/runsc/cmd/exec.go @@ -61,6 +61,9 @@ type Exec struct { // passFDs are user-supplied FDs from the host to be exposed to the // sandboxed app. passFDs fdMappings + + // execFD is the host file descriptor used for program execution. + execFD int } // Name implements subcommands.Command.Name. @@ -105,6 +108,7 @@ func (ex *Exec) SetFlags(f *flag.FlagSet) { f.StringVar(&ex.internalPidFile, "internal-pid-file", "", "filename that the container-internal pid will be written to") f.StringVar(&ex.consoleSocket, "console-socket", "", "path to an AF_UNIX socket which will receive a file descriptor referencing the master end of the console's pseudoterminal") f.Var(&ex.passFDs, "pass-fd", "file descriptor passed to the container in M:N format, where M is the host and N is the guest descriptor (can be supplied multiple times)") + f.IntVar(&ex.execFD, "exec-fd", -1, "host file descriptor used for program execution") } // Execute implements subcommands.Command.Execute. It starts a process in an @@ -160,6 +164,11 @@ func (ex *Exec) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm fdMap[mapping.Guest] = file } + var execFile *os.File + if ex.execFD >= 0 { + execFile = os.NewFile(uintptr(ex.execFD), "exec-fd") + } + // Close the underlying file descriptors after we have passed them. defer func() { for _, file := range fdMap { @@ -168,9 +177,13 @@ func (ex *Exec) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm log.Debugf("Failed to close FD %d", fd) } } + + if execFile != nil && execFile.Close() != nil { + log.Debugf("Failed to close exec FD") + } }() - e.FilePayload = control.NewFDMap(fdMap) + e.FilePayload = control.NewFilePayload(fdMap, execFile) // containerd expects an actual process to represent the container being // executed. If detach was specified, starts a child in non-detach mode, @@ -362,11 +375,11 @@ func (ex *Exec) argsFromCLI(argv []string, enableRaw bool) (*control.ExecArgs, e ExtraKGIDs: extraKGIDs, Capabilities: caps, StdioIsPty: ex.consoleSocket != "" || console.IsPty(os.Stdin.Fd()), - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: os.Stdin, 1: os.Stdout, 2: os.Stderr, - }), + }, nil), }, nil } @@ -415,11 +428,11 @@ func argsFromProcess(p *specs.Process, enableRaw bool) (*control.ExecArgs, error ExtraKGIDs: extraKGIDs, Capabilities: caps, StdioIsPty: p.Terminal, - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: os.Stdin, 1: os.Stdout, 2: os.Stderr, - }), + }, nil), }, nil } diff --git a/runsc/cmd/exec_test.go b/runsc/cmd/exec_test.go index 4c858911e..20c804d97 100644 --- a/runsc/cmd/exec_test.go +++ b/runsc/cmd/exec_test.go @@ -75,11 +75,11 @@ func TestCLIArgs(t *testing.T) { expected: control.ExecArgs{ Argv: []string{"ls", "/"}, WorkingDirectory: "/foo/bar", - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: os.Stdin, 1: os.Stdout, 2: os.Stderr, - }), + }, nil), KUID: 0, KGID: 0, ExtraKGIDs: []auth.KGID{1, 2, 3}, @@ -132,11 +132,11 @@ func TestJSONArgs(t *testing.T) { expected: control.ExecArgs{ Argv: []string{"ls", "/"}, WorkingDirectory: "/foo/bar", - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: os.Stdin, 1: os.Stdout, 2: os.Stderr, - }), + }, nil), KUID: 0, KGID: 0, ExtraKGIDs: []auth.KGID{1, 2, 3}, diff --git a/runsc/cmd/run.go b/runsc/cmd/run.go index 1e6dbe90c..ba4fa1aea 100644 --- a/runsc/cmd/run.go +++ b/runsc/cmd/run.go @@ -39,6 +39,9 @@ type Run struct { // passFDs are user-supplied FDs from the host to be exposed to the // sandboxed app. passFDs fdMappings + + // execFD is the host file descriptor used for program execution. + execFD int } // Name implements subcommands.Command.Name. @@ -61,6 +64,7 @@ func (*Run) Usage() string { func (r *Run) SetFlags(f *flag.FlagSet) { f.BoolVar(&r.detach, "detach", false, "detach from the container's process") f.Var(&r.passFDs, "pass-fd", "file descriptor passed to the container in M:N format, where M is the host and N is the guest descriptor (can be supplied multiple times)") + f.IntVar(&r.execFD, "exec-fd", -1, "host file descriptor used for program execution") r.Create.SetFlags(f) } @@ -106,6 +110,11 @@ func (r *Run) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomman fdMap[mapping.Guest] = file } + var execFile *os.File + if r.execFD >= 0 { + execFile = os.NewFile(uintptr(r.execFD), "exec-fd") + } + // Close the underlying file descriptors after we have passed them. defer func() { for _, file := range fdMap { @@ -114,6 +123,10 @@ func (r *Run) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomman log.Debugf("Failed to close FD %d", fd) } } + + if execFile != nil && execFile.Close() != nil { + log.Debugf("Failed to close exec FD") + } }() runArgs := container.Args{ @@ -125,6 +138,7 @@ func (r *Run) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomman UserLog: r.userLog, Attached: !r.detach, PassFiles: fdMap, + ExecFile: execFile, } ws, err := container.Run(conf, runArgs) if err != nil { diff --git a/runsc/container/console_test.go b/runsc/container/console_test.go index 0c98c9364..c668bed38 100644 --- a/runsc/container/console_test.go +++ b/runsc/container/console_test.go @@ -281,9 +281,9 @@ func TestJobControlSignalExec(t *testing.T) { // our PID counts get messed up. Argv: []string{"/bin/bash", "--noprofile", "--norc"}, // Pass the pty replica as FD 0, 1, and 2. - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: ptyReplica, 1: ptyReplica, 2: ptyReplica, - }), + }, nil), StdioIsPty: true, } diff --git a/runsc/container/container.go b/runsc/container/container.go index baf1d6748..d727b77d3 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -180,6 +180,9 @@ type Args struct { // PassFiles are user-supplied files from the host to be exposed to the // sandboxed app. PassFiles map[int]*os.File + + // ExecFile is the host file used for program execution. + ExecFile *os.File } // New creates the container in a new Sandbox process, unless the metadata @@ -301,6 +304,7 @@ func New(conf *config.Config, args Args) (*Container, error) { Attached: args.Attached, OverlayFilestoreFiles: overlayFilestoreFiles, PassFiles: args.PassFiles, + ExecFile: args.ExecFile, } sand, err := sandbox.New(conf, sandArgs) if err != nil { diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 26cc3186f..1a3e34a46 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -69,19 +69,26 @@ func execute(conf *config.Config, cont *Container, name string, arg ...string) ( return cont.executeSync(conf, args) } -func executeCombinedOutput(conf *config.Config, cont *Container, name string, arg ...string) ([]byte, error) { +// executeCombinedOutput executes a process in the container and captures +// stdout and stderr. If execFile is supplied, a host file will be executed. +// Otherwise, the name argument is used to resolve the executable in the guest. +func executeCombinedOutput(conf *config.Config, cont *Container, execFile *os.File, name string, arg ...string) ([]byte, error) { r, w, err := os.Pipe() if err != nil { return nil, err } defer r.Close() + // Unset the filename when we execute via FD. + if execFile != nil { + name = "" + } args := &control.ExecArgs{ Filename: name, Argv: append([]string{name}, arg...), - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: os.Stdin, 1: w, 2: w, - }), + }, execFile), } ws, err := cont.executeSync(conf, args) w.Close() @@ -176,7 +183,7 @@ func blockUntilWaitable(pid int) error { // execPS executes `ps` inside the container and return the processes. func execPS(conf *config.Config, c *Container) ([]*control.Process, error) { - out, err := executeCombinedOutput(conf, c, "/bin/ps", "-e") + out, err := executeCombinedOutput(conf, c, nil, "/bin/ps", "-e") if err != nil { return nil, err } @@ -854,9 +861,9 @@ func TestExec(t *testing.T) { _, err = cont.executeSync(conf, &control.ExecArgs{ Argv: []string{"/nonexist"}, - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ 0: os.NewFile(uintptr(fds[1]), "sock"), - }), + }, nil), }) want := "failed to load /nonexist" if err == nil || !strings.Contains(err.Error(), want) { @@ -1625,7 +1632,7 @@ func TestReadonlyRoot(t *testing.T) { } // Read mounts to check that root is readonly. - out, err := executeCombinedOutput(conf, c, "/bin/sh", "-c", "mount | grep ' / ' | grep -o -e '(.*)'") + out, err := executeCombinedOutput(conf, c, nil, "/bin/sh", "-c", "mount | grep ' / ' | grep -o -e '(.*)'") if err != nil { t.Fatalf("exec failed: %v", err) } @@ -1684,7 +1691,7 @@ func TestReadonlyMount(t *testing.T) { // Read mounts to check that volume is readonly. cmd := fmt.Sprintf("mount | grep ' %s ' | grep -o -e '(.*)'", dir) - out, err := executeCombinedOutput(conf, c, "/bin/sh", "-c", cmd) + out, err := executeCombinedOutput(conf, c, nil, "/bin/sh", "-c", cmd) if err != nil { t.Fatalf("exec failed, err: %v", err) } @@ -2505,7 +2512,7 @@ func TestRlimitsExec(t *testing.T) { t.Fatalf("error starting container: %v", err) } - got, err := executeCombinedOutput(conf, cont, "/bin/sh", "-c", "ulimit -n") + got, err := executeCombinedOutput(conf, cont, nil, "/bin/sh", "-c", "ulimit -n") if err != nil { t.Fatal(err) } @@ -2869,10 +2876,10 @@ func TestFDPassingExec(t *testing.T) { cmd := fmt.Sprintf("cat /proc/self/fd/%d > /proc/self/fd/%d", int(guestRead.Fd()), int(guestWrite.Fd())) execArgs := &control.ExecArgs{ Argv: []string{"/bin/bash", "-c", cmd}, - FilePayload: control.NewFDMap(map[int]*os.File{ + FilePayload: control.NewFilePayload(map[int]*os.File{ int(guestRead.Fd()): guestRead, int(guestWrite.Fd()): guestWrite, - }), + }, nil), } if _, err = cont.Execute(conf, execArgs); err != nil { @@ -2892,3 +2899,136 @@ func TestFDPassingExec(t *testing.T) { t.Errorf("got message %q, want %q", got, msg) } } + +// findInPath finds a filename in the PATH environment variable. +func findInPath(filename string) string { + for _, dir := range strings.Split(os.Getenv("PATH"), ":") { + fullPath := filepath.Join(dir, filename) + if _, err := os.Stat(fullPath); err == nil { + return fullPath + } + } + return "" +} + +// TestExecFDRun checks that an executable from the host can be started inside +// a container. +func TestExecFDRun(t *testing.T) { + // In the guest, read from the host and write the result back to the host. + conf := testutil.TestConfig(t) + // Note that we do not supply the name or path of the echo binary here. + // Thus, the guest does not know the binary path or name either. + // argv[0] inside echo is "can be anything". + spec := testutil.NewSpecWithArgs("can be anything", "hello world") + + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + // Find the echo binary on the host. + echoPath := findInPath("echo") + if echoPath == "" { + t.Fatalf("failed to find echo executable in PATH") + } + + // Open the echo binary as a file. + echoFile, err := os.Open(echoPath) + if err != nil { + t.Fatalf("opening echo binary: %v", err) + } + defer echoFile.Close() + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("creating pipe: %v", err) + } + defer r.Close() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + PassFiles: map[int]*os.File{ + 0: os.Stdin, 1: w, 2: w, + }, + ExecFile: echoFile, + } + + cont, err := New(conf, args) + if err != nil { + t.Fatalf("Creating container: %v", err) + } + defer cont.Destroy() + + if err := cont.Start(conf); err != nil { + t.Fatalf("starting container: %v", err) + } + + w.Close() + + got, err := io.ReadAll(r) + if err != nil { + t.Errorf("reading container output: %v", err) + } + if want := "hello world\n"; string(got) != want { + t.Errorf("got message %q, want %q", got, want) + } +} + +// TestExecFDExec checks that an executable from the host can be started from a +// file descriptor inside an already running container. +func TestExecFDExec(t *testing.T) { + conf := testutil.TestConfig(t) + + // We just sleep here because we want to test execution in an already + // running container. + spec := testutil.NewSpecWithArgs("bash", "-c", "sleep infinity") + + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + + cont, err := New(conf, args) + if err != nil { + t.Fatalf("Creating container: %v", err) + } + defer cont.Destroy() + + if err := cont.Start(conf); err != nil { + t.Fatalf("starting container: %v", err) + } + + // Find the echo binary on the host. + echoPath := findInPath("echo") + if echoPath == "" { + t.Fatalf("failed to find echo executable in PATH") + } + + // Open the echo binary as a file. + echoFile, err := os.Open(echoPath) + if err != nil { + t.Fatalf("opening echo binary: %v", err) + } + defer echoFile.Close() + + // Note that we do not supply the name or path of the echo binary here. + // Thus, the guest does not know the binary path or name either. + // argv[0] inside echo is "can be anything". + got, err := executeCombinedOutput(conf, cont, echoFile, "can be anything", "hello world") + if err != nil { + t.Fatal(err) + } + if want := "hello world\n"; string(got) != want { + t.Errorf("echo result, got: %q, want: %q", got, want) + } +} diff --git a/runsc/container/metric_server_test.go b/runsc/container/metric_server_test.go index 01f7dc92a..d3d86c116 100644 --- a/runsc/container/metric_server_test.go +++ b/runsc/container/metric_server_test.go @@ -184,7 +184,7 @@ func TestContainerMetrics(t *testing.T) { } t.Logf("After container start, fs_opens=%d (snapshotted at %v)", postStartOpens, postStartTimestamp) // The touch operation may fail from permission errors, but the metric should still be incremented. - shOutput, err := executeCombinedOutput(te.sleepConf, cont, "/bin/bash", "-c", fmt.Sprintf("for i in $(seq 1 %d); do touch /tmp/$i || true; done", targetOpens)) + shOutput, err := executeCombinedOutput(te.sleepConf, cont, nil, "/bin/bash", "-c", fmt.Sprintf("for i in $(seq 1 %d); do touch /tmp/$i || true; done", targetOpens)) if err != nil { t.Fatalf("Exec failed: %v; output: %v", err, shOutput) } @@ -290,7 +290,7 @@ func TestContainerMetricsRobustAgainstRestarts(t *testing.T) { if err := cont.Start(te.sleepConf); err != nil { t.Fatalf("Cannot start container: %v", err) } - shOutput, err := executeCombinedOutput(te.sleepConf, cont, "/bin/bash", "-c", fmt.Sprintf("for i in $(seq 1 %d); do touch /tmp/$i || true; done", targetOpens)) + shOutput, err := executeCombinedOutput(te.sleepConf, cont, nil, "/bin/bash", "-c", fmt.Sprintf("for i in $(seq 1 %d); do touch /tmp/$i || true; done", targetOpens)) if err != nil { t.Fatalf("Exec failed: %v; output: %v", err, shOutput) } @@ -327,7 +327,7 @@ func TestContainerMetricsRobustAgainstRestarts(t *testing.T) { // Do a bunch of touches again. The metric server is down during this time. // This verifies that metric value modifications does not depend on the metric server being up. - shOutput, err = executeCombinedOutput(te.sleepConf, cont, "/bin/bash", "-c", fmt.Sprintf("for i in $(seq 1 %d); do touch /tmp/$i || true; done", targetOpens)) + shOutput, err = executeCombinedOutput(te.sleepConf, cont, nil, "/bin/bash", "-c", fmt.Sprintf("for i in $(seq 1 %d); do touch /tmp/$i || true; done", targetOpens)) if err != nil { t.Fatalf("Exec failed: %v; output: %v", err, shOutput) } diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index 2c9be0efa..74a4b9b07 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -2205,7 +2205,7 @@ func TestMultiContainerShm(t *testing.T) { } // Check that file can be found in the other container. - out, err := executeCombinedOutput(conf, containers[1], "/bin/cat", output) + out, err := executeCombinedOutput(conf, containers[1], nil, "/bin/cat", output) if err != nil { t.Fatalf("exec failed: %v", err) } diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index a3d1490f6..2b7bc940d 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -245,6 +245,9 @@ type Args struct { // PassFiles are user-supplied files from the host to be exposed to the // sandboxed app. PassFiles map[int]*os.File + + // ExecFile is the file from the host used for program execution. + ExecFile *os.File } // New creates the sandbox process. The caller must call Destroy() on the @@ -959,6 +962,10 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn cmd.Args = append(cmd.Args, "--attached") } + if args.ExecFile != nil { + donations.Donate("exec-fd", args.ExecFile) + } + nextFD = donations.Transfer(cmd, nextFD) _ = donation.DonateAndTransferCustomFiles(cmd, nextFD, args.PassFiles)