From 8174aec84e72bbacd327eda5f78bbca12b0fdd68 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Thu, 3 Mar 2022 12:48:08 -0800 Subject: [PATCH] Add helper library to handle file donation to child process PiperOrigin-RevId: 432260948 --- runsc/container/BUILD | 1 + runsc/container/container.go | 79 +++----- runsc/donation/BUILD | 15 ++ runsc/donation/donation.go | 110 +++++++++++ runsc/sandbox/BUILD | 1 + runsc/sandbox/sandbox.go | 355 +++++++++++++---------------------- 6 files changed, 290 insertions(+), 271 deletions(-) create mode 100644 runsc/donation/BUILD create mode 100644 runsc/donation/donation.go diff --git a/runsc/container/BUILD b/runsc/container/BUILD index fe2ef3c69..ca6f6a7d3 100644 --- a/runsc/container/BUILD +++ b/runsc/container/BUILD @@ -25,6 +25,7 @@ go_library( "//runsc/cgroup", "//runsc/config", "//runsc/console", + "//runsc/donation", "//runsc/sandbox", "//runsc/specutils", "@com_github_cenkalti_backoff//:go_default_library", diff --git a/runsc/container/container.go b/runsc/container/container.go index 942af0b50..ec606e34a 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -40,6 +40,7 @@ import ( "gvisor.dev/gvisor/runsc/cgroup" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/console" + "gvisor.dev/gvisor/runsc/donation" "gvisor.dev/gvisor/runsc/sandbox" "gvisor.dev/gvisor/runsc/specutils" ) @@ -886,26 +887,12 @@ func (c *Container) waitForStopped() error { } func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bundleDir string, attached bool) ([]*os.File, *os.File, error) { - // Start with the general config flags. - args := conf.ToFlags() + donations := donation.Agency{} + defer donations.Close() - var goferEnds []*os.File - - // nextFD is the next available file descriptor for the gofer process. - // It starts at 3 because 0-2 are used by stdin/stdout/stderr. - nextFD := 3 - - if conf.LogFilename != "" { - logFile, err := os.OpenFile(conf.LogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return nil, nil, fmt.Errorf("opening log file %q: %v", conf.LogFilename, err) - } - defer logFile.Close() - goferEnds = append(goferEnds, logFile) - args = append(args, "--log-fd="+strconv.Itoa(nextFD)) - nextFD++ + if err := donations.OpenAndDonate("log-fd", conf.LogFilename, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { + return nil, nil, err } - if conf.DebugLog != "" { test := "" if len(conf.TestOnlyTestNameEnv) != 0 { @@ -914,27 +901,31 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu test = t } } - debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "gofer", test) - if err != nil { - return nil, nil, fmt.Errorf("opening debug log file in %q: %v", conf.DebugLog, err) + if err := donations.DonateDebugLogFile("debug-log-fd", conf.DebugLog, "gofer", test); err != nil { + return nil, nil, err } - defer debugLogFile.Close() - goferEnds = append(goferEnds, debugLogFile) - args = append(args, "--debug-log-fd="+strconv.Itoa(nextFD)) - nextFD++ } - args = append(args, "gofer", "--bundle", bundleDir) + // Start with the general config flags. + cmd := exec.Command(specutils.ExePath, conf.ToFlags()...) + cmd.SysProcAttr = &unix.SysProcAttr{} + + // Set Args[0] to make easier to spot the gofer process. Otherwise it's + // shown as `exe`. + cmd.Args[0] = "runsc-gofer" + + // Tranfer FDs that need to be present before the "gofer" command. + // Start at 3 because 0, 1, and 2 are taken by stdin/out/err. + nextFD := donations.Transfer(cmd, 3) + + cmd.Args = append(cmd.Args, "gofer", "--bundle", bundleDir) // Open the spec file to donate to the sandbox. specFile, err := specutils.OpenSpec(bundleDir) if err != nil { return nil, nil, fmt.Errorf("opening spec file: %v", err) } - defer specFile.Close() - goferEnds = append(goferEnds, specFile) - args = append(args, "--spec-fd="+strconv.Itoa(nextFD)) - nextFD++ + donations.DonateAndClose("spec-fd", specFile) // Create pipe that allows gofer to send mount list to sandbox after all paths // have been resolved. @@ -942,10 +933,7 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu if err != nil { return nil, nil, err } - defer mountsGofer.Close() - goferEnds = append(goferEnds, mountsGofer) - args = append(args, fmt.Sprintf("--mounts-fd=%d", nextFD)) - nextFD++ + donations.DonateAndClose("mounts-fd", mountsGofer) // Add root mount and then add any other additional mounts. mountCount := 1 @@ -964,27 +952,13 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu sandEnds = append(sandEnds, os.NewFile(uintptr(fds[0]), "sandbox IO FD")) goferEnd := os.NewFile(uintptr(fds[1]), "gofer IO FD") - defer goferEnd.Close() - goferEnds = append(goferEnds, goferEnd) - - args = append(args, fmt.Sprintf("--io-fds=%d", nextFD)) - nextFD++ + donations.DonateAndClose("io-fds", goferEnd) } - binPath := specutils.ExePath - cmd := exec.Command(binPath, args...) - cmd.ExtraFiles = goferEnds - - // Set Args[0] to make easier to spot the gofer process. Otherwise it's - // shown as `exe`. - cmd.Args[0] = "runsc-gofer" - if attached { // The gofer is attached to the lifetime of this process, so it // should synchronously die when this process dies. - cmd.SysProcAttr = &unix.SysProcAttr{ - Pdeathsig: unix.SIGKILL, - } + cmd.SysProcAttr.Pdeathsig = unix.SIGKILL } // Enter new namespaces to isolate from the rest of the system. Don't unshare @@ -1008,8 +982,11 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 0, Gid: 0} } + donations.Transfer(cmd, nextFD) + // Start the gofer in the given namespace. - log.Debugf("Starting gofer: %s %v", binPath, args) + donation.LogDonations(cmd) + log.Debugf("Starting gofer: %s %v", cmd.Path, cmd.Args) if err := specutils.StartInNS(cmd, nss); err != nil { return nil, nil, fmt.Errorf("gofer: %v", err) } diff --git a/runsc/donation/BUILD b/runsc/donation/BUILD new file mode 100644 index 000000000..f355f4b8b --- /dev/null +++ b/runsc/donation/BUILD @@ -0,0 +1,15 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "donation", + srcs = [ + "donation.go", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/log", + "//runsc/specutils", + ], +) diff --git a/runsc/donation/donation.go b/runsc/donation/donation.go new file mode 100644 index 000000000..c6b80601e --- /dev/null +++ b/runsc/donation/donation.go @@ -0,0 +1,110 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package donation tracks files that are being donated to a child process and +// using flags to notified the child process where the FDs are. +package donation + +import ( + "fmt" + "os" + "os/exec" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/runsc/specutils" +) + +// LogDonations logs the FDs we are donating in the command. +func LogDonations(cmd *exec.Cmd) { + for i, f := range cmd.ExtraFiles { + log.Debugf("Donating FD %d: %q", i+3, f.Name()) + } +} + +// Agency keeps track of files that need to be donated to a child process. +type Agency struct { + donations []donation + closePending []*os.File +} + +type donation struct { + flag string + files []*os.File +} + +// Donate sets up the given files to be donated to another process. The FD +// in which the new file will appear in the child process is added as a flag to +// the child process, e.g. --flag=3. +func (f *Agency) Donate(flag string, files ...*os.File) { + f.donations = append(f.donations, donation{flag: flag, files: files}) +} + +// DonateAndClose does the same as Donate, but takes ownership of the files +// passed in. +func (f *Agency) DonateAndClose(flag string, files ...*os.File) { + f.Donate(flag, files...) + f.closePending = append(f.closePending, files...) +} + +// OpenAndDonate is similar to DonateAndClose but handles the opening of the +// file for convenience. It's a noop, if path is empty. +func (f *Agency) OpenAndDonate(flag, path string, flags int) error { + if len(path) == 0 { + return nil + } + file, err := os.OpenFile(path, flags, 0644) + if err != nil { + return err + } + f.DonateAndClose(flag, file) + return nil +} + +// DonateDebugLogFile is similar to DonateAndClose but handles the opening of +// the file using specutils.DebugLogFile() for convenience. It's a noop, if +// path is empty. +func (f *Agency) DonateDebugLogFile(flag, logPattern, command, test string) error { + if len(logPattern) == 0 { + return nil + } + file, err := specutils.DebugLogFile(logPattern, command, test) + if err != nil { + return fmt.Errorf("opening debug log file in %q: %v", logPattern, err) + } + f.DonateAndClose(flag, file) + return nil +} + +// Transfer sets up all files and flags to cmd. It can be called multiple times +// to partially transfer files to cmd. +func (f *Agency) Transfer(cmd *exec.Cmd, nextFD int) int { + for _, d := range f.donations { + for _, file := range d.files { + cmd.ExtraFiles = append(cmd.ExtraFiles, file) + cmd.Args = append(cmd.Args, fmt.Sprintf("--%s=%d", d.flag, nextFD)) + nextFD++ + } + } + // Reset donations made so far in case more transfers are needed. + f.donations = nil + return nextFD +} + +// Close closes any files the agency has taken ownership over. +func (f *Agency) Close() { + for _, file := range f.closePending { + _ = file.Close() + } + f.closePending = nil +} diff --git a/runsc/sandbox/BUILD b/runsc/sandbox/BUILD index 202e5eae7..313881f80 100644 --- a/runsc/sandbox/BUILD +++ b/runsc/sandbox/BUILD @@ -31,6 +31,7 @@ go_library( "//runsc/cgroup", "//runsc/config", "//runsc/console", + "//runsc/donation", "//runsc/specutils", "@com_github_cenkalti_backoff//:go_default_library", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 8b59fa512..8e47603db 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -48,6 +48,7 @@ import ( "gvisor.dev/gvisor/runsc/cgroup" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/console" + "gvisor.dev/gvisor/runsc/donation" "gvisor.dev/gvisor/runsc/specutils" ) @@ -171,7 +172,12 @@ type Args struct { // New creates the sandbox process. The caller must call Destroy() on the // sandbox. func New(conf *config.Config, args *Args) (*Sandbox, error) { - s := &Sandbox{ID: args.ID, CgroupJSON: cgroup.CgroupJSON{Cgroup: args.Cgroup}} + s := &Sandbox{ + ID: args.ID, + CgroupJSON: cgroup.CgroupJSON{Cgroup: args.Cgroup}, + UID: -1, // prevent usage before it's set. + GID: -1, // prevent usage before it's set. + } // The Cleanup object cleans up partially created sandboxes when an error // occurs. Any errors occurring during cleanup itself are ignored. c := cleanup.Make(func() { @@ -428,26 +434,16 @@ func (s *Sandbox) connError(err error) error { // createSandboxProcess starts the sandbox as a subprocess by running the "boot" // command, passing in the bundle dir. func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyncFile *os.File) error { - // nextFD is used to get unused FDs that we can pass to the sandbox. It - // starts at 3 because 0, 1, and 2 are taken by stdin/out/err. - nextFD := 3 + donations := donation.Agency{} + defer donations.Close() - binPath := specutils.ExePath - cmd := exec.Command(binPath, conf.ToFlags()...) - cmd.SysProcAttr = &unix.SysProcAttr{} - - // Open the log files to pass to the sandbox as FDs. // // These flags must come BEFORE the "boot" command in cmd.Args. - if conf.LogFilename != "" { - logFile, err := os.OpenFile(conf.LogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("opening log file %q: %v", conf.LogFilename, err) - } - defer logFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, logFile) - cmd.Args = append(cmd.Args, "--log-fd="+strconv.Itoa(nextFD)) - nextFD++ + // + + // Open the log files to pass to the sandbox as FDs. + if err := donations.OpenAndDonate("log-fd", conf.LogFilename, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { + return err } test := "" @@ -457,46 +453,66 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn test = t } } - if conf.DebugLog != "" { - debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "boot", test) - if err != nil { - return fmt.Errorf("opening debug log file in %q: %v", conf.DebugLog, err) - } - defer debugLogFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, debugLogFile) - cmd.Args = append(cmd.Args, "--debug-log-fd="+strconv.Itoa(nextFD)) - nextFD++ + if err := donations.DonateDebugLogFile("debug-log-fd", conf.DebugLog, "boot", test); err != nil { + return err } - if conf.PanicLog != "" { - panicLogFile, err := specutils.DebugLogFile(conf.PanicLog, "panic", test) - if err != nil { - return fmt.Errorf("opening panic log file in %q: %v", conf.PanicLog, err) - } - defer panicLogFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, panicLogFile) - cmd.Args = append(cmd.Args, "--panic-log-fd="+strconv.Itoa(nextFD)) - nextFD++ + if err := donations.DonateDebugLogFile("panic-log-fd", conf.PanicLog, "panic", test); err != nil { + return err } covFilename := conf.CoverageReport if covFilename == "" { covFilename = os.Getenv("GO_COVERAGE_FILE") } if covFilename != "" && coverage.Available() { - covFile, err := specutils.DebugLogFile(covFilename, "cov", test) - if err != nil { - return fmt.Errorf("opening debug log file in %q: %v", covFilename, err) + if err := donations.DonateDebugLogFile("coverage-fd", covFilename, "cov", test); err != nil { + return err } - defer covFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, covFile) - cmd.Args = append(cmd.Args, "--coverage-fd="+strconv.Itoa(nextFD)) - nextFD++ } + cmd := exec.Command(specutils.ExePath, conf.ToFlags()...) + cmd.SysProcAttr = &unix.SysProcAttr{ + // Detach from this session, otherwise cmd will get SIGHUP and SIGCONT + // when re-parented. + Setsid: true, + } + + // Set Args[0] to make easier to spot the sandbox process. Otherwise it's + // shown as `exe`. + cmd.Args[0] = "runsc-sandbox" + + // Tranfer FDs that need to be present before the "boot" command. + // Start at 3 because 0, 1, and 2 are taken by stdin/out/err. + nextFD := donations.Transfer(cmd, 3) + // Add the "boot" command to the args. // // All flags after this must be for the boot command cmd.Args = append(cmd.Args, "boot", "--bundle="+args.BundleDir) + // If there is a gofer, sends all socket ends to the sandbox. + donations.DonateAndClose("io-fds", args.IOFiles...) + donations.DonateAndClose("mounts-fd", args.MountsFile) + donations.Donate("start-sync-fd", startSyncFile) + if err := donations.OpenAndDonate("user-log-fd", args.UserLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND); err != nil { + return err + } + const profFlags = os.O_CREATE | os.O_WRONLY | os.O_TRUNC + if err := donations.OpenAndDonate("profile-block-fd", conf.ProfileBlock, profFlags); err != nil { + return err + } + if err := donations.OpenAndDonate("profile-cpu-fd", conf.ProfileCPU, profFlags); err != nil { + return err + } + if err := donations.OpenAndDonate("profile-heap-fd", conf.ProfileHeap, profFlags); err != nil { + return err + } + if err := donations.OpenAndDonate("profile-mutex-fd", conf.ProfileMutex, profFlags); err != nil { + return err + } + if err := donations.OpenAndDonate("trace-fd", conf.TraceFile, profFlags); err != nil { + return err + } + // Create a socket for the control server and donate it to the sandbox. addr := boot.ControlSocketAddr(s.ID) sockFD, err := server.CreateSocket(addr) @@ -504,105 +520,22 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn if err != nil { return fmt.Errorf("creating control server socket for sandbox %q: %v", s.ID, err) } - controllerFile := os.NewFile(uintptr(sockFD), "control_server_socket") - defer controllerFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, controllerFile) - cmd.Args = append(cmd.Args, "--controller-fd="+strconv.Itoa(nextFD)) - nextFD++ - - defer args.MountsFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, args.MountsFile) - cmd.Args = append(cmd.Args, "--mounts-fd="+strconv.Itoa(nextFD)) - nextFD++ + donations.DonateAndClose("controller-fd", os.NewFile(uintptr(sockFD), "control_server_socket")) specFile, err := specutils.OpenSpec(args.BundleDir) if err != nil { return err } - defer specFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, specFile) - cmd.Args = append(cmd.Args, "--spec-fd="+strconv.Itoa(nextFD)) - nextFD++ - - cmd.ExtraFiles = append(cmd.ExtraFiles, startSyncFile) - cmd.Args = append(cmd.Args, "--start-sync-fd="+strconv.Itoa(nextFD)) - nextFD++ - - if conf.ProfileBlock != "" { - blockFile, err := os.OpenFile(conf.ProfileBlock, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("opening block profiling file %q: %v", conf.ProfileBlock, err) - } - defer blockFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, blockFile) - cmd.Args = append(cmd.Args, "--profile-block-fd="+strconv.Itoa(nextFD)) - nextFD++ - } - - if conf.ProfileCPU != "" { - cpuFile, err := os.OpenFile(conf.ProfileCPU, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("opening cpu profiling file %q: %v", conf.ProfileCPU, err) - } - defer cpuFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, cpuFile) - cmd.Args = append(cmd.Args, "--profile-cpu-fd="+strconv.Itoa(nextFD)) - nextFD++ - } - - if conf.ProfileHeap != "" { - heapFile, err := os.OpenFile(conf.ProfileHeap, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("opening heap profiling file %q: %v", conf.ProfileHeap, err) - } - defer heapFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, heapFile) - cmd.Args = append(cmd.Args, "--profile-heap-fd="+strconv.Itoa(nextFD)) - nextFD++ - } - - if conf.ProfileMutex != "" { - mutexFile, err := os.OpenFile(conf.ProfileMutex, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("opening mutex profiling file %q: %v", conf.ProfileMutex, err) - } - defer mutexFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, mutexFile) - cmd.Args = append(cmd.Args, "--profile-mutex-fd="+strconv.Itoa(nextFD)) - nextFD++ - } - - if conf.TraceFile != "" { - traceFile, err := os.OpenFile(conf.TraceFile, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("opening trace file %q: %v", conf.TraceFile, err) - } - defer traceFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, traceFile) - cmd.Args = append(cmd.Args, "--trace-fd="+strconv.Itoa(nextFD)) - nextFD++ - } - - // If there is a gofer, sends all socket ends to the sandbox. - for _, f := range args.IOFiles { - defer f.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, f) - cmd.Args = append(cmd.Args, "--io-fds="+strconv.Itoa(nextFD)) - nextFD++ - } + donations.DonateAndClose("spec-fd", specFile) gPlatform, err := platform.Lookup(conf.Platform) if err != nil { return err } - if deviceFile, err := gPlatform.OpenDevice(conf.PlatformDevicePath); err != nil { return fmt.Errorf("opening device file for platform %q: %v", conf.Platform, err) } else if deviceFile != nil { - defer deviceFile.Close() - cmd.ExtraFiles = append(cmd.ExtraFiles, deviceFile) - cmd.Args = append(cmd.Args, "--device-fd="+strconv.Itoa(nextFD)) - nextFD++ + donations.DonateAndClose("device-fd", deviceFile) } // TODO(b/151157106): syscall tests fail by timeout if asyncpreemptoff @@ -611,66 +544,6 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1") } - // The current process' stdio must be passed to the application via the - // --stdio-fds flag. The stdio of the sandbox process itself must not - // be connected to the same FDs, otherwise we risk leaking sandbox - // errors to the application, so we set the sandbox stdio to nil, - // causing them to read/write from the null device. - cmd.Stdin = nil - cmd.Stdout = nil - cmd.Stderr = nil - var stdios [3]*os.File - - // If the console control socket file is provided, then create a new - // pty master/replica pair and set the TTY on the sandbox process. - if args.Spec.Process.Terminal && args.ConsoleSocket != "" { - // console.NewWithSocket will send the master on the given - // socket, and return the replica. - tty, err := console.NewWithSocket(args.ConsoleSocket) - if err != nil { - return fmt.Errorf("setting up console with socket %q: %v", args.ConsoleSocket, err) - } - defer tty.Close() - - // Set the TTY as a controlling TTY on the sandbox process. - cmd.SysProcAttr.Setctty = true - // The Ctty FD must be the FD in the child process's FD table, - // which will be nextFD in this case. - // See https://github.com/golang/go/issues/29458. - cmd.SysProcAttr.Ctty = nextFD - - // Pass the tty as all stdio fds to sandbox. - stdios[0] = tty - stdios[1] = tty - stdios[2] = tty - - if conf.Debug { - // If debugging, send the boot process stdio to the - // TTY, so that it is easier to find. - cmd.Stdin = tty - cmd.Stdout = tty - cmd.Stderr = tty - } - } else { - // If not using a console, pass our current stdio as the - // container stdio via flags. - stdios[0] = os.Stdin - stdios[1] = os.Stdout - stdios[2] = os.Stderr - - if conf.Debug { - // If debugging, send the boot process stdio to the - // this process' stdio, so that is is easier to find. - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - } - - // Detach from this session, otherwise cmd will get SIGHUP and SIGCONT - // when re-parented. - cmd.SysProcAttr.Setsid = true - // nss is the set of namespaces to join or create before starting the sandbox // process. Mount, IPC and UTS namespaces from the host are not used as they // are virtualized inside the sandbox. Be paranoid and run inside an empty @@ -706,7 +579,8 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn nss = append(nss, specs.LinuxNamespace{Type: specs.NetworkNamespace}) } - // These are set to the uid/gid that the sandbox process will use. + // These are set to the uid/gid that the sandbox process will use. May be + // overriden below. s.UID = os.Getuid() s.GID = os.Getgid() @@ -783,18 +657,71 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn } } + // The current process' stdio must be passed to the application via the + // --stdio-fds flag. The stdio of the sandbox process itself must not + // be connected to the same FDs, otherwise we risk leaking sandbox + // errors to the application, so we set the sandbox stdio to nil, + // causing them to read/write from the null device. + cmd.Stdin = nil + cmd.Stdout = nil + cmd.Stderr = nil + var stdios [3]*os.File + + // If the console control socket file is provided, then create a new + // pty master/replica pair and set the TTY on the sandbox process. + if args.Spec.Process.Terminal && args.ConsoleSocket != "" { + // console.NewWithSocket will send the master on the given + // socket, and return the replica. + tty, err := console.NewWithSocket(args.ConsoleSocket) + if err != nil { + return fmt.Errorf("setting up console with socket %q: %v", args.ConsoleSocket, err) + } + defer tty.Close() + + // Set the TTY as a controlling TTY on the sandbox process. + cmd.SysProcAttr.Setctty = true + + // Inconveniently, the Ctty must be the FD in the *child* process's FD + // table. So transfer all files we have so far and make sure the next file + // added to donations is stdin. + // + // See https://github.com/golang/go/issues/29458. + nextFD = donations.Transfer(cmd, nextFD) + cmd.SysProcAttr.Ctty = nextFD + + // Pass the tty as all stdio fds to sandbox. + stdios[0] = tty + stdios[1] = tty + stdios[2] = tty + + if conf.Debug { + // If debugging, send the boot process stdio to the + // TTY, so that it is easier to find. + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + } + } else { + // If not using a console, pass our current stdio as the + // container stdio via flags. + stdios[0] = os.Stdin + stdios[1] = os.Stdout + stdios[2] = os.Stderr + + if conf.Debug { + // If debugging, send the boot process stdio to the + // this process' stdio, so that is is easier to find. + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + } if err := s.configureStdios(conf, stdios[:]); err != nil { return fmt.Errorf("configuring stdios: %w", err) } - for _, file := range stdios { - cmd.ExtraFiles = append(cmd.ExtraFiles, file) - cmd.Args = append(cmd.Args, "--stdio-fds="+strconv.Itoa(nextFD)) - nextFD++ - } - - // Set Args[0] to make easier to spot the sandbox process. Otherwise it's - // shown as `exe`. - cmd.Args[0] = "runsc-sandbox" + // Note: this must be done right after "cmd.SysProcAttr.Ctty" is set above + // because it relies on stdin being the next FD donated. + donations.Donate("stdio-fds", stdios[:]...) mem, err := totalSystemMemory() if err != nil { @@ -838,20 +765,6 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn } cmd.Args = append(cmd.Args, "--total-memory", strconv.FormatUint(mem, 10)) - if args.UserLog != "" { - f, err := os.OpenFile(args.UserLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0664) - if err != nil { - return fmt.Errorf("opening compat log file: %v", err) - } - defer f.Close() - - cmd.ExtraFiles = append(cmd.ExtraFiles, f) - cmd.Args = append(cmd.Args, "--user-log-fd", strconv.Itoa(nextFD)) - nextFD++ - } - - _ = nextFD // All FD assignment is finished. - if args.Attached { // Kill sandbox if parent process exits in attached mode. cmd.SysProcAttr.Pdeathsig = unix.SIGKILL @@ -859,15 +772,14 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn cmd.Args = append(cmd.Args, "--attached") } - // Add container as the last argument. + // nextFD must not be used beyond this point. + _ = donations.Transfer(cmd, nextFD) + + // Add container ID as the last argument. cmd.Args = append(cmd.Args, s.ID) - // Log the FDs we are donating to the sandbox process. - for i, f := range cmd.ExtraFiles { - log.Debugf("Donating FD %d: %q", i+3, f.Name()) - } - - log.Debugf("Starting sandbox: %s %v", binPath, cmd.Args) + donation.LogDonations(cmd) + log.Debugf("Starting sandbox: %s %v", cmd.Path, cmd.Args) log.Debugf("SysProcAttr: %+v", cmd.SysProcAttr) if err := specutils.StartInNS(cmd, nss); err != nil { err := fmt.Errorf("starting sandbox: %v", err) @@ -1396,6 +1308,9 @@ func (s *Sandbox) configureStdios(conf *config.Config, stdios []*os.File) error return nil } + if s.UID < 0 || s.GID < 0 { + panic(fmt.Sprintf("sandbox UID/GID is not set: %d/%d", s.UID, s.GID)) + } for _, file := range stdios { log.Debugf("Changing %q ownership to %d/%d", file.Name(), s.UID, s.GID) if err := file.Chown(s.UID, s.GID); err != nil {