From 595d424651b4baf70aeaf1fe2565b6c68969ae24 Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Mon, 15 May 2023 19:07:02 -0700 Subject: [PATCH] Add rootless support for directfs and hostinet. Note that this "rootless" is different from Docker/Podman rootless containers. This "rootless" feature refers to runsc caller being a non-root user. In Docker/Podman rootless containers, the runsc caller is root user in a new userns (although that root user does not map to root user on host). With this change, invoking runsc as a non-root user works! This is in preparation to make directfs the default in runsc. PiperOrigin-RevId: 532295213 --- runsc/cmd/boot.go | 25 +++++++-- runsc/cmd/gofer.go | 44 +++++++++------- runsc/container/container.go | 68 ++++--------------------- runsc/sandbox/sandbox.go | 98 ++++++++++++++++++++++++++++++++++-- runsc/specutils/namespace.go | 15 ------ 5 files changed, 148 insertions(+), 102 deletions(-) diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 46eec33db..7e0493878 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -144,6 +144,11 @@ type Boot struct { // procMountSyncFD is a file descriptor that has to be closed when the // procfs mount isn't needed anymore. procMountSyncFD int + + // syncUsernsFD is the file descriptor that has to be closed when the + // boot process should invoke setuid/setgid for root user. This is mainly + // used to synchronize rootless user namespace initialization. + syncUsernsFD int } // Name implements subcommands.Command.Name. @@ -169,6 +174,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { f.BoolVar(&b.pidns, "pidns", false, "if true, the sandbox is in its own PID namespace") f.IntVar(&b.cpuNum, "cpu-num", 0, "number of CPUs to create inside the sandbox") f.IntVar(&b.procMountSyncFD, "proc-mount-sync-fd", -1, "file descriptor that has to be written to when /proc isn't needed anymore and can be unmounted") + f.IntVar(&b.syncUsernsFD, "sync-userns-fd", -1, "file descriptor used to synchronize rootless user namespace initialization.") f.Uint64Var(&b.totalMem, "total-memory", 0, "sets the initial amount of total memory to report back to the container") f.BoolVar(&b.attached, "attached", false, "if attached is true, kills the sandbox process when the parent process terminates") f.StringVar(&b.productName, "product-name", "", "value to show in /sys/devices/virtual/dmi/id/product_name") @@ -231,6 +237,10 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } + if b.syncUsernsFD >= 0 { + syncUsernsForRootless(b.syncUsernsFD) + } + if b.setUpRoot { if err := setUpChroot(b.pidns); err != nil { util.Fatalf("error setting up chroot: %v", err) @@ -255,8 +265,8 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } if !b.applyCaps { - // Remove --setup-root arg to call myself. It has already been done. - args := b.prepareArgs("setup-root") + // Remove the args that have already been done before calling self. + args := b.prepareArgs("setup-root", "sync-userns-fd") // Note that we've already read the spec from the spec FD, and // we will read it again after the exec call. This works @@ -300,9 +310,8 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma caps = specutils.MergeCapabilities(caps, directfsSandboxLinuxCaps) } - // Remove --apply-caps and --setup-root arg to call myself. Both have - // already been done. - args := b.prepareArgs("setup-root", "apply-caps") + // Remove the args that have already been done before calling self. + args := b.prepareArgs("setup-root", "sync-userns-fd", "apply-caps") // Note that we've already read the spec from the spec FD, and // we will read it again after the exec call. This works @@ -317,6 +326,12 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma panic("unreachable") } + if b.syncUsernsFD >= 0 { + // syncUsernsFD is set, but runsc hasn't been re-exeuted with a new UID and GID. + // We expect that setCapsAndCallSelf has to be called in this case. + panic("unreachable") + } + // Close specFile to avoid exposing it to the sandbox. if err := specFile.Close(); err != nil { util.Fatalf("closing specFile: %v", err) diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index 5e024948c..de91dbf21 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -130,23 +130,7 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm } if g.syncUsernsFD >= 0 { - f := os.NewFile(uintptr(g.syncUsernsFD), "sync FD") - defer f.Close() - var b [1]byte - if n, err := f.Read(b[:]); n != 0 || err != io.EOF { - util.Fatalf("failed to sync: %v: %v", n, err) - } - - f.Close() - // SETUID changes UID on the current system thread, so we have - // to re-execute current binary. - runtime.LockOSThread() - if _, _, errno := unix.RawSyscall(unix.SYS_SETUID, 0, 0, 0); errno != 0 { - util.Fatalf("failed to set UID: %v", errno) - } - if _, _, errno := unix.RawSyscall(unix.SYS_SETGID, 0, 0, 0); errno != 0 { - util.Fatalf("failed to set GID: %v", errno) - } + syncUsernsForRootless(g.syncUsernsFD) } if g.setUpRoot { @@ -182,7 +166,7 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm if g.syncUsernsFD >= 0 { // syncUsernsFD is set, but runsc hasn't been re-exeuted with a new UID and GID. - // We expcec that setCapsAndCallSelfsetCapsAndCallSelf has to be called in this case. + // We expect that setCapsAndCallSelf has to be called in this case. panic("unreachable") } @@ -598,3 +582,27 @@ func adjustMountOptions(conf *config.Config, path string, opts []string) ([]stri } return rv, nil } + +// syncUsernsForRootless waits on syncUsernsFD to be closed and then sets +// UID/GID to 0. Note that this function calls runtime.LockOSThread(). +// +// Postcondition: All callers must re-exec themselves after this returns. +func syncUsernsForRootless(syncUsernsFD int) { + f := os.NewFile(uintptr(syncUsernsFD), "sync FD") + defer f.Close() + var b [1]byte + if n, err := f.Read(b[:]); n != 0 || err != io.EOF { + util.Fatalf("failed to sync: %v: %v", n, err) + } + + f.Close() + // SETUID changes UID on the current system thread, so we have + // to re-execute current binary. + runtime.LockOSThread() + if _, _, errno := unix.RawSyscall(unix.SYS_SETUID, 0, 0, 0); errno != 0 { + util.Fatalf("failed to set UID: %v", errno) + } + if _, _, errno := unix.RawSyscall(unix.SYS_SETGID, 0, 0, 0); errno != 0 { + util.Fatalf("failed to set GID: %v", errno) + } +} diff --git a/runsc/container/container.go b/runsc/container/container.go index e49e5d68a..ea6d24a80 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -1177,43 +1177,23 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu // namespace so the gofer's view of the filesystem aligns with the // users in the sandbox. if !rootlessEUID { - userNS := specutils.FilterNS([]specs.LinuxNamespaceType{specs.UserNamespace}, spec) - nss = append(nss, userNS...) - specutils.SetUIDGIDMappings(cmd, spec) - if len(userNS) != 0 { + if userNS, ok := specutils.GetNS(specs.UserNamespace, spec); ok { + nss = append(nss, userNS) + specutils.SetUIDGIDMappings(cmd, spec) // We need to set UID and GID to have capabilities in a new user namespace. cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 0, Gid: 0} } } else { - userNS := specutils.FilterNS([]specs.LinuxNamespaceType{specs.UserNamespace}, spec) - if len(userNS) == 0 { + userNS, ok := specutils.GetNS(specs.UserNamespace, spec) + if !ok { return nil, nil, fmt.Errorf("unable to run a rootless container without userns") } - fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + nss = append(nss, userNS) + syncFile, err := sandbox.ConfigureCmdForRootless(cmd, &donations) if err != nil { return nil, nil, err } - syncFile := os.NewFile(uintptr(fds[0]), "sync FD") defer syncFile.Close() - - f := os.NewFile(uintptr(fds[1]), "sync other FD") - donations.DonateAndClose("sync-userns-fd", f) - if cmd.SysProcAttr == nil { - cmd.SysProcAttr = &unix.SysProcAttr{} - } - cmd.SysProcAttr.AmbientCaps = []uintptr{ - unix.CAP_CHOWN, - unix.CAP_DAC_OVERRIDE, - unix.CAP_DAC_READ_SEARCH, - unix.CAP_FOWNER, - unix.CAP_FSETID, - unix.CAP_SYS_CHROOT, - unix.CAP_SETUID, - unix.CAP_SETGID, - unix.CAP_SYS_ADMIN, - unix.CAP_SETPCAP, - } - nss = append(nss, specs.LinuxNamespace{Type: specs.UserNamespace}) } donations.Transfer(cmd, nextFD) @@ -1226,38 +1206,8 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *config.Config, bu } if rootlessEUID { - log.Debugf("Setting user mappings") - args := []string{strconv.Itoa(cmd.Process.Pid)} - for _, idMap := range spec.Linux.UIDMappings { - log.Infof("Mapping host uid %d to container uid %d (size=%d)", - idMap.HostID, idMap.ContainerID, idMap.Size) - args = append(args, - strconv.Itoa(int(idMap.ContainerID)), - strconv.Itoa(int(idMap.HostID)), - strconv.Itoa(int(idMap.Size)), - ) - } - - out, err := exec.Command("newuidmap", args...).CombinedOutput() - log.Debugf("newuidmap: %#v\n%s", args, out) - if err != nil { - return nil, nil, fmt.Errorf("newuidmap failed: %w", err) - } - - args = []string{strconv.Itoa(cmd.Process.Pid)} - for _, idMap := range spec.Linux.GIDMappings { - log.Infof("Mapping host uid %d to container uid %d (size=%d)", - idMap.HostID, idMap.ContainerID, idMap.Size) - args = append(args, - strconv.Itoa(int(idMap.ContainerID)), - strconv.Itoa(int(idMap.HostID)), - strconv.Itoa(int(idMap.Size)), - ) - } - out, err = exec.Command("newgidmap", args...).CombinedOutput() - log.Debugf("newgidmap: %#v\n%s", args, out) - if err != nil { - return nil, nil, fmt.Errorf("newgidmap failed: %w", err) + if err := sandbox.SetUserMappings(spec, cmd.Process.Pid); err != nil { + return nil, nil, err } } diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 91f530005..59b049461 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -816,14 +816,28 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn // filesystem is required. These features require to run inside the user // namespace specified in the spec or the current namespace if none is // configured. + rootlessEUID := unix.Getuid() != 0 + setUserMappings := false if conf.Network == config.NetworkHost || conf.DirectFS { if userns, ok := specutils.GetNS(specs.UserNamespace, args.Spec); ok { log.Infof("Sandbox will be started in container's user namespace: %+v", userns) nss = append(nss, userns) - specutils.SetUIDGIDMappings(cmd, args.Spec) - // We need to set UID and GID to have capabilities in a new user namespace. - cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 0, Gid: 0} + if rootlessEUID { + syncFile, err := ConfigureCmdForRootless(cmd, &donations) + if err != nil { + return err + } + defer syncFile.Close() + setUserMappings = true + } else { + specutils.SetUIDGIDMappings(cmd, args.Spec) + // We need to set UID and GID to have capabilities in a new user namespace. + cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 0, Gid: 0} + } } else { + if rootlessEUID { + return fmt.Errorf("unable to run a rootless container without userns") + } log.Infof("Sandbox will be started in the current user namespace") } // When running in the caller's defined user namespace, apply the same @@ -835,14 +849,13 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn // bind-mount the executable inside it. if conf.TestOnlyAllowRunAsCurrentUserWithoutChroot { log.Warningf("Running sandbox in test mode without chroot. This is only safe in tests!") - } else if specutils.HasCapabilities(capability.CAP_SYS_ADMIN) { + } else if specutils.HasCapabilities(capability.CAP_SYS_ADMIN) || rootlessEUID { log.Infof("Sandbox will be started in minimal chroot") cmd.Args = append(cmd.Args, "--setup-root") } else { return fmt.Errorf("can't run sandbox process in minimal chroot since we don't have CAP_SYS_ADMIN") } } else { - rootlessEUID := unix.Getuid() != 0 // If we have CAP_SETUID and CAP_SETGID, then we can also run // as user nobody. if conf.TestOnlyAllowRunAsCurrentUserWithoutChroot { @@ -1041,6 +1054,11 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn if err != nil { return err } + if setUserMappings { + if err := SetUserMappings(args.Spec, cmd.Process.Pid); err != nil { + return err + } + } s.child = true s.Pid.store(cmd.Process.Pid) @@ -1550,3 +1568,73 @@ func (s *Sandbox) fixPidns(spec *specs.Spec) { } panic("unreachable") } + +// ConfigureCmdForRootless configures cmd to donate a socket FD that can be +// used to synchronize userns configuration. +func ConfigureCmdForRootless(cmd *exec.Cmd, donations *donation.Agency) (*os.File, error) { + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, err + } + f := os.NewFile(uintptr(fds[1]), "sync other FD") + donations.DonateAndClose("sync-userns-fd", f) + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &unix.SysProcAttr{} + } + cmd.SysProcAttr.AmbientCaps = []uintptr{ + // Same as `cap` in cmd/gofer.go. + unix.CAP_CHOWN, + unix.CAP_DAC_OVERRIDE, + unix.CAP_DAC_READ_SEARCH, + unix.CAP_FOWNER, + unix.CAP_FSETID, + unix.CAP_SYS_CHROOT, + // Needed for setuid(2)/setgid(2). + unix.CAP_SETUID, + unix.CAP_SETGID, + // Needed for chroot. + unix.CAP_SYS_ADMIN, + // Needed to be able to clear bounding set (PR_CAPBSET_DROP). + unix.CAP_SETPCAP, + } + return os.NewFile(uintptr(fds[0]), "sync FD"), nil +} + +// SetUserMappings uses newuidmap/newgidmap programs to set up user ID mappings +// for process pid. +func SetUserMappings(spec *specs.Spec, pid int) error { + log.Debugf("Setting user mappings") + args := []string{strconv.Itoa(pid)} + for _, idMap := range spec.Linux.UIDMappings { + log.Infof("Mapping host uid %d to container uid %d (size=%d)", + idMap.HostID, idMap.ContainerID, idMap.Size) + args = append(args, + strconv.Itoa(int(idMap.ContainerID)), + strconv.Itoa(int(idMap.HostID)), + strconv.Itoa(int(idMap.Size)), + ) + } + + out, err := exec.Command("newuidmap", args...).CombinedOutput() + log.Debugf("newuidmap: %#v\n%s", args, out) + if err != nil { + return fmt.Errorf("newuidmap failed: %w", err) + } + + args = []string{strconv.Itoa(pid)} + for _, idMap := range spec.Linux.GIDMappings { + log.Infof("Mapping host uid %d to container uid %d (size=%d)", + idMap.HostID, idMap.ContainerID, idMap.Size) + args = append(args, + strconv.Itoa(int(idMap.ContainerID)), + strconv.Itoa(int(idMap.HostID)), + strconv.Itoa(int(idMap.Size)), + ) + } + out, err = exec.Command("newgidmap", args...).CombinedOutput() + log.Debugf("newgidmap: %#v\n%s", args, out) + if err != nil { + return fmt.Errorf("newgidmap failed: %w", err) + } + return nil +} diff --git a/runsc/specutils/namespace.go b/runsc/specutils/namespace.go index b17505399..3398eec29 100644 --- a/runsc/specutils/namespace.go +++ b/runsc/specutils/namespace.go @@ -91,21 +91,6 @@ func GetNS(nst specs.LinuxNamespaceType, s *specs.Spec) (specs.LinuxNamespace, b return specs.LinuxNamespace{}, false } -// FilterNS returns a slice of namespaces from the spec with types that match -// those in the `filter` slice. -func FilterNS(filter []specs.LinuxNamespaceType, s *specs.Spec) []specs.LinuxNamespace { - if s.Linux == nil { - return nil - } - var out []specs.LinuxNamespace - for _, nst := range filter { - if ns, ok := GetNS(nst, s); ok { - out = append(out, ns) - } - } - return out -} - // setNS sets the namespace of the given type. It must be called with // OSThreadLocked. func setNS(fd, nsType uintptr) error {