Mount cgroups per container in runsc.

Adds support for per container stats in runsc based on cgroups.
1. Removed the 'cgroupfs' config flag.
2. Mounts the cgroups (/sys/fs/cgroup/<controller>) which will be shared
across all containers during root/pause container startup.
3. The container cgroups (eg:/sys/fs/cgroup/controller/<container-id>) are
mounted along with other container mounts before starting the container
process if the cgroups mount is in the spec.

Updates #172

PiperOrigin-RevId: 590752853
This commit is contained in:
Nayana Bidari
2023-12-13 16:47:49 -08:00
committed by gVisor bot
parent d777746776
commit 29234bc44b
11 changed files with 367 additions and 41 deletions
+1 -4
View File
@@ -3,7 +3,4 @@ FROM ubuntu:22.04
ENV DEBIAN_FRONTEND="noninteractive"
RUN apt-get update && apt-get install -y docker.io
CMD bash -xec 'mount -t tmpfs cgroups /sys/fs/cgroup && \
mkdir /sys/fs/cgroup/devices && \
mount -t cgroup -o devices devices /sys/fs/cgroup/devices && \
exec /usr/bin/dockerd --bridge=none --iptables=false --ip6tables=false -D'
CMD exec /usr/bin/dockerd --bridge=none --iptables=false --ip6tables=false -D
+16
View File
@@ -26,6 +26,7 @@ import (
"gvisor.dev/gvisor/pkg/abi/linux"
"gvisor.dev/gvisor/pkg/fd"
"gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/sentry/fdimport"
"gvisor.dev/gvisor/pkg/sentry/fsimpl/host"
"gvisor.dev/gvisor/pkg/sentry/fsimpl/user"
@@ -270,6 +271,21 @@ func (proc *Proc) execAsync(args *ExecArgs) (*kernel.ThreadGroup, kernel.ThreadI
return nil, 0, nil, err
}
// Set cgroups to the new exec task if cgroups are mounted.
cgroupRegistry := proc.Kernel.CgroupRegistry()
initialCgrps := map[kernel.Cgroup]struct{}{}
for _, ctrl := range kernel.CgroupCtrls {
cg, err := cgroupRegistry.FindCgroup(ctx, ctrl, "/"+args.ContainerID)
if err != nil {
log.Warningf("cgroup mount for controller %v not found", ctrl)
continue
}
initialCgrps[cg] = struct{}{}
}
if len(initialCgrps) > 0 {
initArgs.InitialCgroups = initialCgrps
}
tg, tid, err := proc.Kernel.CreateProcess(initArgs)
if err != nil {
return nil, 0, nil, err
+3
View File
@@ -48,6 +48,9 @@ const (
CgroupControllerPIDs = CgroupControllerType("pids")
)
// CgroupCtrls is the list of cgroup controllers.
var CgroupCtrls = []CgroupControllerType{"cpu", "cpuacct", "cpuset", "devices", "job", "memory", "pids"}
// ParseCgroupController parses a string as a CgroupControllerType.
func ParseCgroupController(val string) (CgroupControllerType, error) {
switch val {
+11 -1
View File
@@ -168,6 +168,10 @@ type Loader struct {
// apply to the entire pod.
mountHints *PodMountHints
// cgroupMounts is a map of cgroup mounts that can be reused across
// containers. It is mapped by cgroup controller name.
cgroupMounts map[string]*cgroupMount
// productName is the value to show in
// /sys/devices/virtual/dmi/id/product_name.
productName string
@@ -991,9 +995,15 @@ func (l *Loader) createContainerProcess(info *containerInfo) (*kernel.ThreadGrou
}
l.startGoferMonitor(info)
if l.root.cid == l.sandboxID {
// Mounts cgroups for all the controllers.
if err := l.mountCgroupMounts(info.conf, info.procArgs.Credentials); err != nil {
return nil, nil, err
}
}
// We can share l.sharedMounts with containerMounter since l.mu is locked.
// Hence, mntr must only be used within this function (while l.mu is locked).
mntr := newContainerMounter(info, l.k, l.mountHints, l.sharedMounts, l.productName, l.sandboxID)
mntr := newContainerMounter(info, l.k, l.mountHints, l.sharedMounts, l.productName, l.sandboxID, l.cgroupMounts)
if err := setupContainerVFS(ctx, info, mntr, &info.procArgs); err != nil {
return nil, nil, err
}
+1 -1
View File
@@ -476,7 +476,7 @@ func TestCreateMountNamespace(t *testing.T) {
defer l.Destroy()
defer loaderCleanup()
mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, "", l.sandboxID)
mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, "", l.sandboxID, l.cgroupMounts)
ctx := l.k.SupervisorContext()
creds := auth.NewRootCredentials(l.root.procArgs.Credentials.UserNamespace)
mns, err := mntr.mountAll(ctx, creds, l.root.spec, l.root.conf, &l.root.procArgs)
+1 -1
View File
@@ -71,7 +71,7 @@ func (r *restorer) restore(l *Loader) error {
// Set up the restore environment.
ctx := l.k.SupervisorContext()
// TODO(b/298078576): Need to process hints here probably
mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, l.productName, r.container.cid)
mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, l.productName, l.sandboxID, l.cgroupMounts)
ctx, err = mntr.configureRestore(ctx)
if err != nil {
return fmt.Errorf("configuring filesystem restore: %v", err)
+152 -24
View File
@@ -189,6 +189,22 @@ func setupContainerVFS(ctx context.Context, info *containerInfo, mntr *container
}
procArgs.MountNamespace = mns
// If cgroups are mounted, then only check for the cgroup mounts per
// container. Otherwise the root cgroups will be enabled.
if mntr.cgroupsMounted {
cgroupRegistry := mntr.k.CgroupRegistry()
for _, ctrl := range kernel.CgroupCtrls {
cg, err := cgroupRegistry.FindCgroup(ctx, ctrl, "/"+mntr.containerID)
if err != nil {
return fmt.Errorf("cgroup mount for controller %v not found", ctrl)
}
if procArgs.InitialCgroups == nil {
procArgs.InitialCgroups = make(map[kernel.Cgroup]struct{}, len(kernel.CgroupCtrls))
}
procArgs.InitialCgroups[cg] = struct{}{}
}
}
mnsRoot := mns.Root(rootCtx)
defer mnsRoot.DecRef(rootCtx)
@@ -213,18 +229,20 @@ func setupContainerVFS(ctx context.Context, info *containerInfo, mntr *container
// mandatory mounts that are required by the OCI specification.
//
// This function must NOT add/remove any gofer mounts or change their order.
func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount {
func compileMounts(spec *specs.Spec, conf *config.Config, containerID string) []specs.Mount {
// Keep track of whether proc and sys were mounted.
var procMounted, sysMounted, devMounted, devptsMounted bool
var procMounted, sysMounted, devMounted, devptsMounted, cgroupsMounted bool
var mounts []specs.Mount
// Mount all submounts from the spec.
for _, m := range spec.Mounts {
// Unconditionally drop any cgroupfs mounts. If requested, we'll add our
// own below.
if m.Type == cgroupfs.Name {
// Mount all the cgroup controllers when "/sys/fs/cgroup" mount
// is present. If any other cgroup controller mounts are there,
// it will be a no-op, drop them.
if m.Type == cgroupfs.Name && cgroupsMounted {
continue
}
switch filepath.Clean(m.Destination) {
case "/proc":
procMounted = true
@@ -236,7 +254,10 @@ func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount {
case "/dev/pts":
m.Type = devpts.Name
devptsMounted = true
case "/sys/fs/cgroup":
cgroupsMounted = true
}
mounts = append(mounts, m)
}
@@ -244,23 +265,6 @@ func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount {
// says we SHOULD.
var mandatoryMounts []specs.Mount
if conf.Cgroupfs {
mandatoryMounts = append(mandatoryMounts, specs.Mount{
Type: tmpfs.Name,
Destination: "/sys/fs/cgroup",
})
mandatoryMounts = append(mandatoryMounts, specs.Mount{
Type: cgroupfs.Name,
Destination: "/sys/fs/cgroup/memory",
Options: []string{"memory"},
})
mandatoryMounts = append(mandatoryMounts, specs.Mount{
Type: cgroupfs.Name,
Destination: "/sys/fs/cgroup/cpu",
Options: []string{"cpu"},
})
}
if !procMounted {
mandatoryMounts = append(mandatoryMounts, specs.Mount{
Type: proc.Name,
@@ -399,12 +403,27 @@ type containerMounter struct {
// sandboxID is the ID for the whole sandbox.
sandboxID string
// cgroupMounts is a map of cgroup mounts that can be reused across
// containers. Key is the cgroup controller name string.
cgroupMounts map[string]*cgroupMount
// cgroupsMounted indicates if cgroups are mounted in the container.
// This is used to set the InitialCgroups before starting the container
// process.
cgroupsMounted bool
}
func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountHints, sharedMounts map[string]*vfs.Mount, productName string, sandboxID string) *containerMounter {
type cgroupMount struct {
fs *vfs.Filesystem
root *vfs.Dentry
mount *vfs.Mount
}
func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountHints, sharedMounts map[string]*vfs.Mount, productName string, sandboxID string, cgroupMounts map[string]*cgroupMount) *containerMounter {
return &containerMounter{
root: info.spec.Root,
mounts: compileMounts(info.spec, info.conf),
mounts: compileMounts(info.spec, info.conf, info.procArgs.ContainerID),
goferFDs: fdDispenser{fds: info.goferFDs},
goferFilestoreFDs: fdDispenser{fds: info.goferFilestoreFDs},
devGoferFD: info.devGoferFD,
@@ -415,6 +434,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountH
productName: productName,
containerID: info.procArgs.ContainerID,
sandboxID: sandboxID,
cgroupMounts: cgroupMounts,
}
}
@@ -723,6 +743,11 @@ func (c *containerMounter) mountSubmounts(ctx context.Context, spec *specs.Spec,
if err != nil {
return fmt.Errorf("mount shared mount %q to %q: %v", submount.hint.Name, submount.mount.Destination, err)
}
} else if submount.mount.Type == cgroupfs.Name {
// Mount all the cgroups controllers.
if err := c.mountCgroupSubmounts(ctx, spec, conf, mns, creds, submount); err != nil {
return fmt.Errorf("mount cgroup %q: %w", submount.mount.Destination, err)
}
} else {
mnt, err = c.mountSubmount(ctx, spec, conf, mns, creds, submount)
if err != nil {
@@ -1070,6 +1095,109 @@ func (c *containerMounter) getSharedMount(ctx context.Context, spec *specs.Spec,
return sharedMount, nil
}
// mountCgroupMounts mounts the cgroups which are shared across all containers.
// Postcondition: Initialized l.cgroupMounts on success.
func (l *Loader) mountCgroupMounts(conf *config.Config, creds *auth.Credentials) error {
ctx := l.k.SupervisorContext()
cgroupMounts := make(map[string]*cgroupMount)
for _, sopts := range kernel.CgroupCtrls {
mopts := &vfs.MountOptions{
GetFilesystemOptions: vfs.GetFilesystemOptions{
Data: string(sopts),
InternalMount: true,
},
}
fs, root, err := l.k.VFS().NewFilesystem(ctx, creds, "cgroup", cgroupfs.Name, mopts)
if err != nil {
return err
}
mount := l.k.VFS().NewDisconnectedMount(fs, root, mopts)
// Private so that mounts created by containers do not appear
// in other container's cgroup paths.
l.k.VFS().SetMountPropagation(mount, linux.MS_PRIVATE, false)
cgroupMounts[string(sopts)] = &cgroupMount{
fs: fs,
root: root,
mount: mount,
}
}
l.cgroupMounts = cgroupMounts
log.Infof("created cgroup mounts for controllers %v", kernel.CgroupCtrls)
return nil
}
// mountCgroupSubmounts mounts all the cgroup controller submounts for the
// container. The cgroup submounts are created under the root controller mount
// with containerID as the directory name and then bind mounts this directory
// inside the container's mount namespace.
func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs.Spec, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *mountInfo) error {
root := mns.Root(ctx)
defer root.DecRef(ctx)
// Mount "/sys/fs/cgroup" in the container's mount namespace.
submount.mount.Type = tmpfs.Name
mnt, err := c.mountSubmount(ctx, spec, conf, mns, creds, submount)
if err != nil {
return err
}
if mnt != nil && mnt.ReadOnly() {
// Switch to ReadWrite while we setup submounts.
if err := c.k.VFS().SetMountReadOnly(mnt, false); err != nil {
return fmt.Errorf("failed to set mount at %q readwrite: %w", submount.mount.Destination, err)
}
// Restore back to ReadOnly at the end.
defer func() {
if err := c.k.VFS().SetMountReadOnly(mnt, true); err != nil {
panic(fmt.Sprintf("failed to restore mount at %q back to readonly: %v", submount.mount.Destination, err))
}
}()
}
// Mount all the cgroup controllers in the container's mount namespace.
mountCtx := vfs.WithRoot(vfs.WithMountNamespace(ctx, mns), root)
for _, ctrl := range kernel.CgroupCtrls {
ctrlName := string(ctrl)
cgroupMnt, ok := c.cgroupMounts[ctrlName]
if !ok {
return fmt.Errorf("cgroup mount for controller %s not found", ctrlName)
}
cgroupMntVD := vfs.MakeVirtualDentry(cgroupMnt.mount, cgroupMnt.root)
sourcePop := vfs.PathOperation{
Root: cgroupMntVD,
Start: cgroupMntVD,
// Use the containerID as the cgroup path.
Path: fspath.Parse(c.containerID),
}
if err := c.k.VFS().MkdirAt(mountCtx, creds, &sourcePop, &vfs.MkdirOptions{
Mode: 0755,
}); err != nil {
log.Infof("error in creating directory %v", err)
return err
}
// Bind mount the new cgroup directory into the container's mount namespace.
destination := "/sys/fs/cgroup/" + ctrlName
if err := c.k.VFS().MakeSyntheticMountpoint(mountCtx, destination, root, creds); err != nil {
// Log a warning, but attempt the mount anyway.
log.Warningf("Failed to create mount point %q: %v", destination, err)
}
target := &vfs.PathOperation{
Root: root,
Start: root,
Path: fspath.Parse(destination),
}
if err := c.k.VFS().BindAt(mountCtx, creds, &sourcePop, target, false); err != nil {
log.Infof("error in bind mounting %v", err)
return err
}
}
c.cgroupsMounted = true
return nil
}
// mountSharedMaster mounts the master of a volume that is shared among
// containers in a pod.
func (c *containerMounter) mountSharedMaster(ctx context.Context, spec *specs.Spec, conf *config.Config, mntInfo *mountInfo, creds *auth.Credentials) (*vfs.Mount, error) {
-3
View File
@@ -259,9 +259,6 @@ type Config struct {
// Enables seccomp inside the sandbox.
OCISeccomp bool `flag:"oci-seccomp"`
// Mounts the cgroup filesystem backed by the sentry's cgroupfs.
Cgroupfs bool `flag:"cgroupfs"`
// Don't configure cgroups.
IgnoreCgroups bool `flag:"ignore-cgroups"`
+167
View File
@@ -2666,3 +2666,170 @@ func TestMultiContainerMemoryLeakStress(t *testing.T) {
time.Sleep(time.Second)
}
}
// Tests cgroups are mounted in only containers which have a cgroup mount in
// the spec.
func TestMultiContainerCgroups(t *testing.T) {
_, err := testutil.FindFile("test/cmd/test_app/test_app")
if err != nil {
t.Fatal("error finding test_app:", err)
}
for name, conf := range configs(t, false /* noOverlay */) {
t.Run(name, func(t *testing.T) {
rootDir, cleanup, err := testutil.SetupRootDir()
if err != nil {
t.Fatalf("error creating root dir: %v", err)
}
defer cleanup()
conf.RootDir = rootDir
podSpecs, ids := createSpecs(
[]string{"sleep", "100"},
[]string{"sleep", "100"})
podSpecs[1].Linux = &specs.Linux{
Namespaces: []specs.LinuxNamespace{{Type: "pid"}},
}
mnt0 := specs.Mount{
Destination: "/sys/fs/cgroup",
Type: "cgroup",
Options: nil,
}
// Append cgroups mount for only one container.
podSpecs[0].Mounts = append(podSpecs[0].Mounts, mnt0)
createSharedMount(mnt0, "test-mount", podSpecs...)
containers, cleanup, err := startContainers(conf, podSpecs, ids)
if err != nil {
t.Fatalf("error starting containers: %v", err)
}
defer cleanup()
ctrlFileMap := map[string]string{
"cpu": "cpu.shares",
"cpuacct": "cpuacct.usage",
"cpuset": "cpuset.cpus",
"devices": "devices.allow",
"memory": "memory.usage_in_bytes",
"pids": "pids.current",
}
for ctrl, f := range ctrlFileMap {
ctrlRoot := control.CgroupControlFile{
Controller: ctrl,
Path: "/",
Name: f,
}
ctrl0 := control.CgroupControlFile{
Controller: ctrl,
Path: "/" + containers[0].ID,
Name: f,
}
ctrl1 := control.CgroupControlFile{
Controller: ctrl,
Path: "/" + containers[1].ID,
Name: f,
}
if _, err := containers[0].Sandbox.CgroupsReadControlFile(ctrlRoot); err != nil {
t.Fatalf("error root cgroup mount for %s not found %v", ctrl, err)
}
if _, err := containers[0].Sandbox.CgroupsReadControlFile(ctrl0); err != nil {
t.Fatalf("error %s cgroups not mounted in container0 %v", ctrl, err)
}
if _, err := containers[1].Sandbox.CgroupsReadControlFile(ctrl1); err == nil {
t.Fatalf("error %s cgroups mounted in container1 even when the spec does not have a cgroup mount %v", ctrl, err)
}
}
})
}
}
// Tests the cgroups are mounted in the containers when the spec has a cgroup
// mount. Also, checks memory usage stats from cgroups work correctly when the
// memory is increased for one container.
func TestMultiContainerCgroupsMemoryUsage(t *testing.T) {
_, err := testutil.FindFile("test/cmd/test_app/test_app")
if err != nil {
t.Fatal("error finding test_app:", err)
}
for name, conf := range configs(t, false /* noOverlay */) {
t.Run(name, func(t *testing.T) {
rootDir, cleanup, err := testutil.SetupRootDir()
if err != nil {
t.Fatalf("error creating root dir: %v", err)
}
defer cleanup()
conf.RootDir = rootDir
podSpecs, ids := createSpecs(
[]string{"sleep", "100"},
[]string{"sleep", "10"})
podSpecs[1].Linux = &specs.Linux{
Namespaces: []specs.LinuxNamespace{{Type: "pid"}},
}
mnt0 := specs.Mount{
Destination: "/sys/fs/cgroup",
Type: "cgroup",
Options: nil,
}
// Append cgroups mount for both containers.
podSpecs[0].Mounts = append(podSpecs[0].Mounts, mnt0)
podSpecs[1].Mounts = append(podSpecs[1].Mounts, mnt0)
createSharedMount(mnt0, "test-mount", podSpecs...)
containers, cleanup, err := startContainers(conf, podSpecs, ids)
if err != nil {
t.Fatalf("error starting containers: %v", err)
}
defer cleanup()
ctrlRoot := control.CgroupControlFile{
Controller: "memory",
Path: "/",
Name: "memory.usage_in_bytes",
}
ctrl0 := control.CgroupControlFile{
Controller: "memory",
Path: "/" + containers[0].ID,
Name: "memory.usage_in_bytes",
}
ctrl1 := control.CgroupControlFile{
Controller: "memory",
Path: "/" + containers[1].ID,
Name: "memory.usage_in_bytes",
}
usageTotal, err := containers[0].Sandbox.CgroupsReadControlFile(ctrlRoot)
if err != nil {
t.Fatalf("error getting total usage %v", err)
}
usage0, err := containers[0].Sandbox.CgroupsReadControlFile(ctrl0)
if err != nil {
t.Fatalf("error getting container0 usage %v", err)
}
usage1, err := containers[1].Sandbox.CgroupsReadControlFile(ctrl1)
if err != nil {
t.Fatalf("error getting container1 usage %v", err)
}
if usageTotal < (usage0 + usage1) {
t.Fatalf("error total usage is less total %v container0_usage %v container1_usage %v", usageTotal, usage0, usage1)
}
// Wait for the second container to exit and check that the new usage must
// be less than the old usage.
time.Sleep(12 * time.Second)
newUsageTotal, err := containers[0].Sandbox.CgroupsReadControlFile(ctrlRoot)
if err != nil {
t.Fatalf("error getting total usage %v", err)
}
if newUsageTotal >= usageTotal {
t.Fatalf("error new total usage %v is not less than old total usage %v", newUsageTotal, usageTotal)
}
})
}
}
+7 -3
View File
@@ -307,7 +307,6 @@ func TestProcfsDump(t *testing.T) {
spec.Process.Rlimits = []specs.POSIXRlimit{
{Type: "RLIMIT_NOFILE", Hard: fdLimit.Max, Soft: fdLimit.Cur},
}
conf.Cgroupfs = true
_, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf)
if err != nil {
t.Fatalf("error setting up container: %v", err)
@@ -414,11 +413,16 @@ func TestProcfsDump(t *testing.T) {
}
wantCgroup := []kernel.TaskCgroupEntry{
kernel.TaskCgroupEntry{HierarchyID: 2, Controllers: "memory", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 7, Controllers: "pids", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 6, Controllers: "memory", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 5, Controllers: "job", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 4, Controllers: "devices", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 3, Controllers: "cpuset", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 2, Controllers: "cpuacct", Path: "/"},
kernel.TaskCgroupEntry{HierarchyID: 1, Controllers: "cpu", Path: "/"},
}
if len(procfsDump[0].Cgroup) != len(wantCgroup) {
t.Errorf("expected 2 cgroup controllers, got %+v", procfsDump[0].Cgroup)
t.Errorf("expected 7 cgroup controllers, got %+v", procfsDump[0].Cgroup)
} else {
for i, cgroup := range procfsDump[0].Cgroup {
if cgroup != wantCgroup[i] {
+8 -4
View File
@@ -64,10 +64,14 @@ syscall_test(
test = "//test/syscalls/linux:brk_test",
)
syscall_test(
one_sandbox = False,
test = "//test/syscalls/linux:cgroup_test",
)
# TODO(b/315355651): Fix cgroup tests in runsc. Cgroups are mounted in the
# root/pause container by default and for other containers cgroups are bind
# mounted if the container spec has a cgroup mount. These cgroup tests
# explicitly mount the cgroups which will fail now in runsc.
# syscall_test(
# one_sandbox = False,
# test = "//test/syscalls/linux:cgroup_test",
# )
syscall_test(
add_fusefs = True,