cgroupfs: Implement hierarchical accounting for cpuacct controller.

PiperOrigin-RevId: 438193226

pkg/sentry/fsimpl/cgroupfs/base.go

@@ -40,6 +40,11 @@ import (
 type controllerCommon struct {
 	ty kernel.CgroupControllerType
 	fs *filesystem
+	// parent is the parent controller if any. Immutable.
+	//
+	// Note that we don't have to update this on renames, since cgroup
+	// directories can't be moved to a different parent directory.
+	parent controller
 }
 
 func (c *controllerCommon) init(ty kernel.CgroupControllerType, fs *filesystem) {
@@ -47,9 +52,15 @@ func (c *controllerCommon) init(ty kernel.CgroupControllerType, fs *filesystem)
 	c.fs = fs
 }
 
-func (c *controllerCommon) cloneFrom(other *controllerCommon) {
-	c.ty = other.ty
-	c.fs = other.fs
+func (c *controllerCommon) cloneFromParent(parent controller) {
+	c.ty = parent.Type()
+	c.fs = parent.Filesystem()
+	c.parent = parent
+}
+
+// Filesystem implements controller.Filesystem.
+func (c *controllerCommon) Filesystem() *filesystem {
+	return c.fs
 }
 
 // Type implements kernel.CgroupController.Type.
@@ -85,7 +96,10 @@ func (c *controllerCommon) RootCgroup() kernel.Cgroup {
 type controller interface {
 	kernel.CgroupController
 
-	// Clone creates a new controller based on the internal state of the current
+	// Filesystem returns the cgroupfs filesystem backing this controller.
+	Filesystem() *filesystem
+
+	// Clone creates a new controller based on the internal state of this
 	// controller. This is used to initialize a sub-cgroup based on the state of
 	// the parent.
 	Clone() controller
@@ -94,6 +108,18 @@ type controller interface {
 	// control files defined by this controller.
 	AddControlFiles(ctx context.Context, creds *auth.Credentials, c *cgroupInode, contents map[string]kernfs.Inode)
 
+	// Enter is called when a task initially moves into a cgroup. This is
+	// distinct from migration because the task isn't migrating away from a
+	// cgroup. Enter is called when a task is created and joins its initial
+	// cgroup, or when cgroupfs is mounted and existing tasks are moved into
+	// cgroups.
+	Enter(t *kernel.Task)
+
+	// Leave is called when a task leaves a cgroup. This is distinct from
+	// migration because the task isn't migrating to another cgroup. Leave is
+	// called when a task exits.
+	Leave(t *kernel.Task)
+
 	// PrepareMigrate signals the controller that a migration is about to
 	// happen. The controller should check for any conditions that would prevent
 	// the migration. If PrepareMigrate succeeds, the controller must
@@ -186,6 +212,7 @@ func (c *cgroupInode) Controllers() []kernel.CgroupController {
 func (c *cgroupInode) tasks() []*kernel.Task {
 	c.fs.tasksMu.RLock()
 	defer c.fs.tasksMu.RUnlock()
+
 	ts := make([]*kernel.Task, 0, len(c.ts))
 	for t := range c.ts {
 		ts = append(ts, t)
@@ -196,15 +223,23 @@ func (c *cgroupInode) tasks() []*kernel.Task {
 // Enter implements kernel.CgroupImpl.Enter.
 func (c *cgroupInode) Enter(t *kernel.Task) {
 	c.fs.tasksMu.Lock()
+	defer c.fs.tasksMu.Unlock()
+
 	c.ts[t] = struct{}{}
-	c.fs.tasksMu.Unlock()
+	for _, ctl := range c.controllers {
+		ctl.Enter(t)
+	}
 }
 
 // Leave implements kernel.CgroupImpl.Leave.
 func (c *cgroupInode) Leave(t *kernel.Task) {
 	c.fs.tasksMu.Lock()
+	defer c.fs.tasksMu.Unlock()
+
+	for _, ctl := range c.controllers {
+		ctl.Leave(t)
+	}
 	delete(c.ts, t)
-	c.fs.tasksMu.Unlock()
 }
 
 // PrepareMigrate implements kernel.CgroupImpl.PrepareMigrate.
@@ -229,14 +264,14 @@ func (c *cgroupInode) PrepareMigrate(t *kernel.Task, src *kernel.Cgroup) error {
 
 // CommitMigrate implements kernel.CgroupImpl.CommitMigrate.
 func (c *cgroupInode) CommitMigrate(t *kernel.Task, src *kernel.Cgroup) {
+	c.fs.tasksMu.Lock()
+	defer c.fs.tasksMu.Unlock()
+
 	for srcType, srcCtl := range src.CgroupImpl.(*cgroupInode).controllers {
 		c.controllers[srcType].CommitMigrate(t, srcCtl)
 	}
 
 	srcI := src.CgroupImpl.(*cgroupInode)
-	c.fs.tasksMu.Lock()
-	defer c.fs.tasksMu.Unlock()
-
 	delete(srcI.ts, t)
 	c.ts[t] = struct{}{}
 }
@@ -375,17 +410,23 @@ func parseInt64FromString(ctx context.Context, src usermem.IOSequence) (val, len
 	return val, int64(n), nil
 }
 
-// controllerNoopMigrate partially implements controller. It stubs the migration
+// controllerStateless partially implements controller. It stubs the migration
 // methods with noops for a stateless controller.
-type controllerNoopMigrate struct{}
+type controllerStateless struct{}
+
+// Enter implements controller.Enter.
+func (*controllerStateless) Enter(t *kernel.Task) {}
+
+// Leave implements controller.Leave.
+func (*controllerStateless) Leave(t *kernel.Task) {}
 
 // PrepareMigrate implements controller.PrepareMigrate.
-func (*controllerNoopMigrate) PrepareMigrate(t *kernel.Task, src controller) error {
+func (*controllerStateless) PrepareMigrate(t *kernel.Task, src controller) error {
 	return nil
 }
 
 // CommitMigrate implements controller.CommitMigrate.
-func (*controllerNoopMigrate) CommitMigrate(t *kernel.Task, src controller) {}
+func (*controllerStateless) CommitMigrate(t *kernel.Task, src controller) {}
 
 // AbortMigrate implements controller.AbortMigrate.
-func (*controllerNoopMigrate) AbortMigrate(t *kernel.Task, src controller) {}
+func (*controllerStateless) AbortMigrate(t *kernel.Task, src controller) {}

pkg/sentry/fsimpl/cgroupfs/cgroupfs.go

@@ -541,6 +541,14 @@ func (d *dir) RmDir(ctx context.Context, name string, child kernfs.Inode) error
 	return err
 }
 
+func (d *dir) forEachChildDir(fn func(*dir)) {
+	d.OrderedChildren.ForEachChild(func(_ string, i kernfs.Inode) {
+		if childI, ok := i.(*cgroupInode); ok {
+			fn(&childI.dir)
+		}
+	})
+}
+
 // controllerFile represents a generic control file that appears within a cgroup
 // directory.
 //

pkg/sentry/fsimpl/cgroupfs/cpu.go

@@ -23,7 +23,7 @@ import (
 // +stateify savable
 type cpuController struct {
 	controllerCommon
-	controllerNoopMigrate
+	controllerStateless
 
 	// CFS bandwidth control parameters, values in microseconds.
 	cfsPeriod int64
@@ -67,7 +67,7 @@ func (c *cpuController) Clone() controller {
 		cfsQuota:  c.cfsQuota,
 		shares:    c.shares,
 	}
-	new.controllerCommon.cloneFrom(&c.controllerCommon)
+	new.controllerCommon.cloneFromParent(c)
 	return new
 }
 

pkg/sentry/fsimpl/cgroupfs/cpuacct.go

@@ -21,29 +21,61 @@ import (
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs"
+	"gvisor.dev/gvisor/pkg/sentry/kernel"
 	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
 	"gvisor.dev/gvisor/pkg/sentry/usage"
+	"gvisor.dev/gvisor/pkg/sync"
 )
 
+// cpuacctController tracks CPU usage for tasks managed by the controller. The
+// sentry already tracks CPU usage per task; the controller tries to avoid
+// duplicate bookkeeping. When a task moves into a cpuacct cgroup, for currently
+// running tasks we simple refer to the tasks themselves when asked to report
+// usage. Things get more interesting when tasks leave the cgroup, since we need
+// to attribute the usage across multiple cgroups.
+//
+// On migration, we attribute the task's usage up to the point of migration to
+// the src cgroup, and keep track of how much of the overall usage to discount
+// at the dst cgroup.
+//
+// On task exit, we attribute all unaccounted usage to the current cgroup and
+// stop tracking the task.
+//
 // +stateify savable
 type cpuacctController struct {
 	controllerCommon
-	controllerNoopMigrate
+	controllerStateless
+
+	mu sync.Mutex `state:"nosave"`
+
+	// taskCommittedCharges tracks charges for a task already attributed to this
+	// cgroup. This is used to avoid double counting usage for live
+	// tasks. Protected by mu.
+	taskCommittedCharges map[*kernel.Task]usage.CPUStats
+
+	// usage is the cumulative CPU time used by past tasks in this cgroup. Note
+	// that this doesn't include usage by live tasks currently in the
+	// cgroup. Protected by mu.
+	usage usage.CPUStats
 }
 
 var _ controller = (*cpuacctController)(nil)
 
 func newCPUAcctController(fs *filesystem) *cpuacctController {
-	c := &cpuacctController{}
+	c := &cpuacctController{
+		taskCommittedCharges: make(map[*kernel.Task]usage.CPUStats),
+	}
 	c.controllerCommon.init(controllerCPUAcct, fs)
 	return c
 }
 
 // Clone implements controller.Clone.
 func (c *cpuacctController) Clone() controller {
-	new := &cpuacctController{}
-	new.controllerCommon.cloneFrom(&new.controllerCommon)
-	return c
+	new := &cpuacctController{
+		taskCommittedCharges: make(map[*kernel.Task]usage.CPUStats),
+	}
+	new.controllerCommon.cloneFromParent(c)
+	return new
 }
 
 // AddControlFiles implements controller.AddControlFiles.
@@ -55,20 +87,81 @@ func (c *cpuacctController) AddControlFiles(ctx context.Context, creds *auth.Cre
 	contents["cpuacct.usage_sys"] = c.fs.newControllerFile(ctx, creds, &cpuacctUsageSysData{cpuacctCG})
 }
 
+// Enter implements controller.Enter.
+func (c *cpuacctController) Enter(t *kernel.Task) {}
+
+// Leave implements controller.Leave.
+func (c *cpuacctController) Leave(t *kernel.Task) {
+	charge := t.CPUStats()
+	c.mu.Lock()
+	outstandingCharge := charge.DifferenceSince(c.taskCommittedCharges[t])
+	c.usage.Accumulate(outstandingCharge)
+	delete(c.taskCommittedCharges, t)
+	c.mu.Unlock()
+}
+
+// PrepareMigrate implements controller.PrepareMigrate.
+func (c *cpuacctController) PrepareMigrate(t *kernel.Task, src controller) error {
+	return nil
+}
+
+// CommitMigrate implements controller.CommitMigrate.
+func (c *cpuacctController) CommitMigrate(t *kernel.Task, src controller) {
+	charge := t.CPUStats()
+
+	// Commit current charge to src and stop tracking t at src.
+	srcCtl := src.(*cpuacctController)
+	srcCtl.mu.Lock()
+	srcTaskCharge := srcCtl.taskCommittedCharges[t]
+	outstandingCharge := charge.DifferenceSince(srcTaskCharge)
+	srcCtl.usage.Accumulate(outstandingCharge)
+	delete(srcCtl.taskCommittedCharges, t)
+	srcCtl.mu.Unlock()
+
+	// Start tracking charge at dst, excluding the charge at src.
+	c.mu.Lock()
+	c.taskCommittedCharges[t] = charge
+	c.mu.Unlock()
+}
+
+// AbortMigrate implements controller.AbortMigrate.
+func (c *cpuacctController) AbortMigrate(t *kernel.Task, src controller) {}
+
 // +stateify savable
 type cpuacctCgroup struct {
 	*cgroupInode
 }
 
-func (c *cpuacctCgroup) collectCPUStats() usage.CPUStats {
-	var cs usage.CPUStats
-	c.fs.tasksMu.RLock()
-	// Note: This isn't very accurate, since the tasks are potentially
-	// still running as we accumulate their stats.
+func (c *cpuacctCgroup) cpuacctController() *cpuacctController {
+	return c.controllers[controllerCPUAcct].(*cpuacctController)
+}
+
+// checklocks:c.fs.tasksMu
+func (c *cpuacctCgroup) collectCPUStatsLocked(acc *usage.CPUStats) {
+	ctl := c.cpuacctController()
 	for t := range c.ts {
-		cs.Accumulate(t.CPUStats())
+		charge := t.CPUStats()
+		ctl.mu.Lock()
+		outstandingCharge := charge.DifferenceSince(ctl.taskCommittedCharges[t])
+		ctl.mu.Unlock()
+		acc.Accumulate(outstandingCharge)
 	}
-	c.fs.tasksMu.RUnlock()
+	ctl.mu.Lock()
+	acc.Accumulate(ctl.usage)
+	ctl.mu.Unlock()
+
+	c.forEachChildDir(func(d *dir) {
+		cg := cpuacctCgroup{d.cgi}
+		cg.collectCPUStatsLocked(acc)
+	})
+}
+
+func (c *cpuacctCgroup) collectCPUStats() usage.CPUStats {
+	c.fs.tasksMu.RLock()
+	defer c.fs.tasksMu.RUnlock()
+
+	var cs usage.CPUStats
+	c.collectCPUStatsLocked(&cs)
 	return cs
 }
 

pkg/sentry/fsimpl/cgroupfs/cpuset.go

@@ -34,7 +34,7 @@ import (
 // +stateify savable
 type cpusetController struct {
 	controllerCommon
-	controllerNoopMigrate
+	controllerStateless
 
 	maxCpus uint32
 	maxMems uint32
@@ -73,7 +73,7 @@ func (c *cpusetController) Clone() controller {
 		cpus:    &cpus,
 		mems:    &mems,
 	}
-	new.controllerCommon.cloneFrom(&c.controllerCommon)
+	new.controllerCommon.cloneFromParent(c)
 	return new
 }
 

pkg/sentry/fsimpl/cgroupfs/job.go

@@ -23,7 +23,7 @@ import (
 // +stateify savable
 type jobController struct {
 	controllerCommon
-	controllerNoopMigrate
+	controllerStateless
 
 	id int64
 }
@@ -41,7 +41,7 @@ func (c *jobController) Clone() controller {
 	new := &jobController{
 		id: c.id,
 	}
-	new.controllerCommon.cloneFrom(&c.controllerCommon)
+	new.controllerCommon.cloneFromParent(c)
 	return new
 }
 

pkg/sentry/fsimpl/cgroupfs/memory.go

@@ -30,7 +30,7 @@ import (
 // +stateify savable
 type memoryController struct {
 	controllerCommon
-	controllerNoopMigrate
+	controllerStateless
 
 	limitBytes            int64
 	softLimitBytes        int64
@@ -72,7 +72,7 @@ func (c *memoryController) Clone() controller {
 		softLimitBytes:        c.softLimitBytes,
 		moveChargeAtImmigrate: c.moveChargeAtImmigrate,
 	}
-	new.controllerCommon.cloneFrom(&c.controllerCommon)
+	new.controllerCommon.cloneFromParent(c)
 	return new
 }
 

pkg/sentry/fsimpl/kernfs/inode_impl_util.go

@@ -486,6 +486,16 @@ func (o *OrderedChildren) Lookup(ctx context.Context, name string) (Inode, error
 	return s.inode, nil
 }
 
+// ForEachChild calls fn on all childrens tracked by this ordered children.
+func (o *OrderedChildren) ForEachChild(fn func(string, Inode)) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+
+	for name, slot := range o.set {
+		fn(name, slot.inode)
+	}
+}
+
 // IterDirents implements Inode.IterDirents.
 func (o *OrderedChildren) IterDirents(ctx context.Context, mnt *vfs.Mount, cb vfs.IterDirentsCallback, offset, relOffset int64) (newOffset int64, err error) {
 	// All entries from OrderedChildren have already been handled in

pkg/sentry/kernel/cgroup.go

@@ -102,6 +102,13 @@ func (ctx *CgroupMigrationContext) Abort() {
 // Commit completes a migration.
 func (ctx *CgroupMigrationContext) Commit() {
 	ctx.dst.CommitMigrate(ctx.t, &ctx.src)
+
+	ctx.t.mu.Lock()
+	delete(ctx.t.cgroups, ctx.src)
+	ctx.src.DecRef(ctx.t)
+	ctx.dst.IncRef()
+	ctx.t.cgroups[ctx.dst] = struct{}{}
+	ctx.t.mu.Unlock()
 }
 
 // CgroupImpl is the common interface to cgroups.

pkg/sentry/usage/cpu.go

@@ -44,3 +44,14 @@ func (s *CPUStats) Accumulate(s2 CPUStats) {
 	s.SysTime += s2.SysTime
 	s.VoluntarySwitches += s2.VoluntarySwitches
 }
+
+// DifferenceSince computes s - earlierSample.
+//
+// Precondition: s >= earlierSample.
+func (s *CPUStats) DifferenceSince(earlierSample CPUStats) CPUStats {
+	return CPUStats{
+		UserTime:          s.UserTime - earlierSample.UserTime,
+		SysTime:           s.SysTime - earlierSample.SysTime,
+		VoluntarySwitches: s.VoluntarySwitches - earlierSample.VoluntarySwitches,
+	}
+}

test/syscalls/linux/cgroup.cc

@@ -508,6 +508,143 @@ TEST(CPUAcctCgroup, CPUAcctStat) {
   EXPECT_THAT(Atoi<int64_t>(sys_tokens[1]), IsPosixErrorOkAndHolds(Ge(0)));
 }
 
+TEST(CPUAcctCgroup, HierarchicalAccounting) {
+  SKIP_IF(!CgroupsAvailable());
+
+  Mounter m(ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir()));
+  Cgroup root = ASSERT_NO_ERRNO_AND_VALUE(m.MountCgroupfs("cpuacct"));
+  Cgroup child = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("child1"));
+
+  // Root should have non-zero CPU usage since the test itself will be running
+  // in the root cgroup.
+  EXPECT_THAT(root.ReadIntegerControlFile("cpuacct.usage"),
+              IsPosixErrorOkAndHolds(Gt(0)));
+
+  // Child should have zero usage since it is initially empty.
+  EXPECT_THAT(child.ReadIntegerControlFile("cpuacct.usage"),
+              IsPosixErrorOkAndHolds(Eq(0)));
+
+  // Move test into child and confirm child starts incurring usage.
+  const int64_t before_move =
+      ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage"));
+  ASSERT_NO_ERRNO(child.Enter(getpid()));
+  ASSERT_NO_ERRNO(
+      child.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  EXPECT_THAT(child.ReadIntegerControlFile("cpuacct.usage"),
+              IsPosixErrorOkAndHolds(Gt(0)));
+
+  // Root shouldn't lose usage due to the migration.
+  const int64_t after_move =
+      ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage"));
+  EXPECT_GE(after_move, before_move);
+
+  // Root should continue to gain usage after the move since child is a
+  // subcgroup.
+  ASSERT_NO_ERRNO(
+      child.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+  EXPECT_THAT(root.ReadIntegerControlFile("cpuacct.usage"),
+              IsPosixErrorOkAndHolds(Ge(after_move)));
+}
+
+TEST(CPUAcctCgroup, IndirectCharge) {
+  SKIP_IF(!CgroupsAvailable());
+
+  Mounter m(ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir()));
+  Cgroup root = ASSERT_NO_ERRNO_AND_VALUE(m.MountCgroupfs("cpuacct"));
+  Cgroup child1 = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("child1"));
+  Cgroup child2 = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("child2"));
+  Cgroup child2a = ASSERT_NO_ERRNO_AND_VALUE(child2.CreateChild("child2a"));
+
+  ASSERT_NO_ERRNO(child1.Enter(getpid()));
+  ASSERT_NO_ERRNO(
+      child1.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  // Only root and child1 should have usage.
+  for (auto const& cg : {root, child1}) {
+    EXPECT_THAT(cg.ReadIntegerControlFile("cpuacct.usage"),
+                IsPosixErrorOkAndHolds(Gt(0)));
+  }
+  for (auto const& cg : {child2, child2a}) {
+    EXPECT_THAT(cg.ReadIntegerControlFile("cpuacct.usage"),
+                IsPosixErrorOkAndHolds(Eq(0)));
+  }
+
+  ASSERT_NO_ERRNO(child2a.Enter(getpid()));
+  ASSERT_NO_ERRNO(
+      child2a.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  const int64_t snapshot_root =
+      ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t snapshot_child1 =
+      ASSERT_NO_ERRNO_AND_VALUE(child1.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t snapshot_child2 =
+      ASSERT_NO_ERRNO_AND_VALUE(child2.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t snapshot_child2a = ASSERT_NO_ERRNO_AND_VALUE(
+      child2a.ReadIntegerControlFile("cpuacct.usage"));
+
+  ASSERT_NO_ERRNO(
+      child2a.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  // Root, child2 and child2a should've accumulated new usage. Child1 should
+  // not.
+  const int64_t now_root =
+      ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t now_child1 =
+      ASSERT_NO_ERRNO_AND_VALUE(child1.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t now_child2 =
+      ASSERT_NO_ERRNO_AND_VALUE(child2.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t now_child2a = ASSERT_NO_ERRNO_AND_VALUE(
+      child2a.ReadIntegerControlFile("cpuacct.usage"));
+
+  EXPECT_GT(now_root, snapshot_root);
+  EXPECT_GT(now_child2, snapshot_child2);
+  EXPECT_GT(now_child2a, snapshot_child2a);
+  EXPECT_EQ(now_child1, snapshot_child1);
+}
+
+TEST(CPUAcctCgroup, NoDoubleAccounting) {
+  SKIP_IF(!CgroupsAvailable());
+
+  Mounter m(ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir()));
+  Cgroup root = ASSERT_NO_ERRNO_AND_VALUE(m.MountCgroupfs("cpuacct"));
+  Cgroup parent = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("parent"));
+  Cgroup a = ASSERT_NO_ERRNO_AND_VALUE(parent.CreateChild("a"));
+  Cgroup b = ASSERT_NO_ERRNO_AND_VALUE(parent.CreateChild("b"));
+
+  ASSERT_NO_ERRNO(a.Enter(getpid()));
+  ASSERT_NO_ERRNO(
+      a.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  ASSERT_NO_ERRNO(b.Enter(getpid()));
+  ASSERT_NO_ERRNO(
+      b.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  ASSERT_NO_ERRNO(root.Enter(getpid()));
+  ASSERT_NO_ERRNO(
+      root.PollControlFileForChange("cpuacct.usage", absl::Seconds(30)));
+
+  // The usage for parent, a & b should now be frozen, since they no longer have
+  // any tasks. Root will continue to accumulate usage.
+  const int64_t usage_root =
+      ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t usage_parent =
+      ASSERT_NO_ERRNO_AND_VALUE(parent.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t usage_a =
+      ASSERT_NO_ERRNO_AND_VALUE(a.ReadIntegerControlFile("cpuacct.usage"));
+  const int64_t usage_b =
+      ASSERT_NO_ERRNO_AND_VALUE(b.ReadIntegerControlFile("cpuacct.usage"));
+
+  EXPECT_GT(usage_root, 0);
+  EXPECT_GT(usage_parent, 0);
+  EXPECT_GT(usage_a, 0);
+  EXPECT_GT(usage_b, 0);
+  EXPECT_EQ(usage_parent, usage_a + usage_b);
+  EXPECT_GE(usage_parent, usage_a);
+  EXPECT_GE(usage_parent, usage_b);
+  EXPECT_GE(usage_root, usage_parent);
+}
+
 // WriteAndVerifyControlValue attempts to write val to a cgroup file at path,
 // and verify the value by reading it afterwards.
 PosixError WriteAndVerifyControlValue(const Cgroup& c, std::string_view path,

test/util/cgroup_util.cc

@@ -92,6 +92,37 @@ PosixErrorOr<absl::flat_hash_set<pid_t>> Cgroup::Tasks() const {
   return ParsePIDList(buf);
 }
 
+PosixError Cgroup::PollControlFileForChange(absl::string_view name,
+                                            absl::Duration timeout) const {
+  const absl::Duration poll_interval = absl::Milliseconds(10);
+  const absl::Time deadline = absl::Now() + timeout;
+  const std::string alias_path = absl::StrFormat("[cg#%d]/%s", id_, name);
+
+  ASSIGN_OR_RETURN_ERRNO(const int64_t initial_value,
+                         ReadIntegerControlFile(name));
+
+  while (true) {
+    ASSIGN_OR_RETURN_ERRNO(const int64_t current_value,
+                           ReadIntegerControlFile(name));
+    if (current_value != initial_value) {
+      std::cerr << absl::StreamFormat(
+                       "Control file '%s' changed from '%d' to '%d'",
+                       alias_path, initial_value, current_value)
+                << std::endl;
+      return NoError();
+    }
+    if (absl::Now() >= deadline) {
+      return PosixError(ETIME, absl::StrCat(alias_path, " didn't change in ",
+                                            absl::FormatDuration(timeout)));
+    }
+    std::cerr << absl::StreamFormat(
+                     "Waiting for control file '%s' to change from '%d'...",
+                     alias_path, initial_value)
+              << std::endl;
+    absl::SleepFor(poll_interval);
+  }
+}
+
 PosixError Cgroup::ContainsCallingProcess() const {
   ASSIGN_OR_RETURN_ERRNO(const absl::flat_hash_set<pid_t> procs, Procs());
   ASSIGN_OR_RETURN_ERRNO(const absl::flat_hash_set<pid_t> tasks, Tasks());

test/util/cgroup_util.h

@@ -70,6 +70,10 @@ class Cgroup {
   PosixError WriteIntegerControlFile(absl::string_view name,
                                      int64_t value) const;
 
+  // Waits for a control file's value to change.
+  PosixError PollControlFileForChange(absl::string_view name,
+                                      absl::Duration timeout) const;
+
   // Returns the thread ids of the leaders of thread groups managed by this
   // cgroup.
   PosixErrorOr<absl::flat_hash_set<pid_t>> Procs() const;