diff --git a/pkg/sentry/fsimpl/cgroupfs/base.go b/pkg/sentry/fsimpl/cgroupfs/base.go index ada829793..ee19e5e44 100644 --- a/pkg/sentry/fsimpl/cgroupfs/base.go +++ b/pkg/sentry/fsimpl/cgroupfs/base.go @@ -40,6 +40,11 @@ import ( type controllerCommon struct { ty kernel.CgroupControllerType fs *filesystem + // parent is the parent controller if any. Immutable. + // + // Note that we don't have to update this on renames, since cgroup + // directories can't be moved to a different parent directory. + parent controller } func (c *controllerCommon) init(ty kernel.CgroupControllerType, fs *filesystem) { @@ -47,9 +52,15 @@ func (c *controllerCommon) init(ty kernel.CgroupControllerType, fs *filesystem) c.fs = fs } -func (c *controllerCommon) cloneFrom(other *controllerCommon) { - c.ty = other.ty - c.fs = other.fs +func (c *controllerCommon) cloneFromParent(parent controller) { + c.ty = parent.Type() + c.fs = parent.Filesystem() + c.parent = parent +} + +// Filesystem implements controller.Filesystem. +func (c *controllerCommon) Filesystem() *filesystem { + return c.fs } // Type implements kernel.CgroupController.Type. @@ -85,7 +96,10 @@ func (c *controllerCommon) RootCgroup() kernel.Cgroup { type controller interface { kernel.CgroupController - // Clone creates a new controller based on the internal state of the current + // Filesystem returns the cgroupfs filesystem backing this controller. + Filesystem() *filesystem + + // Clone creates a new controller based on the internal state of this // controller. This is used to initialize a sub-cgroup based on the state of // the parent. Clone() controller @@ -94,6 +108,18 @@ type controller interface { // control files defined by this controller. AddControlFiles(ctx context.Context, creds *auth.Credentials, c *cgroupInode, contents map[string]kernfs.Inode) + // Enter is called when a task initially moves into a cgroup. This is + // distinct from migration because the task isn't migrating away from a + // cgroup. Enter is called when a task is created and joins its initial + // cgroup, or when cgroupfs is mounted and existing tasks are moved into + // cgroups. + Enter(t *kernel.Task) + + // Leave is called when a task leaves a cgroup. This is distinct from + // migration because the task isn't migrating to another cgroup. Leave is + // called when a task exits. + Leave(t *kernel.Task) + // PrepareMigrate signals the controller that a migration is about to // happen. The controller should check for any conditions that would prevent // the migration. If PrepareMigrate succeeds, the controller must @@ -186,6 +212,7 @@ func (c *cgroupInode) Controllers() []kernel.CgroupController { func (c *cgroupInode) tasks() []*kernel.Task { c.fs.tasksMu.RLock() defer c.fs.tasksMu.RUnlock() + ts := make([]*kernel.Task, 0, len(c.ts)) for t := range c.ts { ts = append(ts, t) @@ -196,15 +223,23 @@ func (c *cgroupInode) tasks() []*kernel.Task { // Enter implements kernel.CgroupImpl.Enter. func (c *cgroupInode) Enter(t *kernel.Task) { c.fs.tasksMu.Lock() + defer c.fs.tasksMu.Unlock() + c.ts[t] = struct{}{} - c.fs.tasksMu.Unlock() + for _, ctl := range c.controllers { + ctl.Enter(t) + } } // Leave implements kernel.CgroupImpl.Leave. func (c *cgroupInode) Leave(t *kernel.Task) { c.fs.tasksMu.Lock() + defer c.fs.tasksMu.Unlock() + + for _, ctl := range c.controllers { + ctl.Leave(t) + } delete(c.ts, t) - c.fs.tasksMu.Unlock() } // PrepareMigrate implements kernel.CgroupImpl.PrepareMigrate. @@ -229,14 +264,14 @@ func (c *cgroupInode) PrepareMigrate(t *kernel.Task, src *kernel.Cgroup) error { // CommitMigrate implements kernel.CgroupImpl.CommitMigrate. func (c *cgroupInode) CommitMigrate(t *kernel.Task, src *kernel.Cgroup) { + c.fs.tasksMu.Lock() + defer c.fs.tasksMu.Unlock() + for srcType, srcCtl := range src.CgroupImpl.(*cgroupInode).controllers { c.controllers[srcType].CommitMigrate(t, srcCtl) } srcI := src.CgroupImpl.(*cgroupInode) - c.fs.tasksMu.Lock() - defer c.fs.tasksMu.Unlock() - delete(srcI.ts, t) c.ts[t] = struct{}{} } @@ -375,17 +410,23 @@ func parseInt64FromString(ctx context.Context, src usermem.IOSequence) (val, len return val, int64(n), nil } -// controllerNoopMigrate partially implements controller. It stubs the migration +// controllerStateless partially implements controller. It stubs the migration // methods with noops for a stateless controller. -type controllerNoopMigrate struct{} +type controllerStateless struct{} + +// Enter implements controller.Enter. +func (*controllerStateless) Enter(t *kernel.Task) {} + +// Leave implements controller.Leave. +func (*controllerStateless) Leave(t *kernel.Task) {} // PrepareMigrate implements controller.PrepareMigrate. -func (*controllerNoopMigrate) PrepareMigrate(t *kernel.Task, src controller) error { +func (*controllerStateless) PrepareMigrate(t *kernel.Task, src controller) error { return nil } // CommitMigrate implements controller.CommitMigrate. -func (*controllerNoopMigrate) CommitMigrate(t *kernel.Task, src controller) {} +func (*controllerStateless) CommitMigrate(t *kernel.Task, src controller) {} // AbortMigrate implements controller.AbortMigrate. -func (*controllerNoopMigrate) AbortMigrate(t *kernel.Task, src controller) {} +func (*controllerStateless) AbortMigrate(t *kernel.Task, src controller) {} diff --git a/pkg/sentry/fsimpl/cgroupfs/cgroupfs.go b/pkg/sentry/fsimpl/cgroupfs/cgroupfs.go index de6277e2b..ad3f7286f 100644 --- a/pkg/sentry/fsimpl/cgroupfs/cgroupfs.go +++ b/pkg/sentry/fsimpl/cgroupfs/cgroupfs.go @@ -541,6 +541,14 @@ func (d *dir) RmDir(ctx context.Context, name string, child kernfs.Inode) error return err } +func (d *dir) forEachChildDir(fn func(*dir)) { + d.OrderedChildren.ForEachChild(func(_ string, i kernfs.Inode) { + if childI, ok := i.(*cgroupInode); ok { + fn(&childI.dir) + } + }) +} + // controllerFile represents a generic control file that appears within a cgroup // directory. // diff --git a/pkg/sentry/fsimpl/cgroupfs/cpu.go b/pkg/sentry/fsimpl/cgroupfs/cpu.go index 9c7f25ddd..5a8f1c9e5 100644 --- a/pkg/sentry/fsimpl/cgroupfs/cpu.go +++ b/pkg/sentry/fsimpl/cgroupfs/cpu.go @@ -23,7 +23,7 @@ import ( // +stateify savable type cpuController struct { controllerCommon - controllerNoopMigrate + controllerStateless // CFS bandwidth control parameters, values in microseconds. cfsPeriod int64 @@ -67,7 +67,7 @@ func (c *cpuController) Clone() controller { cfsQuota: c.cfsQuota, shares: c.shares, } - new.controllerCommon.cloneFrom(&c.controllerCommon) + new.controllerCommon.cloneFromParent(c) return new } diff --git a/pkg/sentry/fsimpl/cgroupfs/cpuacct.go b/pkg/sentry/fsimpl/cgroupfs/cpuacct.go index ae353fb33..69bf277a0 100644 --- a/pkg/sentry/fsimpl/cgroupfs/cpuacct.go +++ b/pkg/sentry/fsimpl/cgroupfs/cpuacct.go @@ -21,29 +21,61 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs" + "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" "gvisor.dev/gvisor/pkg/sentry/usage" + "gvisor.dev/gvisor/pkg/sync" ) +// cpuacctController tracks CPU usage for tasks managed by the controller. The +// sentry already tracks CPU usage per task; the controller tries to avoid +// duplicate bookkeeping. When a task moves into a cpuacct cgroup, for currently +// running tasks we simple refer to the tasks themselves when asked to report +// usage. Things get more interesting when tasks leave the cgroup, since we need +// to attribute the usage across multiple cgroups. +// +// On migration, we attribute the task's usage up to the point of migration to +// the src cgroup, and keep track of how much of the overall usage to discount +// at the dst cgroup. +// +// On task exit, we attribute all unaccounted usage to the current cgroup and +// stop tracking the task. +// // +stateify savable type cpuacctController struct { controllerCommon - controllerNoopMigrate + controllerStateless + + mu sync.Mutex `state:"nosave"` + + // taskCommittedCharges tracks charges for a task already attributed to this + // cgroup. This is used to avoid double counting usage for live + // tasks. Protected by mu. + taskCommittedCharges map[*kernel.Task]usage.CPUStats + + // usage is the cumulative CPU time used by past tasks in this cgroup. Note + // that this doesn't include usage by live tasks currently in the + // cgroup. Protected by mu. + usage usage.CPUStats } var _ controller = (*cpuacctController)(nil) func newCPUAcctController(fs *filesystem) *cpuacctController { - c := &cpuacctController{} + c := &cpuacctController{ + taskCommittedCharges: make(map[*kernel.Task]usage.CPUStats), + } c.controllerCommon.init(controllerCPUAcct, fs) return c } // Clone implements controller.Clone. func (c *cpuacctController) Clone() controller { - new := &cpuacctController{} - new.controllerCommon.cloneFrom(&new.controllerCommon) - return c + new := &cpuacctController{ + taskCommittedCharges: make(map[*kernel.Task]usage.CPUStats), + } + new.controllerCommon.cloneFromParent(c) + return new } // AddControlFiles implements controller.AddControlFiles. @@ -55,20 +87,81 @@ func (c *cpuacctController) AddControlFiles(ctx context.Context, creds *auth.Cre contents["cpuacct.usage_sys"] = c.fs.newControllerFile(ctx, creds, &cpuacctUsageSysData{cpuacctCG}) } +// Enter implements controller.Enter. +func (c *cpuacctController) Enter(t *kernel.Task) {} + +// Leave implements controller.Leave. +func (c *cpuacctController) Leave(t *kernel.Task) { + charge := t.CPUStats() + c.mu.Lock() + outstandingCharge := charge.DifferenceSince(c.taskCommittedCharges[t]) + c.usage.Accumulate(outstandingCharge) + delete(c.taskCommittedCharges, t) + c.mu.Unlock() +} + +// PrepareMigrate implements controller.PrepareMigrate. +func (c *cpuacctController) PrepareMigrate(t *kernel.Task, src controller) error { + return nil +} + +// CommitMigrate implements controller.CommitMigrate. +func (c *cpuacctController) CommitMigrate(t *kernel.Task, src controller) { + charge := t.CPUStats() + + // Commit current charge to src and stop tracking t at src. + srcCtl := src.(*cpuacctController) + srcCtl.mu.Lock() + srcTaskCharge := srcCtl.taskCommittedCharges[t] + outstandingCharge := charge.DifferenceSince(srcTaskCharge) + srcCtl.usage.Accumulate(outstandingCharge) + delete(srcCtl.taskCommittedCharges, t) + srcCtl.mu.Unlock() + + // Start tracking charge at dst, excluding the charge at src. + c.mu.Lock() + c.taskCommittedCharges[t] = charge + c.mu.Unlock() +} + +// AbortMigrate implements controller.AbortMigrate. +func (c *cpuacctController) AbortMigrate(t *kernel.Task, src controller) {} + // +stateify savable type cpuacctCgroup struct { *cgroupInode } -func (c *cpuacctCgroup) collectCPUStats() usage.CPUStats { - var cs usage.CPUStats - c.fs.tasksMu.RLock() - // Note: This isn't very accurate, since the tasks are potentially - // still running as we accumulate their stats. +func (c *cpuacctCgroup) cpuacctController() *cpuacctController { + return c.controllers[controllerCPUAcct].(*cpuacctController) +} + +// checklocks:c.fs.tasksMu +func (c *cpuacctCgroup) collectCPUStatsLocked(acc *usage.CPUStats) { + ctl := c.cpuacctController() for t := range c.ts { - cs.Accumulate(t.CPUStats()) + charge := t.CPUStats() + ctl.mu.Lock() + outstandingCharge := charge.DifferenceSince(ctl.taskCommittedCharges[t]) + ctl.mu.Unlock() + acc.Accumulate(outstandingCharge) } - c.fs.tasksMu.RUnlock() + ctl.mu.Lock() + acc.Accumulate(ctl.usage) + ctl.mu.Unlock() + + c.forEachChildDir(func(d *dir) { + cg := cpuacctCgroup{d.cgi} + cg.collectCPUStatsLocked(acc) + }) +} + +func (c *cpuacctCgroup) collectCPUStats() usage.CPUStats { + c.fs.tasksMu.RLock() + defer c.fs.tasksMu.RUnlock() + + var cs usage.CPUStats + c.collectCPUStatsLocked(&cs) return cs } diff --git a/pkg/sentry/fsimpl/cgroupfs/cpuset.go b/pkg/sentry/fsimpl/cgroupfs/cpuset.go index e6aa4a2a4..8307d15a8 100644 --- a/pkg/sentry/fsimpl/cgroupfs/cpuset.go +++ b/pkg/sentry/fsimpl/cgroupfs/cpuset.go @@ -34,7 +34,7 @@ import ( // +stateify savable type cpusetController struct { controllerCommon - controllerNoopMigrate + controllerStateless maxCpus uint32 maxMems uint32 @@ -73,7 +73,7 @@ func (c *cpusetController) Clone() controller { cpus: &cpus, mems: &mems, } - new.controllerCommon.cloneFrom(&c.controllerCommon) + new.controllerCommon.cloneFromParent(c) return new } diff --git a/pkg/sentry/fsimpl/cgroupfs/job.go b/pkg/sentry/fsimpl/cgroupfs/job.go index 31e25a15f..1cdc3fe64 100644 --- a/pkg/sentry/fsimpl/cgroupfs/job.go +++ b/pkg/sentry/fsimpl/cgroupfs/job.go @@ -23,7 +23,7 @@ import ( // +stateify savable type jobController struct { controllerCommon - controllerNoopMigrate + controllerStateless id int64 } @@ -41,7 +41,7 @@ func (c *jobController) Clone() controller { new := &jobController{ id: c.id, } - new.controllerCommon.cloneFrom(&c.controllerCommon) + new.controllerCommon.cloneFromParent(c) return new } diff --git a/pkg/sentry/fsimpl/cgroupfs/memory.go b/pkg/sentry/fsimpl/cgroupfs/memory.go index c484ba626..8bcde8cb8 100644 --- a/pkg/sentry/fsimpl/cgroupfs/memory.go +++ b/pkg/sentry/fsimpl/cgroupfs/memory.go @@ -30,7 +30,7 @@ import ( // +stateify savable type memoryController struct { controllerCommon - controllerNoopMigrate + controllerStateless limitBytes int64 softLimitBytes int64 @@ -72,7 +72,7 @@ func (c *memoryController) Clone() controller { softLimitBytes: c.softLimitBytes, moveChargeAtImmigrate: c.moveChargeAtImmigrate, } - new.controllerCommon.cloneFrom(&c.controllerCommon) + new.controllerCommon.cloneFromParent(c) return new } diff --git a/pkg/sentry/fsimpl/kernfs/inode_impl_util.go b/pkg/sentry/fsimpl/kernfs/inode_impl_util.go index 5bea0a605..d314e128e 100644 --- a/pkg/sentry/fsimpl/kernfs/inode_impl_util.go +++ b/pkg/sentry/fsimpl/kernfs/inode_impl_util.go @@ -486,6 +486,16 @@ func (o *OrderedChildren) Lookup(ctx context.Context, name string) (Inode, error return s.inode, nil } +// ForEachChild calls fn on all childrens tracked by this ordered children. +func (o *OrderedChildren) ForEachChild(fn func(string, Inode)) { + o.mu.RLock() + defer o.mu.RUnlock() + + for name, slot := range o.set { + fn(name, slot.inode) + } +} + // IterDirents implements Inode.IterDirents. func (o *OrderedChildren) IterDirents(ctx context.Context, mnt *vfs.Mount, cb vfs.IterDirentsCallback, offset, relOffset int64) (newOffset int64, err error) { // All entries from OrderedChildren have already been handled in diff --git a/pkg/sentry/kernel/cgroup.go b/pkg/sentry/kernel/cgroup.go index 071b204c2..3635734ea 100644 --- a/pkg/sentry/kernel/cgroup.go +++ b/pkg/sentry/kernel/cgroup.go @@ -102,6 +102,13 @@ func (ctx *CgroupMigrationContext) Abort() { // Commit completes a migration. func (ctx *CgroupMigrationContext) Commit() { ctx.dst.CommitMigrate(ctx.t, &ctx.src) + + ctx.t.mu.Lock() + delete(ctx.t.cgroups, ctx.src) + ctx.src.DecRef(ctx.t) + ctx.dst.IncRef() + ctx.t.cgroups[ctx.dst] = struct{}{} + ctx.t.mu.Unlock() } // CgroupImpl is the common interface to cgroups. diff --git a/pkg/sentry/usage/cpu.go b/pkg/sentry/usage/cpu.go index bfc282d69..5bf7736c0 100644 --- a/pkg/sentry/usage/cpu.go +++ b/pkg/sentry/usage/cpu.go @@ -44,3 +44,14 @@ func (s *CPUStats) Accumulate(s2 CPUStats) { s.SysTime += s2.SysTime s.VoluntarySwitches += s2.VoluntarySwitches } + +// DifferenceSince computes s - earlierSample. +// +// Precondition: s >= earlierSample. +func (s *CPUStats) DifferenceSince(earlierSample CPUStats) CPUStats { + return CPUStats{ + UserTime: s.UserTime - earlierSample.UserTime, + SysTime: s.SysTime - earlierSample.SysTime, + VoluntarySwitches: s.VoluntarySwitches - earlierSample.VoluntarySwitches, + } +} diff --git a/test/syscalls/linux/cgroup.cc b/test/syscalls/linux/cgroup.cc index 6de704c38..a950f0989 100644 --- a/test/syscalls/linux/cgroup.cc +++ b/test/syscalls/linux/cgroup.cc @@ -508,6 +508,143 @@ TEST(CPUAcctCgroup, CPUAcctStat) { EXPECT_THAT(Atoi(sys_tokens[1]), IsPosixErrorOkAndHolds(Ge(0))); } +TEST(CPUAcctCgroup, HierarchicalAccounting) { + SKIP_IF(!CgroupsAvailable()); + + Mounter m(ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir())); + Cgroup root = ASSERT_NO_ERRNO_AND_VALUE(m.MountCgroupfs("cpuacct")); + Cgroup child = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("child1")); + + // Root should have non-zero CPU usage since the test itself will be running + // in the root cgroup. + EXPECT_THAT(root.ReadIntegerControlFile("cpuacct.usage"), + IsPosixErrorOkAndHolds(Gt(0))); + + // Child should have zero usage since it is initially empty. + EXPECT_THAT(child.ReadIntegerControlFile("cpuacct.usage"), + IsPosixErrorOkAndHolds(Eq(0))); + + // Move test into child and confirm child starts incurring usage. + const int64_t before_move = + ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage")); + ASSERT_NO_ERRNO(child.Enter(getpid())); + ASSERT_NO_ERRNO( + child.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + EXPECT_THAT(child.ReadIntegerControlFile("cpuacct.usage"), + IsPosixErrorOkAndHolds(Gt(0))); + + // Root shouldn't lose usage due to the migration. + const int64_t after_move = + ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage")); + EXPECT_GE(after_move, before_move); + + // Root should continue to gain usage after the move since child is a + // subcgroup. + ASSERT_NO_ERRNO( + child.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + EXPECT_THAT(root.ReadIntegerControlFile("cpuacct.usage"), + IsPosixErrorOkAndHolds(Ge(after_move))); +} + +TEST(CPUAcctCgroup, IndirectCharge) { + SKIP_IF(!CgroupsAvailable()); + + Mounter m(ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir())); + Cgroup root = ASSERT_NO_ERRNO_AND_VALUE(m.MountCgroupfs("cpuacct")); + Cgroup child1 = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("child1")); + Cgroup child2 = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("child2")); + Cgroup child2a = ASSERT_NO_ERRNO_AND_VALUE(child2.CreateChild("child2a")); + + ASSERT_NO_ERRNO(child1.Enter(getpid())); + ASSERT_NO_ERRNO( + child1.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + // Only root and child1 should have usage. + for (auto const& cg : {root, child1}) { + EXPECT_THAT(cg.ReadIntegerControlFile("cpuacct.usage"), + IsPosixErrorOkAndHolds(Gt(0))); + } + for (auto const& cg : {child2, child2a}) { + EXPECT_THAT(cg.ReadIntegerControlFile("cpuacct.usage"), + IsPosixErrorOkAndHolds(Eq(0))); + } + + ASSERT_NO_ERRNO(child2a.Enter(getpid())); + ASSERT_NO_ERRNO( + child2a.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + const int64_t snapshot_root = + ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage")); + const int64_t snapshot_child1 = + ASSERT_NO_ERRNO_AND_VALUE(child1.ReadIntegerControlFile("cpuacct.usage")); + const int64_t snapshot_child2 = + ASSERT_NO_ERRNO_AND_VALUE(child2.ReadIntegerControlFile("cpuacct.usage")); + const int64_t snapshot_child2a = ASSERT_NO_ERRNO_AND_VALUE( + child2a.ReadIntegerControlFile("cpuacct.usage")); + + ASSERT_NO_ERRNO( + child2a.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + // Root, child2 and child2a should've accumulated new usage. Child1 should + // not. + const int64_t now_root = + ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage")); + const int64_t now_child1 = + ASSERT_NO_ERRNO_AND_VALUE(child1.ReadIntegerControlFile("cpuacct.usage")); + const int64_t now_child2 = + ASSERT_NO_ERRNO_AND_VALUE(child2.ReadIntegerControlFile("cpuacct.usage")); + const int64_t now_child2a = ASSERT_NO_ERRNO_AND_VALUE( + child2a.ReadIntegerControlFile("cpuacct.usage")); + + EXPECT_GT(now_root, snapshot_root); + EXPECT_GT(now_child2, snapshot_child2); + EXPECT_GT(now_child2a, snapshot_child2a); + EXPECT_EQ(now_child1, snapshot_child1); +} + +TEST(CPUAcctCgroup, NoDoubleAccounting) { + SKIP_IF(!CgroupsAvailable()); + + Mounter m(ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir())); + Cgroup root = ASSERT_NO_ERRNO_AND_VALUE(m.MountCgroupfs("cpuacct")); + Cgroup parent = ASSERT_NO_ERRNO_AND_VALUE(root.CreateChild("parent")); + Cgroup a = ASSERT_NO_ERRNO_AND_VALUE(parent.CreateChild("a")); + Cgroup b = ASSERT_NO_ERRNO_AND_VALUE(parent.CreateChild("b")); + + ASSERT_NO_ERRNO(a.Enter(getpid())); + ASSERT_NO_ERRNO( + a.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + ASSERT_NO_ERRNO(b.Enter(getpid())); + ASSERT_NO_ERRNO( + b.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + ASSERT_NO_ERRNO(root.Enter(getpid())); + ASSERT_NO_ERRNO( + root.PollControlFileForChange("cpuacct.usage", absl::Seconds(30))); + + // The usage for parent, a & b should now be frozen, since they no longer have + // any tasks. Root will continue to accumulate usage. + const int64_t usage_root = + ASSERT_NO_ERRNO_AND_VALUE(root.ReadIntegerControlFile("cpuacct.usage")); + const int64_t usage_parent = + ASSERT_NO_ERRNO_AND_VALUE(parent.ReadIntegerControlFile("cpuacct.usage")); + const int64_t usage_a = + ASSERT_NO_ERRNO_AND_VALUE(a.ReadIntegerControlFile("cpuacct.usage")); + const int64_t usage_b = + ASSERT_NO_ERRNO_AND_VALUE(b.ReadIntegerControlFile("cpuacct.usage")); + + EXPECT_GT(usage_root, 0); + EXPECT_GT(usage_parent, 0); + EXPECT_GT(usage_a, 0); + EXPECT_GT(usage_b, 0); + EXPECT_EQ(usage_parent, usage_a + usage_b); + EXPECT_GE(usage_parent, usage_a); + EXPECT_GE(usage_parent, usage_b); + EXPECT_GE(usage_root, usage_parent); +} + // WriteAndVerifyControlValue attempts to write val to a cgroup file at path, // and verify the value by reading it afterwards. PosixError WriteAndVerifyControlValue(const Cgroup& c, std::string_view path, diff --git a/test/util/cgroup_util.cc b/test/util/cgroup_util.cc index 55dacd319..e0e423f97 100644 --- a/test/util/cgroup_util.cc +++ b/test/util/cgroup_util.cc @@ -92,6 +92,37 @@ PosixErrorOr> Cgroup::Tasks() const { return ParsePIDList(buf); } +PosixError Cgroup::PollControlFileForChange(absl::string_view name, + absl::Duration timeout) const { + const absl::Duration poll_interval = absl::Milliseconds(10); + const absl::Time deadline = absl::Now() + timeout; + const std::string alias_path = absl::StrFormat("[cg#%d]/%s", id_, name); + + ASSIGN_OR_RETURN_ERRNO(const int64_t initial_value, + ReadIntegerControlFile(name)); + + while (true) { + ASSIGN_OR_RETURN_ERRNO(const int64_t current_value, + ReadIntegerControlFile(name)); + if (current_value != initial_value) { + std::cerr << absl::StreamFormat( + "Control file '%s' changed from '%d' to '%d'", + alias_path, initial_value, current_value) + << std::endl; + return NoError(); + } + if (absl::Now() >= deadline) { + return PosixError(ETIME, absl::StrCat(alias_path, " didn't change in ", + absl::FormatDuration(timeout))); + } + std::cerr << absl::StreamFormat( + "Waiting for control file '%s' to change from '%d'...", + alias_path, initial_value) + << std::endl; + absl::SleepFor(poll_interval); + } +} + PosixError Cgroup::ContainsCallingProcess() const { ASSIGN_OR_RETURN_ERRNO(const absl::flat_hash_set procs, Procs()); ASSIGN_OR_RETURN_ERRNO(const absl::flat_hash_set tasks, Tasks()); diff --git a/test/util/cgroup_util.h b/test/util/cgroup_util.h index 870b35714..998410c3c 100644 --- a/test/util/cgroup_util.h +++ b/test/util/cgroup_util.h @@ -70,6 +70,10 @@ class Cgroup { PosixError WriteIntegerControlFile(absl::string_view name, int64_t value) const; + // Waits for a control file's value to change. + PosixError PollControlFileForChange(absl::string_view name, + absl::Duration timeout) const; + // Returns the thread ids of the leaders of thread groups managed by this // cgroup. PosixErrorOr> Procs() const;