Better memory reporting for multi-container

Right now, the entire sandbox memory is reported per-container, confusing
users and tools that aggregate per-container memory to compute sandbox/pod
memory. So instead, split memory usage amoung all containers in the
system, except for the root container which is ignored by K8s. This way
pod memory usage is shown correctly in graphs.

Updates #172

PiperOrigin-RevId: 550670618
This commit is contained in:
Fabricio Voznika
2023-07-24 14:20:10 -07:00
committed by gVisor bot
parent 0ef88bbbd8
commit a5fd5015e9
4 changed files with 54 additions and 10 deletions
+25 -4
View File
@@ -15,6 +15,8 @@
package boot
import (
"errors"
"gvisor.dev/gvisor/pkg/sentry/control"
"gvisor.dev/gvisor/pkg/sentry/usage"
)
@@ -81,22 +83,41 @@ type CPUUsage struct {
}
// Event gets the events from the container.
func (cm *containerManager) Event(_ *struct{}, out *EventOut) error {
func (cm *containerManager) Event(cid *string, out *EventOut) error {
*out = EventOut{
Event: Event{
ID: *cid,
Type: "stats",
},
}
// Memory usage.
// TODO(gvisor.dev/issue/172): Per-container accounting.
mem := cm.l.k.MemoryFile()
_ = mem.UpdateUsage() // best effort to update.
_, totalUsage := usage.MemoryAccounting.Copy()
out.Event.Data.Memory.Usage = MemoryEntry{
Usage: totalUsage,
switch containers := cm.l.containerCount(); containers {
case 0:
return errors.New("no container was found")
case 1:
// There is a single container, so total usage can only come from it.
default:
// In the multi-container case, reports 0 for the root (pause) container,
// since it's small and idle. Then equally split the usage to the other
// containers. At least the sum of all containers will correctly account
// for the memory used by the sandbox.
//
// TODO(gvisor.dev/issue/172): Proper per-container accounting.
if *cid == cm.l.sandboxID {
totalUsage = 0
} else {
totalUsage /= uint64(containers - 1)
}
}
out.Event.Data.Memory.Usage.Usage = totalUsage
// PIDs.
// TODO(gvisor.dev/issue/172): Per-container accounting.
out.Event.Data.Pids.Current = uint64(len(cm.l.k.TaskSet().Root.ThreadGroups()))
+16 -1
View File
@@ -1009,7 +1009,7 @@ func (l *Loader) startGoferMonitor(cid string, rootfsGoferFD int32) {
},
}
_, _, err := specutils.RetryEintr(func() (uintptr, uintptr, error) {
// Use ppoll instead of poll because it's already whilelisted in seccomp.
// Use ppoll instead of poll because it's already allowed in seccomp.
n, err := unix.Ppoll(events, nil, nil)
return uintptr(n), 0, err
})
@@ -1608,3 +1608,18 @@ func (l *Loader) importFD(ctx context.Context, f *os.File) (*vfs.FileDescription
hostFD.Release()
return fd, nil
}
func (l *Loader) containerCount() int {
l.mu.Lock()
defer l.mu.Unlock()
containers := 0
for id := range l.processes {
if id.pid == 0 {
// pid==0 represents the init process of a container. There is
// only one of such process per container.
containers++
}
}
return containers
}
+12 -1
View File
@@ -2106,7 +2106,7 @@ func TestMultiContainerEvent(t *testing.T) {
}
// Check events for running containers.
for _, cont := range containers[:2] {
for i, cont := range containers[:2] {
ret, err := cont.Event()
if err != nil {
t.Errorf("Container.Event(%q): %v", cont.ID, err)
@@ -2123,6 +2123,17 @@ func TestMultiContainerEvent(t *testing.T) {
t.Errorf("Wrong number of PIDs, cid: %q, want: %d, got: %d", cont.ID, want, got)
}
switch i {
case 0:
if evt.Data.Memory.Usage.Usage != uint64(0) {
t.Errorf("root container should report 0 memory usage, got: %v", evt.Data.Memory.Usage.Usage)
}
case 1:
if evt.Data.Memory.Usage.Usage == uint64(0) {
t.Error("sub-container should report non-zero memory usage")
}
}
// The exited container should always have a usage of zero.
if exited := ret.ContainerUsage[containers[2].ID]; exited != 0 {
t.Errorf("Exited container should report 0 CPU usage, got: %d", exited)
+1 -4
View File
@@ -577,12 +577,9 @@ func (s *Sandbox) Execute(conf *config.Config, args *control.ExecArgs) (int32, e
func (s *Sandbox) Event(cid string) (*boot.EventOut, error) {
log.Debugf("Getting events for container %q in sandbox %q", cid, s.ID)
var e boot.EventOut
// TODO(b/129292330): Pass in the container id (cid) here. The sandbox
// should return events only for that container.
if err := s.call(boot.ContMgrEvent, nil, &e); err != nil {
if err := s.call(boot.ContMgrEvent, &cid, &e); err != nil {
return nil, fmt.Errorf("retrieving event data from sandbox: %w", err)
}
e.Event.ID = cid
return &e, nil
}