Add Usage controls

Add Usage controls and implement "usage/usagefd" commands.

PiperOrigin-RevId: 390507423

pkg/sentry/control/BUILD

@@ -12,6 +12,7 @@ go_library(
         "pprof.go",
         "proc.go",
         "state.go",
+        "usage.go",
     ],
     visibility = [
         "//:sandbox",
@@ -39,6 +40,7 @@ go_library(
         "//pkg/tcpip/link/sniffer",
         "//pkg/urpc",
         "//pkg/usermem",
+        "@org_golang_x_sys//unix:go_default_library",
     ],
 )
 

pkg/sentry/control/usage.go

@@ -0,0 +1,183 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package control
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/sentry/usage"
+	"gvisor.dev/gvisor/pkg/urpc"
+)
+
+// Usage includes usage-related RPC stubs.
+type Usage struct {
+	Kernel *kernel.Kernel
+}
+
+// MemoryUsageOpts contains usage options.
+type MemoryUsageOpts struct {
+	// Full indicates that a full accounting should be done. If Full is not
+	// specified, then a partial accounting will be done, and Unknown will
+	// contain a majority of memory. See Collect for more information.
+	Full bool `json:"Full"`
+}
+
+// MemoryUsage is a memory usage structure.
+type MemoryUsage struct {
+	Unknown   uint64 `json:"Unknown"`
+	System    uint64 `json:"System"`
+	Anonymous uint64 `json:"Anonymous"`
+	PageCache uint64 `json:"PageCache"`
+	Mapped    uint64 `json:"Mapped"`
+	Tmpfs     uint64 `json:"Tmpfs"`
+	Ramdiskfs uint64 `json:"Ramdiskfs"`
+	Total     uint64 `json:"Total"`
+}
+
+// MemoryUsageFileOpts contains usage file options.
+type MemoryUsageFileOpts struct {
+	// Version is used to ensure both sides agree on the format of the
+	// shared memory buffer.
+	Version uint64 `json:"Version"`
+}
+
+// MemoryUsageFile contains the file handle to the usage file.
+type MemoryUsageFile struct {
+	urpc.FilePayload
+}
+
+// UsageFD returns the file that tracks the memory usage of the application.
+func (u *Usage) UsageFD(opts *MemoryUsageFileOpts, out *MemoryUsageFile) error {
+	// Only support version 1 for now.
+	if opts.Version != 1 {
+		return fmt.Errorf("unsupported version requested: %d", opts.Version)
+	}
+
+	mf := u.Kernel.MemoryFile()
+	*out = MemoryUsageFile{
+		FilePayload: urpc.FilePayload{
+			Files: []*os.File{
+				usage.MemoryAccounting.File,
+				mf.File(),
+			},
+		},
+	}
+
+	return nil
+}
+
+// Collect returns memory used by the sandboxed application.
+func (u *Usage) Collect(opts *MemoryUsageOpts, out *MemoryUsage) error {
+	if opts.Full {
+		// Ensure everything is up to date.
+		if err := u.Kernel.MemoryFile().UpdateUsage(); err != nil {
+			return err
+		}
+
+		// Copy out a snapshot.
+		snapshot, total := usage.MemoryAccounting.Copy()
+		*out = MemoryUsage{
+			System:    snapshot.System,
+			Anonymous: snapshot.Anonymous,
+			PageCache: snapshot.PageCache,
+			Mapped:    snapshot.Mapped,
+			Tmpfs:     snapshot.Tmpfs,
+			Ramdiskfs: snapshot.Ramdiskfs,
+			Total:     total,
+		}
+	} else {
+		// Get total usage from the MemoryFile implementation.
+		total, err := u.Kernel.MemoryFile().TotalUsage()
+		if err != nil {
+			return err
+		}
+
+		// The memory accounting is guaranteed to be accurate only when
+		// UpdateUsage is called. If UpdateUsage is not called, then only Mapped
+		// will be up-to-date.
+		snapshot, _ := usage.MemoryAccounting.Copy()
+		*out = MemoryUsage{
+			Unknown: total,
+			Mapped:  snapshot.Mapped,
+			Total:   total + snapshot.Mapped,
+		}
+
+	}
+
+	return nil
+}
+
+// UsageReduceOpts contains options to Usage.Reduce().
+type UsageReduceOpts struct {
+	// If Wait is true, Reduce blocks until all activity initiated by
+	// Usage.Reduce() has completed.
+	Wait bool `json:"wait"`
+}
+
+// UsageReduceOutput contains output from Usage.Reduce().
+type UsageReduceOutput struct{}
+
+// Reduce requests that the sentry attempt to reduce its memory usage.
+func (u *Usage) Reduce(opts *UsageReduceOpts, out *UsageReduceOutput) error {
+	mf := u.Kernel.MemoryFile()
+	mf.StartEvictions()
+	if opts.Wait {
+		mf.WaitForEvictions()
+	}
+	return nil
+}
+
+// MemoryUsageRecord contains the mapping and platform memory file.
+type MemoryUsageRecord struct {
+	mmap  uintptr
+	stats *usage.RTMemoryStats
+	mf    os.File
+}
+
+// NewMemoryUsageRecord creates a new MemoryUsageRecord from usageFile and
+// platformFile.
+func NewMemoryUsageRecord(usageFile, platformFile os.File) (*MemoryUsageRecord, error) {
+	mmap, _, e := unix.RawSyscall6(unix.SYS_MMAP, 0, usage.RTMemoryStatsSize, unix.PROT_READ, unix.MAP_SHARED, usageFile.Fd(), 0)
+	if e != 0 {
+		return nil, fmt.Errorf("mmap returned %d, want 0", e)
+	}
+
+	m := MemoryUsageRecord{
+		mmap:  mmap,
+		stats: usage.RTMemoryStatsPointer(mmap),
+		mf:    platformFile,
+	}
+
+	runtime.SetFinalizer(&m, finalizer)
+	return &m, nil
+}
+
+func finalizer(m *MemoryUsageRecord) {
+	unix.RawSyscall(unix.SYS_MUNMAP, m.mmap, usage.RTMemoryStatsSize, 0)
+}
+
+// Fetch fetches the usage info from a MemoryUsageRecord.
+func (m *MemoryUsageRecord) Fetch() (mapped, unknown, total uint64, err error) {
+	var stat unix.Stat_t
+	if err := unix.Fstat(int(m.mf.Fd()), &stat); err != nil {
+		return 0, 0, 0, err
+	}
+	fmem := uint64(stat.Blocks) * 512
+	return m.stats.RTMapped, fmem, m.stats.RTMapped + fmem, nil
+}

runsc/boot/controller.go

@@ -114,6 +114,13 @@ const (
 	FsCat = "Fs.Cat"
 )
 
+// Usage related commands (see usage.go for more details).
+const (
+	UsageCollect = "Usage.Collect"
+	UsageUsageFD = "Usage.UsageFD"
+	UsageReduce  = "Usage.Reduce"
+)
+
 // ControlSocketAddr generates an abstract unix socket name for the given ID.
 func ControlSocketAddr(id string) string {
 	return fmt.Sprintf("\x00runsc-sandbox.%s", id)
@@ -157,6 +164,7 @@ func newController(fd int, l *Loader) (*controller, error) {
 	ctrl.srv.Register(&control.Logging{})
 	ctrl.srv.Register(&control.Lifecycle{l.k})
 	ctrl.srv.Register(&control.Fs{l.k})
+	ctrl.srv.Register(&control.Usage{l.k})
 
 	if l.root.conf.ProfileEnable {
 		ctrl.srv.Register(control.NewProfile(l.k))

runsc/cmd/BUILD

@@ -36,6 +36,7 @@ go_library(
         "statefile.go",
         "symbolize.go",
         "syscalls.go",
+        "usage.go",
         "verity_prepare.go",
         "wait.go",
     ],

runsc/cmd/usage.go

@@ -0,0 +1,93 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/google/subcommands"
+	"gvisor.dev/gvisor/runsc/config"
+	"gvisor.dev/gvisor/runsc/container"
+	"gvisor.dev/gvisor/runsc/flag"
+)
+
+// Usage implements subcommands.Command for the "usage" command.
+type Usage struct {
+	full bool
+	fd   bool
+}
+
+// Name implements subcommands.Command.Name.
+func (*Usage) Name() string {
+	return "usage"
+}
+
+// Synopsis implements subcommands.Command.Synopsis.
+func (*Usage) Synopsis() string {
+	return "Usage shows application memory usage across various categories in bytes."
+}
+
+// Usage implements subcommands.Command.Usage.
+func (*Usage) Usage() string {
+	return `usage [flags] <container id> - print memory usages to standard output.`
+}
+
+// SetFlags implements subcommands.Command.SetFlags.
+func (u *Usage) SetFlags(f *flag.FlagSet) {
+	f.BoolVar(&u.full, "full", false, "enumerate all usage by categories")
+	f.BoolVar(&u.fd, "fd", false, "retrieves a subset of usage through the established usage FD")
+}
+
+// Execute implements subcommands.Command.Execute.
+func (u *Usage) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
+	if f.NArg() < 1 {
+		f.Usage()
+		return subcommands.ExitUsageError
+	}
+
+	id := f.Arg(0)
+	conf := args[0].(*config.Config)
+
+	cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{})
+	if err != nil {
+		Fatalf("loading container: %v", err)
+	}
+
+	if !u.fd {
+		m, err := cont.Usage(u.full)
+		if err != nil {
+			Fatalf("usage failed: %v", err)
+		}
+		if err := json.NewEncoder(os.Stdout).Encode(m); err != nil {
+			Fatalf("Encode MemoryUsage failed: %v", err)
+		}
+	} else {
+		m, err := cont.UsageFD()
+		if err != nil {
+			Fatalf("usagefd failed: %v", err)
+		}
+
+		mapped, unknown, total, err := m.Fetch()
+		if err != nil {
+			Fatalf("Fetch memory usage failed: %v", err)
+		}
+
+		fmt.Printf("Mapped %v, Unknown %v, Total %v\n", mapped, unknown, total)
+	}
+	return subcommands.ExitSuccess
+}

runsc/container/container.go

@@ -652,6 +652,24 @@ func (c *Container) Cat(files []string, out *os.File) error {
 	return c.Sandbox.Cat(c.ID, files, out)
 }
 
+// Usage displays memory used by the application.
+func (c *Container) Usage(full bool) (control.MemoryUsage, error) {
+	log.Debugf("Usage in container, cid: %s, full: %v", c.ID, full)
+	return c.Sandbox.Usage(c.ID, full)
+}
+
+// UsageFD shows application memory usage using two donated FDs.
+func (c *Container) UsageFD() (*control.MemoryUsageRecord, error) {
+	log.Debugf("UsageFD in container, cid: %s", c.ID)
+	return c.Sandbox.UsageFD(c.ID)
+}
+
+// Reduce requests that the sentry attempt to reduce its memory usage.
+func (c *Container) Reduce(wait bool) error {
+	log.Debugf("Reduce in container, cid: %s", c.ID)
+	return c.Sandbox.Reduce(c.ID, wait)
+}
+
 // State returns the metadata of the container.
 func (c *Container) State() specs.State {
 	return specs.State{

runsc/container/container_test.go

@@ -2655,3 +2655,127 @@ func TestCat(t *testing.T) {
 		t.Errorf("out got %s, want include %s", buf, want)
 	}
 }
+
+// TestUsage checks that usage generates the expected memory usage.
+func TestUsage(t *testing.T) {
+	spec, conf := sleepSpecConf(t)
+	_, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf)
+	if err != nil {
+		t.Fatalf("error setting up container: %v", err)
+	}
+	defer cleanup()
+
+	args := Args{
+		ID:        testutil.RandomContainerID(),
+		Spec:      spec,
+		BundleDir: bundleDir,
+	}
+
+	cont, err := New(conf, args)
+	if err != nil {
+		t.Fatalf("Creating container: %v", err)
+	}
+	defer cont.Destroy()
+
+	if err := cont.Start(conf); err != nil {
+		t.Fatalf("starting container: %v", err)
+	}
+
+	for _, full := range []bool{false, true} {
+		m, err := cont.Usage(full)
+		if err != nil {
+			t.Fatalf("error usage from container: %v", err)
+		}
+		if m.Mapped == 0 {
+			t.Errorf("Usage mapped got zero")
+		}
+		if m.Total == 0 {
+			t.Errorf("Usage total got zero")
+		}
+		if full {
+			if m.System == 0 {
+				t.Errorf("Usage system got zero")
+			}
+			if m.Anonymous == 0 {
+				t.Errorf("Usage anonymous got zero")
+			}
+		}
+	}
+}
+
+// TestUsageFD checks that usagefd generates the expected memory usage.
+func TestUsageFD(t *testing.T) {
+	spec, conf := sleepSpecConf(t)
+
+	_, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf)
+	if err != nil {
+		t.Fatalf("error setting up container: %v", err)
+	}
+	defer cleanup()
+
+	args := Args{
+		ID:        testutil.RandomContainerID(),
+		Spec:      spec,
+		BundleDir: bundleDir,
+	}
+
+	cont, err := New(conf, args)
+	if err != nil {
+		t.Fatalf("Creating container: %v", err)
+	}
+	defer cont.Destroy()
+
+	if err := cont.Start(conf); err != nil {
+		t.Fatalf("starting container: %v", err)
+	}
+
+	m, err := cont.UsageFD()
+	if err != nil {
+		t.Fatalf("error usageFD from container: %v", err)
+	}
+
+	mapped, unknown, total, err := m.Fetch()
+	if err != nil {
+		t.Fatalf("error Fetch memory usage: %v", err)
+	}
+
+	if mapped == 0 {
+		t.Errorf("UsageFD Mapped got zero")
+	}
+	if unknown == 0 {
+		t.Errorf("UsageFD unknown got zero")
+	}
+	if total == 0 {
+		t.Errorf("UsageFD total got zero")
+	}
+}
+
+// TestReduce checks that reduce call succeeds.
+func TestReduce(t *testing.T) {
+	spec, conf := sleepSpecConf(t)
+	_, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf)
+	if err != nil {
+		t.Fatalf("error setting up container: %v", err)
+	}
+	defer cleanup()
+
+	args := Args{
+		ID:        testutil.RandomContainerID(),
+		Spec:      spec,
+		BundleDir: bundleDir,
+	}
+
+	cont, err := New(conf, args)
+	if err != nil {
+		t.Fatalf("Creating container: %v", err)
+	}
+	defer cont.Destroy()
+
+	if err := cont.Start(conf); err != nil {
+		t.Fatalf("starting container: %v", err)
+	}
+
+	if err := cont.Reduce(false); err != nil {
+		t.Fatalf("error reduce from container: %v", err)
+	}
+}

runsc/sandbox/sandbox.go

@@ -1020,6 +1020,59 @@ func (s *Sandbox) Cat(cid string, files []string, out *os.File) error {
 	return nil
 }
 
+// Usage sends the collect call for a container in the sandbox.
+func (s *Sandbox) Usage(cid string, Full bool) (control.MemoryUsage, error) {
+	log.Debugf("Usage sandbox %q", s.ID)
+	conn, err := s.sandboxConnect()
+	if err != nil {
+		return control.MemoryUsage{}, err
+	}
+	defer conn.Close()
+
+	var m control.MemoryUsage
+	err = conn.Call(boot.UsageCollect, &control.MemoryUsageOpts{
+		Full: Full,
+	}, &m)
+	return m, err
+}
+
+// UsageFD sends the usagefd call for a container in the sandbox.
+func (s *Sandbox) UsageFD(cid string) (*control.MemoryUsageRecord, error) {
+	log.Debugf("Usage sandbox %q", s.ID)
+	conn, err := s.sandboxConnect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	var m control.MemoryUsageFile
+	if err := conn.Call(boot.UsageUsageFD, &control.MemoryUsageFileOpts{
+		Version: 1,
+	}, &m); err != nil {
+		return nil, fmt.Errorf("UsageFD failed: %v", err)
+	}
+
+	if len(m.FilePayload.Files) != 2 {
+		return nil, fmt.Errorf("wants exactly two fds")
+	}
+
+	return control.NewMemoryUsageRecord(*m.FilePayload.Files[0], *m.FilePayload.Files[1])
+}
+
+// Reduce sends the reduce call for a container in the sandbox.
+func (s *Sandbox) Reduce(cid string, wait bool) error {
+	log.Debugf("Reduce sandbox %q", s.ID)
+	conn, err := s.sandboxConnect()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	return conn.Call(boot.UsageReduce, &control.UsageReduceOpts{
+		Wait: wait,
+	}, nil)
+}
+
 // IsRunning returns true if the sandbox or gofer process is running.
 func (s *Sandbox) IsRunning() bool {
 	if s.Pid != 0 {