gVisor: Add runsc metrics-export subcommand.

This subcommand prints a sandbox's instrumentation data in Prometheus format
to stdout.

This change is part of a series of changes to support Prometheus-style metrics
in `runsc`. Doing so requires making several seemingly-odd design decisions,
due to the following architectural constraints:

- Prometheus requires an HTTP server serving the `/metrics` endpoint.
- For performance reasons, the `runsc boot` process cannot run the `netpoller`
  goroutine.
  - Since we don't want to write our own HTTP server implementation, this
    means the HTTP endpoint has to be served by a separate process that
    remains running during the lifetime of the container.
- The `runsc boot` process is untrusted.
  - This means we cannot trust metrics data that comes out of the Sentry.
    Therefore, there needs to be an elaborate dance where we pre-register
    metric metadata before starting any untrusted workload. Then, the server
    relaying the metric data must verify the validity of metric values against
    this metric metadata. This avoids leaking metrics, cardinality blow-ups,
    and other such DoS vectors.
- This feature needs to be easy-to-use in a typical Docker setting.
  - This means having the ability to just say
    `--metrics-server=localhost:1337` in the `runsc` runtime entry in
    `/etc/docker/daemon.json` and have that Just Work(TM), even when multiple
    containers are running.
  - Since only one process may listen on a port at a given time, this means
    the metric server needs to be able to multiplex requests out to multiple
    running sandboxes, and remain alive for the entire duration of either of
    these sandboxes. However, it should also die when there are no sandboxes,
    so that we don't end up with leftover metric servers lying around.
  - For this reason, the metrics server runs *outside* of the usual
    per-container cgroups.
  - This also saves system resources by not running one server per sandbox.
- The metrics server must be exposed to the outside world, and cannot assume
  that its clients are trustworthy.
  - For this reason, a metrics server is bound to a runtime root directory,
    and double-checks all that the sandboxes it is asked to follow actually
    exist in this root directory.

PiperOrigin-RevId: 498067941

pkg/sentry/control/BUILD

@@ -20,6 +20,7 @@ go_library(
         "fs.go",
         "lifecycle.go",
         "logging.go",
+        "metrics.go",
         "pprof.go",
         "proc.go",
         "state.go",
@@ -36,6 +37,8 @@ go_library(
         "//pkg/fd",
         "//pkg/fspath",
         "//pkg/log",
+        "//pkg/metric",
+        "//pkg/prometheus",
         "//pkg/sentry/fdimport",
         "//pkg/sentry/fsimpl/host",
         "//pkg/sentry/fsimpl/user",

pkg/sentry/control/metrics.go

@@ -0,0 +1,37 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package control
+
+import (
+	"gvisor.dev/gvisor/pkg/metric"
+	"gvisor.dev/gvisor/pkg/prometheus"
+)
+
+// Metrics includes metrics-related RPC stubs.
+type Metrics struct{}
+
+// MetricsExportOpts contains metric exporting options.
+type MetricsExportOpts struct{}
+
+// MetricsExportData contains data for all metrics being exported.
+type MetricsExportData struct {
+	Snapshot *prometheus.Snapshot `json:"snapshot"`
+}
+
+// Export export metrics data into MetricsExportData.
+func (u *Metrics) Export(_ *MetricsExportOpts, out *MetricsExportData) error {
+	out.Snapshot = metric.GetSnapshot()
+	return nil
+}

runsc/boot/controller.go

@@ -129,6 +129,11 @@ const (
 	UsageReduce  = "Usage.Reduce"
 )
 
+// Metrics related commands (see metrics.go).
+const (
+	MetricsExport = "Metrics.Export"
+)
+
 // Commands for interacting with cgroupfs within the sandbox.
 const (
 	CgroupsReadControlFiles  = "Cgroups.ReadControlFiles"
@@ -173,6 +178,7 @@ func newController(fd int, l *Loader) (*controller, error) {
 	ctrl.srv.Register(&control.Proc{Kernel: l.k})
 	ctrl.srv.Register(&control.State{Kernel: l.k})
 	ctrl.srv.Register(&control.Usage{Kernel: l.k})
+	ctrl.srv.Register(&control.Metrics{})
 	ctrl.srv.Register(&debug{})
 
 	if eps, ok := l.k.RootNetworkNamespace().Stack().(*netstack.Stack); ok {

runsc/cli/main.go

@@ -94,6 +94,7 @@ func Main(version string) {
 	subcommands.Register(new(cmd.Statefile), debugGroup)
 	subcommands.Register(new(cmd.Symbolize), debugGroup)
 	subcommands.Register(new(cmd.Usage), debugGroup)
+	subcommands.Register(new(cmd.MetricExport), debugGroup)
 	subcommands.Register(new(cmd.ReadControl), debugGroup)
 	subcommands.Register(new(cmd.WriteControl), debugGroup)
 

runsc/cmd/BUILD

@@ -21,6 +21,7 @@ go_library(
         "install.go",
         "kill.go",
         "list.go",
+        "metric_export.go",
         "mitigate.go",
         "mitigate_extras.go",
         "path.go",
@@ -50,6 +51,8 @@ go_library(
         "//pkg/coretag",
         "//pkg/coverage",
         "//pkg/log",
+        "//pkg/metric",
+        "//pkg/prometheus",
         "//pkg/sentry/control",
         "//pkg/sentry/kernel",
         "//pkg/sentry/kernel/auth",

runsc/cmd/boot.go

@@ -29,6 +29,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/coretag"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/metric"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
 	"gvisor.dev/gvisor/runsc/boot"
 	"gvisor.dev/gvisor/runsc/cmd/util"
@@ -365,6 +366,12 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		}
 	}
 
+	// Prepare metrics.
+	// This needs to happen after the kernel is initialized (such that all metrics are registered)
+	// but before the start-sync file is notified, as the parent process needs to query for
+	// registered metrics prior to sending the start signal.
+	metric.Initialize()
+
 	// Notify the parent process the sandbox has booted (and that the controller
 	// is up).
 	startSyncFile := os.NewFile(uintptr(b.startSyncFD), "start-sync file")

runsc/cmd/metric_export.go

@@ -0,0 +1,91 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/google/subcommands"
+	"gvisor.dev/gvisor/pkg/prometheus"
+	"gvisor.dev/gvisor/runsc/cmd/util"
+	"gvisor.dev/gvisor/runsc/config"
+	"gvisor.dev/gvisor/runsc/container"
+	"gvisor.dev/gvisor/runsc/flag"
+)
+
+// MetricExport implements subcommands.Command for the "metric-export" command.
+type MetricExport struct {
+}
+
+// Name implements subcommands.Command.Name.
+func (*MetricExport) Name() string {
+	return "export-metrics"
+}
+
+// Synopsis implements subcommands.Command.Synopsis.
+func (*MetricExport) Synopsis() string {
+	return "export metric data for the sandbox"
+}
+
+// Usage implements subcommands.Command.Usage.
+func (*MetricExport) Usage() string {
+	return `export-metrics <container id> - prints sandbox metric data in Prometheus metric format`
+}
+
+// SetFlags implements subcommands.Command.SetFlags.
+func (m *MetricExport) SetFlags(f *flag.FlagSet) {
+}
+
+// Execute implements subcommands.Command.Execute.
+func (m *MetricExport) Execute(ctx context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus {
+	if f.NArg() < 1 {
+		f.Usage()
+		return subcommands.ExitUsageError
+	}
+
+	id := f.Arg(0)
+	conf := args[0].(*config.Config)
+
+	cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{})
+	if err != nil {
+		util.Fatalf("loading container: %v", err)
+	}
+
+	snapshot, err := cont.Sandbox.ExportMetrics()
+	if err != nil {
+		util.Fatalf("ExportMetrics failed: %v", err)
+	}
+	bufWriter := bufio.NewWriter(os.Stdout)
+	written, err := snapshot.WriteTo(bufWriter, prometheus.ExportOptions{
+		CommentHeader:  fmt.Sprintf("Command-line export for sandbox %s owning container %s", cont.Sandbox.ID, id),
+		ExporterPrefix: conf.MetricExporterPrefix,
+		ExtraLabels: map[string]string{
+			"sandbox":   cont.Sandbox.ID,
+			"container": cont.ID,
+		},
+	})
+	if err != nil {
+		util.Fatalf("Cannot write metrics to stdout: %v", err)
+	}
+	if err = bufWriter.Flush(); err != nil {
+		util.Fatalf("Cannot flush metrics to stdout: %v", err)
+	}
+	util.Infof("Wrote %d bytes of Prometheus metric data to stdout", written)
+
+	return subcommands.ExitSuccess
+}

runsc/config/config.go

@@ -139,6 +139,11 @@ type Config struct {
 	// If unset, a sane platform-specific default will be used.
 	PlatformDevicePath string `flag:"platform_device_path"`
 
+	// MetricExporterPrefix is added as prefix to all metric names.
+	// It is used to follow Prometheus's exporter convention, whereby all metric names should be
+	// prefixed by a name meaningfully identifying the software exporting the metric.
+	MetricExporterPrefix string `flag:"metric-exporter-prefix"`
+
 	// Strace indicates that strace should be enabled.
 	Strace bool `flag:"strace"`
 

runsc/config/flags.go

@@ -51,6 +51,9 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.Bool("allow-flag-override", false, "allow OCI annotations (dev.gvisor.flag.<name>) to override flags for debugging.")
 	flagSet.String("traceback", "system", "golang runtime's traceback level")
 
+	// Metrics flags.
+	flagSet.String("metric-exporter-prefix", "runsc_", "prefix for all metric names, following Prometheus exporter convention")
+
 	// Debugging flags: strace related
 	flagSet.Bool("strace", false, "enable strace.")
 	flagSet.String("strace-syscalls", "", "comma-separated list of syscalls to trace. If --strace is true and this list is empty, then all syscalls will be traced.")

runsc/sandbox/BUILD

@@ -23,6 +23,7 @@ go_library(
         "//pkg/control/server",
         "//pkg/coverage",
         "//pkg/log",
+        "//pkg/prometheus",
         "//pkg/sentry/control",
         "//pkg/sentry/platform",
         "//pkg/sentry/seccheck",

runsc/sandbox/sandbox.go

@@ -39,6 +39,7 @@ import (
 	"gvisor.dev/gvisor/pkg/control/server"
 	"gvisor.dev/gvisor/pkg/coverage"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/prometheus"
 	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
 	"gvisor.dev/gvisor/pkg/sentry/seccheck"
@@ -1178,6 +1179,20 @@ func (s *Sandbox) Reduce(wait bool) error {
 	}, nil)
 }
 
+// ExportMetrics writes Prometheus-formatted metrics data to the given io.Writer.
+func (s *Sandbox) ExportMetrics() (*prometheus.Snapshot, error) {
+	conn, err := s.sandboxConnect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	data := &control.MetricsExportData{}
+	if err = conn.Call(boot.MetricsExport, &control.MetricsExportOpts{}, data); err != nil {
+		return nil, err
+	}
+	return data.Snapshot, nil
+}
+
 // IsRunning returns true if the sandbox or gofer process is running.
 func (s *Sandbox) IsRunning() bool {
 	pid := s.Pid.load()