From 01061a8f207de3985f1f9d94dbf793810ea15a88 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Tue, 27 Dec 2022 18:09:16 -0800 Subject: [PATCH] gVisor: Add `runsc metrics-export` subcommand. This subcommand prints a sandbox's instrumentation data in Prometheus format to stdout. This change is part of a series of changes to support Prometheus-style metrics in `runsc`. Doing so requires making several seemingly-odd design decisions, due to the following architectural constraints: - Prometheus requires an HTTP server serving the `/metrics` endpoint. - For performance reasons, the `runsc boot` process cannot run the `netpoller` goroutine. - Since we don't want to write our own HTTP server implementation, this means the HTTP endpoint has to be served by a separate process that remains running during the lifetime of the container. - The `runsc boot` process is untrusted. - This means we cannot trust metrics data that comes out of the Sentry. Therefore, there needs to be an elaborate dance where we pre-register metric metadata before starting any untrusted workload. Then, the server relaying the metric data must verify the validity of metric values against this metric metadata. This avoids leaking metrics, cardinality blow-ups, and other such DoS vectors. - This feature needs to be easy-to-use in a typical Docker setting. - This means having the ability to just say `--metrics-server=localhost:1337` in the `runsc` runtime entry in `/etc/docker/daemon.json` and have that Just Work(TM), even when multiple containers are running. - Since only one process may listen on a port at a given time, this means the metric server needs to be able to multiplex requests out to multiple running sandboxes, and remain alive for the entire duration of either of these sandboxes. However, it should also die when there are no sandboxes, so that we don't end up with leftover metric servers lying around. - For this reason, the metrics server runs *outside* of the usual per-container cgroups. - This also saves system resources by not running one server per sandbox. - The metrics server must be exposed to the outside world, and cannot assume that its clients are trustworthy. - For this reason, a metrics server is bound to a runtime root directory, and double-checks all that the sandboxes it is asked to follow actually exist in this root directory. PiperOrigin-RevId: 498067941 --- pkg/sentry/control/BUILD | 3 ++ pkg/sentry/control/metrics.go | 37 ++++++++++++++ runsc/boot/controller.go | 6 +++ runsc/cli/main.go | 1 + runsc/cmd/BUILD | 3 ++ runsc/cmd/boot.go | 7 +++ runsc/cmd/metric_export.go | 91 +++++++++++++++++++++++++++++++++++ runsc/config/config.go | 5 ++ runsc/config/flags.go | 3 ++ runsc/sandbox/BUILD | 1 + runsc/sandbox/sandbox.go | 15 ++++++ 11 files changed, 172 insertions(+) create mode 100644 pkg/sentry/control/metrics.go create mode 100644 runsc/cmd/metric_export.go diff --git a/pkg/sentry/control/BUILD b/pkg/sentry/control/BUILD index 28174782d..430eabe4a 100644 --- a/pkg/sentry/control/BUILD +++ b/pkg/sentry/control/BUILD @@ -20,6 +20,7 @@ go_library( "fs.go", "lifecycle.go", "logging.go", + "metrics.go", "pprof.go", "proc.go", "state.go", @@ -36,6 +37,8 @@ go_library( "//pkg/fd", "//pkg/fspath", "//pkg/log", + "//pkg/metric", + "//pkg/prometheus", "//pkg/sentry/fdimport", "//pkg/sentry/fsimpl/host", "//pkg/sentry/fsimpl/user", diff --git a/pkg/sentry/control/metrics.go b/pkg/sentry/control/metrics.go new file mode 100644 index 000000000..801e86a11 --- /dev/null +++ b/pkg/sentry/control/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package control + +import ( + "gvisor.dev/gvisor/pkg/metric" + "gvisor.dev/gvisor/pkg/prometheus" +) + +// Metrics includes metrics-related RPC stubs. +type Metrics struct{} + +// MetricsExportOpts contains metric exporting options. +type MetricsExportOpts struct{} + +// MetricsExportData contains data for all metrics being exported. +type MetricsExportData struct { + Snapshot *prometheus.Snapshot `json:"snapshot"` +} + +// Export export metrics data into MetricsExportData. +func (u *Metrics) Export(_ *MetricsExportOpts, out *MetricsExportData) error { + out.Snapshot = metric.GetSnapshot() + return nil +} diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index 11819403b..bc71e7be6 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -129,6 +129,11 @@ const ( UsageReduce = "Usage.Reduce" ) +// Metrics related commands (see metrics.go). +const ( + MetricsExport = "Metrics.Export" +) + // Commands for interacting with cgroupfs within the sandbox. const ( CgroupsReadControlFiles = "Cgroups.ReadControlFiles" @@ -173,6 +178,7 @@ func newController(fd int, l *Loader) (*controller, error) { ctrl.srv.Register(&control.Proc{Kernel: l.k}) ctrl.srv.Register(&control.State{Kernel: l.k}) ctrl.srv.Register(&control.Usage{Kernel: l.k}) + ctrl.srv.Register(&control.Metrics{}) ctrl.srv.Register(&debug{}) if eps, ok := l.k.RootNetworkNamespace().Stack().(*netstack.Stack); ok { diff --git a/runsc/cli/main.go b/runsc/cli/main.go index b6f43a42e..bb8f161d5 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -94,6 +94,7 @@ func Main(version string) { subcommands.Register(new(cmd.Statefile), debugGroup) subcommands.Register(new(cmd.Symbolize), debugGroup) subcommands.Register(new(cmd.Usage), debugGroup) + subcommands.Register(new(cmd.MetricExport), debugGroup) subcommands.Register(new(cmd.ReadControl), debugGroup) subcommands.Register(new(cmd.WriteControl), debugGroup) diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD index 0cbb45dde..7fc4f150c 100644 --- a/runsc/cmd/BUILD +++ b/runsc/cmd/BUILD @@ -21,6 +21,7 @@ go_library( "install.go", "kill.go", "list.go", + "metric_export.go", "mitigate.go", "mitigate_extras.go", "path.go", @@ -50,6 +51,8 @@ go_library( "//pkg/coretag", "//pkg/coverage", "//pkg/log", + "//pkg/metric", + "//pkg/prometheus", "//pkg/sentry/control", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 520b9c156..aff98e531 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -29,6 +29,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/coretag" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/metric" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/cmd/util" @@ -365,6 +366,12 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } + // Prepare metrics. + // This needs to happen after the kernel is initialized (such that all metrics are registered) + // but before the start-sync file is notified, as the parent process needs to query for + // registered metrics prior to sending the start signal. + metric.Initialize() + // Notify the parent process the sandbox has booted (and that the controller // is up). startSyncFile := os.NewFile(uintptr(b.startSyncFD), "start-sync file") diff --git a/runsc/cmd/metric_export.go b/runsc/cmd/metric_export.go new file mode 100644 index 000000000..bc704d259 --- /dev/null +++ b/runsc/cmd/metric_export.go @@ -0,0 +1,91 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bufio" + "context" + "fmt" + "os" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/pkg/prometheus" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/flag" +) + +// MetricExport implements subcommands.Command for the "metric-export" command. +type MetricExport struct { +} + +// Name implements subcommands.Command.Name. +func (*MetricExport) Name() string { + return "export-metrics" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*MetricExport) Synopsis() string { + return "export metric data for the sandbox" +} + +// Usage implements subcommands.Command.Usage. +func (*MetricExport) Usage() string { + return `export-metrics - prints sandbox metric data in Prometheus metric format` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (m *MetricExport) SetFlags(f *flag.FlagSet) { +} + +// Execute implements subcommands.Command.Execute. +func (m *MetricExport) Execute(ctx context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if f.NArg() < 1 { + f.Usage() + return subcommands.ExitUsageError + } + + id := f.Arg(0) + conf := args[0].(*config.Config) + + cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{}) + if err != nil { + util.Fatalf("loading container: %v", err) + } + + snapshot, err := cont.Sandbox.ExportMetrics() + if err != nil { + util.Fatalf("ExportMetrics failed: %v", err) + } + bufWriter := bufio.NewWriter(os.Stdout) + written, err := snapshot.WriteTo(bufWriter, prometheus.ExportOptions{ + CommentHeader: fmt.Sprintf("Command-line export for sandbox %s owning container %s", cont.Sandbox.ID, id), + ExporterPrefix: conf.MetricExporterPrefix, + ExtraLabels: map[string]string{ + "sandbox": cont.Sandbox.ID, + "container": cont.ID, + }, + }) + if err != nil { + util.Fatalf("Cannot write metrics to stdout: %v", err) + } + if err = bufWriter.Flush(); err != nil { + util.Fatalf("Cannot flush metrics to stdout: %v", err) + } + util.Infof("Wrote %d bytes of Prometheus metric data to stdout", written) + + return subcommands.ExitSuccess +} diff --git a/runsc/config/config.go b/runsc/config/config.go index 90acae375..c43dbe7c9 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -139,6 +139,11 @@ type Config struct { // If unset, a sane platform-specific default will be used. PlatformDevicePath string `flag:"platform_device_path"` + // MetricExporterPrefix is added as prefix to all metric names. + // It is used to follow Prometheus's exporter convention, whereby all metric names should be + // prefixed by a name meaningfully identifying the software exporting the metric. + MetricExporterPrefix string `flag:"metric-exporter-prefix"` + // Strace indicates that strace should be enabled. Strace bool `flag:"strace"` diff --git a/runsc/config/flags.go b/runsc/config/flags.go index b226c9925..3df557f3b 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -51,6 +51,9 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("allow-flag-override", false, "allow OCI annotations (dev.gvisor.flag.) to override flags for debugging.") flagSet.String("traceback", "system", "golang runtime's traceback level") + // Metrics flags. + flagSet.String("metric-exporter-prefix", "runsc_", "prefix for all metric names, following Prometheus exporter convention") + // Debugging flags: strace related flagSet.Bool("strace", false, "enable strace.") flagSet.String("strace-syscalls", "", "comma-separated list of syscalls to trace. If --strace is true and this list is empty, then all syscalls will be traced.") diff --git a/runsc/sandbox/BUILD b/runsc/sandbox/BUILD index 3e08d0a9c..6f0420f93 100644 --- a/runsc/sandbox/BUILD +++ b/runsc/sandbox/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/control/server", "//pkg/coverage", "//pkg/log", + "//pkg/prometheus", "//pkg/sentry/control", "//pkg/sentry/platform", "//pkg/sentry/seccheck", diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index c56a9454e..5cf222b7d 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -39,6 +39,7 @@ import ( "gvisor.dev/gvisor/pkg/control/server" "gvisor.dev/gvisor/pkg/coverage" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/prometheus" "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/seccheck" @@ -1178,6 +1179,20 @@ func (s *Sandbox) Reduce(wait bool) error { }, nil) } +// ExportMetrics writes Prometheus-formatted metrics data to the given io.Writer. +func (s *Sandbox) ExportMetrics() (*prometheus.Snapshot, error) { + conn, err := s.sandboxConnect() + if err != nil { + return nil, err + } + defer conn.Close() + data := &control.MetricsExportData{} + if err = conn.Call(boot.MetricsExport, &control.MetricsExportOpts{}, data); err != nil { + return nil, err + } + return data.Snapshot, nil +} + // IsRunning returns true if the sandbox or gofer process is running. func (s *Sandbox) IsRunning() bool { pid := s.Pid.load()