diff --git a/pkg/sentry/control/BUILD b/pkg/sentry/control/BUILD index 28174782d..430eabe4a 100644 --- a/pkg/sentry/control/BUILD +++ b/pkg/sentry/control/BUILD @@ -20,6 +20,7 @@ go_library( "fs.go", "lifecycle.go", "logging.go", + "metrics.go", "pprof.go", "proc.go", "state.go", @@ -36,6 +37,8 @@ go_library( "//pkg/fd", "//pkg/fspath", "//pkg/log", + "//pkg/metric", + "//pkg/prometheus", "//pkg/sentry/fdimport", "//pkg/sentry/fsimpl/host", "//pkg/sentry/fsimpl/user", diff --git a/pkg/sentry/control/metrics.go b/pkg/sentry/control/metrics.go new file mode 100644 index 000000000..801e86a11 --- /dev/null +++ b/pkg/sentry/control/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package control + +import ( + "gvisor.dev/gvisor/pkg/metric" + "gvisor.dev/gvisor/pkg/prometheus" +) + +// Metrics includes metrics-related RPC stubs. +type Metrics struct{} + +// MetricsExportOpts contains metric exporting options. +type MetricsExportOpts struct{} + +// MetricsExportData contains data for all metrics being exported. +type MetricsExportData struct { + Snapshot *prometheus.Snapshot `json:"snapshot"` +} + +// Export export metrics data into MetricsExportData. +func (u *Metrics) Export(_ *MetricsExportOpts, out *MetricsExportData) error { + out.Snapshot = metric.GetSnapshot() + return nil +} diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index 11819403b..bc71e7be6 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -129,6 +129,11 @@ const ( UsageReduce = "Usage.Reduce" ) +// Metrics related commands (see metrics.go). +const ( + MetricsExport = "Metrics.Export" +) + // Commands for interacting with cgroupfs within the sandbox. const ( CgroupsReadControlFiles = "Cgroups.ReadControlFiles" @@ -173,6 +178,7 @@ func newController(fd int, l *Loader) (*controller, error) { ctrl.srv.Register(&control.Proc{Kernel: l.k}) ctrl.srv.Register(&control.State{Kernel: l.k}) ctrl.srv.Register(&control.Usage{Kernel: l.k}) + ctrl.srv.Register(&control.Metrics{}) ctrl.srv.Register(&debug{}) if eps, ok := l.k.RootNetworkNamespace().Stack().(*netstack.Stack); ok { diff --git a/runsc/cli/main.go b/runsc/cli/main.go index b6f43a42e..bb8f161d5 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -94,6 +94,7 @@ func Main(version string) { subcommands.Register(new(cmd.Statefile), debugGroup) subcommands.Register(new(cmd.Symbolize), debugGroup) subcommands.Register(new(cmd.Usage), debugGroup) + subcommands.Register(new(cmd.MetricExport), debugGroup) subcommands.Register(new(cmd.ReadControl), debugGroup) subcommands.Register(new(cmd.WriteControl), debugGroup) diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD index 0cbb45dde..7fc4f150c 100644 --- a/runsc/cmd/BUILD +++ b/runsc/cmd/BUILD @@ -21,6 +21,7 @@ go_library( "install.go", "kill.go", "list.go", + "metric_export.go", "mitigate.go", "mitigate_extras.go", "path.go", @@ -50,6 +51,8 @@ go_library( "//pkg/coretag", "//pkg/coverage", "//pkg/log", + "//pkg/metric", + "//pkg/prometheus", "//pkg/sentry/control", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index 520b9c156..aff98e531 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -29,6 +29,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/coretag" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/metric" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/cmd/util" @@ -365,6 +366,12 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma } } + // Prepare metrics. + // This needs to happen after the kernel is initialized (such that all metrics are registered) + // but before the start-sync file is notified, as the parent process needs to query for + // registered metrics prior to sending the start signal. + metric.Initialize() + // Notify the parent process the sandbox has booted (and that the controller // is up). startSyncFile := os.NewFile(uintptr(b.startSyncFD), "start-sync file") diff --git a/runsc/cmd/metric_export.go b/runsc/cmd/metric_export.go new file mode 100644 index 000000000..bc704d259 --- /dev/null +++ b/runsc/cmd/metric_export.go @@ -0,0 +1,91 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bufio" + "context" + "fmt" + "os" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/pkg/prometheus" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/flag" +) + +// MetricExport implements subcommands.Command for the "metric-export" command. +type MetricExport struct { +} + +// Name implements subcommands.Command.Name. +func (*MetricExport) Name() string { + return "export-metrics" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*MetricExport) Synopsis() string { + return "export metric data for the sandbox" +} + +// Usage implements subcommands.Command.Usage. +func (*MetricExport) Usage() string { + return `export-metrics - prints sandbox metric data in Prometheus metric format` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (m *MetricExport) SetFlags(f *flag.FlagSet) { +} + +// Execute implements subcommands.Command.Execute. +func (m *MetricExport) Execute(ctx context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if f.NArg() < 1 { + f.Usage() + return subcommands.ExitUsageError + } + + id := f.Arg(0) + conf := args[0].(*config.Config) + + cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, container.LoadOpts{}) + if err != nil { + util.Fatalf("loading container: %v", err) + } + + snapshot, err := cont.Sandbox.ExportMetrics() + if err != nil { + util.Fatalf("ExportMetrics failed: %v", err) + } + bufWriter := bufio.NewWriter(os.Stdout) + written, err := snapshot.WriteTo(bufWriter, prometheus.ExportOptions{ + CommentHeader: fmt.Sprintf("Command-line export for sandbox %s owning container %s", cont.Sandbox.ID, id), + ExporterPrefix: conf.MetricExporterPrefix, + ExtraLabels: map[string]string{ + "sandbox": cont.Sandbox.ID, + "container": cont.ID, + }, + }) + if err != nil { + util.Fatalf("Cannot write metrics to stdout: %v", err) + } + if err = bufWriter.Flush(); err != nil { + util.Fatalf("Cannot flush metrics to stdout: %v", err) + } + util.Infof("Wrote %d bytes of Prometheus metric data to stdout", written) + + return subcommands.ExitSuccess +} diff --git a/runsc/config/config.go b/runsc/config/config.go index 90acae375..c43dbe7c9 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -139,6 +139,11 @@ type Config struct { // If unset, a sane platform-specific default will be used. PlatformDevicePath string `flag:"platform_device_path"` + // MetricExporterPrefix is added as prefix to all metric names. + // It is used to follow Prometheus's exporter convention, whereby all metric names should be + // prefixed by a name meaningfully identifying the software exporting the metric. + MetricExporterPrefix string `flag:"metric-exporter-prefix"` + // Strace indicates that strace should be enabled. Strace bool `flag:"strace"` diff --git a/runsc/config/flags.go b/runsc/config/flags.go index b226c9925..3df557f3b 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -51,6 +51,9 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("allow-flag-override", false, "allow OCI annotations (dev.gvisor.flag.) to override flags for debugging.") flagSet.String("traceback", "system", "golang runtime's traceback level") + // Metrics flags. + flagSet.String("metric-exporter-prefix", "runsc_", "prefix for all metric names, following Prometheus exporter convention") + // Debugging flags: strace related flagSet.Bool("strace", false, "enable strace.") flagSet.String("strace-syscalls", "", "comma-separated list of syscalls to trace. If --strace is true and this list is empty, then all syscalls will be traced.") diff --git a/runsc/sandbox/BUILD b/runsc/sandbox/BUILD index 3e08d0a9c..6f0420f93 100644 --- a/runsc/sandbox/BUILD +++ b/runsc/sandbox/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/control/server", "//pkg/coverage", "//pkg/log", + "//pkg/prometheus", "//pkg/sentry/control", "//pkg/sentry/platform", "//pkg/sentry/seccheck", diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index c56a9454e..5cf222b7d 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -39,6 +39,7 @@ import ( "gvisor.dev/gvisor/pkg/control/server" "gvisor.dev/gvisor/pkg/coverage" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/prometheus" "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/seccheck" @@ -1178,6 +1179,20 @@ func (s *Sandbox) Reduce(wait bool) error { }, nil) } +// ExportMetrics writes Prometheus-formatted metrics data to the given io.Writer. +func (s *Sandbox) ExportMetrics() (*prometheus.Snapshot, error) { + conn, err := s.sandboxConnect() + if err != nil { + return nil, err + } + defer conn.Close() + data := &control.MetricsExportData{} + if err = conn.Call(boot.MetricsExport, &control.MetricsExportOpts{}, data); err != nil { + return nil, err + } + return data.Snapshot, nil +} + // IsRunning returns true if the sandbox or gofer process is running. func (s *Sandbox) IsRunning() bool { pid := s.Pid.load()