From 1e4d19665c234bc7c00e366dbc548838cf058f92 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Tue, 24 Jan 2023 16:18:08 -0800 Subject: [PATCH] Add sandbox metadata to `runsc` metrics. This adds static configuration information to what's exported about each sandbox. In turn, this allows correlating Sentry metrics against salient parts of its configuration (e.g. platform, network mode, etc.). This is exported as key-value labels. This is a bit awkward, but is the only option we have because Prometheus doesn't support string-typed metrics. This change is part of a series of changes to support Prometheus-style metrics in `runsc`. Sample output (from `container_test`, newlines added manually for readability): ``` # HELP testmetric_meta_sandbox_metadata Key-value pairs about per-sandbox # metadata. # TYPE testmetric_meta_sandbox_metadata gauge testmetric_meta_sandbox_metadata{coretags="false", iterationid="8729125096080520243", namespace="foons", network="none", platform="ptrace", pod="foopod", sandbox="test-container-RR2PYJOS5376N6S7SLA4KL6NQJOJVEHG"} 1 1672959320454 ``` PiperOrigin-RevId: 504403468 --- runsc/cmd/metric_server.go | 18 ++++++++++ runsc/config/config.go | 11 ++++++ runsc/container/metric_server_test.go | 12 +++++++ runsc/sandbox/sandbox.go | 7 ++++ test/metricclient/BUILD | 1 + test/metricclient/metricclient.go | 51 +++++++++++++++++++++++++++ 6 files changed, 100 insertions(+) diff --git a/runsc/cmd/metric_server.go b/runsc/cmd/metric_server.go index e2f4948b7..3fa05adf2 100644 --- a/runsc/cmd/metric_server.go +++ b/runsc/cmd/metric_server.go @@ -79,6 +79,11 @@ type servedSandbox struct { // Once set, it is immutable. sandbox *sandbox.Sandbox + // labelsWithMetadata is the union of `extraLabels` and `sandbox.MetricMetadata`. + // This is exported as the set of labels for the `sandbox_metadata` metric. + // Once set, it is immutable. + labelsWithMetadata map[string]string + // verifier allows verifying the data integrity of the metrics we get from this sandbox. // It is not always initialized when the sandbox is discovered, but rather upon first metrics // access to the sandbox. Metric registration data is loaded from the root container's @@ -138,6 +143,13 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) { } s.verifier = verifier } + s.labelsWithMetadata = make(map[string]string, len(s.extraLabels)+len(s.sandbox.MetricMetadata)) + for k, v := range s.extraLabels { + s.labelsWithMetadata[k] = v + } + for k, v := range s.sandbox.MetricMetadata { + s.labelsWithMetadata[k] = v + } return s.sandbox, s.verifier, nil } @@ -408,6 +420,11 @@ var ( Type: prometheus.TypeGauge, Help: "Boolean metric set to 1 for each running sandbox.", } + sandboxMetadataMetric = prometheus.Metric{ + Name: "sandbox_metadata", + Type: prometheus.TypeGauge, + Help: "Key-value pairs about per-sandbox metadata.", + } numRunningSandboxesMetric = prometheus.Metric{ Name: "num_sandboxes_running", Type: prometheus.TypeGauge, @@ -541,6 +558,7 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht metricsMu.Lock() defer metricsMu.Unlock() selfMetrics.Add(prometheus.LabeledIntData(&sandboxPresenceMetric, served.extraLabels, 1)) + selfMetrics.Add(prometheus.LabeledIntData(&sandboxMetadataMetric, served.labelsWithMetadata, 1)) sandboxRunning := int64(0) if isRunning { sandboxRunning = 1 diff --git a/runsc/config/config.go b/runsc/config/config.go index b0d3bb599..693191d0b 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -19,6 +19,7 @@ package config import ( "fmt" + "strconv" "strings" "time" @@ -351,6 +352,16 @@ func (c *Config) GetOverlay2() Overlay2 { return c.Overlay2 } +// MetricMetadata returns key-value pairs that are useful to include in metrics +// exported about the sandbox this config represents. +func (c *Config) MetricMetadata() map[string]string { + metadata := make(map[string]string) + metadata["platform"] = c.Platform + metadata["network"] = c.Network.String() + metadata["coretags"] = strconv.FormatBool(c.EnableCoreTags) + return metadata +} + // FileAccessType tells how the filesystem is accessed. type FileAccessType int diff --git a/runsc/container/metric_server_test.go b/runsc/container/metric_server_test.go index a4a3424af..aeafaf1af 100644 --- a/runsc/container/metric_server_test.go +++ b/runsc/container/metric_server_test.go @@ -141,6 +141,18 @@ func TestContainerMetrics(t *testing.T) { if err != nil { t.Errorf("Cannot get metrics after creating container: %v", err) } + gotMetadata, err := initialData.GetSandboxMetadataMetric(metricclient.WantMetric{ + Metric: "testmetric_meta_sandbox_metadata", + Sandbox: args.ID, + Pod: "foopod", + Namespace: "foons", + }) + if err != nil { + t.Errorf("Cannot get sandbox metadata: %v", err) + } + if gotMetadata["platform"] == "" || gotMetadata["platform"] != te.sleepConf.Platform { + t.Errorf("Invalid platform: Metric metadata says %v, config says %v", gotMetadata["platform"], te.sleepConf.Platform) + } t.Logf("Metrics prior to container start:\n\n%s\n\n", initialData) if err := cont.Start(te.sleepConf); err != nil { t.Fatalf("Cannot start container: %v", err) diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 5a2525d12..43bef6efc 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -163,6 +163,12 @@ type Sandbox struct { // created. RegisteredMetrics *metricpb.MetricRegistration `json:"registeredMetrics"` + // MetricMetadata are key-value pairs that are useful to export about this + // sandbox, but not part of the set of labels that uniquely identify it. + // They are static once initialized, and typically contain high-level + // configuration information about the sandbox. + MetricMetadata map[string]string `json:"metricMetadata"` + // MetricServerAddress is the address of the metric server that this sandbox // intends to export metrics for. // Only populated if exporting metrics was requested when the sandbox was @@ -247,6 +253,7 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) { }, UID: -1, // prevent usage before it's set. GID: -1, // prevent usage before it's set. + MetricMetadata: conf.MetricMetadata(), MetricServerAddress: conf.MetricServer, } if args.Spec != nil && args.Spec.Annotations != nil { diff --git a/test/metricclient/BUILD b/test/metricclient/BUILD index cf9b97080..c46aa5d7c 100644 --- a/test/metricclient/BUILD +++ b/test/metricclient/BUILD @@ -14,6 +14,7 @@ go_library( deps = [ "//pkg/sync", "//runsc/config", + "//runsc/sandbox", "//runsc/specutils", "@com_github_cenkalti_backoff//:go_default_library", "@com_github_prometheus_common//expfmt", diff --git a/test/metricclient/metricclient.go b/test/metricclient/metricclient.go index 34fc37646..c702ad164 100644 --- a/test/metricclient/metricclient.go +++ b/test/metricclient/metricclient.go @@ -36,6 +36,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/sandbox" "gvisor.dev/gvisor/runsc/specutils" ) @@ -359,3 +360,53 @@ func (m MetricData) GetPrometheusContainerInteger(want WantMetric) (int64, time. } return m.GetPrometheusInteger(want.Metric, labels) } + +// GetSandboxMetadataMetric returns the labels attached to the metadata metric for a given sandbox. +func (m MetricData) GetSandboxMetadataMetric(want WantMetric) (map[string]string, error) { + var buf bytes.Buffer + buf.WriteString(string(m)) + parsed, err := (&expfmt.TextParser{}).TextToMetricFamilies(&buf) + if err != nil { + return nil, err + } + metricData, found := parsed[want.Metric] + if !found { + return nil, fmt.Errorf("metric %q not found", want.Metric) + } + foundIndex := -1 + for i, data := range metricData.GetMetric() { + dataLabels := make(map[string]string, len(data.GetLabel())) + for _, label := range data.GetLabel() { + dataLabels[label.GetName()] = label.GetValue() + } + allMatching := true + for wantLabel, wantValue := range map[string]string{ + sandbox.SandboxIDLabel: want.Sandbox, + sandbox.NamespaceLabel: want.Namespace, + sandbox.PodNameLabel: want.Pod, + } { + if dataLabels[wantLabel] != wantValue { + allMatching = false + break + } + } + if allMatching { + if foundIndex != -1 { + return nil, errors.New("found multiple metadata metrics matching requested labels") + } + foundIndex = i + } + } + if foundIndex == -1 { + return nil, errors.New("no metadata metric matching requested labels") + } + data := metricData.GetMetric()[foundIndex] + metadataLabels := make(map[string]string, len(data.GetLabel())) + for _, label := range data.GetLabel() { + if label.GetName() == sandbox.SandboxIDLabel || label.GetName() == sandbox.NamespaceLabel || label.GetName() == sandbox.PodNameLabel { + continue + } + metadataLabels[label.GetName()] = label.GetValue() + } + return metadataLabels, nil +}