Add sandbox metadata to runsc metrics.

This adds static configuration information to what's exported about each
sandbox. In turn, this allows correlating Sentry metrics against salient
parts of its configuration (e.g. platform, network mode, etc.).

This is exported as key-value labels. This is a bit awkward, but is the only
option we have because Prometheus doesn't support string-typed metrics.

This change is part of a series of changes to support Prometheus-style metrics
in `runsc`.

Sample output (from `container_test`, newlines added manually for readability):

```
# HELP testmetric_meta_sandbox_metadata Key-value pairs about per-sandbox
# metadata.
# TYPE testmetric_meta_sandbox_metadata gauge
testmetric_meta_sandbox_metadata{coretags="false",
  iterationid="8729125096080520243",
  namespace="foons",
  network="none",
  platform="ptrace",
  pod="foopod",
  sandbox="test-container-RR2PYJOS5376N6S7SLA4KL6NQJOJVEHG"} 1 1672959320454
```

PiperOrigin-RevId: 504403468

runsc/cmd/metric_server.go

@@ -79,6 +79,11 @@ type servedSandbox struct {
 	// Once set, it is immutable.
 	sandbox *sandbox.Sandbox
 
+	// labelsWithMetadata is the union of `extraLabels` and `sandbox.MetricMetadata`.
+	// This is exported as the set of labels for the `sandbox_metadata` metric.
+	// Once set, it is immutable.
+	labelsWithMetadata map[string]string
+
 	// verifier allows verifying the data integrity of the metrics we get from this sandbox.
 	// It is not always initialized when the sandbox is discovered, but rather upon first metrics
 	// access to the sandbox. Metric registration data is loaded from the root container's
@@ -138,6 +143,13 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) {
 		}
 		s.verifier = verifier
 	}
+	s.labelsWithMetadata = make(map[string]string, len(s.extraLabels)+len(s.sandbox.MetricMetadata))
+	for k, v := range s.extraLabels {
+		s.labelsWithMetadata[k] = v
+	}
+	for k, v := range s.sandbox.MetricMetadata {
+		s.labelsWithMetadata[k] = v
+	}
 	return s.sandbox, s.verifier, nil
 }
 
@@ -408,6 +420,11 @@ var (
 		Type: prometheus.TypeGauge,
 		Help: "Boolean metric set to 1 for each running sandbox.",
 	}
+	sandboxMetadataMetric = prometheus.Metric{
+		Name: "sandbox_metadata",
+		Type: prometheus.TypeGauge,
+		Help: "Key-value pairs about per-sandbox metadata.",
+	}
 	numRunningSandboxesMetric = prometheus.Metric{
 		Name: "num_sandboxes_running",
 		Type: prometheus.TypeGauge,
@@ -541,6 +558,7 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht
 					metricsMu.Lock()
 					defer metricsMu.Unlock()
 					selfMetrics.Add(prometheus.LabeledIntData(&sandboxPresenceMetric, served.extraLabels, 1))
+					selfMetrics.Add(prometheus.LabeledIntData(&sandboxMetadataMetric, served.labelsWithMetadata, 1))
 					sandboxRunning := int64(0)
 					if isRunning {
 						sandboxRunning = 1

runsc/config/config.go

@@ -19,6 +19,7 @@ package config
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
@@ -351,6 +352,16 @@ func (c *Config) GetOverlay2() Overlay2 {
 	return c.Overlay2
 }
 
+// MetricMetadata returns key-value pairs that are useful to include in metrics
+// exported about the sandbox this config represents.
+func (c *Config) MetricMetadata() map[string]string {
+	metadata := make(map[string]string)
+	metadata["platform"] = c.Platform
+	metadata["network"] = c.Network.String()
+	metadata["coretags"] = strconv.FormatBool(c.EnableCoreTags)
+	return metadata
+}
+
 // FileAccessType tells how the filesystem is accessed.
 type FileAccessType int
 

runsc/container/metric_server_test.go

@@ -141,6 +141,18 @@ func TestContainerMetrics(t *testing.T) {
 	if err != nil {
 		t.Errorf("Cannot get metrics after creating container: %v", err)
 	}
+	gotMetadata, err := initialData.GetSandboxMetadataMetric(metricclient.WantMetric{
+		Metric:    "testmetric_meta_sandbox_metadata",
+		Sandbox:   args.ID,
+		Pod:       "foopod",
+		Namespace: "foons",
+	})
+	if err != nil {
+		t.Errorf("Cannot get sandbox metadata: %v", err)
+	}
+	if gotMetadata["platform"] == "" || gotMetadata["platform"] != te.sleepConf.Platform {
+		t.Errorf("Invalid platform: Metric metadata says %v, config says %v", gotMetadata["platform"], te.sleepConf.Platform)
+	}
 	t.Logf("Metrics prior to container start:\n\n%s\n\n", initialData)
 	if err := cont.Start(te.sleepConf); err != nil {
 		t.Fatalf("Cannot start container: %v", err)

runsc/sandbox/sandbox.go

@@ -163,6 +163,12 @@ type Sandbox struct {
 	// created.
 	RegisteredMetrics *metricpb.MetricRegistration `json:"registeredMetrics"`
 
+	// MetricMetadata are key-value pairs that are useful to export about this
+	// sandbox, but not part of the set of labels that uniquely identify it.
+	// They are static once initialized, and typically contain high-level
+	// configuration information about the sandbox.
+	MetricMetadata map[string]string `json:"metricMetadata"`
+
 	// MetricServerAddress is the address of the metric server that this sandbox
 	// intends to export metrics for.
 	// Only populated if exporting metrics was requested when the sandbox was
@@ -247,6 +253,7 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) {
 		},
 		UID:                 -1, // prevent usage before it's set.
 		GID:                 -1, // prevent usage before it's set.
+		MetricMetadata:      conf.MetricMetadata(),
 		MetricServerAddress: conf.MetricServer,
 	}
 	if args.Spec != nil && args.Spec.Annotations != nil {

test/metricclient/BUILD

@@ -14,6 +14,7 @@ go_library(
     deps = [
         "//pkg/sync",
         "//runsc/config",
+        "//runsc/sandbox",
         "//runsc/specutils",
         "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_prometheus_common//expfmt",

test/metricclient/metricclient.go

@@ -36,6 +36,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/sync"
 	"gvisor.dev/gvisor/runsc/config"
+	"gvisor.dev/gvisor/runsc/sandbox"
 	"gvisor.dev/gvisor/runsc/specutils"
 )
 
@@ -359,3 +360,53 @@ func (m MetricData) GetPrometheusContainerInteger(want WantMetric) (int64, time.
 	}
 	return m.GetPrometheusInteger(want.Metric, labels)
 }
+
+// GetSandboxMetadataMetric returns the labels attached to the metadata metric for a given sandbox.
+func (m MetricData) GetSandboxMetadataMetric(want WantMetric) (map[string]string, error) {
+	var buf bytes.Buffer
+	buf.WriteString(string(m))
+	parsed, err := (&expfmt.TextParser{}).TextToMetricFamilies(&buf)
+	if err != nil {
+		return nil, err
+	}
+	metricData, found := parsed[want.Metric]
+	if !found {
+		return nil, fmt.Errorf("metric %q not found", want.Metric)
+	}
+	foundIndex := -1
+	for i, data := range metricData.GetMetric() {
+		dataLabels := make(map[string]string, len(data.GetLabel()))
+		for _, label := range data.GetLabel() {
+			dataLabels[label.GetName()] = label.GetValue()
+		}
+		allMatching := true
+		for wantLabel, wantValue := range map[string]string{
+			sandbox.SandboxIDLabel: want.Sandbox,
+			sandbox.NamespaceLabel: want.Namespace,
+			sandbox.PodNameLabel:   want.Pod,
+		} {
+			if dataLabels[wantLabel] != wantValue {
+				allMatching = false
+				break
+			}
+		}
+		if allMatching {
+			if foundIndex != -1 {
+				return nil, errors.New("found multiple metadata metrics matching requested labels")
+			}
+			foundIndex = i
+		}
+	}
+	if foundIndex == -1 {
+		return nil, errors.New("no metadata metric matching requested labels")
+	}
+	data := metricData.GetMetric()[foundIndex]
+	metadataLabels := make(map[string]string, len(data.GetLabel()))
+	for _, label := range data.GetLabel() {
+		if label.GetName() == sandbox.SandboxIDLabel || label.GetName() == sandbox.NamespaceLabel || label.GetName() == sandbox.PodNameLabel {
+			continue
+		}
+		metadataLabels[label.GetName()] = label.GetValue()
+	}
+	return metadataLabels, nil
+}