diff --git a/runsc/cmd/metric_server.go b/runsc/cmd/metric_server.go index e2f4948b7..3fa05adf2 100644 --- a/runsc/cmd/metric_server.go +++ b/runsc/cmd/metric_server.go @@ -79,6 +79,11 @@ type servedSandbox struct { // Once set, it is immutable. sandbox *sandbox.Sandbox + // labelsWithMetadata is the union of `extraLabels` and `sandbox.MetricMetadata`. + // This is exported as the set of labels for the `sandbox_metadata` metric. + // Once set, it is immutable. + labelsWithMetadata map[string]string + // verifier allows verifying the data integrity of the metrics we get from this sandbox. // It is not always initialized when the sandbox is discovered, but rather upon first metrics // access to the sandbox. Metric registration data is loaded from the root container's @@ -138,6 +143,13 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) { } s.verifier = verifier } + s.labelsWithMetadata = make(map[string]string, len(s.extraLabels)+len(s.sandbox.MetricMetadata)) + for k, v := range s.extraLabels { + s.labelsWithMetadata[k] = v + } + for k, v := range s.sandbox.MetricMetadata { + s.labelsWithMetadata[k] = v + } return s.sandbox, s.verifier, nil } @@ -408,6 +420,11 @@ var ( Type: prometheus.TypeGauge, Help: "Boolean metric set to 1 for each running sandbox.", } + sandboxMetadataMetric = prometheus.Metric{ + Name: "sandbox_metadata", + Type: prometheus.TypeGauge, + Help: "Key-value pairs about per-sandbox metadata.", + } numRunningSandboxesMetric = prometheus.Metric{ Name: "num_sandboxes_running", Type: prometheus.TypeGauge, @@ -541,6 +558,7 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht metricsMu.Lock() defer metricsMu.Unlock() selfMetrics.Add(prometheus.LabeledIntData(&sandboxPresenceMetric, served.extraLabels, 1)) + selfMetrics.Add(prometheus.LabeledIntData(&sandboxMetadataMetric, served.labelsWithMetadata, 1)) sandboxRunning := int64(0) if isRunning { sandboxRunning = 1 diff --git a/runsc/config/config.go b/runsc/config/config.go index b0d3bb599..693191d0b 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -19,6 +19,7 @@ package config import ( "fmt" + "strconv" "strings" "time" @@ -351,6 +352,16 @@ func (c *Config) GetOverlay2() Overlay2 { return c.Overlay2 } +// MetricMetadata returns key-value pairs that are useful to include in metrics +// exported about the sandbox this config represents. +func (c *Config) MetricMetadata() map[string]string { + metadata := make(map[string]string) + metadata["platform"] = c.Platform + metadata["network"] = c.Network.String() + metadata["coretags"] = strconv.FormatBool(c.EnableCoreTags) + return metadata +} + // FileAccessType tells how the filesystem is accessed. type FileAccessType int diff --git a/runsc/container/metric_server_test.go b/runsc/container/metric_server_test.go index a4a3424af..aeafaf1af 100644 --- a/runsc/container/metric_server_test.go +++ b/runsc/container/metric_server_test.go @@ -141,6 +141,18 @@ func TestContainerMetrics(t *testing.T) { if err != nil { t.Errorf("Cannot get metrics after creating container: %v", err) } + gotMetadata, err := initialData.GetSandboxMetadataMetric(metricclient.WantMetric{ + Metric: "testmetric_meta_sandbox_metadata", + Sandbox: args.ID, + Pod: "foopod", + Namespace: "foons", + }) + if err != nil { + t.Errorf("Cannot get sandbox metadata: %v", err) + } + if gotMetadata["platform"] == "" || gotMetadata["platform"] != te.sleepConf.Platform { + t.Errorf("Invalid platform: Metric metadata says %v, config says %v", gotMetadata["platform"], te.sleepConf.Platform) + } t.Logf("Metrics prior to container start:\n\n%s\n\n", initialData) if err := cont.Start(te.sleepConf); err != nil { t.Fatalf("Cannot start container: %v", err) diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 5a2525d12..43bef6efc 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -163,6 +163,12 @@ type Sandbox struct { // created. RegisteredMetrics *metricpb.MetricRegistration `json:"registeredMetrics"` + // MetricMetadata are key-value pairs that are useful to export about this + // sandbox, but not part of the set of labels that uniquely identify it. + // They are static once initialized, and typically contain high-level + // configuration information about the sandbox. + MetricMetadata map[string]string `json:"metricMetadata"` + // MetricServerAddress is the address of the metric server that this sandbox // intends to export metrics for. // Only populated if exporting metrics was requested when the sandbox was @@ -247,6 +253,7 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) { }, UID: -1, // prevent usage before it's set. GID: -1, // prevent usage before it's set. + MetricMetadata: conf.MetricMetadata(), MetricServerAddress: conf.MetricServer, } if args.Spec != nil && args.Spec.Annotations != nil { diff --git a/test/metricclient/BUILD b/test/metricclient/BUILD index cf9b97080..c46aa5d7c 100644 --- a/test/metricclient/BUILD +++ b/test/metricclient/BUILD @@ -14,6 +14,7 @@ go_library( deps = [ "//pkg/sync", "//runsc/config", + "//runsc/sandbox", "//runsc/specutils", "@com_github_cenkalti_backoff//:go_default_library", "@com_github_prometheus_common//expfmt", diff --git a/test/metricclient/metricclient.go b/test/metricclient/metricclient.go index 34fc37646..c702ad164 100644 --- a/test/metricclient/metricclient.go +++ b/test/metricclient/metricclient.go @@ -36,6 +36,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/sandbox" "gvisor.dev/gvisor/runsc/specutils" ) @@ -359,3 +360,53 @@ func (m MetricData) GetPrometheusContainerInteger(want WantMetric) (int64, time. } return m.GetPrometheusInteger(want.Metric, labels) } + +// GetSandboxMetadataMetric returns the labels attached to the metadata metric for a given sandbox. +func (m MetricData) GetSandboxMetadataMetric(want WantMetric) (map[string]string, error) { + var buf bytes.Buffer + buf.WriteString(string(m)) + parsed, err := (&expfmt.TextParser{}).TextToMetricFamilies(&buf) + if err != nil { + return nil, err + } + metricData, found := parsed[want.Metric] + if !found { + return nil, fmt.Errorf("metric %q not found", want.Metric) + } + foundIndex := -1 + for i, data := range metricData.GetMetric() { + dataLabels := make(map[string]string, len(data.GetLabel())) + for _, label := range data.GetLabel() { + dataLabels[label.GetName()] = label.GetValue() + } + allMatching := true + for wantLabel, wantValue := range map[string]string{ + sandbox.SandboxIDLabel: want.Sandbox, + sandbox.NamespaceLabel: want.Namespace, + sandbox.PodNameLabel: want.Pod, + } { + if dataLabels[wantLabel] != wantValue { + allMatching = false + break + } + } + if allMatching { + if foundIndex != -1 { + return nil, errors.New("found multiple metadata metrics matching requested labels") + } + foundIndex = i + } + } + if foundIndex == -1 { + return nil, errors.New("no metadata metric matching requested labels") + } + data := metricData.GetMetric()[foundIndex] + metadataLabels := make(map[string]string, len(data.GetLabel())) + for _, label := range data.GetLabel() { + if label.GetName() == sandbox.SandboxIDLabel || label.GetName() == sandbox.NamespaceLabel || label.GetName() == sandbox.PodNameLabel { + continue + } + metadataLabels[label.GetName()] = label.GetValue() + } + return metadataLabels, nil +}