From 4e75dc46503d679c7c5c0f1327e4b5c6df8f6b93 Mon Sep 17 00:00:00 2001
From: Etienne Perot <eperot@google.com>
Date: Fri, 27 Jan 2023 15:54:28 -0800
Subject: [PATCH] `runsc metric-server`: Add per-sandbox start timestamp
 metric.

This synthetic metric contains the Unix timestamp that each sandbox was
started at.

This is useful for counter metrics, such that rates of change over time can
be properly on a per-sandbox basis.

PiperOrigin-RevId: 505228878
---
 pkg/prometheus/prometheus.go |  9 +++++++--
 runsc/cmd/metric_server.go   | 36 +++++++++++++++++++++---------------
 2 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go
index c5831bc31..024f5d9b4 100644
--- a/pkg/prometheus/prometheus.go
+++ b/pkg/prometheus/prometheus.go
@@ -222,7 +222,7 @@ type Data struct {
 
 // NewIntData returns a new Data struct with the given metric and value.
 func NewIntData(metric *Metric, val int64) *Data {
-	return &Data{Metric: metric, Number: &Number{Int: val}}
+	return LabeledIntData(metric, nil, val)
 }
 
 // LabeledIntData returns a new Data struct with the given metric, labels, and value.
@@ -232,7 +232,12 @@ func LabeledIntData(metric *Metric, labels map[string]string, val int64) *Data {
 
 // NewFloatData returns a new Data struct with the given metric and value.
 func NewFloatData(metric *Metric, val float64) *Data {
-	return &Data{Metric: metric, Number: &Number{Float: val}}
+	return LabeledFloatData(metric, nil, val)
+}
+
+// LabeledFloatData returns a new Data struct with the given metric, labels, and value.
+func LabeledFloatData(metric *Metric, labels map[string]string, val float64) *Data {
+	return &Data{Metric: metric, Labels: labels, Number: &Number{Float: val}}
 }
 
 // ExportOptions contains options that control how metric data is exported in Prometheus format.
diff --git a/runsc/cmd/metric_server.go b/runsc/cmd/metric_server.go
index ac2239b25..956cdf39e 100644
--- a/runsc/cmd/metric_server.go
+++ b/runsc/cmd/metric_server.go
@@ -475,6 +475,11 @@ var (
 		Type: prometheus.TypeGauge,
 		Help: "Key-value pairs about per-sandbox metadata.",
 	}
+	sandboxCreationMetric = prometheus.Metric{
+		Name: "sandbox_creation_time_seconds",
+		Type: prometheus.TypeGauge,
+		Help: "When the sandbox was created, as a unix timestamp in milliseconds.",
+	}
 	numRunningSandboxesMetric = prometheus.Metric{
 		Name: "num_sandboxes_running",
 		Type: prometheus.TypeGauge,
@@ -595,12 +600,13 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht
 		go func(metricsMu *sync.Mutex, meta *metaMetrics, selfMetrics *prometheus.Snapshot) {
 			defer wg.Done()
 			for s := range loadedSandboxCh {
-				served, sand, verifier, err := s.served, s.sandbox, s.verifier, s.err
+				served, sand, verifier, loadErr := s.served, s.sandbox, s.verifier, s.err
 				isRunning := false
 				var snapshot *prometheus.Snapshot
-				if err == nil {
+				sandboxErr := loadErr
+				if loadErr == nil {
 					queryCtx, queryCtxCancel := context.WithTimeout(ctx, perSandboxTime)
-					snapshot, err = queryMetrics(queryCtx, sand, verifier)
+					snapshot, sandboxErr = queryMetrics(queryCtx, sand, verifier)
 					queryCtxCancel()
 					isRunning = sand.IsRunning()
 				}
@@ -608,25 +614,25 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht
 					metricsMu.Lock()
 					defer metricsMu.Unlock()
 					selfMetrics.Add(prometheus.LabeledIntData(&sandboxPresenceMetric, served.extraLabels, 1))
-					selfMetrics.Add(prometheus.LabeledIntData(&sandboxMetadataMetric, served.labelsWithMetadata, 1))
 					sandboxRunning := int64(0)
 					if isRunning {
 						sandboxRunning = 1
+						meta.numRunningSandboxes++
 					}
 					selfMetrics.Add(prometheus.LabeledIntData(&sandboxRunningMetric, served.extraLabels, sandboxRunning))
-					if err != nil && !isRunning {
-						// The sandbox either hasn't started running yet, or it ran and has gone away between the
-						// start of the function and now. It is normal that metrics are not exported for this
-						// sandbox in this case, so do not report this as an error.
+					if loadErr == nil {
+						selfMetrics.Add(prometheus.LabeledIntData(&sandboxMetadataMetric, served.labelsWithMetadata, 1))
+						selfMetrics.Add(prometheus.LabeledFloatData(&sandboxCreationMetric, served.extraLabels, float64(served.createdAt.Unix())+(float64(served.createdAt.Nanosecond())/1e9)))
+					}
+					if sandboxErr != nil {
+						// If the sandbox isn't running, it is normal that metrics are not exported for it, so
+						// do not report this case as an error.
+						if isRunning {
+							meta.numCannotExportSandboxes++
+							log.Warningf("Could not export metrics from sandbox %s: %v", served.rootContainerID.SandboxID, sandboxErr)
+						}
 						return
 					}
-					if err != nil {
-						meta.numRunningSandboxes++
-						meta.numCannotExportSandboxes++
-						log.Warningf("Could not export metrics from sandbox %s: %v", served.rootContainerID.SandboxID, err)
-						return
-					}
-					meta.numRunningSandboxes++
 					snapshotCh <- snapshotAndOptions{
 						snapshot: snapshot,
 						options: prometheus.SnapshotExportOptions{