From debdc9910f45207bb822495c5361317a3d0bfac1 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Mon, 10 Apr 2023 15:55:59 -0700 Subject: [PATCH] `runsc metric-server`: Add metric for interesting traits of sandbox specs. This is useful to export data about a sandbox that isn't gVisor configuration details (that would be the role of the `sandbox_metadata`) metric. `sandbox_spec` exports data about the specs of containers within a sandbox. Currently that is the following bits of information: - Does any of the container run as UID 0? Useful to check whether it has all capabilities. - What OCI runtime version do the containers use? Other interesting details might be number of containers, number of read-only vs read-write mounts, and so on. But those might be better off as separate metrics since they are numerical and could even be per-container rather than per-pod. PiperOrigin-RevId: 523234187 --- runsc/cmd/metric_server.go | 45 ++++++++++++++++++++++++++- runsc/container/container.go | 5 +++ runsc/container/metric_server_test.go | 18 +++++++++-- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/runsc/cmd/metric_server.go b/runsc/cmd/metric_server.go index ca0a05c98..e8e57eca7 100644 --- a/runsc/cmd/metric_server.go +++ b/runsc/cmd/metric_server.go @@ -96,6 +96,10 @@ type servedSandbox struct { // added for the whole sandbox. capabilities []linux.Capability + // specMetadataLabels is the set of label exported as part of the + // `spec_metadata` metric. + specMetadataLabels map[string]string + // verifier allows verifying the data integrity of the metrics we get from this sandbox. // It is not always initialized when the sandbox is discovered, but rather upon first metrics // access to the sandbox. Metric registration data is loaded from the root container's @@ -141,6 +145,36 @@ func sandboxPrometheusLabels(rootContainer *container.Container) (map[string]str return labels, nil } +// ComputeSpecMetadata returns the labels for the `spec_metadata` metric. +// It merges data from the Specs of multiple containers running within the +// same sandbox. +// It must support being called with zero containers; this is used to determine +// its set of labels, the keys of which must be static regardless of how many +// containers are passed to this function. +func ComputeSpecMetadata(allContainers []*container.Container) map[string]string { + const ( + unknownOCIVersion = "UNKNOWN" + inconsistentOCIVersion = "INCONSISTENT" + ) + + hasUID0Container := false + ociVersion := unknownOCIVersion + for _, cont := range allContainers { + if cont.RunsAsUID0() { + hasUID0Container = true + } + if ociVersion == unknownOCIVersion { + ociVersion = cont.Spec.Version + } else if ociVersion != cont.Spec.Version { + ociVersion = inconsistentOCIVersion + } + } + return map[string]string{ + "hasuid0": strconv.FormatBool(hasUID0Container), + "ociversion": ociVersion, + } +} + // load loads the sandbox being monitored and initializes its metric verifier. // If it returns an error other than container.ErrStateFileLocked, the sandbox is either // non-existent, or has not requested instrumentation to be enabled, or does not have @@ -213,6 +247,9 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) { } } + // Compute spec metadata. + s.specMetadataLabels = ComputeSpecMetadata(allContainers) + s.sandbox = rootContainer.Sandbox s.createdAt = rootContainer.CreatedAt } @@ -573,7 +610,12 @@ var ( Help: "Linux capabilities added within containers of the sandbox.", } SandboxCapabilitiesMetricLabel = "capability" - SandboxCreationMetric = prometheus.Metric{ + SpecMetadataMetric = prometheus.Metric{ + Name: "spec_metadata", + Type: prometheus.TypeGauge, + Help: "Key-value pairs about OCI spec metadata.", + } + SandboxCreationMetric = prometheus.Metric{ Name: "sandbox_creation_time_seconds", Type: prometheus.TypeGauge, Help: "When the sandbox was created, as a unix timestamp in milliseconds.", @@ -766,6 +808,7 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht SandboxCapabilitiesMetricLabel: cap.TrimmedString(), }, 1).SetExternalLabels(served.extraLabels)) } + selfMetrics.Add(prometheus.LabeledIntData(&SpecMetadataMetric, served.specMetadataLabels, 1).SetExternalLabels(served.extraLabels)) createdAt := float64(served.createdAt.Unix()) + (float64(served.createdAt.Nanosecond()) / 1e9) selfMetrics.Add(prometheus.LabeledFloatData(&SandboxCreationMetric, nil, createdAt).SetExternalLabels(served.extraLabels)) } diff --git a/runsc/container/container.go b/runsc/container/container.go index e744a645f..a4aeb0043 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -1325,6 +1325,11 @@ func (c *Container) HasCapabilityInAnySet(capability linux.Capability) bool { return false } +// RunsAsUID0 returns true if the container process runs with UID 0 (root). +func (c *Container) RunsAsUID0() bool { + return c.Spec.Process.User.UID == 0 +} + func (c *Container) requireStatus(action string, statuses ...Status) error { for _, s := range statuses { if c.Status == s { diff --git a/runsc/container/metric_server_test.go b/runsc/container/metric_server_test.go index affc5d587..7b392040e 100644 --- a/runsc/container/metric_server_test.go +++ b/runsc/container/metric_server_test.go @@ -154,7 +154,7 @@ func TestContainerMetrics(t *testing.T) { if err != nil { t.Errorf("Cannot get metrics after creating container: %v", err) } - gotMetadata, err := initialData.GetSandboxMetadataMetric(metricclient.WantMetric{ + gotSandboxMetadata, err := initialData.GetSandboxMetadataMetric(metricclient.WantMetric{ Metric: "testmetric_meta_sandbox_metadata", Sandbox: args.ID, Pod: "foopod", @@ -163,8 +163,20 @@ func TestContainerMetrics(t *testing.T) { if err != nil { t.Errorf("Cannot get sandbox metadata: %v", err) } - if gotMetadata["platform"] == "" || gotMetadata["platform"] != te.sleepConf.Platform { - t.Errorf("Invalid platform: Metric metadata says %v, config says %v", gotMetadata["platform"], te.sleepConf.Platform) + if gotSandboxMetadata["platform"] == "" || gotSandboxMetadata["platform"] != te.sleepConf.Platform { + t.Errorf("Invalid platform: Metric metadata says %v, config says %v", gotSandboxMetadata["platform"], te.sleepConf.Platform) + } + gotSpecMetadata, err := initialData.GetSandboxMetadataMetric(metricclient.WantMetric{ + Metric: "testmetric_meta_spec_metadata", + Sandbox: args.ID, + Pod: "foopod", + Namespace: "foons", + }) + if err != nil { + t.Errorf("Cannot get spec metadata: %v", err) + } + if gotSpecMetadata["hasuid0"] == "" || (gotSpecMetadata["hasuid0"] != "true" && gotSpecMetadata["hasuid0"] != "false") { + t.Errorf("Invalid or absent hasuid0 key from spec metadata: %v", gotSpecMetadata["hasuid0"]) } t.Logf("Metrics prior to container start:\n\n%s\n\n", initialData) if err := cont.Start(te.sleepConf); err != nil {