diff --git a/g3doc/user_guide/observability.md b/g3doc/user_guide/observability.md index 6fbd83870..f07d6f569 100644 --- a/g3doc/user_guide/observability.md +++ b/g3doc/user_guide/observability.md @@ -307,5 +307,11 @@ own metrics as well. All metrics have documentation and type annotations in the labels contain useful metadata about the sandbox, such as the version number, [platform](platforms.md), and [network type](networking.md) being used. +* `sandbox_capabilities`: A per-sandbox, per-capability metric that carries + the union of all capabilities present on at least one container of the + sandbox. Can optionally be filtered to only a subset of capabilities using + the `runsc-capability-filter` GET parameter on `/metrics` requests (regular + expression). Useful for auditing and aggregating the capabilities you rely + on across multiple sandboxes. * `sandbox_creation_time_seconds`: A per-sandbox Unix timestamp representing the time at which this sandbox was created. diff --git a/pkg/abi/linux/capability.go b/pkg/abi/linux/capability.go index 9b98483d1..76f64cbfe 100644 --- a/pkg/abi/linux/capability.go +++ b/pkg/abi/linux/capability.go @@ -14,6 +14,10 @@ package linux +import ( + "strings" +) + // A Capability represents the ability to perform a privileged operation. type Capability int @@ -157,6 +161,23 @@ func (cp Capability) String() string { } } +// TrimmedString returns the capability name without the "CAP_" prefix. +func (cp Capability) TrimmedString() string { + const capPrefix = "CAP_" + s := cp.String() + if !strings.HasPrefix(s, capPrefix) { + return s + } + // This could use strings.TrimPrefix, but that function doesn't guarantee + // that it won't allocate a new string, whereas string slicing does. + // In the case of this function, since Capability.String returns a constant + // string, the underlying set of bytes backing that string will never be + // garbage-collected. Therefore, we always want to use a string slice that + // points to this same constant set of bytes, rather than risking + // allocating a new string. + return s[len(capPrefix):] +} + // CapabilityFromString converts a string to a capability. // If the capability doesn't exist, its second return value is `false`. // The capability name is expected to include the "CAP_" prefix. diff --git a/runsc/cmd/metric_server.go b/runsc/cmd/metric_server.go index 16337820d..853c7643f 100644 --- a/runsc/cmd/metric_server.go +++ b/runsc/cmd/metric_server.go @@ -37,6 +37,7 @@ import ( "time" "github.com/google/subcommands" + "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/prometheus" @@ -89,6 +90,12 @@ type servedSandbox struct { // Once set, it is immutable. createdAt time.Time + // capabilities is the union of the capability set of the containers within `sandbox`. + // It is used to export a per-sandbox metric representing which capabilities are in use. + // For monitoring purposes, a capability added in a container means it is considered + // added for the whole sandbox. + capabilities []linux.Capability + // verifier allows verifying the data integrity of the metrics we get from this sandbox. // It is not always initialized when the sandbox is discovered, but rather upon first metrics // access to the sandbox. Metric registration data is loaded from the root container's @@ -143,16 +150,25 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) { s.mu.Lock() defer s.mu.Unlock() if s.sandbox == nil { - cont, err := container.Load(s.rootDir, s.rootContainerID, container.LoadOpts{ - Exact: true, - SkipCheck: true, - TryLock: container.TryAcquire, - RootContainer: true, + allContainers, err := container.LoadSandbox(s.rootDir, s.rootContainerID.SandboxID, container.LoadOpts{ + TryLock: container.TryAcquire, }) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("cannot load sandbox %q: %v", s.rootContainerID.SandboxID, err) } - sandboxMetricAddr := strings.ReplaceAll(cont.Sandbox.MetricServerAddress, "%RUNTIME_ROOT%", s.rootDir) + var rootContainer *container.Container + for _, cont := range allContainers { + if cont.IsSandboxRoot() { + if rootContainer != nil { + return nil, nil, fmt.Errorf("multiple root contains found for sandbox ID %q: %v and %v", s.rootContainerID.SandboxID, cont, rootContainer) + } + rootContainer = cont + } + } + if rootContainer == nil { + return nil, nil, fmt.Errorf("no root container found for sandbox ID %q", s.rootContainerID.SandboxID) + } + sandboxMetricAddr := strings.ReplaceAll(rootContainer.Sandbox.MetricServerAddress, "%RUNTIME_ROOT%", s.rootDir) if sandboxMetricAddr == "" { return nil, nil, errors.New("sandbox did not request instrumentation") } @@ -161,7 +177,7 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) { } // Update label data as read from the state file. // Do not store empty labels. - authoritativeLabels, err := sandboxPrometheusLabels(cont) + authoritativeLabels, err := sandboxPrometheusLabels(rootContainer) if err != nil { return nil, nil, fmt.Errorf("cannot compute Prometheus labels of sandbox: %v", err) } @@ -177,8 +193,28 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) { delete(s.extraLabels, label) } } - s.sandbox = cont.Sandbox - s.createdAt = cont.CreatedAt + + // Compute capability set. + allCaps := linux.AllCapabilities() + capSet := make([]linux.Capability, 0, len(allCaps)) + for _, cap := range allCaps { + for _, cont := range allContainers { + if cont.HasCapabilityInAnySet(cap) { + capSet = append(capSet, cap) + break + } + } + } + if len(capSet) > 0 { + // Reallocate a slice with minimum size, since it will be long-lived. + s.capabilities = make([]linux.Capability, len(capSet)) + for i, capLabels := range capSet { + s.capabilities[i] = capLabels + } + } + + s.sandbox = rootContainer.Sandbox + s.createdAt = rootContainer.CreatedAt } if s.verifier == nil { registeredMetrics, err := s.sandbox.GetRegisteredMetrics() @@ -280,6 +316,17 @@ type MetricServer struct { // is consistently passing in the same value for this parameter in each successive request. lastValidMetricFilter string + // lastValidCapabilityFilterStr stores the last value of the "runsc-capability-filter" parameter + // for /metrics requests. + // It represents the last-known compilable regular expression that was passed to /metrics. + // It is used to avoid re-verifying this parameter in the common case where a single scraper + // is consistently passing in the same value for this parameter in each successive request. + lastValidCapabilityFilterStr string + + // lastValidCapabilityFilterReg is the compiled regular expression corresponding to + // lastValidCapabilityFilterStr. + lastValidCapabilityFilterReg *regexp.Regexp + // numSandboxes counts the number of sandboxes that have ever been registered on this server. // Used to distinguish between the case where this metrics serve has sat there doing nothing // because no sandbox ever registered against it (which is unexpected), vs the case where it has @@ -441,7 +488,7 @@ func (m *MetricServer) refreshSandboxesLocked() { continue } - // This is redundant with one of the checks performed below in servedSandbox.load(), but this + // This is redundant with one of the checks performed below in servedSandbox.load, but this // avoids log spam for the non-error case of sandboxes that didn't request instrumentation. sandboxMetricAddr := strings.ReplaceAll(cont.Sandbox.MetricServerAddress, "%RUNTIME_ROOT%", m.rootDir) if sandboxMetricAddr != m.address { @@ -520,7 +567,13 @@ var ( Type: prometheus.TypeGauge, Help: "Key-value pairs about per-sandbox metadata.", } - SandboxCreationMetric = prometheus.Metric{ + SandboxCapabilitiesMetric = prometheus.Metric{ + Name: "sandbox_capabilities", + Type: prometheus.TypeGauge, + Help: "Linux capabilities added within containers of the sandbox.", + } + SandboxCapabilitiesMetricLabel = "capability" + SandboxCreationMetric = prometheus.Metric{ Name: "sandbox_creation_time_seconds", Type: prometheus.TypeGauge, Help: "When the sandbox was created, as a unix timestamp in milliseconds.", @@ -560,6 +613,8 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht defer ctxCancel() metricsFilter := req.URL.Query().Get("runsc-sandbox-metrics-filter") + var capabilityFilterReg *regexp.Regexp + capabilityFilterStr := req.URL.Query().Get("runsc-capability-filter") m.mu.Lock() @@ -571,6 +626,20 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht } m.lastValidMetricFilter = metricsFilter } + if capabilityFilterStr != "" { + if capabilityFilterStr != m.lastValidCapabilityFilterStr { + reg, err := regexp.Compile(capabilityFilterStr) + if err != nil { + m.mu.Unlock() + return httpResult{http.StatusBadRequest, errors.New("provided capability filter is not a valid regular expression")} + } + m.lastValidCapabilityFilterStr = capabilityFilterStr + m.lastValidCapabilityFilterReg = reg + capabilityFilterReg = reg + } else { + capabilityFilterReg = m.lastValidCapabilityFilterReg + } + } m.refreshSandboxesLocked() @@ -688,6 +757,14 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht selfMetrics.Add(prometheus.LabeledIntData(&SandboxRunningMetric, nil, sandboxRunning).SetExternalLabels(served.extraLabels)) if loadErr == nil { selfMetrics.Add(prometheus.LabeledIntData(&SandboxMetadataMetric, sand.MetricMetadata, 1).SetExternalLabels(served.extraLabels)) + for _, cap := range served.capabilities { + if capabilityFilterReg != nil && !capabilityFilterReg.MatchString(cap.String()) && !capabilityFilterReg.MatchString(cap.TrimmedString()) { + continue + } + selfMetrics.Add(prometheus.LabeledIntData(&SandboxCapabilitiesMetric, map[string]string{ + SandboxCapabilitiesMetricLabel: cap.TrimmedString(), + }, 1).SetExternalLabels(served.extraLabels)) + } createdAt := float64(served.createdAt.Unix()) + (float64(served.createdAt.Nanosecond()) / 1e9) selfMetrics.Add(prometheus.LabeledFloatData(&SandboxCreationMetric, nil, createdAt).SetExternalLabels(served.extraLabels)) } diff --git a/runsc/container/container.go b/runsc/container/container.go index baf1d6748..98eea8fce 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -1301,6 +1301,26 @@ func (c *Container) IsSandboxRunning() bool { return c.Sandbox != nil && c.Sandbox.IsRunning() } +// HasCapabilityInAnySet returns true if the given capability is in any of the +// capability sets of the container process. +func (c *Container) HasCapabilityInAnySet(capability linux.Capability) bool { + capString := capability.String() + for _, set := range [5][]string{ + c.Spec.Process.Capabilities.Bounding, + c.Spec.Process.Capabilities.Effective, + c.Spec.Process.Capabilities.Inheritable, + c.Spec.Process.Capabilities.Permitted, + c.Spec.Process.Capabilities.Ambient, + } { + for _, c := range set { + if c == capString { + return true + } + } + } + return false +} + func (c *Container) requireStatus(action string, statuses ...Status) error { for _, s := range statuses { if c.Status == s { @@ -1310,6 +1330,11 @@ func (c *Container) requireStatus(action string, statuses ...Status) error { return fmt.Errorf("cannot %s container %q in state %s", action, c.ID, c.Status) } +// IsSandboxRoot returns true if this container is its sandbox's root container. +func (c *Container) IsSandboxRoot() bool { + return isRoot(c.Spec) +} + func isRoot(spec *specs.Spec) bool { return specutils.SpecContainerType(spec) != specutils.ContainerTypeContainer } @@ -1350,7 +1375,7 @@ func adjustSandboxOOMScoreAdj(s *sandbox.Sandbox, spec *specs.Spec, rootDir stri return nil } - containers, err := loadSandbox(rootDir, s.ID) + containers, err := LoadSandbox(rootDir, s.ID, LoadOpts{}) if err != nil { return fmt.Errorf("loading sandbox containers: %v", err) } diff --git a/runsc/container/metric_server_test.go b/runsc/container/metric_server_test.go index 2da9b2a02..03c122ec3 100644 --- a/runsc/container/metric_server_test.go +++ b/runsc/container/metric_server_test.go @@ -27,6 +27,7 @@ import ( "github.com/google/go-cmp/cmp" specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/test/testutil" "gvisor.dev/gvisor/runsc/config" @@ -636,6 +637,90 @@ func TestContainerMetricsFilter(t *testing.T) { } } +// TestContainerCapabilityFilter verifies the ability to filter capabilities in /metrics requests. +func TestContainerCapabilityFilter(t *testing.T) { + te, cleanup := setupMetrics(t, false /* forceTempUDS */) + defer cleanup() + te.sleepSpec.Process.Capabilities.Bounding = append( + te.sleepSpec.Process.Capabilities.Bounding, + linux.CAP_SYS_NICE.String(), + linux.CAP_NET_RAW.String()) + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: te.sleepSpec, + BundleDir: te.bundleDir, + } + cont, err := New(te.sleepConf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(te.sleepConf); err != nil { + t.Fatalf("Cannot start container: %v", err) + } + + for _, test := range []struct { + name string + filter string + want map[linux.Capability]bool + }{ + { + name: "unfiltered", + filter: "", + want: map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: true}, + }, + { + name: "all filtered out", + filter: "^$", + want: map[linux.Capability]bool{linux.CAP_SYS_NICE: false, linux.CAP_NET_RAW: false}, + }, + { + name: "simple filter with prefix", + filter: fmt.Sprintf("^%s$", linux.CAP_SYS_NICE.String()), + want: map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: false}, + }, + { + name: "simple filter without prefix", + filter: fmt.Sprintf("^%s$", linux.CAP_SYS_NICE.TrimmedString()), + want: map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: false}, + }, + { + name: "unfiltered again to test regexp caching", + filter: "", + want: map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: true}, + }, + } { + t.Run(test.name, func(t *testing.T) { + var params map[string]string + if test.filter != "" { + params = map[string]string{ + "runsc-capability-filter": test.filter, + } + } + data, err := te.client.GetMetrics(te.testCtx, params) + if err != nil { + t.Fatalf("Cannot get metrics: %v", err) + } + for cap, want := range test.want { + got, _, err := data.GetPrometheusContainerInteger(metricclient.WantMetric{ + Metric: "testmetric_meta_sandbox_capabilities", + Sandbox: args.ID, + ExtraLabels: map[string]string{"capability": cap.TrimmedString()}, + }) + if err != nil && want { + t.Errorf("Cannot get testmetric_meta_sandbox_capabilities[capability=%q]: %v", cap.TrimmedString(), err) + } else if err == nil && !want { + t.Errorf("Unexpectedly able to get testmetric_meta_sandbox_capabilities[capability=%q]: %v", cap.TrimmedString(), got) + } + } + if t.Failed() { + t.Logf("Metric data:\n\n%s\n\n", data) + } + }) + } +} + func TestMetricServerChecksRootDirectoryAccess(t *testing.T) { te, cleanup := setupMetrics(t /* forceTempUDS= */, false) defer cleanup() diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index 2c9be0efa..f9e853245 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -1817,7 +1817,7 @@ func TestMultiContainerLoadSandbox(t *testing.T) { // Load the sandbox and check that the correct containers were returned. id := wants[0].Sandbox.ID - gots, err := loadSandbox(conf.RootDir, id) + gots, err := LoadSandbox(conf.RootDir, id, LoadOpts{}) if err != nil { t.Fatalf("loadSandbox()=%v", err) } diff --git a/runsc/container/state_file.go b/runsc/container/state_file.go index 29b53d6f6..ac2566016 100644 --- a/runsc/container/state_file.go +++ b/runsc/container/state_file.go @@ -173,18 +173,23 @@ func listMatch(rootDir string, id FullID) ([]FullID, error) { return out, nil } -// loadSandbox loads all containers that belong to the sandbox with the given +// LoadSandbox loads all containers that belong to the sandbox with the given // ID. -func loadSandbox(rootDir, id string) ([]*Container, error) { +func LoadSandbox(rootDir, id string, opts LoadOpts) ([]*Container, error) { cids, err := listMatch(rootDir, FullID{SandboxID: id}) if err != nil { return nil, err } + // Override load options that don't make sense in the context of this function. + opts.SkipCheck = true // We're loading all containers irrespective of status. + opts.RootContainer = false // We're loading all containers, not just the root one. + opts.Exact = true // We'll iterate over exact container IDs below. + // Load the container metadata. var containers []*Container for _, cid := range cids { - container, err := Load(rootDir, cid, LoadOpts{Exact: true, SkipCheck: true}) + container, err := Load(rootDir, cid, opts) if err != nil { // Container file may not exist if it raced with creation/deletion or // directory was left behind. Load provides a snapshot in time, so it's diff --git a/test/metricclient/metricclient.go b/test/metricclient/metricclient.go index 423124d7e..f08e354b8 100644 --- a/test/metricclient/metricclient.go +++ b/test/metricclient/metricclient.go @@ -340,9 +340,9 @@ func (m MetricData) GetPrometheusInteger(metricName string, wantLabels map[strin data := metricData.GetMetric()[foundIndex] // Convert the value of this data point to an int regardless of its underlying Prometheus type. var floatValue float64 - if data.GetCounter().Value != nil { + if data.GetCounter() != nil && data.GetCounter().Value != nil { floatValue = data.GetCounter().GetValue() - } else if data.GetGauge().Value != nil { + } else if data.GetGauge() != nil && data.GetGauge().Value != nil { floatValue = data.GetGauge().GetValue() } else { return 0, time.Time{}, fmt.Errorf("metric is not numerical: %v", data)