runsc metric-server: Add metric to track the presence of capabilities.

PiperOrigin-RevId: 521848278

g3doc/user_guide/observability.md

@@ -307,5 +307,11 @@ own metrics as well. All metrics have documentation and type annotations in the
     labels contain useful metadata about the sandbox, such as the version
     number, [platform](platforms.md), and [network type](networking.md) being
     used.
+*   `sandbox_capabilities`: A per-sandbox, per-capability metric that carries
+    the union of all capabilities present on at least one container of the
+    sandbox. Can optionally be filtered to only a subset of capabilities using
+    the `runsc-capability-filter` GET parameter on `/metrics` requests (regular
+    expression). Useful for auditing and aggregating the capabilities you rely
+    on across multiple sandboxes.
 *   `sandbox_creation_time_seconds`: A per-sandbox Unix timestamp representing
     the time at which this sandbox was created.

pkg/abi/linux/capability.go

@@ -14,6 +14,10 @@
 
 package linux
 
+import (
+	"strings"
+)
+
 // A Capability represents the ability to perform a privileged operation.
 type Capability int
 
@@ -157,6 +161,23 @@ func (cp Capability) String() string {
 	}
 }
 
+// TrimmedString returns the capability name without the "CAP_" prefix.
+func (cp Capability) TrimmedString() string {
+	const capPrefix = "CAP_"
+	s := cp.String()
+	if !strings.HasPrefix(s, capPrefix) {
+		return s
+	}
+	// This could use strings.TrimPrefix, but that function doesn't guarantee
+	// that it won't allocate a new string, whereas string slicing does.
+	// In the case of this function, since Capability.String returns a constant
+	// string, the underlying set of bytes backing that string will never be
+	// garbage-collected. Therefore, we always want to use a string slice that
+	// points to this same constant set of bytes, rather than risking
+	// allocating a new string.
+	return s[len(capPrefix):]
+}
+
 // CapabilityFromString converts a string to a capability.
 // If the capability doesn't exist, its second return value is `false`.
 // The capability name is expected to include the "CAP_" prefix.

runsc/cmd/metric_server.go

@@ -37,6 +37,7 @@ import (
 	"time"
 
 	"github.com/google/subcommands"
+	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/prometheus"
@@ -89,6 +90,12 @@ type servedSandbox struct {
 	// Once set, it is immutable.
 	createdAt time.Time
 
+	// capabilities is the union of the capability set of the containers within `sandbox`.
+	// It is used to export a per-sandbox metric representing which capabilities are in use.
+	// For monitoring purposes, a capability added in a container means it is considered
+	// added for the whole sandbox.
+	capabilities []linux.Capability
+
 	// verifier allows verifying the data integrity of the metrics we get from this sandbox.
 	// It is not always initialized when the sandbox is discovered, but rather upon first metrics
 	// access to the sandbox. Metric registration data is loaded from the root container's
@@ -143,16 +150,25 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.sandbox == nil {
-		cont, err := container.Load(s.rootDir, s.rootContainerID, container.LoadOpts{
-			Exact:         true,
-			SkipCheck:     true,
-			TryLock:       container.TryAcquire,
-			RootContainer: true,
+		allContainers, err := container.LoadSandbox(s.rootDir, s.rootContainerID.SandboxID, container.LoadOpts{
+			TryLock: container.TryAcquire,
 		})
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, fmt.Errorf("cannot load sandbox %q: %v", s.rootContainerID.SandboxID, err)
 		}
-		sandboxMetricAddr := strings.ReplaceAll(cont.Sandbox.MetricServerAddress, "%RUNTIME_ROOT%", s.rootDir)
+		var rootContainer *container.Container
+		for _, cont := range allContainers {
+			if cont.IsSandboxRoot() {
+				if rootContainer != nil {
+					return nil, nil, fmt.Errorf("multiple root contains found for sandbox ID %q: %v and %v", s.rootContainerID.SandboxID, cont, rootContainer)
+				}
+				rootContainer = cont
+			}
+		}
+		if rootContainer == nil {
+			return nil, nil, fmt.Errorf("no root container found for sandbox ID %q", s.rootContainerID.SandboxID)
+		}
+		sandboxMetricAddr := strings.ReplaceAll(rootContainer.Sandbox.MetricServerAddress, "%RUNTIME_ROOT%", s.rootDir)
 		if sandboxMetricAddr == "" {
 			return nil, nil, errors.New("sandbox did not request instrumentation")
 		}
@@ -161,7 +177,7 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) {
 		}
 		// Update label data as read from the state file.
 		// Do not store empty labels.
-		authoritativeLabels, err := sandboxPrometheusLabels(cont)
+		authoritativeLabels, err := sandboxPrometheusLabels(rootContainer)
 		if err != nil {
 			return nil, nil, fmt.Errorf("cannot compute Prometheus labels of sandbox: %v", err)
 		}
@@ -177,8 +193,28 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) {
 				delete(s.extraLabels, label)
 			}
 		}
-		s.sandbox = cont.Sandbox
-		s.createdAt = cont.CreatedAt
+
+		// Compute capability set.
+		allCaps := linux.AllCapabilities()
+		capSet := make([]linux.Capability, 0, len(allCaps))
+		for _, cap := range allCaps {
+			for _, cont := range allContainers {
+				if cont.HasCapabilityInAnySet(cap) {
+					capSet = append(capSet, cap)
+					break
+				}
+			}
+		}
+		if len(capSet) > 0 {
+			// Reallocate a slice with minimum size, since it will be long-lived.
+			s.capabilities = make([]linux.Capability, len(capSet))
+			for i, capLabels := range capSet {
+				s.capabilities[i] = capLabels
+			}
+		}
+
+		s.sandbox = rootContainer.Sandbox
+		s.createdAt = rootContainer.CreatedAt
 	}
 	if s.verifier == nil {
 		registeredMetrics, err := s.sandbox.GetRegisteredMetrics()
@@ -280,6 +316,17 @@ type MetricServer struct {
 	// is consistently passing in the same value for this parameter in each successive request.
 	lastValidMetricFilter string
 
+	// lastValidCapabilityFilterStr stores the last value of the "runsc-capability-filter" parameter
+	// for /metrics requests.
+	// It represents the last-known compilable regular expression that was passed to /metrics.
+	// It is used to avoid re-verifying this parameter in the common case where a single scraper
+	// is consistently passing in the same value for this parameter in each successive request.
+	lastValidCapabilityFilterStr string
+
+	// lastValidCapabilityFilterReg is the compiled regular expression corresponding to
+	// lastValidCapabilityFilterStr.
+	lastValidCapabilityFilterReg *regexp.Regexp
+
 	// numSandboxes counts the number of sandboxes that have ever been registered on this server.
 	// Used to distinguish between the case where this metrics serve has sat there doing nothing
 	// because no sandbox ever registered against it (which is unexpected), vs the case where it has
@@ -441,7 +488,7 @@ func (m *MetricServer) refreshSandboxesLocked() {
 			continue
 		}
 
-		// This is redundant with one of the checks performed below in servedSandbox.load(), but this
+		// This is redundant with one of the checks performed below in servedSandbox.load, but this
 		// avoids log spam for the non-error case of sandboxes that didn't request instrumentation.
 		sandboxMetricAddr := strings.ReplaceAll(cont.Sandbox.MetricServerAddress, "%RUNTIME_ROOT%", m.rootDir)
 		if sandboxMetricAddr != m.address {
@@ -520,7 +567,13 @@ var (
 		Type: prometheus.TypeGauge,
 		Help: "Key-value pairs about per-sandbox metadata.",
 	}
-	SandboxCreationMetric = prometheus.Metric{
+	SandboxCapabilitiesMetric = prometheus.Metric{
+		Name: "sandbox_capabilities",
+		Type: prometheus.TypeGauge,
+		Help: "Linux capabilities added within containers of the sandbox.",
+	}
+	SandboxCapabilitiesMetricLabel = "capability"
+	SandboxCreationMetric          = prometheus.Metric{
 		Name: "sandbox_creation_time_seconds",
 		Type: prometheus.TypeGauge,
 		Help: "When the sandbox was created, as a unix timestamp in milliseconds.",
@@ -560,6 +613,8 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht
 	defer ctxCancel()
 
 	metricsFilter := req.URL.Query().Get("runsc-sandbox-metrics-filter")
+	var capabilityFilterReg *regexp.Regexp
+	capabilityFilterStr := req.URL.Query().Get("runsc-capability-filter")
 
 	m.mu.Lock()
 
@@ -571,6 +626,20 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht
 		}
 		m.lastValidMetricFilter = metricsFilter
 	}
+	if capabilityFilterStr != "" {
+		if capabilityFilterStr != m.lastValidCapabilityFilterStr {
+			reg, err := regexp.Compile(capabilityFilterStr)
+			if err != nil {
+				m.mu.Unlock()
+				return httpResult{http.StatusBadRequest, errors.New("provided capability filter is not a valid regular expression")}
+			}
+			m.lastValidCapabilityFilterStr = capabilityFilterStr
+			m.lastValidCapabilityFilterReg = reg
+			capabilityFilterReg = reg
+		} else {
+			capabilityFilterReg = m.lastValidCapabilityFilterReg
+		}
+	}
 
 	m.refreshSandboxesLocked()
 
@@ -688,6 +757,14 @@ func (m *MetricServer) serveMetrics(w http.ResponseWriter, req *http.Request) ht
 					selfMetrics.Add(prometheus.LabeledIntData(&SandboxRunningMetric, nil, sandboxRunning).SetExternalLabels(served.extraLabels))
 					if loadErr == nil {
 						selfMetrics.Add(prometheus.LabeledIntData(&SandboxMetadataMetric, sand.MetricMetadata, 1).SetExternalLabels(served.extraLabels))
+						for _, cap := range served.capabilities {
+							if capabilityFilterReg != nil && !capabilityFilterReg.MatchString(cap.String()) && !capabilityFilterReg.MatchString(cap.TrimmedString()) {
+								continue
+							}
+							selfMetrics.Add(prometheus.LabeledIntData(&SandboxCapabilitiesMetric, map[string]string{
+								SandboxCapabilitiesMetricLabel: cap.TrimmedString(),
+							}, 1).SetExternalLabels(served.extraLabels))
+						}
 						createdAt := float64(served.createdAt.Unix()) + (float64(served.createdAt.Nanosecond()) / 1e9)
 						selfMetrics.Add(prometheus.LabeledFloatData(&SandboxCreationMetric, nil, createdAt).SetExternalLabels(served.extraLabels))
 					}

runsc/container/container.go

@@ -1301,6 +1301,26 @@ func (c *Container) IsSandboxRunning() bool {
 	return c.Sandbox != nil && c.Sandbox.IsRunning()
 }
 
+// HasCapabilityInAnySet returns true if the given capability is in any of the
+// capability sets of the container process.
+func (c *Container) HasCapabilityInAnySet(capability linux.Capability) bool {
+	capString := capability.String()
+	for _, set := range [5][]string{
+		c.Spec.Process.Capabilities.Bounding,
+		c.Spec.Process.Capabilities.Effective,
+		c.Spec.Process.Capabilities.Inheritable,
+		c.Spec.Process.Capabilities.Permitted,
+		c.Spec.Process.Capabilities.Ambient,
+	} {
+		for _, c := range set {
+			if c == capString {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func (c *Container) requireStatus(action string, statuses ...Status) error {
 	for _, s := range statuses {
 		if c.Status == s {
@@ -1310,6 +1330,11 @@ func (c *Container) requireStatus(action string, statuses ...Status) error {
 	return fmt.Errorf("cannot %s container %q in state %s", action, c.ID, c.Status)
 }
 
+// IsSandboxRoot returns true if this container is its sandbox's root container.
+func (c *Container) IsSandboxRoot() bool {
+	return isRoot(c.Spec)
+}
+
 func isRoot(spec *specs.Spec) bool {
 	return specutils.SpecContainerType(spec) != specutils.ContainerTypeContainer
 }
@@ -1350,7 +1375,7 @@ func adjustSandboxOOMScoreAdj(s *sandbox.Sandbox, spec *specs.Spec, rootDir stri
 		return nil
 	}
 
-	containers, err := loadSandbox(rootDir, s.ID)
+	containers, err := LoadSandbox(rootDir, s.ID, LoadOpts{})
 	if err != nil {
 		return fmt.Errorf("loading sandbox containers: %v", err)
 	}

runsc/container/metric_server_test.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 	"gvisor.dev/gvisor/runsc/config"
@@ -636,6 +637,90 @@ func TestContainerMetricsFilter(t *testing.T) {
 	}
 }
 
+// TestContainerCapabilityFilter verifies the ability to filter capabilities in /metrics requests.
+func TestContainerCapabilityFilter(t *testing.T) {
+	te, cleanup := setupMetrics(t, false /* forceTempUDS */)
+	defer cleanup()
+	te.sleepSpec.Process.Capabilities.Bounding = append(
+		te.sleepSpec.Process.Capabilities.Bounding,
+		linux.CAP_SYS_NICE.String(),
+		linux.CAP_NET_RAW.String())
+
+	args := Args{
+		ID:        testutil.RandomContainerID(),
+		Spec:      te.sleepSpec,
+		BundleDir: te.bundleDir,
+	}
+	cont, err := New(te.sleepConf, args)
+	if err != nil {
+		t.Fatalf("error creating container: %v", err)
+	}
+	defer cont.Destroy()
+	if err := cont.Start(te.sleepConf); err != nil {
+		t.Fatalf("Cannot start container: %v", err)
+	}
+
+	for _, test := range []struct {
+		name   string
+		filter string
+		want   map[linux.Capability]bool
+	}{
+		{
+			name:   "unfiltered",
+			filter: "",
+			want:   map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: true},
+		},
+		{
+			name:   "all filtered out",
+			filter: "^$",
+			want:   map[linux.Capability]bool{linux.CAP_SYS_NICE: false, linux.CAP_NET_RAW: false},
+		},
+		{
+			name:   "simple filter with prefix",
+			filter: fmt.Sprintf("^%s$", linux.CAP_SYS_NICE.String()),
+			want:   map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: false},
+		},
+		{
+			name:   "simple filter without prefix",
+			filter: fmt.Sprintf("^%s$", linux.CAP_SYS_NICE.TrimmedString()),
+			want:   map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: false},
+		},
+		{
+			name:   "unfiltered again to test regexp caching",
+			filter: "",
+			want:   map[linux.Capability]bool{linux.CAP_SYS_NICE: true, linux.CAP_NET_RAW: true},
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var params map[string]string
+			if test.filter != "" {
+				params = map[string]string{
+					"runsc-capability-filter": test.filter,
+				}
+			}
+			data, err := te.client.GetMetrics(te.testCtx, params)
+			if err != nil {
+				t.Fatalf("Cannot get metrics: %v", err)
+			}
+			for cap, want := range test.want {
+				got, _, err := data.GetPrometheusContainerInteger(metricclient.WantMetric{
+					Metric:      "testmetric_meta_sandbox_capabilities",
+					Sandbox:     args.ID,
+					ExtraLabels: map[string]string{"capability": cap.TrimmedString()},
+				})
+				if err != nil && want {
+					t.Errorf("Cannot get testmetric_meta_sandbox_capabilities[capability=%q]: %v", cap.TrimmedString(), err)
+				} else if err == nil && !want {
+					t.Errorf("Unexpectedly able to get testmetric_meta_sandbox_capabilities[capability=%q]: %v", cap.TrimmedString(), got)
+				}
+			}
+			if t.Failed() {
+				t.Logf("Metric data:\n\n%s\n\n", data)
+			}
+		})
+	}
+}
+
 func TestMetricServerChecksRootDirectoryAccess(t *testing.T) {
 	te, cleanup := setupMetrics(t /* forceTempUDS= */, false)
 	defer cleanup()

runsc/container/multi_container_test.go

@@ -1817,7 +1817,7 @@ func TestMultiContainerLoadSandbox(t *testing.T) {
 
 	// Load the sandbox and check that the correct containers were returned.
 	id := wants[0].Sandbox.ID
-	gots, err := loadSandbox(conf.RootDir, id)
+	gots, err := LoadSandbox(conf.RootDir, id, LoadOpts{})
 	if err != nil {
 		t.Fatalf("loadSandbox()=%v", err)
 	}

runsc/container/state_file.go

@@ -173,18 +173,23 @@ func listMatch(rootDir string, id FullID) ([]FullID, error) {
 	return out, nil
 }
 
-// loadSandbox loads all containers that belong to the sandbox with the given
+// LoadSandbox loads all containers that belong to the sandbox with the given
 // ID.
-func loadSandbox(rootDir, id string) ([]*Container, error) {
+func LoadSandbox(rootDir, id string, opts LoadOpts) ([]*Container, error) {
 	cids, err := listMatch(rootDir, FullID{SandboxID: id})
 	if err != nil {
 		return nil, err
 	}
 
+	// Override load options that don't make sense in the context of this function.
+	opts.SkipCheck = true      // We're loading all containers irrespective of status.
+	opts.RootContainer = false // We're loading all containers, not just the root one.
+	opts.Exact = true          // We'll iterate over exact container IDs below.
+
 	// Load the container metadata.
 	var containers []*Container
 	for _, cid := range cids {
-		container, err := Load(rootDir, cid, LoadOpts{Exact: true, SkipCheck: true})
+		container, err := Load(rootDir, cid, opts)
 		if err != nil {
 			// Container file may not exist if it raced with creation/deletion or
 			// directory was left behind. Load provides a snapshot in time, so it's

test/metricclient/metricclient.go

@@ -340,9 +340,9 @@ func (m MetricData) GetPrometheusInteger(metricName string, wantLabels map[strin
 	data := metricData.GetMetric()[foundIndex]
 	// Convert the value of this data point to an int regardless of its underlying Prometheus type.
 	var floatValue float64
-	if data.GetCounter().Value != nil {
+	if data.GetCounter() != nil && data.GetCounter().Value != nil {
 		floatValue = data.GetCounter().GetValue()
-	} else if data.GetGauge().Value != nil {
+	} else if data.GetGauge() != nil && data.GetGauge().Value != nil {
 		floatValue = data.GetGauge().GetValue()
 	} else {
 		return 0, time.Time{}, fmt.Errorf("metric is not numerical: %v", data)