gVisor: Implement runsc metric-server which serves Prometheus metrics.

The metrics server implements the following interface:

- GET `/metrics`: Serves Prometheus metrics.
- POST `/runsc-metrics`: Contains administrative endpoints.
  All of them require the `root` argument to be specified, and match the one
  that the server expects. This is used to avoid confusing multiple instances
  of the  `runsc` metrics server.
  - POST `/runsc-metrics/healthcheck`: Used by clients to verify that the
    server is running and is the expected metric server.

This change has no tests, but coverage is provided in a later change that
provides an end-to-end container tests that the metric server works and
exports data faithfully.

This change is part of a series of changes to support Prometheus-style metrics
in `runsc`. Doing so requires making several seemingly-odd design decisions,
due to the following architectural constraints:

- Prometheus requires an HTTP server serving the `/metrics` endpoint.
- For performance reasons, the `runsc boot` process cannot run the `netpoller`
  goroutine.
  - Since we don't want to write our own HTTP server implementation, this
    means the HTTP endpoint has to be served by a separate process that
    remains running during the lifetime of the container.
- The `runsc boot` process is untrusted.
  - This means we cannot trust metrics data that comes out of the Sentry.
    Therefore, there needs to be an elaborate dance where we pre-register
    metric metadata before starting any untrusted workload. Then, the server
    relaying the metric data must verify the validity of metric values against
    this metric metadata. This avoids leaking metrics, cardinality blow-ups,
    and other such DoS vectors.
- This feature needs to be easy-to-use in a typical Docker setting.
  - This means having the ability to just say
    `--metrics-server=localhost:1337` in the `runsc` runtime entry in
    `/etc/docker/daemon.json` and have that Just Work(TM), even when multiple
    containers are running.
  - Since only one process may listen on a port at a given time, this means
    the metric server needs to be able to multiplex requests out to multiple
    running sandboxes, and remain alive for the entire duration of either of
    these sandboxes.
  - For this reason, the metrics server runs *outside* of the usual
    per-container cgroups.
  - This also saves system resources by not running one server per sandbox.
- The metrics server must be exposed to the outside world, and cannot assume
  that its clients are trustworthy.
  - For this reason, a metrics server is bound to a runtime root directory,
    and double-checks all that the sandboxes it is asked to follow actually
    exist in this root directory.

PiperOrigin-RevId: 503254345

runsc/cli/main.go

@@ -103,6 +103,7 @@ func Main(version string) {
 	subcommands.Register(new(cmd.Boot), internalGroup)
 	subcommands.Register(new(cmd.Gofer), internalGroup)
 	subcommands.Register(new(cmd.Umount), internalGroup)
+	subcommands.Register(new(cmd.MetricServer), internalGroup)
 
 	// Register with the main command line.
 	config.RegisterFlags(flag.CommandLine)

runsc/cmd/BUILD

@@ -22,6 +22,7 @@ go_library(
         "kill.go",
         "list.go",
         "metric_export.go",
+        "metric_server.go",
         "mitigate.go",
         "mitigate_extras.go",
         "path.go",
@@ -48,6 +49,7 @@ go_library(
     ],
     deps = [
         "//pkg/abi/linux",
+        "//pkg/atomicbitops",
         "//pkg/coretag",
         "//pkg/coverage",
         "//pkg/log",
@@ -59,6 +61,7 @@ go_library(
         "//pkg/sentry/platform",
         "//pkg/state/pretty",
         "//pkg/state/statefile",
+        "//pkg/sync",
         "//pkg/unet",
         "//pkg/urpc",
         "//runsc/boot",
@@ -71,6 +74,7 @@ go_library(
         "//runsc/fsgofer/filter",
         "//runsc/mitigate",
         "//runsc/profile",
+        "//runsc/sandbox",
         "//runsc/specutils",
         "@com_github_google_subcommands//:go_default_library",
         "@com_github_opencontainers_runtime_spec//specs-go:go_default_library",

runsc/cmd/metric_export.go

@@ -74,7 +74,7 @@ func (m *MetricExport) Execute(ctx context.Context, f *flag.FlagSet, args ...any
 	}, map[*prometheus.Snapshot]prometheus.SnapshotExportOptions{
 		snapshot: {
 			ExporterPrefix: conf.MetricExporterPrefix,
-			ExtraLabels:    map[string]string{"sandbox": cont.Sandbox.ID},
+			ExtraLabels:    cont.Sandbox.PrometheusLabels(),
 		},
 	})
 	if err != nil {

runsc/config/config.go

@@ -139,6 +139,17 @@ type Config struct {
 	// If unset, a sane platform-specific default will be used.
 	PlatformDevicePath string `flag:"platform_device_path"`
 
+	// MetricServer, if set, indicates that metrics should be exported on this address.
+	// This may either be 1) "addr:port" to export metrics on a specific network interface address,
+	// 2) ":port" for exporting metrics on all addresses, or 3) an absolute path to a Unix Domain
+	// Socket.
+	// The substring "%ID%" will be replaced by the container ID, and "%RUNTIME_ROOT%" by the root.
+	// This flag must be specified *both* as part of the `runsc metric-server` arguments (so that the
+	// metric server knows which address to bind to), and as part of the `runsc create` arguments (as
+	// an indication that the container being created wishes that its metrics should be exported).
+	// The value of this flag must also match across the two command lines.
+	MetricServer string `flag:"metric-server"`
+
 	// MetricExporterPrefix is added as prefix to all metric names.
 	// It is used to follow Prometheus's exporter convention, whereby all metric names should be
 	// prefixed by a name meaningfully identifying the software exporting the metric.

runsc/config/flags.go

@@ -52,6 +52,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.String("traceback", "system", "golang runtime's traceback level")
 
 	// Metrics flags.
+	flagSet.String("metric-server", "", "if set, export metrics on this address. This may either be 1) 'addr:port' to export metrics on a specific network interface address, 2) ':port' for exporting metrics on all interfaces, or 3) an absolute path to a Unix Domain Socket. The substring '%ID%' will be replaced by the container ID, and '%RUNTIME_ROOT%' by the root. This flag must be specified in both `runsc metric-server` and `runsc create`, and their values must match.")
 	flagSet.String("metric-exporter-prefix", "runsc_", "prefix for all metric names, following Prometheus exporter convention")
 
 	// Debugging flags: strace related

runsc/container/container.go

@@ -379,7 +379,7 @@ func New(conf *config.Config, args Args) (*Container, error) {
 func (c *Container) Start(conf *config.Config) error {
 	log.Debugf("Start container, cid: %s", c.ID)
 
-	if err := c.Saver.lock(); err != nil {
+	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
 	}
 	unlock := cleanup.Make(c.Saver.UnlockOrDie)
@@ -463,7 +463,7 @@ func (c *Container) Start(conf *config.Config) error {
 // to restore a container from its state file.
 func (c *Container) Restore(spec *specs.Spec, conf *config.Config, restoreFile string) error {
 	log.Debugf("Restore container, cid: %s", c.ID)
-	if err := c.Saver.lock(); err != nil {
+	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
 	}
 	defer c.Saver.UnlockOrDie()
@@ -649,7 +649,7 @@ func (c *Container) Checkpoint(f *os.File) error {
 // The call only succeeds if the container's status is created or running.
 func (c *Container) Pause() error {
 	log.Debugf("Pausing container, cid: %s", c.ID)
-	if err := c.Saver.lock(); err != nil {
+	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
 	}
 	defer c.Saver.UnlockOrDie()
@@ -669,7 +669,7 @@ func (c *Container) Pause() error {
 // The call only succeeds if the container's status is paused.
 func (c *Container) Resume() error {
 	log.Debugf("Resuming container, cid: %s", c.ID)
-	if err := c.Saver.lock(); err != nil {
+	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
 	}
 	defer c.Saver.UnlockOrDie()
@@ -710,7 +710,7 @@ func (c *Container) Processes() ([]*control.Process, error) {
 func (c *Container) Destroy() error {
 	log.Debugf("Destroy container, cid: %s", c.ID)
 
-	if err := c.Saver.lock(); err != nil {
+	if err := c.Saver.lock(BlockAcquire); err != nil {
 		return err
 	}
 	defer func() {

runsc/container/container_test.go

@@ -2670,7 +2670,7 @@ func TestSaveSystemdCgroup(t *testing.T) {
 	defer cont.Destroy()
 
 	cont.CompatCgroup = cgroup.CgroupJSON{Cgroup: cgroup.CreateMockSystemdCgroup()}
-	if err := cont.Saver.lock(); err != nil {
+	if err := cont.Saver.lock(BlockAcquire); err != nil {
 		t.Fatalf("cannot lock container metadata file: %v", err)
 	}
 	if err := cont.saveLocked(); err != nil {
@@ -2678,7 +2678,7 @@ func TestSaveSystemdCgroup(t *testing.T) {
 	}
 	cont.Saver.unlock()
 	loadCont := Container{}
-	cont.Saver.load(&loadCont)
+	cont.Saver.load(&loadCont, LoadOpts{})
 	if !reflect.DeepEqual(cont.CompatCgroup, loadCont.CompatCgroup) {
 		t.Errorf("CompatCgroup not properly saved: want %v, got %v", cont.CompatCgroup, loadCont.CompatCgroup)
 	}

runsc/container/state_file.go

@@ -16,6 +16,7 @@ package container
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -31,6 +32,21 @@ import (
 
 const stateFileExtension = "state"
 
+// ErrStateFileLocked is returned by Load() when the state file is locked
+// and TryLock is enabled.
+var ErrStateFileLocked = errors.New("state file locked")
+
+// TryLock represents whether we should block waiting for the lock to be acquired or not.
+type TryLock bool
+
+const (
+	// BlockAcquire means we will block until the lock can be acquired.
+	BlockAcquire TryLock = false
+
+	// TryAcquire means we will fail fast if the lock cannot be acquired.
+	TryAcquire TryLock = true
+)
+
 // LoadOpts provides options for Load()ing a container.
 type LoadOpts struct {
 	// Exact tells whether the search should be exact. See Load() for more.
@@ -39,6 +55,11 @@ type LoadOpts struct {
 	// SkipCheck tells Load() to skip checking if container is runnning.
 	SkipCheck bool
 
+	// TryLock tells Load() to fail if the container state file cannot be locked,
+	// as opposed to blocking until it is available.
+	// When the state file cannot be locked, it will error with ErrStateFileLocked.
+	TryLock TryLock
+
 	// RootContainer when true matches the search only with the root container of
 	// a sandbox. This is used when looking for a sandbox given that root
 	// container and sandbox share the same ID.
@@ -74,7 +95,7 @@ func Load(rootDir string, id FullID, opts LoadOpts) (*Container, error) {
 	defer state.close()
 
 	c := &Container{}
-	if err := state.load(c); err != nil {
+	if err := state.load(c, opts); err != nil {
 		if os.IsNotExist(err) {
 			// Preserve error so that callers can distinguish 'not found' errors.
 			return nil, err
@@ -272,13 +293,23 @@ type StateFile struct {
 }
 
 // lock globally locks all locking operations for the container.
-func (s *StateFile) lock() error {
+func (s *StateFile) lock(tryLock TryLock) error {
 	s.once.Do(func() {
 		s.flock = flock.New(s.lockPath())
 	})
 
-	if err := s.flock.Lock(); err != nil {
-		return fmt.Errorf("acquiring lock on %q: %v", s.flock, err)
+	if tryLock {
+		gotLock, err := s.flock.TryLock()
+		if err != nil {
+			return fmt.Errorf("acquiring lock on %q: %v", s.flock, err)
+		}
+		if !gotLock {
+			return ErrStateFileLocked
+		}
+	} else {
+		if err := s.flock.Lock(); err != nil {
+			return fmt.Errorf("acquiring lock on %q: %v", s.flock, err)
+		}
 	}
 	return nil
 }
@@ -287,7 +318,7 @@ func (s *StateFile) lock() error {
 // is done to ensure that more than one creation didn't race to create
 // containers with the same ID.
 func (s *StateFile) LockForNew() error {
-	if err := s.lock(); err != nil {
+	if err := s.lock(BlockAcquire); err != nil {
 		return err
 	}
 
@@ -327,7 +358,7 @@ func (s *StateFile) UnlockOrDie() {
 
 // SaveLocked saves 'v' to the state file.
 //
-// Preconditions: lock() must been called before.
+// Preconditions: lock(*) must been called before.
 func (s *StateFile) SaveLocked(v any) error {
 	if !s.flock.Locked() {
 		panic("saveLocked called without lock held")
@@ -343,8 +374,8 @@ func (s *StateFile) SaveLocked(v any) error {
 	return nil
 }
 
-func (s *StateFile) load(v any) error {
-	if err := s.lock(); err != nil {
+func (s *StateFile) load(v any, opts LoadOpts) error {
+	if err := s.lock(opts.TryLock); err != nil {
 		return err
 	}
 	defer s.UnlockOrDie()

runsc/sandbox/BUILD

@@ -23,6 +23,7 @@ go_library(
         "//pkg/control/server",
         "//pkg/coverage",
         "//pkg/log",
+        "//pkg/metric:metric_go_proto",
         "//pkg/prometheus",
         "//pkg/sentry/control",
         "//pkg/sentry/platform",
@@ -44,6 +45,7 @@ go_library(
         "@com_github_opencontainers_runtime_spec//specs-go:go_default_library",
         "@com_github_syndtr_gocapability//capability:go_default_library",
         "@com_github_vishvananda_netlink//:go_default_library",
+        "@org_golang_google_protobuf//encoding/prototext:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",
     ],
 )

runsc/sandbox/sandbox.go

@@ -33,12 +33,14 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/syndtr/gocapability/capability"
 	"golang.org/x/sys/unix"
+	"google.golang.org/protobuf/encoding/prototext"
 	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/control/client"
 	"gvisor.dev/gvisor/pkg/control/server"
 	"gvisor.dev/gvisor/pkg/coverage"
 	"gvisor.dev/gvisor/pkg/log"
+	metricpb "gvisor.dev/gvisor/pkg/metric/metric_go_proto"
 	"gvisor.dev/gvisor/pkg/prometheus"
 	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
@@ -54,6 +56,16 @@ import (
 	"gvisor.dev/gvisor/runsc/specutils"
 )
 
+const (
+	// namespaceAnnotation is a pod annotation populated by containerd.
+	// It contains the name of the pod that a sandbox is in when running in Kubernetes.
+	podNameAnnotation = "io.kubernetes.cri.sandbox-name"
+
+	// namespaceAnnotation is a pod annotation populated by containerd.
+	// It contains the namespace of the pod that a sandbox is in when running in Kubernetes.
+	namespaceAnnotation = "io.kubernetes.cri.sandbox-namespace"
+)
+
 // pid is an atomic type that implements JSON marshal/unmarshal interfaces.
 type pid struct {
 	val atomicbitops.Int64
@@ -96,6 +108,14 @@ type Sandbox struct {
 	// ID as the first container run in the sandbox.
 	ID string `json:"id"`
 
+	// PodName is the name of the Kubernetes Pod (if any) that this sandbox
+	// represents. Unset if not running under containerd or Kubernetes.
+	PodName string `json:"podName"`
+
+	// Namespace is the Kubernetes namespace (if any) of the pod that this
+	// sandbox represents. Unset if not running under containerd or Kubernetes.
+	Namespace string `json:"namespace"`
+
 	// Pid is the pid of the running sandbox. May be 0 if the sandbox
 	// is not running.
 	Pid pid `json:"pid"`
@@ -113,6 +133,13 @@ type Sandbox struct {
 	// started, before it may be modified.
 	OriginalOOMScoreAdj int `json:"originalOomScoreAdj"`
 
+	// RegisteredMetrics is the set of metrics registered in the sandbox.
+	// Used for verifying metric data integrity after containers are started.
+	// Only populated if exporting metrics was requested when the sandbox was
+	// created.
+	// This is a textproto string of metricpb.MetricRegistration.
+	RegisteredMetrics string `json:"registeredMetrics"`
+
 	// child is set if a sandbox process is a child of the current process.
 	//
 	// This field isn't saved to json, because only a creator of sandbox
@@ -189,6 +216,10 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) {
 		UID: -1, // prevent usage before it's set.
 		GID: -1, // prevent usage before it's set.
 	}
+	if args.Spec != nil && args.Spec.Annotations != nil {
+		s.PodName = args.Spec.Annotations[podNameAnnotation]
+		s.Namespace = args.Spec.Annotations[namespaceAnnotation]
+	}
 	// The Cleanup object cleans up partially created sandboxes when an error
 	// occurs. Any errors occurring during cleanup itself are ignored.
 	c := cleanup.Make(func() {
@@ -1072,7 +1103,23 @@ func (s *Sandbox) UsageFD() (*control.MemoryUsageRecord, error) {
 	return control.NewMemoryUsageRecord(*m.FilePayload.Files[0], *m.FilePayload.Files[1])
 }
 
-// ExportMetrics writes Prometheus-formatted metrics data to the given io.Writer.
+// GetRegisteredMetrics returns metric registration data from the sandbox.
+// This data is meant to be used as a way to sanity-check any exported metrics data during the
+// lifetime of the sandbox in order to avoid a compromised sandbox from being able to produce
+// bogus metrics.
+// This returns an error if the sandbox has not requested instrumentation during creation time.
+func (s *Sandbox) GetRegisteredMetrics() (*metricpb.MetricRegistration, error) {
+	if s.RegisteredMetrics == "" {
+		return nil, errors.New("sandbox did not request instrumentation when it was created")
+	}
+	registeredMetrics := &metricpb.MetricRegistration{}
+	if err := prototext.Unmarshal([]byte(s.RegisteredMetrics), registeredMetrics); err != nil {
+		return nil, err
+	}
+	return registeredMetrics, nil
+}
+
+// ExportMetrics returns a snapshot of metric values from the sandbox in Prometheus format.
 func (s *Sandbox) ExportMetrics() (*prometheus.Snapshot, error) {
 	log.Debugf("Metrics export sandbox %q", s.ID)
 	data := &control.MetricsExportData{}
@@ -1082,6 +1129,26 @@ func (s *Sandbox) ExportMetrics() (*prometheus.Snapshot, error) {
 	return data.Snapshot, nil
 }
 
+// Label names used to identify each sandbox.
+const (
+	SandboxIDLabel = "sandbox"
+	PodNameLabel   = "pod"
+	NamespaceLabel = "namespace"
+)
+
+// PrometheusLabels returns a set of Prometheus labels that identifies the sandbox.
+func (s *Sandbox) PrometheusLabels() map[string]string {
+	labels := make(map[string]string, 3)
+	labels[SandboxIDLabel] = s.ID
+	if s.PodName != "" {
+		labels[PodNameLabel] = s.PodName
+	}
+	if s.Namespace != "" {
+		labels[NamespaceLabel] = s.Namespace
+	}
+	return labels
+}
+
 // IsRunning returns true if the sandbox or gofer process is running.
 func (s *Sandbox) IsRunning() bool {
 	pid := s.Pid.load()