runsc metric-server: Optimize memory usage and allocation-heavy functions.

This is an effort to reduce it to be a well-behaved background process.

With 110 sandboxes running, at rest, this goes from

```
VmRSS:   72376 kB
RssAnon: 51944 kB
```

to:

```
VmRSS:   45864 kB
RssAnon: 25788 kB
```

This GCs much more aggressively, including after every single request, which
means we do spend disproportionately more CPU in order to get that low memory
usage. From my testing, serving requests takes about 12% more CPU, and it's
all spent in GC.

The optimizations that went into this are:

- Add a method in `state` to discard the global type maps.
- Add a custom "packed" number type in `prometheus` library that encodes small
  integers and floating-point numbers in 32 bits whenever possible without
  loss of precision, otherwise they are encoded in their full 64-bit glory and
  the 32-bit representation is used as a pointer to the 64-bit representation.
  These are stored either per-sandbox (for static-after-sandbox-creation
  numbers like distribution bucket boundaries), or per-metric-retrieval
  attempt otherwise.
- Use string interning for commonly-seen strings across sandboxes, like metric
  names and label names. Label values are also interned, but only at a
  per-sandbox granularity.
- Reworked allocation-heavy functions like `OrderedLabels` and some string
  rendering functions to be (almost) allocation-free. This doesn't reduce
  memory usage at rest, and does increase their CPU cost, but in return it
  significantly cuts down on the percentage of CPU time spent in GC
  (>50% -> 25%) enough to justify spending the extra CPU in these functions.

PiperOrigin-RevId: 515181387

pkg/prometheus/prometheus_test.go

@@ -874,7 +874,8 @@ func TestVerifier(t *testing.T) {
 			}
 			at(testTime, func() {
 				t.Logf("Test is running with simulated time: %v", testTime)
-				verifier, err := NewVerifier(test.Registration)
+				verifier, cleanup, err := NewVerifier(test.Registration)
+				defer cleanup()
 				if err != nil && !test.WantVerifierCreationErr {
 					t.Fatalf("unexpected verifier creation error: %v", err)
 				}
@@ -1459,3 +1460,145 @@ func TestGroupSameNameMetrics(t *testing.T) {
 		t.Errorf("Seen metrics: %v\nWant metrics: %v", seenMetrics, wantSeenMetrics)
 	}
 }
+
+func TestNumberPacker(t *testing.T) {
+	interestingIntegers := map[uint64]struct{}{
+		uint64(0):                  struct{}{},
+		uint64(0x5555555555555555): struct{}{},
+		uint64(0xaaaaaaaaaaaaaaaa): struct{}{},
+		uint64(0xffffffffffffffff): struct{}{},
+	}
+	for numBits := 0; numBits < 2; numBits++ {
+		newIntegers := map[uint64]struct{}{}
+		for interestingInt := range interestingIntegers {
+			for i := 0; i < 64; i++ {
+				newIntegers[interestingInt|(1<<i)] = struct{}{}
+				newIntegers[interestingInt & ^(1<<i)] = struct{}{}
+			}
+		}
+		for newInt := range newIntegers {
+			interestingIntegers[newInt] = struct{}{}
+		}
+	}
+	for _, i := range []int64{
+		math.MinInt,
+		math.MaxInt,
+		math.MinInt8,
+		math.MaxInt8,
+		math.MaxUint8,
+		math.MinInt16,
+		math.MaxInt16,
+		math.MaxUint16,
+		math.MinInt32,
+		math.MaxInt32,
+		math.MaxUint32,
+		math.MinInt64,
+		math.MaxInt64,
+	} {
+		for d := int64(-3); d <= int64(3); d++ {
+			interestingIntegers[uint64(i+d)] = struct{}{}
+		}
+	}
+	interestingIntegers[0] = struct{}{}
+	interestingIntegers[1] = struct{}{}
+	interestingIntegers[2] = struct{}{}
+	interestingIntegers[3] = struct{}{}
+	interestingIntegers[math.MaxUint64-3] = struct{}{}
+	interestingIntegers[math.MaxUint64-2] = struct{}{}
+	interestingIntegers[math.MaxUint64-1] = struct{}{}
+	interestingIntegers[math.MaxUint64] = struct{}{}
+
+	p := &numberPacker{}
+	t.Run("integers", func(t *testing.T) {
+		seenDirectInteger := false
+		seenIndirectInteger := false
+		for interestingInt := range interestingIntegers {
+			orig := NewInt(int64(interestingInt))
+			packed, err := p.pack(orig)
+			if err != nil {
+				t.Fatalf("integer %v (bits=%x): cannot pack: %v", orig, interestingInt, err)
+			}
+			unpacked := p.unpack(packed)
+			if !orig.SameType(unpacked) || orig.Int != unpacked.Int {
+				t.Errorf("integer %v (bits=%x): got packed=%v => unpacked version %v (int: %d)", orig, interestingInt, uint32(packed), unpacked, unpacked.Int)
+			}
+			seenDirectInteger = seenDirectInteger || (uint32(packed)&storageField) == storageFieldDirect
+			seenIndirectInteger = seenIndirectInteger || (uint32(packed)&storageField) == storageFieldIndirect
+		}
+		if !seenDirectInteger {
+			t.Error("did not encounter any integer that could be packed directly")
+		}
+		if !seenIndirectInteger {
+			t.Error("did not encounter any integer that was packed indirectly")
+		}
+	})
+	t.Run("packing_efficiency", func(t *testing.T) {
+		// Verify that we actually saved space by not packing every number in numberPacker itself.
+		if len(p.data) >= len(interestingIntegers) {
+			t.Errorf("packer had %d data points stored in its data, but we expected some of it to not be stored in it (tried to pack %d integers total)", len(p.data), len(interestingIntegers))
+		}
+	})
+	t.Run("floats", func(t *testing.T) {
+		interestingFloats := make(map[float64]struct{}, len(interestingIntegers)+21*21+17)
+		for divExp := -10; divExp < 10; divExp++ {
+			div := math.Pow(10, float64(divExp))
+			for i := -10; i < 10; i++ {
+				interestingFloats[float64(i)*div] = struct{}{}
+			}
+		}
+		interestingFloats[0.0] = struct{}{}
+		interestingFloats[math.NaN()] = struct{}{}
+		interestingFloats[math.Inf(1)] = struct{}{}
+		interestingFloats[math.Inf(-1)] = struct{}{}
+		interestingFloats[math.Pi] = struct{}{}
+		interestingFloats[math.Sqrt2] = struct{}{}
+		interestingFloats[math.E] = struct{}{}
+		interestingFloats[math.SqrtE] = struct{}{}
+		interestingFloats[math.Ln2] = struct{}{}
+		interestingFloats[math.MaxFloat32] = struct{}{}
+		interestingFloats[-math.MaxFloat32] = struct{}{}
+		interestingFloats[math.MaxFloat64] = struct{}{}
+		interestingFloats[-math.MaxFloat64] = struct{}{}
+		interestingFloats[math.SmallestNonzeroFloat32] = struct{}{}
+		interestingFloats[-math.SmallestNonzeroFloat32] = struct{}{}
+		interestingFloats[math.SmallestNonzeroFloat64] = struct{}{}
+		interestingFloats[-math.SmallestNonzeroFloat64] = struct{}{}
+		for interestingInt := range interestingIntegers {
+			interestingFloats[math.Float64frombits(interestingInt)] = struct{}{}
+		}
+		seenDirectFloat := false
+		seenIndirectFloat := false
+		for interestingFloat := range interestingFloats {
+			orig := NewFloat(interestingFloat)
+			packed, err := p.pack(orig)
+			if err != nil {
+				t.Fatalf("float %v (64bits=%x, 32bits=%x. float32-encodable=%v): cannot pack: %v", orig, math.Float64bits(interestingFloat), math.Float32bits(float32(interestingFloat)), float64(float32(interestingFloat)) == interestingFloat, err)
+			}
+			unpacked := p.unpack(packed)
+			switch {
+			case interestingFloat == 0: // Zero-valued float becomes an integer.
+				if !unpacked.IsInteger() {
+					t.Errorf("Zero-valued float %v: got non-integer number: %v", orig, unpacked)
+				} else if unpacked.Int != 0 {
+					t.Errorf("Zero-valued float %v: got non-zero integer: %d", orig, unpacked.Int)
+				}
+			case math.IsNaN(orig.Float):
+				if !math.IsNaN(unpacked.Float) {
+					t.Errorf("NaN float %v: got non-NaN unpacked version %v", orig, unpacked)
+				}
+			default: // Not NaN, not integer
+				if !orig.SameType(unpacked) || orig.Float != unpacked.Float {
+					t.Errorf("float %v (64bits=%x, 32bits=%x, float32-encodable=%v): got packed=%x => unpacked version %v (float: %f)", orig, math.Float64bits(interestingFloat), math.Float32bits(float32(interestingFloat)), float64(float32(interestingFloat)) == interestingFloat, uint32(packed), unpacked, unpacked.Float)
+				}
+			}
+			seenDirectFloat = seenDirectFloat || (uint32(packed)&storageField) == storageFieldDirect
+			seenIndirectFloat = seenIndirectFloat || (uint32(packed)&storageField) == storageFieldIndirect
+		}
+		if !seenDirectFloat {
+			t.Error("did not encounter any float that could be packed directly")
+		}
+		if !seenIndirectFloat {
+			t.Error("did not encounter any float that was packed indirectly")
+		}
+	})
+}

pkg/prometheus/prometheus_verify.go

@@ -32,28 +32,231 @@ const (
 	maxExportStaleness = 10 * time.Second
 )
 
+// internedStringMap allows for interning strings.
+type internedStringMap map[string]*string
+
+// Intern returns the interned version of the given string.
+// If it is not already interned in the map, this function interns it.
+func (m internedStringMap) Intern(s string) string {
+	if existing, found := m[s]; found {
+		return *existing
+	}
+	m[s] = &s
+	return s
+}
+
+// globalInternMap is a string intern map used for globally-relevant data that repeats across
+// verifiers, such as metric names and field names, but not field values or combinations of field
+// values.
+var (
+	globalInternMu  sync.Mutex
+	verifierCount   uint64
+	globalInternMap = make(internedStringMap)
+)
+
+// globalIntern returns the interned version of the given string.
+// If it is not already interned in the map, this function interns it.
+func globalIntern(s string) string {
+	globalInternMu.Lock()
+	defer globalInternMu.Unlock()
+	return globalInternMap.Intern(s)
+}
+
+func globalInternVerifierCreated() {
+	globalInternMu.Lock()
+	defer globalInternMu.Unlock()
+	verifierCount++
+}
+
+func globalInternVerifierReleased() {
+	globalInternMu.Lock()
+	defer globalInternMu.Unlock()
+	verifierCount--
+	if verifierCount <= 0 {
+		verifierCount = 0
+		// No more verifiers active, so release the global map to not keep consuming needless resources.
+		globalInternMap = make(internedStringMap)
+	}
+}
+
+// numberPacker holds packedNumber data. It is useful to store large amounts of Number structs in a
+// small memory footprint.
+type numberPacker struct {
+	data []uint64
+}
+
+// packedNumber is a non-serializable but smaller-memory-footprint container for a numerical value.
+// It can be unpacked out to a Number struct.
+// This contains 4 bytes where we try to pack as much as possible.
+// For the overhwelmingly-common case of integers that fit in 30 bits (i.e. 30 bits where the first
+// 2 bits are zero, we store them directly here. Otherwise, we store the offset of a 64-bit number
+// within numberPacker.
+// Layout, going from highest to lowest bit:
+// Bit 0 is the type: 0 for integer, 1 for float.
+// Bit 1 is 0 if the number's value is stored within the next 30 bits, or 1 if the next 30 bits
+// refer to an offset within numberPacker instead.
+// In the case of a float, the next two bits (bits 2 and 3) may be used to encode a special value:
+//   - 00 means not a special value
+//   - 01 means NaN
+//   - 10 means -infinity
+//   - 11 means +infinity
+//
+// When not using a special value, the 32-bit exponent must fit in 5 bits, and is encoded using a
+// bias of 2^4, meaning it ranges from -15 (encoded as 0b00000) to 16 (encoded as 0b11111), and an
+// exponent of 0 is encoded as 0b01111.
+// Floats that do not fit within this range must be encoded indirectly as float64s, similar to
+// integers that don't fit in 30 bits.
+type packedNumber uint32
+
+// Useful masks and other bit-twiddling stuff for packedNumber.
+const (
+	typeField                = uint32(1 << 31)
+	typeFieldInteger         = uint32(0)
+	typeFieldFloat           = uint32(typeField)
+	storageField             = uint32(1 << 30)
+	storageFieldDirect       = uint32(0)
+	storageFieldIndirect     = uint32(storageField)
+	valueField               = uint32(1<<30 - 1)
+	float32ExponentField     = uint32(0x7f800000)
+	float32ExponentShift     = uint32(23)
+	float32ExponentBias      = uint32(127)
+	float32FractionField     = uint32(0x7fffff)
+	packedFloatExponentField = uint32(0x0f800000)
+	packedFloatExponentBias  = uint32(15)
+	packedFloatNaN           = packedNumber(typeFieldFloat | storageFieldDirect | 0x10000000)
+	packedFloatNegInf        = packedNumber(typeFieldFloat | storageFieldDirect | 0x20000000)
+	packedFloatInf           = packedNumber(typeFieldFloat | storageFieldDirect | 0x30000000)
+)
+
+// errOutOfPackerMemory is returned when the number cannot be packed into a numberPacker.
+var errOutOfPackerMemory = errors.New("out of numberPacker memory")
+
+// pack packs a Number into a packedNumber.
+func (p *numberPacker) pack(n *Number) (packedNumber, error) {
+	if n.Float == 0.0 {
+		v := n.Int
+		if v >= 0 && v <= int64(valueField) {
+			// We can store the integer value directly.
+			return packedNumber(typeFieldInteger | storageFieldDirect | uint32(v)), nil
+		}
+		// Need to allocate a new entry in packedNumber.
+		newIndex := uint32(len(p.data))
+		if newIndex > valueField {
+			return 0, errOutOfPackerMemory
+		}
+		p.data = append(p.data, uint64(v))
+		return packedNumber(typeFieldInteger | storageFieldIndirect | newIndex), nil
+	}
+	// n is a float.
+	v := n.Float
+	if math.IsNaN(v) {
+		return packedFloatNaN, nil
+	}
+	if v == math.Inf(-1) {
+		return packedFloatNegInf, nil
+	}
+	if v == math.Inf(1) {
+		return packedFloatInf, nil
+	}
+	if v >= 0.0 && float64(float32(v)) == v {
+		float32Bits := math.Float32bits(float32(v))
+		exponent := (float32Bits&float32ExponentField)>>float32ExponentShift - float32ExponentBias
+		packedExponent := (exponent + packedFloatExponentBias) << float32ExponentShift
+		if packedExponent&packedFloatExponentField == packedExponent {
+			float32Fraction := float32Bits & float32FractionField
+			return packedNumber(typeFieldFloat | storageFieldDirect | packedExponent | float32Fraction), nil
+		}
+	}
+	// Need to allocate a new entry in packedNumber.
+	newIndex := uint32(len(p.data))
+	if newIndex > valueField {
+		return 0, errOutOfPackerMemory
+	}
+	p.data = append(p.data, math.Float64bits(v))
+	return packedNumber(typeFieldFloat | storageFieldIndirect | newIndex), nil
+}
+
+func (p *numberPacker) mustPack(n *Number) packedNumber {
+	packed, err := p.pack(n)
+	if err != nil {
+		panic(err)
+	}
+	return packed
+}
+
+func (p *numberPacker) mustPackInt(val int64) packedNumber {
+	n := Number{Int: val}
+	return p.mustPack(&n)
+}
+
+func (p *numberPacker) mustPackFloat(val float64) packedNumber {
+	n := Number{Float: val}
+	return p.mustPack(&n)
+}
+
+// unpack unpacks a packedNumber back into a Number.
+func (p *numberPacker) unpack(n packedNumber) *Number {
+	switch uint32(n) & typeField {
+	case typeFieldInteger:
+		switch uint32(n) & storageField {
+		case storageFieldDirect:
+			return NewInt(int64(uint32(n) & valueField))
+		case storageFieldIndirect:
+			return NewInt(int64(p.data[uint32(n)&valueField]))
+		}
+	case typeFieldFloat:
+		switch uint32(n) & storageField {
+		case storageFieldDirect:
+			switch n {
+			case packedFloatNaN:
+				return NewFloat(math.NaN())
+			case packedFloatNegInf:
+				return NewFloat(math.Inf(-1))
+			case packedFloatInf:
+				return NewFloat(math.Inf(1))
+			default:
+				exponent := ((uint32(n) & packedFloatExponentField) >> float32ExponentShift) - packedFloatExponentBias
+				float32Bits := ((exponent + float32ExponentBias) << float32ExponentShift) | (uint32(n) & float32FractionField)
+				return NewFloat(float64(math.Float32frombits(float32Bits)))
+			}
+		case storageFieldIndirect:
+			return NewFloat(math.Float64frombits(p.data[uint32(n)&valueField]))
+		}
+	}
+	panic("unreachable")
+}
+
+func (p *numberPacker) mustUnpackInt(n packedNumber) int64 {
+	num := p.unpack(n)
+	if !num.IsInteger() {
+		panic("not an integer")
+	}
+	return num.Int
+}
+
 // verifiableMetric verifies a single metric within a Verifier.
 type verifiableMetric struct {
 	metadata              *pb.MetricMetadata
 	wantMetric            Metric
-	numFields             int
+	numFields             uint32
+	verifier              *Verifier
 	allowedFieldValues    map[string]map[string]struct{}
-	wantBucketUpperBounds []Number
+	wantBucketUpperBounds []packedNumber
 
 	// The following fields are used to verify that values are actually increasing monotonically.
 	// They are only read and modified when the parent Verifier.mu is held.
 	// They are mapped by their combination of field values.
 
 	// lastCounterValue is used for counter metrics.
-	lastCounterValue map[string]Number
+	lastCounterValue map[string]packedNumber
 
 	// lastBucketSamples is used for distribution ("histogram") metrics.
-	lastBucketSamples map[string][]uint64
+	lastBucketSamples map[string][]packedNumber
 }
 
 // newVerifiableMetric creates a new verifiableMetric that can verify the
 // values of a metric with the given metadata.
-func newVerifiableMetric(metadata *pb.MetricMetadata) (*verifiableMetric, error) {
+func newVerifiableMetric(metadata *pb.MetricMetadata, verifier *Verifier) (*verifiableMetric, error) {
 	if metadata.GetName() == "" || metadata.GetPrometheusName() == "" {
 		return nil, errors.New("metric has no name")
 	}
@@ -65,7 +268,7 @@ func newVerifiableMetric(metadata *pb.MetricMetadata) (*verifiableMetric, error)
 			return nil, fmt.Errorf("invalid character %c in prometheus metric name %q", r, metadata.GetPrometheusName())
 		}
 	}
-	numFields := len(metadata.GetFields())
+	numFields := uint32(len(metadata.GetFields()))
 	var allowedFieldValues map[string]map[string]struct{}
 	if numFields > 0 {
 		seenFields := make(map[string]struct{}, numFields)
@@ -84,16 +287,17 @@ func newVerifiableMetric(metadata *pb.MetricMetadata) (*verifiableMetric, error)
 				if _, alreadyExists := fieldValues[value]; alreadyExists {
 					return nil, fmt.Errorf("field %s has duplicate allowed value %q", fieldName, value)
 				}
-				fieldValues[value] = struct{}{}
+				fieldValues[globalIntern(value)] = struct{}{}
 			}
-			allowedFieldValues[fieldName] = fieldValues
+			allowedFieldValues[globalIntern(fieldName)] = fieldValues
 		}
 	}
 	v := &verifiableMetric{
 		metadata: metadata,
+		verifier: verifier,
 		wantMetric: Metric{
-			Name: metadata.GetPrometheusName(),
-			Help: metadata.GetDescription(),
+			Name: globalIntern(metadata.GetPrometheusName()),
+			Help: globalIntern(metadata.GetDescription()),
 		},
 		numFields:          numFields,
 		allowedFieldValues: allowedFieldValues,
@@ -104,7 +308,7 @@ func newVerifiableMetric(metadata *pb.MetricMetadata) (*verifiableMetric, error)
 		v.wantMetric.Type = TypeGauge
 		if metadata.GetCumulative() {
 			v.wantMetric.Type = TypeCounter
-			v.lastCounterValue = make(map[string]Number, numFieldCombinations)
+			v.lastCounterValue = make(map[string]packedNumber, numFieldCombinations)
 		}
 	case pb.MetricMetadata_TYPE_DISTRIBUTION:
 		v.wantMetric.Type = TypeHistogram
@@ -112,12 +316,12 @@ func newVerifiableMetric(metadata *pb.MetricMetadata) (*verifiableMetric, error)
 		if numBuckets <= 1 || numBuckets > 256 {
 			return nil, fmt.Errorf("unsupported number of buckets: %d", numBuckets)
 		}
-		v.wantBucketUpperBounds = make([]Number, numBuckets)
+		v.wantBucketUpperBounds = make([]packedNumber, numBuckets)
 		for i, boundary := range metadata.GetDistributionBucketLowerBounds() {
-			v.wantBucketUpperBounds[i] = Number{Int: boundary}
+			v.wantBucketUpperBounds[i] = verifier.permanentPacker.mustPackInt(boundary)
 		}
-		v.wantBucketUpperBounds[numBuckets-1] = Number{Float: math.Inf(1)}
-		v.lastBucketSamples = make(map[string][]uint64, numFieldCombinations)
+		v.wantBucketUpperBounds[numBuckets-1] = verifier.permanentPacker.mustPackFloat(math.Inf(1))
+		v.lastBucketSamples = make(map[string][]packedNumber, numFieldCombinations)
 	default:
 		return nil, fmt.Errorf("invalid type: %v", metadata.GetType())
 	}
@@ -133,13 +337,14 @@ func (v *verifiableMetric) numFieldCombinations() int {
 // field values that have already been seen. `verify` should populate this.
 // `dataToFieldsSeen` is passed across calls to `verify` and other methods of `verifiableMetric`.
 // It is used to store the canonical representation of the field values seen for each *Data.
+// Precondition: `Verifier.mu` is held.
 func (v *verifiableMetric) verify(data *Data, metricFieldsSeen map[string]struct{}, dataToFieldsSeen map[*Data]string) error {
 	if *data.Metric != v.wantMetric {
 		return fmt.Errorf("invalid metric definition: got %+v want %+v", data.Metric, v.wantMetric)
 	}
 
 	// Verify fields.
-	if len(data.Labels) != v.numFields {
+	if uint32(len(data.Labels)) != v.numFields {
 		return fmt.Errorf("invalid number of fields: got %d want %d", len(data.Labels), v.numFields)
 	}
 	var fieldValues strings.Builder
@@ -193,7 +398,7 @@ func (v *verifiableMetric) verify(data *Data, metricFieldsSeen map[string]struct
 			return fmt.Errorf("invalid number of buckets: got %d want %d", len(data.HistogramValue.Buckets), len(v.wantBucketUpperBounds))
 		}
 		for i, b := range data.HistogramValue.Buckets {
-			if want := v.wantBucketUpperBounds[i]; b.UpperBound != want {
+			if want := v.wantBucketUpperBounds[i]; b.UpperBound != *v.verifier.permanentPacker.unpack(want) {
 				return fmt.Errorf("invalid upper bound for bucket %d (0-based): got %v want %v", i, b.UpperBound, want)
 			}
 		}
@@ -202,6 +407,7 @@ func (v *verifiableMetric) verify(data *Data, metricFieldsSeen map[string]struct
 	}
 
 	// All passed. Update the maps that are shared across calls.
+	fieldValuesStr = v.verifier.internMap.Intern(fieldValuesStr)
 	dataToFieldsSeen[data] = fieldValuesStr
 	metricFieldsSeen[fieldValuesStr] = struct{}{}
 	return nil
@@ -209,10 +415,10 @@ func (v *verifiableMetric) verify(data *Data, metricFieldsSeen map[string]struct
 
 // verifyIncrement verifies that incremental metrics are monotonically increasing.
 // Preconditions: `verify` has succeeded on the given `data`, and `Verifier.mu` is held.
-func (v *verifiableMetric) verifyIncrement(data *Data, fieldValues string) error {
+func (v *verifiableMetric) verifyIncrement(data *Data, fieldValues string, packer *numberPacker) error {
 	switch v.wantMetric.Type {
 	case TypeCounter:
-		last := v.lastCounterValue[fieldValues]
+		last := packer.unpack(v.lastCounterValue[v.verifier.internMap.Intern(fieldValues)])
 		if !last.SameType(data.Number) {
 			return fmt.Errorf("counter number type changed: %v vs %v", last, data.Number)
 		}
@@ -220,14 +426,14 @@ func (v *verifiableMetric) verifyIncrement(data *Data, fieldValues string) error
 			return fmt.Errorf("counter value decreased from %v to %v", last, data.Number)
 		}
 	case TypeHistogram:
-		lastBucketSamples := v.lastBucketSamples[fieldValues]
+		lastBucketSamples := v.lastBucketSamples[v.verifier.internMap.Intern(fieldValues)]
 		if lastBucketSamples == nil {
-			lastBucketSamples = make([]uint64, len(v.wantBucketUpperBounds))
-			v.lastBucketSamples[fieldValues] = lastBucketSamples
+			lastBucketSamples = make([]packedNumber, len(v.wantBucketUpperBounds))
+			v.lastBucketSamples[v.verifier.internMap.Intern(fieldValues)] = lastBucketSamples
 		}
 		for i, b := range data.HistogramValue.Buckets {
-			if lastBucketSamples[i] > b.Samples {
-				return fmt.Errorf("number of samples in bucket %d (0-based) decreased from %d to %d", i, lastBucketSamples[i], b.Samples)
+			if uint64(packer.mustUnpackInt(lastBucketSamples[i])) > b.Samples {
+				return fmt.Errorf("number of samples in bucket %d (0-based) decreased from %d to %d", i, packer.mustUnpackInt(lastBucketSamples[i]), b.Samples)
 			}
 		}
 	}
@@ -236,14 +442,14 @@ func (v *verifiableMetric) verifyIncrement(data *Data, fieldValues string) error
 
 // update updates incremental metrics' "last seen" data.
 // Preconditions: `verifyIncrement` has succeeded on the given `data`, and `Verifier.mu` is held.
-func (v *verifiableMetric) update(data *Data, fieldValues string) {
+func (v *verifiableMetric) update(data *Data, fieldValues string, packer *numberPacker) {
 	switch v.wantMetric.Type {
 	case TypeCounter:
-		v.lastCounterValue[fieldValues] = *data.Number
+		v.lastCounterValue[v.verifier.internMap.Intern(fieldValues)] = packer.mustPack(data.Number)
 	case TypeHistogram:
-		lastBucketSamples := v.lastBucketSamples[fieldValues]
+		lastBucketSamples := v.lastBucketSamples[v.verifier.internMap.Intern(fieldValues)]
 		for i, b := range data.HistogramValue.Buckets {
-			lastBucketSamples[i] = b.Samples
+			lastBucketSamples[i] = packer.mustPackInt(int64(b.Samples))
 		}
 	}
 }
@@ -253,29 +459,50 @@ func (v *verifiableMetric) update(data *Data, fieldValues string) {
 // A single Verifier should be used per sandbox. It is expected to be reused across exports such
 // that it can enforce the export snapshot timestamp is strictly monotonically increasing.
 type Verifier struct {
-	knownMetrics  map[string]*verifiableMetric
-	mu            sync.Mutex
+	knownMetrics map[string]*verifiableMetric
+
+	// permanentPacker is used to pack numbers seen at Verifier initialization time only.
+	permanentPacker *numberPacker
+
+	// mu protects the fields below.
+	mu sync.Mutex
+
+	// internMap is used to intern strings relevant to this verifier only.
+	// Globally-relevant strings should be interned in globalInternMap.
+	internMap internedStringMap
+
+	// lastPacker is a reference to the numberPacker used to pack numbers in the last successful
+	// verification round.
+	lastPacker *numberPacker
+
+	// lastTimestamp is the snapshot timestamp of the last successfully-verified snapshot.
 	lastTimestamp time.Time
 }
 
 // NewVerifier returns a new metric verifier that can verify the integrity of snapshots against
 // the given metric registration data.
-func NewVerifier(registration *pb.MetricRegistration) (*Verifier, error) {
-	knownMetrics := make(map[string]*verifiableMetric)
+// It returns a cleanup function that must be called when the Verifier is no longer needed.
+func NewVerifier(registration *pb.MetricRegistration) (*Verifier, func(), error) {
+	globalInternVerifierCreated()
+	verifier := &Verifier{
+		knownMetrics:    make(map[string]*verifiableMetric),
+		permanentPacker: &numberPacker{},
+		internMap:       make(internedStringMap),
+	}
 	for _, metric := range registration.GetMetrics() {
 		metricName := metric.GetPrometheusName()
-		if _, alreadyExists := knownMetrics[metricName]; alreadyExists {
-			return nil, fmt.Errorf("metric %q registered twice", metricName)
+		if _, alreadyExists := verifier.knownMetrics[metricName]; alreadyExists {
+			globalInternVerifierReleased()
+			return nil, func() {}, fmt.Errorf("metric %q registered twice", metricName)
 		}
-		verifiableM, err := newVerifiableMetric(metric)
+		verifiableM, err := newVerifiableMetric(metric, verifier)
 		if err != nil {
-			return nil, fmt.Errorf("metric %q: %v", metricName, err)
+			globalInternVerifierReleased()
+			return nil, func() {}, fmt.Errorf("metric %q: %v", metricName, err)
 		}
-		knownMetrics[metricName] = verifiableM
+		verifier.knownMetrics[globalIntern(metricName)] = verifiableM
 	}
-	return &Verifier{
-		knownMetrics: knownMetrics,
-	}, nil
+	return verifier, globalInternVerifierReleased, nil
 }
 
 // Verify verifies the integrity of a snapshot against the metric registration data of the Verifier.
@@ -292,6 +519,10 @@ func (v *Verifier) Verify(snapshot *Snapshot) error {
 		return fmt.Errorf("snapshot is too old; it is from %v, expected at least %v (%v from now)", snapshot.When, now.Add(-maxExportStaleness), maxExportStaleness)
 	}
 
+	// Start critical section.
+	v.mu.Lock()
+	defer v.mu.Unlock()
+
 	// Metrics checks.
 	fieldsSeen := make(map[string]map[string]struct{}, len(v.knownMetrics))
 	dataToFieldsSeen := make(map[*Data]string, len(snapshot.Data))
@@ -301,6 +532,7 @@ func (v *Verifier) Verify(snapshot *Snapshot) error {
 		if !found {
 			return fmt.Errorf("snapshot contains unknown metric %q", metricName)
 		}
+		metricName = globalIntern(metricName)
 		metricFieldsSeen, found := fieldsSeen[metricName]
 		if !found {
 			metricFieldsSeen = make(map[string]struct{}, verifiableM.numFieldCombinations())
@@ -311,22 +543,21 @@ func (v *Verifier) Verify(snapshot *Snapshot) error {
 		}
 	}
 
-	// Start the critical section.
-	v.mu.Lock()
-	defer v.mu.Unlock()
 	if v.lastTimestamp.After(snapshot.When) {
 		return fmt.Errorf("consecutive snapshots are not chronologically ordered: last verified snapshot was exported at %v, this one is from %v", v.lastTimestamp, snapshot.When)
 	}
 	for _, data := range snapshot.Data {
-		if err = v.knownMetrics[data.Metric.Name].verifyIncrement(data, dataToFieldsSeen[data]); err != nil {
+		if err = v.knownMetrics[data.Metric.Name].verifyIncrement(data, dataToFieldsSeen[data], v.lastPacker); err != nil {
 			return fmt.Errorf("metric %q: %v", data.Metric.Name, err)
 		}
 	}
 
 	// All checks succeeded, update last-seen data.
+	newPacker := &numberPacker{}
 	v.lastTimestamp = snapshot.When
 	for _, data := range snapshot.Data {
-		v.knownMetrics[data.Metric.Name].update(data, dataToFieldsSeen[data])
+		v.knownMetrics[globalIntern(data.Metric.Name)].update(data, v.internMap.Intern(dataToFieldsSeen[data]), newPacker)
 	}
+	v.lastPacker = newPacker
 	return nil
 }

pkg/state/types.go

@@ -324,6 +324,14 @@ var globalTypeDatabase = map[string]reflect.Type{}
 // reverseTypeDatabase is a reverse mapping.
 var reverseTypeDatabase = map[reflect.Type]string{}
 
+// Release releases references to global type databases.
+// Must only be called in contexts where they will definitely never be used,
+// in order to save memory.
+func Release() {
+	globalTypeDatabase = nil
+	reverseTypeDatabase = nil
+}
+
 // Register registers a type.
 //
 // This must be called on init and only done once.

runsc/cmd/BUILD

@@ -64,6 +64,7 @@ go_library(
         "//pkg/sentry/kernel",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/platform",
+        "//pkg/state",
         "//pkg/state/pretty",
         "//pkg/state/statefile",
         "//pkg/sync",

runsc/cmd/metric_server.go

@@ -28,6 +28,7 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
+	"runtime/debug"
 	"runtime/pprof"
 	"strconv"
 	"strings"
@@ -38,6 +39,7 @@ import (
 	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/prometheus"
+	"gvisor.dev/gvisor/pkg/state"
 	"gvisor.dev/gvisor/pkg/sync"
 	"gvisor.dev/gvisor/runsc/cmd/util"
 	"gvisor.dev/gvisor/runsc/config"
@@ -100,6 +102,9 @@ type servedSandbox struct {
 	// be deleted from the server.
 	// Once set, it is immutable.
 	verifier *prometheus.Verifier
+
+	// cleanupVerifier holds a reference to the cleanup function of the verifier.
+	cleanupVerifier func()
 }
 
 // sandboxPrometheusLabels returns a set of Prometheus labels that identifies the sandbox running
@@ -183,11 +188,12 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) {
 		if err != nil {
 			return nil, nil, err
 		}
-		verifier, err := prometheus.NewVerifier(registeredMetrics)
+		verifier, cleanup, err := prometheus.NewVerifier(registeredMetrics)
 		if err != nil {
 			return nil, nil, err
 		}
 		s.verifier = verifier
+		s.cleanupVerifier = cleanup
 	}
 	s.labelsWithMetadata = make(map[string]string, len(s.extraLabels)+len(s.sandbox.MetricMetadata))
 	for k, v := range s.extraLabels {
@@ -199,25 +205,36 @@ func (s *servedSandbox) load() (*sandbox.Sandbox, *prometheus.Verifier, error) {
 	return s.sandbox, s.verifier, nil
 }
 
+func (s *servedSandbox) cleanup() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.cleanupVerifier != nil {
+		s.cleanupVerifier()
+	}
+}
+
 // queryMetrics queries the sandbox for metrics data.
 func queryMetrics(ctx context.Context, sand *sandbox.Sandbox, verifier *prometheus.Verifier) (*prometheus.Snapshot, error) {
 	ch := make(chan struct {
 		snapshot *prometheus.Snapshot
 		err      error
 	}, 1)
-	defer close(ch)
+	canceled := make(chan struct{}, 1)
+	defer close(canceled)
 	go func() {
 		snapshot, err := sand.ExportMetrics()
 		select {
+		case <-canceled:
 		case ch <- struct {
 			snapshot *prometheus.Snapshot
 			err      error
 		}{snapshot, err}:
-		default:
+			close(ch)
 		}
 	}()
 	select {
 	case <-ctx.Done():
+		canceled <- struct{}{}
 		return nil, ctx.Err()
 	case ret := <-ch:
 		if ret.err != nil {
@@ -356,16 +373,19 @@ func (m *MetricServer) refreshSandboxesLocked() {
 		}
 		if !found {
 			log.Warningf("Sandbox %s no longer exists but did not explicitly unregister. Removing it.", sandboxID)
+			sandbox.cleanup()
 			delete(m.sandboxes, sandboxID)
 			continue
 		}
 		if _, _, err := sandbox.load(); err != nil && err != container.ErrStateFileLocked {
 			log.Warningf("Sandbox %s cannot be loaded, deleting it: %v", sandboxID, err)
+			sandbox.cleanup()
 			delete(m.sandboxes, sandboxID)
 			continue
 		}
 		if !sandbox.sandbox.IsRunning() {
 			log.Infof("Sandbox %s is no longer running, deleting it.", sandboxID)
+			sandbox.cleanup()
 			delete(m.sandboxes, sandboxID)
 			continue
 		}
@@ -444,6 +464,7 @@ func (m *MetricServer) refreshSandboxesLocked() {
 		if _, _, err := served.load(); err != nil && err != container.ErrStateFileLocked {
 			log.Warningf("Sandbox %q cannot be loaded, ignoring it: %v", sid, err)
 			m.lastStateFileStat[sid] = stat
+			served.cleanup()
 			continue
 		}
 		m.sandboxes[sid] = served
@@ -845,6 +866,8 @@ func logRequest(f func(w http.ResponseWriter, req *http.Request) httpResult) fun
 			http.Error(w, result.err.Error(), result.code)
 			log.Warningf("Request: %s %s: Failed with HTTP code %d: %v", req.Method, req.URL.Path, result.code, result.err)
 		}
+		// Run GC after every request to keep memory usage as predictable and as flat as possible.
+		runtime.GC()
 	}
 }
 
@@ -979,6 +1002,9 @@ func (m *MetricServer) Execute(ctx context.Context, f *flag.FlagSet, args ...any
 		log.Warningf("Profiling HTTP endpoints are exposed; this should only be used for development!")
 		mux.HandleFunc("/runsc-metrics/profile-cpu", logRequest(m.profileCPU))
 		mux.HandleFunc("/runsc-metrics/profile-heap", logRequest(m.profileHeap))
+	} else {
+		// Disable memory profiling, since we don't expose it.
+		runtime.MemProfileRate = 0
 	}
 	mux.HandleFunc("/metrics", logRequest(m.serveMetrics))
 	mux.HandleFunc("/", logRequest(m.serveIndex))
@@ -994,9 +1020,18 @@ func (m *MetricServer) Execute(ctx context.Context, f *flag.FlagSet, args ...any
 		log.Infof("Wrote PID %d to file %v.", m.pid, m.pidFile)
 	}
 
+	// If not modified by the user from the environment, set the Go GC percentage lower than default.
+	if _, hasEnv := os.LookupEnv("GOGC"); !hasEnv {
+		debug.SetGCPercent(40)
+	}
+
+	// Run GC immediately to get rid of all the initialization-related memory bloat and start from
+	// a clean slate.
+	state.Release()
+	runtime.GC()
+
 	// Initialization complete.
 	log.Infof("Server serving on %s for root directory %s.", conf.MetricServer, conf.RootDir)
-
 	serveErr := m.srv.Serve(listener)
 	log.Infof("Server has stopped accepting requests.")
 	m.mu.Lock()