Add a stress test for multicontainers to check for memory leak.

The test runs 50 subcontainers that do a lot of filesystem work and checks that
after the containers exit, there is no increase in sandbox memory usage.

PiperOrigin-RevId: 501364377

runsc/container/multi_container_test.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -2291,3 +2292,117 @@ func TestMultiContainerOverlayLeaks(t *testing.T) {
 		t.Errorf("overlay filestore usage changed: old = %d, new = %d", oldOverlayUsage, newOverlayUsage)
 	}
 }
+
+// Test that spawning many subcontainers that do a lot of filesystem operations
+// does not lead to memory leaks.
+func TestMultiContainerMemoryLeakStress(t *testing.T) {
+	conf := testutil.TestConfig(t)
+	app, err := testutil.FindFile("test/cmd/test_app/test_app")
+	if err != nil {
+		t.Fatal("error finding test_app:", err)
+	}
+
+	rootDir, cleanup, err := testutil.SetupRootDir()
+	if err != nil {
+		t.Fatalf("error creating root dir: %v", err)
+	}
+	defer cleanup()
+	conf.RootDir = rootDir
+
+	// Configure root overlay (backed by memory) so that containers can create
+	// files in the root directory.
+	conf.Overlay2 = config.Overlay2{
+		RootMount: true,
+	}
+
+	// Root container will just sleep.
+	sleep := []string{"sleep", "1000"}
+
+	// Subcontainers will do a lot of filesystem work. Create a lot of them.
+	createFsTree := []string{app, "fsTreeCreate", "--depth=10", "--file-per-level=10", "--file-size=1048576"}
+	const warmupContainers = 5
+	const stressContainers = 45
+	cmds := make([][]string, 0, warmupContainers+stressContainers+1)
+	cmds = append(cmds, sleep)
+	for i := 0; i < warmupContainers+stressContainers; i++ {
+		cmds = append(cmds, createFsTree)
+	}
+	testSpecs, ids := createSpecs(cmds...)
+	// Make sure none of the root filesystems are read-only, otherwise we won't
+	// be able to create the file.
+	for _, s := range testSpecs {
+		s.Root.Readonly = false
+	}
+
+	// Start the root container.
+	rootCont, cleanup, err := startContainers(conf, testSpecs[:1], ids[:1])
+	if err != nil {
+		t.Fatalf("error starting containers: %v", err)
+	}
+	defer cleanup()
+
+	// Warm up the sandbox.
+	warmUpContainers, cleanUp2, err := startContainers(conf, testSpecs[1:1+warmupContainers], ids[1:1+warmupContainers])
+	if err != nil {
+		t.Fatalf("error starting containers: %v", err)
+	}
+	defer cleanUp2()
+	// Wait for all warm up subcontainers to stop.
+	for i, c := range warmUpContainers {
+		// Wait for the sub-container to stop.
+		if ws, err := c.Wait(); err != nil {
+			t.Errorf("failed to wait for warm up subcontainer number %d: %v", i, err)
+		} else if es := ws.ExitStatus(); es != 0 {
+			t.Errorf("warm up subcontainer number %d exited with non-zero status %d", i, es)
+		}
+	}
+
+	// Give the reclaimer goroutine some time to reclaim.
+	time.Sleep(3 * time.Second)
+
+	// Measure the memory usage after the warm up.
+	oldUsage, err := rootCont[0].Sandbox.Usage(true /* Full */)
+	if err != nil {
+		t.Fatalf("sandbox.Usage failed: %v", err)
+	}
+
+	// Hammer the sandbox with sub containers.
+	subConts, cleanup3, err := startContainers(conf, testSpecs[1+warmupContainers:1+warmupContainers+stressContainers], ids[1+warmupContainers:1+warmupContainers+stressContainers])
+	if err != nil {
+		t.Fatalf("error starting containers: %v", err)
+	}
+	defer cleanup3()
+	// Wait for all subcontainers to stop.
+	for i, c := range subConts {
+		if ws, err := c.Wait(); err != nil {
+			t.Errorf("failed to wait for subcontainer number %d: %v", i, err)
+		} else if es := ws.ExitStatus(); es != 0 {
+			t.Errorf("subcontainer number %d exited with non-zero status %d", i, es)
+		}
+	}
+
+	// Give the reclaimer goroutine some time to reclaim.
+	time.Sleep(3 * time.Second)
+
+	// Compare memory usage.
+	newUsage, err := rootCont[0].Sandbox.Usage(true /* Full */)
+	if err != nil {
+		t.Fatalf("sandbox.Usage failed: %v", err)
+	}
+	// Note that all fields of control.MemoryUsage are exported and uint64.
+	oldUsageV := reflect.ValueOf(oldUsage)
+	newUsageV := reflect.ValueOf(newUsage)
+	numFields := oldUsageV.NumField()
+	for i := 0; i < numFields; i++ {
+		name := oldUsageV.Type().Field(i).Name
+		oldVal := oldUsageV.Field(i).Interface().(uint64)
+		newVal := newUsageV.Field(i).Interface().(uint64)
+		if newVal <= oldVal {
+			continue
+		}
+
+		if ((newVal-oldVal)*100)/oldVal > 5 {
+			t.Errorf("%s usage increased by more than 5%%: old=%d, new=%d", name, oldVal, newVal)
+		}
+	}
+}