diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go index 61553d5f2..6626962f6 100644 --- a/runsc/container/multi_container_test.go +++ b/runsc/container/multi_container_test.go @@ -21,6 +21,7 @@ import ( "os" "path" "path/filepath" + "reflect" "strings" "testing" "time" @@ -2291,3 +2292,117 @@ func TestMultiContainerOverlayLeaks(t *testing.T) { t.Errorf("overlay filestore usage changed: old = %d, new = %d", oldOverlayUsage, newOverlayUsage) } } + +// Test that spawning many subcontainers that do a lot of filesystem operations +// does not lead to memory leaks. +func TestMultiContainerMemoryLeakStress(t *testing.T) { + conf := testutil.TestConfig(t) + app, err := testutil.FindFile("test/cmd/test_app/test_app") + if err != nil { + t.Fatal("error finding test_app:", err) + } + + rootDir, cleanup, err := testutil.SetupRootDir() + if err != nil { + t.Fatalf("error creating root dir: %v", err) + } + defer cleanup() + conf.RootDir = rootDir + + // Configure root overlay (backed by memory) so that containers can create + // files in the root directory. + conf.Overlay2 = config.Overlay2{ + RootMount: true, + } + + // Root container will just sleep. + sleep := []string{"sleep", "1000"} + + // Subcontainers will do a lot of filesystem work. Create a lot of them. + createFsTree := []string{app, "fsTreeCreate", "--depth=10", "--file-per-level=10", "--file-size=1048576"} + const warmupContainers = 5 + const stressContainers = 45 + cmds := make([][]string, 0, warmupContainers+stressContainers+1) + cmds = append(cmds, sleep) + for i := 0; i < warmupContainers+stressContainers; i++ { + cmds = append(cmds, createFsTree) + } + testSpecs, ids := createSpecs(cmds...) + // Make sure none of the root filesystems are read-only, otherwise we won't + // be able to create the file. + for _, s := range testSpecs { + s.Root.Readonly = false + } + + // Start the root container. + rootCont, cleanup, err := startContainers(conf, testSpecs[:1], ids[:1]) + if err != nil { + t.Fatalf("error starting containers: %v", err) + } + defer cleanup() + + // Warm up the sandbox. + warmUpContainers, cleanUp2, err := startContainers(conf, testSpecs[1:1+warmupContainers], ids[1:1+warmupContainers]) + if err != nil { + t.Fatalf("error starting containers: %v", err) + } + defer cleanUp2() + // Wait for all warm up subcontainers to stop. + for i, c := range warmUpContainers { + // Wait for the sub-container to stop. + if ws, err := c.Wait(); err != nil { + t.Errorf("failed to wait for warm up subcontainer number %d: %v", i, err) + } else if es := ws.ExitStatus(); es != 0 { + t.Errorf("warm up subcontainer number %d exited with non-zero status %d", i, es) + } + } + + // Give the reclaimer goroutine some time to reclaim. + time.Sleep(3 * time.Second) + + // Measure the memory usage after the warm up. + oldUsage, err := rootCont[0].Sandbox.Usage(true /* Full */) + if err != nil { + t.Fatalf("sandbox.Usage failed: %v", err) + } + + // Hammer the sandbox with sub containers. + subConts, cleanup3, err := startContainers(conf, testSpecs[1+warmupContainers:1+warmupContainers+stressContainers], ids[1+warmupContainers:1+warmupContainers+stressContainers]) + if err != nil { + t.Fatalf("error starting containers: %v", err) + } + defer cleanup3() + // Wait for all subcontainers to stop. + for i, c := range subConts { + if ws, err := c.Wait(); err != nil { + t.Errorf("failed to wait for subcontainer number %d: %v", i, err) + } else if es := ws.ExitStatus(); es != 0 { + t.Errorf("subcontainer number %d exited with non-zero status %d", i, es) + } + } + + // Give the reclaimer goroutine some time to reclaim. + time.Sleep(3 * time.Second) + + // Compare memory usage. + newUsage, err := rootCont[0].Sandbox.Usage(true /* Full */) + if err != nil { + t.Fatalf("sandbox.Usage failed: %v", err) + } + // Note that all fields of control.MemoryUsage are exported and uint64. + oldUsageV := reflect.ValueOf(oldUsage) + newUsageV := reflect.ValueOf(newUsage) + numFields := oldUsageV.NumField() + for i := 0; i < numFields; i++ { + name := oldUsageV.Type().Field(i).Name + oldVal := oldUsageV.Field(i).Interface().(uint64) + newVal := newUsageV.Field(i).Interface().(uint64) + if newVal <= oldVal { + continue + } + + if ((newVal-oldVal)*100)/oldVal > 5 { + t.Errorf("%s usage increased by more than 5%%: old=%d, new=%d", name, oldVal, newVal) + } + } +}