From 596e8d22b9027c249a88880806da69d531d7a728 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Thu, 11 Apr 2024 19:02:00 -0700 Subject: [PATCH] Increase `sandbox.waitForStopped` timeout yet again. For humongous containers (>256GiB of RAM), this can exceed 45s even on beefy machines. I considered making the timeout scale based on the memory usage of the sandbox, but this is only an approximate proxy metric for time to shut down, and I don't think we can reliably get it once the sandbox process is already shutting down. PiperOrigin-RevId: 624009617 --- runsc/sandbox/sandbox.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 07864ccfd..d175187c7 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -1451,7 +1451,10 @@ func (s *Sandbox) destroyContainer(cid string) error { return nil } +// waitForStopped waits for the sandbox to actually stop. +// This should only be called when the sandbox is known to be shutting down. func (s *Sandbox) waitForStopped() error { + const waitTimeout = 2 * time.Minute if s.child { s.statusMu.Lock() defer s.statusMu.Unlock() @@ -1467,8 +1470,7 @@ func (s *Sandbox) waitForStopped() error { s.Pid.store(0) return nil } - - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), waitTimeout) defer cancel() b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx) op := func() error {