kernel: fix lock order inversion in ThreadGroup.Release()

PiperOrigin-RevId: 681199251

pkg/sentry/kernel/kernel.go

@@ -19,12 +19,13 @@
 // Lock order (outermost locks must be taken first):
 //
 //	Kernel.extMu
-//		ThreadGroup.timerMu
-//		  ktime.Timer.mu (for IntervalTimer) and Kernel.cpuClockMu
-//		    TaskSet.mu
-//		      SignalHandlers.mu
-//		        Task.mu
-//		    runningTasksMu
+//	  TTY.mu
+//	  ThreadGroup.timerMu
+//	    ktime.Timer.mu (for IntervalTimer) and Kernel.cpuClockMu
+//	      TaskSet.mu
+//	        SignalHandlers.mu
+//	          Task.mu
+//	      runningTasksMu
 //
 // Locking SignalHandlers.mu in multiple SignalHandlers requires locking
 // TaskSet.mu exclusively first. Locking Task.mu in multiple Tasks at the same

pkg/sentry/kernel/thread_group.go

@@ -343,15 +343,22 @@ func (tg *ThreadGroup) Release(ctx context.Context) {
 	}
 	clear(tg.timers) // nil maps can't be saved
 	// Disassociate from the tty if we have one.
+	var tty *TTY
 	if tg.tty != nil {
-		tg.tty.mu.Lock() // FIXME(b/370763686)
-		if tg.tty.tg == tg {
-			tg.tty.tg = nil
-		}
-		tg.tty.mu.Unlock()
-		tg.tty = nil
+		// Can't lock tty.mu due to lock ordering.
+		tty = tg.tty
 	}
 	tg.signalHandlers.mu.Unlock()
+	if tty != nil {
+		tty.mu.Lock()
+		tg.signalHandlers.mu.Lock()
+		tg.tty = nil
+		if tty.tg == tg {
+			tty.tg = nil
+		}
+		tg.signalHandlers.mu.Unlock()
+		tty.mu.Unlock()
+	}
 	for _, it := range its {
 		it.DestroyTimer()
 	}