From 03500a8da647758b36fcaa5a26a83e0336f1b2e1 Mon Sep 17 00:00:00 2001 From: jpolo1224 Date: Mon, 24 Aug 2026 14:30:19 -0400 Subject: [PATCH] Poll the hang watchdog off the RSX thread check_frame_stall() runs from do_local_task, on the RSX thread's own FIFO loop. That works for a guest-side hang with the RSX idle -- every hang chased so far -- and is useless for the opposite case, where the RSX thread is the one stuck. It never returns to do_local_task, so the detector that would report the hang is starved by the hang. Tales of Xillia 2 (BLUS31397) is exactly that. Reproduced on device: guest mutex traffic at zero for minutes while sys_event_queue_receive and sys_timer_usleep tick at flat identical rates, rsx::thread accumulating 4.99s of CPU per 5s of wall clock, and NV406E_SEMAPHORE_ACQUIRE its costliest method at 1.59ms a call. The RSX is spinning on a guest semaphore the stopped guest will never write, and nothing reported any of it. The same condition is now polled once a second from the PPU syscall usage thread, which is independent and keeps running. That half only dumps; the on-screen message and the native-UI flip stay on the RSX side, because the overlay is not safe to drive from another thread. Both share one dump budget so they cannot produce four dumps between them. --- rpcs3/Emu/Cell/lv2/lv2.cpp | 9 +++++++++ rpcs3/Emu/RSX/RSXThread.cpp | 39 +++++++++++++++++++++++++++++++++++++ rpcs3/Emu/RSX/RSXThread.h | 4 ++++ 3 files changed, 52 insertions(+) diff --git a/rpcs3/Emu/Cell/lv2/lv2.cpp b/rpcs3/Emu/Cell/lv2/lv2.cpp index 676df5ab4..68b905ebb 100644 --- a/rpcs3/Emu/Cell/lv2/lv2.cpp +++ b/rpcs3/Emu/Cell/lv2/lv2.cpp @@ -25,6 +25,7 @@ #include "sys_ppu_thread.h" #include "sys_process.h" #include "sys_prx.h" +#include "Emu/RSX/RSXThread.h" #include "sys_rsx.h" #include "sys_rwlock.h" #include "sys_semaphore.h" @@ -1246,6 +1247,14 @@ public: { thread_ctrl::wait_until(&sleep_until, 1'000'000); + // Hang watchdog. This thread is independent of the RSX thread, which is the whole + // point: a hang where the RSX spins inside a method handler starves the stall check + // that lives on it. Cheap -- two atomic loads and a clock read unless it fires. + if (!Emu.IsPaused() && !Emu.IsStopped()) + { + rsx::poll_frame_stall_watchdog(); + } + const bool is_paused = Emu.IsPaused(); // Force-print all if paused diff --git a/rpcs3/Emu/RSX/RSXThread.cpp b/rpcs3/Emu/RSX/RSXThread.cpp index 4db620d9d..6c0ad3e4c 100644 --- a/rpcs3/Emu/RSX/RSXThread.cpp +++ b/rpcs3/Emu/RSX/RSXThread.cpp @@ -1433,6 +1433,45 @@ namespace rsx // idm::unlocked deliberately: this runs on the RSX thread, and taking the id lock here to // diagnose a hang would add exactly the kind of dependency being diagnosed. A torn read of // a diagnostic line costs nothing. + // Independent watchdog, called from a thread that is NOT the RSX thread. + // + // check_frame_stall() runs from do_local_task, on the RSX thread's own FIFO loop. That is + // fine for a guest-side hang with the RSX idle, and useless for the opposite case: Tales of + // Xillia 2 hangs with rsx::thread spinning at 100% CPU inside NV406E_SEMAPHORE_ACQUIRE, + // waiting on a guest semaphore the stopped guest will never write. Spinning inside a method + // handler it never returns to do_local_task, so the detector that would report the hang is + // starved by the hang itself. Measured on the device: +4.99s of thread CPU across 5s of wall + // clock, with guest mutex traffic at exactly zero for minutes. + // + // So the same condition is polled from the PPU syscall usage thread, which ticks once a + // second and keeps running regardless. This half only DUMPS -- the on-screen message and the + // native-UI flip stay on the RSX side, because the overlay is not safe to drive from here. + void poll_frame_stall_watchdog() + { + const u64 now = get_system_time(); + + if (g_progr_text || !g_last_frame_time) + { + return; + } + + if (now - g_last_frame_time < 30'000'000) + { + return; + } + + // Shares the RSX side's counter and its two-sample budget, so the two paths can never + // produce four dumps between them. + if (const u32 taken = g_frame_stall_dumps; taken < 2) + { + g_frame_stall_dumps = taken + 1; + + rsx_log.error("No frame presented in %us and the RSX thread is not polling: dumping " + "guest threads from the watchdog.", (now - g_last_frame_time) / 1'000'000); + + dump_guest_threads_stalled(); + } + } static void dump_guest_threads_stalled() { std::string out; diff --git a/rpcs3/Emu/RSX/RSXThread.h b/rpcs3/Emu/RSX/RSXThread.h index b8bc59ad8..6f3195450 100644 --- a/rpcs3/Emu/RSX/RSXThread.h +++ b/rpcs3/Emu/RSX/RSXThread.h @@ -38,6 +38,10 @@ extern rsx::frame_capture_data frame_capture; namespace rsx { + // Polled once a second from the PPU syscall usage thread. The RSX-side stall check cannot + // report a hang in which the RSX thread itself is spinning; see the definition. + void poll_frame_stall_watchdog(); + class RSXDMAWriter; struct context;