Merge pull request #14037 from jordan-woyak/presentation-timing

Add "Rush Frame Presentation" and "Smooth Early Presentation" settings.

Source/Core/VideoCommon/BPStructs.cpp

@@ -13,7 +13,6 @@
 #include "Common/EnumMap.h"
 #include "Common/Logging/Log.h"
 
-#include "Core/CoreTiming.h"
 #include "Core/DolphinAnalytics.h"
 #include "Core/FifoPlayer/FifoPlayer.h"
 #include "Core/FifoPlayer/FifoRecorder.h"
@@ -359,14 +358,8 @@ static void BPWritten(PixelShaderManager& pixel_shader_manager, XFStateManager&
 
       if (g_ActiveConfig.bImmediateXFB)
       {
-        // TODO: GetTicks is not sane from the GPU thread.
-        // This value is currently used for frame dumping and the custom shader "time_ms" value.
-        // Frame dumping has more calls that aren't sane from the GPU thread.
-        // i.e. Frame dumping is not sane in "Dual Core" mode in general.
-        const u64 ticks = system.GetCoreTiming().GetTicks();
-
         // below div two to convert from bytes to pixels - it expects width, not stride
-        g_presenter->ImmediateSwap(destAddr, destStride / 2, destStride, height, ticks);
+        g_presenter->ImmediateSwap(destAddr, destStride / 2, destStride, height);
       }
       else
       {

Source/Core/VideoCommon/FrameDumpFFMpeg.cpp

@@ -65,6 +65,7 @@ namespace
 {
 AVRational GetTimeBaseForCurrentRefreshRate(s64 max_denominator)
 {
+  // TODO: GetTargetRefreshRate* are not safe from GPU thread.
   auto& vi = Core::System::GetInstance().GetVideoInterface();
   int num;
   int den;
@@ -368,6 +369,7 @@ void FFMpegFrameDump::AddFrame(const FrameData& frame)
   // Calculate presentation timestamp from ticks since start.
   const s64 pts = av_rescale_q(
       frame.state.ticks - m_context->start_ticks,
+      // TODO: GetTicksPerSecond is not safe from GPU thread.
       AVRational{1, int(Core::System::GetInstance().GetSystemTimers().GetTicksPerSecond())},
       m_context->codec->time_base);
 

Source/Core/VideoCommon/PerformanceMetrics.cpp

@@ -23,6 +23,8 @@ void PerformanceMetrics::Reset()
 
   m_speed = 0;
   m_max_speed = 0;
+
+  m_frame_presentation_offset = DT{};
 }
 
 void PerformanceMetrics::CountFrame()
@@ -98,6 +100,11 @@ double PerformanceMetrics::GetMaxSpeed() const
   return m_max_speed.load(std::memory_order_relaxed);
 }
 
+void PerformanceMetrics::SetLatestFramePresentationOffset(DT offset)
+{
+  m_frame_presentation_offset.store(offset, std::memory_order_relaxed);
+}
+
 void PerformanceMetrics::DrawImGuiStats(const float backbuffer_scale)
 {
   m_vps_counter.UpdateStats();
@@ -293,6 +300,10 @@ void PerformanceMetrics::DrawImGuiStats(const float backbuffer_scale)
                            DT_ms(m_fps_counter.GetDtAvg()).count());
         ImGui::TextColored(ImVec4(r, g, b, 1.0f), " ±:%6.2lfms",
                            DT_ms(m_fps_counter.GetDtStd()).count());
+
+        const auto offset =
+            DT_ms(m_frame_presentation_offset.load(std::memory_order_relaxed)).count();
+        ImGui::TextColored(ImVec4(r, g, b, 1.0f), "ofs:%5.1lfms", offset);
       }
     }
     ImGui::End();

Source/Core/VideoCommon/PerformanceMetrics.h

@@ -43,6 +43,9 @@ public:
   double GetSpeed() const;
   double GetMaxSpeed() const;
 
+  // Call from any thread.
+  void SetLatestFramePresentationOffset(DT offset);
+
   // ImGui Functions
   void DrawImGuiStats(const float backbuffer_scale);
 
@@ -55,6 +58,8 @@ private:
   std::atomic<double> m_speed{};
   std::atomic<double> m_max_speed{};
 
+  std::atomic<DT> m_frame_presentation_offset{};
+
   struct PerfSample
   {
     TimePoint clock_time;

Source/Core/VideoCommon/Present.cpp

@@ -5,6 +5,7 @@
 
 #include "Common/ChunkFile.h"
 #include "Core/Config/GraphicsSettings.h"
+#include "Core/Config/MainSettings.h"
 #include "Core/CoreTiming.h"
 #include "Core/HW/VideoInterface.h"
 #include "Core/Host.h"
@@ -162,9 +163,12 @@ void Presenter::ViSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height,
 {
   bool is_duplicate = FetchXFB(xfb_addr, fb_width, fb_stride, fb_height, ticks);
 
-  PresentInfo present_info;
-  present_info.emulated_timestamp = ticks;
-  present_info.present_count = m_present_count++;
+  PresentInfo present_info{
+      .present_count = m_present_count++,
+      .emulated_timestamp = ticks,
+      .intended_present_time = presentation_time,
+  };
+
   if (is_duplicate)
   {
     present_info.frame_count = m_frame_count - 1;  // Previous frame
@@ -201,33 +205,43 @@ void Presenter::ViSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height,
 
   if (!is_duplicate || !g_ActiveConfig.bSkipPresentingDuplicateXFBs)
   {
-    Present(presentation_time);
+    Present(&present_info);
     ProcessFrameDumping(ticks);
 
     video_events.after_present_event.Trigger(present_info);
   }
 }
 
-void Presenter::ImmediateSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height, u64 ticks)
+void Presenter::ImmediateSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height)
 {
+  const u64 ticks = m_next_swap_estimated_ticks;
+
   FetchXFB(xfb_addr, fb_width, fb_stride, fb_height, ticks);
 
-  PresentInfo present_info;
-  present_info.emulated_timestamp = ticks;  // TODO: This should be the time of the next VI field
-  present_info.frame_count = m_frame_count++;
-  present_info.reason = PresentInfo::PresentReason::Immediate;
-  present_info.present_count = m_present_count++;
+  PresentInfo present_info{
+      .frame_count = m_frame_count++,
+      .present_count = m_present_count++,
+      .reason = PresentInfo::PresentReason::Immediate,
+      .emulated_timestamp = ticks,
+      .intended_present_time = m_next_swap_estimated_time,
+  };
 
   auto& video_events = GetVideoEvents();
 
   video_events.before_present_event.Trigger(present_info);
 
-  Present();
+  Present(&present_info);
   ProcessFrameDumping(ticks);
 
   video_events.after_present_event.Trigger(present_info);
 }
 
+void Presenter::SetNextSwapEstimatedTime(u64 ticks, TimePoint host_time)
+{
+  m_next_swap_estimated_ticks = ticks;
+  m_next_swap_estimated_time = host_time;
+}
+
 void Presenter::ProcessFrameDumping(u64 ticks) const
 {
   if (g_frame_dumper->IsFrameDumping() && m_xfb_entry)
@@ -819,7 +833,7 @@ void Presenter::RenderXFBToScreen(const MathUtil::Rectangle<int>& target_rc,
   }
 }
 
-void Presenter::Present(std::optional<TimePoint> presentation_time)
+void Presenter::Present(PresentInfo* present_info)
 {
   m_present_count++;
 
@@ -873,8 +887,16 @@ void Presenter::Present(std::optional<TimePoint> presentation_time)
   {
     std::lock_guard<std::mutex> guard(m_swap_mutex);
 
-    if (presentation_time.has_value())
-      Core::System::GetInstance().GetCoreTiming().SleepUntil(*presentation_time);
+    if (present_info != nullptr)
+    {
+      const auto present_time = GetUpdatedPresentationTime(present_info->intended_present_time);
+
+      Core::System::GetInstance().GetCoreTiming().SleepUntil(present_time);
+
+      // Perhaps in the future a more accurate time can be acquired from the various backends.
+      present_info->actual_present_time = Clock::now();
+      present_info->present_time_accuracy = PresentInfo::PresentTimeAccuracy::PresentInProgress;
+    }
 
     g_gfx->PresentBackbuffer();
   }
@@ -892,6 +914,34 @@ void Presenter::Present(std::optional<TimePoint> presentation_time)
   g_gfx->EndUtilityDrawing();
 }
 
+TimePoint Presenter::GetUpdatedPresentationTime(TimePoint intended_presentation_time)
+{
+  const auto now = Clock::now();
+  const auto arrival_offset = std::min(now - intended_presentation_time, DT{});
+
+  if (!Config::Get(Config::MAIN_SMOOTH_EARLY_PRESENTATION))
+  {
+    m_presentation_time_offset = arrival_offset;
+
+    // When SmoothEarlyPresentation is off and ImmediateXFB or RushFramePresentation are on,
+    //  present as soon as possible as the goal is to achieve low input latency.
+    if (g_ActiveConfig.bImmediateXFB || Config::Get(Config::MAIN_RUSH_FRAME_PRESENTATION))
+      return now;
+
+    return intended_presentation_time;
+  }
+
+  // Adjust slowly backward in time but quickly forward in time.
+  // This keeps the pacing moderately smooth even if games produce regular sporadic bumps.
+  // This was tuned to handle the terrible pacing in Brawl with "Immediate XFB".
+  // Super Mario Galaxy 1 + 2 still perform poorly here in SingleCore mode.
+  const auto adjustment_divisor = (arrival_offset < m_presentation_time_offset) ? 100 : 2;
+
+  m_presentation_time_offset += (arrival_offset - m_presentation_time_offset) / adjustment_divisor;
+
+  return intended_presentation_time + m_presentation_time_offset;
+}
+
 void Presenter::SetKeyMap(const DolphinKeyMap& key_map)
 {
   if (m_onscreen_ui)
@@ -931,8 +981,10 @@ void Presenter::DoState(PointerWrap& p)
     // This technically counts as the end of the frame
     GetVideoEvents().after_frame_event.Trigger(Core::System::GetInstance());
 
-    ImmediateSwap(m_last_xfb_addr, m_last_xfb_width, m_last_xfb_stride, m_last_xfb_height,
-                  m_last_xfb_ticks);
+    m_next_swap_estimated_ticks = m_last_xfb_ticks;
+    m_next_swap_estimated_time = Clock::now();
+
+    ImmediateSwap(m_last_xfb_addr, m_last_xfb_width, m_last_xfb_stride, m_last_xfb_height);
   }
 }
 

Source/Core/VideoCommon/Present.h

@@ -37,9 +37,11 @@ public:
 
   void ViSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height, u64 ticks,
               TimePoint presentation_time);
-  void ImmediateSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height, u64 ticks);
+  void ImmediateSwap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height);
 
-  void Present(std::optional<TimePoint> presentation_time = std::nullopt);
+  void SetNextSwapEstimatedTime(u64 ticks, TimePoint host_time);
+
+  void Present(PresentInfo* present_info = nullptr);
   void ClearLastXfbId() { m_last_xfb_id = std::numeric_limits<u64>::max(); }
 
   bool Initialize();
@@ -167,6 +169,18 @@ private:
   u32 m_last_xfb_height = MAX_XFB_HEIGHT;
 
   Common::EventHook m_config_changed;
+
+  // Updates state for the SmoothEarlyPresentation setting if enabled.
+  // Returns the desired presentation time regardless.
+  TimePoint GetUpdatedPresentationTime(TimePoint intended_presentation_time);
+
+  // Used by the SmoothEarlyPresentation setting.
+  DT m_presentation_time_offset{};
+
+  // Calculated from the previous swap time and current refresh rate.
+  // Can be used for presentation of ImmediateXFB swaps which don't have timing information.
+  u64 m_next_swap_estimated_ticks = 0;
+  TimePoint m_next_swap_estimated_time{Clock::now()};
 };
 
 }  // namespace VideoCommon

Source/Core/VideoCommon/VideoBackendBase.cpp

@@ -21,6 +21,8 @@
 #include "Core/Core.h"
 #include "Core/CoreTiming.h"
 #include "Core/DolphinAnalytics.h"
+#include "Core/HW/SystemTimers.h"
+#include "Core/HW/VideoInterface.h"
 #include "Core/System.h"
 
 // TODO: ugly
@@ -93,16 +95,35 @@ std::string VideoBackendBase::BadShaderFilename(const char* shader_stage, int co
 void VideoBackendBase::Video_OutputXFB(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height,
                                        u64 ticks)
 {
-  if (m_initialized && g_presenter && !g_ActiveConfig.bImmediateXFB)
+  if (!m_initialized || !g_presenter)
+    return;
+
+  auto& system = Core::System::GetInstance();
+  auto& core_timing = system.GetCoreTiming();
+
+  if (!g_ActiveConfig.bImmediateXFB)
   {
-    auto& system = Core::System::GetInstance();
     system.GetFifo().SyncGPU(Fifo::SyncGPUReason::Swap);
 
-    const TimePoint presentation_time = system.GetCoreTiming().GetTargetHostTime(ticks);
+    const TimePoint presentation_time = core_timing.GetTargetHostTime(ticks);
     AsyncRequests::GetInstance()->PushEvent([=] {
       g_presenter->ViSwap(xfb_addr, fb_width, fb_stride, fb_height, ticks, presentation_time);
     });
   }
+
+  // Inform the Presenter of the next estimated swap time.
+
+  auto& vi = system.GetVideoInterface();
+  const s64 refresh_rate_den = vi.GetTargetRefreshRateDenominator();
+  const s64 refresh_rate_num = vi.GetTargetRefreshRateNumerator();
+
+  const auto next_swap_estimated_ticks =
+      ticks + (system.GetSystemTimers().GetTicksPerSecond() * refresh_rate_den / refresh_rate_num);
+  const auto next_swap_estimated_time = core_timing.GetTargetHostTime(next_swap_estimated_ticks);
+
+  AsyncRequests::GetInstance()->PushEvent([=] {
+    g_presenter->SetNextSwapEstimatedTime(next_swap_estimated_ticks, next_swap_estimated_time);
+  });
 }
 
 u32 VideoBackendBase::Video_GetQueryResult(PerfQueryType type)

Source/Core/VideoCommon/VideoEvents.h

@@ -34,14 +34,12 @@ struct PresentInfo
   PresentReason reason = PresentReason::Immediate;
 
   // The exact emulated time of the when real hardware would have presented this frame
-  // FIXME: Immediate should predict the timestamp of this present
   u64 emulated_timestamp = 0;
 
-  // TODO:
-  // u64 intended_present_time = 0;
+  TimePoint intended_present_time{};
 
   // AfterPresent only: The actual time the frame was presented
-  u64 actual_present_time = 0;
+  TimePoint actual_present_time{};
 
   enum class PresentTimeAccuracy
   {