Merge pull request #12723 from mitaclaw/cached-interpreter-2.0

Cached Interpreter 2.0

Source/Core/Common/CodeBlock.h

@@ -17,7 +17,7 @@ namespace Common
 // having to prefix them with gen-> or something similar.
 // Example implementation:
 // class JIT : public CodeBlock<ARMXEmitter> {}
-template <class T>
+template <class T, bool executable = true>
 class CodeBlock : public T
 {
 private:
@@ -53,7 +53,10 @@ public:
   {
     region_size = size;
     total_region_size = size;
-    region = static_cast<u8*>(Common::AllocateExecutableMemory(total_region_size));
+    if constexpr (executable)
+      region = static_cast<u8*>(Common::AllocateExecutableMemory(total_region_size));
+    else
+      region = static_cast<u8*>(Common::AllocateMemoryPages(total_region_size));
     T::SetCodePtr(region, region + size);
   }
 

Source/Core/Core/CMakeLists.txt

@@ -481,8 +481,10 @@ add_library(core
   PowerPC/BreakPoints.h
   PowerPC/CachedInterpreter/CachedInterpreter.cpp
   PowerPC/CachedInterpreter/CachedInterpreter.h
-  PowerPC/CachedInterpreter/InterpreterBlockCache.cpp
-  PowerPC/CachedInterpreter/InterpreterBlockCache.h
+  PowerPC/CachedInterpreter/CachedInterpreterBlockCache.cpp
+  PowerPC/CachedInterpreter/CachedInterpreterBlockCache.h
+  PowerPC/CachedInterpreter/CachedInterpreterEmitter.cpp
+  PowerPC/CachedInterpreter/CachedInterpreterEmitter.h
   PowerPC/ConditionRegister.cpp
   PowerPC/ConditionRegister.h
   PowerPC/Expression.cpp

Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.h

@@ -3,14 +3,27 @@
 
 #pragma once
 
-#include <vector>
+#include <cstddef>
+
+#include <rangeset/rangesizeset.h>
 
 #include "Common/CommonTypes.h"
-#include "Core/PowerPC/CachedInterpreter/InterpreterBlockCache.h"
+#include "Core/PowerPC/CachedInterpreter/CachedInterpreterBlockCache.h"
+#include "Core/PowerPC/CachedInterpreter/CachedInterpreterEmitter.h"
 #include "Core/PowerPC/JitCommon/JitBase.h"
 #include "Core/PowerPC/PPCAnalyst.h"
 
-class CachedInterpreter : public JitBase
+namespace CoreTiming
+{
+class CoreTimingManager;
+}
+namespace CPU
+{
+enum class State;
+}
+class Interpreter;
+
+class CachedInterpreter : public JitBase, public CachedInterpreterCodeBlock
 {
 public:
   explicit CachedInterpreter(Core::System& system);
@@ -30,32 +43,94 @@ public:
   void SingleStep() override;
 
   void Jit(u32 address) override;
+  void Jit(u32 address, bool clear_cache_and_retry_on_failure);
+  bool DoJit(u32 address, JitBlock* b, u32 nextPC);
 
   JitBaseBlockCache* GetBlockCache() override { return &m_block_cache; }
   const char* GetName() const override { return "Cached Interpreter"; }
   const CommonAsmRoutinesBase* GetAsmRoutines() override { return nullptr; }
 
 private:
-  struct Instruction;
-
-  u8* GetCodePtr();
   void ExecuteOneBlock();
 
   bool HandleFunctionHooking(u32 address);
 
-  static void EndBlock(CachedInterpreter& cached_interpreter, UGeckoInstruction data);
-  static void UpdateNumLoadStoreInstructions(CachedInterpreter& cached_interpreter,
-                                             UGeckoInstruction data);
-  static void UpdateNumFloatingPointInstructions(CachedInterpreter& cached_interpreter,
-                                                 UGeckoInstruction data);
-  static void WritePC(CachedInterpreter& cached_interpreter, UGeckoInstruction data);
-  static void WriteBrokenBlockNPC(CachedInterpreter& cached_interpreter, UGeckoInstruction data);
-  static bool CheckFPU(CachedInterpreter& cached_interpreter, u32 data);
-  static bool CheckDSI(CachedInterpreter& cached_interpreter, u32 data);
-  static bool CheckProgramException(CachedInterpreter& cached_interpreter, u32 data);
-  static bool CheckBreakpoint(CachedInterpreter& cached_interpreter, u32 data);
-  static bool CheckIdle(CachedInterpreter& cached_interpreter, u32 idle_pc);
+  // Finds a free memory region and sets the code emitter to point at that region.
+  // Returns false if no free memory region can be found.
+  bool SetEmitterStateToFreeCodeRegion();
 
-  BlockCache m_block_cache{*this};
-  std::vector<Instruction> m_code;
+  void FreeRanges();
+  void ResetFreeMemoryRanges();
+
+  struct EndBlockOperands;
+  struct InterpretOperands;
+  struct InterpretAndCheckExceptionsOperands;
+  struct HLEFunctionOperands;
+  struct WriteBrokenBlockNPCOperands;
+  struct CheckHaltOperands;
+  struct CheckIdleOperands;
+
+  static s32 EndBlock(PowerPC::PowerPCState& ppc_state, const EndBlockOperands& operands);
+  template <bool write_pc>
+  static s32 Interpret(PowerPC::PowerPCState& ppc_state, const InterpretOperands& operands);
+  template <bool write_pc>
+  static s32 InterpretAndCheckExceptions(PowerPC::PowerPCState& ppc_state,
+                                         const InterpretAndCheckExceptionsOperands& operands);
+  static s32 HLEFunction(PowerPC::PowerPCState& ppc_state, const HLEFunctionOperands& operands);
+  static s32 WriteBrokenBlockNPC(PowerPC::PowerPCState& ppc_state,
+                                 const WriteBrokenBlockNPCOperands& operands);
+  static s32 CheckFPU(PowerPC::PowerPCState& ppc_state, const CheckHaltOperands& operands);
+  static s32 CheckBreakpoint(PowerPC::PowerPCState& ppc_state, const CheckHaltOperands& operands);
+  static s32 CheckIdle(PowerPC::PowerPCState& ppc_state, const CheckIdleOperands& operands);
+
+  HyoutaUtilities::RangeSizeSet<u8*> m_free_ranges;
+  CachedInterpreterBlockCache m_block_cache;
+};
+
+struct CachedInterpreter::EndBlockOperands
+{
+  u32 downcount;
+  u32 num_load_stores;
+  u32 num_fp_inst;
+  u32 : 32;
+};
+
+struct CachedInterpreter::InterpretOperands
+{
+  Interpreter& interpreter;
+  void (*func)(Interpreter&, UGeckoInstruction);  // Interpreter::Instruction
+  u32 current_pc;
+  UGeckoInstruction inst;
+};
+
+struct CachedInterpreter::InterpretAndCheckExceptionsOperands : InterpretOperands
+{
+  PowerPC::PowerPCManager& power_pc;
+  u32 downcount;
+};
+
+struct CachedInterpreter::HLEFunctionOperands
+{
+  Core::System& system;
+  u32 current_pc;
+  u32 hook_index;
+};
+
+struct CachedInterpreter::WriteBrokenBlockNPCOperands
+{
+  u32 current_pc;
+  u32 : 32;
+};
+
+struct CachedInterpreter::CheckHaltOperands
+{
+  PowerPC::PowerPCManager& power_pc;
+  u32 current_pc;
+  u32 downcount;
+};
+
+struct CachedInterpreter::CheckIdleOperands
+{
+  CoreTiming::CoreTimingManager& core_timing;
+  u32 idle_pc;
 };

Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreterBlockCache.cpp

@@ -0,0 +1,41 @@
+// Copyright 2024 Dolphin Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "Core/PowerPC/CachedInterpreter/CachedInterpreterBlockCache.h"
+
+#include "Core/PowerPC/CachedInterpreter/CachedInterpreterEmitter.h"
+#include "Core/PowerPC/JitCommon/JitBase.h"
+
+CachedInterpreterBlockCache::CachedInterpreterBlockCache(JitBase& jit) : JitBaseBlockCache{jit}
+{
+}
+
+void CachedInterpreterBlockCache::Init()
+{
+  JitBaseBlockCache::Init();
+  ClearRangesToFree();
+}
+
+void CachedInterpreterBlockCache::DestroyBlock(JitBlock& block)
+{
+  JitBaseBlockCache::DestroyBlock(block);
+
+  if (block.near_begin != block.near_end)
+    m_ranges_to_free_on_next_codegen.emplace_back(block.near_begin, block.near_end);
+}
+
+void CachedInterpreterBlockCache::ClearRangesToFree()
+{
+  m_ranges_to_free_on_next_codegen.clear();
+}
+
+void CachedInterpreterBlockCache::WriteLinkBlock(const JitBlock::LinkData& source,
+                                                 const JitBlock* dest)
+{
+}
+
+void CachedInterpreterBlockCache::WriteDestroyBlock(const JitBlock& block)
+{
+  CachedInterpreterEmitter emitter(block.normalEntry, block.near_end);
+  emitter.Write(CachedInterpreterEmitter::PoisonCallback);
+}

Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreterBlockCache.h

@@ -0,0 +1,35 @@
+// Copyright 2024 Dolphin Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include <utility>
+#include <vector>
+
+#include "Common/CommonTypes.h"
+#include "Core/PowerPC/JitCommon/JitCache.h"
+
+class JitBase;
+
+class CachedInterpreterBlockCache final : public JitBaseBlockCache
+{
+public:
+  explicit CachedInterpreterBlockCache(JitBase& jit);
+
+  void Init() override;
+
+  void DestroyBlock(JitBlock& block) override;
+
+  void ClearRangesToFree();
+
+  const std::vector<std::pair<u8*, u8*>>& GetRangesToFree() const
+  {
+    return m_ranges_to_free_on_next_codegen;
+  };
+
+private:
+  void WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest) override;
+  void WriteDestroyBlock(const JitBlock& block) override;
+
+  std::vector<std::pair<u8*, u8*>> m_ranges_to_free_on_next_codegen;
+};

Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreterEmitter.cpp

@@ -0,0 +1,39 @@
+// Copyright 2024 Dolphin Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "Core/PowerPC/CachedInterpreter/CachedInterpreterEmitter.h"
+
+#include <algorithm>
+#include <cstring>
+
+#include "Common/Assert.h"
+#include "Common/MsgHandler.h"
+
+void CachedInterpreterEmitter::Write(AnyCallback callback, const void* operands, std::size_t size)
+{
+  DEBUG_ASSERT(reinterpret_cast<std::uintptr_t>(m_code) % alignof(AnyCallback) == 0);
+  if (m_code + sizeof(callback) + size >= m_code_end)
+  {
+    m_write_failed = true;
+    return;
+  }
+  std::memcpy(m_code, &callback, sizeof(callback));
+  m_code += sizeof(callback);
+  std::memcpy(m_code, operands, size);
+  m_code += size;
+}
+
+s32 CachedInterpreterEmitter::PoisonCallback(PowerPC::PowerPCState& ppc_state, const void* operands)
+{
+  ASSERT_MSG(DYNA_REC, false,
+             "The Cached Interpreter reached a poisoned callback. This should never happen!");
+  return 0;
+}
+
+void CachedInterpreterCodeBlock::PoisonMemory()
+{
+  DEBUG_ASSERT(reinterpret_cast<std::uintptr_t>(region) % alignof(AnyCallback) == 0);
+  DEBUG_ASSERT(region_size % sizeof(AnyCallback) == 0);
+  std::fill(reinterpret_cast<AnyCallback*>(region),
+            reinterpret_cast<AnyCallback*>(region + region_size), AnyCallbackCast(PoisonCallback));
+}

Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreterEmitter.h

@@ -0,0 +1,84 @@
+// Copyright 2024 Dolphin Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+
+#include "Common/CodeBlock.h"
+#include "Common/CommonTypes.h"
+
+namespace PowerPC
+{
+struct PowerPCState;
+}
+
+class CachedInterpreterEmitter
+{
+protected:
+  // The return value of most callbacks is the distance in memory to the next callback.
+  // If a callback returns 0, the block will be exited. The return value is signed to
+  // support block-linking. 32-bit return values seem to perform better than 64-bit ones.
+  template <class Operands>
+  using Callback = s32 (*)(PowerPC::PowerPCState& ppc_state, const Operands& operands);
+  using AnyCallback = s32 (*)(PowerPC::PowerPCState& ppc_state, const void* operands);
+
+  template <class Operands>
+  static AnyCallback AnyCallbackCast(Callback<Operands> callback)
+  {
+    return reinterpret_cast<AnyCallback>(callback);
+  }
+  static consteval AnyCallback AnyCallbackCast(AnyCallback callback) { return callback; }
+
+public:
+  CachedInterpreterEmitter() = default;
+  explicit CachedInterpreterEmitter(u8* begin, u8* end) : m_code(begin), m_code_end(end) {}
+
+  template <class Operands>
+  void Write(Callback<Operands> callback, const Operands& operands)
+  {
+    // I would use std::is_trivial_v, but almost every operands struct uses
+    // references instead of pointers to make the callback functions nicer.
+    static_assert(
+        std::is_trivially_copyable_v<Operands> && std::is_trivially_destructible_v<Operands> &&
+        alignof(Operands) <= alignof(AnyCallback) && sizeof(Operands) % alignof(AnyCallback) == 0);
+    Write(AnyCallbackCast(callback), &operands, sizeof(Operands));
+  }
+  void Write(AnyCallback callback) { Write(callback, nullptr, 0); }
+
+  const u8* GetCodePtr() const { return m_code; }
+  u8* GetWritableCodePtr() { return m_code; }
+  const u8* GetCodeEnd() const { return m_code_end; };
+  u8* GetWritableCodeEnd() { return m_code_end; };
+  // Should be checked after a block of code has been generated to see if the code has been
+  // successfully written to memory. Do not call the generated code when this returns true!
+  bool HasWriteFailed() const { return m_write_failed; }
+
+  void SetCodePtr(u8* begin, u8* end)
+  {
+    m_code = begin;
+    m_code_end = end;
+    m_write_failed = false;
+  };
+
+  static s32 PoisonCallback(PowerPC::PowerPCState& ppc_state, const void* operands);
+
+private:
+  void Write(AnyCallback callback, const void* operands, std::size_t size);
+
+  // Pointer to memory where code will be emitted to.
+  u8* m_code = nullptr;
+  // Pointer past the end of the memory region we're allowed to emit to.
+  // Writes that would reach this memory are refused and will set the m_write_failed flag instead.
+  u8* m_code_end = nullptr;
+  // Set to true when a write request happens that would write past m_code_end.
+  // Must be cleared with SetCodePtr() afterwards.
+  bool m_write_failed = false;
+};
+
+class CachedInterpreterCodeBlock : public Common::CodeBlock<CachedInterpreterEmitter, false>
+{
+private:
+  void PoisonMemory() override;
+};

Source/Core/Core/PowerPC/CachedInterpreter/InterpreterBlockCache.cpp

@@ -1,14 +0,0 @@
-// Copyright 2016 Dolphin Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#include "Core/PowerPC/CachedInterpreter/InterpreterBlockCache.h"
-
-#include "Core/PowerPC/JitCommon/JitBase.h"
-
-BlockCache::BlockCache(JitBase& jit) : JitBaseBlockCache{jit}
-{
-}
-
-void BlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest)
-{
-}

Source/Core/Core/PowerPC/CachedInterpreter/InterpreterBlockCache.h

@@ -1,17 +0,0 @@
-// Copyright 2016 Dolphin Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#pragma once
-
-#include "Core/PowerPC/JitCommon/JitCache.h"
-
-class JitBase;
-
-class BlockCache final : public JitBaseBlockCache
-{
-public:
-  explicit BlockCache(JitBase& jit);
-
-private:
-  void WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest) override;
-};

Source/Core/DolphinLib.props

@@ -428,7 +428,8 @@
     <ClInclude Include="Core\PatchEngine.h" />
     <ClInclude Include="Core\PowerPC\BreakPoints.h" />
     <ClInclude Include="Core\PowerPC\CachedInterpreter\CachedInterpreter.h" />
-    <ClInclude Include="Core\PowerPC\CachedInterpreter\InterpreterBlockCache.h" />
+    <ClInclude Include="Core\PowerPC\CachedInterpreter\CachedInterpreterBlockCache.h" />
+    <ClInclude Include="Core\PowerPC\CachedInterpreter\CachedInterpreterEmitter.h" />
     <ClInclude Include="Core\PowerPC\ConditionRegister.h" />
     <ClInclude Include="Core\PowerPC\CPUCoreBase.h" />
     <ClInclude Include="Core\PowerPC\Expression.h" />
@@ -1089,7 +1090,8 @@
     <ClCompile Include="Core\PatchEngine.cpp" />
     <ClCompile Include="Core\PowerPC\BreakPoints.cpp" />
     <ClCompile Include="Core\PowerPC\CachedInterpreter\CachedInterpreter.cpp" />
-    <ClCompile Include="Core\PowerPC\CachedInterpreter\InterpreterBlockCache.cpp" />
+    <ClCompile Include="Core\PowerPC\CachedInterpreter\CachedInterpreterBlockCache.cpp" />
+    <ClCompile Include="Core\PowerPC\CachedInterpreter\CachedInterpreterEmitter.cpp" />
     <ClCompile Include="Core\PowerPC\ConditionRegister.cpp" />
     <ClCompile Include="Core\PowerPC\Expression.cpp" />
     <ClCompile Include="Core\PowerPC\GDBStub.cpp" />