Core: Create fastmem mappings for page address translation

Previously we've only been setting up fastmem mappings for block address
translation, but now we also do it for page address translation. This
increases performance when games access memory using page tables, but
decreases performance when games set up page tables.

The tlbie instruction is used as an indication that the mappings need to
be updated.

There are some accuracy downsides:

* The TLB is now effectively infinitely large, which matters if games
  don't use tlbie when modifying page tables.
* The R and C bits for page table entries get set pessimistically rather
  than when the page is actually accessed.

No games are known to be broken by these inaccuracies, but unfortunately
the second inaccuracy causes a large performance regression in Rogue
Squadron 3. You still get the old, more accurate behavior if Enable
Write-Back Cache is on.

Source/Core/Core/HW/Memmap.cpp

@@ -11,6 +11,7 @@
 #include <algorithm>
 #include <array>
 #include <cstring>
+#include <map>
 #include <memory>
 #include <span>
 #include <tuple>
@@ -41,7 +42,8 @@
 
 namespace Memory
 {
-MemoryManager::MemoryManager(Core::System& system) : m_system(system)
+MemoryManager::MemoryManager(Core::System& system)
+    : m_page_size(m_arena.GetPageSize()), m_system(system)
 {
 }
 
@@ -233,13 +235,19 @@ bool MemoryManager::InitFastmemArena()
   return true;
 }
 
-void MemoryManager::UpdateLogicalMemory(const PowerPC::BatTable& dbat_table)
+void MemoryManager::UpdateDBATMappings(const PowerPC::BatTable& dbat_table)
 {
-  for (auto& entry : m_logical_mapped_entries)
+  for (auto& entry : m_dbat_mapped_entries)
   {
     m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size);
   }
-  m_logical_mapped_entries.clear();
+  m_dbat_mapped_entries.clear();
+
+  for (auto& entry : m_page_table_mapped_entries)
+  {
+    m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size);
+  }
+  m_page_table_mapped_entries.clear();
 
   m_logical_page_mappings.fill(nullptr);
 
@@ -288,13 +296,12 @@ void MemoryManager::UpdateLogicalMemory(const PowerPC::BatTable& dbat_table)
             void* mapped_pointer = m_arena.MapInMemoryRegion(position, mapped_size, base);
             if (!mapped_pointer)
             {
-              PanicAlertFmt(
-                  "Memory::UpdateLogicalMemory(): Failed to map memory region at 0x{:08X} "
-                  "(size 0x{:08X}) into logical fastmem region at 0x{:08X}.",
-                  intersection_start, mapped_size, logical_address);
-              exit(0);
+              PanicAlertFmt("Memory::UpdateDBATMappings(): Failed to map memory region at 0x{:08X} "
+                            "(size 0x{:08X}) into logical fastmem region at 0x{:08X}.",
+                            intersection_start, mapped_size, logical_address);
+              continue;
             }
-            m_logical_mapped_entries.push_back({mapped_pointer, mapped_size});
+            m_dbat_mapped_entries.push_back({mapped_pointer, mapped_size});
           }
 
           m_logical_page_mappings[i] =
@@ -305,6 +312,50 @@ void MemoryManager::UpdateLogicalMemory(const PowerPC::BatTable& dbat_table)
   }
 }
 
+void MemoryManager::UpdatePageTableMappings(const std::map<u32, u32>& page_mappings)
+{
+  if (!m_is_fastmem_arena_initialized || m_page_size > PowerPC::HW_PAGE_SIZE)
+    return;
+
+  for (auto& entry : m_page_table_mapped_entries)
+  {
+    m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size);
+  }
+  m_page_table_mapped_entries.clear();
+
+  for (const auto [logical_address, translated_address] : page_mappings)
+  {
+    constexpr u32 logical_size = PowerPC::HW_PAGE_SIZE;
+    for (const auto& physical_region : m_physical_regions)
+    {
+      if (!physical_region.active)
+        continue;
+
+      const u32 mapping_address = physical_region.physical_address;
+      const u32 mapping_end = mapping_address + physical_region.size;
+      const u32 intersection_start = std::max(mapping_address, translated_address);
+      const u32 intersection_end = std::min(mapping_end, translated_address + logical_size);
+      if (intersection_start >= intersection_end)
+        continue;
+
+      // Found an overlapping region; map it.
+      const u32 position = physical_region.shm_position + intersection_start - mapping_address;
+      u8* const base = m_logical_base + logical_address + intersection_start - translated_address;
+      const u32 mapped_size = intersection_end - intersection_start;
+
+      void* const mapped_pointer = m_arena.MapInMemoryRegion(position, mapped_size, base);
+      if (!mapped_pointer)
+      {
+        PanicAlertFmt("Memory::UpdatePageTableMappings(): Failed to map memory region at 0x{:08X} "
+                      "(size 0x{:08X}) into logical fastmem region at 0x{:08X}.",
+                      intersection_start, mapped_size, logical_address);
+        continue;
+      }
+      m_page_table_mapped_entries.push_back({mapped_pointer, mapped_size});
+    }
+  }
+}
+
 void MemoryManager::DoState(PointerWrap& p)
 {
   const u32 current_ram_size = GetRamSize();
@@ -386,11 +437,17 @@ void MemoryManager::ShutdownFastmemArena()
     m_arena.UnmapFromMemoryRegion(base, region.size);
   }
 
-  for (auto& entry : m_logical_mapped_entries)
+  for (auto& entry : m_dbat_mapped_entries)
   {
     m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size);
   }
-  m_logical_mapped_entries.clear();
+  m_dbat_mapped_entries.clear();
+
+  for (auto& entry : m_page_table_mapped_entries)
+  {
+    m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size);
+  }
+  m_page_table_mapped_entries.clear();
 
   m_arena.ReleaseMemoryRegion();
 

Source/Core/Core/HW/Memmap.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <array>
+#include <map>
 #include <memory>
 #include <span>
 #include <string>
@@ -99,7 +100,8 @@ public:
   void ShutdownFastmemArena();
   void DoState(PointerWrap& p);
 
-  void UpdateLogicalMemory(const PowerPC::BatTable& dbat_table);
+  void UpdateDBATMappings(const PowerPC::BatTable& dbat_table);
+  void UpdatePageTableMappings(const std::map<u32, u32>& page_mappings);
 
   void Clear();
 
@@ -208,6 +210,8 @@ private:
   // The MemArena class
   Common::MemArena m_arena;
 
+  const size_t m_page_size;
+
   // Dolphin allocates memory to represent four regions:
   // - 32MB RAM (actually 24MB on hardware), available on GameCube and Wii
   // - 64MB "EXRAM", RAM only available on Wii
@@ -248,7 +252,8 @@ private:
   // TODO: Do we want to handle the mirrors of the GC RAM?
   std::array<PhysicalMemoryRegion, 4> m_physical_regions{};
 
-  std::vector<LogicalMemoryView> m_logical_mapped_entries;
+  std::vector<LogicalMemoryView> m_dbat_mapped_entries;
+  std::vector<LogicalMemoryView> m_page_table_mapped_entries;
 
   std::array<void*, PowerPC::BAT_PAGE_COUNT> m_physical_page_mappings{};
   std::array<void*, PowerPC::BAT_PAGE_COUNT> m_logical_page_mappings{};

Source/Core/Core/PowerPC/JitCommon/JitBase.cpp

@@ -19,6 +19,7 @@
 #include "Core/CoreTiming.h"
 #include "Core/HW/CPU.h"
 #include "Core/MemTools.h"
+#include "Core/PowerPC/MMU.h"
 #include "Core/PowerPC/PPCAnalyst.h"
 #include "Core/PowerPC/PowerPC.h"
 #include "Core/System.h"
@@ -120,6 +121,8 @@ bool JitBase::DoesConfigNeedRefresh() const
 
 void JitBase::RefreshConfig()
 {
+  const bool wanted_page_table_mappings = WantsPageTableMappings();
+
   for (const auto& [member, config_info] : JIT_SETTINGS)
     this->*member = Config::Get(*config_info);
 
@@ -141,6 +144,18 @@ void JitBase::RefreshConfig()
   jo.memcheck = m_system.IsMMUMode() || m_system.IsPauseOnPanicMode() || any_watchpoints;
   jo.fp_exceptions = m_enable_float_exceptions;
   jo.div_by_zero_exceptions = m_enable_div_by_zero_exceptions;
+
+  if (!wanted_page_table_mappings && WantsPageTableMappings())
+  {
+    // Mustn't call this if we're still initializing
+    if (Core::IsRunning(m_system))
+      m_system.GetMMU().PageTableUpdated();
+  }
+}
+
+bool JitBase::WantsPageTableMappings() const
+{
+  return jo.fastmem;
 }
 
 void JitBase::InitFastmemArena()

Source/Core/Core/PowerPC/JitCommon/JitBase.h

@@ -217,6 +217,8 @@ public:
 
   virtual const CommonAsmRoutinesBase* GetAsmRoutines() = 0;
 
+  virtual bool WantsPageTableMappings() const;
+
   virtual bool HandleFault(uintptr_t access_address, SContext* ctx) = 0;
   bool HandleStackFault();
 

Source/Core/Core/PowerPC/JitInterface.cpp

@@ -77,6 +77,16 @@ CPUCoreBase* JitInterface::GetCore() const
   return m_jit.get();
 }
 
+#ifndef _ARCH_32
+bool JitInterface::WantsPageTableMappings() const
+{
+  if (!m_jit)
+    return false;
+
+  return m_jit->WantsPageTableMappings();
+}
+#endif
+
 void JitInterface::UpdateMembase()
 {
   if (!m_jit)

Source/Core/Core/PowerPC/JitInterface.h

@@ -45,6 +45,12 @@ public:
   CPUCoreBase* InitJitCore(PowerPC::CPUCore core);
   CPUCoreBase* GetCore() const;
 
+#ifdef _ARCH_32
+  constexpr bool WantsPageTableMappings() const { return false; }
+#else
+  bool WantsPageTableMappings() const;
+#endif
+
   void UpdateMembase();
   void JitBlockLogDump(const Core::CPUThreadGuard& guard, std::FILE* file) const;
   void WipeBlockProfilingData(const Core::CPUThreadGuard& guard);

Source/Core/Core/PowerPC/MMU.cpp

@@ -33,6 +33,7 @@
 #include "Common/Align.h"
 #include "Common/Assert.h"
 #include "Common/BitUtils.h"
+#include "Common/ChunkFile.h"
 #include "Common/CommonTypes.h"
 #include "Common/Logging/Log.h"
 
@@ -58,6 +59,22 @@ MMU::MMU(Core::System& system, Memory::MemoryManager& memory, PowerPC::PowerPCMa
 
 MMU::~MMU() = default;
 
+void MMU::Reset()
+{
+  m_page_mappings.clear();
+  m_memory.UpdatePageTableMappings(m_page_mappings);
+}
+
+void MMU::DoState(PointerWrap& p)
+{
+  // Instead of storing m_page_mappings in savestates, we *could* recalculate it based on memory
+  // here in DoState, but this could lead to us getting a more up-to-date set of page mappings
+  // than we had when the savestate was created, which could be a problem for TAS determinism.
+  p.Do(m_page_mappings);
+  // If we're reading a savestate, PowerPCManager::DoState::DoState will call MMU::DBATUpdated,
+  // which will call MemoryManager::UpdatePageTableMappings with m_page_mappings.
+}
+
 // Overloaded byteswap functions, for use within the templated functions below.
 [[maybe_unused]] static u8 bswap(u8 val)
 {
@@ -1217,10 +1234,13 @@ void MMU::SDRUpdated()
 
   m_ppc_state.pagetable_base = htaborg << 16;
   m_ppc_state.pagetable_mask = (htabmask << 16) | 0xffc0;
+
+  PageTableUpdated();
 }
 
 void MMU::SRUpdated()
 {
+  PageTableUpdated();
 }
 
 enum class TLBLookupResult
@@ -1310,6 +1330,130 @@ void MMU::InvalidateTLBEntry(u32 address)
 
   m_ppc_state.tlb[PowerPC::DATA_TLB_INDEX][entry_index].Invalidate();
   m_ppc_state.tlb[PowerPC::INST_TLB_INDEX][entry_index].Invalidate();
+
+  PageTableUpdated();
+}
+
+void MMU::PageTableUpdated()
+{
+  m_page_mappings.clear();
+
+  if (!m_system.GetJitInterface().WantsPageTableMappings())
+  {
+    // If the JIT has no use for page table mappings, setting them up would be a waste of time.
+    // Skipping setting them up also comes with the bonus of skipping the inaccurate behavior of
+    // setting the R and C bits of PTE2 as soon as a page is mapped.
+    m_memory.UpdatePageTableMappings(m_page_mappings);
+    return;
+  }
+
+  const u32 page_table_mask = m_ppc_state.pagetable_mask;
+  const u32 page_table_base = m_ppc_state.pagetable_base;
+  const u32 page_table_end =
+      Common::AlignUp(page_table_base | page_table_mask, PAGE_TABLE_MIN_SIZE);
+  const u32 page_table_size = page_table_end - page_table_base;
+
+  u8* page_table_view = m_system.GetMemory().GetPointerForRange(page_table_base, page_table_size);
+  if (!page_table_view)
+  {
+    WARN_LOG_FMT(POWERPC, "Failed to read page table at {:#010x}-{:#010x}", page_table_base,
+                 page_table_end);
+    m_memory.UpdatePageTableMappings(m_page_mappings);
+    return;
+  }
+
+  const auto read_page_table = [&](u32 H) {
+    for (u32 i = 0; i <= page_table_mask; i += 64)
+    {
+      for (u32 j = 0; j < 64; j += 8)
+      {
+        const u32 pte_addr = (page_table_base | (i & page_table_mask)) + j;
+
+        UPTE_Lo pte1(Common::swap32(page_table_view + pte_addr - page_table_base));
+        UPTE_Hi pte2(Common::swap32(page_table_view + pte_addr - page_table_base + 4));
+
+        if (!pte1.V)
+          continue;
+
+        if (pte1.H != H)
+          continue;
+
+        // There are quirks related to uncached memory that can't be correctly emulated by fast
+        // accesses, so we don't map uncached memory. (However, no software at all is known to
+        // trigger these quirks through page address translation, only through block address
+        // translation.)
+        const bool wi = (pte2.WIMG & 0b1100) != 0;
+        if (wi)
+          continue;
+
+        // Due to hash masking, the upper bits of page_index_from_hash might not match the actual
+        // page index. But these bits fully overlap with the API (abbreviated page index), so we can
+        // overwrite these bits with the API from pte1 and thereby get the correct page index.
+        //
+        // In other words: logical_address.API must be written to after logical_address.page_index!
+        u32 page_index_from_hash = (i / 64) ^ pte1.VSID;
+        if (pte1.H)
+          page_index_from_hash = ~page_index_from_hash;
+        EffectiveAddress logical_address;
+        logical_address.offset = 0;
+        logical_address.page_index = page_index_from_hash;
+        logical_address.API = pte1.API;
+
+        // If the hash mask is large enough that one or more bits specified in pte1.API can also be
+        // obtained from page_index_from_hash, check that those bits match.
+        const u32 api_mask = ((page_table_mask & ~page_table_base) >> 16) & 0x3f;
+        if ((pte1.API & api_mask) != ((page_index_from_hash >> 10) & api_mask))
+          continue;
+
+        for (u32 k = 0; k < std::size(m_ppc_state.sr); ++k)
+        {
+          const auto sr = UReg_SR{m_ppc_state.sr[k]};
+          if (sr.VSID != pte1.VSID || sr.T != 0)
+            continue;
+
+          logical_address.SR = k;
+
+          // Block address translation takes priority over page address translation.
+          if (m_dbat_table[logical_address.Hex >> PowerPC::BAT_INDEX_SHIFT] &
+              PowerPC::BAT_MAPPED_BIT)
+          {
+            continue;
+          }
+
+          // Fast accesses don't support memchecks, so force slow accesses by removing fastmem
+          // mappings for all overlapping virtual pages.
+          constexpr u32 logical_size = PowerPC::HW_PAGE_SIZE;
+          if (m_power_pc.GetMemChecks().OverlapsMemcheck(logical_address.Hex, logical_size))
+            continue;
+
+          const u32 physical_address = pte2.RPN << 12;
+
+          // Important: This doesn't overwrite anything already present in m_page_mappings.
+          m_page_mappings.emplace(logical_address.Hex, physical_address);
+
+          // HACK: We set R and C, which indicate whether a page have been read from and written to
+          // respectively, when a page is mapped rather than when it's actually accessed. The latter
+          // is probably possible using some fault handling logic, but for now it seems like more
+          // work than it's worth.
+          if (!pte2.R || !pte2.C)
+          {
+            pte2.R = 1;
+            pte2.C = 1;
+
+            const u32 pte2_swapped = Common::swap32(pte2.Hex);
+            std::memcpy(page_table_view + pte_addr - page_table_base + 4, &pte2_swapped,
+                        sizeof(pte2_swapped));
+          }
+        }
+      }
+    }
+  };
+
+  // We need to read all H=0 PTEs first, because H=0 takes priority over H=1.
+  read_page_table(0);
+  read_page_table(1);
+
+  m_memory.UpdatePageTableMappings(m_page_mappings);
 }
 
 // Page Address Translation
@@ -1537,7 +1681,8 @@ void MMU::DBATUpdated()
   }
 
 #ifndef _ARCH_32
-  m_memory.UpdateLogicalMemory(m_dbat_table);
+  m_memory.UpdateDBATMappings(m_dbat_table);
+  m_memory.UpdatePageTableMappings(m_page_mappings);
 #endif
 
   // IsOptimizable*Address and dcbz depends on the BAT mapping, so we need a flush here.

Source/Core/Core/PowerPC/MMU.h

@@ -5,6 +5,7 @@
 
 #include <array>
 #include <cstddef>
+#include <map>
 #include <optional>
 #include <string>
 
@@ -12,6 +13,8 @@
 #include "Common/CommonTypes.h"
 #include "Common/TypeUtils.h"
 
+class PointerWrap;
+
 namespace Core
 {
 class CPUThreadGuard;
@@ -73,6 +76,8 @@ constexpr size_t HW_PAGE_MASK = HW_PAGE_SIZE - 1;
 constexpr u32 HW_PAGE_INDEX_SHIFT = 12;
 constexpr u32 HW_PAGE_INDEX_MASK = 0x3f;
 
+constexpr u32 PAGE_TABLE_MIN_SIZE = 0x10000;
+
 // Return value of MMU::TryReadInstruction().
 struct TryReadInstResult
 {
@@ -117,6 +122,9 @@ public:
   MMU& operator=(MMU&& other) = delete;
   ~MMU();
 
+  void Reset();
+  void DoState(PointerWrap& p);
+
   // Routines for debugger UI, cheats, etc. to access emulated memory from the
   // perspective of the CPU.  Not for use by core emulation routines.
   // Use "Host" prefix.
@@ -239,6 +247,7 @@ public:
   void SDRUpdated();
   void SRUpdated();
   void InvalidateTLBEntry(u32 address);
+  void PageTableUpdated();
   void DBATUpdated();
   void IBATUpdated();
 
@@ -318,6 +327,10 @@ private:
   PowerPC::PowerPCManager& m_power_pc;
   PowerPC::PowerPCState& m_ppc_state;
 
+  // STATE_TO_SAVE
+  std::map<u32, u32> m_page_mappings;
+  // END STATE_TO_SAVE
+
   BatTable m_ibat_table;
   BatTable m_dbat_table;
 };

Source/Core/Core/PowerPC/PowerPC.cpp

@@ -103,6 +103,9 @@ void PowerPCManager::DoState(PointerWrap& p)
   m_ppc_state.iCache.DoState(memory, p);
   m_ppc_state.dCache.DoState(memory, p);
 
+  auto& mmu = m_system.GetMMU();
+  mmu.DoState(p);
+
   if (p.IsReadMode())
   {
     if (!m_ppc_state.m_enable_dcache)
@@ -114,7 +117,6 @@ void PowerPCManager::DoState(PointerWrap& p)
     RoundingModeUpdated(m_ppc_state);
     RecalculateAllFeatureFlags(m_ppc_state);
 
-    auto& mmu = m_system.GetMMU();
     mmu.IBATUpdated();
     mmu.DBATUpdated();
   }
@@ -280,6 +282,7 @@ void PowerPCManager::Reset()
   ResetRegisters();
   m_ppc_state.iCache.Reset(m_system.GetJitInterface());
   m_ppc_state.dCache.Reset();
+  m_system.GetMMU().Reset();
 }
 
 void PowerPCManager::ScheduleInvalidateCacheThreadSafe(u32 address)

Source/Core/Core/State.cpp

@@ -94,7 +94,7 @@ static size_t s_state_writes_in_queue;
 static std::condition_variable s_state_write_queue_is_empty;
 
 // Don't forget to increase this after doing changes on the savestate system
-constexpr u32 STATE_VERSION = 175;  // Last changed in PR 13751
+constexpr u32 STATE_VERSION = 176;  // Last changed in PR 13768
 
 // Increase this if the StateExtendedHeader definition changes
 constexpr u32 EXTENDED_HEADER_VERSION = 1;  // Last changed in PR 12217