Merge pull request #3454 from mmastrac/gqr_fixes

JIT perf improvements for quantized loads/stores

Source/Core/Core/PowerPC/Jit64/Jit.cpp

@@ -672,27 +672,20 @@ const u8* Jit64::DoJit(u32 em_address, PPCAnalyst::CodeBuffer* code_buf, JitBloc
   js.skipInstructions = 0;
   js.carryFlagSet = false;
   js.carryFlagInverted = false;
-  js.assumeNoPairedQuantize = false;
+  js.constantGqr.clear();
 
-  // If the block only uses one GQR and the GQR is zero at compile time, make a guess that the block
-  // never uses quantized loads/stores. Many paired-heavy games use largely float loads and stores,
+  // Assume that GQR values don't change often at runtime. Many paired-heavy games use largely float
+  // loads and stores,
   // which are significantly faster when inlined (especially in MMU mode, where this lets them use
   // fastmem).
-  // Insert a check that the GQR is still zero at the start of the block in case our guess turns out
-  // wrong.
-  // TODO: support any other constant GQR value, not merely zero/unquantized: we can optimize
-  // quantized
-  // loadstores too, it'd just be more code.
-  if (code_block.m_gqr_used.Count() == 1 &&
-      js.pairedQuantizeAddresses.find(js.blockStart) == js.pairedQuantizeAddresses.end())
+  if (js.pairedQuantizeAddresses.find(js.blockStart) == js.pairedQuantizeAddresses.end())
   {
-    int gqr = *code_block.m_gqr_used.begin();
-    if (!code_block.m_gqr_modified[gqr] && !GQR(gqr))
+    // If there are GQRs used but not set, we'll treat those as constant and optimize them
+    BitSet8 gqr_static = ComputeStaticGQRs(code_block);
+    if (gqr_static)
     {
-      CMP(32, PPCSTATE(spr[SPR_GQR0 + gqr]), Imm8(0));
-      FixupBranch failure = J_CC(CC_NZ, true);
       SwitchToFarCode();
-      SetJumpTarget(failure);
+      const u8* target = GetCodePtr();
       MOV(32, PPCSTATE(pc), Imm32(js.blockStart));
       ABI_PushRegistersAndAdjustStack({}, 0);
       ABI_CallFunctionC((void*)&JitInterface::CompileExceptionCheck,
@@ -700,7 +693,16 @@ const u8* Jit64::DoJit(u32 em_address, PPCAnalyst::CodeBuffer* code_buf, JitBloc
       ABI_PopRegistersAndAdjustStack({}, 0);
       JMP(asm_routines.dispatcher, true);
       SwitchToNearCode();
-      js.assumeNoPairedQuantize = true;
+
+      // Insert a check that the GQRs are still the value we expect at
+      // the start of the block in case our guess turns out wrong.
+      for (int gqr : gqr_static)
+      {
+        u32 value = GQR(gqr);
+        js.constantGqr[gqr] = value;
+        CMP_or_TEST(32, PPCSTATE(spr[SPR_GQR0 + gqr]), Imm32(value));
+        J_CC(CC_NZ, target);
+      }
     }
   }
 
@@ -947,7 +949,12 @@ const u8* Jit64::DoJit(u32 em_address, PPCAnalyst::CodeBuffer* code_buf, JitBloc
   return normalEntry;
 }
 
-BitSet32 Jit64::CallerSavedRegistersInUse()
+BitSet8 Jit64::ComputeStaticGQRs(const PPCAnalyst::CodeBlock& cb) const
+{
+  return cb.m_gqr_used & ~cb.m_gqr_modified;
+}
+
+BitSet32 Jit64::CallerSavedRegistersInUse() const
 {
   BitSet32 result;
   for (int i = 0; i < NUMXREGS; i++)

Source/Core/Core/PowerPC/Jit64/Jit.h

@@ -64,7 +64,8 @@ public:
   void Jit(u32 em_address) override;
   const u8* DoJit(u32 em_address, PPCAnalyst::CodeBuffer* code_buf, JitBlock* b, u32 nextPC);
 
-  BitSet32 CallerSavedRegistersInUse();
+  BitSet32 CallerSavedRegistersInUse() const;
+  BitSet8 ComputeStaticGQRs(const PPCAnalyst::CodeBlock&) const;
 
   JitBlockCache* GetBlockCache() override { return &blocks; }
   void Trace();

Source/Core/Core/PowerPC/Jit64/Jit_LoadStorePaired.cpp

@@ -35,8 +35,12 @@ void Jit64::psq_stXX(UGeckoInstruction inst)
   int w = indexed ? inst.Wx : inst.W;
   FALLBACK_IF(!a);
 
+  auto it = js.constantGqr.find(i);
+  bool gqrIsConstant = it != js.constantGqr.end();
+  u32 gqrValue = gqrIsConstant ? it->second & 0xffff : 0;
+
   gpr.Lock(a, b);
-  if (js.assumeNoPairedQuantize)
+  if (gqrIsConstant && gqrValue == 0)
   {
     int storeOffset = 0;
     gpr.BindToRegister(a, true, update);
@@ -125,25 +129,68 @@ void Jit64::psq_stXX(UGeckoInstruction inst)
   // In memcheck mode, don't update the address until the exception check
   if (update && !jo.memcheck)
     MOV(32, gpr.R(a), R(RSCRATCH_EXTRA));
-  // Some games (e.g. Dirt 2) incorrectly set the unused bits which breaks the lookup table code.
-  // Hence, we need to mask out the unused bits. The layout of the GQR register is
-  // UU[SCALE]UUUUU[TYPE] where SCALE is 6 bits and TYPE is 3 bits, so we have to AND with
-  // 0b0011111100000111, or 0x3F07.
-  MOV(32, R(RSCRATCH2), Imm32(0x3F07));
-  AND(32, R(RSCRATCH2), PPCSTATE(spr[SPR_GQR0 + i]));
-  MOVZX(32, 8, RSCRATCH, R(RSCRATCH2));
 
-  if (w)
+  if (gqrIsConstant)
   {
-    // One value
-    CVTSD2SS(XMM0, fpr.R(s));
-    CALLptr(MScaled(RSCRATCH, SCALE_8, (u32)(u64)asm_routines.singleStoreQuantized));
+// Paired stores don't yield any real change in performance right now, but if we can
+// improve fastmem support this might change
+//#define INLINE_PAIRED_STORES
+#ifdef INLINE_PAIRED_STORES
+    if (w)
+    {
+      // One value
+      CVTSD2SS(XMM0, fpr.R(s));
+      GenQuantizedStore(true, static_cast<EQuantizeType>(gqrValue & 0x7), (gqrValue & 0x3F00) >> 8);
+    }
+    else
+    {
+      // Pair of values
+      CVTPD2PS(XMM0, fpr.R(s));
+      GenQuantizedStore(false, static_cast<EQuantizeType>(gqrValue & 0x7),
+                        (gqrValue & 0x3F00) >> 8);
+    }
+#else
+    // We know what GQR is here, so we can load RSCRATCH2 and call into the store method directly
+    // with just the scale bits.
+    int type = gqrValue & 0x7;
+    MOV(32, R(RSCRATCH2), Imm32(gqrValue & 0x3F00));
+
+    if (w)
+    {
+      // One value
+      CVTSD2SS(XMM0, fpr.R(s));
+      CALL(asm_routines.singleStoreQuantized[type]);
+    }
+    else
+    {
+      // Pair of values
+      CVTPD2PS(XMM0, fpr.R(s));
+      CALL(asm_routines.pairedStoreQuantized[type]);
+    }
+#endif
   }
   else
   {
-    // Pair of values
-    CVTPD2PS(XMM0, fpr.R(s));
-    CALLptr(MScaled(RSCRATCH, SCALE_8, (u32)(u64)asm_routines.pairedStoreQuantized));
+    // Some games (e.g. Dirt 2) incorrectly set the unused bits which breaks the lookup table code.
+    // Hence, we need to mask out the unused bits. The layout of the GQR register is
+    // UU[SCALE]UUUUU[TYPE] where SCALE is 6 bits and TYPE is 3 bits, so we have to AND with
+    // 0b0011111100000111, or 0x3F07.
+    MOV(32, R(RSCRATCH2), Imm32(0x3F07));
+    AND(32, R(RSCRATCH2), PPCSTATE(spr[SPR_GQR0 + i]));
+    MOVZX(32, 8, RSCRATCH, R(RSCRATCH2));
+
+    if (w)
+    {
+      // One value
+      CVTSD2SS(XMM0, fpr.R(s));
+      CALLptr(MScaled(RSCRATCH, SCALE_8, (u32)(u64)asm_routines.singleStoreQuantized));
+    }
+    else
+    {
+      // Pair of values
+      CVTPD2PS(XMM0, fpr.R(s));
+      CALLptr(MScaled(RSCRATCH, SCALE_8, (u32)(u64)asm_routines.pairedStoreQuantized));
+    }
   }
 
   if (update && jo.memcheck)
@@ -173,8 +220,13 @@ void Jit64::psq_lXX(UGeckoInstruction inst)
   int w = indexed ? inst.Wx : inst.W;
   FALLBACK_IF(!a);
 
+  auto it = js.constantGqr.find(i);
+  bool gqrIsConstant = it != js.constantGqr.end();
+  u32 gqrValue = gqrIsConstant ? it->second >> 16 : 0;
+
   gpr.Lock(a, b);
-  if (js.assumeNoPairedQuantize)
+
+  if (gqrIsConstant && gqrValue == 0)
   {
     s32 loadOffset = 0;
     gpr.BindToRegister(a, true, update);
@@ -302,16 +354,24 @@ void Jit64::psq_lXX(UGeckoInstruction inst)
   // In memcheck mode, don't update the address until the exception check
   if (update && !jo.memcheck)
     MOV(32, gpr.R(a), R(RSCRATCH_EXTRA));
-  MOV(32, R(RSCRATCH2), Imm32(0x3F07));
 
-  // Get the high part of the GQR register
-  OpArg gqr = PPCSTATE(spr[SPR_GQR0 + i]);
-  gqr.AddMemOffset(2);
+  if (gqrIsConstant)
+  {
+    GenQuantizedLoad(w == 1, static_cast<EQuantizeType>(gqrValue & 0x7), (gqrValue & 0x3F00) >> 8);
+  }
+  else
+  {
+    MOV(32, R(RSCRATCH2), Imm32(0x3F07));
 
-  AND(32, R(RSCRATCH2), gqr);
-  MOVZX(32, 8, RSCRATCH, R(RSCRATCH2));
+    // Get the high part of the GQR register
+    OpArg gqr = PPCSTATE(spr[SPR_GQR0 + i]);
+    gqr.AddMemOffset(2);
 
-  CALLptr(MScaled(RSCRATCH, SCALE_8, (u32)(u64)(&asm_routines.pairedLoadQuantized[w * 8])));
+    AND(32, R(RSCRATCH2), gqr);
+    MOVZX(32, 8, RSCRATCH, R(RSCRATCH2));
+
+    CALLptr(MScaled(RSCRATCH, SCALE_8, (u32)(u64)(&asm_routines.pairedLoadQuantized[w * 8])));
+  }
 
   MemoryExceptionCheck();
   CVTPS2PD(fpr.RX(s), R(XMM0));

Source/Core/Core/PowerPC/Jit64Common/Jit64AsmCommon.h

@@ -7,16 +7,31 @@
 #include "Core/PowerPC/JitCommon/JitAsmCommon.h"
 #include "Core/PowerPC/JitCommon/Jit_Util.h"
 
-class CommonAsmRoutines : public CommonAsmRoutinesBase, public EmuCodeBlock
-{
-protected:
-  void GenQuantizedLoads();
-  void GenQuantizedStores();
-  void GenQuantizedSingleStores();
+enum EQuantizeType : u32;
 
+class QuantizedMemoryRoutines : public EmuCodeBlock
+{
+public:
+  void GenQuantizedLoad(bool single, EQuantizeType type, int quantize);
+  void GenQuantizedStore(bool single, EQuantizeType type, int quantize);
+
+private:
+  void GenQuantizedLoadFloat(bool single, bool isInline);
+  void GenQuantizedStoreFloat(bool single, bool isInline);
+};
+
+class CommonAsmRoutines : public CommonAsmRoutinesBase, public QuantizedMemoryRoutines
+{
 public:
   void GenFifoWrite(int size);
   void GenFrsqrte();
   void GenFres();
   void GenMfcr();
+
+protected:
+  const u8* GenQuantizedLoadRuntime(bool single, EQuantizeType type);
+  const u8* GenQuantizedStoreRuntime(bool single, EQuantizeType type);
+  void GenQuantizedLoads();
+  void GenQuantizedStores();
+  void GenQuantizedSingleStores();
 };

Source/Core/Core/PowerPC/JitCommon/JitBase.h

@@ -8,6 +8,7 @@
 //#define JIT_LOG_GPR     // Enables logging of the PPC general purpose regs
 //#define JIT_LOG_FPR     // Enables logging of the PPC floating point regs
 
+#include <map>
 #include <unordered_set>
 
 #include "Common/CommonTypes.h"
@@ -88,6 +89,7 @@ protected:
     int revertFprLoad;
 
     bool assumeNoPairedQuantize;
+    std::map<u8, u32> constantGqr;
     bool firstFPInstructionFound;
     bool isLastInstruction;
     int skipInstructions;
@@ -130,7 +132,7 @@ public:
   virtual bool HandleStackFault() { return false; }
 };
 
-class Jitx86Base : public JitBase, public EmuCodeBlock
+class Jitx86Base : public JitBase, public QuantizedMemoryRoutines
 {
 protected:
   bool BackPatch(u32 emAddress, SContext* ctx);