Merge pull request #13900 from JosJuice/jit-fma-double-rounding

Jit: Implement error-free transformation for single-precision FMA

Source/Core/Common/Arm64Emitter.cpp

@@ -3156,6 +3156,10 @@ void ARM64FloatEmitter::DUP(u8 size, ARM64Reg Rd, ARM64Reg Rn, u8 index)
 
   EmitCopy(IsQuad(Rd), 0, imm5, 0, Rd, Rn);
 }
+void ARM64FloatEmitter::EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(1, 0, 3, Rd, Rn, Rm);
+}
 void ARM64FloatEmitter::FABS(u8 size, ARM64Reg Rd, ARM64Reg Rn)
 {
   Emit2RegMisc(IsQuad(Rd), 0, 2 | (size >> 6), 0xF, Rd, Rn);
@@ -3505,6 +3509,53 @@ void ARM64FloatEmitter::UCVTF(ARM64Reg Rd, ARM64Reg Rn, int scale)
   EmitConversion2(sf, 0, false, type, 0, 3, 64 - scale, Rd, Rn);
 }
 
+// Comparison
+void ARM64FloatEmitter::CMEQ(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(1, MathUtil::IntLog2(size) - 3, 0x11, Rd, Rn, Rm);
+}
+void ARM64FloatEmitter::CMEQ(u8 size, ARM64Reg Rd, ARM64Reg Rn)
+{
+  Emit2RegMisc(IsQuad(Rd), 0, MathUtil::IntLog2(size) - 3, 0x9, Rd, Rn);
+}
+void ARM64FloatEmitter::CMGE(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(0, MathUtil::IntLog2(size) - 3, 0x7, Rd, Rn, Rm);
+}
+void ARM64FloatEmitter::CMGE(u8 size, ARM64Reg Rd, ARM64Reg Rn)
+{
+  Emit2RegMisc(IsQuad(Rd), 1, MathUtil::IntLog2(size) - 3, 0x8, Rd, Rn);
+}
+void ARM64FloatEmitter::CMGT(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(0, MathUtil::IntLog2(size) - 3, 0x6, Rd, Rn, Rm);
+}
+void ARM64FloatEmitter::CMGT(u8 size, ARM64Reg Rd, ARM64Reg Rn)
+{
+  Emit2RegMisc(IsQuad(Rd), 0, MathUtil::IntLog2(size) - 3, 0x8, Rd, Rn);
+}
+void ARM64FloatEmitter::CMHI(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(1, MathUtil::IntLog2(size) - 3, 0x6, Rd, Rn, Rm);
+}
+void ARM64FloatEmitter::CMHS(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(1, MathUtil::IntLog2(size) - 3, 0x7, Rd, Rn, Rm);
+}
+void ARM64FloatEmitter::CMLE(u8 size, ARM64Reg Rd, ARM64Reg Rn)
+{
+  Emit2RegMisc(IsQuad(Rd), 1, MathUtil::IntLog2(size) - 3, 0x9, Rd, Rn);
+}
+void ARM64FloatEmitter::CMLT(u8 size, ARM64Reg Rd, ARM64Reg Rn)
+{
+  Emit2RegMisc(IsQuad(Rd), 0, MathUtil::IntLog2(size) - 3, 0xA, Rd, Rn);
+}
+void ARM64FloatEmitter::CMTST(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
+{
+  EmitThreeSame(0, MathUtil::IntLog2(size) - 3, 0x11, Rd, Rn, Rm);
+}
+
+// Float comparison
 void ARM64FloatEmitter::FCMP(ARM64Reg Rn, ARM64Reg Rm)
 {
   EmitCompare(0, 0, 0, 0, Rn, Rm);
@@ -3664,7 +3715,7 @@ void ARM64FloatEmitter::SHL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift)
 {
   ASSERT_MSG(DYNA_REC, shift < src_size, "Shift amount must be less than the element size! {} {}",
              shift, src_size);
-  EmitShiftImm(1, 0, src_size | shift, 0b01010, Rd, Rn);
+  EmitShiftImm(IsQuad(Rd), 0, src_size | shift, 0b01010, Rd, Rn);
 }
 
 void ARM64FloatEmitter::SSHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift, bool upper)
@@ -3674,11 +3725,18 @@ void ARM64FloatEmitter::SSHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift,
   EmitShiftImm(upper, 0, src_size | shift, 0b10100, Rd, Rn);
 }
 
+void ARM64FloatEmitter::SSHR(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift)
+{
+  ASSERT_MSG(DYNA_REC, shift < src_size, "Shift amount must be less than the element size! {} {}",
+             shift, src_size);
+  EmitShiftImm(IsQuad(Rd), 0, src_size * 2 - shift, 0b00000, Rd, Rn);
+}
+
 void ARM64FloatEmitter::URSHR(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift)
 {
   ASSERT_MSG(DYNA_REC, shift < src_size, "Shift amount must be less than the element size! {} {}",
              shift, src_size);
-  EmitShiftImm(1, 1, src_size * 2 - shift, 0b00100, Rd, Rn);
+  EmitShiftImm(IsQuad(Rd), 1, src_size * 2 - shift, 0b00100, Rd, Rn);
 }
 
 void ARM64FloatEmitter::USHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift, bool upper)

Source/Core/Common/Arm64Emitter.h

@@ -800,6 +800,7 @@ public:
     ARM64Reg zr = Is64Bit(Rd) ? ARM64Reg::ZR : ARM64Reg::WZR;
     CSINV(Rd, zr, zr, (CCFlags)((u32)cond ^ 1));
   }
+  void CNEG(ARM64Reg Rd, ARM64Reg Rn, CCFlags cond) { CSNEG(Rd, Rn, Rn, (CCFlags)((u32)cond ^ 1)); }
   void NEG(ARM64Reg Rd, ARM64Reg Rs) { SUB(Rd, Is64Bit(Rd) ? ARM64Reg::ZR : ARM64Reg::WZR, Rs); }
   void NEG(ARM64Reg Rd, ARM64Reg Rs, ArithOption Option)
   {
@@ -1281,6 +1282,7 @@ public:
   void BIT(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
   void BSL(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
   void DUP(u8 size, ARM64Reg Rd, ARM64Reg Rn, u8 index);
+  void EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
   void FABS(u8 size, ARM64Reg Rd, ARM64Reg Rn);
   void FADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
   void FMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
@@ -1342,6 +1344,19 @@ public:
   void SCVTF(ARM64Reg Rd, ARM64Reg Rn, int scale);
   void UCVTF(ARM64Reg Rd, ARM64Reg Rn, int scale);
 
+  // Comparison
+  void CMEQ(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
+  void CMEQ(u8 size, ARM64Reg Rd, ARM64Reg Rn);
+  void CMGE(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
+  void CMGE(u8 size, ARM64Reg Rd, ARM64Reg Rn);
+  void CMGT(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
+  void CMGT(u8 size, ARM64Reg Rd, ARM64Reg Rn);
+  void CMHI(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
+  void CMHS(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
+  void CMLE(u8 size, ARM64Reg Rd, ARM64Reg Rn);
+  void CMLT(u8 size, ARM64Reg Rd, ARM64Reg Rn);
+  void CMTST(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
+
   // Float comparison
   void FCMP(ARM64Reg Rn, ARM64Reg Rm);
   void FCMP(ARM64Reg Rn);
@@ -1380,6 +1395,7 @@ public:
   void SHL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
   void SSHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
   void SSHLL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
+  void SSHR(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
   void URSHR(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
   void USHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
   void USHLL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);

Source/Core/Common/x64Emitter.cpp

@@ -2519,19 +2519,19 @@ void XEmitter::PUNPCKLQDQ(X64Reg dest, const OpArg& arg)
   WriteSSEOp(0x66, 0x6C, dest, arg);
 }
 
-void XEmitter::PSRLW(X64Reg reg, int shift)
+void XEmitter::PSRLW(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x71, (X64Reg)2, R(reg));
   Write8(shift);
 }
 
-void XEmitter::PSRLD(X64Reg reg, int shift)
+void XEmitter::PSRLD(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x72, (X64Reg)2, R(reg));
   Write8(shift);
 }
 
-void XEmitter::PSRLQ(X64Reg reg, int shift)
+void XEmitter::PSRLQ(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x73, (X64Reg)2, R(reg));
   Write8(shift);
@@ -2542,38 +2542,38 @@ void XEmitter::PSRLQ(X64Reg reg, const OpArg& arg)
   WriteSSEOp(0x66, 0xd3, reg, arg);
 }
 
-void XEmitter::PSRLDQ(X64Reg reg, int shift)
+void XEmitter::PSRLDQ(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x73, (X64Reg)3, R(reg));
   Write8(shift);
 }
 
-void XEmitter::PSLLW(X64Reg reg, int shift)
+void XEmitter::PSLLW(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x71, (X64Reg)6, R(reg));
   Write8(shift);
 }
 
-void XEmitter::PSLLD(X64Reg reg, int shift)
+void XEmitter::PSLLD(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x72, (X64Reg)6, R(reg));
   Write8(shift);
 }
 
-void XEmitter::PSLLQ(X64Reg reg, int shift)
+void XEmitter::PSLLQ(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x73, (X64Reg)6, R(reg));
   Write8(shift);
 }
 
-void XEmitter::PSLLDQ(X64Reg reg, int shift)
+void XEmitter::PSLLDQ(X64Reg reg, u8 shift)
 {
   WriteSSEOp(0x66, 0x73, (X64Reg)7, R(reg));
   Write8(shift);
 }
 
 // WARNING not REX compatible
-void XEmitter::PSRAW(X64Reg reg, int shift)
+void XEmitter::PSRAW(X64Reg reg, u8 shift)
 {
   if (reg > 7)
     PanicAlertFmt("The PSRAW-emitter does not support regs above 7");
@@ -2585,7 +2585,7 @@ void XEmitter::PSRAW(X64Reg reg, int shift)
 }
 
 // WARNING not REX compatible
-void XEmitter::PSRAD(X64Reg reg, int shift)
+void XEmitter::PSRAD(X64Reg reg, u8 shift)
 {
   if (reg > 7)
     PanicAlertFmt("The PSRAD-emitter does not support regs above 7");
@@ -2695,6 +2695,11 @@ void XEmitter::BLENDPD(X64Reg dest, const OpArg& arg, u8 blend)
   Write8(blend);
 }
 
+void XEmitter::PCMPEQQ(X64Reg dest, const OpArg& arg)
+{
+  WriteSSE41Op(0x66, 0x3829, dest, arg);
+}
+
 void XEmitter::PAND(X64Reg dest, const OpArg& arg)
 {
   WriteSSEOp(0x66, 0xDB, dest, arg);
@@ -3038,6 +3043,12 @@ void XEmitter::VPXOR(X64Reg regOp1, X64Reg regOp2, const OpArg& arg)
   WriteAVXOp(0x66, 0xEF, regOp1, regOp2, arg);
 }
 
+void XEmitter::VPSLLQ(X64Reg regOp1, X64Reg regOp2, u8 shift)
+{
+  WriteAVXOp(0x66, 0x73, (X64Reg)6, regOp1, R(regOp2));
+  Write8(shift);
+}
+
 void XEmitter::VMOVAPS(const OpArg& arg, X64Reg regOp)
 {
   WriteAVXOp(0x00, 0x29, regOp, X64Reg::INVALID_REG, arg);

Source/Core/Common/x64Emitter.h

@@ -801,19 +801,19 @@ public:
   void PSHUFLW(X64Reg dest, const OpArg& arg, u8 shuffle);
   void PSHUFHW(X64Reg dest, const OpArg& arg, u8 shuffle);
 
-  void PSRLW(X64Reg reg, int shift);
-  void PSRLD(X64Reg reg, int shift);
-  void PSRLQ(X64Reg reg, int shift);
+  void PSRLW(X64Reg reg, u8 shift);
+  void PSRLD(X64Reg reg, u8 shift);
+  void PSRLQ(X64Reg reg, u8 shift);
   void PSRLQ(X64Reg reg, const OpArg& arg);
-  void PSRLDQ(X64Reg reg, int shift);
+  void PSRLDQ(X64Reg reg, u8 shift);
 
-  void PSLLW(X64Reg reg, int shift);
-  void PSLLD(X64Reg reg, int shift);
-  void PSLLQ(X64Reg reg, int shift);
-  void PSLLDQ(X64Reg reg, int shift);
+  void PSLLW(X64Reg reg, u8 shift);
+  void PSLLD(X64Reg reg, u8 shift);
+  void PSLLQ(X64Reg reg, u8 shift);
+  void PSLLDQ(X64Reg reg, u8 shift);
 
-  void PSRAW(X64Reg reg, int shift);
-  void PSRAD(X64Reg reg, int shift);
+  void PSRAW(X64Reg reg, u8 shift);
+  void PSRAD(X64Reg reg, u8 shift);
 
   // SSE4: data type conversions
   void PMOVSXBW(X64Reg dest, const OpArg& arg);
@@ -836,6 +836,9 @@ public:
   void BLENDPS(X64Reg dest, const OpArg& arg, u8 blend);
   void BLENDPD(X64Reg dest, const OpArg& arg, u8 blend);
 
+  // SSE4: compare instructions
+  void PCMPEQQ(X64Reg dest, const OpArg& arg);
+
   // AVX
   void VADDSS(X64Reg regOp1, X64Reg regOp2, const OpArg& arg);
   void VSUBSS(X64Reg regOp1, X64Reg regOp2, const OpArg& arg);
@@ -878,6 +881,8 @@ public:
   void VPOR(X64Reg regOp1, X64Reg regOp2, const OpArg& arg);
   void VPXOR(X64Reg regOp1, X64Reg regOp2, const OpArg& arg);
 
+  void VPSLLQ(X64Reg regOp1, X64Reg regOp2, u8 shift);
+
   void VMOVAPS(const OpArg& arg, X64Reg regOp);
 
   void VZEROUPPER();

Source/Core/Core/Config/MainSettings.cpp

@@ -222,6 +222,7 @@ const Info<bool> MAIN_DIVIDE_BY_ZERO_EXCEPTIONS{{System::Main, "Core", "DivByZer
                                                 false};
 const Info<bool> MAIN_FPRF{{System::Main, "Core", "FPRF"}, false};
 const Info<bool> MAIN_ACCURATE_NANS{{System::Main, "Core", "AccurateNaNs"}, false};
+const Info<bool> MAIN_ACCURATE_FMADDS{{System::Main, "Core", "AccurateFmadds"}, true};
 const Info<bool> MAIN_DISABLE_ICACHE{{System::Main, "Core", "DisableICache"}, false};
 const Info<float> MAIN_EMULATION_SPEED{{System::Main, "Core", "EmulationSpeed"}, 1.0f};
 #if defined(ANDROID)

Source/Core/Core/Config/MainSettings.h

@@ -128,6 +128,7 @@ extern const Info<bool> MAIN_FLOAT_EXCEPTIONS;
 extern const Info<bool> MAIN_DIVIDE_BY_ZERO_EXCEPTIONS;
 extern const Info<bool> MAIN_FPRF;
 extern const Info<bool> MAIN_ACCURATE_NANS;
+extern const Info<bool> MAIN_ACCURATE_FMADDS;
 extern const Info<bool> MAIN_DISABLE_ICACHE;
 extern const Info<float> MAIN_EMULATION_SPEED;
 extern const Info<bool> MAIN_PRECISION_FRAME_TIMING;

Source/Core/Core/ConfigLoaders/NetPlayConfigLoader.cpp

@@ -80,6 +80,7 @@ public:
     layer->Set(Config::MAIN_DIVIDE_BY_ZERO_EXCEPTIONS, m_settings.divide_by_zero_exceptions);
     layer->Set(Config::MAIN_FPRF, m_settings.fprf);
     layer->Set(Config::MAIN_ACCURATE_NANS, m_settings.accurate_nans);
+    layer->Set(Config::MAIN_ACCURATE_FMADDS, m_settings.accurate_fmadds);
     layer->Set(Config::MAIN_DISABLE_ICACHE, m_settings.disable_icache);
     layer->Set(Config::MAIN_SYNC_ON_SKIP_IDLE, m_settings.sync_on_skip_idle);
     layer->Set(Config::MAIN_SYNC_GPU, m_settings.sync_gpu);

Source/Core/Core/NetPlayProto.h

@@ -68,6 +68,7 @@ struct NetSettings
   bool divide_by_zero_exceptions = false;
   bool fprf = false;
   bool accurate_nans = false;
+  bool accurate_fmadds = false;
   bool disable_icache = false;
   bool sync_on_skip_idle = false;
   bool sync_gpu = false;

Source/Core/Core/NetPlayServer.cpp

@@ -1425,6 +1425,7 @@ bool NetPlayServer::SetupNetSettings()
   settings.divide_by_zero_exceptions = Config::Get(Config::MAIN_DIVIDE_BY_ZERO_EXCEPTIONS);
   settings.fprf = Config::Get(Config::MAIN_FPRF);
   settings.accurate_nans = Config::Get(Config::MAIN_ACCURATE_NANS);
+  settings.accurate_fmadds = Config::Get(Config::MAIN_ACCURATE_FMADDS);
   settings.disable_icache = Config::Get(Config::MAIN_DISABLE_ICACHE);
   settings.sync_on_skip_idle = Config::Get(Config::MAIN_SYNC_ON_SKIP_IDLE);
   settings.sync_gpu = Config::Get(Config::MAIN_SYNC_GPU);

Source/Core/Core/PowerPC/Interpreter/Interpreter_FPUtils.h

@@ -342,12 +342,12 @@ inline FPResult NI_madd_msub(PowerPC::PowerPCState& ppc_state, double a, double
   //    - This will cause `d` to round to 100...00, meaning it will tie then round upwards.
   // 3. Tying up to even because `c` is too small
   //    a. The highest bit of `d` is 1, the rest of the bits of `d` are 0 (this means it ties)
-  //    b. The lowest bit of `f` is 1 (this means it ties to even downwards)
+  //    b. The lowest bit of `f` is 1 (this means it ties to even upwards)
   //    c. `c` is negative and does not round `d` downwards
   //    -  This is similar to the first one but in reverse, rounding up instead of down.
   // 4. Tying down because `d` rounded down
   //    a. The highest and lowest bits of `d` are 1, the rest of the bits of `d` are 0
-  //    b. The lowest bit of `f` is 0 (this means it ties to even upwards)
+  //    b. The lowest bit of `f` is 0 (this means it ties to even downwards)
   //    c. `c` is negative, and the highest bit of c is 1,
   //       and at least one other bit of c is nonzero
   //    - The backwards counterpart to case 2, this will cause `d` to round back down to 100..00,
@@ -375,12 +375,6 @@ inline FPResult NI_madd_msub(PowerPC::PowerPCState& ppc_state, double a, double
   // - Correct ordering of NaN checking (for both double and single precision)
   // - Rounding frC up
   // - Rounding only once for single precision inputs (this will be the large majority of cases!)
-  //   - Currently this is interpreter-only.
-  //     This can be implemented in the JIT just as easily, though.
-  //     Eventually the JITs should hopefully support detecting back to back
-  //     single-precision operations, which will lead to no overhead at all.
-  //     In the cases where JITs can't do this, an alternative method is used, as
-  //     is done in the interpreter as well.
   // - Rounding only once for double precision inputs
   //   - This is a side effect of how we handle single-precision inputs: By doing
   //     error calculations rather than checking if every input is a float, we ensure that we know
@@ -421,7 +415,7 @@ inline FPResult NI_madd_msub(PowerPC::PowerPCState& ppc_state, double a, double
     const double b_sign = sub ? -b : b;
     result.value = std::fma(a, c_round, b_sign);
 
-    // We then check if we're currently tying in rounding directioh
+    // We then check if we're currently tying in rounding direction
     const u64 result_bits = std::bit_cast<u64>(result.value);
 
     // The mask of the `d` bits as shown in the above comments
@@ -432,9 +426,8 @@ inline FPResult NI_madd_msub(PowerPC::PowerPCState& ppc_state, double a, double
 
     // Because we check this entire mask which includes a 1 bit, we can be sure that
     // if this result passes, the input is not an infinity that would become a NaN.
-    // This means that, for the JITs, if they only wanted to check for a subset of these
-    // bits (e.g. only checking if the last one was 0), then using the zero flag for a branch,
-    // they would have to check if the result was NaN before here.
+    // If we had only checked for a subset of these bits (e.g. only checking if the last
+    // one was 0), we would have needed to also check if the exponent was all ones.
     if ((result_bits & D_MASK) == EVEN_TIE)
     {
       // Because we have a tie, we now compute any error in the FMA calculation

Source/Core/Core/PowerPC/Jit64/Jit.cpp

@@ -1284,9 +1284,9 @@ BitSet8 Jit64::ComputeStaticGQRs(const PPCAnalyst::CodeBlock& cb) const
   return cb.m_gqr_used & ~cb.m_gqr_modified;
 }
 
-BitSet32 Jit64::CallerSavedRegistersInUse() const
+BitSet32 Jit64::CallerSavedRegistersInUse(BitSet32 additional_registers) const
 {
-  BitSet32 in_use = gpr.RegistersInUse() | (fpr.RegistersInUse() << 16);
+  BitSet32 in_use = gpr.RegistersInUse() | (fpr.RegistersInUse() << 16) | additional_registers;
   return in_use & ABI_ALL_CALLER_SAVED;
 }
 

Source/Core/Core/PowerPC/Jit64/Jit.h

@@ -77,7 +77,7 @@ public:
   // Returns false if no free memory region can be found for either of the two.
   bool SetEmitterStateToFreeCodeRegion();
 
-  BitSet32 CallerSavedRegistersInUse() const;
+  BitSet32 CallerSavedRegistersInUse(BitSet32 additional_registers = {}) const;
   BitSet8 ComputeStaticGQRs(const PPCAnalyst::CodeBlock&) const;
 
   void IntializeSpeculativeConstants();
@@ -153,9 +153,10 @@ public:
   void FinalizeSingleResult(Gen::X64Reg output, const Gen::OpArg& input, bool packed = true,
                             bool duplicate = false);
   void FinalizeDoubleResult(Gen::X64Reg output, const Gen::OpArg& input);
-  void HandleNaNs(UGeckoInstruction inst, Gen::X64Reg xmm, Gen::X64Reg clobber,
-                  std::optional<Gen::OpArg> Ra, std::optional<Gen::OpArg> Rb,
-                  std::optional<Gen::OpArg> Rc);
+  [[nodiscard]] Gen::FixupBranch HandleNaNs(UGeckoInstruction inst, Gen::X64Reg xmm,
+                                            Gen::X64Reg clobber, std::optional<Gen::OpArg> Ra,
+                                            std::optional<Gen::OpArg> Rb,
+                                            std::optional<Gen::OpArg> Rc);
 
   void MultiplyImmediate(u32 imm, int a, int d, bool overflow);
 

Source/Core/Core/PowerPC/Jit64/JitAsm.cpp

@@ -265,6 +265,10 @@ void Jit64AsmRoutineManager::GenerateCommon()
   GenMfcr();
   cdts = AlignCode4();
   GenConvertDoubleToSingle();
+  fmadds_eft = AlignCode4();
+  GenerateFmaddsEft();
+  ps_madd_eft = AlignCode4();
+  GeneratePsMaddEft();
 
   GenQuantizedLoads();
   GenQuantizedSingleLoads();

Source/Core/Core/PowerPC/Jit64/Jit_Paired.cpp

@@ -100,12 +100,19 @@ void Jit64::ps_muls(UGeckoInstruction inst)
   default:
     PanicAlertFmt("ps_muls WTF!!!");
   }
+
   if (round_input)
     Force25BitPrecision(XMM1, R(Rc_duplicated), XMM0);
   else if (XMM1 != Rc_duplicated)
     MOVAPD(XMM1, Rc_duplicated);
   MULPD(XMM1, Ra);
-  HandleNaNs(inst, XMM1, XMM0, Ra, std::nullopt, Rc_duplicated);
+
+  if (m_accurate_nans)
+  {
+    const FixupBranch handled_nans = HandleNaNs(inst, XMM1, XMM0, Ra, std::nullopt, Rc_duplicated);
+    SetJumpTarget(handled_nans);
+  }
+
   FinalizeSingleResult(Rd, R(XMM1));
 }
 

Source/Core/Core/PowerPC/Jit64Common/EmuCodeBlock.cpp

@@ -741,7 +741,8 @@ void EmuCodeBlock::JitClearCA()
 // Abstract between AVX and SSE: automatically handle 3-operand instructions
 void EmuCodeBlock::avx_op(void (XEmitter::*avxOp)(X64Reg, X64Reg, const OpArg&),
                           void (XEmitter::*sseOp)(X64Reg, const OpArg&), X64Reg regOp,
-                          const OpArg& arg1, const OpArg& arg2, bool packed, bool reversible)
+                          const OpArg& arg1, const OpArg& arg2, bool packed, bool reversible,
+                          X64Reg scratch)
 {
   if (arg1.IsSimpleReg(regOp))
   {
@@ -778,19 +779,19 @@ void EmuCodeBlock::avx_op(void (XEmitter::*avxOp)(X64Reg, X64Reg, const OpArg&),
   else
   {
     // The ugly case: Not reversible, and we have regOp == arg2 without AVX or with arg1 == memory
-    if (!arg1.IsSimpleReg(XMM0))
-      MOVAPD(XMM0, arg1);
+    if (!arg1.IsSimpleReg(scratch))
+      MOVAPD(scratch, arg1);
     if (cpu_info.bAVX)
     {
-      (this->*avxOp)(regOp, XMM0, arg2);
+      (this->*avxOp)(regOp, scratch, arg2);
     }
     else
     {
-      (this->*sseOp)(XMM0, arg2);
+      (this->*sseOp)(scratch, arg2);
       if (packed)
-        MOVAPD(regOp, R(XMM0));
+        MOVAPD(regOp, R(scratch));
       else
-        MOVSD(regOp, R(XMM0));
+        MOVSD(regOp, R(scratch));
     }
   }
 }
@@ -798,7 +799,7 @@ void EmuCodeBlock::avx_op(void (XEmitter::*avxOp)(X64Reg, X64Reg, const OpArg&),
 // Abstract between AVX and SSE: automatically handle 3-operand instructions
 void EmuCodeBlock::avx_op(void (XEmitter::*avxOp)(X64Reg, X64Reg, const OpArg&, u8),
                           void (XEmitter::*sseOp)(X64Reg, const OpArg&, u8), X64Reg regOp,
-                          const OpArg& arg1, const OpArg& arg2, u8 imm)
+                          const OpArg& arg1, const OpArg& arg2, u8 imm, X64Reg scratch)
 {
   if (arg1.IsSimpleReg(regOp))
   {
@@ -816,21 +817,40 @@ void EmuCodeBlock::avx_op(void (XEmitter::*avxOp)(X64Reg, X64Reg, const OpArg&,
   else
   {
     // The ugly case: regOp == arg2 without AVX, or with arg1 == memory
-    if (!arg1.IsSimpleReg(XMM0))
-      MOVAPD(XMM0, arg1);
+    if (!arg1.IsSimpleReg(scratch))
+      MOVAPD(scratch, arg1);
     if (cpu_info.bAVX)
     {
-      (this->*avxOp)(regOp, XMM0, arg2, imm);
+      (this->*avxOp)(regOp, scratch, arg2, imm);
     }
     else
     {
-      (this->*sseOp)(XMM0, arg2, imm);
-      if (regOp != XMM0)
-        MOVAPD(regOp, R(XMM0));
+      (this->*sseOp)(scratch, arg2, imm);
+      if (regOp != scratch)
+        MOVAPD(regOp, R(scratch));
     }
   }
 }
 
+// Abstract between AVX and SSE: automatically handle 3-operand instructions
+void EmuCodeBlock::avx_op(void (XEmitter::*avxOp)(X64Reg, X64Reg, u8),
+                          void (XEmitter::*sseOp)(X64Reg, u8), X64Reg regOp1, X64Reg regOp2, u8 imm)
+{
+  if (regOp1 == regOp2)
+  {
+    (this->*sseOp)(regOp1, imm);
+  }
+  else if (cpu_info.bAVX)
+  {
+    (this->*avxOp)(regOp1, regOp2, imm);
+  }
+  else
+  {
+    MOVAPD(regOp1, R(regOp2));
+    (this->*sseOp)(regOp1, imm);
+  }
+}
+
 alignas(16) static const u64 psMantissaTruncate[2] = {0xFFFFFFFFF8000000ULL, 0xFFFFFFFFF8000000ULL};
 alignas(16) static const u64 psRoundBit[2] = {0x8000000, 0x8000000};
 
@@ -842,8 +862,9 @@ void EmuCodeBlock::Force25BitPrecision(X64Reg output, const OpArg& input, X64Reg
 {
   if (m_jit.jo.accurateSinglePrecision)
   {
+    DEBUG_ASSERT(output != tmp);
     // mantissa = (mantissa & ~0xFFFFFFF) + ((mantissa & (1ULL << 27)) << 1);
-    if (input.IsSimpleReg() && cpu_info.bAVX)
+    if (input.IsSimpleReg() && !input.IsSimpleReg(tmp) && cpu_info.bAVX)
     {
       VPAND(tmp, input.GetSimpleReg(), MConst(psRoundBit));
       VPAND(output, input.GetSimpleReg(), MConst(psMantissaTruncate));

Source/Core/Core/PowerPC/Jit64Common/EmuCodeBlock.h

@@ -113,10 +113,14 @@ public:
   void avx_op(void (Gen::XEmitter::*avxOp)(Gen::X64Reg, Gen::X64Reg, const Gen::OpArg&),
               void (Gen::XEmitter::*sseOp)(Gen::X64Reg, const Gen::OpArg&), Gen::X64Reg regOp,
               const Gen::OpArg& arg1, const Gen::OpArg& arg2, bool packed = true,
-              bool reversible = false);
+              bool reversible = false, Gen::X64Reg scratch = Gen::XMM0);
   void avx_op(void (Gen::XEmitter::*avxOp)(Gen::X64Reg, Gen::X64Reg, const Gen::OpArg&, u8),
               void (Gen::XEmitter::*sseOp)(Gen::X64Reg, const Gen::OpArg&, u8), Gen::X64Reg regOp,
-              const Gen::OpArg& arg1, const Gen::OpArg& arg2, u8 imm);
+              const Gen::OpArg& arg1, const Gen::OpArg& arg2, u8 imm,
+              Gen::X64Reg scratch = Gen::XMM0);
+  void avx_op(void (Gen::XEmitter::*avxOp)(Gen::X64Reg, Gen::X64Reg, u8),
+              void (Gen::XEmitter::*sseOp)(Gen::X64Reg, u8), Gen::X64Reg regOp1, Gen::X64Reg regOp2,
+              u8 imm);
 
   void Force25BitPrecision(Gen::X64Reg output, const Gen::OpArg& input, Gen::X64Reg tmp);
 

Source/Core/Core/PowerPC/Jit64Common/Jit64AsmCommon.cpp

@@ -326,6 +326,98 @@ void CommonAsmRoutines::GenMfcr()
   Common::JitRegister::Register(start, GetCodePtr(), "JIT_Mfcr");
 }
 
+// Inputs:
+// XMM0: First error term
+// XMM1: Result with potentially incorrect rounding
+// XMM2: Second error term, negated
+//
+// Outputs result with corrected rounding in XMM1.
+// Clobbers RSCRATCH, RSCRATCH2, XMM0, XMM2, and flags.
+void CommonAsmRoutines::GenerateFmaddsEft()
+{
+  // Check if XMM1 is an even tie, i.e. check (input & 0x1fffffff) == 0x10000000
+  MOVQ_xmm(R(RSCRATCH), XMM1);
+  MOV(32, R(RSCRATCH2), Imm32(0x80000000));
+  LEA(32, RSCRATCH2, MComplex(RSCRATCH2, RSCRATCH, SCALE_8, 0));
+  TEST(32, R(RSCRATCH2), R(RSCRATCH2));
+  FixupBranch even_tie = J_CC(CCFlags::CC_Z);
+
+  const u8* ret = GetCodePtr();
+  RET();
+
+  // Check if the error is 0
+  SetJumpTarget(even_tie);
+  SUBSD(XMM0, R(XMM2));
+  XORPD(XMM2, R(XMM2));
+  UCOMISD(XMM0, R(XMM2));
+  J_CC(CCFlags::CC_E, ret);
+
+  // Round XMM1 up or down
+  MOVQ_xmm(R(RSCRATCH2), XMM0);
+  XOR(64, R(RSCRATCH2), R(RSCRATCH));
+  SAR(64, R(RSCRATCH2), Imm8(63));
+  OR(64, R(RSCRATCH2), Imm8(1));
+  ADD(64, R(RSCRATCH), R(RSCRATCH2));
+  MOVQ_xmm(XMM1, R(RSCRATCH));
+  RET();
+}
+
+alignas(16) static const __m128i double_msb = _mm_set_epi64x(0x8000000000000000,
+                                                             0x8000000000000000);
+alignas(16) static const __m128i double_lsb = _mm_set_epi64x(1, 1);
+
+// Inputs:
+// XMM0: First error terms
+// XMM1: Results with potentially incorrect rounding
+// XMM2: Second error terms, negated
+//
+// Outputs results with corrected rounding in XMM1. Clobbers RSCRATCH, XMM0-XMM3, and flags.
+void CommonAsmRoutines::GeneratePsMaddEft()
+{
+  // Check if XMM1 has an even tie, i.e. check (input & 0x1fffffff) == 0x10000000
+  avx_op(&XEmitter::VPSLLQ, &XEmitter::PSLLQ, XMM3, XMM1, 35);
+  if (cpu_info.bSSE4_1)
+  {
+    PCMPEQQ(XMM3, MConst(double_msb));
+  }
+  else
+  {
+    PCMPEQW(XMM3, MConst(double_msb));
+    PSHUFD(XMM3, R(XMM3), 0xF5);
+  }
+
+  // Just for performance, exit early if there is no even tie
+  if (cpu_info.bSSE4_1)
+  {
+    PTEST(XMM3, R(XMM3));
+  }
+  else
+  {
+    PMOVMSKB(RSCRATCH, R(XMM3));
+    TEST(32, R(RSCRATCH), R(RSCRATCH));
+  }
+  FixupBranch even_tie = J_CC(CCFlags::CC_NZ);
+  RET();
+  SetJumpTarget(even_tie);
+
+  // Check if the error is zero
+  SUBPD(XMM0, R(XMM2));
+  XORPD(XMM2, R(XMM2));
+  CMPPD(XMM2, R(XMM0), CMP_EQ);
+
+  // Store -1 or 1 in XMM0 depending on whether we're rounding down or up
+  PXOR(XMM0, R(XMM1));
+  PSRAD(XMM0, 31);
+  PSHUFD(XMM0, R(XMM0), 0xF5);
+  POR(XMM0, MConst(double_lsb));
+
+  // Round the elements that have both a non-zero error and an even tie
+  PANDN(XMM2, R(XMM3));
+  PAND(XMM0, R(XMM2));
+  PADDQ(XMM1, R(XMM0));
+  RET();
+}
+
 // Safe + Fast Quantizers, originally from JITIL by magumagu
 alignas(16) static const float m_65535[4] = {65535.0f, 65535.0f, 65535.0f, 65535.0f};
 alignas(16) static const float m_32767 = 32767.0f;

Source/Core/Core/PowerPC/Jit64Common/Jit64AsmCommon.h

@@ -33,6 +33,8 @@ public:
 
 protected:
   void GenConvertDoubleToSingle();
+  void GenerateFmaddsEft();
+  void GeneratePsMaddEft();
   const u8* GenQuantizedLoadRuntime(bool single, EQuantizeType type);
   const u8* GenQuantizedStoreRuntime(bool single, EQuantizeType type);
   void GenQuantizedLoads();

Source/Core/Core/PowerPC/JitArm64/Jit.h

@@ -324,6 +324,8 @@ protected:
   void GenerateConvertDoubleToSingle();
   void GenerateConvertSingleToDouble();
   void GenerateFPRF(bool single);
+  void GenerateFmaddsEft();
+  void GeneratePsMaddEft();
   void GenerateQuantizedLoads();
   void GenerateQuantizedStores();