diff --git a/pcsx2/R5900OpcodeTables.cpp b/pcsx2/R5900OpcodeTables.cpp index 3644483b5b..b5395f6f14 100644 --- a/pcsx2/R5900OpcodeTables.cpp +++ b/pcsx2/R5900OpcodeTables.cpp @@ -25,7 +25,7 @@ namespace R5900 { // Generates an entry for the given opcode name. // Assumes the default function naming schemes for interpreter and recompiler functions. -#ifdef _M_X86 // TODO(Stenzek): Remove me once EE/VU/IOP recs are added. +#if defined(_M_X86) || defined(ARCH_ARM64) // ARM64 EE recompiler is supported # define MakeOpcode( name, cycles, flags ) \ static const OPCODE name = { \ #name, \ diff --git a/pcsx2/arm64/BaseblockEx-arm64.h b/pcsx2/arm64/BaseblockEx-arm64.h new file mode 100644 index 0000000000..bca1f2f1cb --- /dev/null +++ b/pcsx2/arm64/BaseblockEx-arm64.h @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64-specific BaseBlocks with block linking that stays signal-safe. +// +// x86 BaseBlocks (pcsx2/x86/BaseblockEx.h) uses a std::multimap to track +// pending link sites and rewrites them whenever a block is added or +// removed. The multimap is not used here because Remove() runs from the +// SIGSEGV fastmem handler, where mutating an STL container is unsafe. +// +// This implementation supports block linking while keeping Remove() signal-safe: +// - Link()/New() touch the multimap, but they only run from the +// compile path (single-threaded, never from a signal). +// - Remove() does NOT walk the link map. Instead it overwrites the +// first 4 bytes of each removed block with `B JITCompile`, so any +// stale link still resolves correctly via the dispatcher (which +// can re-patch the link to the freshly compiled target on its next +// dispatch). Block memory isn't reclaimed until a full reset, so +// this 4-byte rewrite always lands on memory the recompiler still +// owns. +// +// The patch site for each link is the address of a single B instruction +// emitted by SetBranchImm (see iR5900-arm64.cpp). Aligned 32-bit stores +// are atomic on AArch64, and `cacheflush(2)` on the patch site is +// async-signal-safe, so the redirect-stub Remove() write is signal-safe. +// +// Range: B imm26 covers ±128 MB. The EE recompiler region is 64 MB +// (HostMemoryMap::EErecSize), and JITCompile lives in the same region, +// so all link sites are reachable with a single B. + +#pragma once + +#include + +#include "x86/BaseblockEx.h" // BASEBLOCK, BASEBLOCKEX, BaseBlockArray, recLUT_SetPage + +class Arm64BaseBlocks +{ +protected: + using linkmap_t = std::multimap; + + BaseBlockArray blocks; + linkmap_t links; + uptr jitcompile = 0; + + // Encode a B imm26 from `site` to `target`. ARM64 B encoding: + // bits[31:26] = 000101 + // bits[25:0] = sign-extended ((target - site) >> 2) + static u32 EncodeB(uptr site, uptr target) + { + const intptr_t off = static_cast(target) - static_cast(site); + pxAssertRel((off & 3) == 0, "Branch offset not 4-byte aligned"); + const intptr_t imm26 = off >> 2; + pxAssertRel(imm26 >= -(1 << 25) && imm26 < (1 << 25), "Branch offset out of B imm26 range"); + return 0x14000000u | (static_cast(imm26) & 0x03FFFFFFu); + } + + static void PatchAtomic(uptr site, u32 instr) + { + // 4-byte aligned word stores are atomic on AArch64. + *reinterpret_cast(site) = instr; + // Then make sure cores fetching instructions see the new word. + __builtin___clear_cache(reinterpret_cast(site), + reinterpret_cast(site) + 4); + } + +public: + Arm64BaseBlocks() + : blocks(0x4000) + { + } + + void SetJITCompile(const void* recompiler_) + { + jitcompile = reinterpret_cast(recompiler_); + } + + // Register a link site that wants to branch directly to the block at + // `pc`. Patches immediately if a block already exists; otherwise + // records the site so New(pc, ...) can patch it later. + void Link(u32 pc, void* patch_site) + { + pxAssertRel(jitcompile, "SetJITCompile() must be called before Link()"); + + BASEBLOCKEX* target = Get(pc); + const uptr target_addr = (target && target->startpc == pc) + ? target->fnptr : jitcompile; + PatchAtomic(reinterpret_cast(patch_site), + EncodeB(reinterpret_cast(patch_site), target_addr)); + + links.insert({pc, reinterpret_cast(patch_site)}); + } + + BASEBLOCKEX* New(u32 startpc, uptr fnptr) + { + // Patch any pending links waiting for a block at this PC. After + // patching they go directly to fnptr instead of routing through + // JITCompile. + const auto range = links.equal_range(startpc); + for (auto it = range.first; it != range.second; ++it) + PatchAtomic(it->second, EncodeB(it->second, fnptr)); + + return blocks.insert(startpc, fnptr); + } + + int LastIndex(u32 startpc) const + { + if (blocks.size() == 0) + return -1; + + int imin = 0, imax = (int)blocks.size() - 1, imid; + + while (imin != imax) + { + imid = (imin + imax + 1) >> 1; + + if (blocks[imid].startpc > startpc) + imax = imid - 1; + else + imin = imid; + } + + if (IsDevBuild) + { + if (imin != 0) + pxAssert(blocks[imin].startpc <= startpc); + if (imin < (int)blocks.size() - 1) + pxAssert(blocks[imin + 1].startpc > startpc); + } + + return imin; + } + + __fi int Index(u32 startpc) const + { + int idx = LastIndex(startpc); + + if ((idx == -1) || (startpc < blocks[idx].startpc) || + ((blocks[idx].size) && (startpc >= blocks[idx].startpc + blocks[idx].size * 4))) + return -1; + else + return idx; + } + + __fi BASEBLOCKEX* operator[](int idx) + { + if (idx < 0 || idx >= (int)blocks.size()) + return 0; + + return &blocks[idx]; + } + + __fi BASEBLOCKEX* Get(u32 startpc) + { + return (*this)[Index(startpc)]; + } + + // Signal-safe: writes a redirect stub at each removed block's entry + // point so any stale link still resolves through JITCompile, then + // erases from the flat sorted array. Does NOT touch the link map — + // stale entries there are harmless (they just trigger a re-patch on + // the next compile cycle for the same PC). + __fi void Remove(int first, int last) + { + pxAssert(first <= last); + + if (jitcompile) + { + for (int i = first; i <= last; ++i) + { + const uptr site = blocks[i].fnptr; + PatchAtomic(site, EncodeB(site, jitcompile)); + } + } + + blocks.erase(first, last + 1); + } + + __fi void Reset() + { + blocks.clear(); + links.clear(); + } + +#ifdef PCSX2_RECOMPILER_TESTS + // Test-only introspection. Returns true iff a link patch site within the + // block containing src_pc targets a block at dst_pc. The link multimap + // is keyed by destination PC, so the entries for dst_pc are walked to check + // whether the patch site lies inside [block.fnptr, block.fnptr + x86size), + // where x86size is BASEBLOCKEX's (legacy-named) host machine-code byte size. + // O(L_d + log B) where L_d is the number of links to dst_pc and B is the + // block count. + bool IsLinked(u32 src_pc, u32 dst_pc) const + { + const int idx = LastIndex(src_pc); + if (idx < 0) + return false; + const BASEBLOCKEX& b = blocks[idx]; + if (src_pc < b.startpc || src_pc >= b.startpc + b.size * 4) + return false; + const uptr lo = b.fnptr; + const uptr hi = b.fnptr + b.x86size; + const auto range = links.equal_range(dst_pc); + for (auto it = range.first; it != range.second; ++it) + { + if (it->second >= lo && it->second < hi) + return true; + } + return false; + } +#endif +}; diff --git a/pcsx2/arm64/iCOP0-arm64.cpp b/pcsx2/arm64/iCOP0-arm64.cpp new file mode 100644 index 0000000000..6acb488f55 --- /dev/null +++ b/pcsx2/arm64/iCOP0-arm64.cpp @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE COP0 Instruction Codegen — NEON-based + +#include "arm64/iR5900-arm64.h" +#include "arm64/AsmHelpers.h" + +#include "Hw.h" +#include "Memory.h" + +namespace a64 = vixl::aarch64; + +namespace Interp = R5900::Interpreter::OpcodeImpl::COP0; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { +namespace COP0 { + +// COP0 branch (BC0F/T/FL/TL) — native DMAC-condition test. The branch +// condition is (((DMAC_STAT | ~DMAC_PCR) & 0x3ff) == 0x3ff): BC0T branches +// when true, BC0F when false. Emitting the test inline avoids a costly +// iFlushCall + interpreter dispatch per iteration in DMAC-poll spin loops. +// BC0 uses the same SaveBranchState/recompileNextInstruction/SetBranchImm +// branch-emission pattern as recBC1F/recBC1T on this target. + +// Emit the DMAC condition test, leaving the result in the flags (CMP vs 0x3ff). +// 32-bit loads are fine: the result is masked to the low 10 bits, matching x86. +static void _setupBranchTestBC0() +{ + _eeFlushAllDirty(); + armLoadPtr(RWARG1, &psHu32(DMAC_PCR)); + armAsm->Mvn(RWARG1, RWARG1); // ~PCR + armLoadPtr(RWSCRATCH, &psHu32(DMAC_STAT)); + armAsm->Orr(RWARG1, RWARG1, RWSCRATCH); // STAT | ~PCR + armAsm->And(RWARG1, RWARG1, 0x3ff); + armAsm->Cmp(RWARG1, 0x3ff); // EQ ⇔ condition true +} + +static a64::Label* s_pBC0Label = nullptr; + +static void recSetBranchBC0(bool branchOnTrue) +{ + _setupBranchTestBC0(); + s_pBC0Label = new a64::Label(); + // Emit the "skip the taken-branch" jump: the negation of the branch cond. + if (branchOnTrue) + armAsm->B(s_pBC0Label, a64::ne); // BC0T: skip (fall through) when condition false + else + armAsm->B(s_pBC0Label, a64::eq); // BC0F: skip when condition true +} + +static void recBindBC0Label() +{ + armAsm->Bind(s_pBC0Label); + delete s_pBC0Label; + s_pBC0Label = nullptr; +} + +void recBC0F() +{ + const u32 branchTo = ((s32)_Imm_ * 4) + pc; + const bool swap = TrySwapDelaySlot(0, 0, 0, false); + recSetBranchBC0(false); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBC0Label(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +void recBC0T() +{ + const u32 branchTo = ((s32)_Imm_ * 4) + pc; + const bool swap = TrySwapDelaySlot(0, 0, 0, false); + recSetBranchBC0(true); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBC0Label(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +void recBC0FL() +{ + const u32 branchTo = ((s32)_Imm_ * 4) + pc; + recSetBranchBC0(false); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBC0Label(); + LoadBranchState(); + SetBranchImm(pc); +} + +void recBC0TL() +{ + const u32 branchTo = ((s32)_Imm_ * 4) + pc; + recSetBranchBC0(true); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBC0Label(); + LoadBranchState(); + SetBranchImm(pc); +} + +REC_FUNC(TLBR); +REC_FUNC(TLBP); +REC_FUNC(TLBWI); +REC_FUNC(TLBWR); + +REC_SYS(ERET); +REC_SYS(EI); + +// DI — inline, non-branching. Unlike EI (which must branch so newly-enabled +// interrupts fire), disabling interrupts needs no block exit, so DI is emitted +// inline and the block stays open. Mirrors x86 iCOP0.cpp. The next instruction +// is recompiled BEFORE applying DI so the interrupt-mask change takes effect one +// instruction late (fixes booting in Jak X, Namco 50th, Spongebob, The +// Incredibles, etc.). +void recDI() +{ + if (!g_recompilingDelaySlot) + recompileNextInstruction(false, false); // DI delayed by one instruction + + // Clear Status.EIE (bit 16) unless in user mode with no exception level: + // clear iff (EXL|ERL|EDI) set OR KSU == 0 (kernel). Matches Interp::DI + // (COP0.cpp:708-717) and the x86 TEST 0x20006 / TEST 0x18 guard. + a64::Label doClear; + a64::Label done; + armLoadEERegPtr(RWSCRATCH, &cpuRegs.CP0.r[12]); // Status + armAsm->Tst(RWSCRATCH, 0x20006); // EXL | ERL | EDI + armAsm->B(&doClear, a64::ne); + armAsm->Tst(RWSCRATCH, 0x18); // KSU (non-zero => user mode) + armAsm->B(&done, a64::ne); + armAsm->Bind(&doClear); + armAsm->And(RWSCRATCH, RWSCRATCH, ~static_cast(0x10000)); // clear EIE + armStoreEERegPtr(RWSCRATCH, &cpuRegs.CP0.r[12]); + armAsm->Bind(&done); +} + +#ifdef FORCE_INTERP_COP0 +REC_FUNC(MFC0); +REC_FUNC(MTC0); +#else + +// Apply pending block cycles to RECCYCLE and flush to cpuRegs.cycle so the +// interpreter helper (which reads cpuRegs.cycle directly) sees the right +// value. The helper is called inline mid-block (not via DispatcherEvent), +// so the caller must follow up with emitReloadCycle() once it returns to +// keep RECCYCLE in sync with anything the helper wrote. +static void emitFlushBlockCycles() +{ + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); +} + +static void emitReloadCycle() +{ + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); +} + +// MFC0: rt = sign_extend(COP0[rd]) +void recMFC0() +{ + // Count (rd=9) must still tick even when writing to $zero — interpreter + // MFC0 increments CP0.n.Count and updates lastCOP0Cycle before checking + // _Rt_. Match that gate exactly. + if (!_Rt_ && _Rd_ != 9) + return; + + switch (_Rd_) + { + case 9: // Count — inline cycle update; no iFlushCall/interp call + { // (matches interp COP0.cpp:564-575). + // Bring cpuRegs.cycle / RECCYCLE current first (Count reads it). + emitFlushBlockCycles(); // RECCYCLE (x25) == updated cycle, also in memory + // incr = cycle - lastCOP0Cycle; if (incr == 0) incr++; (interp :566-568) + armAsm->Ldr(RXSCRATCH, armCpuRegMem(&cpuRegs.lastCOP0Cycle)); + armAsm->Sub(RXSCRATCH, RECCYCLE, RXSCRATCH); + armAsm->Cmp(RXSCRATCH, 0); + armAsm->Csinc(RXSCRATCH, RXSCRATCH, a64::xzr, a64::ne); // 0 -> 1 + // CP0.n.Count += incr (32-bit register; low 32 of incr) + armAsm->Ldr(RWARG1, armCpuRegMem(&cpuRegs.CP0.r[9])); + armAsm->Add(RWARG1, RWARG1, RWSCRATCH); + armAsm->Str(RWARG1, armCpuRegMem(&cpuRegs.CP0.r[9])); + // lastCOP0Cycle = cycle (interp :569) + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.lastCOP0Cycle)); + if (!_Rt_) + return; + // rt = sign_extend(CP0.r[9]) (interp :571-577) + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.CP0.r[9]); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); + return; + } + + case 25: // Performance counters — cycle-dependent + iFlushCall(FLUSH_INTERPRETER); + emitFlushBlockCycles(); + armEmitCall((void*)Interp::MFC0); + emitReloadCycle(); + return; + + case 24: // Debug breakpoint register — ignore + return; + + case 12: // Status — mask reserved bits, matching interp MFC0 case 12 (COP0.cpp) + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.CP0.r[_Rd_]); + // 0xf0c79c1f is not a valid AArch64 logical immediate, so materialize it + // in a scratch register. Use RWARG1 (caller-saved, dead here), not the + // reserved address scratch x17 (RSCRATCHADDR), which armLoad*Ptr clobbers. + armAsm->Mov(RWARG1, 0xf0c79c1fu); + armAsm->And(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); + return; + + default: + // Simple case: rt = sign_extend(cpuRegs.CP0.r[rd]) + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.CP0.r[_Rd_]); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); + return; + } +} + +// MTC0: COP0[rd] = rt +void recMTC0() +{ + switch (_Rd_) + { + case 9: // Count — inline; no iFlushCall/interp call + { // (matches interp COP0.cpp:583-585): lastCOP0Cycle = cycle; + // CP0.r[9] = rt[31:0]. Bring cycle current first. + emitFlushBlockCycles(); // RECCYCLE (x25) == updated cycle, also in memory + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.lastCOP0Cycle)); + if (GPR_IS_CONST1(_Rt_)) + { + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.CP0.r[9])); + } + else + { + _deleteEEreg(_Rt_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rt_].UL[0]); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.CP0.r[9])); + } + return; + } + + case 12: // Status — has side effects (interrupt check) + case 16: // Config — has side effects + case 25: // Performance counters + iFlushCall(FLUSH_INTERPRETER); + emitFlushBlockCycles(); + armEmitCall((void*)Interp::MTC0); + emitReloadCycle(); + return; + + case 24: // Debug breakpoint register — log-only in interp (COP0.cpp:599-601) + return; + + default: + // Simple case: cpuRegs.CP0.r[rd] = rt[31:0] + if (GPR_IS_CONST1(_Rt_)) + { + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.CP0.r[_Rd_])); + } + else + { + _deleteEEreg(_Rt_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rt_].UL[0]); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.CP0.r[_Rd_])); + } + return; + } +} + +#endif // !FORCE_INTERP_COP0 + +} // namespace COP0 +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iCOP2-arm64.cpp b/pcsx2/arm64/iCOP2-arm64.cpp new file mode 100644 index 0000000000..422d6abe09 --- /dev/null +++ b/pcsx2/arm64/iCOP2-arm64.cpp @@ -0,0 +1,1952 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 native COP2 (VU0 macro mode) codegen using NEON. +// Memory-based: loads VF regs from VU0.VF[], computes with NEON, stores back. +// MAC/status flags are updated via C helper calls for correctness. +// No VU register allocator — each instruction is self-contained. + +#include "arm64/iR5900-arm64.h" + +namespace a64 = vixl::aarch64; + +// ======================================================================== +// COP2 instruction field decoding (VU encoding within EE instruction) +// ======================================================================== +// VU fields reuse EE instruction bit positions: +// _Ft_ = bits 20-16 (same as _Rt_) +// _Fs_ = bits 15-11 (same as _Rd_) +// _Fd_ = bits 10-6 (same as _Sa_) +// dest = bits 24-21 (XYZW write mask) + +#define _Ft_cop2 _Rt_ +#define _Fs_cop2 _Rd_ +#define _Fd_cop2 _Sa_ + +#define _X_cop2 ((cpuRegs.code >> 24) & 0x1) +#define _Y_cop2 ((cpuRegs.code >> 23) & 0x1) +#define _Z_cop2 ((cpuRegs.code >> 22) & 0x1) +#define _W_cop2 ((cpuRegs.code >> 21) & 0x1) +#define _XYZW_cop2 ((cpuRegs.code >> 21) & 0xF) + +// Broadcast field for bc variants (bits 1-0 of function code) +#define _bc_cop2 (cpuRegs.code & 0x3) + +// Fsf/Ftf fields for scalar source selection +#define _Fsf_cop2 ((cpuRegs.code >> 21) & 0x3) +#define _Ftf_cop2 ((cpuRegs.code >> 23) & 0x3) + +// ======================================================================== +// NEON scratch register assignments for COP2 +// ======================================================================== +// q30 (RQSCRATCH) = fs operand / result +// q31 (RQSCRATCH2) = ft operand +// q29 (RQSCRATCH3) = dest mask / ACC / temp + +// ======================================================================== +// Dest field mask table — 16 entries for each XYZW combination +// ======================================================================== +// Each entry is a 128-bit mask: lane = 0xFFFFFFFF if written, 0 if not. +// XYZW is 4 bits: X=bit3, Y=bit2, Z=bit1, W=bit0 +// Lane order in NEON: [0]=x, [1]=y, [2]=z, [3]=w +alignas(16) static const u32 s_cop2DestMasks[16][4] = { + {0x00000000, 0x00000000, 0x00000000, 0x00000000}, // 0000 + {0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF}, // 000W + {0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000}, // 00Z0 + {0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF}, // 00ZW + {0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000}, // 0Y00 + {0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF}, // 0Y0W + {0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000}, // 0YZ0 + {0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, // 0YZW + {0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000}, // X000 + {0xFFFFFFFF, 0x00000000, 0x00000000, 0xFFFFFFFF}, // X00W + {0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000}, // X0Z0 + {0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF}, // X0ZW + {0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000}, // XY00 + {0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF}, // XY0W + {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000}, // XYZ0 + {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, // XYZW +}; + +// ======================================================================== +// VF register load/store helpers +// ======================================================================== + +// Load VU0.VF[reg] into NEON Q register +static void cop2LoadVF(const a64::VRegister& qreg, int vfReg) +{ + armAsm->Ldr(qreg, armVU0Mem(&VU0.VF[vfReg])); +} + +// Store NEON Q register to VU0.VF[reg] +static void cop2StoreVF(const a64::VRegister& qreg, int vfReg) +{ + armAsm->Str(qreg, armVU0Mem(&VU0.VF[vfReg])); +} + +// Load VU0.ACC into NEON Q register +static void cop2LoadACC(const a64::VRegister& qreg) +{ + armAsm->Ldr(qreg, armVU0Mem(&VU0.ACC)); +} + +// Store NEON Q register to VU0.ACC +static void cop2StoreACC(const a64::VRegister& qreg) +{ + armAsm->Str(qreg, armVU0Mem(&VU0.ACC)); +} + +// ======================================================================== +// Dest field masking +// ======================================================================== + +// Apply dest mask: merge 'result' in RQSCRATCH into VU0.VF[fdReg], writing +// only the lanes selected by `xyzw`. The variants without an explicit `xyzw` +// read it from the instruction (_XYZW_cop2); VOPMSUB / VOPMULA force xyzw=0xE +// since PS2 hardware always writes XYZ regardless of the encoded dest field. +static void cop2ApplyDestMaskExplicit(int fdReg, int xyzw) +{ + if (xyzw == 0xF) + { + if (fdReg != 0) + cop2StoreVF(RQSCRATCH, fdReg); + return; + } + + if (xyzw == 0) + return; + + if (fdReg == 0) + return; + + cop2LoadVF(RQSCRATCH3, fdReg); + + armMoveAddressToReg(RSCRATCHADDR, &s_cop2DestMasks[xyzw]); + armAsm->Ldr(RQSCRATCH2, a64::MemOperand(RSCRATCHADDR)); + + armAsm->Bsl(RQSCRATCH2.V16B(), RQSCRATCH.V16B(), RQSCRATCH3.V16B()); + cop2StoreVF(RQSCRATCH2, fdReg); +} + +static void cop2ApplyDestMask(int fdReg) +{ + cop2ApplyDestMaskExplicit(fdReg, _XYZW_cop2); +} + +static void cop2ApplyDestMaskACCExplicit(const a64::VRegister& result, int xyzw) +{ + if (xyzw == 0xF) + { + cop2StoreACC(result); + return; + } + + if (xyzw == 0) + return; + + if (result.GetCode() != RQSCRATCH.GetCode()) + armAsm->Mov(RQSCRATCH.V16B(), result.V16B()); + + cop2LoadACC(RQSCRATCH3); + + armMoveAddressToReg(RSCRATCHADDR, &s_cop2DestMasks[xyzw]); + armAsm->Ldr(RQSCRATCH2, a64::MemOperand(RSCRATCHADDR)); + + armAsm->Bsl(RQSCRATCH2.V16B(), RQSCRATCH.V16B(), RQSCRATCH3.V16B()); + cop2StoreACC(RQSCRATCH2); +} + +static void cop2ApplyDestMaskACC(const a64::VRegister& result) +{ + cop2ApplyDestMaskACCExplicit(result, _XYZW_cop2); +} + +// NOTE: MAC/status flag updates are deferred — VU0.macflag/statusflag are not +// updated here. Most games don't read COP2 flags. When flag support is needed, +// emit a C call to update flags per-instruction. The interpreter fallback ops +// (DIV, CLIP, etc.) still update flags correctly. + +// COP2 accesses VU0 memory, not cpuRegs GPRs — no EE register flush needed. + +// ======================================================================== +// PS2 VU float clamping +// ======================================================================== +// PS2 VU has no infinities — overflow clamps to ±FLT_MAX (0x7f7fffff). +// NEON FPCR has FZ=1 (denormals flushed to zero), so only post-op clamping is needed. +// FMINNM/FMAXNM match x86 MINPS/MAXPS semantics: NaN → non-NaN operand. + +alignas(16) static const u32 s_cop2MaxFloat[4] = {0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff}; + +// VCLIP positive per-lane clip-bit weights ([+x@bit0, +y@bit2, +z@bit4]; lane w +// unused). The negative weights ([-x@bit1, -y@bit3, -z@bit5]) are these << 1, so +// only one constant is needed. After Cmgt the positive/negative masks are +// weighted per lane and a horizontal Addv collapses them into the 6-bit field +// (the +/- bits per axis are mutually exclusive and the lane contributions +// occupy disjoint bit ranges, so the add never carries between bits). +alignas(16) static const u32 s_cop2ClipWeightPos[4] = {0x01, 0x04, 0x10, 0x00}; + +// Clamp RQSCRATCH to [-FLT_MAX, +FLT_MAX] (removes infinities and NaNs) +// FMINNM/FMAXNM match x86 MINPS/MAXPS semantics: NaN → non-NaN operand. +static void cop2ClampResult() +{ + armMoveAddressToReg(RSCRATCHADDR, &s_cop2MaxFloat); + armAsm->Ldr(RQSCRATCH2, a64::MemOperand(RSCRATCHADDR)); + armAsm->Fneg(RQSCRATCH3.V4S(), RQSCRATCH2.V4S()); // -FLT_MAX + + armAsm->Fminnm(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); // clamp to +FLT_MAX + armAsm->Fmaxnm(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH3.V4S()); // clamp to -FLT_MAX +} + +// Clamp an arbitrary operand register `qreg` to [-FLT_MAX, +FLT_MAX], using +// tmpHi/tmpLo to hold the ±FLT_MAX bounds. Input-operand variant of +// cop2ClampResult — used to pre-clamp an FMAC operand before the arithmetic +// (matching x86 mVU's cFs/cFt operand clamps) rather than clamping the result. +static void cop2ClampReg(const a64::VRegister& qreg, + const a64::VRegister& tmpHi, const a64::VRegister& tmpLo) +{ + armMoveAddressToReg(RSCRATCHADDR, &s_cop2MaxFloat); + armAsm->Ldr(tmpHi, a64::MemOperand(RSCRATCHADDR)); + armAsm->Fneg(tmpLo.V4S(), tmpHi.V4S()); + armAsm->Fminnm(qreg.V4S(), qreg.V4S(), tmpHi.V4S()); + armAsm->Fmaxnm(qreg.V4S(), qreg.V4S(), tmpLo.V4S()); +} + +// Single-temp variant of cop2ClampReg: clamps `qreg` to [-FLT_MAX, +FLT_MAX] +// using just one scratch register (it negates the +FLT_MAX bound in place +// between the two clamps). Needed when pre-clamping a broadcast FMAC operand, +// where Fs/Ft already occupy two of the three q-scratch regs and only one is +// free. +static void cop2ClampRegOneTmp(const a64::VRegister& qreg, const a64::VRegister& tmp) +{ + armMoveAddressToReg(RSCRATCHADDR, &s_cop2MaxFloat); + armAsm->Ldr(tmp, a64::MemOperand(RSCRATCHADDR)); + armAsm->Fminnm(qreg.V4S(), qreg.V4S(), tmp.V4S()); // clamp to +FLT_MAX + armAsm->Fneg(tmp.V4S(), tmp.V4S()); // -FLT_MAX + armAsm->Fmaxnm(qreg.V4S(), qreg.V4S(), tmp.V4S()); // clamp to -FLT_MAX +} + +// Pre-clamp the broadcast-MUL operands per mVU_FMACa: (_XYZW_PS)?(cFs|cFt):cFs. +// cFs (clamp Fs) is applied on every mask; cFt (clamp the broadcast Ft) only +// when all four lanes are active. Fs is in RQSCRATCH, the already-broadcast Ft +// in RQSCRATCH2, RQSCRATCH3 is the scratch. This catches operand overflow before +// it propagates through the multiply (TOTA, Disgaea, Ice Age on VU0), instead of +// only clamping the product afterward. +static void cop2EmitMulInputClamp() +{ + cop2ClampRegOneTmp(RQSCRATCH, RQSCRATCH3); // cFs (every mask) + if (_XYZW_cop2 == 0xf) + cop2ClampRegOneTmp(RQSCRATCH2, RQSCRATCH3); // cFt (full mask only) +} + +// ======================================================================== +// PS2 VU integer-comparison MAX/MINI +// ======================================================================== +// PS2 VMAX/VMINI use signed integer comparison on float bit patterns, +// NOT IEEE FMAX/FMIN. This handles NaN and negative values correctly: +// fp_max(a,b) = both_neg ? min_s32(a,b) : max_s32(a,b) +// Implemented as: selection = CMGT(a,b) XOR both_neg_mask, then BSL. +// +// Expects: RQSCRATCH = a (from VF[fs]), RQSCRATCH2 = b (from VF[ft] or broadcast) +// Result: RQSCRATCH = fp_max(a, b) or fp_min(a, b) +// Clobbers: RQSCRATCH, RQSCRATCH2 preserved, RQSCRATCH3 used as scratch. + +static void cop2EmitIntegerMax(int fsReg) +{ + // q30=a, q31=b, q29=scratch + armAsm->And(RQSCRATCH3.V16B(), RQSCRATCH.V16B(), RQSCRATCH2.V16B()); // both_neg test + armAsm->Sshr(RQSCRATCH3.V4S(), RQSCRATCH3.V4S(), 31); // broadcast sign → mask + armAsm->Cmgt(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); // a > b (signed int) + armAsm->Eor(RQSCRATCH.V16B(), RQSCRATCH.V16B(), RQSCRATCH3.V16B()); // selection = CMGT XOR both_neg + cop2LoadVF(RQSCRATCH3, fsReg); // reload a into q29 + armAsm->Bsl(RQSCRATCH.V16B(), RQSCRATCH3.V16B(), RQSCRATCH2.V16B()); // sel ? a : b +} + +static void cop2EmitIntegerMin(int fsReg) +{ + // Same as max but BSL operands swapped: sel ? b : a + armAsm->And(RQSCRATCH3.V16B(), RQSCRATCH.V16B(), RQSCRATCH2.V16B()); + armAsm->Sshr(RQSCRATCH3.V4S(), RQSCRATCH3.V4S(), 31); + armAsm->Cmgt(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + armAsm->Eor(RQSCRATCH.V16B(), RQSCRATCH.V16B(), RQSCRATCH3.V16B()); + cop2LoadVF(RQSCRATCH3, fsReg); + armAsm->Bsl(RQSCRATCH.V16B(), RQSCRATCH2.V16B(), RQSCRATCH3.V16B()); // sel ? b : a +} + +// ======================================================================== +// MAC/Status flag update infrastructure +// ======================================================================== +// Implements mVUupdateFlags + mVUallocSFLAGc/d semantics. +// The status flag is stored in a "denormalized" format during macro mode: +// Bits 0-3: Zero sticky per lane (ZS) +// Bits 4-7: Sign sticky per lane (SS) +// Bits 8-11: Zero current per lane (Z) +// Bits 12-15: Sign current per lane (S) +// Bits 16+: D/I/O/U flags (from divide ops) +// +// The "normalized" format in VU0.VI[REG_STATUS_FLAG] has: +// Bit 0: Z (any current zero), Bit 1: S (any current sign) +// Bit 6: ZS (any sticky zero), Bit 7: SS (any sticky sign) +// Bits 2-5,8+: D/I/O/U flags + +// Runtime storage for denormalized status flag during macro op. Plain static +// (not thread_local): COP2/VU0 macro mode runs only on the EE thread (VU0 is +// lockstep with the EE; MTVU offloads VU1 only), and the JIT bakes this address +// in at emit time — a fixed global address is correct and avoids materializing a +// thread-local slot that only ever has one instance. +static u32 s_cop2DenormStatusFlag; + +// Emit code to denormalize status flag from VU0.VI[REG_STATUS_FLAG] +// into s_cop2DenormStatusFlag (mVUallocSFLAGd). +// Denormalized = ((norm >> 3) & 0x18) | ((norm << 11) & 0x1800) | ((norm << 14) & 0x3cf0000) +static void cop2EmitDenormalizeStatusFlag() +{ + // Load normalized status flag + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); + + // tmp2 = norm + const a64::Register tmp1 = a64::w1; + const a64::Register tmp2 = a64::w2; + armAsm->Mov(tmp2, RWSCRATCH); + + // reg = (norm >> 3) & 0x18 + armAsm->Lsr(RWSCRATCH, tmp2, 3); + armAsm->And(RWSCRATCH, RWSCRATCH, 0x18); + + // tmp1 = (norm << 11) & 0x1800 + armAsm->Lsl(tmp1, tmp2, 11); + armAsm->And(tmp1, tmp1, 0x1800); + armAsm->Orr(RWSCRATCH, RWSCRATCH, tmp1); + + // tmp2 = (norm << 14) & 0x3cf0000 + armAsm->Lsl(tmp2, tmp2, 14); + armAsm->Mov(a64::w3, 0x3cf0000); + armAsm->And(tmp2, tmp2, a64::w3); + armAsm->Orr(RWSCRATCH, RWSCRATCH, tmp2); + + // Store denormalized flag + armMoveAddressToReg(RSCRATCHADDR, &s_cop2DenormStatusFlag); + armAsm->Str(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); +} + +// Emit code to normalize status flag from s_cop2DenormStatusFlag +// back to VU0.VI[REG_STATUS_FLAG] (mVUallocSFLAGc). +static void cop2EmitNormalizeStatusFlag() +{ + // Load denormalized flag + armMoveAddressToReg(RSCRATCHADDR, &s_cop2DenormStatusFlag); + armAsm->Ldr(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + const a64::Register result = a64::w1; + armAsm->Mov(result, a64::wzr); // result = 0 + + // Z bit (norm bit 0): set if any of denorm bits 8-11 + armAsm->Tst(RWSCRATCH, 0x0f00); + armAsm->Cset(a64::w2, a64::ne); + armAsm->Orr(result, result, a64::w2); // bit 0 + + // S bit (norm bit 1): set if any of denorm bits 12-15 + armAsm->Tst(RWSCRATCH, 0xf000); + armAsm->Cset(a64::w2, a64::ne); + armAsm->Orr(result, result, a64::Operand(a64::w2, a64::LSL, 1)); // bit 1 + + // 'result' now holds the current Z (bit0) / S (bit1). Sticky bits are not + // derived from a separate denorm range — they accumulate from the current + // Z/S below (preserve old sticky, OR current into both current and sticky + // positions), matching the interpreter's statusflag-shift behavior. + // + // VI[STATUS] = (VI[STATUS] & 0xFC0) | (Z/S) | ((Z/S) << 6) + // + // This MUST mirror the interpreter's COP2 macro oracle SYNCMSFLAGS() + // (VUops.cpp): preserve 0xFC0 — NOT 0xFF0; the 0xFF0 mask belongs to the + // FMAC-pipeline-flush path, not the COP2 macro path — then write the low + // nibble into bits 0-3 and shifted into the sticky bits 6-9. + // + // LIMITATION: cop2EmitFlagUpdate() computes only Z (Fcmeq) and S (Cmlt) here; + // it never sets the U (underflow, exp==0) or O (overflow, exp==255) bits that + // interp's VU_STAT_UPDATE (VUflags.cpp) can produce. So 'result' only ever + // holds bits 0-1, and masking it with 0x3 is exact. DO NOT widen the 0xFC0 + // preserve to 0xFF0 (or the 0x3 result-mask to 0xF) without first teaching + // cop2EmitFlagUpdate to compute U/O — a bare widen keeps stale bits the + // interpreter clears and regresses EeVu0Cop2Macro.VaddXyzwSumsLanes. No game + // in the corpus reads U/O after a COP2 macro FMAC, so this gap is latent. + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); + armAsm->And(RWSCRATCH, RWSCRATCH, 0xFC0); // preserve existing sticky (bits 6-11) + armAsm->And(result, result, 0x3); // keep only Z/S (bits 0-1) + armAsm->Orr(RWSCRATCH, RWSCRATCH, result); // OR in current Z/S + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::Operand(result, a64::LSL, 6)); // OR shifted into sticky + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); +} + +// Emit code to update MAC and status flags from the result in RQSCRATCH. +// Implements mVUupdateFlags behavior. +// xyzw = dest field mask (which lanes were written). +// Uses RQSCRATCH2, RQSCRATCH3 as temporaries. +static void cop2EmitFlagUpdate(int xyzw) +{ + // Flags are updated unconditionally for correctness; no liveness-based + // skip is applied here. + + if (xyzw == 0) return; + + // Save result — flag extraction clobbers NEON scratch registers + // Use q28 to preserve the result while extracting flags from it + a64::VRegister savedResult = a64::VRegister(28, 128); + armAsm->Mov(savedResult.V16B(), RQSCRATCH.V16B()); + + // --- Extract sign bits from result --- + // CMLT produces all-1s per lane if negative. armEmitPackLaneBits expects + // all-1s/0 lanes (no Ushr needed) — AND-with-weights gives back the weight + // when the lane is set and 0 otherwise. + armAsm->Cmlt(RQSCRATCH2.V4S(), savedResult.V4S(), 0); + + // --- Extract zero bits --- + // FCMEQ produces all-1s per lane if == 0.0 + armAsm->Fcmeq(RQSCRATCH3.V4S(), savedResult.V4S(), 0); + + // --- Pack 4 lane bits into GPR in PS2 MAC flag order --- + // PS2 MAC flag: bit0=W, bit1=Z, bit2=Y, bit3=X (reverse of NEON lane order + // [0]=x, [1]=y, [2]=z, [3]=w). reverse=true picks weight vector {8,4,2,1}. + // RQSCRATCH (q30) is free here — savedResult lives in q28. + const a64::Register signBits = a64::w1; + const a64::Register zeroBits = a64::w2; + armEmitPackLaneBits(signBits, RQSCRATCH2, RQSCRATCH, /*reverse=*/true); + armEmitPackLaneBits(zeroBits, RQSCRATCH3, RQSCRATCH, /*reverse=*/true); + + // --- Apply XYZW dest mask --- + // _XYZW_cop2 = X(bit3) Y(bit2) Z(bit1) W(bit0) — matches PS2 MAC order + // Lanes not in dest mask should have their flag bits cleared. + armAsm->And(signBits, signBits, xyzw); + armAsm->And(zeroBits, zeroBits, xyzw); + + // --- Build MAC flag: (sign << 4) | zero --- + const a64::Register macFlag = a64::w3; + armAsm->Lsl(macFlag, signBits, 4); + armAsm->Orr(macFlag, macFlag, zeroBits); + + // --- Write MAC flag to VU0.VI[REG_MAC_FLAG] --- + armAsm->Str(macFlag, armVU0Mem(&VU0.VI[REG_MAC_FLAG])); + + // --- Update denormalized status flag --- + // Load current denorm flag + armMoveAddressToReg(RSCRATCHADDR, &s_cop2DenormStatusFlag); + armAsm->Ldr(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + // Clear current (non-sticky) bits 8-15 + armAsm->Mov(a64::w4, 0xFF00); + armAsm->Bic(RWSCRATCH, RWSCRATCH, a64::w4); + + // OR macFlag into sticky bits (0-7) — accumulates over time + armAsm->Orr(RWSCRATCH, RWSCRATCH, macFlag); + + // OR (macFlag << 8) into current bits (8-15) — this instruction's result + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::Operand(macFlag, a64::LSL, 8)); + + // Store back. RSCRATCHADDR still holds &s_cop2DenormStatusFlag from the load + // above (none of the Mov/Bic/Orr between touch it), so no reload is needed. + armAsm->Str(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + // Restore result to RQSCRATCH for subsequent cop2ApplyDestMask + armAsm->Mov(RQSCRATCH.V16B(), savedResult.V16B()); +} + +// ======================================================================== +// COP2 Macro Mode Setup/Teardown +// ======================================================================== +// ARM64 setupMacroOp/endMacroOp (see microVU_Macro.inl for the x86 version). +// Mode flags: 0x01=read Q, 0x02=write Q, 0x10=update status/MAC flags. + +// cop2EmitConditionalSync is declared in iR5900-arm64.h (callable from +// recVTLB-arm64.cpp for LQC2/SQC2); definition is later in this file. + +void setupMacroOp_arm64(int mode) +{ + // VU0 sync is gated on EEINST analysis (EEINST_COP2_SYNC_VU0 / FINISH_VU0). + // In the common case where the analysis says no sync is needed, this emits + // zero instructions (per-op recXXX gates sync via COP2_Interlock / + // mVUSyncVU0 / mVUFinishVU0). + cop2EmitConditionalSync(false, _vu0FinishMicro); + + if (mode & 0x10) // Status/MAC flags will be updated + { + // Always denormalize the status flag; no liveness-based skip is applied. + cop2EmitDenormalizeStatusFlag(); + } + + if (mode & 0x01) // Q register will be read — load into RQSCRATCH3 + { + // Q is loaded per-instruction by the Q-variant ops (ADDq etc.) + // No global load needed here — the Q-variant ops load Q inline. + } + + // microVU0 state setup so mVU-reuse wrappers (REC_COP2_mVU0_ARM64) can + // drive mVU_LQI/SQI/MFIR/MTIR/... directly from macro-mode dispatch. + // Hand-rolled arithmetic ops (recCOP2_VADDx etc.) don't read this state, + // so unconditional setup is a cheap no-op cost for them. + mVUmacroSetupCOP2State(mode, g_pCurInstInfo ? g_pCurInstInfo->info : 0u); +} + +void endMacroOp_arm64(int mode) +{ + if (mode & 0x02) // Q register was written + { + // DIV/SQRT/RSQRT write Q inline — no global store needed here. + } + + if (mode & 0x10) // Status/MAC flags were updated + { + // Always normalize status flag back to VU0.VI[REG_STATUS_FLAG]. + // Each COP2 macro instruction is self-contained, so the normalized + // flag must be written every time. The vuFlagHack optimization + // (skipping normalization when no one reads the flag) requires + // correct denormalized flag persistence across instructions, + // which is not yet supported. + cop2EmitNormalizeStatusFlag(); + } + + // microVU0 state teardown — flushPartialForCOP2 + cop2=0 + regAlloc reset. + mVUmacroEndCOP2State(); +} + +// Macro for COP2 arithmetic ops that go through the setup/teardown pipeline. +// opFunc emits the actual NEON arithmetic + flag update. +#define REC_COP2_ARM64(f, mode) \ + void recCOP2_V##f() \ + { \ + setupMacroOp_arm64(mode); \ + cop2Op_##f(); \ + endMacroOp_arm64(mode); \ + } + +// ======================================================================== +// COP2 Transfer ops: QMFC2, QMTC2, CFC2, CTC2 +// ======================================================================== +// These move data between EE GPRs and VU0 registers. +// VU0 sync is conditional on VU0 actually running (VPU_STAT bit 0). +// Sync is skipped in the common case where VU0 micro isn't executing. + +extern void vu0Sync(); +extern void _vu0FinishMicro(); +extern void _vu0WaitMicro(); + +// Emit conditional VU0 sync: uses EEINST analysis flags when available, +// falls back to runtime VPU_STAT check otherwise. +// Implements the COP2_Interlock + mVUSyncVU0/mVUFinishVU0 sync protocol. +void cop2EmitConditionalSync(bool interlock, void (*finishFunc)()) +{ + // Handle interlock (bit 0 set): COP2_Interlock pattern + if (interlock) + { + // Interlock requires sync — check if analysis says VU0 could be running + if (g_pCurInstInfo->info & EEINST_COP2_SYNC_VU0) + { + // Lighter flush than FLUSH_EVERYTHING: FLUSH_FREE_XMM | FLUSH_FREE_VU0 + // skips callee-saved EE-GPR writebacks (callee-saved survives the C + // call) while still evicting caller-saved GPR + all NEON. + iFlushCall(FLUSH_FREE_XMM | FLUSH_FREE_VU0); + + // Apply block cycles to RECCYCLE (the pinned cpuRegs.cycle). + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + // Runtime: skip if VU0 not running + a64::Label skipSync; + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[REG_VPU_STAT])); + armAsm->Tbz(RWSCRATCH, 0, &skipSync); + + // Flush RECCYCLE before vu0Sync — it reads cpuRegs.cycle to + // determine how many VU0 micro cycles to run. Reload after, + // since vu0Sync may advance cpuRegs.cycle. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armEmitCall((void*)vu0Sync); + if (finishFunc) + armEmitCall((void*)finishFunc); + + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armAsm->Bind(&skipSync); + } + // else: analysis says no VU0 program between COP2 ops, safe to skip + return; + } + + // Non-interlock: check analysis flags for sync/finish + const bool needsSync = (g_pCurInstInfo->info & EEINST_COP2_SYNC_VU0) != 0; + const bool needsFinish = (g_pCurInstInfo->info & EEINST_COP2_FINISH_VU0) != 0; + + if (!needsSync && !needsFinish) + return; // Analysis says no sync needed + + // Lighter flush — see interlock branch above for rationale. + iFlushCall(FLUSH_FREE_XMM | FLUSH_FREE_VU0); + + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + // Runtime: skip if VU0 not running + a64::Label skipSync; + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[REG_VPU_STAT])); + armAsm->Tbz(RWSCRATCH, 0, &skipSync); + + // Flush + reload around the C call (see comment above). + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + if (needsSync) + armEmitCall((void*)vu0Sync); + else + armEmitCall((void*)_vu0FinishMicro); + + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armAsm->Bind(&skipSync); +} + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +// QMFC2: cpuRegs.GPR[rt] = VU0.VF[fs] (128-bit copy, VF → EE GPR) +void recCOP2_QMFC2() +{ + iFlushCall(FLUSH_EVERYTHING); + cop2EmitConditionalSync(cpuRegs.code & 1, _vu0FinishMicro); + + if (_Rt_ == 0) return; + GPR_DEL_CONST(_Rt_); + + // 128-bit copy: VU0.VF[fs] → cpuRegs.GPR.r[rt] + armAsm->Ldr(RQSCRATCH, armVU0Mem(&VU0.VF[_Rd_])); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rt_])); +} + +// QMTC2: VU0.VF[fs] = cpuRegs.GPR[rt] (128-bit copy, EE GPR → VF) +void recCOP2_QMTC2() +{ + iFlushCall(FLUSH_EVERYTHING); + cop2EmitConditionalSync(cpuRegs.code & 1, _vu0WaitMicro); + + if (_Rd_ == 0) return; // VF[0] is read-only + + // 128-bit copy: cpuRegs.GPR.r[rt] → VU0.VF[fs] + if (GPR_IS_CONST1(_Rt_)) + { + armMoveAddressToReg(RSCRATCHADDR, &g_cpuConstRegs[_Rt_]); + armAsm->Ldr(RQSCRATCH, a64::MemOperand(RSCRATCHADDR)); + } + else + { + armAsm->Ldr(RQSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rt_])); + } + armAsm->Str(RQSCRATCH, armVU0Mem(&VU0.VF[_Rd_])); +} + +// CFC2: cpuRegs.GPR[rt] = sign_extend_32_to_64(VU0.VI[fs]) +void recCOP2_CFC2() +{ + iFlushCall(FLUSH_EVERYTHING); + cop2EmitConditionalSync(cpuRegs.code & 1, _vu0FinishMicro); + + if (_Rt_ == 0) return; + GPR_DEL_CONST(_Rt_); + + if (_Rd_ == REG_R) + { + // REG_R: mask to 23 bits, write only UL[0] + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[REG_R])); + armAsm->And(RWSCRATCH, RWSCRATCH, 0x7FFFFF); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rt_].UL[0])); + } + else + { + // General VI: load 32-bit, sign-extend to UL[0]+UL[1] + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[_Rd_])); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rt_].UL[0])); + // Sign-extend: UL[1] = (UL[0] & 0x80000000) ? 0xFFFFFFFF : 0 + armAsm->Asr(RWSCRATCH, RWSCRATCH, 31); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rt_].UL[1])); + } +} + +// CTC2: cpuRegs.GPR[rt] → VU0.VI[fs] (with special-case registers) +// _Fs_ is known at compile time, so dispatch happens at compile time. +// FBRST and CMSAR1 fall back to interpreter (complex side effects). +// CTC2() is in global namespace (VU0.cpp), referenced via ::CTC2. + +void recCOP2_CTC2() +{ + const int fs = _Rd_; // _Fs_ in VU encoding = _Rd_ in EE encoding + + if (fs == 0) return; // VI[0] is read-only + + // Read-only registers — no-op + if (fs == REG_MAC_FLAG || fs == REG_TPC || fs == REG_VPU_STAT) + return; + + // FBRST and CMSAR1 have complex side effects — use interpreter + if (fs == REG_FBRST || fs == REG_CMSAR1) + { + recCall(::CTC2); + return; + } + + // For all other cases: flush + conditional sync, then inline write + iFlushCall(FLUSH_EVERYTHING); + cop2EmitConditionalSync(cpuRegs.code & 1, _vu0WaitMicro); + + // Load source value from cpuRegs.GPR[rt].UL[0] + if (GPR_IS_CONST1(_Rt_)) + { + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + } + else + { + armAsm->Ldr(RWSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rt_].UL[0])); + } + + if (fs == REG_R) + { + // REG_R: (value & 0x7FFFFF) | 0x3F800000 + armAsm->And(RWSCRATCH, RWSCRATCH, 0x7FFFFF); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x3F800000); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.VI[REG_R])); + } + else if (fs == REG_CLIP_FLAG) + { + // REG_CLIP_FLAG: write to both clipflag and VI + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.clipflag)); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.VI[REG_CLIP_FLAG])); + } + else if (fs == REG_STATUS_FLAG) + { + // STATUS_FLAG: take only the 0xFC0 field from the GPR, preserve the + // low-6 sticky bits in VI[STATUS], then denormalize the result + // (mVUallocSFLAGd) and broadcast it into all four lanes of + // micro_statusflags — microVU reads that array for flag sync, so a raw + // 32-bit overwrite of VI[STATUS] alone leaves it stale and corrupts VU + // flag state. RWSCRATCH = GPR[_Rt_].UL[0] here (== 0 for _Rt_==0, so + // the RMW degrades to STATUS &= 0x3F). + armAsm->And(RWSCRATCH, RWSCRATCH, 0xFC0); // masked field from GPR + + armAsm->Ldr(RWARG2, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); + armAsm->And(RWARG2, RWARG2, 0x3F); // preserve sticky bits 0-5 + armAsm->Orr(RWARG2, RWARG2, RWSCRATCH); // RWARG2 = new normalized STATUS + armAsm->Str(RWARG2, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); + + // Denormalize the new STATUS (in RWARG2) into RWSCRATCH: + // denorm = ((s>>3)&0x18) | ((s<<11)&0x1800) | ((s<<14)&0x3cf0000) + armAsm->Lsr(RWSCRATCH, RWARG2, 3); + armAsm->And(RWSCRATCH, RWSCRATCH, 0x18); + armAsm->Lsl(a64::w2, RWARG2, 11); + armAsm->And(a64::w2, a64::w2, 0x1800); + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::w2); + armAsm->Lsl(a64::w3, RWARG2, 14); + armAsm->Mov(a64::w4, 0x3cf0000); // not a valid logical-imm; materialize + armAsm->And(a64::w3, a64::w3, a64::w4); + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::w3); + + // Broadcast the denormalized value into all 4 lanes of micro_statusflags. + armAsm->Dup(RQSCRATCH.V4S(), RWSCRATCH); + armAsm->Str(RQSCRATCH, armVU0Mem(&VU0.micro_statusflags)); + } + else + { + // Default: write 32-bit value to VI register + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.VI[fs])); + } +} + +// ======================================================================== +// COP2 Integer ops: IADD, ISUB, IADDI, IAND, IOR +// ======================================================================== +// 16-bit VI register operations. VU field encoding: +// _Id_ = _Sa_ & 0xF (destination VI), _Is_ = _Rd_ & 0xF, _It_ = _Rt_ & 0xF + +#define _Id_cop2 (_Sa_ & 0xF) +#define _Is_cop2 (_Rd_ & 0xF) +#define _It_cop2 (_Rt_ & 0xF) + +// IADD: VI[id] = VI[is] + VI[it] +void recCOP2_VIADD() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Id_cop2 == 0) return; + + armAsm->Ldrsh(RWSCRATCH, armVU0Mem(&VU0.VI[_Is_cop2])); + armAsm->Ldrsh(RWARG2, armVU0Mem(&VU0.VI[_It_cop2])); + armAsm->Add(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Strh(RWSCRATCH, armVU0Mem(&VU0.VI[_Id_cop2])); +} + +// ISUB: VI[id] = VI[is] - VI[it] +void recCOP2_VISUB() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Id_cop2 == 0) return; + + armAsm->Ldrsh(RWSCRATCH, armVU0Mem(&VU0.VI[_Is_cop2])); + armAsm->Ldrsh(RWARG2, armVU0Mem(&VU0.VI[_It_cop2])); + armAsm->Sub(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Strh(RWSCRATCH, armVU0Mem(&VU0.VI[_Id_cop2])); +} + +// IADDI: VI[it] = VI[is] + sign_ext_5bit_imm +void recCOP2_VIADDI() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_It_cop2 == 0) return; + + // 5-bit immediate at bits 10-6, sign-extended + s16 imm = ((_Sa_ & 0x1F)); + imm = ((imm & 0x10) ? (s16)(0xFFF0 | imm) : imm); + + armAsm->Ldrsh(RWSCRATCH, armVU0Mem(&VU0.VI[_Is_cop2])); + armAsm->Add(RWSCRATCH, RWSCRATCH, imm); + armAsm->Strh(RWSCRATCH, armVU0Mem(&VU0.VI[_It_cop2])); +} + +// IAND: VI[id] = VI[is] & VI[it] +void recCOP2_VIAND() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Id_cop2 == 0) return; + + armAsm->Ldrh(RWSCRATCH, armVU0Mem(&VU0.VI[_Is_cop2])); + armAsm->Ldrh(RWARG2, armVU0Mem(&VU0.VI[_It_cop2])); + armAsm->And(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Strh(RWSCRATCH, armVU0Mem(&VU0.VI[_Id_cop2])); +} + +// IOR: VI[id] = VI[is] | VI[it] +void recCOP2_VIOR() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Id_cop2 == 0) return; + + armAsm->Ldrh(RWSCRATCH, armVU0Mem(&VU0.VI[_Is_cop2])); + armAsm->Ldrh(RWARG2, armVU0Mem(&VU0.VI[_It_cop2])); + armAsm->Orr(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Strh(RWSCRATCH, armVU0Mem(&VU0.VI[_Id_cop2])); +} + +// ======================================================================== +// SIMPLE template: VMOVE, VMR32, VNOP, VWAITQ, VABS +// ======================================================================== + +// VMOVE: VF[ft] = VF[fs] (masked by dest) +void recCOP2_VMOVE() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; // VF0 is read-only + + const int xyzw = _XYZW_cop2; + if (xyzw == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2ApplyDestMask(_Ft_cop2); +} + +// VMR32: rotate VF[fs] lanes right by one, store to VF[ft] (masked) +// x=y, y=z, z=w, w=x (rotate left in element order) +void recCOP2_VMR32() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + const int xyzw = _XYZW_cop2; + if (xyzw == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + // EXT rotates: target lane order is [y,z,w,x] from [x,y,z,w] + // That's a left rotation by 1 lane = EXT #4 (4 bytes) + armAsm->Ext(RQSCRATCH.V16B(), RQSCRATCH.V16B(), RQSCRATCH.V16B(), 4); + cop2ApplyDestMask(_Ft_cop2); +} + +// VNOP: no operation +void recCOP2_VNOP() +{ +} + +// VWAITQ: wait for Q register (no-op in macro mode) +void recCOP2_VWAITQ() +{ +} + +// VABS: VF[ft] = abs(VF[fs]) (masked) +void recCOP2_VABS() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + const int xyzw = _XYZW_cop2; + if (xyzw == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armAsm->Fabs(RQSCRATCH.V4S(), RQSCRATCH.V4S()); + cop2ApplyDestMask(_Ft_cop2); +} + +// ======================================================================== +// VEC_ARITH template: VADD, VSUB, VMUL +// Pattern: VF[fd] = VF[fs] OP VF[ft] (masked by dest) +// ======================================================================== + +void recCOP2_VADD() +{ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; + setupMacroOp_arm64(0x110); + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + armAsm->Fadd(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + cop2ClampResult(); + cop2EmitFlagUpdate(_XYZW_cop2); + cop2ApplyDestMask(_Fd_cop2); + + endMacroOp_arm64(0x110); +} + +void recCOP2_VSUB() +{ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; + setupMacroOp_arm64(0x110); + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + armAsm->Fsub(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + cop2ClampResult(); + cop2EmitFlagUpdate(_XYZW_cop2); + cop2ApplyDestMask(_Fd_cop2); + + endMacroOp_arm64(0x110); +} + +void recCOP2_VMUL() +{ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; + setupMacroOp_arm64(0x110); + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + cop2ClampResult(); + cop2EmitFlagUpdate(_XYZW_cop2); + cop2ApplyDestMask(_Fd_cop2); + + endMacroOp_arm64(0x110); +} + +void recCOP2_VMAX() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Fd_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + cop2EmitIntegerMax(_Fs_cop2); + cop2ApplyDestMask(_Fd_cop2); +} + +void recCOP2_VMINI() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Fd_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + cop2EmitIntegerMin(_Fs_cop2); + cop2ApplyDestMask(_Fd_cop2); +} + +// ======================================================================== +// Broadcast helpers for _BC variants +// ======================================================================== + +// Load VF[ft] and broadcast lane 'bc' (0=x, 1=y, 2=z, 3=w) to all lanes +static void cop2LoadBroadcast(const a64::VRegister& qreg, int vfReg, int bc) +{ + cop2LoadVF(qreg, vfReg); + armAsm->Dup(qreg.V4S(), qreg.V4S(), bc); +} + +// ======================================================================== +// ADD_BC / SUB_BC / MUL_BC template +// Pattern: VF[fd] = VF[fs] OP VF[ft].bc (broadcast one lane) +// ======================================================================== + +// Helper macro for broadcast binary ops (with input/output clamping + flags). +// mulClamp=true pre-clamps the FMAC operands per mVU_MULx cFs/cFt (MUL family); +// ADD/SUB pass false (ADD clampType=0; SUB's input clamp is a separate concern). +#define COP2_BC_OP(name, neonOp, bc, mulClamp) \ + void recCOP2_V##name() \ + { \ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadBroadcast(RQSCRATCH2, _Ft_cop2, bc); \ + if (mulClamp) cop2EmitMulInputClamp(); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + endMacroOp_arm64(0x110); \ + } + +// ADDx/y/z/w +COP2_BC_OP(ADDx, Fadd, 0, false) +COP2_BC_OP(ADDy, Fadd, 1, false) +COP2_BC_OP(ADDz, Fadd, 2, false) +COP2_BC_OP(ADDw, Fadd, 3, false) + +// SUBx/y/z/w +COP2_BC_OP(SUBx, Fsub, 0, false) +COP2_BC_OP(SUBy, Fsub, 1, false) +COP2_BC_OP(SUBz, Fsub, 2, false) +COP2_BC_OP(SUBw, Fsub, 3, false) + +// MULx/y/z/w — pre-clamp Fs (and Ft on full mask) per mVU_MULx cFs/cFt spec +COP2_BC_OP(MULx, Fmul, 0, true) +COP2_BC_OP(MULy, Fmul, 1, true) +COP2_BC_OP(MULz, Fmul, 2, true) +COP2_BC_OP(MULw, Fmul, 3, true) + +// MAXx/y/z/w — PS2 integer comparison, not IEEE FMAX +#define COP2_BC_MAX(name, bc) \ + void recCOP2_V##name() \ + { \ + cop2EmitConditionalSync(false, _vu0FinishMicro); \ + if (_Fd_cop2 == 0) return; \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadBroadcast(RQSCRATCH2, _Ft_cop2, bc); \ + cop2EmitIntegerMax(_Fs_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + } + +// MINIx/y/z/w — PS2 integer comparison, not IEEE FMIN +#define COP2_BC_MINI(name, bc) \ + void recCOP2_V##name() \ + { \ + cop2EmitConditionalSync(false, _vu0FinishMicro); \ + if (_Fd_cop2 == 0) return; \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadBroadcast(RQSCRATCH2, _Ft_cop2, bc); \ + cop2EmitIntegerMin(_Fs_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + } + +COP2_BC_MAX(MAXx, 0) +COP2_BC_MAX(MAXy, 1) +COP2_BC_MAX(MAXz, 2) +COP2_BC_MAX(MAXw, 3) + +COP2_BC_MINI(MINIx, 0) +COP2_BC_MINI(MINIy, 1) +COP2_BC_MINI(MINIz, 2) +COP2_BC_MINI(MINIw, 3) + +// MAXi/MINIi — broadcast I register, PS2 integer comparison +void recCOP2_VMAXi() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Fd_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_I]); + cop2EmitIntegerMax(_Fs_cop2); + cop2ApplyDestMask(_Fd_cop2); +} + +void recCOP2_VMINIi() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Fd_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_I]); + cop2EmitIntegerMin(_Fs_cop2); + cop2ApplyDestMask(_Fd_cop2); +} + +// ======================================================================== +// ADDq/SUBq/MULq — broadcast Q register +// ======================================================================== + +#define COP2_Q_OP(name, neonOp) \ + void recCOP2_V##name() \ + { \ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; \ + setupMacroOp_arm64(0x111); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_Q]); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + endMacroOp_arm64(0x111); \ + } + +COP2_Q_OP(ADDq, Fadd) +COP2_Q_OP(SUBq, Fsub) +COP2_Q_OP(MULq, Fmul) + +// ADDi/SUBi/MULi — broadcast I register +#define COP2_I_OP(name, neonOp) \ + void recCOP2_V##name() \ + { \ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_I]); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + endMacroOp_arm64(0x110); \ + } + +COP2_I_OP(ADDi, Fadd) +COP2_I_OP(SUBi, Fsub) +COP2_I_OP(MULi, Fmul) + +// ======================================================================== +// MADD/MSUB variants: VF[fd] = ACC ± VF[fs] * VF[ft] +// ======================================================================== + +// MADD/MSUB use separate FMUL+FADD/FSUB (not FMLA/FMLS) to match PS2 VU +// intermediate rounding. PS2 rounds the multiply result before adding to ACC. + +void recCOP2_VMADD() +{ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; + setupMacroOp_arm64(0x110); + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + cop2LoadACC(RQSCRATCH3); + armAsm->Fadd(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + cop2ClampResult(); + cop2EmitFlagUpdate(_XYZW_cop2); + cop2ApplyDestMask(_Fd_cop2); + + endMacroOp_arm64(0x110); +} + +void recCOP2_VMSUB() +{ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; + setupMacroOp_arm64(0x110); + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2LoadVF(RQSCRATCH2, _Ft_cop2); + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + cop2LoadACC(RQSCRATCH3); + armAsm->Fsub(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + cop2ClampResult(); + cop2EmitFlagUpdate(_XYZW_cop2); + cop2ApplyDestMask(_Fd_cop2); + + endMacroOp_arm64(0x110); +} + +// MADD/MSUB broadcast variants: separate FMUL + FADD/FSUB. +// +// MADDx/y/z/w pre-clamp Fs before the multiply (clampFs=true): mVU_MADDx passes +// cFs, and the interpreter routes Fs through vuDouble, so an Inf/NaN Fs against +// a zero broadcast Ft must become FLT_MAX*0 = 0 rather than Inf*0 = NaN folded +// to +/-FLT_MAX by the result clamp. MSUBx/y/z/w use mVU_FMACd (clampType=0, +// no cFs) — that Fs divergence is shared/by-design, so MSUB keeps clampFs=false. +// The MADDw extras (cACC|cFt) are a separate concern. RQSCRATCH2/RQSCRATCH3 are +// free as the ±FLT_MAX bounds here (Ft/ACC are loaded after the clamp). +#define COP2_MADD_BC(name, addOp, bc, clampFs) \ + void recCOP2_V##name() \ + { \ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + if (clampFs) \ + cop2ClampReg(RQSCRATCH, RQSCRATCH2, RQSCRATCH3); \ + cop2LoadBroadcast(RQSCRATCH2, _Ft_cop2, bc); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + endMacroOp_arm64(0x110); \ + } + +COP2_MADD_BC(MADDx, Fadd, 0, true) +COP2_MADD_BC(MADDy, Fadd, 1, true) +COP2_MADD_BC(MADDz, Fadd, 2, true) +COP2_MADD_BC(MADDw, Fadd, 3, true) + +COP2_MADD_BC(MSUBx, Fsub, 0, false) +COP2_MADD_BC(MSUBy, Fsub, 1, false) +COP2_MADD_BC(MSUBz, Fsub, 2, false) +COP2_MADD_BC(MSUBw, Fsub, 3, false) + +// MADDq/MSUBq — broadcast Q +#define COP2_MADD_Q(name, addOp) \ + void recCOP2_V##name() \ + { \ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; \ + setupMacroOp_arm64(0x111); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_Q]); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + endMacroOp_arm64(0x111); \ + } + +COP2_MADD_Q(MADDq, Fadd) +COP2_MADD_Q(MSUBq, Fsub) + +// MADDi/MSUBi — broadcast I +#define COP2_MADD_I(name, addOp) \ + void recCOP2_V##name() \ + { \ + if (_Fd_cop2 == 0 && _XYZW_cop2 == 0) return; \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_I]); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMask(_Fd_cop2); \ + endMacroOp_arm64(0x110); \ + } + +COP2_MADD_I(MADDi, Fadd) +COP2_MADD_I(MSUBi, Fsub) + +// OPMSUB: VF[fd].xyz = ACC.xyz - VF[fs].yzx * VF[ft].zxy (cross product subtract) +// PS2 always writes XYZ only, ignoring the instruction's dest field. +void recCOP2_VOPMSUB() +{ + if (_Fd_cop2 == 0) return; + setupMacroOp_arm64(0x110); + + cop2LoadVF(RQSCRATCH, _Fs_cop2); // fs = [x,y,z,w] + cop2LoadVF(RQSCRATCH2, _Ft_cop2); // ft = [x,y,z,w] + cop2LoadACC(RQSCRATCH3); // ACC + + // Build fs.yzx: EXT #4 gives [y,z,w,x], fix lane 2 (w→x) + a64::VRegister fsRot = a64::VRegister(28, 128); + armAsm->Ext(fsRot.V16B(), RQSCRATCH.V16B(), RQSCRATCH.V16B(), 4); // [y,z,w,x] + armAsm->Ins(fsRot.V4S(), 2, RQSCRATCH.V4S(), 0); // [y,z,x,x] + + // Build ft.zxy: RQSCRATCH2 still holds ft from the load above (the fsRot + // construction and ACC load only touch RQSCRATCH/v28/RQSCRATCH3), so reuse it. + a64::VRegister ftRot = a64::VRegister(27, 128); + armAsm->Ext(ftRot.V16B(), RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), 8); // [z,w,x,y] + armAsm->Ins(ftRot.V4S(), 1, RQSCRATCH2.V4S(), 0); // [z,x,x,y] + armAsm->Ins(ftRot.V4S(), 2, RQSCRATCH2.V4S(), 1); // [z,x,y,y] + + // ACC - fs.yzx * ft.zxy (separate FMUL+FSUB for PS2 rounding) + armAsm->Fmul(RQSCRATCH.V4S(), fsRot.V4S(), ftRot.V4S()); + armAsm->Fsub(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + cop2ClampResult(); + // OPMSUB always updates XYZ flags only (0xE), W MAC flag cleared. + // PS2 hardware ignores the W bit of the instruction's dest field — + // only XYZ are ever written. Force the mask to XYZ regardless of encoding. + cop2EmitFlagUpdate(0xE); + cop2ApplyDestMaskExplicit(_Fd_cop2, _XYZW_cop2 & 0xE); + + endMacroOp_arm64(0x110); +} + +// ======================================================================== +// Accumulator write variants (xxxA): result goes to ACC instead of VF[fd] +// ======================================================================== + +// VADDA/VSUBA/VMULA: ACC = VF[fs] OP VF[ft] +#define COP2_ACCUM_OP(name, neonOp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadVF(RQSCRATCH2, _Ft_cop2); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x110); \ + } + +COP2_ACCUM_OP(ADDA, Fadd) +COP2_ACCUM_OP(SUBA, Fsub) +COP2_ACCUM_OP(MULA, Fmul) + +// Broadcast accumulator variants: ACC = VF[fs] OP VF[ft].bc +// mulClamp=true pre-clamps the FMAC operands (cFs every mask + cFt on the full +// mask) per mVU_MULAx cFs/cFt; ADD/SUB pass false (clampType=0). +#define COP2_ACCUM_BC(name, neonOp, bc, mulClamp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadBroadcast(RQSCRATCH2, _Ft_cop2, bc); \ + if (mulClamp) cop2EmitMulInputClamp(); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x110); \ + } + +// ADDAx/y/z/w +COP2_ACCUM_BC(ADDAx, Fadd, 0, false) +COP2_ACCUM_BC(ADDAy, Fadd, 1, false) +COP2_ACCUM_BC(ADDAz, Fadd, 2, false) +COP2_ACCUM_BC(ADDAw, Fadd, 3, false) + +// SUBAx/y/z/w +COP2_ACCUM_BC(SUBAx, Fsub, 0, false) +COP2_ACCUM_BC(SUBAy, Fsub, 1, false) +COP2_ACCUM_BC(SUBAz, Fsub, 2, false) +COP2_ACCUM_BC(SUBAw, Fsub, 3, false) + +// MULAx/y/z/w — pre-clamp Fs (and Ft on full mask) before the multiply per +// mVU_MULAx: `(_XYZW_PS)?(cFs|cFt):cFs` (TOTA, DoM). cFs catches an Inf/NaN +// Fs against a zero broadcast (Inf*0 = NaN -> result-clamped ±FLT_MAX instead +// of the interpreter's vuDouble(Fs)-clamped 0). MULAw uses the same path to +// ensure the always-on cFs is applied. +COP2_ACCUM_BC(MULAx, Fmul, 0, true) +COP2_ACCUM_BC(MULAy, Fmul, 1, true) +COP2_ACCUM_BC(MULAz, Fmul, 2, true) +COP2_ACCUM_BC(MULAw, Fmul, 3, true) + +// ACCUMq variants +#define COP2_ACCUM_Q(name, neonOp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x111); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_Q]); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x111); \ + } + +COP2_ACCUM_Q(ADDAq, Fadd) +COP2_ACCUM_Q(SUBAq, Fsub) +COP2_ACCUM_Q(MULAq, Fmul) + +// ACCUMi variants +#define COP2_ACCUM_I(name, neonOp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_I]); \ + armAsm->neonOp(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x110); \ + } + +COP2_ACCUM_I(ADDAi, Fadd) +COP2_ACCUM_I(SUBAi, Fsub) +COP2_ACCUM_I(MULAi, Fmul) + +// MADDA/MSUBA variants: ACC = ACC ± VF[fs] * VF[ft] +// Separate FMUL+FADD/FSUB for PS2 intermediate rounding. +#define COP2_MADDA_OP(name, addOp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadVF(RQSCRATCH2, _Ft_cop2); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x110); \ + } + +COP2_MADDA_OP(MADDA, Fadd) +COP2_MADDA_OP(MSUBA, Fsub) + +// MADDA/MSUBA broadcast variants: ACC = ACC ± VF[fs] * VF[ft].bc +#define COP2_MADDA_BC(name, addOp, bc) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + cop2LoadBroadcast(RQSCRATCH2, _Ft_cop2, bc); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x110); \ + } + +COP2_MADDA_BC(MADDAx, Fadd, 0) +COP2_MADDA_BC(MADDAy, Fadd, 1) +COP2_MADDA_BC(MADDAz, Fadd, 2) +COP2_MADDA_BC(MADDAw, Fadd, 3) + +COP2_MADDA_BC(MSUBAx, Fsub, 0) +COP2_MADDA_BC(MSUBAy, Fsub, 1) +COP2_MADDA_BC(MSUBAz, Fsub, 2) +COP2_MADDA_BC(MSUBAw, Fsub, 3) + +// MADDAq/MSUBAq +#define COP2_MADDA_Q(name, addOp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x111); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_Q]); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x111); \ + } + +COP2_MADDA_Q(MADDAq, Fadd) +COP2_MADDA_Q(MSUBAq, Fsub) + +// MADDAi/MSUBAi +#define COP2_MADDA_I(name, addOp) \ + void recCOP2_V##name() \ + { \ + setupMacroOp_arm64(0x110); \ + cop2LoadVF(RQSCRATCH, _Fs_cop2); \ + armLd1rVU0(RQSCRATCH2.V4S(), &VU0.VI[REG_I]); \ + armAsm->Fmul(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); \ + cop2LoadACC(RQSCRATCH3); \ + armAsm->addOp(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); \ + cop2ClampResult(); \ + cop2EmitFlagUpdate(_XYZW_cop2); \ + cop2ApplyDestMaskACC(RQSCRATCH); \ + endMacroOp_arm64(0x110); \ + } + +COP2_MADDA_I(MADDAi, Fadd) +COP2_MADDA_I(MSUBAi, Fsub) + +// OPMULA: ACC.xyz = VF[fs].yzx * VF[ft].zxy (cross product to accumulator) +// PS2 always writes XYZ only, ignoring the instruction's dest field. +void recCOP2_VOPMULA() +{ + setupMacroOp_arm64(0x110); + cop2LoadVF(RQSCRATCH, _Fs_cop2); // fs = [x,y,z,w] + cop2LoadVF(RQSCRATCH2, _Ft_cop2); // ft = [x,y,z,w] + + // Build fs.yzx: EXT #4 gives [y,z,w,x], fix lane 2 (w→x) + a64::VRegister fsRot = a64::VRegister(28, 128); + armAsm->Ext(fsRot.V16B(), RQSCRATCH.V16B(), RQSCRATCH.V16B(), 4); // [y,z,w,x] + armAsm->Ins(fsRot.V4S(), 2, RQSCRATCH.V4S(), 0); // [y,z,x,x] + + // Build ft.zxy: EXT #8 gives [z,w,x,y], fix lanes 1,2 + a64::VRegister ftRot = a64::VRegister(27, 128); + armAsm->Ext(ftRot.V16B(), RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), 8); // [z,w,x,y] + armAsm->Ins(ftRot.V4S(), 1, RQSCRATCH2.V4S(), 0); // [z,x,x,y] + armAsm->Ins(ftRot.V4S(), 2, RQSCRATCH2.V4S(), 1); // [z,x,y,y] + + armAsm->Fmul(RQSCRATCH.V4S(), fsRot.V4S(), ftRot.V4S()); + cop2ClampResult(); + // OPMULA always updates XYZ flags only (0xE), W MAC flag cleared. + // PS2 hardware writes ACC.xyz only; ACC.w is preserved regardless of mask. + cop2EmitFlagUpdate(0xE); + + cop2ApplyDestMaskACCExplicit(RQSCRATCH, _XYZW_cop2 & 0xE); + endMacroOp_arm64(0x110); +} + +// ======================================================================== +// Conversion ops: ITOF0/4/12/15, FTOI0/4/12/15 +// ======================================================================== + +void recCOP2_VITOF0() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armAsm->Scvtf(RQSCRATCH.V4S(), RQSCRATCH.V4S()); + cop2ApplyDestMask(_Ft_cop2); +} + +void recCOP2_VITOF4() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armAsm->Scvtf(RQSCRATCH.V4S(), RQSCRATCH.V4S(), 4); + cop2ApplyDestMask(_Ft_cop2); +} + +void recCOP2_VITOF12() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armAsm->Scvtf(RQSCRATCH.V4S(), RQSCRATCH.V4S(), 12); + cop2ApplyDestMask(_Ft_cop2); +} + +void recCOP2_VITOF15() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + armAsm->Scvtf(RQSCRATCH.V4S(), RQSCRATCH.V4S(), 15); + cop2ApplyDestMask(_Ft_cop2); +} + +// Float→signed-int convert (Fcvtzs) with NaN saturation, for COP2 macro-mode +// VFTOIx. ARM64 NEON Fcvtzs returns 0 for a NaN input, but the PS2 — like +// mVU_FTOIx (microVU_Upper-arm64.inl) and the interpreter — saturates NaN to a +// sign-based INT_MAX/INT_MIN. Finite overflow and ±Inf already saturate +// correctly in Fcvtzs; only NaN lanes need the fixup. Source lanes are in +// RQSCRATCH and the converted+saturated result is left there; `fbits` is the +// fixed-point fraction (0/4/12/15). Uses RQSCRATCH2/RQSCRATCH3 as temps. +// +// Uses the same sign-based BIF pattern as mVU_FTOIx, but materializes the +// 0x7FFFFFFF constant with MVNI (NOT(0x80<<24)) instead of loading +// mVUglob.absclip, since the COP2 macro path does not set up the mVUglob base +// register. +static void cop2EmitFtoiSaturated(int fbits) +{ + // Build the saturation value and NaN mask from the source float BEFORE the + // convert clobbers RQSCRATCH. + armAsm->Sshr(RQSCRATCH2.V4S(), RQSCRATCH.V4S(), 31); // 0xffffffff if sign set + armAsm->Mvni(RQSCRATCH3.V4S(), 0x80, a64::LSL, 24); // 0x7fffffff (INT_MAX) per lane + armAsm->Eor(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH3.V16B()); // +NaN→0x7fffffff, -NaN→0x80000000 + armAsm->Fcmeq(RQSCRATCH3.V4S(), RQSCRATCH.V4S(), RQSCRATCH.V4S()); // 0xffffffff where NOT NaN + + if (fbits) + armAsm->Fcvtzs(RQSCRATCH.V4S(), RQSCRATCH.V4S(), fbits); + else + armAsm->Fcvtzs(RQSCRATCH.V4S(), RQSCRATCH.V4S()); + + // NaN lanes (notNan==0): replace Fcvtzs's 0 with the saturation value. + // BIF: dst bit <- src bit where mask bit is 0. + armAsm->Bif(RQSCRATCH.V16B(), RQSCRATCH2.V16B(), RQSCRATCH3.V16B()); +} + +void recCOP2_VFTOI0() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2EmitFtoiSaturated(0); + cop2ApplyDestMask(_Ft_cop2); +} + +void recCOP2_VFTOI4() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2EmitFtoiSaturated(4); + cop2ApplyDestMask(_Ft_cop2); +} + +void recCOP2_VFTOI12() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2EmitFtoiSaturated(12); + cop2ApplyDestMask(_Ft_cop2); +} + +void recCOP2_VFTOI15() +{ + cop2EmitConditionalSync(false, _vu0FinishMicro); + if (_Ft_cop2 == 0) return; + + cop2LoadVF(RQSCRATCH, _Fs_cop2); + cop2EmitFtoiSaturated(15); + cop2ApplyDestMask(_Ft_cop2); +} + +// ======================================================================== +// Division ops: VDIV, VSQRT, VRSQRT +// ======================================================================== +// These are scalar operations on single VF lanes, writing to the Q register. +// In macro mode, the result is immediately available (no pipeline delay). +// After computing Q, sync: copy to VI[REG_Q] and update D/I status flags. +// Complex edge cases (div-by-zero, negative sqrt) are handled with branches. + +// Emit SYNCFDIV: copy VU0.q to VU0.VI[REG_Q], update D/I status flags. +// statusflag = (statusflag & 0x3CF) | (statusflag_DI & 0x30) | ((statusflag_DI & 0x30) << 6) +static void cop2EmitSyncFDiv() +{ + // Copy q to VI[REG_Q] + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.q)); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.VI[REG_Q])); + + // Update status flag: (old & 0x3CF) | (statusflag & 0x30) | ((statusflag & 0x30) << 6) + armAsm->Ldr(a64::w1, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); + armAsm->And(a64::w1, a64::w1, 0x3CF); // clear D/I bits + + armAsm->Ldr(a64::w2, armVU0Mem(&VU0.statusflag)); + armAsm->And(a64::w2, a64::w2, 0x30); // D/I current bits + + armAsm->Orr(a64::w1, a64::w1, a64::w2); // current D/I + armAsm->Orr(a64::w1, a64::w1, a64::Operand(a64::w2, a64::LSL, 6)); // sticky D/I + + armAsm->Str(a64::w1, armVU0Mem(&VU0.VI[REG_STATUS_FLAG])); +} + +// VDIV: Q = VF[fs].fsf / VF[ft].ftf +void recCOP2_VDIV() +{ + const int fsf = _Fsf_cop2; + const int ftf = _Ftf_cop2; + + // Clear D/I flags + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Mov(RWARG1, 0x30); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG1); // clear D/I bits + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + + // Load fs scalar and ft scalar + armAsm->Ldr(RSSCRATCH, armVU0Mem(&VU0.VF[_Fs_cop2].UL[fsf])); // s30 = fs[fsf] + armAsm->Ldr(RSSCRATCH2, armVU0Mem(&VU0.VF[_Ft_cop2].UL[ftf])); // s31 = ft[ftf] + + // Check ft == 0 + a64::Label ftNonZero, done; + armAsm->Fcmp(RSSCRATCH2, 0.0); + armAsm->B(a64::ne, &ftNonZero); + + // ft == 0: set D/I flags, Q = ±FLT_MAX based on sign XOR + { + // Check if fs == 0 too → invalid (D flag = 0x10), else divide-by-zero (I flag = 0x20) + armAsm->Fcmp(RSSCRATCH, 0.0); + armAsm->Mov(a64::w1, 0x10); // invalid (0/0) + armAsm->Mov(a64::w2, 0x20); // div-by-zero + armAsm->Csel(a64::w1, a64::w1, a64::w2, a64::eq); + + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::w1); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + + // Q = sign(fs) XOR sign(ft) ? -FLT_MAX : +FLT_MAX + armAsm->Ldr(a64::w1, armVU0Mem(&VU0.VF[_Fs_cop2].UL[fsf])); + armAsm->Ldr(a64::w2, armVU0Mem(&VU0.VF[_Ft_cop2].UL[ftf])); + armAsm->Eor(a64::w1, a64::w1, a64::w2); + armAsm->Mov(a64::w2, 0x7F7FFFFF); // +FLT_MAX + armAsm->Mov(a64::w3, 0xFF7FFFFF); // -FLT_MAX (encoded as two MOVs by vixl) + armAsm->Tst(a64::w1, 0x80000000); + armAsm->Csel(RWSCRATCH, a64::w3, a64::w2, a64::ne); + + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.q)); + } + armAsm->B(&done); + + // ft != 0: Q = fs / ft, then clamp + armAsm->Bind(&ftNonZero); + { + armAsm->Fdiv(RSSCRATCH, RSSCRATCH, RSSCRATCH2); + // Clamp result against ±FLT_MAX held in callee-saved s8/s9. + armAsm->Fminnm(RSSCRATCH, RSSCRATCH, a64::s8); + armAsm->Fmaxnm(RSSCRATCH, RSSCRATCH, a64::s9); + armAsm->Str(RSSCRATCH, armVU0Mem(&VU0.q)); + } + + armAsm->Bind(&done); + cop2EmitSyncFDiv(); +} + +// VSQRT: Q = sqrt(|VF[ft].ftf|) +void recCOP2_VSQRT() +{ + const int ftf = _Ftf_cop2; + + // Clear D/I flags + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Mov(RWARG1, 0x30); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG1); // clear D/I bits + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + + // Load ft scalar + armAsm->Ldr(RSSCRATCH, armVU0Mem(&VU0.VF[_Ft_cop2].UL[ftf])); + + // If ft < 0, set invalid flag (D flag = 0x10) + a64::Label notNeg; + armAsm->Fcmp(RSSCRATCH, 0.0); + armAsm->B(a64::ge, ¬Neg); + { + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x10); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + } + armAsm->Bind(¬Neg); + + // Q = sqrt(|ft|) + armAsm->Fabs(RSSCRATCH, RSSCRATCH); + armAsm->Fsqrt(RSSCRATCH, RSSCRATCH); + + // Clamp against ±FLT_MAX held in callee-saved s8/s9. + armAsm->Fminnm(RSSCRATCH, RSSCRATCH, a64::s8); + armAsm->Fmaxnm(RSSCRATCH, RSSCRATCH, a64::s9); + + armAsm->Str(RSSCRATCH, armVU0Mem(&VU0.q)); + + cop2EmitSyncFDiv(); +} + +// VRSQRT: Q = VF[fs].fsf / sqrt(|VF[ft].ftf|) +void recCOP2_VRSQRT() +{ + const int fsf = _Fsf_cop2; + const int ftf = _Ftf_cop2; + + // Clear D/I flags + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Mov(RWARG1, 0x30); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG1); // clear D/I bits + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + + // Load ft scalar + armAsm->Ldr(RSSCRATCH2, armVU0Mem(&VU0.VF[_Ft_cop2].UL[ftf])); // s31 = ft[ftf] + + // Load fs scalar + armAsm->Ldr(RSSCRATCH, armVU0Mem(&VU0.VF[_Fs_cop2].UL[fsf])); // s30 = fs[fsf] + + // Check ft == 0 → div-by-zero + a64::Label ftNonZero, done; + armAsm->Fcmp(RSSCRATCH2, 0.0); + armAsm->B(a64::ne, &ftNonZero); + + // ft == 0: set div-by-zero flag (0x20), Q based on signs + { + armAsm->Fcmp(RSSCRATCH, 0.0); + + // fs == 0: set invalid flag too (0x10), Q = ±0 + a64::Label fsNonZero; + armAsm->B(a64::ne, &fsNonZero); + { + // D/I flags: 0x30 (both invalid and div-by-zero) + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x30); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + + // Q = sign(fs) XOR sign(ft) ? -0 : +0 + armAsm->Ldr(a64::w1, armVU0Mem(&VU0.VF[_Fs_cop2].UL[fsf])); + armAsm->Ldr(a64::w2, armVU0Mem(&VU0.VF[_Ft_cop2].UL[ftf])); + armAsm->Eor(a64::w1, a64::w1, a64::w2); + armAsm->And(RWSCRATCH, a64::w1, 0x80000000); // just sign bit, or 0 + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.q)); + armAsm->B(&done); + } + + // fs != 0: Q = ±FLT_MAX + armAsm->Bind(&fsNonZero); + { + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x20); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + + armAsm->Ldr(a64::w1, armVU0Mem(&VU0.VF[_Fs_cop2].UL[fsf])); + armAsm->Ldr(a64::w2, armVU0Mem(&VU0.VF[_Ft_cop2].UL[ftf])); + armAsm->Eor(a64::w1, a64::w1, a64::w2); + armAsm->Mov(a64::w2, 0x7F7FFFFF); + armAsm->Mov(a64::w3, 0xFF7FFFFF); + armAsm->Tst(a64::w1, 0x80000000); + armAsm->Csel(RWSCRATCH, a64::w3, a64::w2, a64::ne); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.q)); + armAsm->B(&done); + } + } + + // ft != 0: normal path + armAsm->Bind(&ftNonZero); + { + // If ft < 0, set invalid flag + a64::Label notNeg; + armAsm->Fcmp(RSSCRATCH2, 0.0); + armAsm->B(a64::ge, ¬Neg); + { + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x10); + armAsm->Str(RWSCRATCH, armVU0Mem(&VU0.statusflag)); + } + armAsm->Bind(¬Neg); + + // Q = fs / sqrt(|ft|) + armAsm->Fabs(RSSCRATCH2, RSSCRATCH2); + armAsm->Fsqrt(RSSCRATCH2, RSSCRATCH2); + armAsm->Fdiv(RSSCRATCH, RSSCRATCH, RSSCRATCH2); + + // Clamp against ±FLT_MAX held in callee-saved s8/s9. + armAsm->Fminnm(RSSCRATCH, RSSCRATCH, a64::s8); + armAsm->Fmaxnm(RSSCRATCH, RSSCRATCH, a64::s9); + + armAsm->Str(RSSCRATCH, armVU0Mem(&VU0.q)); + } + + armAsm->Bind(&done); + cop2EmitSyncFDiv(); +} + +// ======================================================================== +// CLIP: 6-plane frustum clip test +// ======================================================================== +// Compares VF[fs].xyz against ±|VF[ft].w| using signed integer comparison. +// Result: 6 bits shifted into clipflag history (24-bit rolling window). +// Bit layout per test: bit0=+x, bit1=-x, bit2=+y, bit3=-y, bit4=+z, bit5=-z + +void recCOP2_VCLIP() +{ + // Load ft.w as integer, compute |ft.w| with denormal handling + // If denormal (exponent == 0), use 0x007fffff instead + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VF[_Ft_cop2].UL[3])); // w lane + + // value = (raw & 0x7f800000) ? (raw & 0x7fffffff) : 0x007fffff + armAsm->Mov(a64::w1, RWSCRATCH); + armAsm->And(a64::w2, a64::w1, 0x7F800000); // exponent field + armAsm->And(a64::w1, a64::w1, 0x7FFFFFFF); // |raw| = clear sign + armAsm->Mov(a64::w3, 0x007FFFFF); // denormal replacement + armAsm->Cmp(a64::w2, 0); + armAsm->Csel(a64::w1, a64::w1, a64::w3, a64::ne); // w1 = clip value + + // Shift clipflag left by 6 + armAsm->Ldr(a64::w4, armVU0Mem(&VU0.clipflag)); + armAsm->Lsl(a64::w4, a64::w4, 6); + + // Load fs = [x,y,z,w] as integers for the lane comparisons. + armAsm->Ldr(RQSCRATCH, armVU0Mem(&VU0.VF[_Fs_cop2])); // q30 = [x,y,z,w] + + // Vectorized signed-integer clip test (matches the interp's + // (s32)(fs.lane ^ {0,0x80000000}) > value exactly — Cmgt is SCMGT). The + // scalar 6× UMOV/CMP/CSET loop collapses to two NEON compares plus a + // weighted horizontal add. + // pos = (s32)fs > value → +x,+y,+z lanes + // neg = (s32)(fs^signbit) > value → -x,-y,-z lanes + armAsm->Dup(RQSCRATCH3.V4S(), a64::w1); // q29 = [value × 4] + armAsm->Movi(RQSCRATCH2.V4S(), 0x80, a64::LSL, 24); // q31 = [0x80000000 × 4] + armAsm->Eor(RQSCRATCH2.V16B(), RQSCRATCH.V16B(), RQSCRATCH2.V16B()); // q31 = fs ^ sign + a64::VRegister posMask = a64::VRegister(28, 128); + armAsm->Cmgt(posMask.V4S(), RQSCRATCH.V4S(), RQSCRATCH3.V4S()); // pos mask + armAsm->Cmgt(RQSCRATCH2.V4S(), RQSCRATCH2.V4S(), RQSCRATCH3.V4S()); // neg mask + + // Weight each lane by its clip bit and fold to a 6-bit field. The negative + // weights are the positive ones << 1 ([1,4,16,0] -> [2,8,32,0]), so a single + // constant load plus a Shl covers both. +/- per axis are mutually exclusive + // and the weights are disjoint bits, so Add+Addv = OR (no carries). + a64::VRegister weight = a64::VRegister(27, 128); + armMoveAddressToReg(RSCRATCHADDR, &s_cop2ClipWeightPos); + armAsm->Ldr(weight, a64::MemOperand(RSCRATCHADDR)); // [1,4,16,0] + armAsm->And(posMask.V16B(), posMask.V16B(), weight.V16B()); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), weight.V16B()); + armAsm->Shl(RQSCRATCH2.V4S(), RQSCRATCH2.V4S(), 1); // neg weights = pos << 1 + armAsm->Add(posMask.V4S(), posMask.V4S(), RQSCRATCH2.V4S()); + armAsm->Addv(posMask.S(), posMask.V4S()); // sum lanes → scalar + armAsm->Umov(a64::w2, posMask.V4S(), 0); // 6-bit clip field + + // Merge into clipflag and mask to 24 bits + armAsm->Orr(a64::w4, a64::w4, a64::w2); + armAsm->And(a64::w4, a64::w4, 0xFFFFFF); + + // Store clipflag and sync to VI[REG_CLIP_FLAG] + armAsm->Str(a64::w4, armVU0Mem(&VU0.clipflag)); + armAsm->Str(a64::w4, armVU0Mem(&VU0.VI[REG_CLIP_FLAG])); + + // Broadcast the new clipflag into all 4 lanes of micro_clipflags. A + // subsequent VU0 microprogram loads its clip-flag instances directly from + // the VURegs::micro_clipflags field in the mVU Execute prologue — without + // this they would be stale (pre-VCLIP). RQSCRATCH is free here (its earlier + // fs load is consumed). + armAsm->Dup(RQSCRATCH.V4S(), a64::w4); + armAsm->Str(RQSCRATCH, armVU0Mem(&VU0.micro_clipflags)); +} + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 + +// ======================================================================== +// cop2flags — determines which control flags a COP2 instruction writes. +// Used by the analysis pass (iR5900Analysis.cpp) for flag optimization. +// Returns: 0=none, 1=status, 2=MAC, 3=both, 4=clip +// Architecture-independent — identical to x86 version. +// ======================================================================== + +int cop2flags(u32 code) +{ + if (code >> 26 != 022) + return 0; // not COP2 + if ((code >> 25 & 1) == 0) + return 0; // a branch or transfer instruction + + switch (code >> 2 & 15) + { + case 15: + switch (code >> 6 & 0x1f) + { + case 4: // ITOF* + case 5: // FTOI* + case 12: // MOVE MR32 + case 13: // LQI SQI LQD SQD + case 15: // MTIR MFIR ILWR ISWR + case 16: // RNEXT RGET RINIT RXOR + return 0; + case 7: // MULAq, ABS, MULAi, CLIP + if ((code & 3) == 1) // ABS + return 0; + if ((code & 3) == 3) // CLIP + return 4; + return 3; + case 11: // SUBA, MSUBA, OPMULA, NOP + if ((code & 3) == 3) // NOP + return 0; + return 3; + case 14: // DIV, SQRT, RSQRT, WAITQ + if ((code & 3) == 3) // WAITQ + return 0; + return 1; + default: + break; + } + break; + case 4: // MAXbc + case 5: // MINbc + case 12: // IADD, ISUB, IADDI + case 13: // IAND, IOR + case 14: // VCALLMS, VCALLMSR + return 0; + case 7: + if ((code & 1) == 1) // MAXi, MINIi + return 0; + return 3; + case 10: + if ((code & 3) == 3) // MAX + return 0; + return 3; + case 11: + if ((code & 3) == 3) // MINI + return 0; + return 3; + default: + break; + } + return 3; +} diff --git a/pcsx2/arm64/iCore-arm64.cpp b/pcsx2/arm64/iCore-arm64.cpp new file mode 100644 index 0000000000..0bb548ecc5 --- /dev/null +++ b/pcsx2/arm64/iCore-arm64.cpp @@ -0,0 +1,1197 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +#include "Config.h" +#include "R3000A.h" +#include "R5900.h" +#include "Vif.h" +#include "VU.h" +#include "arm64/iR5900-arm64.h" +#include "arm64/iR3000A-arm64.h" + +#include "common/Assertions.h" +#include "common/Console.h" + +namespace a64 = vixl::aarch64; + +//#define RALOG(...) fprintf(stderr, __VA_ARGS__) +#define RALOG(...) + + +//////////////////////////////////////////////////////////////////////////////// +// IOP constant propagation externs +// These are defined in the IOP recompiler, but the register allocator needs +// them to handle PSX register allocation correctly. + +extern u32 g_psxConstRegs[32]; +extern u32 g_psxHasConstReg, g_psxFlushedConstReg; + +#define PSX_IS_CONST1(reg) ((reg) < 32 && (g_psxHasConstReg & (1 << (reg)))) +#define PSX_DEL_CONST(reg) \ + { \ + if ((reg) < 32) \ + g_psxHasConstReg &= ~(1 << (reg)); \ + } + +//////////////////////////////////////////////////////////////////////////////// +// Shared state + +EEINST* g_pCurInstInfo = nullptr; + +u16 g_arm64AllocCounter = 0; +u16 g_neonAllocCounter = 0; + +// EE constant propagation state +alignas(16) GPR_reg64 g_cpuConstRegs[32] = {}; +u32 g_cpuHasConstReg = 0, g_cpuFlushedConstReg = 0; + +//////////////////////////////////////////////////////////////////////////////// +// ARM64 GPR Register Allocator + +_arm64gprregs arm64gprs[NUM_ARM_GPR_REGS], s_saveArm64GPRregs[NUM_ARM_GPR_REGS]; +static uint g_arm64checknext = 0; + +_arm64neonregs arm64neon[NUM_ARM_NEON_REGS], s_saveArm64NEONregs[NUM_ARM_NEON_REGS]; + +// ARM64 register allocation policy: +// x0-x3: argument/return registers (caller-saved, allocatable) +// x4-x15: caller-saved temporaries (allocatable) +// x16: VIXL intra-procedure scratch — NOT allocatable +// x17: RSCRATCHADDR — NOT allocatable +// x18: platform reserved — NOT allocatable +// x19: RFASTMEMBASE — NOT allocatable (reserved for fastmem base) +// x20: RSTATE — NOT allocatable (reserved for cpuRegs pointer) +// x21: RPSXSTATE — NOT allocatable (reserved for psxRegs pointer in IOP JIT) +// x22-x23: callee-saved (allocatable) +// x24: RVU0 — NOT allocatable (reserved for &VU0 pointer in EE COP2 JIT) +// x25: RECCYCLE — NOT allocatable (reserved for cpuRegs.cycle) +// x26-x28: callee-saved (allocatable) +// x29: frame pointer — NOT allocatable +// x30: link register — NOT allocatable + +// Bitmask of allocatable aarch64 GPRs. Bit `n` set ↔ x_n is in the pool. +// Cleared bits, all-pinned/scratch as documented above: +// bit 8 — x8 : RXSCRATCH/RWSCRATCH (value scratch) +// bits 9-10 — x9/x10 : load/store address + value scratch +// bits 16-18 — x16 (vixl), x17 (RSCRATCHADDR), x18 (platform reserved) +// bit 19 — x19 : RFASTMEMBASE +// bit 20 — x20 : RSTATE (cpuRegs base pointer) +// bit 21 — x21 : RPSXSTATE (psxRegs base; shared alloc table with EE) +// bit 24 — x24 : RVU0 (pinned &VU0 for iCOP2) +// bit 25 — x25 : RECCYCLE (pinned cpuRegs.cycle) +// bits 29-30 — x29/x30 : FP, LR — never allocatable +// Inner allocator loop runs 31× per cache miss and was nine sequential +// `if (armreg == N) return false` branches per probe; collapse to one +// LSR + AND + cbz against this mask. +static constexpr uint32_t ALLOCATABLE_MASK = ~((1u << 8) + | (1u << 9) | (1u << 10) + | (7u << 16) + | (1u << 19) | (1u << 20) | (1u << 21) + | (1u << 24) | (1u << 25) + | (3u << 29)); + +bool _isAllocatableArm64GPR(int armreg) +{ + return ((ALLOCATABLE_MASK >> armreg) & 1u) != 0u; +} + +void _initArm64GPRregs() +{ + std::memset(arm64gprs, 0, sizeof(arm64gprs)); + g_arm64AllocCounter = 0; + g_arm64checknext = 0; +} + +bool _hasArm64GPR(int type, int reg, int required_mode) +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == type && arm64gprs[i].reg == reg) + return ((arm64gprs[i].mode & required_mode) == required_mode); + } + return false; +} + +int _getFreeArm64GPR(int mode) +{ + int tempi = -1; + u32 bestcount = 0x10000; + + // First pass: find a completely free register + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + const int reg = (g_arm64checknext + i) % NUM_ARM_GPR_REGS; + if (arm64gprs[reg].inuse || !_isAllocatableArm64GPR(reg)) + continue; + + if ((mode & MODE_CALLEESAVED) && !armIsCalleeSavedRegister(reg)) + continue; + + if ((mode & MODE_COP2) && mVUIsReservedCOP2(reg)) + continue; + + g_arm64checknext = (reg + 1) % NUM_ARM_GPR_REGS; + return reg; + } + + // Second pass: evict by LRU, prefer temps first + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (!_isAllocatableArm64GPR(i)) + continue; + if ((mode & MODE_CALLEESAVED) && !armIsCalleeSavedRegister(i)) + continue; + if ((mode & MODE_COP2) && mVUIsReservedCOP2(i)) + continue; + + pxAssert(arm64gprs[i].inuse); + if (arm64gprs[i].needed) + continue; + + if (arm64gprs[i].type == ARM64TYPE_TEMP) + { + _freeArm64GPR(i); + return i; + } + + if (arm64gprs[i].counter < bestcount) + { + tempi = i; + bestcount = arm64gprs[i].counter; + } + } + + if (tempi != -1) + { + _freeArm64GPR(tempi); + return tempi; + } + + pxFailRel("ARM64 GPR register allocation error"); + return -1; +} + +void _writebackArm64GPR(int armreg) +{ + switch (arm64gprs[armreg].type) + { + case ARM64TYPE_GPR: + RALOG("Writing back ARM64 GPR %d for guest reg %d\n", armreg, arm64gprs[armreg].reg); + armStoreEERegPtr(armXRegister(armreg), &cpuRegs.GPR.r[arm64gprs[armreg].reg].UD[0]); + break; + + case ARM64TYPE_FPRC: + RALOG("Writing back ARM64 GPR %d for guest FPCR %d\n", armreg, arm64gprs[armreg].reg); + armStoreEERegPtr(armWRegister(armreg), &fpuRegs.fprc[arm64gprs[armreg].reg]); + break; + + case ARM64TYPE_VIREG: + RALOG("Writing back ARM64 GPR %d for guest VI %d\n", armreg, arm64gprs[armreg].reg); + armAsm->Strh(armWRegister(armreg), armVU0Mem(&VU0.VI[arm64gprs[armreg].reg].UL)); + break; + + case ARM64TYPE_PCWRITEBACK: + RALOG("Writing back PC writeback from ARM64 GPR %d\n", armreg); + armAsm->Str(armWRegister(armreg), armCpuRegMem(&cpuRegs.pcWriteback)); + break; + + case ARM64TYPE_PSX: + RALOG("Writing back ARM64 GPR %d for guest PSX reg %d\n", armreg, arm64gprs[armreg].reg); + armAsm->Str(armWRegister(armreg), armPsxRegMem(&psxRegs.GPR.r[arm64gprs[armreg].reg])); + break; + + case ARM64TYPE_PSX_PCWRITEBACK: + RALOG("Writing back PSX PC writeback from ARM64 GPR %d\n", armreg); + armAsm->Str(armWRegister(armreg), armPsxRegMem(&psxRegs.pcWriteback)); + break; + + default: + break; + } +} + +void _freeArm64GPR(int armreg) +{ + pxAssert(armreg >= 0 && armreg < NUM_ARM_GPR_REGS); + if (!arm64gprs[armreg].inuse) + return; + + if (arm64gprs[armreg].mode & MODE_WRITE) + _writebackArm64GPR(armreg); + + arm64gprs[armreg].inuse = 0; + arm64gprs[armreg].mode = 0; +} + +void _freeArm64GPRWithoutWriteback(int armreg) +{ + pxAssert(armreg >= 0 && armreg < NUM_ARM_GPR_REGS); + arm64gprs[armreg].inuse = 0; + arm64gprs[armreg].mode = 0; +} + +void _freeArm64GPRregs() +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse) + _freeArm64GPR(i); + } +} + +void _flushArm64GPRregs() +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && (arm64gprs[i].mode & MODE_WRITE)) + { + _writebackArm64GPR(i); + arm64gprs[i].mode &= ~MODE_WRITE; + arm64gprs[i].mode |= MODE_READ; + } + } +} + +int _checkArm64GPR(int type, int reg, int mode) +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == type && arm64gprs[i].reg == reg) + { + arm64gprs[i].mode |= mode; + arm64gprs[i].counter = g_arm64AllocCounter++; + arm64gprs[i].needed = 1; + return i; + } + } + return -1; +} + +int _allocArm64GPR(int type, int reg, int mode) +{ + if (type == ARM64TYPE_GPR || type == ARM64TYPE_PSX) + pxAssertMsg(reg >= 0 && reg < 34, "Register index out of bounds."); + + int hostNEONreg = (type == ARM64TYPE_GPR) ? _checkNEONreg(NEONTYPE_GPRREG, reg, 0) : -1; + + // Check if already allocated + if (type != ARM64TYPE_TEMP) + { + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (!arm64gprs[i].inuse || arm64gprs[i].type != type || arm64gprs[i].reg != reg) + continue; + + if (type == ARM64TYPE_VIREG && reg < 0) + continue; + + if (type == ARM64TYPE_GPR && (mode & MODE_WRITE)) + { + if (GPR_IS_CONST1(reg)) + GPR_DEL_CONST(reg); + if (hostNEONreg >= 0) + { + pxAssert(!(arm64neon[hostNEONreg].mode & MODE_WRITE)); + _freeNEONreg(hostNEONreg); + } + } + else if (type == ARM64TYPE_PSX && (mode & MODE_WRITE)) + { + if (PSX_IS_CONST1(reg)) + PSX_DEL_CONST(reg); + } + + arm64gprs[i].counter = g_arm64AllocCounter++; + arm64gprs[i].mode |= mode & ~MODE_CALLEESAVED; + arm64gprs[i].needed = true; + return i; + } + } + + // Need to allocate a new register + const int regnum = _getFreeArm64GPR(mode); + arm64gprs[regnum].type = type; + arm64gprs[regnum].reg = reg; + arm64gprs[regnum].mode = mode & ~MODE_CALLEESAVED; + arm64gprs[regnum].counter = g_arm64AllocCounter++; + arm64gprs[regnum].needed = true; + arm64gprs[regnum].inuse = true; + + if (mode & MODE_READ) + { + switch (type) + { + case ARM64TYPE_GPR: + { + if (reg == 0) + { + // r0 is always zero + armAsm->Mov(armWRegister(regnum), 0); + } + else if (hostNEONreg >= 0) + { + // Value is in a NEON register, extract lower 64 bits + RALOG("Copying guest reg %d from NEON %d to GPR %d\n", reg, hostNEONreg, regnum); + armAsm->Mov(armXRegister(regnum), armQRegister(hostNEONreg).V2D(), 0); + + if (arm64neon[hostNEONreg].mode & MODE_WRITE) + { + _freeNEONreg(hostNEONreg); + } + } + else if (GPR_IS_CONST1(reg)) + { + RALOG("Loading constant %lld for guest reg %d to GPR %d\n", + (long long)g_cpuConstRegs[reg].SD[0], reg, regnum); + armAsm->Mov(armXRegister(regnum), g_cpuConstRegs[reg].SD[0]); + g_cpuFlushedConstReg |= (1u << reg); + arm64gprs[regnum].mode |= MODE_WRITE; + } + else + { + RALOG("Loading guest reg %d to GPR %d\n", reg, regnum); + armLoadEERegPtr(armXRegister(regnum), &cpuRegs.GPR.r[reg].UD[0]); + } + } + break; + + case ARM64TYPE_FPRC: + RALOG("Loading guest FPCR %d to GPR %d\n", reg, regnum); + armLoadEERegPtr(armWRegister(regnum), &fpuRegs.fprc[reg]); + break; + + case ARM64TYPE_PSX: + { + if (reg == 0) + { + armAsm->Mov(armWRegister(regnum), 0); + } + else if (PSX_IS_CONST1(reg)) + { + armAsm->Mov(armWRegister(regnum), g_psxConstRegs[reg]); + g_psxFlushedConstReg |= (1u << reg); + arm64gprs[regnum].mode |= MODE_WRITE; + } + else + { + armLoadPsxRegPtr(armWRegister(regnum), &psxRegs.GPR.r[reg]); + } + } + break; + + case ARM64TYPE_VIREG: + { + RALOG("Loading guest VI reg %d to GPR %d\n", reg, regnum); + armAsm->Ldrh(armWRegister(regnum), armVU0Mem(&VU0.VI[reg].US[0])); + } + break; + + default: + break; + } + } + + if (type == ARM64TYPE_GPR && (mode & MODE_WRITE)) + { + if (reg < 32 && GPR_IS_CONST1(reg)) + GPR_DEL_CONST(reg); + if (hostNEONreg >= 0) + { + // We're about to write this guest reg into the scalar GPR, so the + // cached NEON copy is superseded — discard it WITHOUT writeback + // (mirrors _allocGPRtoNEONreg and x86 _allocGPRtoXMMreg). Writing + // it back would store a stale value the new GPR's flush overwrites. + _freeNEONregWithoutWriteback(hostNEONreg); + } + } + else if (type == ARM64TYPE_PSX && (mode & MODE_WRITE)) + { + if (reg < 32 && PSX_IS_CONST1(reg)) + PSX_DEL_CONST(reg); + } + + return regnum; +} + +void _addNeededArm64GPR(int type, int reg) +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == type && arm64gprs[i].reg == reg) + arm64gprs[i].needed = 1; + } +} + +void _clearNeededArm64GPRregs() +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].needed && arm64gprs[i].type == ARM64TYPE_TEMP) + _freeArm64GPR(i); + arm64gprs[i].needed = 0; + } +} + +void _flushConstReg(int reg) +{ + if (GPR_IS_CONST1(reg) && !(g_cpuFlushedConstReg & (1 << reg))) + { + armAsm->Mov(RXSCRATCH, static_cast(g_cpuConstRegs[reg].SD[0])); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[reg].UD[0])); + g_cpuFlushedConstReg |= (1 << reg); + if (reg == 0) + DevCon.Warning("Flushing r0!"); + } +} + +void _flushConstRegs(bool delete_const) +{ + for (u32 i = 0; i < 32; i++) + { + if (!GPR_IS_CONST1(i) || g_cpuFlushedConstReg & (1u << i)) + continue; + + armAsm->Mov(RXSCRATCH, static_cast(g_cpuConstRegs[i].UD[0])); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[i].UD[0])); + g_cpuFlushedConstReg |= 1u << i; + } + + if (delete_const) + { + // Clear ALL const state, including already-flushed registers. + // After an interpreter call, the interpreter may have modified any + // register — stale const flags would cause subsequent native code + // to use outdated values from g_cpuConstRegs instead of memory. + g_cpuHasConstReg = 1; // keep r0 (always zero) + g_cpuFlushedConstReg = 1; + } +} + +void _validateRegs() +{ +#ifdef PCSX2_DEVBUILD + for (s8 guestreg = 0; guestreg < 32; guestreg++) + { + u32 gprreg = 0, gprmode = 0; + u32 neonreg = 0, neonmode = 0; + for (int hostreg = 0; hostreg < NUM_ARM_GPR_REGS; hostreg++) + { + if (arm64gprs[hostreg].inuse && arm64gprs[hostreg].type == ARM64TYPE_GPR && arm64gprs[hostreg].reg == guestreg) + { + pxAssertMsg(gprreg == 0 && gprmode == 0, "register not already allocated in GPR"); + gprreg = hostreg; + gprmode = arm64gprs[hostreg].mode; + } + } + for (int hostreg = 0; hostreg < NUM_ARM_NEON_REGS; hostreg++) + { + if (arm64neon[hostreg].inuse && arm64neon[hostreg].type == NEONTYPE_GPRREG && arm64neon[hostreg].reg == guestreg) + { + pxAssertMsg(neonreg == 0 && neonmode == 0, "register not already allocated in NEON"); + neonreg = hostreg; + neonmode = arm64neon[hostreg].mode; + } + } + + if ((gprmode | neonmode) & MODE_WRITE) + pxAssertMsg((gprmode & MODE_WRITE) != (neonmode & MODE_WRITE), "only one of GPR/NEON is in write state"); + } +#endif +} + +// Type-specific convenience wrappers over _addNeededArm64GPR. +void _addNeededGPRtoArm64GPR(int gprreg) { _addNeededArm64GPR(ARM64TYPE_GPR, gprreg); } +void _addNeededPSXtoArm64GPR(int gprreg) { _addNeededArm64GPR(ARM64TYPE_PSX, gprreg); } + +void _deleteGPRtoArm64GPR(int reg, int flush) +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == ARM64TYPE_GPR && arm64gprs[i].reg == reg) + { + switch (flush) + { + case DELETE_REG_FREE: _freeArm64GPR(i); break; + case DELETE_REG_FLUSH: + if (arm64gprs[i].mode & MODE_WRITE) + { + _writebackArm64GPR(i); + // Drop MODE_WRITE (keep MODE_READ) so a later + // _freeArm64GPR won't store the same value again. + arm64gprs[i].mode = (arm64gprs[i].mode & ~MODE_WRITE) | MODE_READ; + } + break; + case DELETE_REG_FLUSH_AND_FREE: _freeArm64GPR(i); break; + case DELETE_REG_FREE_NO_WRITEBACK: _freeArm64GPRWithoutWriteback(i); break; + } + return; + } + } +} + +void _deletePSXtoArm64GPR(int reg, int flush) +{ + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == ARM64TYPE_PSX && arm64gprs[i].reg == reg) + { + switch (flush) + { + case DELETE_REG_FREE: _freeArm64GPR(i); break; + case DELETE_REG_FLUSH: + if (arm64gprs[i].mode & MODE_WRITE) + { + _writebackArm64GPR(i); + // Drop MODE_WRITE (keep MODE_READ) so a later + // _freeArm64GPR won't store the same value again. + arm64gprs[i].mode = (arm64gprs[i].mode & ~MODE_WRITE) | MODE_READ; + } + break; + case DELETE_REG_FLUSH_AND_FREE: _freeArm64GPR(i); break; + case DELETE_REG_FREE_NO_WRITEBACK: _freeArm64GPRWithoutWriteback(i); break; + } + return; + } + } +} + +int _allocIfUsedGPRtoArm64(int gprreg, int mode) +{ + return EEINST_USEDTEST(gprreg) ? _allocArm64GPR(ARM64TYPE_GPR, gprreg, mode) : -1; +} + +int _allocIfUsedVItoArm64(int vireg, int mode) +{ + return EEINST_VIUSEDTEST(vireg) ? _allocArm64GPR(ARM64TYPE_VIREG, vireg, mode) : -1; +} + +//////////////////////////////////////////////////////////////////////////////// +// ARM64 NEON Register Allocator + +void _initArm64NEONregs() +{ + std::memset(arm64neon, 0, sizeof(arm64neon)); + g_neonAllocCounter = 0; +} + +// Reserved NEON scalars for PS2 FPU clamp constants (held across the JIT +// session). s8 = +FLT_MAX, s9 = -FLT_MAX. Loaded in the EE dispatcher and +// mVU dispatcher prologues; used by fpuClampResult and iCOP2 scalar +// VDIV/VSQRT/VRSQRT. Lower 64 bits are callee-saved per AAPCS64, so the +// values survive every armEmitCall path without compile-time tracking. +// v8/v9 are skipped by every _getFreeArm64NEON search loop below — no +// allocator codepath can land on them. +static constexpr u32 NEON_RESERVED_FPU_MAX = 8; +static constexpr u32 NEON_RESERVED_FPU_MIN = 9; + +// Callee-saved NEON range available to the allocator: q10-q15 +// (indices 8/9 reserved above). EE GPR values allocated here survive FPU +// interpreter calls without flushing. +static constexpr u32 NEON_CALLEE_SAVED_START = 10; +static constexpr u32 NEON_CALLEE_SAVED_END = 16; // exclusive + +int _getFreeArm64NEON(u32 minreg, u32 maxreg) +{ + int tempi = -1; + u32 bestcount = 0x10000; + + // Check for free registers + for (u32 i = minreg; i < maxreg; i++) + { + if (i == NEON_RESERVED_FPU_MAX || i == NEON_RESERVED_FPU_MIN) + continue; + if (!arm64neon[i].inuse) + return i; + } + + // Check for dead regs + tempi = -1; + bestcount = 0xffff; + for (u32 i = minreg; i < maxreg; i++) + { + if (i == NEON_RESERVED_FPU_MAX || i == NEON_RESERVED_FPU_MIN) + continue; + pxAssert(arm64neon[i].inuse); + if (arm64neon[i].needed) + continue; + + pxAssert(arm64neon[i].type != NEONTYPE_TEMP); + + if (arm64neon[i].counter < bestcount) + { + switch (arm64neon[i].type) + { + case NEONTYPE_GPRREG: + if (EEINST_USEDTEST(arm64neon[i].reg)) + continue; + break; + case NEONTYPE_FPREG: + if (FPUINST_USEDTEST(arm64neon[i].reg)) + continue; + break; + case NEONTYPE_VFREG: + if (EEINST_VFUSEDTEST(arm64neon[i].reg)) + continue; + break; + } + + tempi = i; + bestcount = arm64neon[i].counter; + } + } + + if (tempi != -1) + { + _freeNEONreg(tempi); + return tempi; + } + + // Last resort: take the LRU register + bestcount = 0xffff; + for (u32 i = minreg; i < maxreg; i++) + { + if (i == NEON_RESERVED_FPU_MAX || i == NEON_RESERVED_FPU_MIN) + continue; + pxAssert(arm64neon[i].inuse); + if (arm64neon[i].needed) + continue; + + if (arm64neon[i].counter < bestcount) + { + tempi = i; + bestcount = arm64neon[i].counter; + } + } + + if (tempi != -1) + { + _freeNEONreg(tempi); + return tempi; + } + + pxFailRel("ARM64 NEON register allocation error"); + return -1; +} + +// Overload for backward compatibility (full range) +int _getFreeArm64NEON(u32 maxreg) +{ + return _getFreeArm64NEON(0, maxreg); +} + +int _allocTempNEONreg() +{ + const int neonreg = _getFreeArm64NEON(); + arm64neon[neonreg].inuse = 1; + arm64neon[neonreg].type = NEONTYPE_TEMP; + arm64neon[neonreg].needed = 1; + arm64neon[neonreg].counter = g_neonAllocCounter++; + return neonreg; +} + +int _checkNEONreg(int type, int reg, int mode) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && (arm64neon[i].type == (type & 0xff)) && (arm64neon[i].reg == reg)) + { + if (type == NEONTYPE_GPRREG && (mode & MODE_WRITE)) + return _allocGPRtoNEONreg(reg, mode); + + arm64neon[i].mode |= mode; + arm64neon[i].counter = g_neonAllocCounter++; + arm64neon[i].needed = 1; + return i; + } + } + return -1; +} + +bool _hasNEONreg(int type, int reg, int required_mode) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == type && arm64neon[i].reg == reg) + return ((arm64neon[i].mode & required_mode) == required_mode); + } + return false; +} + +int _allocFPtoNEONreg(int fpreg, int mode) +{ + // Check if already allocated + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (!arm64neon[i].inuse || arm64neon[i].type != NEONTYPE_FPREG || arm64neon[i].reg != fpreg) + continue; + + // Slot already holds the live value (MODE_READ → loaded from memory, + // MODE_WRITE → freshly written; both are authoritative over memory). + // Reloading here would clobber a MODE_WRITE-only live value with + // stale memory, breaking chained ops where the next read consumes + // the previous write. Mirrors _allocGPRtoNEONreg's reuse path. + arm64neon[i].counter = g_neonAllocCounter++; + arm64neon[i].needed = 1; + arm64neon[i].mode |= mode; + return i; + } + + // New allocation + const int neonreg = _getFreeArm64NEON(); + arm64neon[neonreg].inuse = 1; + arm64neon[neonreg].type = NEONTYPE_FPREG; + arm64neon[neonreg].reg = fpreg; + arm64neon[neonreg].mode = mode; + arm64neon[neonreg].needed = 1; + arm64neon[neonreg].counter = g_neonAllocCounter++; + + if (mode & MODE_READ) + { + armLoadEERegPtr(armSRegister(neonreg), &fpuRegs.fpr[fpreg].f); + } + + return neonreg; +} + +int _allocGPRtoNEONreg(int gprreg, int mode) +{ + const int hostGPRreg = _checkArm64GPR(ARM64TYPE_GPR, gprreg, MODE_READ); + + // Check if already in NEON + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (!arm64neon[i].inuse || arm64neon[i].type != NEONTYPE_GPRREG || arm64neon[i].reg != gprreg) + continue; + + if (mode & MODE_WRITE && hostGPRreg >= 0) + { + // Dual-dirty (NEON MODE_WRITE + arm64gpr MODE_WRITE for the same + // guest reg) means a scalar op left a pending lower-64 write. + // Flush it before freeing so the value isn't lost. This case is + // legitimate, not an error: eeRecompileCodeXMM can reuse a + // MMI-written slot for a subsequent MMI Rd while the scalar GPR + // allocator still holds an unrelated MODE_WRITE entry for the + // same guest reg. + if (arm64gprs[hostGPRreg].mode & MODE_WRITE) + _writebackArm64GPR(hostGPRreg); + _freeArm64GPRWithoutWriteback(hostGPRreg); + } + + if (mode & MODE_WRITE && GPR_IS_CONST1(gprreg)) + GPR_DEL_CONST(gprreg); + + arm64neon[i].counter = g_neonAllocCounter++; + arm64neon[i].needed = true; + arm64neon[i].mode |= mode; + return i; + } + + // Allocate EE GPRs to callee-saved NEON range so they survive C + // function calls (FPU interpreter, etc.) without flushing. + const int neonreg = _getFreeArm64NEON(NEON_CALLEE_SAVED_START, NEON_CALLEE_SAVED_END); + arm64neon[neonreg].inuse = 1; + arm64neon[neonreg].type = NEONTYPE_GPRREG; + arm64neon[neonreg].reg = gprreg; + arm64neon[neonreg].mode = mode; + arm64neon[neonreg].needed = 1; + arm64neon[neonreg].counter = g_neonAllocCounter++; + + if (mode & MODE_READ) + { + if (gprreg == 0) + { + armAsm->Movi(armQRegister(neonreg).V2D(), 0); + } + else if (GPR_IS_CONST1(gprreg)) + { + // Load full 128 bits from memory, replace lower 64 with constant + armLoadEERegPtr(armQRegister(neonreg), &cpuRegs.GPR.r[gprreg].UQ); + armAsm->Mov(RXSCRATCH, static_cast(g_cpuConstRegs[gprreg].SD[0])); + armAsm->Ins(armQRegister(neonreg).V2D(), 0, RXSCRATCH); + arm64neon[neonreg].mode |= MODE_WRITE; + g_cpuFlushedConstReg |= (1u << gprreg); + + if (hostGPRreg >= 0) + _freeArm64GPRWithoutWriteback(hostGPRreg); + } + else if (hostGPRreg >= 0) + { + // Load full 128, replace lower if dirty + armLoadEERegPtr(armQRegister(neonreg), &cpuRegs.GPR.r[gprreg].UQ); + if (arm64gprs[hostGPRreg].mode & MODE_WRITE) + { + armAsm->Ins(armQRegister(neonreg).V2D(), 0, armXRegister(hostGPRreg)); + _freeArm64GPRWithoutWriteback(hostGPRreg); + arm64neon[neonreg].mode |= MODE_WRITE; + } + } + else + { + armLoadEERegPtr(armQRegister(neonreg), &cpuRegs.GPR.r[gprreg].UQ); + } + } + + if (mode & MODE_WRITE && gprreg < 32 && GPR_IS_CONST1(gprreg)) + GPR_DEL_CONST(gprreg); + if (mode & MODE_WRITE && hostGPRreg >= 0) + _freeArm64GPRWithoutWriteback(hostGPRreg); + + return neonreg; +} + +int _allocFPACCtoNEONreg(int mode) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (!arm64neon[i].inuse || arm64neon[i].type != NEONTYPE_FPACC) + continue; + + // Same invariant as _allocFPtoNEONreg: the slot already holds the + // authoritative value (loaded or freshly written). Reloading would + // clobber a MODE_WRITE-only ACC with stale memory, so a later read of + // ACC must consume the value emitted earlier in the same block rather + // than the pre-block memory image. + arm64neon[i].counter = g_neonAllocCounter++; + arm64neon[i].needed = 1; + arm64neon[i].mode |= mode; + return i; + } + + const int neonreg = _getFreeArm64NEON(); + arm64neon[neonreg].inuse = 1; + arm64neon[neonreg].type = NEONTYPE_FPACC; + arm64neon[neonreg].reg = 0; + arm64neon[neonreg].mode = mode; + arm64neon[neonreg].needed = 1; + arm64neon[neonreg].counter = g_neonAllocCounter++; + + if (mode & MODE_READ) + { + armLoadEERegPtr(armSRegister(neonreg), &fpuRegs.ACC.f); + } + + return neonreg; +} + +int _allocVFtoNEONreg(int vfreg, int mode) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (!arm64neon[i].inuse || arm64neon[i].type != NEONTYPE_VFREG || arm64neon[i].reg != vfreg) + continue; + + if (!(arm64neon[i].mode & MODE_READ) && (mode & MODE_READ)) + { + armLoadPtr(armQRegister(i), &VU0.VF[vfreg]); + arm64neon[i].mode |= MODE_READ; + } + + arm64neon[i].counter = g_neonAllocCounter++; + arm64neon[i].needed = 1; + arm64neon[i].mode |= mode; + return i; + } + + const int neonreg = _getFreeArm64NEON(); + arm64neon[neonreg].inuse = 1; + arm64neon[neonreg].type = NEONTYPE_VFREG; + arm64neon[neonreg].reg = vfreg; + arm64neon[neonreg].mode = mode; + arm64neon[neonreg].needed = 1; + arm64neon[neonreg].counter = g_neonAllocCounter++; + + if (mode & MODE_READ) + armLoadPtr(armQRegister(neonreg), &VU0.VF[vfreg]); + + return neonreg; +} + +void _writebackNEONreg(int neonreg) +{ + switch (arm64neon[neonreg].type) + { + case NEONTYPE_GPRREG: + { + // EE GPRs are 128-bit. Store the full Q register so MMI ops (which + // write all 128 bits via eeRecompileCodeXMM) preserve their upper + // 64-bit lanes through the writeback. _allocGPRtoNEONreg always + // loads 128 bits on MODE_READ, so writeback symmetry is required. + const int reg = arm64neon[neonreg].reg; + if (reg == NEONGPR_LO) + armStorePtr(armQRegister(neonreg), &cpuRegs.LO.UQ); + else if (reg == NEONGPR_HI) + armStorePtr(armQRegister(neonreg), &cpuRegs.HI.UQ); + else + armStorePtr(armQRegister(neonreg), &cpuRegs.GPR.r[reg].UQ); + } + break; + + case NEONTYPE_FPREG: + { + armStoreEERegPtr(armSRegister(neonreg), &fpuRegs.fpr[arm64neon[neonreg].reg].f); + } + break; + + case NEONTYPE_FPACC: + { + armStoreEERegPtr(armSRegister(neonreg), &fpuRegs.ACC.f); + } + break; + + case NEONTYPE_VFREG: + armStorePtr(armQRegister(neonreg), &VU0.VF[arm64neon[neonreg].reg]); + break; + + default: + break; + } +} + +void _freeNEONreg(int neonreg) +{ + pxAssert(neonreg >= 0 && neonreg < NUM_ARM_NEON_REGS); + if (!arm64neon[neonreg].inuse) + return; + + if (arm64neon[neonreg].mode & MODE_WRITE) + _writebackNEONreg(neonreg); + + arm64neon[neonreg].inuse = 0; + arm64neon[neonreg].mode = 0; +} + +void _freeNEONregWithoutWriteback(int neonreg) +{ + pxAssert(neonreg >= 0 && neonreg < NUM_ARM_NEON_REGS); + arm64neon[neonreg].inuse = 0; + arm64neon[neonreg].mode = 0; +} + +void _freeNEONregs() +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse) + _freeNEONreg(i); + } +} + +void _flushNEONreg(int neonreg) +{ + if (arm64neon[neonreg].inuse && (arm64neon[neonreg].mode & MODE_WRITE)) + { + _writebackNEONreg(neonreg); + arm64neon[neonreg].mode &= ~MODE_WRITE; + arm64neon[neonreg].mode |= MODE_READ; + } +} + +void _flushNEONregs() +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + _flushNEONreg(i); +} + +void _addNeededFPtoNEONreg(int fpreg) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_FPREG && arm64neon[i].reg == fpreg) + arm64neon[i].needed = 1; + } +} + +void _addNeededFPACCtoNEONreg() +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_FPACC) + arm64neon[i].needed = 1; + } +} + +void _addNeededGPRtoNEONreg(int gprreg) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_GPRREG && arm64neon[i].reg == gprreg) + arm64neon[i].needed = 1; + } +} + +void _clearNeededNEONregs() +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].needed && arm64neon[i].type == NEONTYPE_TEMP) + _freeNEONreg(i); + arm64neon[i].needed = 0; + } +} + +void _deleteGPRtoNEONreg(int reg, int flush) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_GPRREG && arm64neon[i].reg == reg) + { + switch (flush) + { + case DELETE_REG_FREE: _freeNEONreg(i); break; + case DELETE_REG_FLUSH: + if (arm64neon[i].mode & MODE_WRITE) + { + _writebackNEONreg(i); + // Drop MODE_WRITE (keep MODE_READ) so a later + // _freeNEONreg won't store the same value again. + arm64neon[i].mode = (arm64neon[i].mode & ~MODE_WRITE) | MODE_READ; + } + break; + case DELETE_REG_FLUSH_AND_FREE: _freeNEONreg(i); break; + case DELETE_REG_FREE_NO_WRITEBACK: _freeNEONregWithoutWriteback(i); break; + } + return; + } + } +} + +void _deleteFPtoNEONreg(int reg, int flush) +{ + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_FPREG && arm64neon[i].reg == reg) + { + switch (flush) + { + case DELETE_REG_FREE: _freeNEONreg(i); break; + case DELETE_REG_FLUSH: + if (arm64neon[i].mode & MODE_WRITE) + { + _writebackNEONreg(i); + // Drop MODE_WRITE (keep MODE_READ) so a later + // _freeNEONreg won't store the same value again. + arm64neon[i].mode = (arm64neon[i].mode & ~MODE_WRITE) | MODE_READ; + } + break; + case DELETE_REG_FLUSH_AND_FREE: _freeNEONreg(i); break; + case DELETE_REG_FREE_NO_WRITEBACK: _freeNEONregWithoutWriteback(i); break; + } + return; + } + } +} + +void _reallocateNEONreg(int neonreg, int newtype, int newreg, int newmode, bool writeback) +{ + if (arm64neon[neonreg].inuse && writeback) + _writebackNEONreg(neonreg); + + arm64neon[neonreg].inuse = 1; + arm64neon[neonreg].type = newtype; + arm64neon[neonreg].reg = newreg; + arm64neon[neonreg].mode = newmode; + arm64neon[neonreg].needed = 1; + arm64neon[neonreg].counter = g_neonAllocCounter++; +} + +int _allocIfUsedGPRtoNEON(int gprreg, int mode) +{ + return EEINST_XMMUSEDTEST(gprreg) ? _allocGPRtoNEONreg(gprreg, mode) : -1; +} + +int _allocIfUsedFPUtoNEON(int fpureg, int mode) +{ + return FPUINST_USEDTEST(fpureg) ? _allocFPtoNEONreg(fpureg, mode) : -1; +} + +void _flushCOP2regs() +{ + // Flush any VU registers cached in host regs + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_VFREG) + _freeNEONreg(i); + } + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == ARM64TYPE_VIREG) + _freeArm64GPR(i); + } +} + +// Stubs for COP2 reserved register management +void mVUFreeCOP2GPR(int hostreg) +{ +} + +bool mVUIsReservedCOP2(int hostreg) +{ + return false; +} + +void mVUFreeCOP2NEONreg(int hostreg) +{ +} + +//////////////////////////////////////////////////////////////////////////////// +// Architecture-independent utility functions + +void _recClearInst(EEINST* pinst) +{ + std::memset(pinst, 0, sizeof(EEINST)); + std::memset(pinst->regs, EEINST_LIVE, sizeof(pinst->regs)); + std::memset(pinst->fpuregs, EEINST_LIVE, sizeof(pinst->fpuregs)); + std::memset(pinst->vfregs, EEINST_LIVE, sizeof(pinst->vfregs)); + std::memset(pinst->viregs, EEINST_LIVE, sizeof(pinst->viregs)); +} + +u32 _recIsRegReadOrWritten(EEINST* pinst, int size, u8 xmmtype, u8 reg) +{ + u32 inst = 1; + while (size-- > 0) + { + for (u32 i = 0; i < std::size(pinst->writeType); ++i) + { + if ((pinst->writeType[i] == xmmtype) && (pinst->writeReg[i] == reg)) + return inst; + } + for (u32 i = 0; i < std::size(pinst->readType); ++i) + { + if ((pinst->readType[i] == xmmtype) && (pinst->readReg[i] == reg)) + return inst; + } + ++inst; + pinst++; + } + return 0; +} + +void _recFillRegister(EEINST& pinst, int type, int reg, int write) +{ + if (write) + { + for (u32 i = 0; i < std::size(pinst.writeType); ++i) + { + if (pinst.writeType[i] == NEONTYPE_TEMP) + { + pinst.writeType[i] = type; + pinst.writeReg[i] = reg; + return; + } + } + pxAssume(false); + } + else + { + for (u32 i = 0; i < std::size(pinst.readType); ++i) + { + if (pinst.readType[i] == NEONTYPE_TEMP) + { + pinst.readType[i] = type; + pinst.readReg[i] = reg; + return; + } + } + pxAssume(false); + } +} diff --git a/pcsx2/arm64/iCore-arm64.h b/pcsx2/arm64/iCore-arm64.h new file mode 100644 index 0000000000..4332713de6 --- /dev/null +++ b/pcsx2/arm64/iCore-arm64.h @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +#pragma once + +#include "arm64/AsmHelpers.h" +#include "VUmicro.h" + +// ARM64 Register Allocator +// Mirrors the x86 allocator in x86/iCore.h but adapted for ARM64 register conventions. + +//#define RALOG(...) fprintf(stderr, __VA_ARGS__) +#define RALOG(...) + +//////////////////////////////////////////////////////////////////////////////// +// Shared Register Allocation Flags (same as x86 — shared with instruction codegen) + +#define MODE_READ 1 +#define MODE_WRITE 2 +#define MODE_CALLEESAVED 0x20 +#define MODE_COP2 0x40 + +#define PROCESS_EE_XMM 0x02 + +#define PROCESS_EE_S 0x04 +#define PROCESS_EE_T 0x08 +#define PROCESS_EE_D 0x10 + +#define PROCESS_EE_LO 0x40 +#define PROCESS_EE_HI 0x80 +#define PROCESS_EE_ACC 0x40 + +// Extract host register index from info bitmask. +// ARM64 needs 5 bits per field (registers 0-28), unlike x86 which uses 4 bits (0-15). +// Fields are packed into a 32-bit info word with 5-bit register indices. +// +// NOTE: EEREC_LO, EEREC_HI and EEREC_ACC intentionally decode the SAME field +// (bits 23..27). Five distinct 5-bit fields plus the presence flags overflow the +// 32-bit word, so LO/HI/ACC share one slot. This is only sound because no op needs +// two of them live simultaneously through the allocator: integer MULT/DIV/MADD and +// PMFHL load LO/HI directly from memory (bypassing the allocator), and ACC is used +// exclusively by FPU ops (never alongside LO/HI). eeRecompileCodeXMM asserts the one +// dangerous combination (LO+HI); an op needing both must bypass the allocator too. +#define EEREC_S (((info) >> 8) & 0x1f) +#define EEREC_T (((info) >> 13) & 0x1f) +#define EEREC_D (((info) >> 18) & 0x1f) +#define EEREC_LO (((info) >> 23) & 0x1f) +#define EEREC_HI (((info) >> 23) & 0x1f) +#define EEREC_ACC (((info) >> 23) & 0x1f) + +#define PROCESS_EE_SET_S(reg) (((reg) << 8) | PROCESS_EE_S) +#define PROCESS_EE_SET_T(reg) (((reg) << 13) | PROCESS_EE_T) +#define PROCESS_EE_SET_D(reg) (((reg) << 18) | PROCESS_EE_D) +#define PROCESS_EE_SET_LO(reg) (((reg) << 23) | PROCESS_EE_LO) +#define PROCESS_EE_SET_HI(reg) (((reg) << 23) | PROCESS_EE_HI) +#define PROCESS_EE_SET_ACC(reg) (((reg) << 23) | PROCESS_EE_ACC) + +#define PROCESS_CONSTS 1 +#define PROCESS_CONSTT 2 + +//////////////////////////////////////////////////////////////////////////////// +// NEON (128-bit) Register Allocation — equivalent to XMM on x86 + +enum xmminfo : u16 +{ + XMMINFO_READLO = 0x001, + XMMINFO_READHI = 0x002, + XMMINFO_WRITELO = 0x004, + XMMINFO_WRITEHI = 0x008, + XMMINFO_WRITED = 0x010, + XMMINFO_READD = 0x020, + XMMINFO_READS = 0x040, + XMMINFO_READT = 0x080, + XMMINFO_READACC = 0x200, + XMMINFO_WRITEACC = 0x400, + XMMINFO_WRITET = 0x800, + + XMMINFO_64BITOP = 0x1000, + XMMINFO_FORCEREGS = 0x2000, + XMMINFO_FORCEREGT = 0x4000, + XMMINFO_NORENAME = 0x8000 +}; + +//////////////////////////////////////////////////////////////////////////////// +// ARM64 GPR Register Allocation + +// Total number of ARM64 GPR registers we track (x0-x30 = 31) +static constexpr int NUM_ARM_GPR_REGS = 31; + +// Total number of ARM64 NEON registers we track (q0-q28, excluding q29-q31 scratch) +static constexpr int NUM_ARM_NEON_REGS = 29; + +enum arm64gprtype : u8 +{ + ARM64TYPE_TEMP = 0, + ARM64TYPE_GPR = 1, // EE GPR (lower 64 bits) + ARM64TYPE_FPRC = 2, // FPU control register + ARM64TYPE_VIREG = 3, // VU integer register + ARM64TYPE_PCWRITEBACK = 4, + ARM64TYPE_PSX = 5, // IOP GPR + ARM64TYPE_PSX_PCWRITEBACK = 6 +}; + +struct _arm64gprregs +{ + u8 inuse; + s8 reg; // guest register index + u8 mode; // MODE_READ / MODE_WRITE + u8 needed; // pinned for current instruction + u8 type; // ARM64TYPE_* + u16 counter; // LRU counter + u32 extra; // extra info (e.g., IOP constant regs) +}; + +// NEON register types — same values as XMM types for compatibility +#define NEONTYPE_TEMP 0 +#define NEONTYPE_GPRREG 1 // EE GPR (full 128 bits) +#define NEONTYPE_FPREG 6 // FPU register +#define NEONTYPE_FPACC 7 // FPU accumulator +#define NEONTYPE_VFREG 8 // VU VF register + +// x86 type aliases — used by shared analysis code (iR5900Analysis.cpp) +#define XMMTYPE_TEMP NEONTYPE_TEMP +#define XMMTYPE_GPRREG NEONTYPE_GPRREG +#define XMMTYPE_FPREG NEONTYPE_FPREG +#define XMMTYPE_VFREG NEONTYPE_VFREG +#define X86TYPE_VIREG ARM64TYPE_VIREG +#define X86TYPE_GPR ARM64TYPE_GPR + +// Register index aliases for analysis (same values as x86) +#define XMMGPR_LO NEONGPR_LO // 33 +#define XMMGPR_HI NEONGPR_HI // 32 +#define XMMFPU_ACC NEONFPU_ACC // 32 + +#define NEONGPR_LO 33 +#define NEONGPR_HI 32 +#define NEONFPU_ACC 32 + +enum : int +{ + DELETE_REG_FREE = 0, + DELETE_REG_FLUSH = 1, + DELETE_REG_FLUSH_AND_FREE = 2, + DELETE_REG_FREE_NO_WRITEBACK = 3 +}; + +struct _arm64neonregs +{ + u8 inuse; + s8 reg; // guest register index + u8 type; // NEONTYPE_* + u8 mode; // MODE_READ / MODE_WRITE + u8 needed; // pinned for current instruction + u16 counter; // LRU counter +}; + +//////////////////////////////////////////////////////////////////////////////// +// ARM64 GPR allocator functions + +extern _arm64gprregs arm64gprs[NUM_ARM_GPR_REGS], s_saveArm64GPRregs[NUM_ARM_GPR_REGS]; + +bool _isAllocatableArm64GPR(int armreg); +void _initArm64GPRregs(); +int _getFreeArm64GPR(int mode); +int _allocArm64GPR(int type, int reg, int mode); +int _checkArm64GPR(int type, int reg, int mode); +bool _hasArm64GPR(int type, int reg, int required_mode = 0); +void _addNeededArm64GPR(int type, int reg); +void _clearNeededArm64GPRregs(); +void _freeArm64GPR(int armreg); +void _freeArm64GPRWithoutWriteback(int armreg); +void _freeArm64GPRregs(); +void _flushArm64GPRregs(); +void _flushConstRegs(bool delete_const); +void _flushConstReg(int reg); +void _validateRegs(); +void _writebackArm64GPR(int armreg); + +void mVUFreeCOP2GPR(int hostreg); +bool mVUIsReservedCOP2(int hostreg); + +//////////////////////////////////////////////////////////////////////////////// +// ARM64 NEON allocator functions + +extern _arm64neonregs arm64neon[NUM_ARM_NEON_REGS], s_saveArm64NEONregs[NUM_ARM_NEON_REGS]; + +void _initArm64NEONregs(); +int _getFreeArm64NEON(u32 maxreg = NUM_ARM_NEON_REGS); +int _allocTempNEONreg(); +int _allocFPtoNEONreg(int fpreg, int mode); +int _allocGPRtoNEONreg(int gprreg, int mode); +int _allocFPACCtoNEONreg(int mode); +void _reallocateNEONreg(int neonreg, int newtype, int newreg, int newmode, bool writeback = true); +int _checkNEONreg(int type, int reg, int mode); +bool _hasNEONreg(int type, int reg, int required_mode = 0); +void _addNeededFPtoNEONreg(int fpreg); +void _addNeededFPACCtoNEONreg(); +void _addNeededGPRtoArm64GPR(int gprreg); +void _addNeededPSXtoArm64GPR(int gprreg); +void _addNeededGPRtoNEONreg(int gprreg); +void _clearNeededNEONregs(); +void _deleteGPRtoArm64GPR(int reg, int flush); +void _deletePSXtoArm64GPR(int reg, int flush); +void _deleteGPRtoNEONreg(int reg, int flush); +void _deleteFPtoNEONreg(int reg, int flush); +void _freeNEONreg(int neonreg); +void _freeNEONregWithoutWriteback(int neonreg); +void _freeNEONregs(); +void _writebackNEONreg(int neonreg); +int _allocVFtoNEONreg(int vfreg, int mode); +void mVUFreeCOP2NEONreg(int hostreg); +void _flushCOP2regs(); +void _flushNEONreg(int neonreg); +void _flushNEONregs(); + +//////////////////////////////////////////////////////////////////////////////// +// Instruction Info — architecture-independent, shared with x86 +// (EEINST, liveness analysis, etc.) + +#define EEINST_LIVE 1 +#define EEINST_LASTUSE 8 +#define EEINST_XMM 0x20 // keep name for compat — means "will use NEON/128-bit" +#define EEINST_USED 0x40 + +#define EEINST_COP2_DENORMALIZE_STATUS_FLAG 0x100 +#define EEINST_COP2_NORMALIZE_STATUS_FLAG 0x200 +#define EEINST_COP2_STATUS_FLAG 0x400 +#define EEINST_COP2_MAC_FLAG 0x800 +#define EEINST_COP2_CLIP_FLAG 0x1000 +#define EEINST_COP2_SYNC_VU0 0x2000 +#define EEINST_COP2_FINISH_VU0 0x4000 +#define EEINST_COP2_FLUSH_VU0_REGISTERS 0x8000 + +struct EEINST +{ + u16 info; + u8 regs[34]; // HI=32, LO=33 + u8 fpuregs[33]; // ACC=32 + u8 vfregs[34]; // ACC=32, I=33 + u8 viregs[16]; + + u8 writeType[3], writeReg[3]; + u8 readType[4], readReg[4]; +}; + +extern EEINST* g_pCurInstInfo; +extern void _recClearInst(EEINST* pinst); +extern u32 _recIsRegReadOrWritten(EEINST* pinst, int size, u8 xmmtype, u8 reg); +extern void _recFillRegister(EEINST& pinst, int type, int reg, int write); + +#define EE_WRITE_DEAD_VALUES 1 + +static __fi bool EEINST_USEDTEST(u32 reg) +{ + return (g_pCurInstInfo->regs[reg] & (EEINST_USED | EEINST_LASTUSE)) == EEINST_USED; +} + +static __fi bool EEINST_XMMUSEDTEST(u32 reg) +{ + return (g_pCurInstInfo->regs[reg] & (EEINST_USED | EEINST_XMM | EEINST_LASTUSE)) == (EEINST_USED | EEINST_XMM); +} + +static __fi bool EEINST_VFUSEDTEST(u32 reg) +{ + return (g_pCurInstInfo->vfregs[reg] & (EEINST_USED | EEINST_LASTUSE)) == EEINST_USED; +} + +static __fi bool EEINST_VIUSEDTEST(u32 reg) +{ + return (g_pCurInstInfo->viregs[reg] & (EEINST_USED | EEINST_LASTUSE)) == EEINST_USED; +} + +static __fi bool EEINST_LIVETEST(u32 reg) +{ + return EE_WRITE_DEAD_VALUES || ((g_pCurInstInfo->regs[reg] & EEINST_LIVE) != 0); +} + +static __fi bool EEINST_RENAMETEST(u32 reg) +{ + return (reg == 0 || !EEINST_USEDTEST(reg) || !EEINST_LIVETEST(reg)); +} + +static __fi bool FPUINST_ISLIVE(u32 reg) { return !!(g_pCurInstInfo->fpuregs[reg] & EEINST_LIVE); } +static __fi bool FPUINST_LASTUSE(u32 reg) { return !!(g_pCurInstInfo->fpuregs[reg] & EEINST_LASTUSE); } + +static __fi bool FPUINST_USEDTEST(u32 reg) +{ + return (g_pCurInstInfo->fpuregs[reg] & (EEINST_USED | EEINST_LASTUSE)) == EEINST_USED; +} + +static __fi bool FPUINST_LIVETEST(u32 reg) +{ + return EE_WRITE_DEAD_VALUES || FPUINST_ISLIVE(reg); +} + +static __fi bool FPUINST_RENAMETEST(u32 reg) +{ + return (!EEINST_USEDTEST(reg) || !EEINST_LIVETEST(reg)); +} + +extern u16 g_arm64AllocCounter; +extern u16 g_neonAllocCounter; + +// Allocates only if later instructions use this register +int _allocIfUsedGPRtoArm64(int gprreg, int mode); +int _allocIfUsedVItoArm64(int vireg, int mode); +int _allocIfUsedGPRtoNEON(int gprreg, int mode); +int _allocIfUsedFPUtoNEON(int fpureg, int mode); + +//////////////////////////////////////////////////////////////////////////////// +// Flush call parameters — same values as x86 for compatibility + +#define FLUSH_NONE 0x000 +#define FLUSH_CONSTANT_REGS 0x001 +#define FLUSH_FLUSH_XMM 0x002 // flush NEON regs (keep name for compat) +#define FLUSH_FREE_XMM 0x004 // flush + free NEON regs +#define FLUSH_ALL_X86 0x020 // flush ARM64 GPRs (keep name for compat) +#define FLUSH_FREE_TEMP_X86 0x040 // flush + free temp ARM64 GPRs +#define FLUSH_FREE_NONTEMP_X86 0x080 // free non-temp ARM64 GPRs +#define FLUSH_FREE_VU0 0x100 +#define FLUSH_PC 0x200 +#define FLUSH_CODE 0x800 + +#define FLUSH_EVERYTHING 0x1ff +#define FLUSH_INTERPRETER 0xfff +#define FLUSH_FULLVTLB 0x000 +#define FLUSH_NODESTROY (FLUSH_CONSTANT_REGS | FLUSH_FLUSH_XMM | FLUSH_ALL_X86) diff --git a/pcsx2/arm64/iFPU-arm64.cpp b/pcsx2/arm64/iFPU-arm64.cpp new file mode 100644 index 0000000000..c1a08c6d96 --- /dev/null +++ b/pcsx2/arm64/iFPU-arm64.cpp @@ -0,0 +1,924 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE FPU (COP1) Instruction Codegen — NEON-based +// Transfer ops (MFC1/MTC1/CFC1/CTC1): native with NEON allocation. +// Branch ops (BC1F/BC1T): native, read fprc[31] directly. +// Arithmetic ops: interpreter fallback (PS2 float clamping needed). + +#include "arm64/iR5900-arm64.h" + +#include + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { +namespace COP1 { + +namespace Interp = R5900::Interpreter::OpcodeImpl::COP1; + +#ifdef FORCE_INTERP_FPU +REC_FUNC(CFC1); +REC_FUNC(CTC1); +REC_FUNC(MFC1); +REC_FUNC(MTC1); +REC_SYS(BC1F); +REC_SYS(BC1T); +REC_SYS(BC1FL); +REC_SYS(BC1TL); +#else + +#define _Ft_ _Rt_ +#define _Fs_ _Rd_ +#define _Fd_ _Sa_ + +#define FPUflagC 0x00800000 +#define FPUflagI 0x00020000 +#define FPUflagD 0x00010000 +#define FPUflagSI 0x00000040 +#define FPUflagSD 0x00000020 + +//------------------------------------------------------------------ +// CFC1 — rt = fprc[fs] (read FPU control register) +//------------------------------------------------------------------ +void recCFC1() +{ + if (!_Rt_) return; + + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + + if (_Fs_ >= 16) + { + // FCR31: mask out always-zero bits, set always-one bits + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->And(RWSCRATCH, RWSCRATCH, 0x0083c078); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x01000001); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + } + else + { + // FCR0: read-only revision register + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[0]); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + } + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); +} + +//------------------------------------------------------------------ +// CTC1 — fprc[fs] = rt (write FPU control register) +//------------------------------------------------------------------ +void recCTC1() +{ + if (_Fs_ != 31) return; + + if (GPR_IS_CONST1(_Rt_)) + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + else + { + _deleteEEreg(_Rt_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rt_].UL[0]); + } + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[_Fs_]); +} + +//------------------------------------------------------------------ +// MFC1 — rt = sign_extend(fpr[fs]) (move 32-bit float to GPR) +//------------------------------------------------------------------ +void recMFC1() +{ + if (!_Rt_) return; + + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + + // FPR-side allocator coherence: fpr[fs] may be live in NEON (e.g. a + // preceding ADD_S wrote it, possibly MODE_WRITE-only). If it is already + // resident, read it straight from the host reg instead of flushing it to + // memory and reloading (store→load-forward stall on A53). + // MFC1 doesn't modify fpr[fs], so leave the allocator slot intact. Only + // the not-resident case falls back to the memory load. + const int fsreg = _checkNEONreg(NEONTYPE_FPREG, _Fs_, MODE_READ); + if (fsreg >= 0) + { + armAsm->Fmov(RWSCRATCH, armSRegister(fsreg)); + } + else + { + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fpr[_Fs_].UL); + } + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); +} + +//------------------------------------------------------------------ +// MTC1 — fpr[fs] = rt[31:0] (move GPR lower 32 bits to FPR) +//------------------------------------------------------------------ +void recMTC1() +{ + if (GPR_IS_CONST1(_Rt_)) + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + else + { + _deleteEEreg(_Rt_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rt_].UL[0]); + } + + // If fpr[fs] is already resident in NEON, write the new bits straight into + // the host reg and mark it dirty (MODE_WRITE), keeping it hot for a + // following FPU op; the block epilogue flushes the S-reg to fpr[fs].f. + // MTC1 overwrites fpr[fs] wholesale, so any prior MODE_WRITE-only value + // in the slot is dead and correctly discarded by overwriting lane 0. + // Not-resident → store to memory. + const int fsreg = _checkNEONreg(NEONTYPE_FPREG, _Fs_, MODE_WRITE); + if (fsreg >= 0) + { + armAsm->Fmov(armSRegister(fsreg), RWSCRATCH); + } + else + { + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fpr[_Fs_].UL); + } +} + +//------------------------------------------------------------------ +// BC1F / BC1T — branch on FPU condition flag +//------------------------------------------------------------------ + +// FPU branch setup: flush state and test fprc[31] condition flag. +// Emits conditional forward branch (skip label), matching EE branch pattern. +// bne=false: BC1F (skip if C set), bne=true: BC1T (skip if C clear) +static a64::Label* s_pBC1Label = nullptr; + +static void recSetBranchBC1(bool branchOnTrue) +{ + _eeFlushAllDirty(); + + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + + // FPUflagC (0x00800000) is a single fixed bit (23), so the Tst+B.cond pair + // collapses to one test-bit-and-branch (Tbnz/Tbz). The forward branch + // skips the delay slot on the not-taken edge. + static_assert(FPUflagC == (1u << 23), "FPUflagC must be a single bit for Tbz/Tbnz"); + s_pBC1Label = new a64::Label(); + if (branchOnTrue) + armAsm->Tbz(RWSCRATCH, 23, s_pBC1Label); // BC1T: skip taken if C clear + else + armAsm->Tbnz(RWSCRATCH, 23, s_pBC1Label); // BC1F: skip taken if C set +} + +static void recBindBC1Label() +{ + armAsm->Bind(s_pBC1Label); + delete s_pBC1Label; + s_pBC1Label = nullptr; +} + +void recBC1F() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + const bool swap = TrySwapDelaySlot(0, 0, 0, true); + recSetBranchBC1(false); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBC1Label(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +void recBC1T() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + const bool swap = TrySwapDelaySlot(0, 0, 0, true); + recSetBranchBC1(true); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBC1Label(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +void recBC1FL() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + recSetBranchBC1(false); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBC1Label(); + LoadBranchState(); + SetBranchImm(pc); +} + +void recBC1TL() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + recSetBranchBC1(true); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBC1Label(); + LoadBranchState(); + SetBranchImm(pc); +} + +#undef _Ft_ +#undef _Fs_ +#undef _Fd_ + +#endif // !FORCE_INTERP_FPU + +//------------------------------------------------------------------ +// FPU Arithmetic — lightweight interpreter call +// FPU ops only touch fpuRegs memory, not cpuRegs.GPR. EE GPRs are +// in callee-saved NEON registers (q8-q15) that survive C calls. +// Only flush PC/code for exception handling — skip NEON flush. +//------------------------------------------------------------------ + +static void recFPUCall(void (*func)()) +{ + // Flush PC and code (needed if FPU op triggers an exception) + armAsm->Mov(RWSCRATCH, pc); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.pc)); + + armAsm->Mov(RWSCRATCH, cpuRegs.code); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.code)); + + // FPU allocator coherence: the interpreter reads fpuRegs.fpr[] and + // fpuRegs.ACC directly, and those values can live in NEON slots + // (MODE_WRITE-only) until block-end flush — so writeback every + // FPREG/FPACC slot before the call. EE GPRs in callee-saved q8-q15 + // survive (FPU interpreter doesn't touch cpuRegs.GPR), so iFlushCall's + // full eviction is not needed here. + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && + (arm64neon[i].type == NEONTYPE_FPREG || arm64neon[i].type == NEONTYPE_FPACC)) + { + _freeNEONreg(i); + } + } + + armEmitCall((void*)func); +} + + +#define _Ft_ _Rt_ +#define _Fs_ _Rd_ +#define _Fd_ _Sa_ + +// PS2 FPU max representable float (no infinity) — exactly ±FLT_MAX +// under IEEE-754 single precision: 0x7F7FFFFF / 0xFF7FFFFF. +// Load-bearing for the EE/mVU dispatcher prologues that park these bit +// patterns in s8/s9 — keep this assert if the type of FLT_MAX ever changes. +static_assert(FLT_MAX == 3.40282346638528859811704183484516925e+38f, + "FLT_MAX must be the IEEE-754 0x7F7FFFFF bit pattern (PS2 FPU clamp upper bound)."); + +// Clamp a result in `fpr` to PS2 float range (no inf/nan). +// +// Branchless Fminnm/Fmaxnm: the Number variants are NaN-eating (matching +// x86 MINSS/MAXSS), so NaN routes through Fminnm to +max and Fmaxnm +// passes it through. Both ±Inf get clamped to ±max. +// +// The ±FLT_MAX bounds live in callee-saved s8/s9 — loaded once at JIT +// session entry in `_DynGen_EnterRecompiledCode` and held across every +// armEmitCall via AAPCS64. v8/v9 are excluded from the NEON allocator +// pool so no codegen path can clobber them. 2 host insns per clamp. +// +// NaN sign is not preserved (matches x86 fpuFloat / ClampValues; the +// PS2 FPU has no NaN concept so this is design-correct). +static void fpuClampResult(const a64::VRegister& fpr) +{ + armAsm->Fminnm(fpr, fpr, a64::s8); + armAsm->Fmaxnm(fpr, fpr, a64::s9); +} + +// One-sided positive clamp for results that are statically non-negative +// (ABS.S). Fabs clears the sign bit, so the value is always >= 0 (NaN -> +// +0x7FFFFFFF), which makes the lower Fmaxnm(-FLT_MAX) of fpuClampResult dead. +// Like x86 recABS_S_xmm, only the positive clamp is needed (the ABS result is +// never negative), so only one Fminnm vs 0x7f7fffff is emitted. +// Saves one NEON insn per ABS.S. +static void fpuClampResultPositive(const a64::VRegister& fpr) +{ + armAsm->Fminnm(fpr, fpr, a64::s8); +} + +// Sign-preserving operand clamp for FPU comparisons (C.cond.S). +// +// Mirrors the x86 JIT's fpuFloat3 (PMIN.SD vs 0x7f7fffff then PMIN.UD vs +// 0xff7fffff): +NaN->+fMax, -NaN->-fMax, +Inf->+fMax, -Inf->-fMax. The PS2 +// FPU has no Inf/NaN concept, so both compare operands must be clamped first; +// a raw Fcmp on an unclamped NaN would go unordered (all-false) where the PS2 +// wants an ordered compare against ±FLT_MAX. +// +// Integer SMIN/UMIN preserve the sign bit — unlike fpuClampResult's +// Fminnm/Fmaxnm, which fold every NaN to +fMax and would mis-order -NaN. +// s8/s9 already hold the 0x7f7fffff / 0xff7fffff bit patterns. Only lane 0 +// (the scalar S reg) is consumed by the following Fcmp, so the upper V4S +// lanes are don't-care. (Denormal flush-to-zero is intentionally omitted to +// match fpuFloat3; that is a pre-existing shared JIT-vs-interp behavior.) +static void fpuClampCompareOperand(const a64::VRegister& s) +{ + const a64::VRegister v(s.GetCode(), a64::kQRegSize); + armAsm->Smin(v.V4S(), v.V4S(), a64::VRegister(8, a64::kQRegSize).V4S()); + armAsm->Umin(v.V4S(), v.V4S(), a64::VRegister(9, a64::kQRegSize).V4S()); +} + +// Source-operand clamp for the FPU arithmetic family, gated on +// CHECK_FPU_EXTRA_OVERFLOW (per-game GameDB clampMode>=2). When enabled, the +// PS2 FPU recs clamp each fpr source to ±fMax *before* the op — matching the +// interpreter (which routes every operand through fpuDouble) and x86 +// recCommutativeOp/recMADDtemp (fpuFloat2 under the same gate). This catches +// Inf*0 -> NaN / (+Inf)+(-Inf) -> NaN poison where an fpr was filled with raw +// Inf/NaN bits via MOV.S/LWC1/MTC1; without it the op produces a NaN that +// the result clamp folds to +fMax, diverging from the interpreter's +// clamp-then-compute (e.g. fMax*0 = 0). +// +// Copies into `scratch` rather than mutating the allocator-resident source +// (vs x86's in-place fpuFloat2) so a later read of the same fpr in this block +// still sees the unclamped value. Sign-preserving (fpuClampCompareOperand), +// so -Inf -> -fMax. Only fpr-sourced operands (S/T) need this; ACC is written +// only by always-clamping acc-ops and can never be poisoned. In the default +// config (flag off) this emits nothing and returns the source reg. +static a64::VRegister fpuClampInput(const a64::VRegister& src, const a64::VRegister& scratch) +{ + if (!CHECK_FPU_EXTRA_OVERFLOW) + return src; + armAsm->Fmov(scratch, src); + fpuClampCompareOperand(scratch); + return scratch; +} + +// FpuMulHack (Tales of Destiny Remake gamefix, EmuConfig.Gamefixes.FpuMulHack). +// x86 routes every FPU multiply (MUL/MULA/MADD/MSUB) through FPU_MUL, which — +// when the gamefix is on — patches the single specific product 0.25 * (π/2) +// (0x3e800000 * 0x40490fdb) to 0x3f490fda so the game stops hanging in one +// late-game room. Emit `dst = (hit) ? 0x3f490fda : s*t`; callers clamp/accumulate +// dst as they normally would (the magic value is an ordinary small float, so a +// following fpuClampResult is a no-op). In the default config (gamefix off) this +// is a bare Fmul — zero added cost. +static void emitFpuMul(const a64::VRegister& dst, const a64::VRegister& s, const a64::VRegister& t) +{ + if (!CHECK_FPUMULHACK) + { + armAsm->Fmul(dst, s, t); + return; + } + + a64::Label noHack, done; + armAsm->Fmov(RWARG1, s); + armAsm->Fmov(RWARG2, t); + armAsm->Mov(RWSCRATCH, 0x3e800000); + armAsm->Cmp(RWARG1, RWSCRATCH); + armAsm->B(&noHack, a64::ne); + armAsm->Mov(RWSCRATCH, 0x40490fdb); + armAsm->Cmp(RWARG2, RWSCRATCH); + armAsm->B(&noHack, a64::ne); + armAsm->Mov(RWSCRATCH, 0x3f490fda); + armAsm->Fmov(dst, RWSCRATCH); + armAsm->B(&done); + armAsm->Bind(&noHack); + armAsm->Fmul(dst, s, t); + armAsm->Bind(&done); +} + +//------------------------------------------------------------------ +// Simple FPU ops — no clamping needed +//------------------------------------------------------------------ + +static void recMOV_S_xmm(int info) +{ + // MOV.S is a raw bit-copy (PS2 FPR[fd] = FPR[fs]); no clamp/NaN logic. + // Skip the emit entirely when fd and fs alias the same host reg (guest + // fs==fd): the allocator hands back EEREC_D==EEREC_S and the Fmov would be + // an identity self-move. + if (EEREC_D != EEREC_S) + armAsm->Fmov(armSRegister(EEREC_D), armSRegister(EEREC_S)); +} + +void recMOV_S() +{ + eeFPURecompileCode(recMOV_S_xmm, Interp::MOV_S, + XMMINFO_WRITED | XMMINFO_READS); +} + +static void recABS_S_xmm(int info) +{ + armAsm->Fabs(armSRegister(EEREC_D), armSRegister(EEREC_S)); + // ABS output is always non-negative -> one-sided positive clamp. + fpuClampResultPositive(armSRegister(EEREC_D)); +} + +void recABS_S() +{ + eeFPURecompileCode(recABS_S_xmm, Interp::ABS_S, + XMMINFO_WRITED | XMMINFO_READS); +} + +static void recNEG_S_xmm(int info) +{ + armAsm->Fneg(armSRegister(EEREC_D), armSRegister(EEREC_S)); + fpuClampResult(armSRegister(EEREC_D)); +} + +void recNEG_S() +{ + eeFPURecompileCode(recNEG_S_xmm, Interp::NEG_S, + XMMINFO_WRITED | XMMINFO_READS); +} + +//------------------------------------------------------------------ +// FPU Comparisons — set/clear fprc[31] condition bit +//------------------------------------------------------------------ + +void recC_F() +{ + // Always false — clear condition bit + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Mov(RWARG1, FPUflagC); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG1); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); +} + +void recC_EQ() +{ + _deleteFPtoNEONreg(_Fs_, DELETE_REG_FLUSH_AND_FREE); + _deleteFPtoNEONreg(_Ft_, DELETE_REG_FLUSH_AND_FREE); + armLoadEERegPtr(RSSCRATCH, &fpuRegs.fpr[_Fs_].f); + armLoadEERegPtr(RSSCRATCH2, &fpuRegs.fpr[_Ft_].f); + fpuClampCompareOperand(RSSCRATCH); + fpuClampCompareOperand(RSSCRATCH2); + armAsm->Fcmp(RSSCRATCH, RSSCRATCH2); + armAsm->Mov(a64::w0, 0); + armAsm->Cset(a64::w0, a64::eq); + // Set or clear FPUflagC based on result (w0 holds cset result, don't clobber) + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Mov(RWARG2, FPUflagC); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::Operand(a64::w0, a64::LSL, 23)); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); +} + +void recC_LT() +{ + _deleteFPtoNEONreg(_Fs_, DELETE_REG_FLUSH_AND_FREE); + _deleteFPtoNEONreg(_Ft_, DELETE_REG_FLUSH_AND_FREE); + armLoadEERegPtr(RSSCRATCH, &fpuRegs.fpr[_Fs_].f); + armLoadEERegPtr(RSSCRATCH2, &fpuRegs.fpr[_Ft_].f); + fpuClampCompareOperand(RSSCRATCH); + fpuClampCompareOperand(RSSCRATCH2); + armAsm->Fcmp(RSSCRATCH, RSSCRATCH2); + armAsm->Mov(a64::w0, 0); + armAsm->Cset(a64::w0, a64::lt); + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Mov(RWARG2, FPUflagC); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::Operand(a64::w0, a64::LSL, 23)); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); +} + +void recC_LE() +{ + _deleteFPtoNEONreg(_Fs_, DELETE_REG_FLUSH_AND_FREE); + _deleteFPtoNEONreg(_Ft_, DELETE_REG_FLUSH_AND_FREE); + armLoadEERegPtr(RSSCRATCH, &fpuRegs.fpr[_Fs_].f); + armLoadEERegPtr(RSSCRATCH2, &fpuRegs.fpr[_Ft_].f); + fpuClampCompareOperand(RSSCRATCH); + fpuClampCompareOperand(RSSCRATCH2); + armAsm->Fcmp(RSSCRATCH, RSSCRATCH2); + armAsm->Mov(a64::w0, 0); + armAsm->Cset(a64::w0, a64::le); + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Mov(RWARG2, FPUflagC); armAsm->Bic(RWSCRATCH, RWSCRATCH, RWARG2); + armAsm->Orr(RWSCRATCH, RWSCRATCH, a64::Operand(a64::w0, a64::LSL, 23)); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); +} + +//------------------------------------------------------------------ +// FPU Arithmetic — native with PS2 clamping (no inf/nan) +//------------------------------------------------------------------ + +// "Full" / DOUBLE-precision emitters (iFPUd-arm64.cpp), selected per-op when +// CHECK_FPU_FULL (GameDB eeClampMode:3). Default config uses the fast paths below. +namespace DOUBLE { +void recADD_S_xmm(int info); +void recSUB_S_xmm(int info); +void recADDA_S_xmm(int info); +void recSUBA_S_xmm(int info); +void recMUL_S_xmm(int info); +void recMULA_S_xmm(int info); +void recMADD_S_xmm(int info); +void recMSUB_S_xmm(int info); +void recMADDA_S_xmm(int info); +void recMSUBA_S_xmm(int info); +} // namespace DOUBLE + +static void recADD_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + armAsm->Fadd(armSRegister(EEREC_D), s, t); + fpuClampResult(armSRegister(EEREC_D)); +} + +void recADD_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recADD_S_xmm : recADD_S_xmm, Interp::ADD_S, + XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); +} + +static void recSUB_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + armAsm->Fsub(armSRegister(EEREC_D), s, t); + fpuClampResult(armSRegister(EEREC_D)); +} + +void recSUB_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recSUB_S_xmm : recSUB_S_xmm, Interp::SUB_S, + XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); +} + +static void recMUL_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + emitFpuMul(armSRegister(EEREC_D), s, t); + fpuClampResult(armSRegister(EEREC_D)); +} + +void recMUL_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recMUL_S_xmm : recMUL_S_xmm, Interp::MUL_S, + XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); +} + +// Emit: ldr x9, [addr]; msr FPCR, x9 — load a 64-bit FPCR bitmask from `addr`. +// The PS2 FPU divides/sqrts in round-to-nearest while ADD/MUL round toward zero, +// so DIV must briefly swap FPCR to FPUDivFPCR and restore FPUFPCR after (mirrors +// x86 recDIV_S_xmm's xLDMXCSR pair). Uses x8/x9 as scratch. +static void emitLoadFPCR(const void* addr) +{ + armMoveAddressToReg(a64::x8, const_cast(addr)); + armAsm->Ldr(a64::x9, a64::MemOperand(a64::x8)); + armAsm->Msr(a64::FPCR, a64::x9); +} + +// Native DIV.S — port of x86 recDIVhelper1 (CHECK_FPU_EXTRA_FLAGS is always on) +// + the recDIV_S_xmm FPCR round-mode swap. Matches interp DIV_S / +// checkDivideByZero: +// - divisor == 0 (exp field 0; FZ in FPCR flushes denormals so the float +// compare catches them too): result = sign(Fs^Ft) | 0x7f7fffff (±fMax), +// and set I|SI for 0/0, D|SD for x/0; +// - otherwise native Fdiv (round-to-nearest) then ±fMax result clamp. +// I|D are cleared first to match the divide-by-zero result-shape and sticky +// flag semantics of the interpreter. +static void recDIV_S_xmm(int info) +{ + const bool swapFpcr = EmuConfig.Cpu.FPUFPCR.bitmask != EmuConfig.Cpu.FPUDivFPCR.bitmask; + if (swapFpcr) + emitLoadFPCR(&EmuConfig.Cpu.FPUDivFPCR.bitmask); + + // Copy both operands into temps: EEREC_D may alias EEREC_S/EEREC_T, and the + // div-by-zero path needs the raw (pre-clamp) dividend/divisor sign bits. + const int dreg = _allocTempNEONreg(); + const int treg = _allocTempNEONreg(); + armAsm->Fmov(armSRegister(dreg), armSRegister(EEREC_S)); + armAsm->Fmov(armSRegister(treg), armSRegister(EEREC_T)); + + // Clear I|D. + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Bic(RWSCRATCH, RWSCRATCH, FPUflagI | FPUflagD); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + + a64::Label normal, setMax, xDiv, end; + + armAsm->Fcmp(armSRegister(treg), 0.0); + armAsm->B(&normal, a64::ne); // divisor != 0 → normal divide (unordered too) + + // Divisor is zero: distinguish 0/0 (I|SI) from x/0 (D|SD). + armAsm->Fcmp(armSRegister(dreg), 0.0); + armAsm->B(&xDiv, a64::ne); + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Orr(RWSCRATCH, RWSCRATCH, FPUflagI | FPUflagSI); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->B(&setMax); + armAsm->Bind(&xDiv); + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Orr(RWSCRATCH, RWSCRATCH, FPUflagD | FPUflagSD); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + + armAsm->Bind(&setMax); + // result = sign(Fs ^ Ft) | 0x7f7fffff + armAsm->Fmov(RWARG1, armSRegister(dreg)); + armAsm->Fmov(RWARG2, armSRegister(treg)); + armAsm->Eor(RWARG1, RWARG1, RWARG2); + armAsm->And(RWARG1, RWARG1, 0x80000000); + armAsm->Orr(RWARG1, RWARG1, 0x7f7fffff); + armAsm->Fmov(armSRegister(EEREC_D), RWARG1); + armAsm->B(&end); + + armAsm->Bind(&normal); + if (CHECK_FPU_EXTRA_OVERFLOW) + { + fpuClampCompareOperand(armSRegister(dreg)); + fpuClampCompareOperand(armSRegister(treg)); + } + armAsm->Fdiv(armSRegister(EEREC_D), armSRegister(dreg), armSRegister(treg)); + fpuClampResult(armSRegister(EEREC_D)); + + armAsm->Bind(&end); + + _freeNEONreg(dreg); + _freeNEONreg(treg); + + if (swapFpcr) + emitLoadFPCR(&EmuConfig.Cpu.FPUFPCR.bitmask); +} + +void recDIV_S() +{ + eeFPURecompileCode(recDIV_S_xmm, Interp::DIV_S, + XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); +} + +static void recSQRT_S_xmm(int info) +{ + const a64::VRegister ft = armSRegister(EEREC_T); + + // PS2 SQRT.S rounds to nearest regardless of the configured FCR31 rounding + // mode (same hardware quirk as DIV.S — see recDIV_S_xmm + the emitLoadFPCR + // comment). The EE rec runs under host FPCR = FPUFPCR (ChopZero by default), + // so swap to the nearest-rounding FPUDivFPCR around the Fsqrt and restore + // FPUFPCR after. Mirrors x86 recSQRT_S_xmm (iFPU.cpp:1745-1782). + const bool swapFpcr = EmuConfig.Cpu.FPUFPCR.bitmask != EmuConfig.Cpu.FPUDivFPCR.bitmask; + if (swapFpcr) + emitLoadFPCR(&EmuConfig.Cpu.FPUDivFPCR.bitmask); + + // PS2 SQRT.S flag handling (interp SQRT_S, FPU.cpp; CHECK_FPU_EXTRA_FLAGS + // is always on): clear I|D unconditionally, then set I|SI when Ft is a + // negative *non-zero* value (exp field nonzero AND sign bit set). The + // ±0 / denormal-as-zero case (exp field == 0) sets no flag. Read the Ft + // bits before Fabs clobbers EEREC_D, which may alias EEREC_T. + armAsm->Fmov(RWARG1, ft); + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Bic(RWSCRATCH, RWSCRATCH, FPUflagI | FPUflagD); + a64::Label skipFlag; + armAsm->Tst(RWARG1, 0x7F800000); // exp field + armAsm->B(&skipFlag, a64::eq); // ±0/denorm → no flag + armAsm->Tbz(RWARG1, 31, &skipFlag); // positive → no flag + armAsm->Orr(RWSCRATCH, RWSCRATCH, FPUflagI | FPUflagSI); + armAsm->Bind(&skipFlag); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + + // PS2 takes sqrt of |ft| → Fabs first. + armAsm->Fabs(armSRegister(EEREC_D), ft); + armAsm->Fsqrt(armSRegister(EEREC_D), armSRegister(EEREC_D)); + fpuClampResult(armSRegister(EEREC_D)); + + if (swapFpcr) + emitLoadFPCR(&EmuConfig.Cpu.FPUFPCR.bitmask); +} + +void recSQRT_S() +{ + eeFPURecompileCode(recSQRT_S_xmm, Interp::SQRT_S, + XMMINFO_WRITED | XMMINFO_READT); +} + +void recRSQRT_S() +{ + // Defer to the interpreter: interp RSQRT_S (FPU.cpp) sets D|SD when Ft + // (divisor) is zero and I|SI when Ft is negative, and its Ft==0 branch + // returns ±posFmax keyed off the Ft sign (not Fs) — neither the sticky + // flags nor that result shape are reproducible by a raw Fdiv. RSQRT is + // rare, so the interpreter call is the lowest-risk match and keeps emitted + // code small. Same shape as recDIV_S. + recFPUCall(Interp::RSQRT_S); +} + +// PS2 FPU has no NaN concept — match x86 MAXSS/MINSS NaN-eating semantics +// with Fmaxnm/Fminnm (Fmax/Fmin IEEE-propagate NaN, same trap as mVUclamp1). +// No clamp needed: MAX/MIN cannot widen finite inputs. +static void recMAX_S_xmm(int info) +{ + armAsm->Fmaxnm(armSRegister(EEREC_D), armSRegister(EEREC_S), armSRegister(EEREC_T)); +} + +void recMAX_S() +{ + eeFPURecompileCode(recMAX_S_xmm, Interp::MAX_S, + XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); +} + +static void recMIN_S_xmm(int info) +{ + armAsm->Fminnm(armSRegister(EEREC_D), armSRegister(EEREC_S), armSRegister(EEREC_T)); +} + +void recMIN_S() +{ + eeFPURecompileCode(recMIN_S_xmm, Interp::MIN_S, + XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); +} + +//------------------------------------------------------------------ +// FPU Accumulator ops — ACC = fs OP ft, then fd = ACC OP fs2 +//------------------------------------------------------------------ + +static void recADDA_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + armAsm->Fadd(armSRegister(EEREC_ACC), s, t); + fpuClampResult(armSRegister(EEREC_ACC)); +} + +void recADDA_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recADDA_S_xmm : recADDA_S_xmm, Interp::ADDA_S, + XMMINFO_WRITEACC | XMMINFO_READS | XMMINFO_READT); +} + +static void recSUBA_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + armAsm->Fsub(armSRegister(EEREC_ACC), s, t); + fpuClampResult(armSRegister(EEREC_ACC)); +} + +void recSUBA_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recSUBA_S_xmm : recSUBA_S_xmm, Interp::SUBA_S, + XMMINFO_WRITEACC | XMMINFO_READS | XMMINFO_READT); +} + +static void recMULA_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + emitFpuMul(armSRegister(EEREC_ACC), s, t); + fpuClampResult(armSRegister(EEREC_ACC)); +} + +void recMULA_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recMULA_S_xmm : recMULA_S_xmm, Interp::MULA_S, + XMMINFO_WRITEACC | XMMINFO_READS | XMMINFO_READT); +} + +// fd = ACC + fs * ft. PS2 ISA mandates two separate roundings (mul then +// add), so don't fuse into FMA. RSSCRATCH (s30) is the non-pool scratch +// for the intermediate product — leaves EEREC_S/T allocator-resident. +static void recMADD_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + emitFpuMul(RSSCRATCH, s, t); + fpuClampResult(RSSCRATCH); + armAsm->Fadd(armSRegister(EEREC_D), armSRegister(EEREC_ACC), RSSCRATCH); + fpuClampResult(armSRegister(EEREC_D)); +} + +void recMADD_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recMADD_S_xmm : recMADD_S_xmm, Interp::MADD_S, + XMMINFO_WRITED | XMMINFO_READACC | XMMINFO_READS | XMMINFO_READT); +} + +// fd = ACC - fs * ft +static void recMSUB_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + emitFpuMul(RSSCRATCH, s, t); + fpuClampResult(RSSCRATCH); + armAsm->Fsub(armSRegister(EEREC_D), armSRegister(EEREC_ACC), RSSCRATCH); + fpuClampResult(armSRegister(EEREC_D)); +} + +void recMSUB_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recMSUB_S_xmm : recMSUB_S_xmm, Interp::MSUB_S, + XMMINFO_WRITED | XMMINFO_READACC | XMMINFO_READS | XMMINFO_READT); +} + +// ACC = ACC + fs * ft. Unlike MADD_S, interp MADDA_S (FPU.cpp) adds the raw +// fs*ft product without routing it through fpuDouble — only the final ACC is +// overflow-clamped. So do NOT clamp the intermediate product here, else an +// overflowing product clamped to +-fMax cancels an opposite-signed ACC instead +// of overflowing the accumulate. +static void recMADDA_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + emitFpuMul(RSSCRATCH, s, t); + armAsm->Fadd(armSRegister(EEREC_ACC), armSRegister(EEREC_ACC), RSSCRATCH); + fpuClampResult(armSRegister(EEREC_ACC)); +} + +void recMADDA_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recMADDA_S_xmm : recMADDA_S_xmm, Interp::MADDA_S, + XMMINFO_WRITEACC | XMMINFO_READACC | XMMINFO_READS | XMMINFO_READT); +} + +// ACC = ACC - fs * ft. Same as MADDA_S: interp MSUBA_S does not clamp the +// intermediate product, only the final ACC. +static void recMSUBA_S_xmm(int info) +{ + const a64::VRegister s = fpuClampInput(armSRegister(EEREC_S), RSSCRATCH); + const a64::VRegister t = fpuClampInput(armSRegister(EEREC_T), RSSCRATCH2); + emitFpuMul(RSSCRATCH, s, t); + armAsm->Fsub(armSRegister(EEREC_ACC), armSRegister(EEREC_ACC), RSSCRATCH); + fpuClampResult(armSRegister(EEREC_ACC)); +} + +void recMSUBA_S() +{ + eeFPURecompileCode(CHECK_FPU_FULL ? DOUBLE::recMSUBA_S_xmm : recMSUBA_S_xmm, Interp::MSUBA_S, + XMMINFO_WRITEACC | XMMINFO_READACC | XMMINFO_READS | XMMINFO_READT); +} + +// CVT.S: fd = (float)int_bits_of(fpr[fs]) +static void recCVT_S_xmm(int info) +{ + armAsm->Fmov(RWSCRATCH, armSRegister(EEREC_S)); + armAsm->Scvtf(armSRegister(EEREC_D), RWSCRATCH); +} + +void recCVT_S() +{ + eeFPURecompileCode(recCVT_S_xmm, Interp::CVT_S, + XMMINFO_WRITED | XMMINFO_READS); +} + +// CVT.W: fd_bits = (int32_t)fpr[fs] truncating toward zero. +// PS2 clamps overflow to INT32_MAX/MIN — ARM64 Fcvtzs saturates by default, +// matching interp for the finite-overflow and ±Inf cases. The one divergence +// is NaN: ARM Fcvtzs yields 0, but the PS2 (interp CVT_W, FPU.cpp) saturates +// NaN by sign — positive NaN → 0x7fffffff, negative NaN → 0x80000000. Fix up +// the NaN case only (cold branch over the source-sign select). +static void recCVT_W_xmm(int info) +{ + const a64::VRegister fs = armSRegister(EEREC_S); + armAsm->Fcvtzs(RWSCRATCH, fs); + a64::Label done; + armAsm->Fcmp(fs, fs); // NaN → unordered (V set) + armAsm->B(&done, a64::vc); // ordered → keep Fcvtzs result + armAsm->Fmov(RWARG1, fs); // NaN: broadcast source sign + armAsm->Asr(RWARG1, RWARG1, 31); // 0 if +, 0xFFFFFFFF if - + armAsm->Eor(RWSCRATCH, RWARG1, 0x7fffffff); // + → 0x7fffffff, − → 0x80000000 + armAsm->Bind(&done); + armAsm->Fmov(armSRegister(EEREC_D), RWSCRATCH); +} + +void recCVT_W() +{ + eeFPURecompileCode(recCVT_W_xmm, Interp::CVT_W, + XMMINFO_WRITED | XMMINFO_READS); +} + +#undef _Ft_ +#undef _Fs_ +#undef _Fd_ + +} // namespace COP1 +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iFPUd-arm64.cpp b/pcsx2/arm64/iFPUd-arm64.cpp new file mode 100644 index 0000000000..8bb723f19e --- /dev/null +++ b/pcsx2/arm64/iFPUd-arm64.cpp @@ -0,0 +1,412 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE FPU (COP1) — "Full" / DOUBLE-precision codegen. +// +// This is the arm64 port of pcsx2/x86/iFPUd.cpp: the PS2-accurate FPU that +// widens each single to IEEE double, performs the op in double, then narrows +// back to a PS2 single with the hardware's overflow/underflow/clamp semantics. +// It is selected only when CHECK_FPU_FULL (EmuConfig.Cpu.Recompiler.fpuFullMode, +// the GameDB `eeClampMode:3` path — FFX, Max Payne, Dark Cloud 2, Klonoa 2 …). +// Default config runs the single-precision fast path in iFPU-arm64.cpp. +// +// The algorithm is translated from the x86 semantics; the codegen follows the +// iFPU-arm64.cpp idioms (scalar Fcvt, GPR bit-twiddle via Fmov, the +// armLoadEERegPtr fprc[31]/ACCflag accessors). The shared interpreter +// (FPU.cpp fpuDouble) has no double path, so this codegen has no interpreter +// counterpart. + +#include "arm64/iR5900-arm64.h" + +#include + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { +namespace COP1 { +namespace DOUBLE { + +#define _Ft_ _Rt_ +#define _Fs_ _Rd_ +#define _Fd_ _Sa_ + +#define FPUflagO 0x00008000 +#define FPUflagU 0x00004000 +#define FPUflagSO 0x00000010 +#define FPUflagSU 0x00000008 + +// ---- PS2 single -> IEEE double -------------------------------------------- +// +// A PS2 single with exponent field 0xff is a *normal* large number (1.m * 2^128), +// but IEEE reads exp 0xff as Inf/NaN — so a plain cvtss2sd would corrupt it. +// For those (and only those) lower the exponent by one in the single domain, +// widen exactly, then raise the exponent by one in the double domain. Mirrors +// x86 ToDouble (xPSUB.D one_exp / xCVTSS2SD / xPADD.Q dbl_one_exp). +// +// Operates in place on temp NEON reg `idx`: reads the S lane, writes the D lane. +static void ToDouble(int idx) +{ + const a64::VRegister s = armSRegister(idx); + const a64::VRegister d = armDRegister(idx); + + a64::Label simple, done; + armAsm->Fmov(RWSCRATCH, s); + armAsm->And(RWARG1, RWSCRATCH, 0x7f800000); + armAsm->Cmp(RWARG1, 0x7f800000); + armAsm->B(&simple, a64::ne); + + // Complex: exp field == 0xff (Inf/NaN to IEEE, finite to PS2). + armAsm->Sub(RWSCRATCH, RWSCRATCH, 0x00800000); // lower exponent by one (single) + armAsm->Fmov(s, RWSCRATCH); + armAsm->Fcvt(d, s); // cvtss2sd (now finite) + armAsm->Fmov(RXSCRATCH, d); + armAsm->Mov(RXARG1, static_cast(1) << 52); // dbl_one_exp + armAsm->Add(RXSCRATCH, RXSCRATCH, RXARG1); // raise exponent by one (double) + armAsm->Fmov(d, RXSCRATCH); + armAsm->B(&done); + + armAsm->Bind(&simple); + armAsm->Fcvt(d, s); + + armAsm->Bind(&done); +} + +// ---- IEEE double -> PS2 single (full overflow/underflow/flag handling) ----- +// +// Port of x86 ToPS2FPU_Full. `idx` holds the double result (D lane); `absidx` +// is a scratch NEON reg. On return the PS2 single is in `idx`'s S lane. +// Comparisons are done on the integer bit pattern of |x| — valid because every +// operand here is a finite double, so unsigned-integer order == magnitude order +// (sidesteps NaN/unordered, which never reach this point for ADD/SUB/MUL). +static void ToPS2FPU_Full(int idx, bool flags, int /*absidx*/, bool acc, bool addsub) +{ + const a64::VRegister s = armSRegister(idx); + const a64::VRegister d = armDRegister(idx); + + if (flags) + { + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Bic(RWSCRATCH, RWSCRATCH, FPUflagO | FPUflagU); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + if (acc) + { + armLoadEERegPtr(RWSCRATCH, &fpuRegs.ACCflag); + armAsm->Bic(RWSCRATCH, RWSCRATCH, 1); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.ACCflag); + } + } + + // abs = |reg| (integer, low 63 bits) + armAsm->Fmov(RXSCRATCH, d); + armAsm->And(RXARG1, RXSCRATCH, 0x7fffffffffffffffULL); + + a64::Label toComplex, toUnderflow, toOverflow, end; + + armAsm->Mov(RXARG2, static_cast(1151) << 52); // dbl_cvt_overflow (2^128) + armAsm->Cmp(RXARG1, RXARG2); + armAsm->B(&toComplex, a64::hs); + + armAsm->Mov(RXARG2, static_cast(897) << 52); // dbl_underflow (2^-126) + armAsm->Cmp(RXARG1, RXARG2); + armAsm->B(&toUnderflow, a64::lo); + + // In-range: plain narrow. + armAsm->Fcvt(s, d); + armAsm->B(&end); + + armAsm->Bind(&toComplex); + armAsm->Mov(RXARG2, static_cast(1152) << 52); // dbl_ps2_overflow (2^129) + armAsm->Cmp(RXARG1, RXARG2); + armAsm->B(&toOverflow, a64::hs); + + // Large but PS2-representable (exp-0xff range): lower double exp, narrow, + // raise single exp — the inverse of ToDouble's complex path. + armAsm->Mov(RXARG2, static_cast(1) << 52); + armAsm->Sub(RXSCRATCH, RXSCRATCH, RXARG2); + armAsm->Fmov(d, RXSCRATCH); + armAsm->Fcvt(s, d); + armAsm->Fmov(RWSCRATCH, s); + armAsm->Add(RWSCRATCH, RWSCRATCH, 0x00800000); + armAsm->Fmov(s, RWSCRATCH); + armAsm->B(&end); + + armAsm->Bind(&toOverflow); + // Beyond PS2 range: narrow then clamp to +/-max (keep sign, set all other bits). + armAsm->Fcvt(s, d); + armAsm->Fmov(RWSCRATCH, s); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x7fffffff); + armAsm->Fmov(s, RWSCRATCH); + if (flags) + { + armLoadEERegPtr(RWARG1, &fpuRegs.fprc[31]); + armAsm->Orr(RWARG1, RWARG1, FPUflagO | FPUflagSO); + armStoreEERegPtr(RWARG1, &fpuRegs.fprc[31]); + if (acc) + { + armLoadEERegPtr(RWARG1, &fpuRegs.ACCflag); + armAsm->Orr(RWARG1, RWARG1, 1); + armStoreEERegPtr(RWARG1, &fpuRegs.ACCflag); + } + } + armAsm->B(&end); + + armAsm->Bind(&toUnderflow); + a64::Label uDone; + if (flags) + { + // Set U|SU unless the result is exactly +/-0. + armAsm->Fmov(RXSCRATCH, d); + armAsm->And(RXARG1, RXSCRATCH, 0x7fffffffffffffffULL); + a64::Label isZero; + armAsm->Cbz(RXARG1, &isZero); + armLoadEERegPtr(RWARG2, &fpuRegs.fprc[31]); + armAsm->Orr(RWARG2, RWARG2, FPUflagU | FPUflagSU); + armStoreEERegPtr(RWARG2, &fpuRegs.fprc[31]); + if (addsub) + { + // ADD/SUB leave the (post-normalization) mantissa bits in place; + // reconstruct a PS2 denormal single: bits[22:0] = dbl_mant[51:29], + // bit31 = sign, exp = 0. (x86 PSLL.Q 12 / PSRL.Q 41 / sign<<31 / POR.) + armAsm->Fmov(RXSCRATCH, d); + armAsm->Lsl(RXARG1, RXSCRATCH, 12); + armAsm->Lsr(RXARG1, RXARG1, 41); + armAsm->Lsr(RXARG2, RXSCRATCH, 63); + armAsm->Lsl(RXARG2, RXARG2, 31); + armAsm->Orr(RWSCRATCH, RWARG1, RWARG2); + armAsm->Fmov(s, RWSCRATCH); + armAsm->B(&uDone); + } + armAsm->Bind(&isZero); + } + // Flush to +/-0 (keep sign). + armAsm->Fcvt(s, d); + armAsm->Fmov(RWSCRATCH, s); + armAsm->And(RWSCRATCH, RWSCRATCH, 0x80000000); + armAsm->Fmov(s, RWSCRATCH); + + armAsm->Bind(&uDone); + armAsm->Bind(&end); +} + +// ---- PS2 add/sub guard-bit emulation -------------------------------------- +// +// The EE FPU has no guard bits to the right of the mantissa; subtraction (and +// add of mixed signs) can shift the mantissa left and expose what would have +// been guard bits. This masks the low mantissa bits of the smaller operand by +// the exponent difference so they read as zero. Port of x86 FPU_ADD_SUB; both +// operands (single, in temp NEON regs `idxd`/`idxt`) are mutated in place. +static void FPU_ADD_SUB(int idxd, int idxt) +{ + const a64::VRegister sd = armSRegister(idxd); + const a64::VRegister st = armSRegister(idxt); + + armAsm->Fmov(RWARG1, sd); // d bits + armAsm->Fmov(RWARG2, st); // t bits + armAsm->Ubfx(RWARG3, RWARG1, 23, 8); // expd + armAsm->Ubfx(RWSCRATCH, RWARG2, 23, 8); // expt + armAsm->Sub(RWARG3, RWARG3, RWSCRATCH); // diff = expd - expt (signed) + + a64::Label caseD25, casePos, caseEq, caseDn25, done; + armAsm->Cmp(RWARG3, 25); + armAsm->B(&caseD25, a64::ge); + armAsm->Cmp(RWARG3, 0); + armAsm->B(&casePos, a64::gt); + armAsm->B(&caseEq, a64::eq); + armAsm->Cmn(RWARG3, 25); // cmp diff, -25 + armAsm->B(&caseDn25, a64::le); + + // diff in -24..-1 (expd < expt): mask tempd's low (-diff-1) bits. + armAsm->Neg(RWSCRATCH, RWARG3); + armAsm->Sub(RWSCRATCH, RWSCRATCH, 1); + armAsm->Mov(RWARG4, 0xffffffff); + armAsm->Lsl(RWARG4, RWARG4, RWSCRATCH); + armAsm->And(RWARG1, RWARG1, RWARG4); + armAsm->Fmov(sd, RWARG1); + armAsm->B(&done); + + armAsm->Bind(&caseD25); + // diff >= 25 (expt much smaller): tempt keeps only its sign. + armAsm->And(RWARG2, RWARG2, 0x80000000); + armAsm->Fmov(st, RWARG2); + armAsm->B(&done); + + armAsm->Bind(&casePos); + // diff in 1..24 (expt smaller): mask tempt's low (diff-1) bits. + armAsm->Sub(RWSCRATCH, RWARG3, 1); + armAsm->Mov(RWARG4, 0xffffffff); + armAsm->Lsl(RWARG4, RWARG4, RWSCRATCH); + armAsm->And(RWARG2, RWARG2, RWARG4); + armAsm->Fmov(st, RWARG2); + armAsm->B(&done); + + armAsm->Bind(&caseDn25); + // diff <= -25 (expd much smaller): tempd keeps only its sign. + armAsm->And(RWARG1, RWARG1, 0x80000000); + armAsm->Fmov(sd, RWARG1); + + armAsm->Bind(&caseEq); // diff == 0: nothing + armAsm->Bind(&done); +} + +// ---- Op cores -------------------------------------------------------------- + +// Copy an allocator-resident FP source (EEREC_S/EEREC_T) into a fresh temp so +// ToDouble can mutate it without corrupting the guest fpr slot. +static int copySrc(int eerec) +{ + const int idx = _allocTempNEONreg(); + armAsm->Fmov(armSRegister(idx), armSRegister(eerec)); + return idx; +} + +// ADD/SUB/ADDA/SUBA: FPU_ADD_SUB guard mask -> widen -> op in double -> narrow. +static void recFPUOp(int info, int eeRecDst, int op /*0=add,1=sub*/, bool acc) +{ + const int sreg = copySrc(EEREC_S); + const int treg = copySrc(EEREC_T); + + FPU_ADD_SUB(sreg, treg); + ToDouble(sreg); + ToDouble(treg); + + if (op == 0) + armAsm->Fadd(armDRegister(sreg), armDRegister(sreg), armDRegister(treg)); + else + armAsm->Fsub(armDRegister(sreg), armDRegister(sreg), armDRegister(treg)); + + ToPS2FPU_Full(sreg, true, treg, acc, true); + armAsm->Fmov(armSRegister(eeRecDst), armSRegister(sreg)); + + _freeNEONreg(sreg); + _freeNEONreg(treg); +} + +// MUL/MULA: widen -> multiply in double -> narrow. (FPUMULHACK — the Tales of +// Destiny gamefix — is intentionally not folded in here; default off.) +static void recMULop(int info, int eeRecDst, bool acc) +{ + const int sreg = copySrc(EEREC_S); + const int treg = copySrc(EEREC_T); + + ToDouble(sreg); + ToDouble(treg); + armAsm->Fmul(armDRegister(sreg), armDRegister(sreg), armDRegister(treg)); + + ToPS2FPU_Full(sreg, true, treg, acc, false); + armAsm->Fmov(armSRegister(eeRecDst), armSRegister(sreg)); + + _freeNEONreg(sreg); + _freeNEONreg(treg); +} + +// MADD/MSUB/MADDA/MSUBA: (Fd or ACC) = ACC +/- Fs*Ft, with two PS2-accurate +// roundings (the multiply, then the accumulate) and overflow propagation from +// BOTH the product and the prior ACC. Port of x86 recMaddsub. +// +// The control flow mirrors x86: do the full-mode multiply (which may raise O), +// guard-mask ACC against the product, then branch on whether the product +// overflowed (FPUflagO) or the incoming ACC was already saturated (ACCflag&1). +// If either did, the accumulate is dominated by a 2^128-class term and the +// result is just +/-max with the dominant sign — skip the double add entirely. +// Only when both are finite is the accumulation performed in double. +static void recMaddsub(int info, int eeRecDst, int op /*0=add,1=sub*/, bool acc) +{ + const int sreg = copySrc(EEREC_S); + const int treg = copySrc(EEREC_T); + + // --- multiply stage: sreg = ToPS2FPU(ToDouble(s) * ToDouble(t)). Sets O on + // product overflow; acc=false so it never touches ACCflag here. --- + ToDouble(sreg); + ToDouble(treg); + armAsm->Fmul(armDRegister(sreg), armDRegister(sreg), armDRegister(treg)); + ToPS2FPU_Full(sreg, true, treg, false, false); + + // --- reload ACC (allocator-resident) into treg, then guard-mask it against + // the single-precision product. --- + armAsm->Fmov(armSRegister(treg), armSRegister(EEREC_ACC)); + FPU_ADD_SUB(treg, sreg); + + a64::Label mulovf, accovf, operation, skipall; + + // product overflowed? -> mulovf + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Tst(RWSCRATCH, FPUflagO); + armAsm->B(&mulovf, a64::ne); + ToDouble(sreg); + + // prior ACC saturated? -> accovf + armLoadEERegPtr(RWSCRATCH, &fpuRegs.ACCflag); + armAsm->Tst(RWSCRATCH, 1); + armAsm->B(&accovf, a64::ne); + ToDouble(treg); + armAsm->B(&operation); + + armAsm->Bind(&mulovf); + // Product is a saturated single; for SUB negate its sign, then it becomes + // the (single) accumulate result. Falls through into accovf. + if (op == 1) + { + armAsm->Fmov(RWSCRATCH, armSRegister(sreg)); + armAsm->Eor(RWSCRATCH, RWSCRATCH, 0x80000000); + armAsm->Fmov(armSRegister(sreg), RWSCRATCH); + } + armAsm->Fmov(armSRegister(treg), armSRegister(sreg)); + + armAsm->Bind(&accovf); + // SetMaxValue(treg): keep sign, set all lower bits -> +/-PS2 max. + armAsm->Fmov(RWSCRATCH, armSRegister(treg)); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 0x7fffffff); + armAsm->Fmov(armSRegister(treg), RWSCRATCH); + // Clear O|U then raise O|SO (and ACCflag for the *A variants). + armLoadEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + armAsm->Bic(RWSCRATCH, RWSCRATCH, FPUflagO | FPUflagU); + armAsm->Orr(RWSCRATCH, RWSCRATCH, FPUflagO | FPUflagSO); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.fprc[31]); + if (acc) + { + armLoadEERegPtr(RWSCRATCH, &fpuRegs.ACCflag); + armAsm->Orr(RWSCRATCH, RWSCRATCH, 1); + armStoreEERegPtr(RWSCRATCH, &fpuRegs.ACCflag); + } + armAsm->B(&skipall); + + armAsm->Bind(&operation); + // Both finite: accumulate in double, narrow with flags. + if (op == 1) + armAsm->Fsub(armDRegister(treg), armDRegister(treg), armDRegister(sreg)); + else + armAsm->Fadd(armDRegister(treg), armDRegister(treg), armDRegister(sreg)); + ToPS2FPU_Full(treg, true, sreg, acc, true); + + armAsm->Bind(&skipall); + armAsm->Fmov(armSRegister(eeRecDst), armSRegister(treg)); + + _freeNEONreg(sreg); + _freeNEONreg(treg); +} + +// ---- Per-opcode DOUBLE emitters (called by the CHECK_FPU_FULL branch in +// iFPU-arm64.cpp via eeFPURecompileCode) ------------------------------- + +void recADD_S_xmm(int info) { recFPUOp(info, EEREC_D, 0, false); } +void recSUB_S_xmm(int info) { recFPUOp(info, EEREC_D, 1, false); } +void recADDA_S_xmm(int info) { recFPUOp(info, EEREC_ACC, 0, true); } +void recSUBA_S_xmm(int info) { recFPUOp(info, EEREC_ACC, 1, true); } +void recMUL_S_xmm(int info) { recMULop(info, EEREC_D, false); } +void recMULA_S_xmm(int info) { recMULop(info, EEREC_ACC, true); } +void recMADD_S_xmm(int info) { recMaddsub(info, EEREC_D, 0, false); } +void recMSUB_S_xmm(int info) { recMaddsub(info, EEREC_D, 1, false); } +void recMADDA_S_xmm(int info) { recMaddsub(info, EEREC_ACC, 0, true); } +void recMSUBA_S_xmm(int info) { recMaddsub(info, EEREC_ACC, 1, true); } + +#undef _Ft_ +#undef _Fs_ +#undef _Fd_ + +} // namespace DOUBLE +} // namespace COP1 +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iMMI-arm64.cpp b/pcsx2/arm64/iMMI-arm64.cpp new file mode 100644 index 0000000000..a1dcde61e2 --- /dev/null +++ b/pcsx2/arm64/iMMI-arm64.cpp @@ -0,0 +1,1433 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE MMI (Multimedia Instructions) Codegen — NEON-based +// +// All MMI instructions are 128-bit SIMD operations on the EE's 128-bit GPRs. +// NEON Q registers are used throughout: load from cpuRegs.GPR, operate, store back. + +#include "arm64/iR5900-arm64.h" +#include "arm64/AsmHelpers.h" +#include "common/Assertions.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { +namespace MMI { + +namespace Interp = R5900::Interpreter::OpcodeImpl::MMI; + +// ============================================================================ +// Helpers for 128-bit GPR load/store +// ============================================================================ + +// Flush any const propagation state for a register and invalidate allocations. +// Must be called before reading a register's 128-bit value from memory, +// since const prop only tracks the lower 64 bits. +static void mmiFlushReg(int reg) +{ + if (reg == 0) return; + if (GPR_IS_CONST1(reg)) + { + // Const prop only has lower 64 bits — flush to memory so upper 64 bits + // are preserved alongside the correct lower 64 bits. + _flushEEreg(reg); + } + _deleteEEreg(reg, 1); +} + +// Prepare destination: invalidate const/alloc state (the full 128 bits will be overwritten) +static void mmiInvalidateDest(int reg) +{ + if (reg == 0) return; + _deleteEEreg(reg, 0); + GPR_DEL_CONST(reg); +} + +// Load 128-bit GPR into a NEON Q register +static void mmiLoadReg(const a64::VRegister& qreg, int gpr) +{ + if (gpr == 0) + { + // r0 is always zero + armAsm->Movi(qreg.V16B(), 0); + } + else + { + armAsm->Ldr(qreg, armCpuRegMem(&cpuRegs.GPR.r[gpr].UQ)); + } +} + +// Store 128-bit NEON Q register to GPR +static void mmiStoreReg(int gpr, const a64::VRegister& qreg) +{ + pxAssert(gpr != 0); + armAsm->Str(qreg, armCpuRegMem(&cpuRegs.GPR.r[gpr].UQ)); +} + +// Standard 3-operand MMI: rd = rs OP rt (128-bit). +// +// Routes through eeRecompileCodeXMM so consecutive MMI ops on the same guest +// register stay register-resident in the allocator-managed NEON pool instead +// of bouncing through memory. Allocator handles const tracking, GPR-side +// eviction, NaN-zero of r0, and "already in NEON" reuse. +// +// Each user gets three locals: +// qs — VRegister view of EEREC_S (Rs input, MODE_READ) +// qt — VRegister view of EEREC_T (Rt input, MODE_READ) +// qd — VRegister view of EEREC_D (Rd output, MODE_WRITE) +// +// Functions that need a fourth temp can use RQSCRATCH / RQSCRATCH2 — both are +// outside the allocator pool. Do NOT clobber qs or qt +// before the final write to qd, otherwise the allocator's MODE_READ state +// for them is invalidated. +#define MMI_3OP_SETUP() \ + if (!_Rd_) return; \ + int info = eeRecompileCodeXMM(XMMINFO_READS | XMMINFO_READT | XMMINFO_WRITED); \ + const a64::VRegister qs = armQRegister(EEREC_S); \ + const a64::VRegister qt = armQRegister(EEREC_T); \ + const a64::VRegister qd = armQRegister(EEREC_D); \ + (void)info + +// 2-operand: rd = OP(rt). +// qt — VRegister view of EEREC_T (Rt input, MODE_READ) +// qd — VRegister view of EEREC_D (Rd output, MODE_WRITE) +#define MMI_2OP_SETUP() \ + if (!_Rd_) return; \ + int info = eeRecompileCodeXMM(XMMINFO_READT | XMMINFO_WRITED); \ + const a64::VRegister qt = armQRegister(EEREC_T); \ + const a64::VRegister qd = armQRegister(EEREC_D); \ + (void)info + +// ============================================================================ +// Logical Operations (128-bit) +// ============================================================================ + +void recPAND() +{ + MMI_3OP_SETUP(); + armAsm->And(qd.V16B(), qs.V16B(), qt.V16B()); +} + +void recPOR() +{ + if (!_Rd_) + return; + + // `por rd, r0, rt` is the canonical PS2 128-bit register-move idiom and is + // common. Special-case an r0 operand to avoid allocating r0 into a NEON reg + // and materialize a zero just to OR it in (conditional XMMINFO, + // Movi when both r0, register-copy when one is r0). + const bool s_zero = (_Rs_ == 0); + const bool t_zero = (_Rt_ == 0); + int info = eeRecompileCodeXMM((s_zero ? 0 : XMMINFO_READS) | (t_zero ? 0 : XMMINFO_READT) | XMMINFO_WRITED); + const a64::VRegister qd = armQRegister(EEREC_D); + + if (s_zero && t_zero) + armAsm->Movi(qd.V2D(), 0); + else if (s_zero) + armAsm->Mov(qd.V16B(), armQRegister(EEREC_T).V16B()); + else if (t_zero) + armAsm->Mov(qd.V16B(), armQRegister(EEREC_S).V16B()); + else + armAsm->Orr(qd.V16B(), armQRegister(EEREC_S).V16B(), armQRegister(EEREC_T).V16B()); +} + +void recPXOR() +{ + MMI_3OP_SETUP(); + armAsm->Eor(qd.V16B(), qs.V16B(), qt.V16B()); +} + +void recPNOR() +{ + MMI_3OP_SETUP(); + armAsm->Orr(qd.V16B(), qs.V16B(), qt.V16B()); + armAsm->Not(qd.V16B(), qd.V16B()); +} + +// ============================================================================ +// Packed Arithmetic — Signed +// ============================================================================ + +void recPADDW() +{ + MMI_3OP_SETUP(); + armAsm->Add(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPSUBW() +{ + MMI_3OP_SETUP(); + armAsm->Sub(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPADDH() +{ + MMI_3OP_SETUP(); + armAsm->Add(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPSUBH() +{ + MMI_3OP_SETUP(); + armAsm->Sub(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPADDB() +{ + MMI_3OP_SETUP(); + armAsm->Add(qd.V16B(), qs.V16B(), qt.V16B()); +} + +void recPSUBB() +{ + MMI_3OP_SETUP(); + armAsm->Sub(qd.V16B(), qs.V16B(), qt.V16B()); +} + +// ============================================================================ +// Packed Arithmetic — Unsigned +// ============================================================================ + +void recPADDUW() +{ + MMI_3OP_SETUP(); + armAsm->Uqadd(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPSUBUW() +{ + MMI_3OP_SETUP(); + armAsm->Uqsub(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPADDUH() +{ + MMI_3OP_SETUP(); + armAsm->Uqadd(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPSUBUH() +{ + MMI_3OP_SETUP(); + armAsm->Uqsub(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPADDUB() +{ + MMI_3OP_SETUP(); + armAsm->Uqadd(qd.V16B(), qs.V16B(), qt.V16B()); +} + +void recPSUBUB() +{ + MMI_3OP_SETUP(); + armAsm->Uqsub(qd.V16B(), qs.V16B(), qt.V16B()); +} + +// ============================================================================ +// Packed Arithmetic — Saturating Signed +// ============================================================================ + +void recPADDSW() +{ + MMI_3OP_SETUP(); + armAsm->Sqadd(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPSUBSW() +{ + MMI_3OP_SETUP(); + armAsm->Sqsub(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPADDSH() +{ + MMI_3OP_SETUP(); + armAsm->Sqadd(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPSUBSH() +{ + MMI_3OP_SETUP(); + armAsm->Sqsub(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPADDSB() +{ + MMI_3OP_SETUP(); + armAsm->Sqadd(qd.V16B(), qs.V16B(), qt.V16B()); +} + +void recPSUBSB() +{ + MMI_3OP_SETUP(); + armAsm->Sqsub(qd.V16B(), qs.V16B(), qt.V16B()); +} + +// ============================================================================ +// Packed Compare — Greater Than (signed) +// ============================================================================ + +void recPCGTW() +{ + MMI_3OP_SETUP(); + armAsm->Cmgt(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPCGTH() +{ + MMI_3OP_SETUP(); + armAsm->Cmgt(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPCGTB() +{ + MMI_3OP_SETUP(); + armAsm->Cmgt(qd.V16B(), qs.V16B(), qt.V16B()); +} + +// ============================================================================ +// Packed Compare — Equal +// ============================================================================ + +void recPCEQW() +{ + MMI_3OP_SETUP(); + armAsm->Cmeq(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPCEQH() +{ + MMI_3OP_SETUP(); + armAsm->Cmeq(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPCEQB() +{ + MMI_3OP_SETUP(); + armAsm->Cmeq(qd.V16B(), qs.V16B(), qt.V16B()); +} + +// ============================================================================ +// Packed Min/Max (signed) +// ============================================================================ + +void recPMAXW() +{ + MMI_3OP_SETUP(); + armAsm->Smax(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPMINW() +{ + MMI_3OP_SETUP(); + armAsm->Smin(qd.V4S(), qs.V4S(), qt.V4S()); +} + +void recPMAXH() +{ + MMI_3OP_SETUP(); + armAsm->Smax(qd.V8H(), qs.V8H(), qt.V8H()); +} + +void recPMINH() +{ + MMI_3OP_SETUP(); + armAsm->Smin(qd.V8H(), qs.V8H(), qt.V8H()); +} + +// ============================================================================ +// Packed Absolute Value (signed) +// ============================================================================ + +void recPABSW() +{ + MMI_2OP_SETUP(); + // PS2 PABSW saturates INT_MIN → INT_MAX (per MMI.cpp _PABSW). NEON Abs + // preserves INT_MIN; Sqabs is the saturating form that matches. + armAsm->Sqabs(qd.V4S(), qt.V4S()); +} + +void recPABSH() +{ + MMI_2OP_SETUP(); + // Mirror of PABSW for halfword lanes. + armAsm->Sqabs(qd.V8H(), qt.V8H()); +} + +// ============================================================================ +// Register Copy / Move +// ============================================================================ + +// PCPYLD: rd = { rs.UD[0], rt.UD[0] } — copy (concatenate) lower doubleword of each source. +void recPCPYLD() +{ + MMI_3OP_SETUP(); + armAsm->Zip1(qd.V2D(), qt.V2D(), qs.V2D()); +} + +// PCPYUD: rd = {rt[127:64], rs[127:64]} — upper doublewords interleaved +void recPCPYUD() +{ + MMI_3OP_SETUP(); + armAsm->Zip2(qd.V2D(), qs.V2D(), qt.V2D()); +} + +// PCPYH: rd = {rt.UH[4] x4, rt.UH[0] x4} — replicate halfwords. +// Register-resident via the allocator (MMI_2OP_SETUP) instead of a +// memory-bounce (Ldr q from Rt + Str q to Rd + const flush), matching +// sibling single-source MMI ops (PABSW/PCPYLD). Saves a full-width load +// + store per execution. +// Broadcast rt.H[4] into scratch FIRST so the qd==qt aliased case stays correct +// (a qd write would otherwise clobber rt before H[4] is read). +void recPCPYH() +{ + MMI_2OP_SETUP(); + armAsm->Dup(RQSCRATCH.V8H(), qt.V8H(), 4); // rt.H[4] x8 (read qt before qd write) + armAsm->Dup(qd.V8H(), qt.V8H(), 0); // qd = rt.H[0] x8 + armAsm->Mov(qd.V2D(), 1, RQSCRATCH.V2D(), 0); // upper 64 <- rt.H[4] x4 +} + +// PMFHI: rd = HI (128-bit) +void recPMFHI() +{ + if (!_Rd_) return; + mmiInvalidateDest(_Rd_); + + armAsm->Ldr(RQSCRATCH, armCpuRegMem(&cpuRegs.HI.UQ)); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +// PMFLO: rd = LO (128-bit) +void recPMFLO() +{ + if (!_Rd_) return; + mmiInvalidateDest(_Rd_); + + armAsm->Ldr(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +// PMTHI: HI = rs (128-bit). Take Rs from its NEON slot when allocated and store +// straight to HI memory. HI is never NEON-resident in the EE rec (no opcode +// passes XMMINFO_*HI — see iR5900Templates-arm64.cpp), so no allocator +// invalidation is needed. +void recPMTHI() +{ + int info = eeRecompileCodeXMM(XMMINFO_READS); + (void)info; + armAsm->Str(armQRegister(EEREC_S), armCpuRegMem(&cpuRegs.HI.UQ)); +} + +// PMTLO: LO = rs (128-bit). LO is never NEON-resident (see recPMTHI). +void recPMTLO() +{ + int info = eeRecompileCodeXMM(XMMINFO_READS); + (void)info; + armAsm->Str(armQRegister(EEREC_S), armCpuRegMem(&cpuRegs.LO.UQ)); +} + +// ============================================================================ +// Packed Shifts (by immediate sa field) +// ============================================================================ + +void recPSLLW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + mmiLoadReg(RQSCRATCH, _Rt_); + if (_Sa_ == 0) + { + mmiStoreReg(_Rd_, RQSCRATCH); + return; + } + armAsm->Shl(RQSCRATCH.V4S(), RQSCRATCH.V4S(), _Sa_); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +void recPSRLW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + mmiLoadReg(RQSCRATCH, _Rt_); + if (_Sa_ == 0) + { + mmiStoreReg(_Rd_, RQSCRATCH); + return; + } + armAsm->Ushr(RQSCRATCH.V4S(), RQSCRATCH.V4S(), _Sa_); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +void recPSRAW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + mmiLoadReg(RQSCRATCH, _Rt_); + if (_Sa_ == 0) + { + mmiStoreReg(_Rd_, RQSCRATCH); + return; + } + armAsm->Sshr(RQSCRATCH.V4S(), RQSCRATCH.V4S(), _Sa_); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +// Halfword shifts: interp uses (_Sa_ & 0xf) per MMI.cpp:228/240/252 — +// only 4 of the 5 sa bits are live since the lane is 16-bit. vixl +// Shl/Ushr/Sshr V8H require shift ∈ [0,15]; mask up front to match +// interp and stay inside the encoder's range. +void recPSLLH() +{ + if (!_Rd_) return; + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + mmiLoadReg(RQSCRATCH, _Rt_); + const u32 sa = _Sa_ & 0xf; + if (sa == 0) + { + mmiStoreReg(_Rd_, RQSCRATCH); + return; + } + armAsm->Shl(RQSCRATCH.V8H(), RQSCRATCH.V8H(), sa); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +void recPSRLH() +{ + if (!_Rd_) return; + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + mmiLoadReg(RQSCRATCH, _Rt_); + const u32 sa = _Sa_ & 0xf; + if (sa == 0) + { + mmiStoreReg(_Rd_, RQSCRATCH); + return; + } + armAsm->Ushr(RQSCRATCH.V8H(), RQSCRATCH.V8H(), sa); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +void recPSRAH() +{ + if (!_Rd_) return; + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + mmiLoadReg(RQSCRATCH, _Rt_); + const u32 sa = _Sa_ & 0xf; + if (sa == 0) + { + mmiStoreReg(_Rd_, RQSCRATCH); + return; + } + armAsm->Sshr(RQSCRATCH.V8H(), RQSCRATCH.V8H(), sa); + mmiStoreReg(_Rd_, RQSCRATCH); +} + +// ============================================================================ +// Pack / Unpack (Extend / Compress) +// ============================================================================ + +// PEXTLW: interleave lower 32-bit words of rs and rt +// rd = {rs.UL[1], rt.UL[1], rs.UL[0], rt.UL[0]} +void recPEXTLW() +{ + MMI_3OP_SETUP(); + armAsm->Zip1(qd.V4S(), qt.V4S(), qs.V4S()); +} + +// PEXTUW: interleave upper 32-bit words of rs and rt +// rd = {rs.UL[3], rt.UL[3], rs.UL[2], rt.UL[2]} +void recPEXTUW() +{ + MMI_3OP_SETUP(); + armAsm->Zip2(qd.V4S(), qt.V4S(), qs.V4S()); +} + +// PEXTLH: interleave lower 16-bit halfwords +void recPEXTLH() +{ + if (!_Rd_) + return; + + // rs==0 fast path: the odd output halfwords are all zero, so this + // is just a zero-extend of rt's lower 4 halfwords to words. Skip requesting + // XMMINFO_READS — otherwise the allocator pins a callee-saved NEON reg and + // materializes a zero vector for r0 just to Zip it in. Zip1(qt,qt) duplicates + // each halfword, then Ushr clears the high half of every word lane. + if (_Rs_ == 0) + { + int info = eeRecompileCodeXMM(XMMINFO_READT | XMMINFO_WRITED); + const a64::VRegister qt = armQRegister(EEREC_T); + const a64::VRegister qd = armQRegister(EEREC_D); + (void)info; + armAsm->Zip1(qd.V8H(), qt.V8H(), qt.V8H()); // {H0,H0,H1,H1,H2,H2,H3,H3} + armAsm->Ushr(qd.V4S(), qd.V4S(), 16); // {0:H0, 0:H1, 0:H2, 0:H3} + return; + } + + MMI_3OP_SETUP(); + armAsm->Zip1(qd.V8H(), qt.V8H(), qs.V8H()); +} + +// PEXTUH: interleave upper 16-bit halfwords +void recPEXTUH() +{ + MMI_3OP_SETUP(); + armAsm->Zip2(qd.V8H(), qt.V8H(), qs.V8H()); +} + +// PEXTLB: interleave lower bytes +void recPEXTLB() +{ + MMI_3OP_SETUP(); + armAsm->Zip1(qd.V16B(), qt.V16B(), qs.V16B()); +} + +// PEXTUB: interleave upper bytes +void recPEXTUB() +{ + MMI_3OP_SETUP(); + armAsm->Zip2(qd.V16B(), qt.V16B(), qs.V16B()); +} + +// PPACW: pack words — {rs.UL[2], rs.UL[0], rt.UL[2], rt.UL[0]} +void recPPACW() +{ + MMI_3OP_SETUP(); + armAsm->Uzp1(qd.V4S(), qt.V4S(), qs.V4S()); +} + +// PPACH: pack halfwords — even halfwords from rs and rt +void recPPACH() +{ + MMI_3OP_SETUP(); + armAsm->Uzp1(qd.V8H(), qt.V8H(), qs.V8H()); +} + +// PPACB: pack bytes — even bytes from rs and rt +void recPPACB() +{ + MMI_3OP_SETUP(); + armAsm->Uzp1(qd.V16B(), qt.V16B(), qs.V16B()); +} + +// PADSBH: rd.UH[0..3] = rs.UH[0..3] - rt.UH[0..3], rd.UH[4..7] = rs.UH[4..7] + rt.UH[4..7] +// Lower 4 halfwords: subtract. Upper 4 halfwords: add. +void recPADSBH() +{ + MMI_3OP_SETUP(); + // Compute the add into a scratch FIRST. If Rd aliases Rs or Rt, the + // allocator hands qd back as the same Q-reg as qs/qt — writing qd in + // the sub step would clobber the source still needed for the add. + armAsm->Add(RQSCRATCH.V8H(), qs.V8H(), qt.V8H()); + // qd = sub result (all 8 halfwords); safe to clobber qs/qt now. + armAsm->Sub(qd.V8H(), qs.V8H(), qt.V8H()); + // Blend: keep lower 64 bits of sub in qd, upper 64 bits from add. + armAsm->Mov(qd.V2D(), 1, RQSCRATCH.V2D(), 1); +} + +// ============================================================================ +// Interleave halfwords +// ============================================================================ + +// PINTH: rd.US[2k]=Rt.US[k], rd.US[2k+1]=Rs.US[k+4], k=0..3 — interleave low 4 +// halfwords of Rt with high 4 of Rs. +void recPINTH() +{ + MMI_3OP_SETUP(); + // Move rs upper 64 → low position of scratch (don't clobber qs). + armAsm->Dup(RQSCRATCH.V2D(), qs.V2D(), 1); // tmp = {rs.UD[1], rs.UD[1]} + // zip1.8h of rt(lower) and rs_upper(lower) gives interleaved result. + armAsm->Zip1(qd.V8H(), qt.V8H(), RQSCRATCH.V8H()); +} + +// PINTEH: rd = {rs.UH[6],rt.UH[6], rs.UH[4],rt.UH[4], rs.UH[2],rt.UH[2], rs.UH[0],rt.UH[0]} +// Interleave even halfwords +void recPINTEH() +{ + MMI_3OP_SETUP(); + // Extract even halfwords from each into scratch — never touch qs/qt. + armAsm->Uzp1(RQSCRATCH.V8H(), qs.V8H(), qs.V8H()); // rs evens in lower 64 + armAsm->Uzp1(RQSCRATCH2.V8H(), qt.V8H(), qt.V8H()); // rt evens in lower 64 + // Zip the lower 64 bits of each into qd. + armAsm->Zip1(qd.V8H(), RQSCRATCH2.V8H(), RQSCRATCH.V8H()); +} + +// ============================================================================ +// Shuffles / Permutations +// ============================================================================ + +// PEXEW: rd = {rt[2], rt[1], rt[0], rt[3]} (lane order) — swap words 0 and 2. +// 2-op idiom (Rev64 + Ext) instead of a scratch snapshot + full copy + 2 lane +// inserts. Both ops read qt fully before writing, so it is alias-safe when the +// allocator hands back qd == qt. +void recPEXEW() +{ + MMI_2OP_SETUP(); + armAsm->Rev64(qd.V4S(), qt.V4S()); // {rt[1],rt[0],rt[3],rt[2]} + armAsm->Ext(qd.V16B(), qd.V16B(), qd.V16B(), 12); // {rt[2],rt[1],rt[0],rt[3]} +} + +// PEXEH: swap halfwords 0↔2 in each 64-bit lane +// rd = {H[2],H[1],H[0],H[3], H[6],H[5],H[4],H[7]} +void recPEXEH() +{ + MMI_2OP_SETUP(); + armAsm->Mov(RQSCRATCH.V16B(), qt.V16B()); + armAsm->Mov(qd.V8H(), RQSCRATCH.V8H()); + armAsm->Mov(qd.V8H(), 0, RQSCRATCH.V8H(), 2); + armAsm->Mov(qd.V8H(), 2, RQSCRATCH.V8H(), 0); + armAsm->Mov(qd.V8H(), 4, RQSCRATCH.V8H(), 6); + armAsm->Mov(qd.V8H(), 6, RQSCRATCH.V8H(), 4); +} + +// PREVH: reverse halfwords within each 64-bit lane +// rd = {H[3],H[2],H[1],H[0], H[7],H[6],H[5],H[4]} +void recPREVH() +{ + MMI_2OP_SETUP(); + armAsm->Rev64(qd.V8H(), qt.V8H()); +} + +// PROT3W: rotate lower 3 words: rd = {rt[1], rt[2], rt[0], rt[3]} (lane order). +// 3-op shuffle (Rev64 + Ext + Zip1) instead of a scratch snapshot + full copy +// + 3 lane inserts. Rev64 and Ext read qt into scratches +// first, so Zip1 → qd is alias-safe when qd == qt. +// rev = {rt[1],rt[0],rt[3],rt[2]} +// ext8 = {rt[2],rt[3],rt[0],rt[1]} +// Zip1(rev,ext8) = {rev[0],ext8[0],rev[1],ext8[1]} = {rt[1],rt[2],rt[0],rt[3]} +void recPROT3W() +{ + MMI_2OP_SETUP(); + armAsm->Rev64(RQSCRATCH.V4S(), qt.V4S()); + armAsm->Ext(RQSCRATCH2.V16B(), qt.V16B(), qt.V16B(), 8); + armAsm->Zip1(qd.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); +} + +// PEXCW: swap words 1 and 2: rd = {rt[0], rt[2], rt[1], rt[3]} (lane order). +// 2-op idiom (Rev64 + Uzp1) instead of a scratch snapshot + full copy + 2 lane +// inserts. Both read their sources fully before +// writing qd, so it is alias-safe when qd == qt. +// rev = {rt[1],rt[0],rt[3],rt[2]} +// Uzp1(qt,rev) = {qt[0],qt[2],rev[0],rev[2]} = {rt[0],rt[2],rt[1],rt[3]} +void recPEXCW() +{ + MMI_2OP_SETUP(); + armAsm->Rev64(RQSCRATCH.V4S(), qt.V4S()); + armAsm->Uzp1(qd.V4S(), qt.V4S(), RQSCRATCH.V4S()); +} + +// PEXCH: swap halfwords 1↔2 within each 64-bit lane +// {H[0],H[2],H[1],H[3], H[4],H[6],H[5],H[7]} +void recPEXCH() +{ + MMI_2OP_SETUP(); + armAsm->Mov(RQSCRATCH.V16B(), qt.V16B()); + armAsm->Mov(qd.V8H(), RQSCRATCH.V8H()); + armAsm->Mov(qd.V8H(), 1, RQSCRATCH.V8H(), 2); + armAsm->Mov(qd.V8H(), 2, RQSCRATCH.V8H(), 1); + armAsm->Mov(qd.V8H(), 5, RQSCRATCH.V8H(), 6); + armAsm->Mov(qd.V8H(), 6, RQSCRATCH.V8H(), 5); +} + +// PEXT5: expand each 32-bit lane's PS2 RGB1555 field into BGRA8 layout. +// Per-lane: +// rd = ((rt & 0x001F) << 3) // R bits [4:0] -> [7:3] +// | ((rt & 0x03E0) << 6) // G bits [9:5] -> [15:11] +// | ((rt & 0x7C00) << 9) // B bits [14:10] -> [23:19] +// | ((rt & 0x8000) << 16); // A bit [15] -> [31] +void recPEXT5() +{ + MMI_2OP_SETUP(); + // Preserve qt in case allocator assigned qd == qt — rt is needed for all + // four shift+mask passes below, but the first write to qd would clobber + // it if they share a slot. + armAsm->Mov(RQSCRATCH3.V16B(), qt.V16B()); + + // Field 0: (rt << 3) & 0x000000F8 -> qd + armAsm->Shl(qd.V4S(), RQSCRATCH3.V4S(), 3); + armAsm->Movi(RQSCRATCH.V4S(), 0xF8); + armAsm->And(qd.V16B(), qd.V16B(), RQSCRATCH.V16B()); + + // Field 1: (rt << 6) & 0x0000F800 -> qd + armAsm->Shl(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), 6); + armAsm->Movi(RQSCRATCH.V4S(), 0xF8, vixl::aarch64::LSL, 8); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH.V16B()); + armAsm->Orr(qd.V16B(), qd.V16B(), RQSCRATCH2.V16B()); + + // Field 2: (rt << 9) & 0x00F80000 -> qd + armAsm->Shl(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), 9); + armAsm->Movi(RQSCRATCH.V4S(), 0xF8, vixl::aarch64::LSL, 16); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH.V16B()); + armAsm->Orr(qd.V16B(), qd.V16B(), RQSCRATCH2.V16B()); + + // Field 3: (rt << 16) & 0x80000000 -> qd + armAsm->Shl(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), 16); + armAsm->Movi(RQSCRATCH.V4S(), 0x80, vixl::aarch64::LSL, 24); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH.V16B()); + armAsm->Orr(qd.V16B(), qd.V16B(), RQSCRATCH2.V16B()); +} + +// PPAC5: pack BGRA8-style 32-bit lanes back into PS2 RGB1555 16-bit layout +// (the inverse of PEXT5). Upper 16 bits of each lane left as garbage from +// the shifted-source — interp does not mask them either. +// Per-lane: +// rd = ((rt >> 3) & 0x001F) +// | ((rt >> 6) & 0x03E0) +// | ((rt >> 9) & 0x7C00) +// | ((rt >> 16) & 0x8000); +void recPPAC5() +{ + MMI_2OP_SETUP(); + armAsm->Mov(RQSCRATCH3.V16B(), qt.V16B()); + + // Field 0: (rt >> 3) & 0x0000001F -> qd + armAsm->Ushr(qd.V4S(), RQSCRATCH3.V4S(), 3); + armAsm->Movi(RQSCRATCH.V4S(), 0x1F); + armAsm->And(qd.V16B(), qd.V16B(), RQSCRATCH.V16B()); + + // Field 1: (rt >> 6) & 0x000003E0 -> qd + // 0x3E0 has two non-zero bytes; vixl's Movi macro materializes it via + // Mov scratch_w + Dup (2 host insns) rather than the single LSL form. + armAsm->Ushr(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), 6); + armAsm->Movi(RQSCRATCH.V4S(), 0x3E0); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH.V16B()); + armAsm->Orr(qd.V16B(), qd.V16B(), RQSCRATCH2.V16B()); + + // Field 2: (rt >> 9) & 0x00007C00 -> qd + armAsm->Ushr(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), 9); + armAsm->Movi(RQSCRATCH.V4S(), 0x7C, vixl::aarch64::LSL, 8); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH.V16B()); + armAsm->Orr(qd.V16B(), qd.V16B(), RQSCRATCH2.V16B()); + + // Field 3: (rt >> 16) & 0x00008000 -> qd + armAsm->Ushr(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), 16); + armAsm->Movi(RQSCRATCH.V4S(), 0x80, vixl::aarch64::LSL, 8); + armAsm->And(RQSCRATCH2.V16B(), RQSCRATCH2.V16B(), RQSCRATCH.V16B()); + armAsm->Orr(qd.V16B(), qd.V16B(), RQSCRATCH2.V16B()); +} + +// ============================================================================ +// Variable shifts — operate on words 0 and 2 only, sign-extend to 64 +// ============================================================================ + +// PSLLVW: rd.SD[0] = sign_ext(rt.UL[0] << (rs.UL[0] & 0x1F)) +// rd.SD[1] = sign_ext(rt.UL[2] << (rs.UL[2] & 0x1F)) +void recPSLLVW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + + // Word 0 + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rt_].UL[0]); + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->And(a64::w1, a64::w1, 0x1F); + armAsm->Lsl(a64::w0, a64::w0, a64::w1); + armAsm->Sxtw(a64::x0, a64::w0); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rd_].UD[0]); + + // Word 2 + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rt_].UL[2]); + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[2]); + armAsm->And(a64::w1, a64::w1, 0x1F); + armAsm->Lsl(a64::w0, a64::w0, a64::w1); + armAsm->Sxtw(a64::x0, a64::w0); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rd_].UD[1]); +} + +void recPSRLVW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rt_].UL[0]); + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->And(a64::w1, a64::w1, 0x1F); + armAsm->Lsr(a64::w0, a64::w0, a64::w1); + armAsm->Sxtw(a64::x0, a64::w0); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rd_].UD[0]); + + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rt_].UL[2]); + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[2]); + armAsm->And(a64::w1, a64::w1, 0x1F); + armAsm->Lsr(a64::w0, a64::w0, a64::w1); + armAsm->Sxtw(a64::x0, a64::w0); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rd_].UD[1]); +} + +void recPSRAVW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rt_].UL[0]); + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->And(a64::w1, a64::w1, 0x1F); + armAsm->Asr(a64::w0, a64::w0, a64::w1); + armAsm->Sxtw(a64::x0, a64::w0); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rd_].UD[0]); + + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rt_].UL[2]); + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[2]); + armAsm->And(a64::w1, a64::w1, 0x1F); + armAsm->Asr(a64::w0, a64::w0, a64::w1); + armAsm->Sxtw(a64::x0, a64::w0); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rd_].UD[1]); +} + +// ============================================================================ +// Multiply / Divide / MAC — at x86 parity via interpreter fallback +// ============================================================================ + +// These stay as REC_FUNC because production x86 stays REC_FUNC too (the +// MMI2_RECOMPILE native code paths in pcsx2/x86/iMMI.cpp live behind a never- +// defined macro): +// +// PMADDW / PMSUBW — interp models the PS2-hardware multiplication errata +// (`temp2 / 0xFFFFFFFF` + the conditional `+ 0x70000000` voodoo on boundary +// Rt values). Any non-errata fast path — NEON, SSE, or scalar — diverges +// from interp on essentially every input. Upstream keeps interp +// authoritative; the same applies here. +// +// PDIVW / PDIVBW / PDIVUW — AArch64 NEON has no integer divide, and x86's +// commented-out "native" path is itself `recCall(Interp::PDIV*)` after a +// targeted `_deleteEEreg(_Rd_, 0)`. There is no codegen to port. +// +// PMADDUW gets a native impl below — its interp is plain u64 arithmetic (no +// errata), so a NEON port matches interp bit-for-bit. +REC_FUNC(PMADDW); +REC_FUNC(PMSUBW); +REC_FUNC(PDIVW); +REC_FUNC(PDIVBW); +REC_FUNC(PDIVUW); + +// PMULTW: 2-lane signed 32x32->64 multiply on even-indexed source words. +// prod[0] = (s64)Rs.SL[0] * (s64)Rt.SL[0] +// prod[1] = (s64)Rs.SL[2] * (s64)Rt.SL[2] +// LO.UD[0..1] = sign-extended low32 of each product +// HI.UD[0..1] = sign-extended high32 of each product +// Rd.SD[0..1] = full 64-bit products +void recPMULTW() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + // Pack even-indexed 32-bit lanes into the low half (SL[0],SL[2] -> S[0],S[1]) + armAsm->Uzp1(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH.V4S()); + armAsm->Uzp1(RQSCRATCH2.V4S(), RQSCRATCH2.V4S(), RQSCRATCH2.V4S()); + + // 2-lane signed 32x32->64 -> { prod0, prod1 } as 2x64 + armAsm->Smull(RQSCRATCH3.V2D(), RQSCRATCH.V2S(), RQSCRATCH2.V2S()); + + // LO = sign-extended low32 of each product + armAsm->Xtn(RQSCRATCH.V2S(), RQSCRATCH3.V2D()); + armAsm->Sxtl(RQSCRATCH.V2D(), RQSCRATCH.V2S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + // HI = sign-extended high32 of each product (shift right narrow + sxtl) + armAsm->Shrn(RQSCRATCH.V2S(), RQSCRATCH3.V2D(), 32); + armAsm->Sxtl(RQSCRATCH.V2D(), RQSCRATCH.V2S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.HI.UQ)); + + if (_Rd_) + mmiStoreReg(_Rd_, RQSCRATCH3); +} + +// PMULTUW: 2-lane unsigned 32x32->64 multiply on even-indexed source words. +// prod[0] = (u64)Rs.UL[0] * (u64)Rt.UL[0] +// prod[1] = (u64)Rs.UL[2] * (u64)Rt.UL[2] +// LO.UD[0..1] = sign-extended low32 of each product (interp casts (s32)) +// HI.UD[0..1] = sign-extended high32 of each product +// Rd.UD[0..1] = full 64-bit products +void recPMULTUW() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + armAsm->Uzp1(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH.V4S()); + armAsm->Uzp1(RQSCRATCH2.V4S(), RQSCRATCH2.V4S(), RQSCRATCH2.V4S()); + + armAsm->Umull(RQSCRATCH3.V2D(), RQSCRATCH.V2S(), RQSCRATCH2.V2S()); + + armAsm->Xtn(RQSCRATCH.V2S(), RQSCRATCH3.V2D()); + armAsm->Sxtl(RQSCRATCH.V2D(), RQSCRATCH.V2S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + armAsm->Shrn(RQSCRATCH.V2S(), RQSCRATCH3.V2D(), 32); + armAsm->Sxtl(RQSCRATCH.V2D(), RQSCRATCH.V2S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.HI.UQ)); + + if (_Rd_) + mmiStoreReg(_Rd_, RQSCRATCH3); +} + +// PMADDUW: 2-lane unsigned 32x32+64->64 multiply-accumulate on even-indexed +// source words. +// tempu[k] = (u64)(LO.UL[2k] | (HI.UL[2k] << 32)) + (u64)Rs.UL[2k] * Rt.UL[2k] +// LO.UD[k] = sign-extended low32 of tempu[k] +// HI.UD[k] = sign-extended high32 of tempu[k] +// Rd.UD[k] = tempu[k] (full u64) +// +// Interp has no PS2 multiplication errata for the unsigned variant — plain u64 +// arithmetic — so this matches interp bit-for-bit (unlike PMADDW/PMSUBW which +// stay REC_FUNC above). +// +// Bypasses the LO/HI allocator path: EE rec's info-word layout packs EEREC_LO +// and EEREC_HI into the same 5-bit field (the EEREC_LO/EEREC_HI info-word +// macros decode the same bits), so it can't produce two distinct register +// indices. An op that needs both LO and HI live must load them from memory. +void recPMADDUW() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + // LO/HI are never NEON-resident in the EE rec (no opcode passes XMMINFO_*LO/HI), + // so the Ldrs below already see fresh memory — no allocator flush needed. + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + // Pack even-indexed 32-bit lanes into the low half (UL[0],UL[2] -> S[0],S[1]) + armAsm->Uzp1(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH.V4S()); + armAsm->Uzp1(RQSCRATCH2.V4S(), RQSCRATCH2.V4S(), RQSCRATCH2.V4S()); + + // 2-lane unsigned 32x32->64 product + armAsm->Umull(RQSCRATCH3.V2D(), RQSCRATCH.V2S(), RQSCRATCH2.V2S()); + + // Compose accumulator: { LO.UL[0] | HI.UL[0]<<32, LO.UL[2] | HI.UL[2]<<32 } + // Trn1.V4S(d, a, b) = { a[0], b[0], a[2], b[2] } -> as V2D, gives LO|HI<<32 per lane. + armAsm->Ldr(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + armAsm->Ldr(RQSCRATCH2, armCpuRegMem(&cpuRegs.HI.UQ)); + armAsm->Trn1(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH2.V4S()); + + // sum = composed + product (2x64 unsigned add) + armAsm->Add(RQSCRATCH3.V2D(), RQSCRATCH.V2D(), RQSCRATCH3.V2D()); + + // LO = sign-extended low32 of each 64-bit lane + armAsm->Xtn(RQSCRATCH.V2S(), RQSCRATCH3.V2D()); + armAsm->Sxtl(RQSCRATCH.V2D(), RQSCRATCH.V2S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + // HI = sign-extended high32 of each 64-bit lane + armAsm->Shrn(RQSCRATCH.V2S(), RQSCRATCH3.V2D(), 32); + armAsm->Sxtl(RQSCRATCH.V2D(), RQSCRATCH.V2S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.HI.UQ)); + + // Rd = full 2x64 unsigned sum + if (_Rd_) + mmiStoreReg(_Rd_, RQSCRATCH3); +} + +// PHMADH: 8-lane signed 16x16->32 multiply, pair-sum (n + n+1): +// sum[k] = Rs.SH[2k]*Rt.SH[2k] + Rs.SH[2k+1]*Rt.SH[2k+1] for k = 0..3 +// firsttemp[k] = Rs.SH[2k+1]*Rt.SH[2k+1] (the second product of each pair) +// LO = { sum[0], firsttemp[0], sum[2], firsttemp[2] } +// HI = { sum[1], firsttemp[1], sum[3], firsttemp[3] } +// Rd = { sum[0], sum[1], sum[2], sum[3] } (post-update LO/HI even lanes) +void recPHMADH() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + // p_lo = { Rs.SH[i] * Rt.SH[i] } for i = 0..3 (as 4x32) + // p_hi = { Rs.SH[i] * Rt.SH[i] } for i = 4..7 (as 4x32) + armAsm->Smull(RQSCRATCH3.V4S(), RQSCRATCH.V4H(), RQSCRATCH2.V4H()); + armAsm->Smull2(RQSCRATCH.V4S(), RQSCRATCH.V8H(), RQSCRATCH2.V8H()); + + // sums = ADDP(p_lo, p_hi).4S = { p0+p1, p2+p3, p4+p5, p6+p7 } + armAsm->Addp(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + // firsts = UZP2(p_lo, p_hi).4S = { p1, p3, p5, p7 } + armAsm->Uzp2(RQSCRATCH3.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + + // LO = TRN1.4S(sums, firsts) = { sum0, p1, sum2, p5 } + armAsm->Trn1(RQSCRATCH.V4S(), RQSCRATCH2.V4S(), RQSCRATCH3.V4S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + // HI = TRN2.4S(sums, firsts) = { sum1, p3, sum3, p7 } + armAsm->Trn2(RQSCRATCH.V4S(), RQSCRATCH2.V4S(), RQSCRATCH3.V4S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.HI.UQ)); + + if (_Rd_) + mmiStoreReg(_Rd_, RQSCRATCH2); +} + +// PHMSBH: 8-lane signed 16x16->32 multiply, pair-diff (n+1 - n): +// sum[k] = Rs.SH[2k+1]*Rt.SH[2k+1] - Rs.SH[2k]*Rt.SH[2k] (k = 0..3) +// firsttemp[k] = Rs.SH[2k+1]*Rt.SH[2k+1] (the second product per pair) +// LO = { sum[0], ~firsttemp[0], sum[2], ~firsttemp[2] } (note: bitwise NOT) +// HI = { sum[1], ~firsttemp[1], sum[3], ~firsttemp[3] } +// Rd = { sum[0], sum[1], sum[2], sum[3] } +void recPHMSBH() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + armAsm->Smull(RQSCRATCH3.V4S(), RQSCRATCH.V4H(), RQSCRATCH2.V4H()); + armAsm->Smull2(RQSCRATCH.V4S(), RQSCRATCH.V8H(), RQSCRATCH2.V8H()); + + // odds = UZP2(p_lo, p_hi).4S = { p1, p3, p5, p7 } (firsttemps) + // evens = UZP1(p_lo, p_hi).4S = { p0, p2, p4, p6 } + armAsm->Uzp2(RQSCRATCH2.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + armAsm->Uzp1(RQSCRATCH3.V4S(), RQSCRATCH3.V4S(), RQSCRATCH.V4S()); + + // sums = odds - evens = { p1-p0, p3-p2, p5-p4, p7-p6 } + armAsm->Sub(RQSCRATCH.V4S(), RQSCRATCH2.V4S(), RQSCRATCH3.V4S()); + + // nfirsts = ~odds (reuse RQSCRATCH3 — evens are dead after Sub) + armAsm->Mvn(RQSCRATCH3.V16B(), RQSCRATCH2.V16B()); + + if (_Rd_) + mmiStoreReg(_Rd_, RQSCRATCH); + + // LO = TRN1.4S(sums, nfirsts) = { sum0, ~p1, sum2, ~p5 } + armAsm->Trn1(RQSCRATCH2.V4S(), RQSCRATCH.V4S(), RQSCRATCH3.V4S()); + armAsm->Str(RQSCRATCH2, armCpuRegMem(&cpuRegs.LO.UQ)); + + // HI = TRN2.4S(sums, nfirsts) = { sum1, ~p3, sum3, ~p7 } + armAsm->Trn2(RQSCRATCH2.V4S(), RQSCRATCH.V4S(), RQSCRATCH3.V4S()); + armAsm->Str(RQSCRATCH2, armCpuRegMem(&cpuRegs.HI.UQ)); +} + +// PMULTH: 8-lane signed 16x16->32 multiply. +// r[i] = Rs.SH[i] * Rt.SH[i] for i in 0..7 +// LO = { r0, r1, r4, r5 } +// HI = { r2, r3, r6, r7 } +// Rd = { r0, r2, r4, r6 } (even-indexed products) +void recPMULTH() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + // q29 = SMULL Rs.4H, Rt.4H -> { r0,r1,r2,r3 } as 4x32 + // q31 = SMULL2 Rs.8H, Rt.8H -> { r4,r5,r6,r7 } as 4x32 (in-place over Rt) + armAsm->Smull(RQSCRATCH3.V4S(), RQSCRATCH.V4H(), RQSCRATCH2.V4H()); + armAsm->Smull2(RQSCRATCH2.V4S(), RQSCRATCH.V8H(), RQSCRATCH2.V8H()); + + // LO = TRN1.2D(prod_lo, prod_hi) = { prod_lo.D[0], prod_hi.D[0] } = { r0,r1,r4,r5 } + armAsm->Trn1(RQSCRATCH.V2D(), RQSCRATCH3.V2D(), RQSCRATCH2.V2D()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + // HI = TRN2.2D(prod_lo, prod_hi) = { prod_lo.D[1], prod_hi.D[1] } = { r2,r3,r6,r7 } + armAsm->Trn2(RQSCRATCH.V2D(), RQSCRATCH3.V2D(), RQSCRATCH2.V2D()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.HI.UQ)); + + if (_Rd_) + { + // Rd = UZP1.4S(prod_lo, prod_hi) = { r0, r2, r4, r6 } (even-indexed) + armAsm->Uzp1(RQSCRATCH.V4S(), RQSCRATCH3.V4S(), RQSCRATCH2.V4S()); + mmiStoreReg(_Rd_, RQSCRATCH); + } +} + +// PMADDH: 8-lane signed 16x16->32 multiply, accumulate into existing LO/HI. +// r[i] = Rs.SH[i] * Rt.SH[i] for i in 0..7 +// LO.UL[0..3] += { r0, r1, r4, r5 } +// HI.UL[0..3] += { r2, r3, r6, r7 } +// Rd = { new_LO[0], new_HI[0], new_LO[2], new_HI[2] } +void recPMADDH() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + armAsm->Smull(RQSCRATCH3.V4S(), RQSCRATCH.V4H(), RQSCRATCH2.V4H()); + armAsm->Smull2(RQSCRATCH2.V4S(), RQSCRATCH.V8H(), RQSCRATCH2.V8H()); + + // q30 = LO_increment = { r0,r1,r4,r5 } + armAsm->Trn1(RQSCRATCH.V2D(), RQSCRATCH3.V2D(), RQSCRATCH2.V2D()); + // q29 = HI_increment = { r2,r3,r6,r7 } + armAsm->Trn2(RQSCRATCH3.V2D(), RQSCRATCH3.V2D(), RQSCRATCH2.V2D()); + + // q31 = old LO; add and store + armAsm->Ldr(RQSCRATCH2, armCpuRegMem(&cpuRegs.LO.UQ)); + armAsm->Add(RQSCRATCH.V4S(), RQSCRATCH2.V4S(), RQSCRATCH.V4S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + // q31 = old HI; add and store + armAsm->Ldr(RQSCRATCH2, armCpuRegMem(&cpuRegs.HI.UQ)); + armAsm->Add(RQSCRATCH3.V4S(), RQSCRATCH2.V4S(), RQSCRATCH3.V4S()); + armAsm->Str(RQSCRATCH3, armCpuRegMem(&cpuRegs.HI.UQ)); + + if (_Rd_) + { + // Rd = TRN1.4S(new_LO, new_HI) = { LO[0], HI[0], LO[2], HI[2] } + armAsm->Trn1(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH3.V4S()); + mmiStoreReg(_Rd_, RQSCRATCH); + } +} + +// PMSUBH: 8-lane signed 16x16->32 multiply, subtract from existing LO/HI. +// r[i] = Rs.SH[i] * Rt.SH[i] for i in 0..7 +// LO.UL[0..3] -= { r0, r1, r4, r5 } +// HI.UL[0..3] -= { r2, r3, r6, r7 } +// Rd = { new_LO[0], new_HI[0], new_LO[2], new_HI[2] } +void recPMSUBH() +{ + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + if (_Rd_) + mmiInvalidateDest(_Rd_); + + mmiLoadReg(RQSCRATCH, _Rs_); + mmiLoadReg(RQSCRATCH2, _Rt_); + + armAsm->Smull(RQSCRATCH3.V4S(), RQSCRATCH.V4H(), RQSCRATCH2.V4H()); + armAsm->Smull2(RQSCRATCH2.V4S(), RQSCRATCH.V8H(), RQSCRATCH2.V8H()); + + armAsm->Trn1(RQSCRATCH.V2D(), RQSCRATCH3.V2D(), RQSCRATCH2.V2D()); + armAsm->Trn2(RQSCRATCH3.V2D(), RQSCRATCH3.V2D(), RQSCRATCH2.V2D()); + + armAsm->Ldr(RQSCRATCH2, armCpuRegMem(&cpuRegs.LO.UQ)); + armAsm->Sub(RQSCRATCH.V4S(), RQSCRATCH2.V4S(), RQSCRATCH.V4S()); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.LO.UQ)); + + armAsm->Ldr(RQSCRATCH2, armCpuRegMem(&cpuRegs.HI.UQ)); + armAsm->Sub(RQSCRATCH3.V4S(), RQSCRATCH2.V4S(), RQSCRATCH3.V4S()); + armAsm->Str(RQSCRATCH3, armCpuRegMem(&cpuRegs.HI.UQ)); + + if (_Rd_) + { + armAsm->Trn1(RQSCRATCH.V4S(), RQSCRATCH.V4S(), RQSCRATCH3.V4S()); + mmiStoreReg(_Rd_, RQSCRATCH); + } +} + +// ============================================================================ +// QFSRV: Quad Funnel Shift Right Variable +// ============================================================================ + +// QFSRV: Rd = {Rs, Rt} >> (sa * 8), truncated to 128 bits. +// cpuRegs.sa is in bytes (0-15). Concatenate Rt (low) and Rs (high) +// into a 256-bit value, shift right by sa bytes, take lower 128 bits. +// Implementation: store {Rt, Rs} to adjacent memory, unaligned load at offset sa. +// Matches x86 approach using tempqw buffer. +alignas(16) static u8 s_qfsrvTemp[32]; + +void recQFSRV() +{ + if (!_Rd_) return; + + mmiFlushReg(_Rs_); + mmiFlushReg(_Rt_); + mmiInvalidateDest(_Rd_); + + // Adjacent-source fast path: when Rs == Rt+1 the 256-bit + // {Rt:Rs} window already exists contiguously in the GPR array + // (GPR.r[Rt] immediately precedes GPR.r[Rt+1]==GPR.r[Rs], 32 bytes), now + // memory-coherent after the flushes above. Read the unaligned 128 bits + // directly at &GPR.r[Rt] + sa and skip the two temp stores. sa is 0..15 so + // the load stays within the two registers' 32 bytes. Gate on Rt != 0 to avoid + // depending on GPR.r[0] holding zero in memory (the slow path Movi's it). + if (_Rt_ != 0 && _Rs_ == _Rt_ + 1) + { + armLoadEERegPtr(RWSCRATCH, &cpuRegs.sa); + armMoveAddressToReg(RSCRATCHADDR, &cpuRegs.GPR.r[_Rt_]); + armAsm->Add(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); + armAsm->Ldr(RQSCRATCH, a64::MemOperand(RSCRATCHADDR)); + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rd_])); + return; + } + + // Store Rt at temp[0:15], Rs at temp[16:31] + mmiLoadReg(RQSCRATCH, _Rt_); + armMoveAddressToReg(RSCRATCHADDR, &s_qfsrvTemp[0]); + armAsm->Str(RQSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + mmiLoadReg(RQSCRATCH, _Rs_); + armMoveAddressToReg(RSCRATCHADDR, &s_qfsrvTemp[16]); + armAsm->Str(RQSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + // Load sa (byte offset) + armLoadEERegPtr(RWSCRATCH, &cpuRegs.sa); + + // Unaligned 128-bit load from temp + sa + armMoveAddressToReg(RSCRATCHADDR, &s_qfsrvTemp[0]); + armAsm->Add(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); // addr = temp + sa + armAsm->Ldr(RQSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + // Store result to Rd + armAsm->Str(RQSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[_Rd_])); +} + +// ============================================================================ +// Other MMI +// ============================================================================ + +// PLZCW: count leading sign bits (excluding the sign bit itself) for words 0 and 1 +void recPLZCW() +{ + if (!_Rd_) return; + mmiFlushReg(_Rs_); + mmiInvalidateDest(_Rd_); + + // Word 0: ARM64 CLS counts leading sign bits excluding the MSB sign bit itself, + // which matches the PS2 PLZCW definition (CountLeadingSignBits - 1). + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->Cls(a64::w0, a64::w0); + armStoreEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rd_].UL[0]); + + // Word 1 + armLoadEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rs_].UL[1]); + armAsm->Cls(a64::w0, a64::w0); + armStoreEERegPtr(a64::w0, &cpuRegs.GPR.r[_Rd_].UL[1]); +} + +// PMFHL — read LO/HI 128-bit register pair, dispatch on sa: +// 0x00 LW : Rd = { LO.UL[0], HI.UL[0], LO.UL[2], HI.UL[2] } +// 0x01 UW : Rd = { LO.UL[1], HI.UL[1], LO.UL[3], HI.UL[3] } +// 0x02 SLW: composed s64 (HI.UL[2k]:LO.UL[2k]) signed-saturated to s32 +// then sign-extended to s64; written to Rd.UD[k] for k=0,1. +// 0x03 LH : Rd.US lanes from even-indexed LO/HI halfwords, interleaved +// at 32-bit-pair granularity. +// 0x04 SH : per-lane s32→s16 signed saturation of LO/HI words, interleaved +// at 32-bit-pair granularity. +// sa >= 5 — interpreter is a no-op (no default-case in MMI.cpp PMFHL); mirror +// that by early-returning without touching Rd. (x86 rec asserts on this path, +// but the interp doesn't, so the recompiled-vs-interp diff would fail an assert +// rather than catch real divergence — matching the interp is the safer choice.) +void recPMFHL() +{ + if (!_Rd_) + return; + if (_Sa_ > 0x04) + return; + + // LO/HI loaded directly from memory rather than via the allocator + // (XMMINFO_READLO | XMMINFO_READHI). Reason: the arm64 EE rec's + // info-word layout packs PROCESS_EE_SET_LO and PROCESS_EE_SET_HI + // into the SAME 5-bit field at bits 23..27 (see iCore-arm64.h — + // EEREC_LO == EEREC_HI == EEREC_ACC). PMFHL needs both LO and HI as + // simultaneous live inputs, which that field collision cannot represent. + // Bypass with explicit Ldrs. LO/HI are never NEON-resident in the EE rec + // (no opcode passes XMMINFO_*LO/HI), so the memory image is already current. + + int info = eeRecompileCodeXMM(XMMINFO_WRITED); + const a64::VRegister qd = armQRegister(EEREC_D); + (void)info; + + // Pre-loaded into reserved scratch quads (outside the allocator pool). + const a64::VRegister qlo = RQSCRATCH; + const a64::VRegister qhi = RQSCRATCH2; + armAsm->Ldr(qlo, armCpuRegMem(&cpuRegs.LO.UQ)); + armAsm->Ldr(qhi, armCpuRegMem(&cpuRegs.HI.UQ)); + + switch (_Sa_) + { + case 0x00: // LW: pick even-indexed words from LO/HI and interleave + // TRN1.V4S → { LO.S[0], HI.S[0], LO.S[2], HI.S[2] } + armAsm->Trn1(qd.V4S(), qlo.V4S(), qhi.V4S()); + break; + + case 0x01: // UW: pick odd-indexed words from LO/HI and interleave + // TRN2.V4S → { LO.S[1], HI.S[1], LO.S[3], HI.S[3] } + armAsm->Trn2(qd.V4S(), qlo.V4S(), qhi.V4S()); + break; + + case 0x02: // SLW: compose s64 (HI:LO) per even-word lane, saturate to s32, sign-extend back + // TRN1.V4S → V2D { (HI[0]:LO[0]), (HI[2]:LO[2]) } (LO in low 32 of each 64) + // SQXTN.V2S — signed-saturating narrow 2x64 → 2x32 (matches interp's + // "in-range -> (s64)(s32)LO.UL[2k]; saturate to INT32_MIN/MAX" bounds). + // SXTL.V2D — sign-extend 2x32 → 2x64 (= the recorded Rd.UD shape). + armAsm->Trn1(qd.V4S(), qlo.V4S(), qhi.V4S()); + armAsm->Sqxtn(qd.V2S(), qd.V2D()); + armAsm->Sxtl(qd.V2D(), qd.V2S()); + break; + + case 0x03: // LH: even halfwords from LO/HI, interleaved at S-pair granularity + // UZP1.V8H(x, x) gathers x's even halfwords into the low 64 bits of x. + // ZIP1.V4S picks S[0]/S[1] of each input → output S[0..3] = + // { (LO[0]:LO[2]), (HI[0]:HI[2]), (LO[4]:LO[6]), (HI[4]:HI[6]) } + // which as V8H = { LO[0], LO[2], HI[0], HI[2], LO[4], LO[6], HI[4], HI[6] }. + armAsm->Uzp1(qlo.V8H(), qlo.V8H(), qlo.V8H()); + armAsm->Uzp1(qhi.V8H(), qhi.V8H(), qhi.V8H()); + armAsm->Zip1(qd.V4S(), qlo.V4S(), qhi.V4S()); + break; + + case 0x04: // SH: signed-saturating narrow 32→16 per word, interleaved at S-pair granularity + // SQXTN.V4H — 4x32 signed-sat narrowed to 4x16 in low 64 of each scratch. + // ZIP1.V4S → output S[0..3] = { sat(LO[0..1]), sat(HI[0..1]), sat(LO[2..3]), sat(HI[2..3]) } + // which as V8H is exactly the interp's PMFHL_CLAMP-per-lane pattern. + armAsm->Sqxtn(qlo.V4H(), qlo.V4S()); + armAsm->Sqxtn(qhi.V4H(), qhi.V4S()); + armAsm->Zip1(qd.V4S(), qlo.V4S(), qhi.V4S()); + break; + } +} + +// PMTHL.LW: even-indexed words of LO/HI receive Rs's four words; the +// odd-indexed words (UL[1] and UL[3] of each) are preserved. Matches +// interp at MMI.cpp:217-224 and x86 BLENDPS/SHUFPS sequence at +// iMMI.cpp:234-248. Strategy: load LO/HI as Q regs, INS lanes 1+3 from +// the prior values to preserve them; lane 0 and lane 2 come from Rs's +// word 0/2 for LO, word 1/3 for HI. +void recPMTHL() +{ + if (_Sa_ != 0) + return; + + mmiFlushReg(_Rs_); + mmiLoadReg(RQSCRATCH, _Rs_); + + // LO_new = [Rs.UL[0], LO.UL[1], Rs.UL[2], LO.UL[3]] + armAsm->Ldr(RQSCRATCH2, armCpuRegMem(&cpuRegs.LO.UQ)); + armAsm->Mov(RQSCRATCH3.V16B(), RQSCRATCH.V16B()); + armAsm->Ins(RQSCRATCH3.V4S(), 1, RQSCRATCH2.V4S(), 1); + armAsm->Ins(RQSCRATCH3.V4S(), 3, RQSCRATCH2.V4S(), 3); + armAsm->Str(RQSCRATCH3, armCpuRegMem(&cpuRegs.LO.UQ)); + + // HI_new = [Rs.UL[1], HI.UL[1], Rs.UL[3], HI.UL[3]] + armAsm->Ldr(RQSCRATCH3, armCpuRegMem(&cpuRegs.HI.UQ)); + armAsm->Ins(RQSCRATCH3.V4S(), 0, RQSCRATCH.V4S(), 1); + armAsm->Ins(RQSCRATCH3.V4S(), 2, RQSCRATCH.V4S(), 3); + armAsm->Str(RQSCRATCH3, armCpuRegMem(&cpuRegs.HI.UQ)); +} + +} // namespace MMI +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900-arm64.cpp b/pcsx2/arm64/iR5900-arm64.cpp new file mode 100644 index 0000000000..2c8fe222a8 --- /dev/null +++ b/pcsx2/arm64/iR5900-arm64.cpp @@ -0,0 +1,2013 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE (R5900) Dynamic Recompiler Core +// Dispatcher, block management, all instructions as interpreter fallbacks. + +#include +#include +#include +#include + +#include "arm64/iR5900-arm64.h" +#include "arm64/iR5900Analysis.h" +#include "arm64/AsmHelpers.h" +#include "Host.h" +#include "R3000A.h" +#include "R5900.h" +#include "arm64/BaseblockEx-arm64.h" +#include "R5900OpcodeTables.h" +#include "Common.h" +#include "VMManager.h" +#include "Config.h" +#include "vtlb.h" +#include "Dmac.h" +#include "GS.h" +#ifdef PCSX2_RECOMPILER_TESTS +#include "ee_divtrace.h" // diagnostic divergence-trace hooks (test builds only) +#endif + +#include "common/Assertions.h" +#include "common/AlignedMalloc.h" +#include "common/Console.h" +#include "common/FastJmp.h" +#include "common/HeapArray.h" +#include "common/Perf.h" + +#include "DebugTools/Breakpoints.h" + +namespace a64 = vixl::aarch64; + +// ===================================================================================================== +// Global State +// ===================================================================================================== + +u32 maxrecmem = 0; +u32 pc; +int g_branch; +u32 target; +u32 s_nBlockCycles; +bool s_nBlockInterlocked; + +bool g_recompilingDelaySlot = false; +bool g_cpuFlushedPC = false; +bool g_cpuFlushedCode = false; + +// Constant propagation — defined here, declared extern in iR5900-arm64.h + +static uptr recLUT[0x10000]; +static u32 hwLUT[0x10000]; + +static __fi u32 HWADDR(u32 mem) { return hwLUT[mem >> 16] + mem; } + +static BASEBLOCK* recRAM = nullptr; +static BASEBLOCK* recROM = nullptr; +static BASEBLOCK* recROM1 = nullptr; +static Arm64BaseBlocks recBlocks; +static u8* recPtr = nullptr; +static u8* recPtrEnd = nullptr; + +static EEINST* s_pInstCache = nullptr; +static u32 s_nInstCacheSize = 0; + +static BASEBLOCK* s_pCurBlock = nullptr; +static BASEBLOCKEX* s_pCurBlockEx = nullptr; + +static u32 s_nEndBlock = 0; +static u32 s_branchTo; +static bool s_nBlockFF; + +static DynamicHeapArray recLutReserve_RAM; +static DynamicHeapArray recLutUnmapped; +static DynamicHeapArray recRAMCopy; +static size_t recLutEntries = 0; + +static ArmConstantPool s_eeConstantPool; + +// Execution state +static fastjmp_buf m_SetJmp_StateCheck; +static bool eeCpuExecuting = false; +static bool eeRecNeedsReset = false; +static bool eeRecExitRequested = false; + +#ifdef PCSX2_RECOMPILER_TESTS +// Harness-entry state. Set by recEeExecuteBlock before entering the JIT, +// observed by recEventTest at every iBranchTest event-due fall-out. +// Production execution (recExecute) leaves g_eeHarnessActive false; the +// predicate short-circuits on it. Compiled out entirely in release builds +// (ENABLE_RECOMPILER_TEST_HOOKS=OFF) so the live VM path carries no +// test-only symbols or branches. +static bool g_eeHarnessActive = false; +static u32 g_eeHarnessParkPc = 0; +static s32 g_eeHarnessCycleBudget = 0; +static u64 g_eeHarnessCycleStart = 0; +static constexpr s32 kExecuteBlockSafetyCap = 1 << 20; +#endif + +// Self-modifying code detection +static u16 manual_page[Ps2MemSize::MainRam / 4096] = {}; +static u8 manual_counter[Ps2MemSize::MainRam / 4096] = {}; + +// Forward declarations +static void recRecompile(const u32 startpc); +static void recResetRaw(); +static void recExitExecution(); +static void recSafeExitExecution(); +#ifdef PCSX2_RECOMPILER_TESTS +static bool harnessShouldExit(); +#endif +static void recError(u32 error); +static void dyna_block_discard(u32 start, u32 sz); +static void dyna_page_reset(u32 start, u32 sz); +static void iopClearRecLUT(BASEBLOCK* base, int count); + +// recBackpropBSC declared in arm64/iR5900Analysis.h + +// ===================================================================================================== +// Native Codegen Verification Mode +// ===================================================================================================== + +#ifdef VERIFY_NATIVE_CODEGEN + +// Snapshot of GPR + HI/LO state before the native instruction executes +static GPR_reg s_verifyGPR[32]; +static GPR_reg s_verifyHI, s_verifyLO; +static u32 s_verifyMismatchCount = 0; + +// VU0 state snapshot for COP2 verification +static VECTOR s_verifyVF[32]; +static VECTOR s_verifyACC; +static REG_VI s_verifyVI[32]; +static u32 s_verifyClipFlag; + +// Called at runtime BEFORE the native instruction: snapshot all state +static void verifySnapshotPre(u32 code, u32 instPC) +{ + memcpy(s_verifyGPR, cpuRegs.GPR.r, sizeof(s_verifyGPR)); + s_verifyHI = cpuRegs.HI; + s_verifyLO = cpuRegs.LO; + + // COP2: also snapshot VU0 state + if ((code >> 26) == 0x12) // COP2 opcode + { + memcpy(s_verifyVF, VU0.VF, sizeof(s_verifyVF)); + s_verifyACC = VU0.ACC; + memcpy(s_verifyVI, VU0.VI, sizeof(s_verifyVI)); + s_verifyClipFlag = VU0.clipflag; + } +} + +// Called at runtime AFTER the native instruction: re-run via interpreter on the snapshot and compare +static void verifyCheckPost(u32 code, u32 instPC) +{ + const bool isCOP2 = (code >> 26) == 0x12; + + // Save native results + GPR_reg nativeGPR[32]; + GPR_reg nativeHI, nativeLO; + memcpy(nativeGPR, cpuRegs.GPR.r, sizeof(nativeGPR)); + nativeHI = cpuRegs.HI; + nativeLO = cpuRegs.LO; + + // Save native VU0 state for COP2 + VECTOR nativeVF[32]; + VECTOR nativeACC; + REG_VI nativeVI[32]; + u32 nativeClipFlag = 0; + if (isCOP2) + { + memcpy(nativeVF, VU0.VF, sizeof(nativeVF)); + nativeACC = VU0.ACC; + memcpy(nativeVI, VU0.VI, sizeof(nativeVI)); + nativeClipFlag = VU0.clipflag; + } + + // Restore pre-instruction state + memcpy(cpuRegs.GPR.r, s_verifyGPR, sizeof(s_verifyGPR)); + cpuRegs.HI = s_verifyHI; + cpuRegs.LO = s_verifyLO; + if (isCOP2) + { + memcpy(VU0.VF, s_verifyVF, sizeof(s_verifyVF)); + VU0.ACC = s_verifyACC; + memcpy(VU0.VI, s_verifyVI, sizeof(s_verifyVI)); + VU0.clipflag = s_verifyClipFlag; + } + + // Run interpreter + const u32 savedCode = cpuRegs.code; + cpuRegs.code = code; + const R5900::OPCODE& opcode = R5900::GetCurrentInstruction(); + if (opcode.interpret) + opcode.interpret(); + cpuRegs.code = savedCode; + + // Compare results + bool mismatch = false; + static const char* gpr_names[] = { + "zero","at","v0","v1","a0","a1","a2","a3", + "t0","t1","t2","t3","t4","t5","t6","t7", + "s0","s1","s2","s3","s4","s5","s6","s7", + "t8","t9","k0","k1","gp","sp","fp","ra" + }; + + for (int i = 1; i < 32; i++) // skip r0 + { + if (cpuRegs.GPR.r[i].UD[0] != nativeGPR[i].UD[0]) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" %s(r%d): native=0x%016llX interp=0x%016llX (pre=0x%016llX)", + gpr_names[i], i, nativeGPR[i].UD[0], cpuRegs.GPR.r[i].UD[0], s_verifyGPR[i].UD[0]); + } + } + + if (cpuRegs.HI.UD[0] != nativeHI.UD[0]) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" HI: native=0x%016llX interp=0x%016llX", nativeHI.UD[0], cpuRegs.HI.UD[0]); + } + if (cpuRegs.LO.UD[0] != nativeLO.UD[0]) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" LO: native=0x%016llX interp=0x%016llX", nativeLO.UD[0], cpuRegs.LO.UD[0]); + } + + // COP2: compare VU0 state (tolerate 1-ULP float differences) + if (isCOP2) + { + auto ulpDiff = [](u32 a, u32 b) -> u32 { + return (a > b) ? (a - b) : (b - a); + }; + + for (int i = 1; i < 32; i++) // skip VF0 + { + bool vfMismatch = false; + for (int lane = 0; lane < 4; lane++) + { + if (ulpDiff(VU0.VF[i].UL[lane], nativeVF[i].UL[lane]) > 100) + vfMismatch = true; + } + if (vfMismatch) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" VF%d: native=[%08X,%08X,%08X,%08X] interp=[%08X,%08X,%08X,%08X]", + i, nativeVF[i].UL[0], nativeVF[i].UL[1], nativeVF[i].UL[2], nativeVF[i].UL[3], + VU0.VF[i].UL[0], VU0.VF[i].UL[1], VU0.VF[i].UL[2], VU0.VF[i].UL[3]); + } + } + bool accMismatch = false; + for (int lane = 0; lane < 4; lane++) + { + if (ulpDiff(VU0.ACC.UL[lane], nativeACC.UL[lane]) > 100) + accMismatch = true; + } + if (accMismatch) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" ACC: native=[%08X,%08X,%08X,%08X] interp=[%08X,%08X,%08X,%08X]", + nativeACC.UL[0], nativeACC.UL[1], nativeACC.UL[2], nativeACC.UL[3], + VU0.ACC.UL[0], VU0.ACC.UL[1], VU0.ACC.UL[2], VU0.ACC.UL[3]); + } + // Check MAC and status flags + if (VU0.VI[REG_MAC_FLAG].UL != nativeVI[REG_MAC_FLAG].UL) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" MAC_FLAG: native=0x%04X interp=0x%04X", nativeVI[REG_MAC_FLAG].UL, VU0.VI[REG_MAC_FLAG].UL); + } + if (VU0.VI[REG_STATUS_FLAG].UL != nativeVI[REG_STATUS_FLAG].UL) + { + if (!mismatch) { Console.Error("VERIFY MISMATCH at pc=0x%08X code=0x%08X:", instPC, code); mismatch = true; } + Console.Error(" STATUS_FLAG: native=0x%04X interp=0x%04X", nativeVI[REG_STATUS_FLAG].UL, VU0.VI[REG_STATUS_FLAG].UL); + } + } + + if (mismatch) + { + const u32 op = code >> 26; + const u32 rs = (code >> 21) & 0x1f; + const u32 rt = (code >> 16) & 0x1f; + const u32 rd = (code >> 11) & 0x1f; + const u32 sa = (code >> 6) & 0x1f; + const u32 funct = code & 0x3f; + Console.Error(" Decode: op=%d rs=%d rt=%d rd=%d sa=%d funct=%d", + op, rs, rt, rd, sa, funct); + s_verifyMismatchCount++; + // Don't assert — remaining mismatches are rounding-induced flag diffs + // (MAC zero flag differs when result is on the boundary of 0.0). + // Log only, no crash. + } + + // Restore native results so execution continues with native values + memcpy(cpuRegs.GPR.r, nativeGPR, sizeof(nativeGPR)); + cpuRegs.HI = nativeHI; + cpuRegs.LO = nativeLO; + if (isCOP2) + { + memcpy(VU0.VF, nativeVF, sizeof(nativeVF)); + VU0.ACC = nativeACC; + memcpy(VU0.VI, nativeVI, sizeof(nativeVI)); + VU0.clipflag = nativeClipFlag; + } +} + +#endif // VERIFY_NATIVE_CODEGEN + +#define GETBLOCK(x) PC_GETBLOCK_(x, recLUT) + +// ===================================================================================================== +// Dynamically Compiled Dispatchers - R5900 ARM64 +// ===================================================================================================== + +static const void* DispatcherEvent = nullptr; +static const void* DispatcherReg = nullptr; +static const void* JITCompile = nullptr; +static const void* EnterRecompiledCode = nullptr; +static const void* DispatchBlockDiscard = nullptr; +static const void* DispatchPageReset = nullptr; +static const void* UnmappedRecLUTPage = nullptr; + +static void recEventTest() +{ + eeEventTestIsActive = true; + _cpuEventTest_Shared(); + eeEventTestIsActive = false; + + if (eeRecExitRequested) + { + eeRecExitRequested = false; + recExitExecution(); + } + +#ifdef PCSX2_RECOMPILER_TESTS + if (harnessShouldExit()) + recExitExecution(); +#endif + + if (eeRecNeedsReset) + { + eeRecNeedsReset = false; + recResetRaw(); + } +} + +#ifdef PCSX2_RECOMPILER_TESTS +// Harness-exit predicate. Returns true when running under recEeExecuteBlock +// AND either the parking PC has been reached or the cycle budget has been +// exhausted. Test-only — release builds drop the call site entirely. +static bool harnessShouldExit() +{ + if (!g_eeHarnessActive) + return false; + if (cpuRegs.pc == g_eeHarnessParkPc) + return true; + const u64 elapsed = cpuRegs.cycle - g_eeHarnessCycleStart; + return elapsed >= static_cast(g_eeHarnessCycleBudget); +} +#endif + +// ARM64 EE dispatcher — same two-level LUT as IOP but using cpuRegs.pc +static const void* _DynGen_DispatcherReg() +{ + u8* retval = armGetCurrentCodePointer(); + + armAsm->Ldr(a64::w0, armCpuRegMem(&cpuRegs.pc)); + + // Two-level LUT lookup: + // base = recLUT[pc >> 16] + // block = *(BASEBLOCK*)(base + pc * sizeof(BASEBLOCK)/4) + // sizeof(BASEBLOCK) = 8, so /4 = *2, hence: base + pc*2 + // Note: use FULL pc as index (not pc & 0xFFFF) because recLUT_SetPage + // adjusts the base address to account for the upper bits. + armAsm->Lsr(a64::w1, a64::w0, 16); + armMoveAddressToReg(RSCRATCHADDR, recLUT); + armAsm->Ldr(RSCRATCHADDR, a64::MemOperand(RSCRATCHADDR, a64::x1, a64::LSL, 3)); + + // Index with full PC: base + pc * 2 (not (pc & 0xFFFF) * 2) + armAsm->Add(RSCRATCHADDR, RSCRATCHADDR, a64::Operand(a64::x0, a64::LSL, 1)); + + armAsm->Ldr(RSCRATCHADDR, a64::MemOperand(RSCRATCHADDR)); + + armAsm->Br(RSCRATCHADDR); + + return retval; +} + +static const void* _DynGen_JITCompile() +{ + u8* retval = armGetCurrentCodePointer(); + + // Flush pinned cycle counter before the C call, then reload after — + // recRecompile itself doesn't modify cpuRegs.cycle, but other paths + // (e.g. block discard) might, and the convention is "every C-call + // boundary syncs RECCYCLE both ways". + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armAsm->Ldr(RWARG1, armCpuRegMem(&cpuRegs.pc)); + armEmitCall((void*)recRecompile); + + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armEmitJmp(DispatcherReg); + + return retval; +} + +static const void* _DynGen_DispatcherEvent() +{ + u8* retval = armGetCurrentCodePointer(); + // Flush pinned cycle for recEventTest (it reads cpuRegs.cycle for + // counter / interrupt scheduling), then reload — the event test may + // modify cpuRegs.cycle (e.g. fast-forwarding to nextEventCycle). + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armEmitCall((void*)recEventTest); + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + return retval; // falls through to DispatcherReg +} + +static const void* _DynGen_EnterRecompiledCode() +{ + u8* retval = armGetCurrentCodePointer(); + + // We never return through this function — we exit via fastjmp_jmp (longjmp). + // fastjmp_set/fastjmp_jmp save and restore callee-saved registers, so we + // don't need armBeginStackFrame. Just align the stack (AArch64 requires + // 16-byte alignment). Match x86 pattern: only adjust SP, don't save regs. + armAsm->Sub(a64::sp, a64::sp, 16); + + // Park PS2 FPU clamp constants in callee-saved scalar NEON registers. + // s8 = +FLT_MAX, s9 = -FLT_MAX. AAPCS64 preserves the lower 64 bits of + // d8-d15 across C calls, so these survive every armEmitCall path inside + // JIT blocks without compile-time tracking. fpuClampResult and iCOP2 + // scalar VDIV/VSQRT/VRSQRT clamps read them directly. v8/v9 are removed + // from the NEON allocator pool (see NEON_RESERVED_FPU_{MAX,MIN} in + // iCore-arm64.cpp), so nothing in JIT codegen can clobber them. + armAsm->Ldr(a64::s8, FLT_MAX); + armAsm->Ldr(a64::s9, -FLT_MAX); + + // Load fastmem base into x19 if enabled + if (CHECK_FASTMEM) + { + armMoveAddressToReg(RSCRATCHADDR, &vtlb_private::vtlbdata.fastmem_base); + armAsm->Ldr(RFASTMEMBASE, a64::MemOperand(RSCRATCHADDR)); + } + + // Load &cpuRegs into RSTATE. Callee-saved, never modified by C, so this + // load happens once per JIT entry. Subsequent cpuRegs.X accesses become + // `Ldr/Str ..., [RSTATE, #offsetof(...)]` instead of materializing the + // full address each time. + armMoveAddressToReg(RSTATE, &cpuRegs); + + // Load pinned cycle counter into RECCYCLE. The convention is that + // RECCYCLE holds cpuRegs.cycle for the entire duration of JIT + // execution, with flush+reload around C calls. + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + // Load &VU0 into RVU0. Same idea as RSTATE: VU0 is a static reference + // (constant address), so iCOP2 codegen can reach every VURegs field via + // [RVU0, #imm12]. Survives both armEmitCall and mVU dispatcher runs. + armMoveAddressToReg(RVU0, &VU0); + + // Jump into dispatcher + armEmitJmp(DispatcherReg); + + // Exit point — restore callee-saved and return + // We get here via fastjmp_jmp, not via normal return + return retval; +} + +static const void* _DynGen_DispatchBlockDiscard() +{ + u8* retval = armGetCurrentCodePointer(); + armEmitCall((void*)dyna_block_discard); + armEmitJmp(DispatcherReg); + return retval; +} + +static const void* _DynGen_DispatchPageReset() +{ + u8* retval = armGetCurrentCodePointer(); + armEmitCall((void*)dyna_page_reset); + armEmitJmp(DispatcherReg); + return retval; +} + +static const void* _DynGen_UnmappedRecLUTPage() +{ + u8* retval = armGetCurrentCodePointer(); + armAsm->Mov(RWARG1, 0); + armEmitCall((void*)(void(*)(u32))recError); + return retval; +} + +static void _DynGen_Dispatchers() +{ + const u8* start = armGetCurrentCodePointer(); + + DispatcherEvent = _DynGen_DispatcherEvent(); + DispatcherReg = _DynGen_DispatcherReg(); + + JITCompile = _DynGen_JITCompile(); + EnterRecompiledCode = _DynGen_EnterRecompiledCode(); + DispatchBlockDiscard = _DynGen_DispatchBlockDiscard(); + DispatchPageReset = _DynGen_DispatchPageReset(); + UnmappedRecLUTPage = _DynGen_UnmappedRecLUTPage(); + + // Block linker needs JITCompile so it can route stale / not-yet-compiled + // link sites through the dispatcher path. + recBlocks.SetJITCompile(JITCompile); + + Perf::any.Register(start, static_cast(armGetCurrentCodePointer() - start), "EE Dispatcher"); +} + +// ===================================================================================================== +// Error handling +// ===================================================================================================== + +static void recError(u32 error) +{ + switch (error) + { + case 0: + Host::ReportErrorAsync("R5900 Exception", + fmt::format("Unrecognized opcode (PC: 0x{:08x})", cpuRegs.pc)); + break; + + case 1: + Host::ReportErrorAsync("R5900 Exception", + fmt::format("Jump to unaligned address (PC: 0x{:08x})", cpuRegs.pc)); + break; + } + + VMManager::SetPaused(true); + Cpu->ExitExecution(); +} + +// ===================================================================================================== +// Code generation helpers +// ===================================================================================================== + +void iFlushCall(int flushtype) +{ + // Free caller-saved registers + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (!arm64gprs[i].inuse) + continue; + + if (!armIsCalleeSavedRegister(i) || + ((flushtype & FLUSH_FREE_NONTEMP_X86) && arm64gprs[i].type != ARM64TYPE_TEMP) || + ((flushtype & FLUSH_FREE_TEMP_X86) && arm64gprs[i].type == ARM64TYPE_TEMP)) + { + _freeArm64GPR(i); + } + } + + // Only the lower 64 bits of v8-v15 are callee-saved per AAPCS64; the + // NEON allocator uses 128-bit slots, so all of them are effectively + // caller-saved across a C call. Always free + writeback. Matches x86 + // iFlushCall (pcsx2/x86/ix86-32/iR5900.cpp:1196-1207) which also + // unconditionally evicts caller-saved XMM regs. + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse) + _freeNEONreg(i); + } + + if (flushtype & FLUSH_ALL_X86) + _flushArm64GPRregs(); + + if (flushtype & FLUSH_CONSTANT_REGS) + _flushConstRegs(true); + + if ((flushtype & FLUSH_PC) && !g_cpuFlushedPC) + { + armAsm->Mov(RWSCRATCH, pc); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.pc)); + g_cpuFlushedPC = true; + } + + if ((flushtype & FLUSH_CODE) && !g_cpuFlushedCode) + { + armAsm->Mov(RWSCRATCH, cpuRegs.code); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.code)); + g_cpuFlushedCode = true; + } +} + +// Flag set by cpuTlbMiss to signal that a TLB exception occurred during +// an interpreter call. The JIT block checks this after recCall and exits +// to the dispatcher if set, so the exception vector gets dispatched. +u32 s_recTlbMissOccurred = 0; + +// Emit the post-interpreter-call TLB-miss exception dispatch. cpuTlbMiss sets +// s_recTlbMissOccurred and moves cpuRegs.pc to the exception vector; when set we +// clear the flag and exit to DispatcherReg rather than continue the block at the +// wrong PC. DispatcherReg/s_recTlbMissOccurred are file-local here, so this is +// the shared entry point used by recCall and recVTLB-arm64.cpp's recUnalignedCall. +void recEmitInterpTlbMissCheck() +{ + // Dispatch to DispatcherReg (not DispatcherEvent, which runs event + // processing that may interfere with the pending exception state). + a64::Label noException; + armMoveAddressToReg(RSCRATCHADDR, &s_recTlbMissOccurred); + armAsm->Ldr(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + armAsm->Cbz(RWSCRATCH, &noException); + armAsm->Str(a64::wzr, a64::MemOperand(RSCRATCHADDR)); // clear flag + armEmitJmp(DispatcherReg); + armAsm->Bind(&noException); +} + +void recCall(void (*func)()) +{ + iFlushCall(FLUSH_INTERPRETER); + + // Flush RECCYCLE → cpuRegs.cycle so the interpreter sees the live cycle + // value (some opcodes — COP0 Count, TLB miss, branch helpers — read it). + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armEmitCall((void*)func); + + // Reload RECCYCLE in case the interpreter modified cpuRegs.cycle. + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + // After interpreter calls, dispatch a pending TLB-miss exception. + recEmitInterpTlbMissCheck(); +} + +void recBranchCall(void (*func)()) +{ + iFlushCall(FLUSH_INTERPRETER); + + // Apply accumulated block cycles to RECCYCLE, then flush to memory + // before the C call — the interpreter's intEventTest reads + // cpuRegs.cycle. Reload after, so the g_branch=2 exit code that + // follows can keep using RECCYCLE. + u32 cycles = scaleblockcycles_clear(); + if (cycles > 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armEmitCall((void*)func); + g_branch = 2; + + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); +} + +// s_nBlockCycles is 3-bit fixed point. Divide by 8 when done! +// Scaling blocks under 40 cycles seems to produce countless problems, so let's try to avoid them. +// Matches x86 scaleblockcycles_calculation() in ix86-32/iR5900.cpp +#define DEFAULT_SCALED_BLOCKS() (s_nBlockCycles >> 3) + +static u32 scaleblockcycles_calculation() +{ + const bool lowcycles = (s_nBlockCycles <= 40); + const s8 cyclerate = EmuConfig.Speedhacks.EECycleRate; + u32 scale_cycles = 0; + + if (cyclerate == 0 || lowcycles || cyclerate < -99 || cyclerate > 3) + scale_cycles = DEFAULT_SCALED_BLOCKS(); + + else if (cyclerate > 1) + scale_cycles = s_nBlockCycles >> (2 + cyclerate); + + else if (cyclerate == 1) + scale_cycles = DEFAULT_SCALED_BLOCKS() / 1.3f; + + else if (cyclerate == -1) + scale_cycles = (s_nBlockCycles <= 80 || s_nBlockCycles > 168 ? 5 : 7) * s_nBlockCycles / 32; + + else + scale_cycles = ((5 + (-2 * (cyclerate + 1))) * s_nBlockCycles) >> 5; + + return (scale_cycles < 1) ? 1 : scale_cycles; +} + +u32 scaleblockcycles_clear() +{ + const u32 scaled = scaleblockcycles_calculation(); + + const s8 cyclerate = EmuConfig.Speedhacks.EECycleRate; + const bool lowcycles = (s_nBlockCycles <= 40); + + if (!lowcycles && cyclerate > 1) + s_nBlockCycles &= (0x1 << (cyclerate + 2)) - 1; + else + s_nBlockCycles &= 0x7; + + return scaled; +} + +void _eeFlushAllDirty() +{ + _flushConstRegs(false); + _flushArm64GPRregs(); + _flushNEONregs(); +} + +void _eeOnWriteReg(int reg, int signext) +{ + GPR_DEL_CONST(reg); +} + +void _deleteEEreg(int reg, int flush) +{ + if (!reg) + return; + if (flush && GPR_IS_CONST1(reg)) + _flushConstReg(reg); + + GPR_DEL_CONST(reg); + _deleteGPRtoArm64GPR(reg, flush ? DELETE_REG_FREE : DELETE_REG_FREE_NO_WRITEBACK); + // NEON side: ALWAYS writeback before free. EE GPRs are 128-bit and scalar + // MIPS ops only overwrite UD[0]; the slot's UD[1] holds the live upper-64 + // from a prior MMI write (MMI routes through eeRecompileCodeXMM, so Rd stays + // live in the slot with MODE_WRITE). Dropping without writeback silently + // zeros UD[1] in memory and breaks the interpreter's "preserve UD[1]" + // contract for LUI/MFLO/MOVZ/ADDIU/... + _deleteGPRtoNEONreg(reg, DELETE_REG_FREE); +} + +void _deleteEEreg128(int reg) +{ + if (!reg) + return; + if (GPR_IS_CONST1(reg)) + _flushConstReg(reg); + + GPR_DEL_CONST(reg); + _deleteGPRtoArm64GPR(reg, DELETE_REG_FREE_NO_WRITEBACK); + _deleteGPRtoNEONreg(reg, DELETE_REG_FREE); +} + +void _flushEEreg(int reg, bool clear) +{ + if (!reg) + return; + + if (GPR_IS_DIRTY_CONST(reg)) + _flushConstReg(reg); + if (clear) + GPR_DEL_CONST(reg); + + // Per-register flush honoring reg/clear, mirroring x86 _flushEEreg. + // clear=false → writeback but keep the allocation; clear=true → also free. + // (The previous arm64 impl flushed ALL registers and ignored reg/clear — + // a behavior-equivalent superset given the lone caller, but a lying API.) + _deleteGPRtoNEONreg(reg, clear ? DELETE_REG_FLUSH_AND_FREE : DELETE_REG_FLUSH); + _deleteGPRtoArm64GPR(reg, clear ? DELETE_REG_FLUSH_AND_FREE : DELETE_REG_FLUSH); +} + +void _eeMoveGPRtoR(const a64::Register& to, int fromgpr, bool allow_preload) +{ + if (fromgpr == 0) + { + // r0 is always zero + if (to.Is64Bits()) + armAsm->Mov(to, a64::xzr); + else + armAsm->Mov(to, a64::wzr); + return; + } + + if (GPR_IS_CONST1(fromgpr)) + { + // Value known at compile time — emit immediate load + if (to.Is64Bits()) + armAsm->Mov(to, g_cpuConstRegs[fromgpr].SD[0]); + else + armAsm->Mov(to, g_cpuConstRegs[fromgpr].UL[0]); + return; + } + + // Check if the register is currently allocated in an ARM64 GPR with + // MODE_READ — meaning the host register holds the current guest value. + // MODE_WRITE-only means it's a destination allocation; the current value + // was never loaded, so the host register contains stale data. + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse && arm64gprs[i].type == ARM64TYPE_GPR && + arm64gprs[i].reg == fromgpr && (arm64gprs[i].mode & MODE_READ)) + { + if (to.Is64Bits()) + armAsm->Mov(to, armXRegister(i)); + else + armAsm->Mov(to, armWRegister(i)); + return; + } + } + + // Check if allocated in a NEON register. A MODE_WRITE-only slot is also + // authoritative — the MMI op that allocated it has written the live value + // to qreg even though the slot was never MODE_READ-loaded. Reading from + // memory in that case would return the pre-MMI stale value. eeRecompileCodeXMM + // passes MODE_WRITE alone (no MODE_READ unless XMMINFO_READD is set) for Rd, + // so every MMI destination lands here. + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse && arm64neon[i].type == NEONTYPE_GPRREG && + arm64neon[i].reg == fromgpr && (arm64neon[i].mode & (MODE_READ | MODE_WRITE))) + { + if (to.Is64Bits()) + armAsm->Fmov(to, armDRegister(i)); // FMOV Xd, Dn + else + armAsm->Fmov(to, armSRegister(i)); // FMOV Wd, Sn (lower 32 bits) + return; + } + } + + // Not allocated anywhere — load from cpuRegs memory + armLoadEERegPtr(to, &cpuRegs.GPR.r[fromgpr].UD[0]); +} + +// ===================================================================================================== +// Branch handling +// ===================================================================================================== + +void SetBranchReg() +{ + g_branch = 1; + + // Flush all GPR/NEON/constant allocations FIRST, while host registers + // still hold correct guest values. iFlushCall writes back delay slot + // results (like addiu sp) before the branch target is loaded into w0. + iFlushCall(FLUSH_EVERYTHING); + + // Now load branch target from pcWriteback (saved by recJR/recJALR) + armLoadEERegPtr(a64::w0, &cpuRegs.pcWriteback); + + // GoemonTlbHack: recJR/recJALR store the raw virtual register target; the + // JIT dispatches in physical space, so translate it before use. Mirrors + // recJ/recJAL (compile-time vtlb_V2P via SetBranchImm) and x86 + // recJR/recJALR (vtlb_DynV2P). The V2P lives in SetBranchReg, whose only + // EE callers are recJR/recJALR, so no other target gets double-translated. + // The iFlushCall(FLUSH_EVERYTHING) above has already spilled guest state; + // vtlb_V2P preserves callee-saved x25 (RECCYCLE) per AAPCS64, so the + // C-call needs no extra save. w0 (== RWARG1) already holds the virtual + // target and receives the translated paddr. + if (EmuConfig.Gamefixes.GoemonTlbHack) + armEmitCall((void*)vtlb_V2P); + + // Store to cpuRegs.pc + armAsm->Str(a64::w0, armCpuRegMem(&cpuRegs.pc)); + + // Alignment check + a64::Label unaligned; + armAsm->Tst(a64::w0, 3); + armAsm->B(&unaligned, a64::ne); + + // Update pinned cycle counter (RECCYCLE = cpuRegs.cycle). + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + // Check events (RECCYCLE >= nextEventCycle → DispatcherEvent flushes + // RECCYCLE itself before calling recEventTest). + armAsm->Ldr(a64::x3, armCpuRegMem(&cpuRegs.nextEventCycle)); + armAsm->Cmp(RECCYCLE, a64::x3); + armEmitCondBranch(a64::ge, DispatcherEvent); + + armEmitJmp(DispatcherReg); + + armAsm->Bind(&unaligned); + armAsm->Mov(RWARG1, 1); + armEmitCall((void*)recError); +} + +void SetBranchImm(u32 imm) +{ + g_branch = 1; + pxAssert(imm); + + armAsm->Mov(RWSCRATCH, imm); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.pc)); + + iFlushCall(FLUSH_EVERYTHING); + + // Update pinned cycle counter. + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + // WaitLoop speedhack: when the block scanner detected this block as a + // pure-nop spin branching back to itself (s_nBlockFF), fast-forward + // RECCYCLE to max(RECCYCLE, nextEventCycle) and jump straight to + // DispatcherEvent. Saves the dozens of iterations the loop would + // otherwise burn waiting for an event to fire. Matches the x86 path in + // iR5900.cpp:iBranchTest under EmuConfig.Speedhacks.WaitLoop. + if (EmuConfig.Speedhacks.WaitLoop && s_nBlockFF && imm == s_branchTo) + { + armAsm->Ldr(a64::x3, armCpuRegMem(&cpuRegs.nextEventCycle)); + armAsm->Cmp(RECCYCLE, a64::x3); + armAsm->Csel(RECCYCLE, RECCYCLE, a64::x3, a64::hi); + armEmitJmp(DispatcherEvent); + return; + } + + // Check events. + armAsm->Ldr(a64::x3, armCpuRegMem(&cpuRegs.nextEventCycle)); + armAsm->Cmp(RECCYCLE, a64::x3); + armEmitCondBranch(a64::ge, DispatcherEvent); + + // Block linking: emit a single B as the patch site. Initially routed + // through JITCompile via recBlocks.Link(); once the target block is + // compiled, recBlocks.New() rewrites this B's imm26 to branch to the + // target's fnptr directly, bypassing the dispatcher. + { + a64::SingleEmissionCheckScope guard(armAsm); + u8* patch_site = armGetCurrentCodePointer(); + armAsm->b(int64_t{0}); // placeholder; recBlocks.Link will overwrite + recBlocks.Link(HWADDR(imm), patch_site); + } +} + +// ===================================================================================================== +// Block state save/restore for delay slots +// ===================================================================================================== + +static _arm64gprregs s_savedGPRs[NUM_ARM_GPR_REGS]; +static _arm64neonregs s_savedNEON[NUM_ARM_NEON_REGS]; +static GPR_reg64 s_savedConstRegs[32]; +static u32 s_savedHasConstReg, s_savedFlushedConstReg; +static u32 s_savedBlockCycles; +static EEINST* s_savedInstInfo; + +void SaveBranchState() +{ + s_savedBlockCycles = s_nBlockCycles; + memcpy(s_savedConstRegs, g_cpuConstRegs, sizeof(g_cpuConstRegs)); + s_savedHasConstReg = g_cpuHasConstReg; + s_savedFlushedConstReg = g_cpuFlushedConstReg; + s_savedInstInfo = g_pCurInstInfo; + memcpy(s_savedGPRs, arm64gprs, sizeof(arm64gprs)); + memcpy(s_savedNEON, arm64neon, sizeof(arm64neon)); +} + +void LoadBranchState() +{ + s_nBlockCycles = s_savedBlockCycles; + memcpy(g_cpuConstRegs, s_savedConstRegs, sizeof(g_cpuConstRegs)); + g_cpuHasConstReg = s_savedHasConstReg; + g_cpuFlushedConstReg = s_savedFlushedConstReg; + g_pCurInstInfo = s_savedInstInfo; + memcpy(arm64gprs, s_savedGPRs, sizeof(arm64gprs)); + memcpy(arm64neon, s_savedNEON, sizeof(arm64neon)); +} + +// ===================================================================================================== +// Instruction recompilation +// ===================================================================================================== + +void recompileNextInstruction(bool delayslot, bool swapped_delay_slot) +{ + const u32 old_code = cpuRegs.code; + EEINST* old_inst_info = g_pCurInstInfo; + + cpuRegs.code = memRead32(pc); + + if (!delayslot) + { + pc += 4; + g_cpuFlushedPC = false; + g_cpuFlushedCode = false; + } + else + { + // For delay slots, increment pc after recompiling (at the end of this function) + g_recompilingDelaySlot = true; + } + + g_pCurInstInfo++; + + // NOP gets cycle counted but no codegen (matching x86 behavior) + if (cpuRegs.code == 0) + { + s_nBlockCycles += 9 * (2 - ((cpuRegs.CP0.n.Config >> 18) & 0x1)); + } + else + { + const R5900::OPCODE& opcode = R5900::GetCurrentInstruction(); + s_nBlockCycles += opcode.cycles * (2 - ((cpuRegs.CP0.n.Config >> 18) & 0x1)); + +#ifdef VERIFY_NATIVE_CODEGEN + // Verification mode: verify native codegen against interpreter for + // instructions in categories that have native codegen enabled. + // Skip COP0/Memory/FPU which are interpreter-only and may have + // timing-sensitive behaviour (MFC0 Count reads cycle counter). + const u32 verifyOp = cpuRegs.code >> 26; + const bool isVerifiableCategory = + // Verify COP2 instructions (opcode 18 = 0x12) + (verifyOp == 0x12); + + if (isVerifiableCategory && opcode.recompile && opcode.interpret) + { + // Step 1: Flush all registers to cpuRegs BEFORE native codegen + iFlushCall(FLUSH_EVERYTHING); + + // Step 2: Emit call to snapshot pre-instruction state + armAsm->Mov(a64::w0, cpuRegs.code); + armAsm->Mov(a64::w1, pc - 4); // current instruction PC + armEmitCall((void*)verifySnapshotPre); + + // Step 3: Run the native codegen + opcode.recompile(); + + // Step 4: Flush native results to cpuRegs + iFlushCall(FLUSH_EVERYTHING); + + // Step 5: Emit call to verify against interpreter + armAsm->Mov(a64::w0, cpuRegs.code); + armAsm->Mov(a64::w1, pc - 4); + armEmitCall((void*)verifyCheckPost); + } + else +#endif + { + // Guard: branch/jump in a delay slot would cause infinite + // compile-time recursion. Use interpreter for the instruction. + const bool isBranchInDelaySlot = delayslot && (opcode.flags & IS_BRANCH); + if (isBranchInDelaySlot || !opcode.recompile) + { + if ((opcode.flags & IS_BRANCH) && !isBranchInDelaySlot) + recBranchCall(opcode.interpret); + else + recCall(opcode.interpret); + } + else + opcode.recompile(); + } + } + + // SP misalignment check disabled: MMI/COP2 instructions legitimately use + // r29 as SIMD data, causing massive false-positive spam. + + if (!swapped_delay_slot) + { + _clearNeededArm64GPRregs(); + _clearNeededNEONregs(); + } + + if (delayslot) + { + pc += 4; + g_cpuFlushedPC = false; + g_cpuFlushedCode = false; + g_recompilingDelaySlot = false; + } + + // When called from TrySwapDelaySlot (swapped_delay_slot=true), restore + // cpuRegs.code so that the caller's _Rs_/_Rt_/_Rd_ macros still work. + // Matches x86 at iR5900.cpp:1918-1921. + if (swapped_delay_slot) + { + cpuRegs.code = old_code; + g_pCurInstInfo = old_inst_info; + } +} + +bool TrySwapDelaySlot(u32 rs, u32 rt, u32 rd, bool allow_loadstore) +{ + if (g_recompilingDelaySlot) + return false; + + const u32 opcode_encoded = memRead32(pc); + if (opcode_encoded == 0) // NOP + { + recompileNextInstruction(true, true); + return true; + } + + return false; +} + +// ===================================================================================================== +// Memory management and block clearing +// ===================================================================================================== + +static void recClear(u32 addr, u32 size) +{ + addr = HWADDR(addr); + const u32 end = addr + size * 4; + + int blockidx = recBlocks.LastIndex(end - 4); + if (blockidx == -1) + return; + + // Track the EE-address span of all blocks we touch so the post-walk + // tail can reset interior BLOCKs across the *full* extent of the + // removed blocks (a straddler can extend well past `end` or below + // `addr`). `ceiling` clamps the tail at the next surviving block's + // startpc so we never trample its interior. + u32 lowerextent = static_cast(-1); + u32 upperextent = 0; + u32 ceiling = static_cast(-1); + + if (BASEBLOCKEX* peb_above = recBlocks[blockidx + 1]) + ceiling = peb_above->startpc; + + int toRemoveLast = blockidx; + + // Walk down through blocks overlapping [addr, end). For each, reset + // BLOCK->fnptr at the block's actual start (the straddle-from-below + // case is load-bearing — Arm64BaseBlocks::Remove() patches only the + // compiled-code stub, so any BLOCK->fnptr left pointing at a stub + // trips the recRecompile fnptr assertion on the next dispatch). + // + // Skip s_pCurBlock if we hit it: it's the block currently being + // compiled, and yanking it mid-emit corrupts the in-progress block. + // Splitting the Remove range around it preserves it. Mirrors x86 + // recClear (pcsx2/x86/ix86-32/iR5900.cpp:786). + while (BASEBLOCKEX* pexblock = recBlocks[blockidx]) + { + const u32 blockstart = pexblock->startpc; + const u32 blockend = blockstart + pexblock->size * 4; + BASEBLOCK* pblock = GETBLOCK(blockstart); + + if (pblock == s_pCurBlock) + { + if (toRemoveLast != blockidx) + recBlocks.Remove(blockidx + 1, toRemoveLast); + toRemoveLast = --blockidx; + continue; + } + + if (blockend <= addr) + { + lowerextent = std::max(lowerextent, blockend); + break; + } + + lowerextent = std::min(lowerextent, blockstart); + upperextent = std::max(upperextent, blockend); + pblock->SetFnptr((uptr)JITCompile); + + --blockidx; + } + + if (toRemoveLast != blockidx) + recBlocks.Remove(blockidx + 1, toRemoveLast); + + upperextent = std::min(upperextent, ceiling); + + // Reset interior BLOCKs across the full removed-block extent. Without + // this, interior fnptrs of straddler blocks can stay non-JITCompile + // from a prior compilation, leading to wrong dispatch on a later JR + // into the middle of a freshly-recompiled block. + if (upperextent > lowerextent) + iopClearRecLUT(GETBLOCK(lowerextent), upperextent - lowerextent); +} + +static void iopClearRecLUT(BASEBLOCK* base, int count) +{ + for (int i = 0; i < count / 4; i++) + base[i].SetFnptr((uptr)JITCompile); +} + +static void dyna_block_discard(u32 start, u32 sz) +{ + DevCon.WriteLn("%.8X rec block discard (sz=%d)", start, sz); + recClear(start, sz); +} + +static void dyna_page_reset(u32 start, u32 sz) +{ + recClear(start & ~0xFFF, 0x400); // clear 4KB page + manual_counter[start >> 12]++; + mmap_MarkCountedRamPage(start); +} + +// Self-modifying code detection — generates inline memory comparison checks +// for blocks in manually-protected pages, and sets up page protection for new pages. +// Port of x86 memory_protect_recompiled_code(). +static void memory_protect_recompiled_code(u32 startpc, u32 size) +{ + u32 inpage_ptr = HWADDR(startpc); + const u32 inpage_sz = size * 4; + + // The kernel context register is stored @ 0x800010C0-0x80001300 + // The EENULL thread context register is stored @ 0x81000-.... + const bool contains_thread_stack = ((startpc >> 12) == 0x81) || ((startpc >> 12) == 0x80001); + + const vtlb_ProtectionMode PageType = contains_thread_stack ? ProtMode_Manual : mmap_GetRamPageInfo(inpage_ptr); + + switch (PageType) + { + case ProtMode_NotRequired: + break; + + case ProtMode_None: + case ProtMode_Write: + mmap_MarkCountedRamPage(inpage_ptr); + manual_page[inpage_ptr >> 12] = 0; + break; + + case ProtMode_Manual: + { + // Set up arguments for DispatchBlockDiscard (w0=addr, w1=size) + armAsm->Mov(a64::w0, inpage_ptr); + armAsm->Mov(a64::w1, inpage_sz / 4); + + u32 lpc = inpage_ptr; + u32 stg = inpage_sz; + + // Generate inline byte-by-byte comparison of compiled block source with current RAM. + // If any word differs, the block is stale and must be discarded. + while (stg > 0) + { + const u32 expected = *(u32*)PSM(lpc); + + // Load current memory word + armMoveAddressToReg(RSCRATCHADDR, (void*)PSM(lpc)); + armAsm->Ldr(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + + // Compare with compile-time snapshot + armAsm->Mov(a64::w9, expected); + armAsm->Cmp(RWSCRATCH, a64::w9); + armEmitCondBranch(a64::ne, DispatchBlockDiscard); + + stg -= 4; + lpc += 4; + } + + // Counted blocks: track how often this block runs. If the counter overflows, + // reset the page to write-protected mode (faster than manual checks). + if (!contains_thread_stack && manual_counter[inpage_ptr >> 12] <= 3) + { + armMoveAddressToReg(RSCRATCHADDR, &manual_page[inpage_ptr >> 12]); + armAsm->Ldrh(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + armAsm->Add(RWSCRATCH, RWSCRATCH, size); + armAsm->Strh(RWSCRATCH, a64::MemOperand(RSCRATCHADDR)); + // Check for u16 overflow (bit 16+ set means wrapped past 0xFFFF) + armAsm->Tst(RWSCRATCH, 0xFFFF0000u); + armEmitCondBranch(a64::ne, DispatchPageReset); + } + break; + } + } +} + +// ===================================================================================================== +// Reserve / Reset / Shutdown / Execute +// ===================================================================================================== + +static void recReserveRAM() +{ + recLutEntries = (Ps2MemSize::MainRam + Ps2MemSize::Rom + Ps2MemSize::Rom1) / 4; + + if (recLutReserve_RAM.size() != recLutEntries) + recLutReserve_RAM.resize(recLutEntries); + + recLutUnmapped.resize(_64kb / 4); + + BASEBLOCK* curpos = recLutReserve_RAM.data(); + recRAM = curpos; + curpos += (Ps2MemSize::MainRam / 4); + recROM = curpos; + curpos += (Ps2MemSize::Rom / 4); + recROM1 = curpos; + curpos += (Ps2MemSize::Rom1 / 4); + + if (recRAMCopy.size() != Ps2MemSize::MainRam) + recRAMCopy.resize(Ps2MemSize::MainRam); +} + +static void recReserve() +{ + Console.WriteLn(Color_Green, "EE: ARM64 Recompiler reserved."); + recPtr = SysMemory::GetEERec(); + recPtrEnd = SysMemory::GetEERecEnd() - _64kb; + + recReserveRAM(); + + pxAssertRel(!s_pInstCache, "InstCache not allocated"); + s_nInstCacheSize = 128; + s_pInstCache = (EEINST*)malloc(sizeof(EEINST) * s_nInstCacheSize); + if (!s_pInstCache) + pxFailRel("Failed to allocate R5900 InstCache array."); + + const u32 poolSize = 65536; + u8* poolBase = SysMemory::GetEERecEnd() - poolSize; + s_eeConstantPool.Init(poolBase, poolSize); +} + +static void recResetRaw() +{ + Console.WriteLn(Color_Green, "iR5900-ARM64 Recompiler reset."); + + armSetAsmPtr(SysMemory::GetEERec(), SysMemory::GetEERecEnd() - SysMemory::GetEERec(), &s_eeConstantPool); + armStartBlock(); + const u8* dispStart = armGetCurrentCodePointer(); + _DynGen_Dispatchers(); + const u8* dispEnd = armGetCurrentCodePointer(); + recPtr = armEndBlock(); + + Console.WriteLn(Color_Green, "EE ARM64: Dispatcher generated at %p (%zu bytes)", dispStart, (size_t)(dispEnd - dispStart)); + + iopClearRecLUT(recLutReserve_RAM.data(), + Ps2MemSize::MainRam + Ps2MemSize::Rom + Ps2MemSize::Rom1); + + BASEBLOCK* unmapped = recLutUnmapped.data(); + + for (int i = 0; i < 0x10000; i++) + recLUT_SetPage(recLUT, hwLUT, unmapped, i, 0, 0); + + for (int i = 0; i < _64kb / 4; i++) + unmapped[i].SetFnptr((uptr)UnmappedRecLUTPage); + + // Map EE RAM (32MB, mirrored) + for (int i = 0; i < 0x200; i++) + { + u32 mask = (Ps2MemSize::MainRam / _64kb) - 1; + recLUT_SetPage(recLUT, hwLUT, recRAM, 0x0000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0x2000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0x3000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0x8000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0xa000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0xb000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0xc000, i, i & mask); + recLUT_SetPage(recLUT, hwLUT, recRAM, 0xd000, i, i & mask); + } + + // Map BIOS ROM + for (int i = 0x1fc0; i < 0x2000; i++) + { + recLUT_SetPage(recLUT, hwLUT, recROM, 0x0000, i, i - 0x1fc0); + recLUT_SetPage(recLUT, hwLUT, recROM, 0x8000, i, i - 0x1fc0); + recLUT_SetPage(recLUT, hwLUT, recROM, 0xa000, i, i - 0x1fc0); + } + + // Map ROM1 + for (int i = 0x1e00; i < 0x1e04; i++) + { + recLUT_SetPage(recLUT, hwLUT, recROM1, 0x0000, i, i - 0x1e00); + recLUT_SetPage(recLUT, hwLUT, recROM1, 0x8000, i, i - 0x1e00); + recLUT_SetPage(recLUT, hwLUT, recROM1, 0xa000, i, i - 0x1e00); + } + + if (s_pInstCache) + memset(s_pInstCache, 0, sizeof(EEINST) * s_nInstCacheSize); + + recBlocks.Reset(); + maxrecmem = 0; + + memset(manual_page, 0, sizeof(manual_page)); + memset(manual_counter, 0, sizeof(manual_counter)); + if (recRAMCopy.data()) + memset(recRAMCopy.data(), 0, recRAMCopy.size()); + + g_branch = 0; +} + +static void recShutdown() +{ + s_eeConstantPool.Destroy(); + recRAMCopy.deallocate(); + recLutReserve_RAM.deallocate(); + recLutUnmapped.deallocate(); + + safe_free(s_pInstCache); + s_nInstCacheSize = 0; + + recPtr = nullptr; + recPtrEnd = nullptr; +} + +static void recResetEE() +{ + if (eeCpuExecuting) + { + eeRecNeedsReset = true; + recSafeExitExecution(); + return; + } + + recResetRaw(); +} + +static void recStep() +{ +} + +static void recExitExecution() +{ + fastjmp_jmp(&m_SetJmp_StateCheck, 1); +} + +static void recSafeExitExecution() +{ + eeRecExitRequested = true; + + if (!eeEventTestIsActive) + { + cpuRegs.nextEventCycle = 0; + } + else + { + if (psxRegs.iopCycleEE > 0) + { + psxRegs.iopBreak += psxRegs.iopCycleEE; + psxRegs.iopCycleEE = 0; + } + } +} + +static void recCancelInstruction() +{ + // Called by interpreter functions (e.g. RaiseAddressError) when an + // exception occurs mid-instruction. For the interpreter, this does a + // longjmp. For the recompiler, set the TLB miss flag so that recCall's + // post-call check dispatches to the exception vector. + s_recTlbMissOccurred = 1; +} + +static void recExecute() +{ + if (eeRecNeedsReset) + { + eeRecNeedsReset = false; + recResetRaw(); + } + + Console.WriteLn(Color_Green, "EE ARM64: Entering recompiled code (pc=0x%08X)", cpuRegs.pc); + + if (!fastjmp_set(&m_SetJmp_StateCheck)) + { + eeCpuExecuting = true; + ((void (*)())EnterRecompiledCode)(); + } + + eeCpuExecuting = false; +} + +#ifdef PCSX2_RECOMPILER_TESTS +// Harness entry. Not part of R5900cpu; called directly by EeRecTestHarness +// for a bounded number of guest cycles ending at park_pc. Forces +// nextEventCycle = cpuRegs.cycle so iBranchTest at every block tail routes +// through DispatcherEvent (where recEventTest's harnessShouldExit check +// can observe parking-PC arrival or cycle exhaust). Returns the cycle +// delta consumed in this run. +s32 recEeExecuteBlock(s32 cycles, u32 park_pc) +{ + const s32 cap = std::min(cycles, kExecuteBlockSafetyCap); + + g_eeHarnessActive = true; + g_eeHarnessParkPc = park_pc; + g_eeHarnessCycleBudget = cap; + g_eeHarnessCycleStart = cpuRegs.cycle; + eeRecExitRequested = false; + + cpuRegs.nextEventCycle = cpuRegs.cycle; + + if (!fastjmp_set(&m_SetJmp_StateCheck)) + { + ((void (*)())EnterRecompiledCode)(); + } + + g_eeHarnessActive = false; + + return static_cast(cpuRegs.cycle - g_eeHarnessCycleStart); +} + +// Test-harness link introspection. Forwards to Arm64BaseBlocks::IsLinked, +// which walks the link multimap for any patch site within the block +// containing src_pc that targets dst_pc. +bool recEeIsBlockLinked(u32 src_pc, u32 dst_pc) +{ + return recBlocks.IsLinked(src_pc, dst_pc); +} +#endif + +// ===================================================================================================== +// Timeout Loop Speedhack +// ===================================================================================================== + +// Detects and skips timeout loops like: +// addiu v0,v0,-1 / nop*N / bne v0,zero,loop / nop +// Instead of spinning, advances the cycle counter and decrements the register. +// Port of x86 recSkipTimeoutLoop(). +static bool recSkipTimeoutLoop(s32 reg, bool is_timeout_loop) +{ + if (!EmuConfig.Speedhacks.WaitLoop || !is_timeout_loop) + return false; + + DevCon.WriteLn("[EE] Skipping timeout loop at 0x%08X -> 0x%08X (reg=%d)", + s_pCurBlockEx->startpc, s_nEndBlock, reg); + + // Logic: skip the loop by advancing cycles based on the register value. + // new_cycles = min(reg * 8 + cycle, nextEventCycle) + // new_reg = reg - (new_cycles - cycle) / 8 + // if new_reg > 0, jump to dispatcher (an event interrupted the loop) + // else loop finished, continue at s_nEndBlock + + // if (cycle >= nextEventCycle) goto DispatcherEvent (u64 comparison) + armAsm->Ldr(a64::x3, armCpuRegMem(&cpuRegs.nextEventCycle)); + armAsm->Cmp(RECCYCLE, a64::x3); + armEmitCondBranch(a64::hs, DispatcherEvent); + + // w4 = reg value (the decrementing counter) + armAsm->Ldr(a64::w4, armCpuRegMem(&cpuRegs.GPR.r[reg].UL[0])); + + // x5 = reg * 8 + cycle (estimated end cycle, u64) + armAsm->Add(a64::x5, RECCYCLE, a64::Operand(a64::x4, a64::LSL, 3)); + + // x5 = min(x5, nextEventCycle) + armAsm->Cmp(a64::x5, a64::x3); + armAsm->Csel(a64::x5, a64::x3, a64::x5, a64::hi); // if x5 > nextEvent, use nextEvent + + // w6 = (new_cycles - old_cycle) >> 3 = iterations consumed (uses old RECCYCLE). + armAsm->Sub(a64::w6, a64::w5, RECCYCLE.W()); + armAsm->Lsr(a64::w6, a64::w6, 3); + + // Commit the new cycle value into RECCYCLE (no memory store — DispatcherEvent + // will flush it if we exit there; otherwise the next block-tail event check + // uses RECCYCLE directly). + armAsm->Mov(RECCYCLE, a64::x5); + + // reg -= iterations consumed + armAsm->Sub(a64::w4, a64::w4, a64::w6); + armAsm->Str(a64::w4, armCpuRegMem(&cpuRegs.GPR.r[reg].UL[0])); + // Also sign-extend to upper 32 bits (EE GPRs are 64-bit for lower half) + armAsm->Sxtw(a64::x4, a64::w4); + armAsm->Str(a64::x4, armCpuRegMem(&cpuRegs.GPR.r[reg].UD[0])); + + // if reg != 0, event interrupted the loop — go to dispatcher + armEmitCbnz(a64::w4, DispatcherEvent); + + // Loop finished — set PC to end of block and dispatch + armAsm->Mov(RWSCRATCH, s_nEndBlock); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.pc)); + armEmitJmp(DispatcherReg); + + g_branch = 1; + pc = s_nEndBlock; + + return true; +} + +// ===================================================================================================== +// Main Recompilation Loop +// ===================================================================================================== + +static void recRecompile(const u32 startpc) +{ + u32 i; + + // Note: startpc=0 is valid (EE RAM address 0). The x86 rec asserts on this + // but it can legitimately happen during BIOS init (e.g., JR $ra with ra=0). + // We allow it since address 0 is properly mapped in recLUT. + + if (recPtr >= recPtrEnd) + eeRecNeedsReset = true; + + // Signal that the ELF entry point is now compiling, so VMManager flips + // HasBootedELF() and applies the per-game GameDB fixes (both game fixes and + // GS hardware fixes — e.g. Baldur's Gate: Dark Alliance's textureInsideRT, + // which fixes the right-half-black menu render). Must fire before the + // deferred reset below — the hook can change settings and flush the JIT. + // Mirrors iR5900.cpp's recRecompile. + if (HWADDR(startpc) == VMManager::Internal::GetCurrentELFEntryPoint()) + VMManager::Internal::EntryPointCompilingOnCPUThread(); + + if (eeRecNeedsReset) + { + eeRecNeedsReset = false; + recResetRaw(); + } + + armSetAsmPtr(recPtr, recPtrEnd - recPtr + _64kb, &s_eeConstantPool); + armStartBlock(); + + s_pCurBlock = GETBLOCK(startpc); + pxAssert(s_pCurBlock->GetFnptr() == (uptr)JITCompile || s_pCurBlock->GetFnptr() == (uptr)UnmappedRecLUTPage); + + // armStartBlock() aligned armAsmPtr to 16 bytes, so the actual block + // code starts at armGetCurrentCodePointer(), not at recPtr. Block + // linking branches to BASEBLOCKEX::fnptr, so it must be the aligned + // address — using recPtr instead lands the branch on padding bytes + // and triggers SIGILL. + const uptr block_fnptr = (uptr)armGetCurrentCodePointer(); + + s_pCurBlockEx = recBlocks.Get(HWADDR(startpc)); + if (!s_pCurBlockEx || s_pCurBlockEx->startpc != HWADDR(startpc)) + s_pCurBlockEx = recBlocks.New(HWADDR(startpc), block_fnptr); + + g_branch = 0; + + s_pCurBlock->SetFnptr(block_fnptr); + s_nBlockCycles = 0; + s_nBlockInterlocked = false; + + pc = startpc; + g_cpuHasConstReg = g_cpuFlushedConstReg = 1; + g_cpuFlushedPC = false; + g_cpuFlushedCode = false; + + _initArm64GPRregs(); + _initArm64NEONregs(); + +#ifdef PCSX2_RECOMPILER_TESTS + // Optional block-entry diagnostic hook (test builds only). Emitted only when + // g_emit_block_hook is set before recReset; production recompiles emit + // nothing. Fires on EVERY block entry, including statically-linked ones, + // because linked branches target block_fnptr — i.e. exactly here. At the + // prologue all guest state is memory-resident (the allocator was just + // re-initialized to memory), so the hook's FingerprintCpu() reads correct + // cpuRegs. We pass startpc as an immediate because cpuRegs.pc is not updated + // on a static-linked entry, and flush RECCYCLE -> cpuRegs.cycle so the hook + // sees the live cycle. RECCYCLE (x25) is callee-saved across the C call, so + // no reload is needed. + if (ee_divtrace::g_emit_block_hook) + { + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armAsm->Mov(RWARG1, startpc); + armEmitCall((void*)ee_divtrace_jit_block_hook); + } +#endif + + // EELOAD detection + ELF-load hooks (mirrors iR5900.cpp's recRecompile). + // These compile-time-detected, run-time-emitted calls are how PCSX2 learns + // which game ELF is booting: eeloadHook() -> ELFLoadingOnCPUThread() sets + // s_elf_entry_point / CRC, which gates HasBootedELF() and therefore ALL + // per-game patches, game fixes, GS hardware fixes, widescreen, symbol import + // and achievements. The emitted calls run at block-execution time; at the + // prologue all guest state is memory-resident, so no iFlushCall is needed, + // and the pinned bases (x24/x25) are callee-saved across the C call. + if (HWADDR(startpc) == EELOAD_START) + { + // The EELOAD _start function is the same across all BIOS versions. + const u32 mainjump = memRead32(EELOAD_START + 0x9c); + if (mainjump >> 26 == 3) // JAL + g_eeloadMain = ((EELOAD_START + 0xa0) & 0xf0000000U) | (mainjump << 2 & 0x0fffffffU); + } + + if (g_eeloadMain && HWADDR(startpc) == HWADDR(g_eeloadMain)) + { + armEmitCall((void*)eeloadHook); + if (VMManager::Internal::IsFastBootInProgress()) + { + // Four known EELOAD versions, identified by the location of the 'jal' to + // the EELOAD function that calls ExecPS2(). The function itself is at the + // same address in all BIOSs after v1.00-v1.10. + const u32 typeAexecjump = memRead32(EELOAD_START + 0x470); // v1.00, v1.01?, v1.10? + const u32 typeBexecjump = memRead32(EELOAD_START + 0x5B0); // v1.20, v1.50, v1.60 (3000x models) + const u32 typeCexecjump = memRead32(EELOAD_START + 0x618); // v1.60 (3900x models) + const u32 typeDexecjump = memRead32(EELOAD_START + 0x600); // v1.70, v1.90, v2.00, v2.20, v2.30 + if ((typeBexecjump >> 26 == 3) || (typeCexecjump >> 26 == 3) || (typeDexecjump >> 26 == 3)) // JAL to 0x822B8 + g_eeloadExec = EELOAD_START + 0x2B8; + else if (typeAexecjump >> 26 == 3) // JAL to 0x82170 + g_eeloadExec = EELOAD_START + 0x170; + else // Unexamined BIOS models: 18000, 3500x, 3700x, 5500x, 7900x (and v1.01/v1.10). + Console.WriteLn("recRecompile: Could not enable launch arguments for fast boot mode; unidentified BIOS version! Please report this to the PCSX2 developers."); + } + } + + if (g_eeloadExec && HWADDR(startpc) == HWADDR(g_eeloadExec)) + armEmitCall((void*)eeloadHook2); + + // Scan for block boundary + i = startpc; + s_nEndBlock = 0xffffffff; + s_branchTo = -1; + + // Timeout loop detection (matches x86 recSkipTimeoutLoop pattern): + // addiu reg,reg,-N / nop*N / bne reg,zero,loop / nop + s32 timeout_reg = -1; + bool is_timeout_loop = true; + bool timeout_has_bne = false; + + while (1) + { + BASEBLOCK* pblock = GETBLOCK(i); + if (i != startpc && pblock->GetFnptr() != (uptr)JITCompile) + { + s_nEndBlock = i; + break; + } + + // 4K page boundary + if (i != startpc && (i & 0xffc) == 0) + { + s_nEndBlock = i; + break; + } + + cpuRegs.code = memRead32(i); + + // Timeout loop pattern matching + if (is_timeout_loop) + { + if ((cpuRegs.code >> 26) == 8 || (cpuRegs.code >> 26) == 9) + { + // addi/addiu — must be first non-nop, decrementing same reg + if (timeout_reg >= 0 || _Rs_ != _Rt_ || _Imm_ >= 0) + is_timeout_loop = false; + else + timeout_reg = _Rs_; + } + else if ((cpuRegs.code >> 26) == 5) + { + // bne — must branch back using the timeout reg vs zero + if (timeout_reg != static_cast(_Rs_) || _Rt_ != 0 || memRead32(i + 4) != 0) + is_timeout_loop = false; + else + timeout_has_bne = true; + } + else if (cpuRegs.code != 0) + { + is_timeout_loop = false; + } + } + + switch (cpuRegs.code >> 26) + { + case 0: // SPECIAL + if (_Funct_ == 8 || _Funct_ == 9) // JR, JALR + { + s_nEndBlock = i + 8; + goto StartRecomp; + } + if (_Funct_ == 12 || _Funct_ == 13) // SYSCALL, BREAK + { + s_nEndBlock = i + 4; // no delay slot + goto StartRecomp; + } + break; + + case 1: // REGIMM + if (_Rt_ < 4 || (_Rt_ >= 16 && _Rt_ < 20)) + { + s_branchTo = _Imm_ * 4 + i + 4; + // Backward branch into the current block: end the block at the + // target so the loop head becomes its own linkable block. + // Mirrors x86 iR5900.cpp:2362 and the COP1/COP2 case below. + if (s_branchTo > startpc && s_branchTo < i) + s_nEndBlock = s_branchTo; + else + s_nEndBlock = i + 8; + goto StartRecomp; + } + break; + + case 2: case 3: // J, JAL + s_branchTo = (_InstrucTarget_ << 2) | ((i + 4) & 0xf0000000); + s_nEndBlock = i + 8; + goto StartRecomp; + + case 4: case 5: case 6: case 7: // BEQ, BNE, BLEZ, BGTZ + case 20: case 21: // BEQL, BNEL + case 22: case 23: // BLEZL, BGTZL + s_branchTo = _Imm_ * 4 + i + 4; + // Backward branch into the current block: split so the loop head + // is its own linkable block. Mirrors x86 iR5900.cpp:2387 and + // the COP1/COP2 case below. + if (s_branchTo > startpc && s_branchTo < i) + s_nEndBlock = s_branchTo; + else + s_nEndBlock = i + 8; + goto StartRecomp; + + case 16: // COP0 + if (_Rs_ == 16 && _Funct_ == 24) // ERET (no delay slot) + { + s_nEndBlock = i + 4; + goto StartRecomp; + } + // Fall through: COP0's branch opcodes line up with COP1/COP2's. + [[fallthrough]]; + + case 17: // COP1 + case 18: // COP2 + if (_Rs_ == 8) // BC0/BC1/BC2 F/T/FL/TL + { + s_branchTo = _Imm_ * 4 + i + 4; + if (s_branchTo > startpc && s_branchTo < i) + s_nEndBlock = s_branchTo; + else + s_nEndBlock = i + 8; + goto StartRecomp; + } + break; + } + + i += 4; + } + +StartRecomp: + + // Self-modifying code detection: generate inline memory checks for manual blocks. + memory_protect_recompiled_code(startpc, (s_nEndBlock - startpc) >> 2); + + // Infinite loop detection + s_nBlockFF = false; + if (s_branchTo == startpc) + { + s_nBlockFF = true; + for (i = startpc; i < s_nEndBlock; i += 4) + { + if (i != s_nEndBlock - 8 && memRead32(i) != 0) + { + s_nBlockFF = false; + break; + } + } + } + else + { + // A timeout loop must branch back to its own start (a self-loop). If the + // block's terminating branch targets anywhere else, it is NOT a timeout + // loop and must be recompiled normally. Mirrors x86 iR5900.cpp:2510-2513. + // Without this guard, the early-exit `bne reg,zero,` at the TOP + // of a counted compute loop gets misdetected as a timeout loop and + // recSkipTimeoutLoop fast-forwards the counter to 0 while skipping the + // loop BODY's real work. timeout_has_bne alone is insufficient because it + // matches the forward early-exit branch without checking the branch target. + is_timeout_loop = false; + } + + // Instruction analysis (backward pass) + { + EEINST* pcur; + + if (s_nInstCacheSize < (s_nEndBlock - startpc) / 4 + 1) + { + free(s_pInstCache); + s_nInstCacheSize = (s_nEndBlock - startpc) / 4 + 10; + s_pInstCache = (EEINST*)malloc(sizeof(EEINST) * s_nInstCacheSize); + pxAssert(s_pInstCache != NULL); + } + + pcur = s_pInstCache + (s_nEndBlock - startpc) / 4; + _recClearInst(pcur); + pcur->info = 0; + + bool has_cop2_instructions = false; + for (i = s_nEndBlock; i > startpc; i -= 4) + { + cpuRegs.code = memRead32(i - 4); + pcur[-1] = pcur[0]; + recBackpropBSC(cpuRegs.code, pcur - 1, pcur); + pcur--; + + has_cop2_instructions |= (_Opcode_ == 022 || _Opcode_ == 066 || _Opcode_ == 076); + } + + // Run COP2 analysis passes — sets EEINST_COP2_SYNC_VU0/FINISH_VU0 flags + // for conditional VU0 synchronization in transfer ops. + if (has_cop2_instructions) + { + R5900::COP2MicroFinishPass().Run(startpc, s_nEndBlock, s_pInstCache + 1); + + if (EmuConfig.Speedhacks.vuFlagHack) + R5900::COP2FlagHackPass().Run(startpc, s_nEndBlock, s_pInstCache + 1); + } + } + + // Try timeout loop speedhack — if detected, skip normal codegen + // Timer-poll loops (mfc0 Count / subu / sltu / bne) are NOT skipped because + // they need to wait for a specific elapsed time — the correct fix is native codegen. + // Require timeout_reg >= 0 (actually found an addiu) to avoid matching all-nop blocks + const bool doRecompilation = !recSkipTimeoutLoop(timeout_reg, is_timeout_loop && timeout_reg >= 0 && timeout_has_bne); + + // Code generation (forward pass) + if (doRecompilation) + { + g_pCurInstInfo = s_pInstCache; + while (!g_branch && pc < s_nEndBlock) + recompileNextInstruction(false, false); + } + + pxAssert((pc - startpc) >> 2 <= 0xffff); + s_pCurBlockEx->size = (pc - startpc) >> 2; + + if (!(pc & 0x10000000)) + maxrecmem = std::max((pc & ~0xa0000000), maxrecmem); + + // Snapshot current block's source to recRAMCopy for future overlap detection. + // Note: The overlap check (comparing old blocks' recRAMCopy vs current memory) is + // disabled because it causes infinite recompilation loops — recRAMCopy starts zeroed + // but memory has real code, so the memcmp always fails. The inline CMP checks from + // memory_protect_recompiled_code are the primary SMC detection mechanism. + if (HWADDR(pc) <= Ps2MemSize::MainRam) + { + memcpy(&recRAMCopy[HWADDR(startpc) / 4], PSM(startpc), pc - startpc); + } + + if (g_branch == 2) + { + // Branch taken — flush and dispatch. recBranchCall already accumulated + // any pre-call cycles into RECCYCLE and reloaded it after the C call, + // so any further scaleblockcycles_clear() result is the post-call delta. + iFlushCall(FLUSH_EVERYTHING); + + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + armAsm->Ldr(a64::x3, armCpuRegMem(&cpuRegs.nextEventCycle)); + armAsm->Cmp(RECCYCLE, a64::x3); + armEmitCondBranch(a64::ge, DispatcherEvent); + + armEmitJmp(DispatcherReg); + } + else + { + if (g_branch) + { + // g_branch == 1: block ended with a branch instruction. + // pc may not equal s_nEndBlock for branch-likely instructions + // where the not-taken path skips the delay slot. + } + else + { + // Non-branch block end (split / page boundary / cycle cap). + // Mirrors x86 iR5900.cpp:2680-2698: + // long block (>6 insns): SetBranchImm(pc) — event check + static-linked B + // short block (≤6 insns): flush + pc + cycle + bare static-linked B + // Short blocks skip the event check entirely; a few-insn block can't + // have advanced cycles far enough to cross an event boundary, so the + // load+cmp+B.ge is dead-weight code per block tail. + if (pc != s_nEndBlock) + Console.Error("EE ARM64: Block end mismatch! startpc=0x%08X pc=0x%08X s_nEndBlock=0x%08X", startpc, pc, s_nEndBlock); + pxAssert(pc == s_nEndBlock); + + const int numinsts = (pc - startpc) / 4; + if (numinsts > 6) + { + SetBranchImm(pc); + } + else + { + iFlushCall(FLUSH_EVERYTHING); + + armAsm->Mov(RWSCRATCH, pc); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.pc)); + + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + a64::SingleEmissionCheckScope guard(armAsm); + u8* patch_site = armGetCurrentCodePointer(); + armAsm->b(int64_t{0}); // placeholder; recBlocks.Link will overwrite + recBlocks.Link(HWADDR(pc), patch_site); + } + } + } + + pxAssert(armGetCurrentCodePointer() < SysMemory::GetEERecEnd()); + + // Size is from the aligned block_fnptr, not the pre-alignment recPtr — + // keeps Perf::ee.RegisterPC consistent with the linker's view of the block. + s_pCurBlockEx->x86size = static_cast((uptr)armGetCurrentCodePointer() - s_pCurBlockEx->fnptr); + + Perf::ee.RegisterPC((void*)s_pCurBlockEx->fnptr, s_pCurBlockEx->x86size, s_pCurBlockEx->startpc); + + recPtr = armEndBlock(); + + pxAssert((g_cpuHasConstReg & g_cpuFlushedConstReg) == g_cpuHasConstReg); + + s_pCurBlock = NULL; + s_pCurBlockEx = NULL; +} + +// ===================================================================================================== +// Thunk helpers for fastmem backpatching +// ===================================================================================================== + +u8* recBeginThunk() +{ + // Check for recompiler cache overflow + if (recPtr >= recPtrEnd) + eeRecNeedsReset = true; + + // Set up assembler to emit thunk code at the current recompiler pointer. + // No constant pool needed for thunks (they're small and self-contained). + armSetAsmPtr(recPtr, recPtrEnd - recPtr + _64kb, nullptr); + u8* aligned = armStartBlock(); + + // Return the aligned address where code actually starts, not recPtr. + // armStartBlock() aligns to 16 bytes — branching to recPtr would hit padding. + return aligned; +} + +u8* recEndThunk() +{ + recPtr = armEndBlock(); + pxAssert(recPtr < SysMemory::GetEERecEnd()); + return recPtr; +} + +// ===================================================================================================== +// R5900cpu struct — public interface +// ===================================================================================================== + +R5900cpu recCpu = { + recReserve, + recShutdown, + recResetEE, + recStep, + recExecute, + recSafeExitExecution, + recCancelInstruction, + recClear, +}; diff --git a/pcsx2/arm64/iR5900-arm64.h b/pcsx2/arm64/iR5900-arm64.h new file mode 100644 index 0000000000..732b6a3551 --- /dev/null +++ b/pcsx2/arm64/iR5900-arm64.h @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +#pragma once + +#include "Config.h" +#include "R5900.h" +#include "R5900OpcodeTables.h" +#include "VU.h" +#include "arm64/iCore-arm64.h" + +// Per-category interpreter fallback toggles. +// Comment out a define to enable native ARM64 codegen for that category. +// #define FORCE_INTERP_BRANCH 1 +// #define FORCE_INTERP_JUMP 1 +// #define FORCE_INTERP_MOVE 1 +// #define FORCE_INTERP_SHIFT 1 +// #define FORCE_INTERP_ALU 1 +// #define FORCE_INTERP_ARITIMM 1 +// #define FORCE_INTERP_MULTDIV 1 +// #define FORCE_INTERP_MEMORY 1 +// #define FORCE_INTERP_COP0 1 +// #define FORCE_INTERP_FPU 1 +// #define FORCE_INTERP_COP2 1 + +// Reserved ARM64 registers for the recompiler +// x19: Fastmem base pointer (callee-saved) +#define RFASTMEMBASE vixl::aarch64::x19 +// x20: Pointer to cpuRegs struct (callee-saved). Loaded once at JIT entry by +// EnterRecompiledCode and never modified for the duration of JIT execution. +// Use armCpuRegMem() to construct cpuRegs-relative MemOperands cheaply. +#define RSTATE vixl::aarch64::x20 +// x24: Pinned &VU0 (callee-saved). Loaded once at JIT entry by +// EnterRecompiledCode; used to address VU0.{VF,VI,ACC,q,statusflag,...} +// fields via single [RVU0, #imm12] load/store in iCOP2-arm64.cpp, +// instead of the 3-mov+ldr abs-addr materialization sequence. +// Survives armEmitCall (callee-saved per AAPCS) and the mVU dispatcher's +// outer Stp/Ldp pair, so cross-EE/mVU dispatches preserve it. +#define RVU0 vixl::aarch64::x24 +// x25: Pinned cpuRegs.cycle (callee-saved). Always valid while in JIT; +// flushed to memory only around C calls (DispatcherEvent, JITCompile, +// recBranchCall). Block-to-block control flow keeps it live, so linked +// branches don't reload the cycle counter from memory each iteration. +#define RECCYCLE vixl::aarch64::x25 + +// Build a MemOperand addressing a cpuRegs field via RSTATE. +// Replaces the 3-instruction `armMoveAddressToReg(RSCRATCHADDR, &cpuRegs.X); +// Ldr/Str ..., [RSCRATCHADDR]` pattern with a single Ldr/Str using a +// signed/unsigned-immediate offset on RSTATE. ARM64 LDR with imm12 covers +// offsets up to 32760 bytes (64-bit) / 16380 bytes (32-bit) — easily larger +// than cpuRegs / fpuRegs combined, so a single instruction suffices for +// every reachable field. +static __fi vixl::aarch64::MemOperand armCpuRegMem(const void* field) +{ + const ptrdiff_t off = reinterpret_cast(field) - reinterpret_cast(&cpuRegs); + return vixl::aarch64::MemOperand(RSTATE, static_cast(off)); +} + +// Pinned-base load/store helpers: when the target is anywhere inside +// _cpuRegistersPack (cpuRegs + fpuRegs), reach it via [RSTATE, #off] in one +// instruction; otherwise fall back to the generic 4-inst armLoadPtr/StorePtr. +static __fi bool armIsCpuRegPtr(const void* field) +{ + const u8* base = reinterpret_cast(&_cpuRegistersPack); + const u8* p = reinterpret_cast(field); + return p >= base && p < base + sizeof(cpuRegistersPack); +} +static __fi void armLoadEERegPtr(const vixl::aarch64::CPURegister& reg, const void* field) +{ + if (armIsCpuRegPtr(field)) + armAsm->Ldr(reg, armCpuRegMem(field)); + else + armLoadPtr(reg, field); +} +static __fi void armStoreEERegPtr(const vixl::aarch64::CPURegister& reg, const void* field) +{ + if (armIsCpuRegPtr(field)) + armAsm->Str(reg, armCpuRegMem(field)); + else + armStorePtr(reg, field); +} + +// Build a MemOperand addressing a VU0 field via RVU0. VURegs is < 2 KB, so +// every reachable field fits in imm12 for byte/halfword/word/doubleword/quad +// ldr/str. Mirrors armCpuRegMem for VU0 — used by iCOP2-arm64.cpp. +static __fi vixl::aarch64::MemOperand armVU0Mem(const void* field) +{ + const ptrdiff_t off = reinterpret_cast(field) - reinterpret_cast(&VU0); + return vixl::aarch64::MemOperand(RVU0, static_cast(off)); +} + +// Emit LD1R from a VU0 field, broadcasting to all lanes. ARM64 LD1R does +// NOT support [base, #imm] addressing — only [base] or post-index. vixl's +// LoadStoreStructAddrModeField silently drops the offset (and the assert +// is gated on VIXL_DEBUG, so Devel builds ship the wrong encoding instead +// of trapping). Materialize the address with a single ADD imm12 instead of +// 3-mov: VURegs fields fit within 4 KB of &VU0, so one ADD suffices. +// Total cost: ADD + LD1R = 2 insns, vs the original 4-insn 3-mov + LD1R. +static __fi void armLd1rVU0(const vixl::aarch64::VRegister& vt, const void* field) +{ + const ptrdiff_t off = reinterpret_cast(field) - reinterpret_cast(&VU0); + armAsm->Add(RSCRATCHADDR, RVU0, static_cast(off)); + armAsm->Ld1r(vt, vixl::aarch64::MemOperand(RSCRATCHADDR)); +} + +extern u32 maxrecmem; +extern u32 pc; // recompiler pc +extern int g_branch; // set for branch +extern u32 target; // branch target +extern u32 s_nBlockCycles; // cycles of current block recompiling +extern bool s_nBlockInterlocked; // Current block has VU0 interlocking + +////////////////////////////////////////////////////////////////////////////////////////// +// Interpreter fallback macros + +#define REC_FUNC(f) \ + void rec##f() \ + { \ + /* Delete destination register's const/alloc state before interpreter call. \ + * The interpreter writes directly to cpuRegs, making any cached const or \ + * allocated register stale. SPECIAL ops (Rd), loads/COP (Rt). */ \ + const u32 _op = cpuRegs.code >> 26; \ + const int _dest = (_op == 0 || _op == 0x1C) ? _Rd_ : _Rt_; \ + if (_dest > 0) \ + _deleteEEreg(_dest, 1); \ + recCall(Interp::f); \ + } + +#define REC_FUNC_DEL(f, delreg) \ + void rec##f() \ + { \ + if ((delreg) > 0) \ + _deleteEEreg(delreg, 1); \ + recCall(Interp::f); \ + } + +#define REC_SYS(f) \ + void rec##f() \ + { \ + recBranchCall(Interp::f); \ + } + +#define REC_SYS_DEL(f, delreg) \ + void rec##f() \ + { \ + if ((delreg) > 0) \ + _deleteEEreg(delreg, 1); \ + recBranchCall(Interp::f); \ + } + +extern bool g_recompilingDelaySlot; + +// Used for generating backpatch thunks for fastmem +u8* recBeginThunk(); +u8* recEndThunk(); + +// Branch processing +bool TrySwapDelaySlot(u32 rs, u32 rt, u32 rd, bool allow_loadstore); +void SaveBranchState(); +void LoadBranchState(); + +void recompileNextInstruction(bool delayslot, bool swapped_delay_slot); +void SetBranchReg(); +void SetBranchImm(u32 imm); + +void iFlushCall(int flushtype); +void recBranchCall(void (*func)()); +void recCall(void (*func)()); +// Emit the post-interpreter-call TLB-miss exception dispatch (defined in +// iR5900-arm64.cpp). DispatcherReg/s_recTlbMissOccurred are file-local there, +// so cross-TU interpreter-call sites (recVTLB-arm64.cpp) route through this. +void recEmitInterpTlbMissCheck(); +u32 scaleblockcycles_clear(); + +// COP2 / VU0 sync emit helper (defined in iCOP2-arm64.cpp). +// interlock=true mirrors x86 COP2_Interlock (CFC2/CTC2/QMFC2/QMTC2 path); +// interlock=false mirrors mVUSyncVU0 / mVUFinishVU0 gating used by LQC2/SQC2 +// and the COP2 macro-arithmetic setup. finishFunc is the secondary helper +// to invoke after vu0Sync (typically _vu0FinishMicro or _vu0WaitMicro); +// pass nullptr for "sync only". Emits zero instructions when EEINST analysis +// flags say no sync is needed. +void cop2EmitConditionalSync(bool interlock, void (*finishFunc)()); + +// COP2 macro-mode microVU0 state setup/teardown (defined in microVU-arm64.cpp). +// Mirrors x86 setupMacroOp/endMacroOp's regAlloc reset, microVU0.cop2 = 1, +// prog.IRinfo.curPC/info[0] init, code = cpuRegs.code, and flag scaffolding. +// Required before invoking any mVU emitter (mVU_LQI/SQI/MFIR/...) from a +// COP2 macro-mode dispatch wrapper. eeinstInfo is g_pCurInstInfo->info (or 0 +// when EEINST analysis isn't live for this site). +void mVUmacroSetupCOP2State(int mode, u32 eeinstInfo); +void mVUmacroEndCOP2State(); + +// COP2 macro-mode setup/teardown wrapper (defined in iCOP2-arm64.cpp). +// Calls cop2EmitConditionalSync, emits status-flag denormalize/normalize when +// mode & 0x10, then runs mVUmacroSetup/EndCOP2State to ready microVU0 for the +// mVU emitter pass. REC_COP2_mVU0_ARM64-style wrappers in iR5900Misc-arm64.cpp +// bracket calls to mVUmacroEmit_ with these. +void setupMacroOp_arm64(int mode); +void endMacroOp_arm64(int mode); + +// COP2 macro-mode emit adapters (defined in microVU-arm64.cpp). Each runs the +// pass1+pass2 dispatch x86 uses in REC_COP2_mVU0 (microVU_Macro.inl:127-133). +// mode is the same mode bits passed to setupMacroOp_arm64; only bit 0x04 +// (requires analysis pass) is observed by the adapter. +void mVUmacroEmit_LQI(int mode); +void mVUmacroEmit_SQI(int mode); +void mVUmacroEmit_LQD(int mode); +void mVUmacroEmit_SQD(int mode); +void mVUmacroEmit_MTIR(int mode); +void mVUmacroEmit_MFIR(int mode); +void mVUmacroEmit_ILWR(int mode); +void mVUmacroEmit_ISWR(int mode); +void mVUmacroEmit_RNEXT(int mode); +void mVUmacroEmit_RGET(int mode); +void mVUmacroEmit_RINIT(int mode); +void mVUmacroEmit_RXOR(int mode); + +namespace R5900 +{ + namespace Dynarec + { + extern void recDoBranchImm(u32 branchTo, u32* jmpSkip, bool isLikely = false, bool swappedDelaySlot = false); + } +} + +//////////////////////////////////////////////////////////////////// +// Constant Propagation + +#define GPR_IS_CONST1(reg) (EE_CONST_PROP && (reg) < 32 && (g_cpuHasConstReg & (1 << (reg)))) +#define GPR_IS_CONST2(reg1, reg2) (EE_CONST_PROP && (g_cpuHasConstReg & (1 << (reg1))) && (g_cpuHasConstReg & (1 << (reg2)))) +#define GPR_IS_DIRTY_CONST(reg) (EE_CONST_PROP && (reg) < 32 && (g_cpuHasConstReg & (1 << (reg))) && (!(g_cpuFlushedConstReg & (1 << (reg))))) +#define GPR_SET_CONST(reg) \ + { \ + if ((reg) < 32) \ + { \ + g_cpuHasConstReg |= (1 << (reg)); \ + g_cpuFlushedConstReg &= ~(1 << (reg)); \ + } \ + } + +#define GPR_DEL_CONST(reg) \ + { \ + if ((reg) < 32) \ + g_cpuHasConstReg &= ~(1 << (reg)); \ + } + +alignas(16) extern GPR_reg64 g_cpuConstRegs[32]; +extern u32 g_cpuHasConstReg, g_cpuFlushedConstReg; + +// Move guest GPR value to an ARM64 register +void _eeMoveGPRtoR(const vixl::aarch64::Register& to, int fromgpr, bool allow_preload = true); + +void _eeFlushAllDirty(); +void _eeOnWriteReg(int reg, int signext); + +// Totally deletes from const, NEON, and GPR entries +// if flush is 1, also flushes to memory +void _deleteEEreg(int reg, int flush); +void _deleteEEreg128(int reg); + +void _flushEEreg(int reg, bool clear = false); + +////////////////////////////////////// +// Templates for code recompilation // +////////////////////////////////////// + +typedef void (*R5900FNPTR)(); +typedef void (*R5900FNPTR_INFO)(int info); + +// Memory-based templates — no register allocation, all operands via cpuRegs memory. +void eeRecompileCodeRC0_MEM(R5900FNPTR constcode, R5900FNPTR_INFO constscode, R5900FNPTR_INFO consttcode, R5900FNPTR_INFO noconstcode, int xmminfo); +void eeRecompileCodeRC1_MEM(R5900FNPTR constcode, R5900FNPTR_INFO noconstcode, int xmminfo); +void eeRecompileCodeRC2_MEM(R5900FNPTR constcode, R5900FNPTR_INFO noconstcode, int xmminfo); + +#define EERECOMPILE_CODERC0_MEM(fn, xmminfo) \ + void rec##fn(void) \ + { \ + eeRecompileCodeRC0_MEM(rec##fn##_const, rec##fn##_consts, rec##fn##_constt, rec##fn##_, (xmminfo)); \ + } + +#define EERECOMPILE_CODEX_MEM(codename, fn, xmminfo) \ + void rec##fn(void) \ + { \ + codename(rec##fn##_const, rec##fn##_, (xmminfo)); \ + } + +#define FPURECOMPILE_CONSTCODE(fn, xmminfo) \ + void rec##fn(void) \ + { \ + if (CHECK_FPU_FULL) \ + eeFPURecompileCode(DOUBLE::rec##fn##_xmm, R5900::Interpreter::OpcodeImpl::COP1::fn, xmminfo); \ + else \ + eeFPURecompileCode(rec##fn##_xmm, R5900::Interpreter::OpcodeImpl::COP1::fn, xmminfo); \ + } + +int eeRecompileCodeXMM(int xmminfo); +void eeFPURecompileCode(R5900FNPTR_INFO xmmcode, R5900FNPTR fpucode, int xmminfo); diff --git a/pcsx2/arm64/iR5900Analysis.h b/pcsx2/arm64/iR5900Analysis.h new file mode 100644 index 0000000000..e2f03af9e0 --- /dev/null +++ b/pcsx2/arm64/iR5900Analysis.h @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 wrapper for the shared instruction analysis pass. +// Routes to ARM64-specific headers instead of x86 iR5900.h/iCore.h. + +#pragma once + +#include "arm64/iR5900-arm64.h" +#include "arm64/iCore-arm64.h" + +// Re-export the shared analysis classes and functions from x86/iR5900Analysis.h +namespace R5900 +{ + class AnalysisPass + { + public: + AnalysisPass(); + virtual ~AnalysisPass(); + + virtual void Run(u32 start, u32 end, EEINST* inst_cache); + + protected: + template + void ForEachInstruction(u32 start, u32 end, EEINST* inst_cache, const F& func); + + template + void DumpAnnotatedBlock(u32 start, u32 end, EEINST* inst_cache, const F& func); + }; + + class COP2FlagHackPass final : public AnalysisPass + { + public: + COP2FlagHackPass(); + ~COP2FlagHackPass(); + + void Run(u32 start, u32 end, EEINST* inst_cache) override; + + private: + void DumpAnnotatedBlock(u32 start, u32 end, EEINST* inst_cache); + + void CommitStatusFlag(); + void CommitMACFlag(); + void CommitClipFlag(); + void CommitAllFlags(); + + bool m_status_denormalized = false; + EEINST* m_last_status_write = nullptr; + EEINST* m_last_mac_write = nullptr; + EEINST* m_last_clip_write = nullptr; + + u32 m_cfc2_pc = 0; + }; + + class COP2MicroFinishPass final : public AnalysisPass + { + public: + COP2MicroFinishPass(); + ~COP2MicroFinishPass(); + + void Run(u32 start, u32 end, EEINST* inst_cache) override; + }; +} // namespace R5900 + +void recBackpropBSC(u32 code, EEINST* prev, EEINST* pinst); diff --git a/pcsx2/arm64/iR5900Arit-arm64.cpp b/pcsx2/arm64/iR5900Arit-arm64.cpp new file mode 100644 index 0000000000..5adcb99f83 --- /dev/null +++ b/pcsx2/arm64/iR5900Arit-arm64.cpp @@ -0,0 +1,456 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE ALU Instruction Codegen — memory-based +// All operands loaded from / stored to cpuRegs.GPR memory. +// No register allocation for scalar ops. + +#include "arm64/iR5900-arm64.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_ALU +REC_FUNC(ADD); +void recADDU() { recADD(); } +REC_FUNC(DADD); +void recDADDU() { recDADD(); } +REC_FUNC(SUB); +void recSUBU() { recSUB(); } +REC_FUNC(DSUB); +void recDSUBU() { recDSUB(); } +REC_FUNC(AND); +REC_FUNC(OR); +REC_FUNC(XOR); +REC_FUNC(NOR); +REC_FUNC(SLT); +REC_FUNC(SLTU); +#else + +// Memory load/store helpers — always use cpuRegs memory +static void memLoadS32() { armLoadEERegPtr(RWARG1, &cpuRegs.GPR.r[_Rs_].UL[0]); } +static void memLoadT32() { armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rt_].UL[0]); } +static void memLoadS64() { armLoadEERegPtr(RXARG1, &cpuRegs.GPR.r[_Rs_].UD[0]); } +static void memLoadT64() { armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); } +static void memStoreD() { armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); } + +/********************************************************* + * Register arithmetic — rd = rs OP rt * + * 32-bit ops sign-extend result to 64 bits * + *********************************************************/ + +//// ADD / ADDU + +static void recADD_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = s64(s32(g_cpuConstRegs[_Rs_].UL[0] + g_cpuConstRegs[_Rt_].UL[0])); +} + +static void recADD_consts(int info) +{ + const s32 cval = g_cpuConstRegs[_Rs_].SL[0]; + memLoadT32(); + if (cval != 0) + armAsm->Add(RWSCRATCH, RWSCRATCH, cval); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recADD_constt(int info) +{ + const s32 cval = g_cpuConstRegs[_Rt_].SL[0]; + memLoadS32(); + if (cval != 0) + { + armAsm->Add(RWSCRATCH, RWARG1, cval); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + } + else + { + armAsm->Sxtw(RXSCRATCH, RWARG1); + } + memStoreD(); +} + +static void recADD_(int info) +{ + memLoadS32(); + memLoadT32(); + armAsm->Add(RWSCRATCH, RWARG1, RWSCRATCH); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(ADD, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); + +void recADDU() { recADD(); } + +//// DADD / DADDU + +static void recDADD_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] + g_cpuConstRegs[_Rt_].UD[0]; +} + +static void recDADD_consts(int info) +{ + const s64 cval = g_cpuConstRegs[_Rs_].SD[0]; + memLoadT64(); + if (cval != 0) + { + armAsm->Mov(RXARG1, cval); + armAsm->Add(RXSCRATCH, RXSCRATCH, RXARG1); + } + memStoreD(); +} + +static void recDADD_constt(int info) +{ + const s64 cval = g_cpuConstRegs[_Rt_].SD[0]; + memLoadS64(); + if (cval != 0) + armAsm->Add(RXSCRATCH, RXARG1, cval); + else + armAsm->Mov(RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recDADD_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Add(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(DADD, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +void recDADDU() { recDADD(); } + +//// SUB / SUBU + +static void recSUB_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = s64(s32(g_cpuConstRegs[_Rs_].UL[0] - g_cpuConstRegs[_Rt_].UL[0])); +} + +static void recSUB_consts(int info) +{ + const s32 cval = g_cpuConstRegs[_Rs_].SL[0]; + memLoadT32(); + armAsm->Mov(RWARG1, cval); + armAsm->Sub(RWSCRATCH, RWARG1, RWSCRATCH); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSUB_constt(int info) +{ + const s32 cval = g_cpuConstRegs[_Rt_].SL[0]; + memLoadS32(); + if (cval != 0) + { + armAsm->Sub(RWSCRATCH, RWARG1, cval); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + } + else + { + armAsm->Sxtw(RXSCRATCH, RWARG1); + } + memStoreD(); +} + +static void recSUB_(int info) +{ + // rs - rs == 0 always; emit a single store of zero. + if (_Rs_ == _Rt_) + { + armStoreEERegPtr(a64::xzr, &cpuRegs.GPR.r[_Rd_].UD[0]); + return; + } + memLoadS32(); + memLoadT32(); + armAsm->Sub(RWSCRATCH, RWARG1, RWSCRATCH); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(SUB, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); + +void recSUBU() { recSUB(); } + +//// DSUB / DSUBU + +static void recDSUB_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] - g_cpuConstRegs[_Rt_].UD[0]; +} + +static void recDSUB_consts(int info) +{ + const s64 cval = g_cpuConstRegs[_Rs_].SD[0]; + memLoadT64(); + armAsm->Mov(RXARG1, cval); + armAsm->Sub(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +static void recDSUB_constt(int info) +{ + const s64 cval = g_cpuConstRegs[_Rt_].SD[0]; + memLoadS64(); + if (cval != 0) + armAsm->Sub(RXSCRATCH, RXARG1, cval); + else + armAsm->Mov(RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recDSUB_(int info) +{ + // rs - rs == 0 always; emit a single store of zero. + if (_Rs_ == _Rt_) + { + armStoreEERegPtr(a64::xzr, &cpuRegs.GPR.r[_Rd_].UD[0]); + return; + } + memLoadS64(); + memLoadT64(); + armAsm->Sub(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(DSUB, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +void recDSUBU() { recDSUB(); } + +//// AND — 64-bit + +static void recAND_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] & g_cpuConstRegs[_Rt_].UD[0]; +} + +static void recAND_consts(int info) +{ + memLoadT64(); + armAsm->And(RXSCRATCH, RXSCRATCH, g_cpuConstRegs[_Rs_].UD[0]); + memStoreD(); +} + +static void recAND_constt(int info) +{ + memLoadS64(); + armAsm->And(RXSCRATCH, RXARG1, g_cpuConstRegs[_Rt_].UD[0]); + memStoreD(); +} + +static void recAND_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->And(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(AND, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// OR — 64-bit + +static void recOR_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] | g_cpuConstRegs[_Rt_].UD[0]; +} + +static void recOR_consts(int info) +{ + const u64 cval = g_cpuConstRegs[_Rs_].UD[0]; + memLoadT64(); + if (cval != 0) + armAsm->Orr(RXSCRATCH, RXSCRATCH, cval); + memStoreD(); +} + +static void recOR_constt(int info) +{ + const u64 cval = g_cpuConstRegs[_Rt_].UD[0]; + memLoadS64(); + if (cval != 0) + armAsm->Orr(RXSCRATCH, RXARG1, cval); + else + armAsm->Mov(RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recOR_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Orr(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(OR, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// XOR — 64-bit + +static void recXOR_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] ^ g_cpuConstRegs[_Rt_].UD[0]; +} + +static void recXOR_consts(int info) +{ + memLoadT64(); + armAsm->Mov(RXARG1, g_cpuConstRegs[_Rs_].UD[0]); + armAsm->Eor(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recXOR_constt(int info) +{ + memLoadS64(); + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rt_].UD[0]); + armAsm->Eor(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +static void recXOR_(int info) +{ + // rs ^ rs == 0 always; skip the two operand loads (mirrors recSUB_). + if (_Rs_ == _Rt_) + { + armAsm->Mov(RXSCRATCH, 0); + memStoreD(); + return; + } + memLoadS64(); + memLoadT64(); + armAsm->Eor(RXSCRATCH, RXARG1, RXSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(XOR, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// NOR — rd = ~(rs | rt), 64-bit + +static void recNOR_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = ~(g_cpuConstRegs[_Rs_].UD[0] | g_cpuConstRegs[_Rt_].UD[0]); +} + +static void recNOR_consts(int info) +{ + const u64 cval = g_cpuConstRegs[_Rs_].UD[0]; + memLoadT64(); + if (cval != 0) + armAsm->Orr(RXSCRATCH, RXSCRATCH, cval); + armAsm->Mvn(RXSCRATCH, RXSCRATCH); + memStoreD(); +} + +static void recNOR_constt(int info) +{ + const u64 cval = g_cpuConstRegs[_Rt_].UD[0]; + memLoadS64(); + if (cval != 0) + armAsm->Orr(RXSCRATCH, RXARG1, cval); + else + armAsm->Mov(RXSCRATCH, RXARG1); + armAsm->Mvn(RXSCRATCH, RXSCRATCH); + memStoreD(); +} + +static void recNOR_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Orr(RXSCRATCH, RXARG1, RXSCRATCH); + armAsm->Mvn(RXSCRATCH, RXSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(NOR, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// SLT — rd = (rs < rt) ? 1 : 0 (signed 64-bit compare) + +static void recSLT_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = (g_cpuConstRegs[_Rs_].SD[0] < g_cpuConstRegs[_Rt_].SD[0]) ? 1 : 0; +} + +static void recSLT_consts(int info) +{ + memLoadT64(); + armAsm->Mov(RXARG1, g_cpuConstRegs[_Rs_].SD[0]); + armAsm->Cmp(RXARG1, RXSCRATCH); + armAsm->Cset(RXSCRATCH, a64::lt); + memStoreD(); +} + +static void recSLT_constt(int info) +{ + memLoadS64(); + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rt_].SD[0]); + armAsm->Cmp(RXARG1, RXSCRATCH); + armAsm->Cset(RXSCRATCH, a64::lt); + memStoreD(); +} + +static void recSLT_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Cmp(RXARG1, RXSCRATCH); + armAsm->Cset(RXSCRATCH, a64::lt); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(SLT, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// SLTU — rd = (rs < rt) ? 1 : 0 (unsigned 64-bit compare) + +static void recSLTU_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = (g_cpuConstRegs[_Rs_].UD[0] < g_cpuConstRegs[_Rt_].UD[0]) ? 1 : 0; +} + +static void recSLTU_consts(int info) +{ + memLoadT64(); + armAsm->Mov(RXARG1, g_cpuConstRegs[_Rs_].UD[0]); + armAsm->Cmp(RXARG1, RXSCRATCH); + armAsm->Cset(RXSCRATCH, a64::lo); + memStoreD(); +} + +static void recSLTU_constt(int info) +{ + memLoadS64(); + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rt_].UD[0]); + armAsm->Cmp(RXARG1, RXSCRATCH); + armAsm->Cset(RXSCRATCH, a64::lo); + memStoreD(); +} + +static void recSLTU_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Cmp(RXARG1, RXSCRATCH); + armAsm->Cset(RXSCRATCH, a64::lo); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(SLTU, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +#endif // !FORCE_INTERP_ALU + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900AritImm-arm64.cpp b/pcsx2/arm64/iR5900AritImm-arm64.cpp new file mode 100644 index 0000000000..f4596a8621 --- /dev/null +++ b/pcsx2/arm64/iR5900AritImm-arm64.cpp @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE ALU Immediate Instruction Codegen — memory-based +// rt = rs OP imm16. All operands via cpuRegs memory. + +#include "arm64/iR5900-arm64.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_ARITIMM +REC_FUNC(ADDI); +void recADDIU() { recADDI(); } +REC_FUNC(DADDI); +void recDADDIU() { recDADDI(); } +REC_FUNC(ANDI); +REC_FUNC(ORI); +REC_FUNC(XORI); +REC_FUNC(SLTI); +REC_FUNC(SLTIU); +#else + +// Memory load/store helpers +static void memLoadS32() { armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rs_].UL[0]); } +static void memLoadS64() { armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); } +static void memStoreT() { armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); } + +//// ADDI / ADDIU — rt = sign_extend(rs + imm) +static void recADDI_const() +{ + g_cpuConstRegs[_Rt_].SD[0] = s64(s32(g_cpuConstRegs[_Rs_].UL[0] + u32(s32(_Imm_)))); +} + +static void recADDI_(int info) +{ + memLoadS32(); + if (_Imm_ != 0) + armAsm->Add(RWSCRATCH, RWSCRATCH, _Imm_); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, ADDI, XMMINFO_WRITET | XMMINFO_READS); + +void recADDIU() { recADDI(); } + +//// DADDI / DADDIU — rt = rs + sign_extend(imm) +static void recDADDI_const() +{ + g_cpuConstRegs[_Rt_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] + u64(s64(_Imm_)); +} + +static void recDADDI_(int info) +{ + memLoadS64(); + if (_Imm_ != 0) + { + // vixl's Add(int64_t) picks the right ADD/SUB-imm encoding and + // materializes via x16 when the immediate is unencodable. + armAsm->Add(RXSCRATCH, RXSCRATCH, static_cast(static_cast(_Imm_))); + } + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, DADDI, XMMINFO_WRITET | XMMINFO_READS | XMMINFO_64BITOP); + +void recDADDIU() { recDADDI(); } + +//// ANDI — rt = rs & zero_extend(imm16) +static void recANDI_const() +{ + g_cpuConstRegs[_Rt_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] & (u64)(u16)_ImmU_; +} + +static void recANDI_(int info) +{ + memLoadS64(); + if (_ImmU_ == 0) + armAsm->Mov(RXSCRATCH, 0); + else + armAsm->And(RXSCRATCH, RXSCRATCH, static_cast(static_cast(_ImmU_))); + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, ANDI, XMMINFO_WRITET | XMMINFO_READS | XMMINFO_64BITOP); + +//// ORI — rt = rs | zero_extend(imm16) +static void recORI_const() +{ + g_cpuConstRegs[_Rt_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] | (u64)(u16)_ImmU_; +} + +static void recORI_(int info) +{ + memLoadS64(); + if (_ImmU_ != 0) + armAsm->Orr(RXSCRATCH, RXSCRATCH, static_cast(static_cast(_ImmU_))); + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, ORI, XMMINFO_WRITET | XMMINFO_READS | XMMINFO_64BITOP); + +//// XORI — rt = rs ^ zero_extend(imm16) +static void recXORI_const() +{ + g_cpuConstRegs[_Rt_].UD[0] = g_cpuConstRegs[_Rs_].UD[0] ^ (u64)(u16)_ImmU_; +} + +static void recXORI_(int info) +{ + memLoadS64(); + if (_ImmU_ != 0) + armAsm->Eor(RXSCRATCH, RXSCRATCH, static_cast(static_cast(_ImmU_))); + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, XORI, XMMINFO_WRITET | XMMINFO_READS | XMMINFO_64BITOP); + +//// SLTI — rt = (rs < sign_extend(imm)) ? 1 : 0 (signed) +static void recSLTI_const() +{ + g_cpuConstRegs[_Rt_].UD[0] = (g_cpuConstRegs[_Rs_].SD[0] < (s64)(s32)_Imm_) ? 1 : 0; +} + +static void recSLTI_(int info) +{ + memLoadS64(); + armAsm->Cmp(RXSCRATCH, static_cast(static_cast(_Imm_))); + armAsm->Cset(RXSCRATCH, a64::lt); + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, SLTI, XMMINFO_WRITET | XMMINFO_READS | XMMINFO_64BITOP); + +//// SLTIU — rt = (rs < sign_extend(imm)) ? 1 : 0 (unsigned) +static void recSLTIU_const() +{ + g_cpuConstRegs[_Rt_].UD[0] = (g_cpuConstRegs[_Rs_].UD[0] < (u64)(s64)(s32)_Imm_) ? 1 : 0; +} + +static void recSLTIU_(int info) +{ + memLoadS64(); + // Sign-extended imm — Cmp condition flags are signedness-agnostic; only + // the Cset (lo = unsigned-less-than) differs from SLTI. + armAsm->Cmp(RXSCRATCH, static_cast(static_cast(_Imm_))); + armAsm->Cset(RXSCRATCH, a64::lo); + memStoreT(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC1_MEM, SLTIU, XMMINFO_WRITET | XMMINFO_READS | XMMINFO_64BITOP); + +#endif // !FORCE_INTERP_ARITIMM + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900Branch-arm64.cpp b/pcsx2/arm64/iR5900Branch-arm64.cpp new file mode 100644 index 0000000000..bbc532d687 --- /dev/null +++ b/pcsx2/arm64/iR5900Branch-arm64.cpp @@ -0,0 +1,530 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Branch Instruction Codegen — NEON-based +// Branches read GPR values for comparison. Values are extracted from +// NEON registers via FMOV or loaded from memory after flush. + +#include "arm64/iR5900-arm64.h" +#include "common/Assertions.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_BRANCH +REC_SYS(BEQ); +REC_SYS(BNE); +REC_SYS(BEQL); +REC_SYS(BNEL); +REC_SYS(BLEZ); +REC_SYS(BGTZ); +REC_SYS(BLTZ); +REC_SYS(BGEZ); +REC_SYS(BLEZL); +REC_SYS(BGTZL); +REC_SYS(BLTZL); +REC_SYS(BGEZL); +REC_SYS(BLTZAL); +REC_SYS(BGEZAL); +REC_SYS(BLTZALL); +REC_SYS(BGEZALL); +#else + +// Thread-local label for the "not taken" forward branch +static thread_local a64::Label* s_pBranchLabel = nullptr; + +// Load a GPR value into an ARM64 X register for comparison. Allocator-aware: +// _eeFlushAllDirty leaves slots in MODE_READ (clean) so any GPR/NEON slot +// holding `gprreg` is still authoritative and can be read via Mov/Fmov, +// saving an LDR per branch operand. Falls back to memory when unallocated. +static void loadGPRtoX(const a64::Register& dst, int gprreg) +{ + _eeMoveGPRtoR(dst, gprreg); +} + +// Emit comparison for BEQ/BNE and set up forward branch. +// +// `bne` selects the emitted forward-skip condition, NOT the instruction: +// bne==0 skips on `ne` (used by BEQ and, with its inverted structure, BNEL); +// bne==1 skips on `eq`. The forward branch jumps over the delay slot on the +// not-taken edge. +// +// Const-operand fast paths (one side const-folded): let vixl pick the optimal +// CMP/CMN-immediate encoding, and for compare-against-zero (BEQ/BNE $zero — the +// most common branch shape) collapse the whole test to a single Cbz/Cbnz with +// no Cmp at all. +// Cbz/Cbnz reach ±1MB, strictly wider than the Tbz already proven safe over +// this same skip distance in recSetBranchL. +static void recSetBranchEQ(int bne, int process) +{ + s_pBranchLabel = new a64::Label(); + + if (process & (PROCESS_CONSTS | PROCESS_CONSTT)) + { + const int constReg = (process & PROCESS_CONSTS) ? _Rs_ : _Rt_; + const int liveReg = (process & PROCESS_CONSTS) ? _Rt_ : _Rs_; + const s64 cval = g_cpuConstRegs[constReg].SD[0]; + + _eeFlushAllDirty(); + loadGPRtoX(RXARG1, liveReg); + + if (cval == 0) + { + // Single test-and-branch against $zero — no Cmp. + // bne==0 skips on ne → live != 0 → Cbnz; bne==1 skips on eq → Cbz. + if (bne) + armAsm->Cbz(RXARG1, s_pBranchLabel); + else + armAsm->Cbnz(RXARG1, s_pBranchLabel); + return; + } + + // vixl emits a single CMP/CMN immediate when cval fits, else materializes. + armAsm->Cmp(RXARG1, cval); + } + else + { + _eeFlushAllDirty(); + loadGPRtoX(RXARG1, _Rs_); + loadGPRtoX(RXSCRATCH, _Rt_); + armAsm->Cmp(RXARG1, RXSCRATCH); + } + + if (bne) + armAsm->B(s_pBranchLabel, a64::eq); + else + armAsm->B(s_pBranchLabel, a64::ne); +} + +// Emit comparison for BLTZ/BGEZ (rs vs 0) and set up forward branch. +// +// The "forward branch" jumps over the delay slot when the BLTZ/BGEZ would +// NOT be taken. For BLTZ (ltz=1) we skip when rs >= 0, i.e. bit 63 of the +// 64-bit GPR is zero → Tbz. For BGEZ (ltz=0) we skip when rs < 0, i.e. +// bit 63 is one → Tbnz. Tbz/Tbnz directly test a bit and branch, so no +// Cmp insn is needed. Shares the centralised setup across all 8 BLTZ/BGEZ/L/AL/ALL +// callers in this file. +static void recSetBranchL(int ltz) +{ + _eeFlushAllDirty(); + loadGPRtoX(RXSCRATCH, _Rs_); + + s_pBranchLabel = new a64::Label(); + if (ltz) + armAsm->Tbz(RXSCRATCH, 63, s_pBranchLabel); + else + armAsm->Tbnz(RXSCRATCH, 63, s_pBranchLabel); +} + +// Bind the forward branch label +static void recBindBranchLabel() +{ + pxAssert(s_pBranchLabel != nullptr); + armAsm->Bind(s_pBranchLabel); + delete s_pBranchLabel; + s_pBranchLabel = nullptr; +} + +//// BEQ — branch if rs == rt +static void recBEQ_const() +{ + u32 branchTo; + if (g_cpuConstRegs[_Rs_].SD[0] == g_cpuConstRegs[_Rt_].SD[0]) + branchTo = ((s32)_Imm_ * 4) + pc; + else + branchTo = pc + 4; + recompileNextInstruction(true, false); + SetBranchImm(branchTo); +} + +static void recBEQ_process(int process) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + if (_Rs_ == _Rt_) + { + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + return; + } + + const bool swap = TrySwapDelaySlot(_Rs_, _Rt_, 0, true); + recSetBranchEQ(0, process); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBranchLabel(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +void recBEQ() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + recBEQ_const(); + else if (GPR_IS_CONST1(_Rs_)) + recBEQ_process(PROCESS_CONSTS); + else if (GPR_IS_CONST1(_Rt_)) + recBEQ_process(PROCESS_CONSTT); + else + recBEQ_process(0); +} + +//// BNE — branch if rs != rt +static void recBNE_const() +{ + u32 branchTo; + if (g_cpuConstRegs[_Rs_].SD[0] != g_cpuConstRegs[_Rt_].SD[0]) + branchTo = ((s32)_Imm_ * 4) + pc; + else + branchTo = pc + 4; + recompileNextInstruction(true, false); + SetBranchImm(branchTo); +} + +static void recBNE_process(int process) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + if (_Rs_ == _Rt_) + { + recompileNextInstruction(true, false); + SetBranchImm(pc); + return; + } + + const bool swap = TrySwapDelaySlot(_Rs_, _Rt_, 0, true); + recSetBranchEQ(1, process); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBranchLabel(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +void recBNE() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + recBNE_const(); + else if (GPR_IS_CONST1(_Rs_)) + recBNE_process(PROCESS_CONSTS); + else if (GPR_IS_CONST1(_Rt_)) + recBNE_process(PROCESS_CONSTT); + else + recBNE_process(0); +} + +//// BEQL — branch likely if rs == rt +static void recBEQL_const() +{ + // Capture the taken target BEFORE recompileNextInstruction advances pc by 4 + // (consistent with recBEQ_const / recBEQL_process). + const u32 branchTo = ((s32)_Imm_ * 4) + pc; + if (g_cpuConstRegs[_Rs_].SD[0] == g_cpuConstRegs[_Rt_].SD[0]) + { + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + } + else + SetBranchImm(pc + 4); +} + +static void recBEQL_process(int process) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + recSetBranchEQ(0, process); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBranchLabel(); + LoadBranchState(); + SetBranchImm(pc); +} + +void recBEQL() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + recBEQL_const(); + else if (GPR_IS_CONST1(_Rs_)) + recBEQL_process(PROCESS_CONSTS); + else if (GPR_IS_CONST1(_Rt_)) + recBEQL_process(PROCESS_CONSTT); + else + recBEQL_process(0); +} + +//// BNEL — branch likely if rs != rt +static void recBNEL_const() +{ + // Capture the taken target BEFORE recompileNextInstruction advances pc by 4 + // (consistent with recBNE_const / recBNEL_process). + const u32 branchTo = ((s32)_Imm_ * 4) + pc; + if (g_cpuConstRegs[_Rs_].SD[0] != g_cpuConstRegs[_Rt_].SD[0]) + { + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + } + else + SetBranchImm(pc + 4); +} + +static void recBNEL_process(int process) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + recSetBranchEQ(0, process); + + SaveBranchState(); + SetBranchImm(pc + 4); + + recBindBranchLabel(); + LoadBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); +} + +void recBNEL() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + recBNEL_const(); + else if (GPR_IS_CONST1(_Rs_)) + recBNEL_process(PROCESS_CONSTS); + else if (GPR_IS_CONST1(_Rt_)) + recBNEL_process(PROCESS_CONSTT); + else + recBNEL_process(0); +} + +/********************************************************* + * Single-register branches: BLTZ, BGEZ, BLEZ, BGTZ * + *********************************************************/ + +static void recBranchSingle(a64::Condition skip_cond) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + if (GPR_IS_CONST1(_Rs_)) + { + bool taken; + s64 val = g_cpuConstRegs[_Rs_].SD[0]; + if (skip_cond == a64::gt) taken = (val <= 0); + else if (skip_cond == a64::le) taken = (val > 0); + else if (skip_cond == a64::ge) taken = (val < 0); + else if (skip_cond == a64::lt) taken = (val >= 0); + else taken = false; + + if (!taken) branchTo = pc + 4; + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + return; + } + + const bool swap = TrySwapDelaySlot(_Rs_, 0, 0, true); + + if (skip_cond == a64::ge || skip_cond == a64::lt) + { + recSetBranchL(skip_cond == a64::ge ? 1 : 0); + } + else + { + _eeFlushAllDirty(); + loadGPRtoX(RXSCRATCH, _Rs_); + armAsm->Cmp(RXSCRATCH, 0); + s_pBranchLabel = new a64::Label(); + armAsm->B(s_pBranchLabel, skip_cond); + } + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBranchLabel(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +static void recBranchSingleLikely(a64::Condition skip_cond) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + if (GPR_IS_CONST1(_Rs_)) + { + bool taken; + s64 val = g_cpuConstRegs[_Rs_].SD[0]; + if (skip_cond == a64::gt) taken = (val <= 0); + else if (skip_cond == a64::le) taken = (val > 0); + else if (skip_cond == a64::ge) taken = (val < 0); + else if (skip_cond == a64::lt) taken = (val >= 0); + else taken = false; + + if (taken) + { + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + } + else + SetBranchImm(pc + 4); + return; + } + + if (skip_cond == a64::ge || skip_cond == a64::lt) + { + recSetBranchL(skip_cond == a64::ge ? 1 : 0); + } + else + { + _eeFlushAllDirty(); + loadGPRtoX(RXSCRATCH, _Rs_); + armAsm->Cmp(RXSCRATCH, 0); + s_pBranchLabel = new a64::Label(); + armAsm->B(s_pBranchLabel, skip_cond); + } + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBranchLabel(); + LoadBranchState(); + SetBranchImm(pc); +} + +void recBLEZ() { recBranchSingle(a64::gt); } +void recBGTZ() { recBranchSingle(a64::le); } +void recBLTZ() { recBranchSingle(a64::ge); } +void recBGEZ() { recBranchSingle(a64::lt); } + +void recBLEZL() { recBranchSingleLikely(a64::gt); } +void recBGTZL() { recBranchSingleLikely(a64::le); } +void recBLTZL() { recBranchSingleLikely(a64::ge); } +void recBGEZL() { recBranchSingleLikely(a64::lt); } + +/********************************************************* + * Branch-and-link: BLTZAL, BGEZAL, BLTZALL, BGEZALL * + *********************************************************/ + +static void recBranchLink(bool ltz) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + _eeOnWriteReg(31, 0); + _eeFlushAllDirty(); + + _deleteEEreg(31, 0); + // Store return address directly to memory + armAsm->Mov(RXSCRATCH, (u64)(pc + 4)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.GPR.n.ra.UD[0])); + + if (GPR_IS_CONST1(_Rs_)) + { + bool taken = ltz ? (g_cpuConstRegs[_Rs_].SD[0] < 0) : (g_cpuConstRegs[_Rs_].SD[0] >= 0); + if (!taken) branchTo = pc + 4; + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + return; + } + + const bool swap = TrySwapDelaySlot(_Rs_, 0, 0, true); + recSetBranchL(ltz ? 1 : 0); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBranchLabel(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +static void recBranchLinkLikely(bool ltz) +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + _eeOnWriteReg(31, 0); + _eeFlushAllDirty(); + + _deleteEEreg(31, 0); + armAsm->Mov(RXSCRATCH, (u64)(pc + 4)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.GPR.n.ra.UD[0])); + + if (GPR_IS_CONST1(_Rs_)) + { + bool taken = ltz ? (g_cpuConstRegs[_Rs_].SD[0] < 0) : (g_cpuConstRegs[_Rs_].SD[0] >= 0); + if (taken) + { + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + } + else + SetBranchImm(pc + 4); + return; + } + + recSetBranchL(ltz ? 1 : 0); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBranchLabel(); + LoadBranchState(); + SetBranchImm(pc); +} + +void recBLTZAL() { recBranchLink(true); } +void recBGEZAL() { recBranchLink(false); } +void recBLTZALL() { recBranchLinkLikely(true); } +void recBGEZALL() { recBranchLinkLikely(false); } + +#endif // !FORCE_INTERP_BRANCH + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900Jump-arm64.cpp b/pcsx2/arm64/iR5900Jump-arm64.cpp new file mode 100644 index 0000000000..57c4e1b3cb --- /dev/null +++ b/pcsx2/arm64/iR5900Jump-arm64.cpp @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Jump Instruction Codegen + +#include "arm64/iR5900-arm64.h" +#include "Config.h" +#include "vtlb.h" +#include "common/Console.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_JUMP +REC_SYS(J); +REC_SYS(JAL); +REC_SYS(JR); +REC_SYS(JALR); +#else + +/********************************************************* + * Jump to target * + * Format: OP target * + *********************************************************/ + +//// J +void recJ() +{ + u32 newpc = (_InstrucTarget_ << 2) + (pc & 0xf0000000); + recompileNextInstruction(true, false); + if (EmuConfig.Gamefixes.GoemonTlbHack) + SetBranchImm(vtlb_V2P(newpc)); + else + SetBranchImm(newpc); +} + +//// JAL — jump and link (r31 = return address) +void recJAL() +{ + u32 newpc = (_InstrucTarget_ << 2) + (pc & 0xf0000000); + _deleteEEreg(31, 0); + if (EE_CONST_PROP) + { + GPR_SET_CONST(31); + g_cpuConstRegs[31].UL[0] = pc + 4; + g_cpuConstRegs[31].UL[1] = 0; + } + else + { + armAsm->Mov(RXSCRATCH, (u64)(pc + 4)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.GPR.r[31].UD[0])); + } + + recompileNextInstruction(true, false); + if (EmuConfig.Gamefixes.GoemonTlbHack) + SetBranchImm(vtlb_V2P(newpc)); + else + SetBranchImm(newpc); +} + +/********************************************************* + * Register jump * + * Format: OP rs, rd * + *********************************************************/ + +//// JR — jump to address in rs +void recJR() +{ + const u32 rs = _Rs_; + + // Save jump target to memory BEFORE delay slot, so it can't be lost + // if the delay slot evicts registers. + _deleteEEreg(rs, 1); // flush rs to memory + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[rs].UL[0]); + armStoreEERegPtr(RWSCRATCH, &cpuRegs.pcWriteback); + + recompileNextInstruction(true, false); + + SetBranchReg(); +} + +//// JALR — jump to rs, link in rd +void recJALR() +{ + const u32 rs = _Rs_; + const u32 rd = _Rd_; + const u32 newpc = pc + 4; + + // Save jump target to memory BEFORE delay slot. + // Must read rs before writing rd in case rd == rs. + _deleteEEreg(rs, 1); // flush rs to memory + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[rs].UL[0]); + armStoreEERegPtr(RWSCRATCH, &cpuRegs.pcWriteback); + + // Write link address to rd + if (rd) + { + _deleteEEreg(rd, 0); + if (EE_CONST_PROP) + { + GPR_SET_CONST(rd); + g_cpuConstRegs[rd].UL[0] = newpc; + g_cpuConstRegs[rd].UL[1] = 0; + } + else + { + armAsm->Mov(RXSCRATCH, (u64)newpc); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[rd].UD[0]); + } + } + + recompileNextInstruction(true, false); + + SetBranchReg(); +} + +#endif // !FORCE_INTERP_JUMP + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900LoadStore-arm64.cpp b/pcsx2/arm64/iR5900LoadStore-arm64.cpp new file mode 100644 index 0000000000..8c977ea48e --- /dev/null +++ b/pcsx2/arm64/iR5900LoadStore-arm64.cpp @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Load/Store — all recXXX implementations are in recVTLB-arm64.cpp +// This file is intentionally empty. diff --git a/pcsx2/arm64/iR5900Misc-arm64.cpp b/pcsx2/arm64/iR5900Misc-arm64.cpp new file mode 100644 index 0000000000..c2cad31248 --- /dev/null +++ b/pcsx2/arm64/iR5900Misc-arm64.cpp @@ -0,0 +1,550 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +#include "arm64/iR5900-arm64.h" +#include "common/Assertions.h" +#include "common/Console.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { + +// Forward declarations for native COP2 codegen (defined in iCOP2-arm64.cpp) +namespace OpcodeImpl { + // Transfer ops + void recCOP2_QMFC2(); + void recCOP2_QMTC2(); + void recCOP2_CFC2(); + // SIMPLE + void recCOP2_VMOVE(); + void recCOP2_VMR32(); + void recCOP2_VNOP(); + void recCOP2_VWAITQ(); + void recCOP2_VABS(); + // VEC_ARITH + void recCOP2_VADD(); + void recCOP2_VSUB(); + void recCOP2_VMUL(); + void recCOP2_VMAX(); + void recCOP2_VMINI(); + // BC variants + void recCOP2_VADDx(); void recCOP2_VADDy(); void recCOP2_VADDz(); void recCOP2_VADDw(); + void recCOP2_VSUBx(); void recCOP2_VSUBy(); void recCOP2_VSUBz(); void recCOP2_VSUBw(); + void recCOP2_VMULx(); void recCOP2_VMULy(); void recCOP2_VMULz(); void recCOP2_VMULw(); + void recCOP2_VMAXx(); void recCOP2_VMAXy(); void recCOP2_VMAXz(); void recCOP2_VMAXw(); + void recCOP2_VMINIx(); void recCOP2_VMINIy(); void recCOP2_VMINIz(); void recCOP2_VMINIw(); + void recCOP2_VMAXi(); void recCOP2_VMINIi(); + // Q/I variants + void recCOP2_VADDq(); void recCOP2_VSUBq(); void recCOP2_VMULq(); + void recCOP2_VADDi(); void recCOP2_VSUBi(); void recCOP2_VMULi(); + // MADD/MSUB + void recCOP2_VMADD(); void recCOP2_VMSUB(); + void recCOP2_VMADDx(); void recCOP2_VMADDy(); void recCOP2_VMADDz(); void recCOP2_VMADDw(); + void recCOP2_VMSUBx(); void recCOP2_VMSUBy(); void recCOP2_VMSUBz(); void recCOP2_VMSUBw(); + void recCOP2_VMADDq(); void recCOP2_VMSUBq(); + void recCOP2_VMADDi(); void recCOP2_VMSUBi(); + void recCOP2_VOPMSUB(); + // Accumulator + void recCOP2_VADDA(); void recCOP2_VSUBA(); void recCOP2_VMULA(); + void recCOP2_VADDAx(); void recCOP2_VADDAy(); void recCOP2_VADDAz(); void recCOP2_VADDAw(); + void recCOP2_VSUBAx(); void recCOP2_VSUBAy(); void recCOP2_VSUBAz(); void recCOP2_VSUBAw(); + void recCOP2_VMULAx(); void recCOP2_VMULAy(); void recCOP2_VMULAz(); void recCOP2_VMULAw(); + void recCOP2_VMULAq(); void recCOP2_VMULAi(); + void recCOP2_VADDAq(); void recCOP2_VSUBAq(); + void recCOP2_VADDAi(); void recCOP2_VSUBAi(); + void recCOP2_VMADDA(); void recCOP2_VMSUBA(); + void recCOP2_VMADDAx(); void recCOP2_VMADDAy(); void recCOP2_VMADDAz(); void recCOP2_VMADDAw(); + void recCOP2_VMSUBAx(); void recCOP2_VMSUBAy(); void recCOP2_VMSUBAz(); void recCOP2_VMSUBAw(); + void recCOP2_VMADDAq(); void recCOP2_VMSUBAq(); + void recCOP2_VMADDAi(); void recCOP2_VMSUBAi(); + void recCOP2_VOPMULA(); + // Conversion + void recCOP2_VITOF0(); void recCOP2_VITOF4(); void recCOP2_VITOF12(); void recCOP2_VITOF15(); + void recCOP2_VFTOI0(); void recCOP2_VFTOI4(); void recCOP2_VFTOI12(); void recCOP2_VFTOI15(); + // Integer ops + void recCOP2_VIADD(); void recCOP2_VISUB(); void recCOP2_VIADDI(); + void recCOP2_VIAND(); void recCOP2_VIOR(); + // CTC2 + void recCOP2_CTC2(); + // Division ops + void recCOP2_VDIV(); + void recCOP2_VSQRT(); + void recCOP2_VRSQRT(); + // Clip + void recCOP2_VCLIP(); +} // namespace OpcodeImpl + +// Branch helper — not implemented on ARM64. Callers (iCOP0/iFPU/COP2 macro +// paths) drive SaveBranchState/SetBranchImm directly instead. Fail loudly +// rather than silently no-op if a future port wires this in by mistake. +void recDoBranchImm(u32 branchTo, u32* jmpSkip, bool isLikely, bool swappedDelaySlot) +{ + pxFailRel("recDoBranchImm is not implemented on ARM64"); +} + +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +void recPREF() {} + +// SYSCALL and BREAK — flush state and call interpreter +void recSYSCALL() +{ + if (GPR_IS_CONST1(3)) + { + // FlushCache (0x64) / iFlushCache (0x68): the EE cache is not modelled, + // so account for the kernel handler cycles inline and skip the call. + // Cycle count from github.com/F0bes/flushcache-cycles. Mirrors x86 recSYSCALL. + // + // This skip leaves v0/v1/at/t0/t1 and EPC at their pre-syscall values, + // whereas the interpreter actually raises cpuException(0x20) and runs the + // BIOS 0x80000180 trampoline, which clobbers them. That JIT-vs-interp + // divergence is REAL but ABI-benign: FlushCache is a syscall, so under + // the MIPS calling convention those are all caller-saved/temporary regs + // (plus EPC, which user code never reads) — correct code never depends on + // them surviving. Upstream PCSX2-x86 ships this skip as a correct, faster + // optimization. + const u8 syscallNum = g_cpuConstRegs[3].UC[0]; + if (syscallNum == 0x64 || syscallNum == 0x68) + { + s_nBlockCycles += 5650; + return; + } + } + recBranchCall(Interp::SYSCALL); +} + +void recBREAK() +{ + recBranchCall(Interp::BREAK); +} + +// ===================================================================================================== +// COP2 (VU0 macro mode) — dispatch table with per-sub-opcode interpreter fallback +// Mirrors the x86 dispatch structure: recCOP2 → recCOP2t[_Rs_] → SPEC1/SPEC2 +// ===================================================================================================== + +// COP2 macro-mode mVU-reuse wrapper. Drives the existing microVU emitter +// (mVU_ in microVU_Lower-arm64.inl) via the mVUmacroEmit_ adapter +// declared in iR5900-arm64.h. Mirrors x86 REC_COP2_mVU0 (microVU_Macro.inl:122). +// Mode bits per x86 microVU_Macro.inl:158-165: +// 0x01 reads Q reg / 0x02 writes Q reg / 0x04 requires analysis pass +// 0x08 writes CLIP / 0x10 writes status/mac / 0x100 requires x86 regs. +#define REC_COP2_mVU0_ARM64(name, mode) \ + static void recV##name() \ + { \ + setupMacroOp_arm64(mode); \ + mVUmacroEmit_##name(mode); \ + endMacroOp_arm64(mode); \ + } + +// Transfer ops — native codegen for QMFC2/QMTC2/CFC2, CTC2 stays interpreter +static void recVQMFC2() { recCOP2_QMFC2(); } +static void recVQMTC2() { recCOP2_QMTC2(); } +static void recVCFC2() { recCOP2_CFC2(); } +static void recVCTC2() { recCOP2_CTC2(); } + +// Branch ops — native COP2 condition branch. CP2COND = bit 8 of +// VU0.VI[REG_VPU_STAT] (COP2.cpp:11). Mirrors x86 _setupBranchTest +// (microVU_Macro.inl) and the recBC1F FPU-branch shape: a lightweight +// _eeFlushAllDirty + a single Tbz/Tbnz on the flag bit, then the standard EE +// branch-imm machinery — avoiding FLUSH_INTERPRETER + C-call + dispatcher +// round-trip overhead. +static a64::Label* s_pBC2Label = nullptr; + +static void recSetBranchCOP2(bool branchOnTrue) +{ + _eeFlushAllDirty(); + armAsm->Ldr(RWSCRATCH, armVU0Mem(&VU0.VI[REG_VPU_STAT])); + + // The forward branch skips the taken path: BC2T (branchOnTrue) is taken when + // CP2COND is set → skip when clear → Tbz; BC2F is taken when clear → skip + // when set → Tbnz. Matches x86 JZ32/JNZ32 in recBC2T/recBC2F. + s_pBC2Label = new a64::Label(); + if (branchOnTrue) + armAsm->Tbz(RWSCRATCH, 8, s_pBC2Label); + else + armAsm->Tbnz(RWSCRATCH, 8, s_pBC2Label); +} + +static void recBindBC2Label() +{ + armAsm->Bind(s_pBC2Label); + delete s_pBC2Label; + s_pBC2Label = nullptr; +} + +// Non-likely (BC2F/BC2T): attempt a delay-slot swap (allow_loadstore=false, +// matching x86 _setupBranchTest's TrySwapDelaySlot(0,0,0,false)). +static void recVBC2F() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + const bool swap = TrySwapDelaySlot(0, 0, 0, false); + recSetBranchCOP2(false); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBC2Label(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +static void recVBC2T() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + const bool swap = TrySwapDelaySlot(0, 0, 0, false); + recSetBranchCOP2(true); + + if (!swap) + { + SaveBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(branchTo); + + recBindBC2Label(); + + if (!swap) + { + pc -= 4; + LoadBranchState(); + recompileNextInstruction(true, false); + } + SetBranchImm(pc); +} + +// Likely (BC2FL/BC2TL): delay slot squashed when not taken; no swap, matching +// the x86 isLikely path (and the interp's `else { cpuRegs.pc += 4; }`). +static void recVBC2FL() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + recSetBranchCOP2(false); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBC2Label(); + LoadBranchState(); + SetBranchImm(pc); +} + +static void recVBC2TL() +{ + u32 branchTo = ((s32)_Imm_ * 4) + pc; + + recSetBranchCOP2(true); + + SaveBranchState(); + recompileNextInstruction(true, false); + SetBranchImm(branchTo); + + recBindBC2Label(); + LoadBranchState(); + SetBranchImm(pc); +} + +// Upper instructions (SPEC1) — native NEON codegen +// BC variants: VF[fd] = VF[fs] OP VF[ft].bc +static void recVADDx() { recCOP2_VADDx(); } static void recVADDy() { recCOP2_VADDy(); } +static void recVADDz() { recCOP2_VADDz(); } static void recVADDw() { recCOP2_VADDw(); } +static void recVSUBx() { recCOP2_VSUBx(); } static void recVSUBy() { recCOP2_VSUBy(); } +static void recVSUBz() { recCOP2_VSUBz(); } static void recVSUBw() { recCOP2_VSUBw(); } +static void recVMADDx() { recCOP2_VMADDx(); } static void recVMADDy() { recCOP2_VMADDy(); } +static void recVMADDz() { recCOP2_VMADDz(); } static void recVMADDw() { recCOP2_VMADDw(); } +static void recVMSUBx() { recCOP2_VMSUBx(); } static void recVMSUBy() { recCOP2_VMSUBy(); } +static void recVMSUBz() { recCOP2_VMSUBz(); } static void recVMSUBw() { recCOP2_VMSUBw(); } +static void recVMAXx() { recCOP2_VMAXx(); } static void recVMAXy() { recCOP2_VMAXy(); } +static void recVMAXz() { recCOP2_VMAXz(); } static void recVMAXw() { recCOP2_VMAXw(); } +static void recVMINIx() { recCOP2_VMINIx(); } static void recVMINIy() { recCOP2_VMINIy(); } +static void recVMINIz() { recCOP2_VMINIz(); } static void recVMINIw() { recCOP2_VMINIw(); } +static void recVMULx() { recCOP2_VMULx(); } static void recVMULy() { recCOP2_VMULy(); } +static void recVMULz() { recCOP2_VMULz(); } static void recVMULw() { recCOP2_VMULw(); } +static void recVMULq() { recCOP2_VMULq(); } static void recVMAXi() { recCOP2_VMAXi(); } +static void recVMULi() { recCOP2_VMULi(); } static void recVMINIi() { recCOP2_VMINIi(); } +static void recVADDq() { recCOP2_VADDq(); } static void recVMADDq() { recCOP2_VMADDq(); } +static void recVADDi() { recCOP2_VADDi(); } static void recVMADDi() { recCOP2_VMADDi(); } +static void recVSUBq() { recCOP2_VSUBq(); } static void recVMSUBq() { recCOP2_VMSUBq(); } +static void recVSUBi() { recCOP2_VSUBi(); } static void recVMSUBi() { recCOP2_VMSUBi(); } +static void recVADD() { recCOP2_VADD(); } static void recVMADD() { recCOP2_VMADD(); } +static void recVMUL() { recCOP2_VMUL(); } static void recVMAX() { recCOP2_VMAX(); } +static void recVSUB() { recCOP2_VSUB(); } static void recVMSUB() { recCOP2_VMSUB(); } +static void recVOPMSUB(){ recCOP2_VOPMSUB(); } static void recVMINI() { recCOP2_VMINI(); } +// Integer ops — native +static void recVIADD() { recCOP2_VIADD(); } static void recVISUB() { recCOP2_VISUB(); } +static void recVIADDI() { recCOP2_VIADDI(); } +static void recVIAND() { recCOP2_VIAND(); } static void recVIOR() { recCOP2_VIOR(); } +// CALLMS/CALLMSR kick off a VU0 microprogram via the interpreter — they are +// NOT EE branches (x86 iR5900Analysis case 56/57 just `break;`) so they +// must NOT exit the recompiled block the way recBranchCall does. Mirror +// x86's INTERPRETATE_COP2_FUNC(CALLMS) (microVU_Macro.inl:142): full +// FLUSH_INTERPRETER flush (so cpuRegs.code is current — VCALLMS reads +// the start PC from `(cpuRegs.code >> 6) & 0x7FFF`), apply pending block +// cycles, call the interpreter (which itself runs _vu0FinishMicro + +// vu0ExecMicro), then reload RECCYCLE in case the interp advanced +// cpuRegs.cycle. Block execution continues at the next opcode. +// Using iFlushCall(FLUSH_INTERPRETER) inline avoids the g_branch=2 block +// exit that recBranchCall would trigger on every CALLMS; the flush cost +// is the same, with no dispatcher round-trip. +static void recVCallmsImpl(void (*func)()) +{ + iFlushCall(FLUSH_INTERPRETER); + + u32 cycles = scaleblockcycles_clear(); + if (cycles != 0) + armAsm->Add(RECCYCLE, RECCYCLE, cycles); + + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armEmitCall((void*)func); + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); +} + +static void recVCALLMS() { recVCallmsImpl(VCALLMS); } +static void recVCALLMSR() { recVCallmsImpl(VCALLMSR); } + +// Lower instructions (SPEC2) — native NEON codegen for accumulator/conversion/simple ops +// Accumulator BC variants +static void recVADDAx() { recCOP2_VADDAx(); } static void recVADDAy() { recCOP2_VADDAy(); } +static void recVADDAz() { recCOP2_VADDAz(); } static void recVADDAw() { recCOP2_VADDAw(); } +static void recVSUBAx() { recCOP2_VSUBAx(); } static void recVSUBAy() { recCOP2_VSUBAy(); } +static void recVSUBAz() { recCOP2_VSUBAz(); } static void recVSUBAw() { recCOP2_VSUBAw(); } +static void recVMADDAx(){ recCOP2_VMADDAx(); } static void recVMADDAy(){ recCOP2_VMADDAy(); } +static void recVMADDAz(){ recCOP2_VMADDAz(); } static void recVMADDAw(){ recCOP2_VMADDAw(); } +static void recVMSUBAx(){ recCOP2_VMSUBAx(); } static void recVMSUBAy(){ recCOP2_VMSUBAy(); } +static void recVMSUBAz(){ recCOP2_VMSUBAz(); } static void recVMSUBAw(){ recCOP2_VMSUBAw(); } +// Conversions — native NEON +static void recVITOF0() { recCOP2_VITOF0(); } static void recVITOF4() { recCOP2_VITOF4(); } +static void recVITOF12() { recCOP2_VITOF12(); } static void recVITOF15() { recCOP2_VITOF15(); } +static void recVFTOI0() { recCOP2_VFTOI0(); } static void recVFTOI4() { recCOP2_VFTOI4(); } +static void recVFTOI12() { recCOP2_VFTOI12(); } static void recVFTOI15() { recCOP2_VFTOI15(); } +// Accumulator MULAx/y/z/w +static void recVMULAx() { recCOP2_VMULAx(); } static void recVMULAy() { recCOP2_VMULAy(); } +static void recVMULAz() { recCOP2_VMULAz(); } static void recVMULAw() { recCOP2_VMULAw(); } +static void recVMULAq() { recCOP2_VMULAq(); } static void recVABS() { recCOP2_VABS(); } +static void recVMULAi() { recCOP2_VMULAi(); } +// CLIP — still interpreter fallback (complex flag logic) +static void recVCLIP() { recCOP2_VCLIP(); } +// Accumulator Q/I variants +static void recVADDAq() { recCOP2_VADDAq(); } static void recVMADDAq(){ recCOP2_VMADDAq(); } +static void recVADDAi() { recCOP2_VADDAi(); } static void recVMADDAi(){ recCOP2_VMADDAi(); } +static void recVSUBAq() { recCOP2_VSUBAq(); } static void recVMSUBAq(){ recCOP2_VMSUBAq(); } +static void recVSUBAi() { recCOP2_VSUBAi(); } static void recVMSUBAi(){ recCOP2_VMSUBAi(); } +// Accumulator full-vector variants +static void recVADDA() { recCOP2_VADDA(); } static void recVMADDA() { recCOP2_VMADDA(); } +static void recVMULA() { recCOP2_VMULA(); } +static void recVSUBA() { recCOP2_VSUBA(); } static void recVMSUBA() { recCOP2_VMSUBA(); } +static void recVOPMULA(){ recCOP2_VOPMULA(); } static void recVNOP() { recCOP2_VNOP(); } +// Simple data movement — native +static void recVMOVE() { recCOP2_VMOVE(); } static void recVMR32() { recCOP2_VMR32(); } +// Load/store — full group native via mVU emit (mode bits from x86 microVU_Macro.inl:276-279). +REC_COP2_mVU0_ARM64(LQI, 0x104); REC_COP2_mVU0_ARM64(SQI, 0x100); +REC_COP2_mVU0_ARM64(LQD, 0x104); REC_COP2_mVU0_ARM64(SQD, 0x100); +// Division ops — native +static void recVDIV() { recCOP2_VDIV(); } +static void recVSQRT() { recCOP2_VSQRT(); } +static void recVRSQRT(){ recCOP2_VRSQRT(); } +static void recVWAITQ() { recCOP2_VWAITQ(); } +REC_COP2_mVU0_ARM64(MTIR, 0x104); REC_COP2_mVU0_ARM64(MFIR, 0x104); +REC_COP2_mVU0_ARM64(ILWR, 0x104); REC_COP2_mVU0_ARM64(ISWR, 0x100); +REC_COP2_mVU0_ARM64(RNEXT, 0x104); REC_COP2_mVU0_ARM64(RGET, 0x104); +REC_COP2_mVU0_ARM64(RINIT, 0x100); REC_COP2_mVU0_ARM64(RXOR, 0x100); + +static void rec_C2UNK() { Console.Error("EE: Unrecognized COP2 opcode %08X", cpuRegs.code); } + +// Dispatch tables — mirror x86 structure +static void recCOP2_BC2(); +static void recCOP2_SPEC1(); +static void recCOP2_SPEC2(); + +static void (*recCOP2t[32])() = { + rec_C2UNK, recVQMFC2, recVCFC2, rec_C2UNK, rec_C2UNK, recVQMTC2, recVCTC2, rec_C2UNK, + recCOP2_BC2, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, + recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, recCOP2_SPEC1, +}; + +static void (*recCOP2_BC2t[32])() = { + recVBC2F, recVBC2T, recVBC2FL, recVBC2TL, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, +}; + +static void (*recCOP2SPECIAL1t[64])() = { + recVADDx, recVADDy, recVADDz, recVADDw, recVSUBx, recVSUBy, recVSUBz, recVSUBw, + recVMADDx, recVMADDy, recVMADDz, recVMADDw, recVMSUBx, recVMSUBy, recVMSUBz, recVMSUBw, + recVMAXx, recVMAXy, recVMAXz, recVMAXw, recVMINIx, recVMINIy, recVMINIz, recVMINIw, + recVMULx, recVMULy, recVMULz, recVMULw, recVMULq, recVMAXi, recVMULi, recVMINIi, + recVADDq, recVMADDq, recVADDi, recVMADDi, recVSUBq, recVMSUBq, recVSUBi, recVMSUBi, + recVADD, recVMADD, recVMUL, recVMAX, recVSUB, recVMSUB, recVOPMSUB, recVMINI, + recVIADD, recVISUB, recVIADDI, rec_C2UNK, recVIAND, recVIOR, rec_C2UNK, rec_C2UNK, + recVCALLMS, recVCALLMSR,rec_C2UNK, rec_C2UNK, recCOP2_SPEC2, recCOP2_SPEC2, recCOP2_SPEC2, recCOP2_SPEC2, +}; + +static void (*recCOP2SPECIAL2t[128])() = { + recVADDAx, recVADDAy, recVADDAz, recVADDAw, recVSUBAx, recVSUBAy, recVSUBAz, recVSUBAw, + recVMADDAx,recVMADDAy, recVMADDAz, recVMADDAw, recVMSUBAx, recVMSUBAy, recVMSUBAz, recVMSUBAw, + recVITOF0, recVITOF4, recVITOF12, recVITOF15, recVFTOI0, recVFTOI4, recVFTOI12, recVFTOI15, + recVMULAx, recVMULAy, recVMULAz, recVMULAw, recVMULAq, recVABS, recVMULAi, recVCLIP, + recVADDAq, recVMADDAq,recVADDAi, recVMADDAi, recVSUBAq, recVMSUBAq, recVSUBAi, recVMSUBAi, + recVADDA, recVMADDA, recVMULA, rec_C2UNK, recVSUBA, recVMSUBA, recVOPMULA, recVNOP, + recVMOVE, recVMR32, rec_C2UNK, rec_C2UNK, recVLQI, recVSQI, recVLQD, recVSQD, + recVDIV, recVSQRT, recVRSQRT, recVWAITQ, recVMTIR, recVMFIR, recVILWR, recVISWR, + recVRNEXT, recVRGET, recVRINIT, recVRXOR, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, + rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, rec_C2UNK, +}; + +static void recCOP2_BC2() { recCOP2_BC2t[_Rt_](); } +static void recCOP2_SPEC1() { recCOP2SPECIAL1t[cpuRegs.code & 0x3f](); } +static void recCOP2_SPEC2() { recCOP2SPECIAL2t[(cpuRegs.code & 0x3) | ((cpuRegs.code >> 4) & 0x7c)](); } + +void recCOP2() +{ +#ifdef FORCE_INTERP_COP2 + // Use interpreter for all COP2 — but branches need recBranchCall + if (_Rs_ == 8) // BC2 branch instructions + recBranchCall(Interp::COP2); + else + recCall(Interp::COP2); +#else + recCOP2t[_Rs_](); +#endif +} + +void recSYNC() {} + +// MFSA — rd = sa (shift amount register) +void recMFSA() +{ + if (!_Rd_) return; + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.sa); + armAsm->Mov(RXSCRATCH, RWSCRATCH); // zero-extend to 64 + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +// MTSA — sa = rs +void recMTSA() +{ + if (GPR_IS_CONST1(_Rs_)) + { + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rs_].UL[0]); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.sa)); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.sa)); + } +} + +// MTSAB — sa = (rs[3:0] ^ imm[3:0]) +void recMTSAB() +{ + if (GPR_IS_CONST1(_Rs_)) + { + u32 val = (g_cpuConstRegs[_Rs_].UL[0] & 0xF) ^ (_Imm_ & 0xF); + armAsm->Mov(RWSCRATCH, val); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.sa)); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->And(RWSCRATCH, RWSCRATCH, 0xF); + armAsm->Eor(RWSCRATCH, RWSCRATCH, _Imm_ & 0xF); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.sa)); + } +} + +// MTSAH — sa = ((rs[2:0] ^ imm[2:0]) << 1) +void recMTSAH() +{ + if (GPR_IS_CONST1(_Rs_)) + { + u32 val = ((g_cpuConstRegs[_Rs_].UL[0] & 0x7) ^ (_Imm_ & 0x7)) << 1; + armAsm->Mov(RWSCRATCH, val); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.sa)); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rs_].UL[0]); + armAsm->Eor(RWSCRATCH, RWSCRATCH, _Imm_ & 0x7); + // ubfiz w, w, #1, #3 extracts bits[2:0] and places them at bit 1 + armAsm->Ubfiz(RWSCRATCH, RWSCRATCH, 1, 3); + armAsm->Str(RWSCRATCH, armCpuRegMem(&cpuRegs.sa)); + } +} + +void recNULL() +{ + Console.Error("EE: Unimplemented op %x", cpuRegs.code); +} + +void recUnknown() +{ + Console.Error("EE: Unrecognized op %x", cpuRegs.code); +} + +void recMMI_Unknown() +{ + Console.Error("EE: Unrecognized MMI op %x", cpuRegs.code); +} + +void recCOP0_Unknown() +{ + Console.Error("EE: Unrecognized COP0 op %x", cpuRegs.code); +} + +void recCOP1_Unknown() +{ + Console.Error("EE: Unrecognized FPU/COP1 op %x", cpuRegs.code); +} + +void recCACHE() {} + +REC_SYS(TGE); +REC_SYS(TGEU); +REC_SYS(TLT); +REC_SYS(TLTU); +REC_SYS(TEQ); +REC_SYS(TNE); +REC_SYS(TGEI); +REC_SYS(TGEIU); +REC_SYS(TLTI); +REC_SYS(TLTIU); +REC_SYS(TEQI); +REC_SYS(TNEI); + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 + +// recBackpropBSC is provided by the shared x86/iR5900Analysis.cpp +// (compiled for ARM64 via ARCH_ARM64 conditional include). + diff --git a/pcsx2/arm64/iR5900Move-arm64.cpp b/pcsx2/arm64/iR5900Move-arm64.cpp new file mode 100644 index 0000000000..786570f678 --- /dev/null +++ b/pcsx2/arm64/iR5900Move-arm64.cpp @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Move Instruction Codegen — memory-based +// All operands via cpuRegs memory. + +#include "arm64/iR5900-arm64.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_MOVE +REC_FUNC(LUI); +REC_FUNC(MFLO); +REC_FUNC(MFHI); +REC_FUNC(MTLO); +REC_FUNC(MTHI); +REC_FUNC(MFLO1); +REC_FUNC(MFHI1); +REC_FUNC(MTLO1); +REC_FUNC(MTHI1); +REC_FUNC(MOVZ); +REC_FUNC(MOVN); +#else + +//// LUI — rt = imm16 << 16 (sign-extended to 64 bits) +void recLUI() +{ + if (!_Rt_) return; + + _deleteEEreg(_Rt_, 0); + + if (EE_CONST_PROP) + { + g_cpuConstRegs[_Rt_].SD[0] = s64(s32((u32)_ImmU_ << 16)); + GPR_SET_CONST(_Rt_); + } + else + { + GPR_DEL_CONST(_Rt_); + armAsm->Mov(RXSCRATCH, s64(s32((u32)_ImmU_ << 16))); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); + } +} + +//// MFLO / MFHI — rd = LO/HI (memory to memory) +void recMFLO() +{ + if (!_Rd_) return; + + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +void recMFHI() +{ + if (!_Rd_) return; + + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.HI.UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +//// MTLO / MTHI — LO/HI = rs (memory to memory) +void recMTLO() +{ + if (GPR_IS_CONST1(_Rs_)) + { + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rs_].SD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[0]); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[0]); + } +} + +void recMTHI() +{ + if (GPR_IS_CONST1(_Rs_)) + { + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rs_].SD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.HI.UD[0]); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.HI.UD[0]); + } +} + +//// MFLO1/MFHI1 — rd = LO1/HI1 (upper 64 bits of LO/HI) +void recMFLO1() +{ + if (!_Rd_) return; + + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[1]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +void recMFHI1() +{ + if (!_Rd_) return; + + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.HI.UD[1]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +//// MTLO1/MTHI1 — LO1/HI1 = rs +void recMTLO1() +{ + if (GPR_IS_CONST1(_Rs_)) + { + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rs_].SD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[1]); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[1]); + } +} + +void recMTHI1() +{ + if (GPR_IS_CONST1(_Rs_)) + { + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rs_].SD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.HI.UD[1]); + } + else + { + _deleteEEreg(_Rs_, 1); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.HI.UD[1]); + } +} + +//// MOVZ — if (rt == 0) then rd = rs +// Memory-based: all loads/stores via cpuRegs + +static void recMOVZtemp_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0]; +} + +static void recMOVZtemp_consts(int info) +{ + // S is const — load T from memory, compare, conditionally store + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); + armAsm->Cmp(RXSCRATCH, 0); + + armAsm->Mov(RXARG1, g_cpuConstRegs[_Rs_].SD[0]); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); + armAsm->Csel(RXSCRATCH, RXARG1, RXSCRATCH, a64::eq); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +static void recMOVZtemp_constt(int info) +{ + // T is constant and zero (checked in wrapper) — unconditional move + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +static void recMOVZtemp_(int info) +{ + // Load T for comparison + armLoadEERegPtr(RXARG1, &cpuRegs.GPR.r[_Rt_].UD[0]); + armAsm->Cmp(RXARG1, 0); + + // Load S + armLoadEERegPtr(RXARG1, &cpuRegs.GPR.r[_Rs_].UD[0]); + + // Load current D, conditional select, store back + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); + armAsm->Csel(RXSCRATCH, RXARG1, RXSCRATCH, a64::eq); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +static EERECOMPILE_CODERC0_MEM(MOVZtemp, XMMINFO_READS | XMMINFO_READT | XMMINFO_READD | XMMINFO_WRITED | XMMINFO_NORENAME); + +void recMOVZ() +{ + if (_Rs_ == _Rd_) + return; + + if (GPR_IS_CONST1(_Rt_) && g_cpuConstRegs[_Rt_].UD[0] != 0) + return; + + recMOVZtemp(); +} + +//// MOVN — if (rt != 0) then rd = rs + +static void recMOVNtemp_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rs_].UD[0]; +} + +static void recMOVNtemp_consts(int info) +{ + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); + armAsm->Cmp(RXSCRATCH, 0); + + armAsm->Mov(RXARG1, g_cpuConstRegs[_Rs_].SD[0]); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); + armAsm->Csel(RXSCRATCH, RXARG1, RXSCRATCH, a64::ne); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +static void recMOVNtemp_constt(int info) +{ + // T is constant and non-zero (checked in wrapper) — unconditional move + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rs_].UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +static void recMOVNtemp_(int info) +{ + armLoadEERegPtr(RXARG1, &cpuRegs.GPR.r[_Rt_].UD[0]); + armAsm->Cmp(RXARG1, 0); + + armLoadEERegPtr(RXARG1, &cpuRegs.GPR.r[_Rs_].UD[0]); + + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); + armAsm->Csel(RXSCRATCH, RXARG1, RXSCRATCH, a64::ne); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +static EERECOMPILE_CODERC0_MEM(MOVNtemp, XMMINFO_READS | XMMINFO_READT | XMMINFO_READD | XMMINFO_WRITED | XMMINFO_NORENAME); + +void recMOVN() +{ + if (_Rs_ == _Rd_) + return; + + if (GPR_IS_CONST1(_Rt_) && g_cpuConstRegs[_Rt_].UD[0] == 0) + return; + + recMOVNtemp(); +} + +#endif // !FORCE_INTERP_MOVE + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900MultDiv-arm64.cpp b/pcsx2/arm64/iR5900MultDiv-arm64.cpp new file mode 100644 index 0000000000..c3374a7d4a --- /dev/null +++ b/pcsx2/arm64/iR5900MultDiv-arm64.cpp @@ -0,0 +1,594 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Multiply/Divide Instruction Codegen — memory-based +// MULT/DIV write to HI:LO registers, optionally Rd. +// ARM64 has native SMULL/UMULL and SDIV/UDIV. +// All operands via cpuRegs memory. + +#include "arm64/iR5900-arm64.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_MULTDIV +REC_FUNC(MULT); +REC_FUNC(MULTU); +REC_FUNC(DIV); +REC_FUNC(DIVU); +#else + +// Load Rs/Rt lower 32 bits from memory (or const) +static void loadRs32() +{ + if (GPR_IS_CONST1(_Rs_)) + armAsm->Mov(a64::w1, g_cpuConstRegs[_Rs_].UL[0]); + else + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rs_].UL[0]); +} + +static void loadRt32() +{ + if (GPR_IS_CONST1(_Rt_)) + armAsm->Mov(a64::w2, g_cpuConstRegs[_Rt_].UL[0]); + else + armLoadEERegPtr(a64::w2, &cpuRegs.GPR.r[_Rt_].UL[0]); +} + +// Write LO and HI from 64-bit result in x0 +// lo = lower 32, hi = upper 32, both sign-extended to 64 bits +static void recWritebackHILO(bool upper) +{ + armAsm->Sxtw(RXSCRATCH, a64::w0); + armAsm->Str(RXSCRATCH, armCpuRegMem(upper ? &cpuRegs.LO.UD[1] : &cpuRegs.LO.UD[0])); + + armAsm->Asr(a64::x0, a64::x0, 32); + armAsm->Sxtw(RXSCRATCH, a64::w0); + armAsm->Str(RXSCRATCH, armCpuRegMem(upper ? &cpuRegs.HI.UD[1] : &cpuRegs.HI.UD[0])); +} + +// Write Rd from LO (memory-based — no register allocation) +static void recWritebackRd() +{ + if (!_Rd_) return; + + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[0]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +//// MULT — signed 32-bit multiply, result in HI:LO, optionally Rd +void recMULT() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + s64 result = (s64)(s32)g_cpuConstRegs[_Rs_].UL[0] * (s64)(s32)g_cpuConstRegs[_Rt_].UL[0]; + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)result); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[0])); + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)(result >> 32)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[0])); + + if (_Rd_) + { + _deleteEEreg(_Rd_, 0); + g_cpuConstRegs[_Rd_].SD[0] = (s32)(u32)result; + GPR_SET_CONST(_Rd_); + } + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Smull(a64::x0, a64::w1, a64::w2); + + recWritebackHILO(false); + recWritebackRd(); +} + +//// MULTU — unsigned 32-bit multiply +void recMULTU() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + u64 result = (u64)g_cpuConstRegs[_Rs_].UL[0] * (u64)g_cpuConstRegs[_Rt_].UL[0]; + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)result); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[0])); + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)(result >> 32)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[0])); + + if (_Rd_) + { + _deleteEEreg(_Rd_, 0); + g_cpuConstRegs[_Rd_].SD[0] = (s32)(u32)result; + GPR_SET_CONST(_Rd_); + } + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Umull(a64::x0, a64::w1, a64::w2); + + recWritebackHILO(false); + recWritebackRd(); +} + +//// DIV — signed 32-bit divide. LO = quotient, HI = remainder. +// PS2 div-by-zero: LO = (rs >= 0 ? -1 : 1), HI = rs (sign-extended into 64-bit +// HI/LO). Matches the interpreter and the PS2 hardware spec. +void recDIV() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + s32 rs = g_cpuConstRegs[_Rs_].SL[0]; + s32 rt = g_cpuConstRegs[_Rt_].SL[0]; + + s32 lo, hi; + if (rt == 0) + { + lo = (rs >= 0) ? -1 : 1; + hi = rs; + } + else if (rs == (s32)0x80000000 && rt == -1) + { + lo = (s32)0x80000000; + hi = 0; + } + else + { + lo = rs / rt; + hi = rs % rt; + } + + armAsm->Mov(RXSCRATCH, (s64)lo); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[0])); + + armAsm->Mov(RXSCRATCH, (s64)hi); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[0])); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + // Branch on rt == 0 → div-by-zero handler. + a64::Label divByZero; + a64::Label done; + armAsm->Cbz(a64::w2, &divByZero); + + // Normal path: SDIV w0, w1, w2; MSUB w3 = w1 - w0 * w2 (remainder). + armAsm->Sdiv(a64::w0, a64::w1, a64::w2); + armAsm->Msub(a64::w3, a64::w0, a64::w2, a64::w1); + armAsm->B(&done); + + // Div-by-zero: w0 = (rs >= 0 ? -1 : 1), w3 = rs. + // Cneg w0, w0, lt: if rs < 0, w0 = -(-1) = 1; else w0 = -1. + armAsm->Bind(&divByZero); + armAsm->Mov(a64::w0, -1); + armAsm->Cmp(a64::w1, 0); + armAsm->Cneg(a64::w0, a64::w0, a64::lt); + armAsm->Mov(a64::w3, a64::w1); // HI = rs + + armAsm->Bind(&done); + + // Store LO = sign_extend(quotient or -1/1) + armAsm->Sxtw(RXSCRATCH, a64::w0); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[0])); + + // Store HI = sign_extend(remainder or rs) + armAsm->Sxtw(RXSCRATCH, a64::w3); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[0])); +} + +//// DIVU — unsigned 32-bit divide. PS2 div-by-zero: LO = -1 (0xffffffff +//// sign-extended), HI = rs. +void recDIVU() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + u32 rs = g_cpuConstRegs[_Rs_].UL[0]; + u32 rt = g_cpuConstRegs[_Rt_].UL[0]; + + s32 lo, hi; + if (rt == 0) + { + lo = -1; + hi = (s32)rs; + } + else + { + lo = (s32)(rs / rt); + hi = (s32)(rs % rt); + } + + armAsm->Mov(RXSCRATCH, (s64)lo); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[0])); + + armAsm->Mov(RXSCRATCH, (s64)hi); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[0])); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + a64::Label divByZero; + a64::Label done; + armAsm->Cbz(a64::w2, &divByZero); + + // Normal path: UDIV w0, w1, w2; MSUB w1 = w1 - w0 * w2 (remainder in-place + // over Rs scratch; Msub permits rd==ra). On div-by-zero w1 still holds Rs, + // so the slow path doesn't need a separate Mov w3, w1. + armAsm->Udiv(a64::w0, a64::w1, a64::w2); + armAsm->Msub(a64::w1, a64::w0, a64::w2, a64::w1); + armAsm->B(&done); + + // Div-by-zero: w0 = -1; w1 already holds Rs (HI). + armAsm->Bind(&divByZero); + armAsm->Mov(a64::w0, -1); + + armAsm->Bind(&done); + + // Store LO = sign_extend(quotient or -1) + armAsm->Sxtw(RXSCRATCH, a64::w0); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[0])); + + // Store HI = sign_extend(remainder or rs) + armAsm->Sxtw(RXSCRATCH, a64::w1); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[0])); +} + +#endif // !FORCE_INTERP_MULTDIV + +// Write Rd from LO1 (pipeline 1) +static void recWritebackRd1() +{ + if (!_Rd_) return; + + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + armLoadEERegPtr(RXSCRATCH, &cpuRegs.LO.UD[1]); + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +//// MULT1 — signed 32-bit multiply, pipeline 1 (HI1:LO1) +void recMULT1() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + s64 result = (s64)(s32)g_cpuConstRegs[_Rs_].UL[0] * (s64)(s32)g_cpuConstRegs[_Rt_].UL[0]; + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)result); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[1])); + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)(result >> 32)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[1])); + + if (_Rd_) + { + _deleteEEreg(_Rd_, 0); + g_cpuConstRegs[_Rd_].SD[0] = (s32)(u32)result; + GPR_SET_CONST(_Rd_); + } + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Smull(a64::x0, a64::w1, a64::w2); + + recWritebackHILO(true); + recWritebackRd1(); +} + +//// MULTU1 — unsigned 32-bit multiply, pipeline 1 +void recMULTU1() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + u64 result = (u64)g_cpuConstRegs[_Rs_].UL[0] * (u64)g_cpuConstRegs[_Rt_].UL[0]; + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)result); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[1])); + + armAsm->Mov(RXSCRATCH, (s64)(s32)(u32)(result >> 32)); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[1])); + + if (_Rd_) + { + _deleteEEreg(_Rd_, 0); + g_cpuConstRegs[_Rd_].SD[0] = (s32)(u32)result; + GPR_SET_CONST(_Rd_); + } + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Umull(a64::x0, a64::w1, a64::w2); + + recWritebackHILO(true); + recWritebackRd1(); +} + +//// DIV1 — signed 32-bit divide, pipeline 1. Same div-by-zero spec as DIV. +void recDIV1() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + s32 rs = g_cpuConstRegs[_Rs_].SL[0]; + s32 rt = g_cpuConstRegs[_Rt_].SL[0]; + + s32 lo, hi; + if (rt == 0) + { + lo = (rs >= 0) ? -1 : 1; + hi = rs; + } + else if (rs == (s32)0x80000000 && rt == -1) + { + lo = (s32)0x80000000; + hi = 0; + } + else + { + lo = rs / rt; + hi = rs % rt; + } + + armAsm->Mov(RXSCRATCH, (s64)lo); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[1])); + + armAsm->Mov(RXSCRATCH, (s64)hi); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[1])); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + a64::Label divByZero; + a64::Label done; + armAsm->Cbz(a64::w2, &divByZero); + + armAsm->Sdiv(a64::w0, a64::w1, a64::w2); + armAsm->Msub(a64::w3, a64::w0, a64::w2, a64::w1); + armAsm->B(&done); + + // Div-by-zero: w0 = (rs >= 0 ? -1 : 1), w3 = rs. See recDIV for Cneg rationale. + armAsm->Bind(&divByZero); + armAsm->Mov(a64::w0, -1); + armAsm->Cmp(a64::w1, 0); + armAsm->Cneg(a64::w0, a64::w0, a64::lt); + armAsm->Mov(a64::w3, a64::w1); // HI = rs + + armAsm->Bind(&done); + + armAsm->Sxtw(RXSCRATCH, a64::w0); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[1])); + + armAsm->Sxtw(RXSCRATCH, a64::w3); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[1])); +} + +//// DIVU1 — unsigned 32-bit divide, pipeline 1. Same div-by-zero spec as DIVU. +void recDIVU1() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + u32 rs = g_cpuConstRegs[_Rs_].UL[0]; + u32 rt = g_cpuConstRegs[_Rt_].UL[0]; + + s32 lo, hi; + if (rt == 0) + { + lo = -1; + hi = (s32)rs; + } + else + { + lo = (s32)(rs / rt); + hi = (s32)(rs % rt); + } + + armAsm->Mov(RXSCRATCH, (s64)lo); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[1])); + + armAsm->Mov(RXSCRATCH, (s64)hi); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[1])); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + a64::Label divByZero; + a64::Label done; + armAsm->Cbz(a64::w2, &divByZero); + + // See recDIVU for the Msub-into-w1 rationale. + armAsm->Udiv(a64::w0, a64::w1, a64::w2); + armAsm->Msub(a64::w1, a64::w0, a64::w2, a64::w1); + armAsm->B(&done); + + armAsm->Bind(&divByZero); + armAsm->Mov(a64::w0, -1); + + armAsm->Bind(&done); + + armAsm->Sxtw(RXSCRATCH, a64::w0); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.LO.UD[1])); + + armAsm->Sxtw(RXSCRATCH, a64::w1); + armAsm->Str(RXSCRATCH, armCpuRegMem(&cpuRegs.HI.UD[1])); +} + +//// MADD — signed multiply-add: HI:LO += Rs * Rt, Rd = LO +void recMADD() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + s64 result = (s64)(s32)g_cpuConstRegs[_Rs_].UL[0] * (s64)(s32)g_cpuConstRegs[_Rt_].UL[0]; + + // Add to existing HI:LO — load, add, store + _eeFlushAllDirty(); + armLoadEERegPtr(a64::w1, &cpuRegs.LO.UL[0]); + armLoadEERegPtr(a64::w2, &cpuRegs.HI.UL[0]); + armAsm->Orr(a64::x1, a64::x1, a64::Operand(a64::x2, a64::LSL, 32)); + armAsm->Mov(RXSCRATCH, result); + armAsm->Add(a64::x0, a64::x1, RXSCRATCH); + + recWritebackHILO(false); + recWritebackRd(); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + // x0 = Rs * Rt (signed 32x32→64) + armAsm->Smull(a64::x0, a64::w1, a64::w2); + + // Load existing HI:LO into x1 + armLoadEERegPtr(a64::w3, &cpuRegs.LO.UL[0]); + armLoadEERegPtr(a64::w4, &cpuRegs.HI.UL[0]); + armAsm->Orr(a64::x3, a64::x3, a64::Operand(a64::x4, a64::LSL, 32)); + + // Add + armAsm->Add(a64::x0, a64::x0, a64::x3); + + recWritebackHILO(false); + recWritebackRd(); +} + +//// MADDU — unsigned multiply-add: HI:LO += Rs * Rt, Rd = LO +void recMADDU() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + u64 result = (u64)g_cpuConstRegs[_Rs_].UL[0] * (u64)g_cpuConstRegs[_Rt_].UL[0]; + + _eeFlushAllDirty(); + armLoadEERegPtr(a64::w1, &cpuRegs.LO.UL[0]); + armLoadEERegPtr(a64::w2, &cpuRegs.HI.UL[0]); + armAsm->Orr(a64::x1, a64::x1, a64::Operand(a64::x2, a64::LSL, 32)); + armAsm->Mov(RXSCRATCH, result); + armAsm->Add(a64::x0, a64::x1, RXSCRATCH); + + recWritebackHILO(false); + recWritebackRd(); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Umull(a64::x0, a64::w1, a64::w2); + + armLoadEERegPtr(a64::w3, &cpuRegs.LO.UL[0]); + armLoadEERegPtr(a64::w4, &cpuRegs.HI.UL[0]); + armAsm->Orr(a64::x3, a64::x3, a64::Operand(a64::x4, a64::LSL, 32)); + + armAsm->Add(a64::x0, a64::x0, a64::x3); + + recWritebackHILO(false); + recWritebackRd(); +} + +//// MADD1 — signed multiply-add, pipeline 1: HI1:LO1 += Rs * Rt, Rd = LO1 +void recMADD1() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + s64 result = (s64)(s32)g_cpuConstRegs[_Rs_].UL[0] * (s64)(s32)g_cpuConstRegs[_Rt_].UL[0]; + + _eeFlushAllDirty(); + armLoadEERegPtr(a64::w1, &cpuRegs.LO.UL[2]); + armLoadEERegPtr(a64::w2, &cpuRegs.HI.UL[2]); + armAsm->Orr(a64::x1, a64::x1, a64::Operand(a64::x2, a64::LSL, 32)); + armAsm->Mov(RXSCRATCH, result); + armAsm->Add(a64::x0, a64::x1, RXSCRATCH); + + recWritebackHILO(true); + recWritebackRd1(); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Smull(a64::x0, a64::w1, a64::w2); + + armLoadEERegPtr(a64::w3, &cpuRegs.LO.UL[2]); // LO1 = LO.UL[2] (upper 64 bits) + armLoadEERegPtr(a64::w4, &cpuRegs.HI.UL[2]); // HI1 = HI.UL[2] + armAsm->Orr(a64::x3, a64::x3, a64::Operand(a64::x4, a64::LSL, 32)); + + armAsm->Add(a64::x0, a64::x0, a64::x3); + + recWritebackHILO(true); + recWritebackRd1(); +} + +//// MADDU1 — unsigned multiply-add, pipeline 1 +void recMADDU1() +{ + if (GPR_IS_CONST2(_Rs_, _Rt_)) + { + u64 result = (u64)g_cpuConstRegs[_Rs_].UL[0] * (u64)g_cpuConstRegs[_Rt_].UL[0]; + + _eeFlushAllDirty(); + armLoadEERegPtr(a64::w1, &cpuRegs.LO.UL[2]); + armLoadEERegPtr(a64::w2, &cpuRegs.HI.UL[2]); + armAsm->Orr(a64::x1, a64::x1, a64::Operand(a64::x2, a64::LSL, 32)); + armAsm->Mov(RXSCRATCH, result); + armAsm->Add(a64::x0, a64::x1, RXSCRATCH); + + recWritebackHILO(true); + recWritebackRd1(); + return; + } + + _eeFlushAllDirty(); + loadRs32(); + loadRt32(); + + armAsm->Umull(a64::x0, a64::w1, a64::w2); + + armLoadEERegPtr(a64::w3, &cpuRegs.LO.UL[2]); + armLoadEERegPtr(a64::w4, &cpuRegs.HI.UL[2]); + armAsm->Orr(a64::x3, a64::x3, a64::Operand(a64::x4, a64::LSL, 32)); + + armAsm->Add(a64::x0, a64::x0, a64::x3); + + recWritebackHILO(true); + recWritebackRd1(); +} + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900Shift-arm64.cpp b/pcsx2/arm64/iR5900Shift-arm64.cpp new file mode 100644 index 0000000000..5cda95986f --- /dev/null +++ b/pcsx2/arm64/iR5900Shift-arm64.cpp @@ -0,0 +1,427 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Shift Instruction Codegen — memory-based +// All operands loaded from / stored to cpuRegs.GPR memory. +// No register allocation for scalar ops. + +#include "arm64/iR5900-arm64.h" + +namespace a64 = vixl::aarch64; + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +#ifdef FORCE_INTERP_SHIFT +REC_FUNC(SLL); +REC_FUNC(SRL); +REC_FUNC(SRA); +REC_FUNC(DSLL); +REC_FUNC(DSRL); +REC_FUNC(DSRA); +REC_FUNC(DSLL32); +REC_FUNC(DSRL32); +REC_FUNC(DSRA32); +REC_FUNC(SLLV); +REC_FUNC(SRLV); +REC_FUNC(SRAV); +REC_FUNC(DSLLV); +REC_FUNC(DSRLV); +REC_FUNC(DSRAV); +#else + +// Memory load/store helpers — always use cpuRegs memory +static void memLoadS32() +{ + armLoadEERegPtr(RWARG1, &cpuRegs.GPR.r[_Rs_].UL[0]); +} + +static void memLoadS64() +{ + armLoadEERegPtr(RXARG1, &cpuRegs.GPR.r[_Rs_].UD[0]); +} + +static void memLoadT32() +{ + armLoadEERegPtr(RWSCRATCH, &cpuRegs.GPR.r[_Rt_].UL[0]); +} + +static void memLoadT64() +{ + armLoadEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rt_].UD[0]); +} + +static void memStoreD() +{ + armStoreEERegPtr(RXSCRATCH, &cpuRegs.GPR.r[_Rd_].UD[0]); +} + +/********************************************************* + * Shift with constant amount — rd = rt SHIFT sa * + * Uses eeRecompileCodeRC2_MEM * + *********************************************************/ + +//// SLL — rd = sign_extend(rt << sa) +static void recSLL_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = (s32)(g_cpuConstRegs[_Rt_].UL[0] << _Sa_); +} + +static void recSLL_(int info) +{ + memLoadT32(); + if (_Sa_ != 0) + armAsm->Lsl(RWSCRATCH, RWSCRATCH, _Sa_); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, SLL, XMMINFO_WRITED | XMMINFO_READT); + +//// SRL — rd = sign_extend(rt >> sa) (logical) +static void recSRL_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = (s32)(g_cpuConstRegs[_Rt_].UL[0] >> _Sa_); +} + +static void recSRL_(int info) +{ + memLoadT32(); + if (_Sa_ != 0) + armAsm->Lsr(RWSCRATCH, RWSCRATCH, _Sa_); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, SRL, XMMINFO_WRITED | XMMINFO_READT); + +//// SRA — rd = sign_extend(rt >> sa) (arithmetic) +static void recSRA_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = (s32)(g_cpuConstRegs[_Rt_].SL[0] >> _Sa_); +} + +static void recSRA_(int info) +{ + memLoadT32(); + if (_Sa_ != 0) + armAsm->Asr(RWSCRATCH, RWSCRATCH, _Sa_); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, SRA, XMMINFO_WRITED | XMMINFO_READT); + +//// DSLL — rd = rt << sa (64-bit) +static void recDSLL_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rt_].UD[0] << _Sa_; +} + +static void recDSLL_(int info) +{ + memLoadT64(); + if (_Sa_ != 0) + armAsm->Lsl(RXSCRATCH, RXSCRATCH, _Sa_); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, DSLL, XMMINFO_WRITED | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSRL +static void recDSRL_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rt_].UD[0] >> _Sa_; +} + +static void recDSRL_(int info) +{ + memLoadT64(); + if (_Sa_ != 0) + armAsm->Lsr(RXSCRATCH, RXSCRATCH, _Sa_); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, DSRL, XMMINFO_WRITED | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSRA +static void recDSRA_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = g_cpuConstRegs[_Rt_].SD[0] >> _Sa_; +} + +static void recDSRA_(int info) +{ + memLoadT64(); + if (_Sa_ != 0) + armAsm->Asr(RXSCRATCH, RXSCRATCH, _Sa_); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, DSRA, XMMINFO_WRITED | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSLL32 — rd = rt << (sa + 32) +static void recDSLL32_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rt_].UD[0] << (_Sa_ + 32); +} + +static void recDSLL32_(int info) +{ + memLoadT64(); + armAsm->Lsl(RXSCRATCH, RXSCRATCH, _Sa_ + 32); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, DSLL32, XMMINFO_WRITED | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSRL32 +static void recDSRL32_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rt_].UD[0] >> (_Sa_ + 32); +} + +static void recDSRL32_(int info) +{ + memLoadT64(); + armAsm->Lsr(RXSCRATCH, RXSCRATCH, _Sa_ + 32); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, DSRL32, XMMINFO_WRITED | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSRA32 +static void recDSRA32_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = g_cpuConstRegs[_Rt_].SD[0] >> (_Sa_ + 32); +} + +static void recDSRA32_(int info) +{ + memLoadT64(); + armAsm->Asr(RXSCRATCH, RXSCRATCH, _Sa_ + 32); + memStoreD(); +} + +EERECOMPILE_CODEX_MEM(eeRecompileCodeRC2_MEM, DSRA32, XMMINFO_WRITED | XMMINFO_READT | XMMINFO_64BITOP); + +/********************************************************* + * Variable shifts — rd = rt SHIFT rs * + * Uses eeRecompileCodeRC0_MEM * + *********************************************************/ + +//// SLLV — rd = sign_extend((rt << rs[4:0])) +static void recSLLV_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = (s32)(g_cpuConstRegs[_Rt_].UL[0] << (g_cpuConstRegs[_Rs_].UL[0] & 0x1f)); +} + +static void recSLLV_consts(int info) +{ + memLoadT32(); + u32 sa = g_cpuConstRegs[_Rs_].UL[0] & 0x1f; + if (sa != 0) + armAsm->Lsl(RWSCRATCH, RWSCRATCH, sa); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSLLV_constt(int info) +{ + memLoadS32(); + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + armAsm->Lsl(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSLLV_(int info) +{ + memLoadS32(); + memLoadT32(); + armAsm->Lsl(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(SLLV, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); + +//// SRLV +static void recSRLV_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = (s32)(g_cpuConstRegs[_Rt_].UL[0] >> (g_cpuConstRegs[_Rs_].UL[0] & 0x1f)); +} + +static void recSRLV_consts(int info) +{ + memLoadT32(); + u32 sa = g_cpuConstRegs[_Rs_].UL[0] & 0x1f; + if (sa != 0) + armAsm->Lsr(RWSCRATCH, RWSCRATCH, sa); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSRLV_constt(int info) +{ + memLoadS32(); + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].UL[0]); + armAsm->Lsr(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSRLV_(int info) +{ + memLoadS32(); + memLoadT32(); + armAsm->Lsr(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(SRLV, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); + +//// SRAV +static void recSRAV_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = (s32)(g_cpuConstRegs[_Rt_].SL[0] >> (g_cpuConstRegs[_Rs_].UL[0] & 0x1f)); +} + +static void recSRAV_consts(int info) +{ + memLoadT32(); + u32 sa = g_cpuConstRegs[_Rs_].UL[0] & 0x1f; + if (sa != 0) + armAsm->Asr(RWSCRATCH, RWSCRATCH, sa); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSRAV_constt(int info) +{ + memLoadS32(); + armAsm->Mov(RWSCRATCH, g_cpuConstRegs[_Rt_].SL[0]); + armAsm->Asr(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +static void recSRAV_(int info) +{ + memLoadS32(); + memLoadT32(); + armAsm->Asr(RWSCRATCH, RWSCRATCH, RWARG1); + armAsm->Sxtw(RXSCRATCH, RWSCRATCH); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(SRAV, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT); + +//// DSLLV — rd = rt << rs[5:0] (64-bit) +static void recDSLLV_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rt_].UD[0] << (g_cpuConstRegs[_Rs_].UL[0] & 0x3f); +} + +static void recDSLLV_consts(int info) +{ + memLoadT64(); + u32 sa = g_cpuConstRegs[_Rs_].UL[0] & 0x3f; + if (sa != 0) + armAsm->Lsl(RXSCRATCH, RXSCRATCH, sa); + memStoreD(); +} + +static void recDSLLV_constt(int info) +{ + memLoadS64(); + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rt_].UD[0]); + armAsm->Lsl(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recDSLLV_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Lsl(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(DSLLV, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSRLV +static void recDSRLV_const() +{ + g_cpuConstRegs[_Rd_].UD[0] = g_cpuConstRegs[_Rt_].UD[0] >> (g_cpuConstRegs[_Rs_].UL[0] & 0x3f); +} + +static void recDSRLV_consts(int info) +{ + memLoadT64(); + u32 sa = g_cpuConstRegs[_Rs_].UL[0] & 0x3f; + if (sa != 0) + armAsm->Lsr(RXSCRATCH, RXSCRATCH, sa); + memStoreD(); +} + +static void recDSRLV_constt(int info) +{ + memLoadS64(); + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rt_].UD[0]); + armAsm->Lsr(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recDSRLV_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Lsr(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(DSRLV, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +//// DSRAV +static void recDSRAV_const() +{ + g_cpuConstRegs[_Rd_].SD[0] = g_cpuConstRegs[_Rt_].SD[0] >> (g_cpuConstRegs[_Rs_].UL[0] & 0x3f); +} + +static void recDSRAV_consts(int info) +{ + memLoadT64(); + u32 sa = g_cpuConstRegs[_Rs_].UL[0] & 0x3f; + if (sa != 0) + armAsm->Asr(RXSCRATCH, RXSCRATCH, sa); + memStoreD(); +} + +static void recDSRAV_constt(int info) +{ + memLoadS64(); + armAsm->Mov(RXSCRATCH, g_cpuConstRegs[_Rt_].SD[0]); + armAsm->Asr(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +static void recDSRAV_(int info) +{ + memLoadS64(); + memLoadT64(); + armAsm->Asr(RXSCRATCH, RXSCRATCH, RXARG1); + memStoreD(); +} + +EERECOMPILE_CODERC0_MEM(DSRAV, XMMINFO_WRITED | XMMINFO_READS | XMMINFO_READT | XMMINFO_64BITOP); + +#endif // !FORCE_INTERP_SHIFT + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/arm64/iR5900Templates-arm64.cpp b/pcsx2/arm64/iR5900Templates-arm64.cpp new file mode 100644 index 0000000000..956f6475dd --- /dev/null +++ b/pcsx2/arm64/iR5900Templates-arm64.cpp @@ -0,0 +1,303 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE Code Generation Templates +// Ports of x86/ix86-32/iR5900Templates.cpp for ARM64 register allocator. +// These templates handle register allocation, constant propagation dispatch, +// and register renaming for the standard instruction patterns. + +#include "arm64/iR5900-arm64.h" +#include "Common.h" +#include "Memory.h" +#include "VU.h" +#include "VUmicro.h" + +namespace a64 = vixl::aarch64; + +//////////////////// +// Code Templates // +//////////////////// + + +//////////////////////////////////// +// Memory-based scalar templates // +// No register allocation — all // +// operands via cpuRegs memory. // +//////////////////////////////////// + +// rd = rs OP rt (memory-based) +void eeRecompileCodeRC0_MEM(R5900FNPTR constcode, R5900FNPTR_INFO constscode, R5900FNPTR_INFO consttcode, R5900FNPTR_INFO noconstcode, int xmminfo) +{ + if (!_Rd_ && (xmminfo & XMMINFO_WRITED)) + return; + + const bool s_is_const = GPR_IS_CONST1(_Rs_); + const bool t_is_const = GPR_IS_CONST1(_Rt_); + + // Both-const: compile-time evaluation + if (s_is_const && t_is_const) + { + if (_Rd_ && (xmminfo & XMMINFO_WRITED)) + { + _deleteEEreg(_Rd_, 0); + GPR_SET_CONST(_Rd_); + } + constcode(); + return; + } + + // Flush source registers to memory (writeback if dirty, then free) + if ((xmminfo & XMMINFO_READS) && !s_is_const) + _deleteEEreg(_Rs_, 1); + if ((xmminfo & XMMINFO_READT) && !t_is_const) + _deleteEEreg(_Rt_, 1); + + // Handle dest register + if (xmminfo & XMMINFO_READD) + _deleteEEreg(_Rd_, 1); // flush so we can read current value + else if (xmminfo & XMMINFO_WRITED) + _deleteEEreg(_Rd_, 0); // discard — we're about to overwrite + + if (xmminfo & XMMINFO_WRITED) + GPR_DEL_CONST(_Rd_); + + u32 info = 0; // No register allocation — codegen reads/writes memory + + if (s_is_const) + { + constscode(info); + return; + } + + if (t_is_const) + { + consttcode(info); + return; + } + + noconstcode(info); +} + +// rt = rs OP imm16 (memory-based) +void eeRecompileCodeRC1_MEM(R5900FNPTR constcode, R5900FNPTR_INFO noconstcode, int xmminfo) +{ + pxAssert((xmminfo & (XMMINFO_READS | XMMINFO_WRITET)) == (XMMINFO_READS | XMMINFO_WRITET)); + + if (!_Rt_) + return; + + // Const: compile-time evaluation + if (GPR_IS_CONST1(_Rs_)) + { + _deleteEEreg(_Rt_, 0); + GPR_SET_CONST(_Rt_); + constcode(); + return; + } + + // Flush source to memory + _deleteEEreg(_Rs_, 1); + + // Discard dest (about to overwrite) + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + + u32 info = 0; + noconstcode(info); +} + +// rd = rt OP sa (memory-based) +void eeRecompileCodeRC2_MEM(R5900FNPTR constcode, R5900FNPTR_INFO noconstcode, int xmminfo) +{ + pxAssert((xmminfo & (XMMINFO_READT | XMMINFO_WRITED)) == (XMMINFO_READT | XMMINFO_WRITED)); + + if (!_Rd_) + return; + + // Const: compile-time evaluation + if (GPR_IS_CONST1(_Rt_)) + { + _deleteEEreg(_Rd_, 0); + GPR_SET_CONST(_Rd_); + constcode(); + return; + } + + // Flush source to memory + _deleteEEreg(_Rt_, 1); + + // Discard dest (about to overwrite) + _deleteEEreg(_Rd_, 0); + GPR_DEL_CONST(_Rd_); + + u32 info = 0; + noconstcode(info); +} + +// 128-bit NEON allocation for MMI/XMM operations +int eeRecompileCodeXMM(int xmminfo) +{ + int info = PROCESS_EE_XMM; + + // EEREC_LO, EEREC_HI and EEREC_ACC all decode from the same 5-bit info-word + // field (see iCore-arm64.h): five distinct 5-bit register fields plus the + // presence flags do not fit in the 32-bit info word. This is safe only because + // no op needs two of {LO, HI, ACC} live at once through the allocator — integer + // MULT/DIV/MADD and PMFHL load LO/HI directly from memory, and ACC is FPU-only. + // Requesting both LO and HI here would OR two register indices into one field + // and silently miscompile, so guard it; an op that genuinely needs both must + // bypass the allocator (see recPMFHL). + pxAssertRel(!((xmminfo & (XMMINFO_READLO | XMMINFO_WRITELO)) && + (xmminfo & (XMMINFO_READHI | XMMINFO_WRITEHI))), + "eeRecompileCodeXMM: LO and HI share an info-word field; an op needing both " + "must bypass the allocator (see recPMFHL)."); + + if (xmminfo & (XMMINFO_READLO | XMMINFO_WRITELO)) + _addNeededGPRtoNEONreg(NEONGPR_LO); + if (xmminfo & (XMMINFO_READHI | XMMINFO_WRITEHI)) + _addNeededGPRtoNEONreg(NEONGPR_HI); + if (xmminfo & XMMINFO_READS) + _addNeededGPRtoNEONreg(_Rs_); + if (xmminfo & XMMINFO_READT) + _addNeededGPRtoNEONreg(_Rt_); + if (xmminfo & XMMINFO_WRITED) + _addNeededGPRtoNEONreg(_Rd_); + + if (xmminfo & XMMINFO_READS) + { + const int reg = _allocGPRtoNEONreg(_Rs_, MODE_READ); + info |= PROCESS_EE_SET_S(reg); + } + if (xmminfo & XMMINFO_READT) + { + const int reg = _allocGPRtoNEONreg(_Rt_, MODE_READ); + info |= PROCESS_EE_SET_T(reg); + } + + if (xmminfo & XMMINFO_WRITED) + { + int readd = MODE_WRITE | ((xmminfo & XMMINFO_READD) ? MODE_READ : 0); + + int regd = _checkNEONreg(NEONTYPE_GPRREG, _Rd_, readd); + if (regd < 0) + { + // TODO: register renaming for NEON + regd = _allocGPRtoNEONreg(_Rd_, readd); + } + info |= PROCESS_EE_SET_D(regd); + } + + // INVARIANT: no EE opcode currently passes XMMINFO_*LO/HI, so these two + // branches never execute and LO/HI are never NEON-resident — which is why + // the MMI mul/mac handlers (recPMADDUW/PMFHL/PMTHI/PMTLO in iMMI-arm64.cpp) + // can Str/Ldr LO/HI straight to memory without an allocator flush. If a + // future op DOES request XMMINFO_*LO/HI, every direct-memory LO/HI handler + // in iMMI-arm64.cpp must regain a _deleteGPRtoNEONreg(NEONGPR_LO/HI) flush. + if (xmminfo & (XMMINFO_READLO | XMMINFO_WRITELO)) + { + info |= PROCESS_EE_SET_LO(_allocGPRtoNEONreg(NEONGPR_LO, + ((xmminfo & XMMINFO_READLO) ? MODE_READ : 0) | ((xmminfo & XMMINFO_WRITELO) ? MODE_WRITE : 0))); + } + if (xmminfo & (XMMINFO_READHI | XMMINFO_WRITEHI)) + { + info |= PROCESS_EE_SET_HI(_allocGPRtoNEONreg(NEONGPR_HI, + ((xmminfo & XMMINFO_READHI) ? MODE_READ : 0) | ((xmminfo & XMMINFO_WRITEHI) ? MODE_WRITE : 0))); + } + + if (xmminfo & XMMINFO_WRITED) + GPR_DEL_CONST(_Rd_); + + _validateRegs(); + return info; +} + +// FPU allocation template +#define _Ft_ _Rt_ +#define _Fs_ _Rd_ +#define _Fd_ _Sa_ + +void eeFPURecompileCode(R5900FNPTR_INFO xmmcode, R5900FNPTR fpucode, int xmminfo) +{ + int mmregs = -1, mmregt = -1, mmregd = -1, mmregacc = -1; + int info = PROCESS_EE_XMM; + + if (xmminfo & XMMINFO_READS) + _addNeededFPtoNEONreg(_Fs_); + if (xmminfo & XMMINFO_READT) + _addNeededFPtoNEONreg(_Ft_); + if (xmminfo & (XMMINFO_WRITED | XMMINFO_READD)) + _addNeededFPtoNEONreg(_Fd_); + if (xmminfo & (XMMINFO_WRITEACC | XMMINFO_READACC)) + _addNeededFPACCtoNEONreg(); + + if (xmminfo & XMMINFO_READT) + mmregt = _allocFPtoNEONreg(_Ft_, MODE_READ); + + if (xmminfo & XMMINFO_READS) + { + mmregs = _allocFPtoNEONreg(_Fs_, MODE_READ); + if ((xmminfo & XMMINFO_READT) && _Fs_ == _Ft_) + mmregt = mmregs; + } + + if (xmminfo & XMMINFO_READD) + { + pxAssert(xmminfo & XMMINFO_WRITED); + mmregd = _allocFPtoNEONreg(_Fd_, MODE_READ); + } + + if (xmminfo & XMMINFO_READACC) + mmregacc = _allocFPACCtoNEONreg(MODE_READ); + + if (xmminfo & XMMINFO_WRITEACC) + { + int readacc = MODE_WRITE | ((xmminfo & XMMINFO_READACC) ? MODE_READ : 0); + mmregacc = _checkNEONreg(NEONTYPE_FPACC, 0, readacc); + if (mmregacc < 0) + mmregacc = _allocFPACCtoNEONreg(readacc); + arm64neon[mmregacc].mode |= MODE_WRITE; + } + else if (xmminfo & XMMINFO_WRITED) + { + int readd = MODE_WRITE | ((xmminfo & XMMINFO_READD) ? MODE_READ : 0); + if (xmminfo & XMMINFO_READD) + mmregd = _allocFPtoNEONreg(_Fd_, readd); + else + mmregd = _checkNEONreg(NEONTYPE_FPREG, _Fd_, readd); + + if (mmregd < 0) + mmregd = _allocFPtoNEONreg(_Fd_, readd); + } + + pxAssert(mmregs >= 0 || mmregt >= 0 || mmregd >= 0 || mmregacc >= 0); + + if (xmminfo & XMMINFO_WRITED) + { + pxAssert(mmregd >= 0); + info |= PROCESS_EE_SET_D(mmregd); + } + if (xmminfo & (XMMINFO_WRITEACC | XMMINFO_READACC)) + { + if (mmregacc >= 0) + info |= PROCESS_EE_SET_ACC(mmregacc) | PROCESS_EE_ACC; + else + pxAssert(!(xmminfo & XMMINFO_WRITEACC)); + } + + if (xmminfo & XMMINFO_READS) + { + if (mmregs >= 0) + info |= PROCESS_EE_SET_S(mmregs); + } + if (xmminfo & XMMINFO_READT) + { + if (mmregt >= 0) + info |= PROCESS_EE_SET_T(mmregt); + } + + xmmcode(info); +} + +#undef _Ft_ +#undef _Fs_ +#undef _Fd_ diff --git a/pcsx2/arm64/recVTLB-arm64.cpp b/pcsx2/arm64/recVTLB-arm64.cpp new file mode 100644 index 0000000000..1cfc8ba91b --- /dev/null +++ b/pcsx2/arm64/recVTLB-arm64.cpp @@ -0,0 +1,1214 @@ +// SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team +// SPDX-License-Identifier: GPL-3.0+ + +// ARM64 EE VTLB Dynamic Code Generation +// +// Two paths for load/store instructions: +// +// 1. Fastmem (primary): Single LDR/STR via RFASTMEMBASE (x19). +// The fastmem area is a 4GB region mapped to PS2 memory. If the page +// is unmapped (MMIO, etc.), the SIGSEGV handler backpatches the faulting +// instruction with a branch to a slow-path thunk. +// +// 2. Softmem (fallback): Inline VTLB page table lookup. Used when fastmem +// is disabled or when a PC has previously faulted (vtlb_IsFaultingPC). + +// FORCE_INTERP_MEMORY is defined in iR5900-arm64.h to force interpreter fallback + +#include "arm64/iR5900-arm64.h" +#include "arm64/AsmHelpers.h" +#include "vtlb.h" +#include "VU.h" +#include "Hw.h" +#include "Memory.h" +#include "common/Assertions.h" + +extern void vu0Sync(); + +namespace a64 = vixl::aarch64; + +using namespace vtlb_private; + + +// ===================================================================================================== +// Softmem — Inline VTLB Lookup (fallback for faulting PCs) +// ===================================================================================================== + +// Generates inline vtlb read code. Result in w0/x0. +// +// Algorithm: +// vmv = vmap[addr >> VTLB_PAGE_BITS] (page table lookup) +// ppf = addr + vmv (combine with mapping) +// if (ppf >= 0) result = *(DataType*)ppf (fast path: direct read) +// else call vtlb_memRead(addr) (slow path: handler dispatch) +// +// addr_reg: ARM64 W register index containing the EE virtual address +// Clobbers: w0, x0, x8, x9, x17 +// Result: in w0/x0 +static void vtlbSoftmemRead(int addr_wreg, u32 bits, bool sign) +{ + pxAssert(bits == 8 || bits == 16 || bits == 32 || bits == 64); + + // Save the original address in w9 (needed for slow path) + if (addr_wreg != 9) + armAsm->Mov(a64::w9, armWRegister(addr_wreg)); + + // Page index: w8 = addr >> VTLB_PAGE_BITS + armAsm->Lsr(a64::w8, a64::w9, VTLB_PAGE_BITS); + + // Load vmap base address into x17 + armMoveAddressToReg(RSCRATCHADDR, vtlbdata.vmap); + + // Load vmap entry: x8 = vmap[page_index] (each entry is 8 bytes = sptr) + armAsm->Ldr(a64::x8, a64::MemOperand(RSCRATCHADDR, a64::x8, a64::LSL, 3)); + + // Compute ppf: x0 = addr + vmv (use 64-bit add, addr zero-extended from w9). + // ADDS sets N from bit 63 of the result, so B.mi handles the slow-path + // branch on the sign bit without a separate Tbnz. + armAsm->Adds(a64::x0, a64::x8, a64::Operand(a64::w9, a64::UXTW)); + + a64::Label slow_path, done; + armAsm->B(&slow_path, a64::mi); + + // --- Fast path: direct memory read from host pointer ppf --- + switch (bits) + { + case 8: + if (sign) + armAsm->Ldrsb(a64::x0, a64::MemOperand(a64::x0)); + else + armAsm->Ldrb(a64::w0, a64::MemOperand(a64::x0)); + break; + case 16: + if (sign) + armAsm->Ldrsh(a64::x0, a64::MemOperand(a64::x0)); + else + armAsm->Ldrh(a64::w0, a64::MemOperand(a64::x0)); + break; + case 32: + if (sign) + armAsm->Ldrsw(a64::x0, a64::MemOperand(a64::x0)); + else + armAsm->Ldr(a64::w0, a64::MemOperand(a64::x0)); + break; + case 64: + armAsm->Ldr(a64::x0, a64::MemOperand(a64::x0)); + break; + } + armAsm->B(&done); + + // --- Slow path: call vtlb_memRead(addr) --- + armAsm->Bind(&slow_path); + armAsm->Mov(a64::w0, a64::w9); // restore original address as argument + + // Spill/reload RECCYCLE around the handler call: page-0F INTC_STAT + // reads invoke IntCHackCheck which mutates cpuRegs.cycle. Without + // this the JIT's pinned x25 stays stale and block-end cycle compare + // never trips on tight INTC polls. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + switch (bits) + { + case 8: armEmitCall((void*)vtlb_memRead); break; + case 16: armEmitCall((void*)vtlb_memRead); break; + case 32: armEmitCall((void*)vtlb_memRead); break; + case 64: armEmitCall((void*)vtlb_memRead); break; + } + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + // Sign-extend if needed (vtlb_memRead returns zero-extended) + if (sign && bits == 8) + armAsm->Sxtb(a64::x0, a64::w0); + else if (sign && bits == 16) + armAsm->Sxth(a64::x0, a64::w0); + else if (sign && bits == 32) + armAsm->Sxtw(a64::x0, a64::w0); + + armAsm->Bind(&done); +} + +// Generates inline vtlb write code. +// addr_reg: W register index with EE virtual address +// value_reg: W/X register index with value to write +// Clobbers: w0, x0, w1, x1, x8, x9, x17 +static void vtlbSoftmemWrite(int addr_wreg, int value_reg, u32 bits) +{ + pxAssert(bits == 8 || bits == 16 || bits == 32 || bits == 64); + + if (addr_wreg != 9) + armAsm->Mov(a64::w9, armWRegister(addr_wreg)); + if (value_reg != 10) + { + if (bits <= 32) + armAsm->Mov(a64::w10, armWRegister(value_reg)); + else + armAsm->Mov(a64::x10, armXRegister(value_reg)); + } + + armAsm->Lsr(a64::w8, a64::w9, VTLB_PAGE_BITS); + armMoveAddressToReg(RSCRATCHADDR, vtlbdata.vmap); + armAsm->Ldr(a64::x8, a64::MemOperand(RSCRATCHADDR, a64::x8, a64::LSL, 3)); + // ADDS sets N from bit 63 of ppf; B.mi (=N) branches on the sign bit + // without the separate Tbnz, saving one instruction per softmem op. + armAsm->Adds(a64::x0, a64::x8, a64::Operand(a64::w9, a64::UXTW)); + + a64::Label slow_path, done; + armAsm->B(&slow_path, a64::mi); + + // --- Fast path: direct memory write --- + switch (bits) + { + case 8: armAsm->Strb(a64::w10, a64::MemOperand(a64::x0)); break; + case 16: armAsm->Strh(a64::w10, a64::MemOperand(a64::x0)); break; + case 32: armAsm->Str(a64::w10, a64::MemOperand(a64::x0)); break; + case 64: armAsm->Str(a64::x10, a64::MemOperand(a64::x0)); break; + } + armAsm->B(&done); + + // --- Slow path: call vtlb_memWrite --- + armAsm->Bind(&slow_path); + armAsm->Mov(a64::w0, a64::w9); + if (bits <= 32) + armAsm->Mov(a64::w1, a64::w10); + else + armAsm->Mov(a64::x1, a64::x10); + + // Spill/reload RECCYCLE: write-side handlers are symmetric to reads — + // any cycle-mutating handler reachable from MMIO must keep the JIT's + // pinned x25 coherent. See vtlbSoftmemRead for full rationale. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + switch (bits) + { + case 8: armEmitCall((void*)vtlb_memWrite); break; + case 16: armEmitCall((void*)vtlb_memWrite); break; + case 32: armEmitCall((void*)vtlb_memWrite); break; + case 64: armEmitCall((void*)vtlb_memWrite); break; + } + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armAsm->Bind(&done); +} + +// ===================================================================================================== +// Load/Store Instruction Implementations +// ===================================================================================================== + +namespace R5900 { +namespace Dynarec { +namespace OpcodeImpl { + +namespace Interp = R5900::Interpreter::OpcodeImpl; + +// ===================================================================================================== +// Fastmem helpers +// ===================================================================================================== + +// Build bitmasks of currently allocated ARM64 GPR and NEON registers. +// The backpatch thunk uses these to save/restore live registers around +// the vtlb C call, preventing corruption of JIT register allocator state. +static void vtlbGetLiveRegisterMasks(u32& gpr_bitmask, u32& fpr_bitmask) +{ + gpr_bitmask = 0; + for (int i = 0; i < NUM_ARM_GPR_REGS; i++) + { + if (arm64gprs[i].inuse) + gpr_bitmask |= (1u << i); + } + + fpr_bitmask = 0; + for (int i = 0; i < NUM_ARM_NEON_REGS; i++) + { + if (arm64neon[i].inuse) + fpr_bitmask |= (1u << i); + } +} + +// Emit a single fastmem load instruction and register backpatch info. +// addr_wreg: W register index holding the 32-bit guest virtual address +// dest_reg: register index where the result goes (W or X based on bits) +// Result is in dest_reg after the load (or after backpatch thunk on fault). +static void vtlbFastmemRead(int addr_wreg, int dest_reg, u32 bits, bool sign) +{ + u32 gpr_bitmask, fpr_bitmask; + vtlbGetLiveRegisterMasks(gpr_bitmask, fpr_bitmask); + + const u8* codeStart = armGetCurrentCodePointer(); + + a64::MemOperand mem(RFASTMEMBASE, armWRegister(addr_wreg), a64::UXTW); + switch (bits) + { + case 8: + if (sign) + armAsm->Ldrsb(armXRegister(dest_reg), mem); + else + armAsm->Ldrb(armWRegister(dest_reg), mem); + break; + case 16: + if (sign) + armAsm->Ldrsh(armXRegister(dest_reg), mem); + else + armAsm->Ldrh(armWRegister(dest_reg), mem); + break; + case 32: + if (sign) + armAsm->Ldrsw(armXRegister(dest_reg), mem); + else + armAsm->Ldr(armWRegister(dest_reg), mem); + break; + case 64: + armAsm->Ldr(armXRegister(dest_reg), mem); + break; + } + + vtlb_AddLoadStoreInfo((uptr)codeStart, 4, pc, gpr_bitmask, fpr_bitmask, + static_cast(addr_wreg), static_cast(dest_reg), + static_cast(bits), sign, true, false); +} + +// Emit a single fastmem store instruction and register backpatch info. +static void vtlbFastmemWrite(int addr_wreg, int value_reg, u32 bits) +{ + u32 gpr_bitmask, fpr_bitmask; + vtlbGetLiveRegisterMasks(gpr_bitmask, fpr_bitmask); + + const u8* codeStart = armGetCurrentCodePointer(); + + a64::MemOperand mem(RFASTMEMBASE, armWRegister(addr_wreg), a64::UXTW); + switch (bits) + { + case 8: armAsm->Strb(armWRegister(value_reg), mem); break; + case 16: armAsm->Strh(armWRegister(value_reg), mem); break; + case 32: armAsm->Str(armWRegister(value_reg), mem); break; + case 64: armAsm->Str(armXRegister(value_reg), mem); break; + } + + vtlb_AddLoadStoreInfo((uptr)codeStart, 4, pc, gpr_bitmask, fpr_bitmask, + static_cast(addr_wreg), static_cast(value_reg), + static_cast(bits), false, false, false); +} + +// Emit a single 128-bit fastmem load (LDR Q0, [RFASTMEMBASE, w_addr, UXTW]). +// Result in q0. Mirrors x86 PCSX2's MOVAPS-via-RFASTMEMBASE pattern +// (ix86-32/recVTLB.cpp). Backpatch thunk extended in RecStubs.cpp. +static void vtlbFastmemRead128(int addr_wreg) +{ + u32 gpr_bitmask, fpr_bitmask; + vtlbGetLiveRegisterMasks(gpr_bitmask, fpr_bitmask); + + const u8* codeStart = armGetCurrentCodePointer(); + armAsm->Ldr(a64::q0, a64::MemOperand(RFASTMEMBASE, armWRegister(addr_wreg), a64::UXTW)); + + vtlb_AddLoadStoreInfo((uptr)codeStart, 4, pc, gpr_bitmask, fpr_bitmask, + static_cast(addr_wreg), /*data_register*/ 0, + /*size_in_bits*/ 128, /*is_signed*/ false, /*is_load*/ true, /*is_fpr*/ true); +} + +// Emit a single 128-bit fastmem store (STR Q0, [RFASTMEMBASE, w_addr, UXTW]). +// Value in q0. Backpatch thunk extended in RecStubs.cpp. +static void vtlbFastmemWrite128(int addr_wreg) +{ + u32 gpr_bitmask, fpr_bitmask; + vtlbGetLiveRegisterMasks(gpr_bitmask, fpr_bitmask); + + const u8* codeStart = armGetCurrentCodePointer(); + armAsm->Str(a64::q0, a64::MemOperand(RFASTMEMBASE, armWRegister(addr_wreg), a64::UXTW)); + + vtlb_AddLoadStoreInfo((uptr)codeStart, 4, pc, gpr_bitmask, fpr_bitmask, + static_cast(addr_wreg), /*data_register*/ 0, + /*size_in_bits*/ 128, /*is_signed*/ false, /*is_load*/ false, /*is_fpr*/ true); +} + +// ===================================================================================================== +// Address computation helpers +// ===================================================================================================== + +// Compute load/store address (rs + imm) into w9. +// Does NOT flush — reads from wherever the value currently lives +// (const propagation, ARM64 GPR, NEON register, or cpuRegs memory). +// For const Rs, the full address is computed at compile time. +static void recComputeAddr() +{ + if (GPR_IS_CONST1(_Rs_)) + { + armAsm->Mov(a64::w9, g_cpuConstRegs[_Rs_].UL[0] + _Imm_); + } + else + { + _eeMoveGPRtoR(a64::w9, _Rs_); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + } +} + +// Prepare store value in w10/x10. +// Does NOT flush — reads from wherever the value currently lives. +static void recPrepStoreValue(u32 bits) +{ + _eeMoveGPRtoR(bits <= 32 ? a64::w10 : a64::x10, _Rt_); +} + +// Store load result from x0 to guest register Rt. +// Invalidates any existing allocation for Rt (const/GPR/NEON) and +// writes the result to cpuRegs memory. +static void recStoreLoadResult() +{ + if (_Rt_) + { + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + armStoreEERegPtr(a64::x0, &cpuRegs.GPR.r[_Rt_].UD[0]); + } +} + +// ===================================================================================================== +// Load implementations +// ===================================================================================================== + +// Const-paddr MMIO shortcut. When Rs is constant and the resolved page is a +// handler (MMIO), emit a direct BL to the registered handler instead of going +// through fastmem-fault → backpatch thunk → vtlb_memRead → page-table dispatch. +// Mirrors x86 vtlb_DynGenReadNonQuad_Const (ix86-32/recVTLB.cpp). +// +// Direct (RAM-backed) const-paddr loads stay on the fastmem path — a single +// LDR off RFASTMEMBASE is already optimal for those. +// +// Returns true if the shortcut emitted the load; caller should bail out. +static bool recLoadConstPaddrMMIOShortcut(u32 bits, bool sign) +{ + if (!GPR_IS_CONST1(_Rs_)) + return false; + + const u32 addr_const = g_cpuConstRegs[_Rs_].UL[0] + _Imm_; + const auto vmv = vtlbdata.vmap[addr_const >> VTLB_PAGE_BITS]; + if (!vmv.isHandler(addr_const)) + return false; + + const u32 paddr = vmv.assumeHandlerGetPAddr(addr_const); + + iFlushCall(FLUSH_INTERPRETER); + + // INTC_STAT inline-load when the speedhack is disabled. With it on + // (the default), fall through to a direct BL of the registered + // hwRead32_page_0F_INTC_HACK handler. + if (bits == 32 && !EmuConfig.Speedhacks.IntcStat && paddr == INTC_STAT) + { + armLoadPtr(a64::w0, &psHu32(INTC_STAT)); + if (sign) + armAsm->Sxtw(a64::x0, a64::w0); + recStoreLoadResult(); + return true; + } + + int szidx = 0; + switch (bits) + { + case 8: szidx = 0; break; + case 16: szidx = 1; break; + case 32: szidx = 2; break; + case 64: szidx = 3; break; + } + armAsm->Mov(a64::w0, paddr); + // Spill/reload RECCYCLE around the registered handler: the const-paddr + // MMIO shortcut targets the same handler set as vtlbSoftmemRead's slow + // path, including page-0F INTC_STAT → IntCHackCheck which mutates + // cpuRegs.cycle. See vtlbSoftmemRead for full rationale. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armEmitCall(vmv.assumeHandlerGetRaw(szidx, false)); + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + // Extend handler return value into x0 for the 64-bit cpuRegs.GPR store. + // AAPCS64 leaves the upper bits of x0 unspecified for sub-word returns. + if (bits < 64) + { + if (sign) + { + switch (bits) + { + case 8: armAsm->Sxtb(a64::x0, a64::w0); break; + case 16: armAsm->Sxth(a64::x0, a64::w0); break; + case 32: armAsm->Sxtw(a64::x0, a64::w0); break; + } + } + else + { + armAsm->Uxtw(a64::x0, a64::w0); + } + } + + recStoreLoadResult(); + return true; +} + +// Generic load: fastmem (primary) with softmem fallback for faulting PCs. +// +// Fastmem path: no flush — the backpatch thunk in RecStubs.cpp saves/ +// restores live regs around the slow-path C call. +// +// Softmem path: FLUSH_CONSTANT_REGS only. iFlushCall already evicts all +// caller-saved GPRs + NEON unconditionally; constants must additionally +// be written back so post-call emit re-reads from cpuRegs.GPR rather than +// trusting now-stale const tracking. PC and CODE are not load-bearing — +// vtlb_memRead doesn't read them, and exception handlers that fire +// from the slow path stash their own PC. +static void recLoad(u32 bits, bool sign) +{ + // Force an event test on EE counter-range reads (the EE timers at + // 0x10000000..0x10001FFF) to improve read + interrupt syncing — namely + // ESPN Games. Follows the upstream x86-master fix + // (iR5900LoadStore.cpp: needs_flush → iFlushCall(FLUSH_INTERPRETER) + + // g_branch=2). Setting g_branch=2 makes the block finalizer end the block + // with the event-test exit (it does the FLUSH_EVERYTHING + cycle accumulate + + // nextEventCycle dispatch itself), so no explicit iFlushCall is needed here. + bool forceEventTest = false; + if (GPR_IS_CONST1(_Rs_) && bits <= 32) + { + const u32 srcadr = g_cpuConstRegs[_Rs_].UL[0] + _Imm_; + forceEventTest = (srcadr & 0xFFFFE000) == 0x10000000; + } + + if (recLoadConstPaddrMMIOShortcut(bits, sign)) + { + if (forceEventTest) + g_branch = 2; + return; + } + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + if (GPR_IS_CONST1(_Rs_)) + { + iFlushCall(FLUSH_CONSTANT_REGS); + armAsm->Mov(a64::w9, g_cpuConstRegs[_Rs_].UL[0] + _Imm_); + } + else + { + _eeMoveGPRtoR(a64::w9, _Rs_); + iFlushCall(FLUSH_CONSTANT_REGS); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + } + + if (useFastmem) + { + vtlbFastmemRead(9, 0, bits, sign); + } + else + { + vtlbSoftmemRead(9, bits, sign); + } + recStoreLoadResult(); + + if (forceEventTest) + g_branch = 2; +} + +#ifdef FORCE_INTERP_MEMORY +REC_FUNC(LB); REC_FUNC(LBU); REC_FUNC(LH); REC_FUNC(LHU); +REC_FUNC(LW); REC_FUNC(LWU); REC_FUNC(LD); +#else +void recLB() { recLoad(8, true); } +void recLBU() { recLoad(8, false); } +void recLH() { recLoad(16, true); } +void recLHU() { recLoad(16, false); } +void recLW() { recLoad(32, true); } +void recLWU() { recLoad(32, false); } +void recLD() { recLoad(64, false); } +#endif + +// ===================================================================================================== +// Store implementations +// ===================================================================================================== + +// Symmetric to recLoadConstPaddrMMIOShortcut: when Rs is constant and the +// resolved page is a handler (MMIO), emit a direct BL to the registered write +// handler instead of going through fastmem-fault → backpatch thunk → +// vtlb_memWrite → page-table dispatch. Mirrors x86 vtlb_DynGenWrite_Const +// (ix86-32/recVTLB.cpp). +// +// Direct (RAM-backed) const-paddr stores stay on the fastmem path — a single +// STR off RFASTMEMBASE is already optimal for those. +// +// Returns true if the shortcut emitted the store; caller should bail out. +static bool recStoreConstPaddrMMIOShortcut(u32 bits) +{ + if (!GPR_IS_CONST1(_Rs_)) + return false; + + const u32 addr_const = g_cpuConstRegs[_Rs_].UL[0] + _Imm_; + const auto vmv = vtlbdata.vmap[addr_const >> VTLB_PAGE_BITS]; + if (!vmv.isHandler(addr_const)) + return false; + + const u32 paddr = vmv.assumeHandlerGetPAddr(addr_const); + + iFlushCall(FLUSH_INTERPRETER); + + int szidx = 0; + switch (bits) + { + case 8: szidx = 0; break; + case 16: szidx = 1; break; + case 32: szidx = 2; break; + case 64: szidx = 3; break; + } + + // AAPCS64: w0 = paddr, w1/x1 = value. After FLUSH_INTERPRETER (0xfff) + // all guest reg state is in memory, so armLoadEERegPtr is correct for + // both const and non-const Rt (including Rt == 0). + armAsm->Mov(a64::w0, paddr); + if (bits <= 32) + armLoadEERegPtr(a64::w1, &cpuRegs.GPR.r[_Rt_].UL[0]); + else + armLoadEERegPtr(a64::x1, &cpuRegs.GPR.r[_Rt_].UD[0]); + + // RECCYCLE coherence — same rationale as recLoadConstPaddrMMIOShortcut. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armEmitCall(vmv.assumeHandlerGetRaw(szidx, true)); + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + return true; +} + +static void recStore(u32 bits) +{ + if (recStoreConstPaddrMMIOShortcut(bits)) + return; + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + // Flush rationale: see recLoad. FLUSH_CONSTANT_REGS only. + if (GPR_IS_CONST1(_Rt_)) + { + if (bits <= 32) + armAsm->Mov(a64::w10, g_cpuConstRegs[_Rt_].UL[0]); + else + armAsm->Mov(a64::x10, g_cpuConstRegs[_Rt_].UD[0]); + } + else + { + _eeMoveGPRtoR(bits <= 32 ? a64::w10 : a64::x10, _Rt_); + } + + if (GPR_IS_CONST1(_Rs_)) + { + iFlushCall(FLUSH_CONSTANT_REGS); + armAsm->Mov(a64::w9, g_cpuConstRegs[_Rs_].UL[0] + _Imm_); + } + else + { + _eeMoveGPRtoR(a64::w9, _Rs_); + iFlushCall(FLUSH_CONSTANT_REGS); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + } + + if (useFastmem) + { + vtlbFastmemWrite(9, 10, bits); + } + else + { + vtlbSoftmemWrite(9, 10, bits); + } +} + +#ifdef FORCE_INTERP_MEMORY +REC_FUNC(SB); REC_FUNC(SH); REC_FUNC(SW); REC_FUNC(SD); +#else +void recSB() { recStore(8); } +void recSH() { recStore(16); } +void recSW() { recStore(32); } +void recSD() { recStore(64); } +#endif + +// ===================================================================================================== +// LQ / SQ — 128-bit quad load/store +// addr = (rs + imm) & ~0xF (silently aligned to 16 bytes) +// ===================================================================================================== + +// Inline VTLB 128-bit read. Result in q0. +static void vtlbSoftmemRead128(int addr_wreg) +{ + if (addr_wreg != 9) + armAsm->Mov(a64::w9, armWRegister(addr_wreg)); + + armAsm->Lsr(a64::w8, a64::w9, VTLB_PAGE_BITS); + armMoveAddressToReg(RSCRATCHADDR, vtlbdata.vmap); + armAsm->Ldr(a64::x8, a64::MemOperand(RSCRATCHADDR, a64::x8, a64::LSL, 3)); + // ADDS sets N from bit 63 of ppf; B.mi (=N) branches on the sign bit + // without the separate Tbnz, saving one instruction per softmem op. + armAsm->Adds(a64::x0, a64::x8, a64::Operand(a64::w9, a64::UXTW)); + + a64::Label slow_path, done; + armAsm->B(&slow_path, a64::mi); + + // Fast path: LDR q0, [x0] + armAsm->Ldr(a64::q0, a64::MemOperand(a64::x0)); + armAsm->B(&done); + + // Slow path: call vtlb_memRead128(addr) — returns r128 in q0 + armAsm->Bind(&slow_path); + armAsm->Mov(a64::w0, a64::w9); + // See vtlbSoftmemRead for the RECCYCLE coherence rationale. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armEmitCall((void*)vtlb_memRead128); + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armAsm->Bind(&done); +} + +// Inline VTLB 128-bit write. Value in q0. +static void vtlbSoftmemWrite128(int addr_wreg) +{ + if (addr_wreg != 9) + armAsm->Mov(a64::w9, armWRegister(addr_wreg)); + + armAsm->Lsr(a64::w8, a64::w9, VTLB_PAGE_BITS); + armMoveAddressToReg(RSCRATCHADDR, vtlbdata.vmap); + armAsm->Ldr(a64::x8, a64::MemOperand(RSCRATCHADDR, a64::x8, a64::LSL, 3)); + // ADDS sets N from bit 63 of ppf; B.mi (=N) branches on the sign bit + // without the separate Tbnz, saving one instruction per softmem op. + armAsm->Adds(a64::x0, a64::x8, a64::Operand(a64::w9, a64::UXTW)); + + a64::Label slow_path, done; + armAsm->B(&slow_path, a64::mi); + + // Fast path: STR q0, [x0] + armAsm->Str(a64::q0, a64::MemOperand(a64::x0)); + armAsm->B(&done); + + // Slow path: call vtlb_memWrite128(addr, value) + // addr in w0, value in q0 (ARM64 ABI: 128-bit passed in q0) + armAsm->Bind(&slow_path); + armAsm->Mov(a64::w0, a64::w9); + // See vtlbSoftmemRead for the RECCYCLE coherence rationale. + armAsm->Str(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + armEmitCall((void*)vtlb_memWrite128); + armAsm->Ldr(RECCYCLE, armCpuRegMem(&cpuRegs.cycle)); + + armAsm->Bind(&done); +} + +void recLQ() +{ + // addr = (rs + imm) & ~0xF — compute from live registers, then flush + recComputeAddr(); + armAsm->And(a64::w9, a64::w9, (u32)~0xF); + iFlushCall(FLUSH_CONSTANT_REGS); + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + if (useFastmem) + vtlbFastmemRead128(9); + else + vtlbSoftmemRead128(9); + + // Store full 128-bit result to GPR[rt] via memory + if (_Rt_) + { + _deleteEEreg(_Rt_, 0); + GPR_DEL_CONST(_Rt_); + armAsm->Str(a64::q0, armCpuRegMem(&cpuRegs.GPR.r[_Rt_].UD[0])); + } +} + +void recSQ() +{ + // Flush rationale: see recLoad. FLUSH_CONSTANT_REGS only. + // Rt and Rs are read from memory after the flush. iFlushCall has + // freed all NEON unconditionally (writeback-on-dirty), so any EE + // GPR allocated in NEON is in memory. EE GPRs in arm64gprs[] are + // not written back by FLUSH_CONSTANT_REGS — relies on allocator + // preferring NEON for full-128-bit guest GPRs. + iFlushCall(FLUSH_CONSTANT_REGS); + armLoadEERegPtr(a64::q0, &cpuRegs.GPR.r[_Rt_].UQ); + + armLoadEERegPtr(a64::w9, &cpuRegs.GPR.r[_Rs_].UL[0]); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + armAsm->And(a64::w9, a64::w9, (u32)~0xF); + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + if (useFastmem) + vtlbFastmemWrite128(9); + else + vtlbSoftmemWrite128(9); +} + +// ===================================================================================================== +// LWC1 / SWC1 — FPU 32-bit load/store +// LWC1: fpr[ft] = mem32(rs + imm) +// SWC1: mem32(rs + imm) = fpr[ft] +// ===================================================================================================== + +void recLWC1() +{ + // On the fast path a single inline LDR off RFASTMEMBASE + backpatch, + // no iFlushCall and no vtlb C call. The result lands in w0 (a plain GPR + // rather than an allocated FPR host reg). Softmem stays as the + // faulting-PC fallback. + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + // Compute address into w9 from live registers. + recComputeAddr(); + + if (useFastmem) + { + vtlbFastmemRead(9, 0, 32, false); + } + else + { + iFlushCall(FLUSH_CONSTANT_REGS); + vtlbSoftmemRead(9, 32, false); + } + + // fpr[ft] in memory is about to be overwritten; the allocator's slot + // (if any) is now stale and must not flush back over the write. + _deleteFPtoNEONreg(_Rt_, DELETE_REG_FREE_NO_WRITEBACK); + // Store to fpuRegs.fpr[ft] + armStoreEERegPtr(a64::w0, &fpuRegs.fpr[_Rt_].UL); +} + +void recSWC1() +{ + // fpr[ft] may be live in NEON with MODE_WRITE-only state; flush dirty + // content to memory and drop the slot before reading via armLoad. + _deleteFPtoNEONreg(_Rt_, DELETE_REG_FLUSH_AND_FREE); + + // Load FPU register value into w10 + armLoadEERegPtr(a64::w10, &fpuRegs.fpr[_Rt_].UL); + + // Inline STR off RFASTMEMBASE + backpatch on the fast path, softmem + // fallback otherwise. + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + // Compute address from live registers. + recComputeAddr(); + + if (useFastmem) + { + vtlbFastmemWrite(9, 10, 32); + } + else + { + iFlushCall(FLUSH_CONSTANT_REGS); + vtlbSoftmemWrite(9, 10, 32); + } +} + +// ===================================================================================================== +// Unaligned load/store (LWL/LWR/LDL/LDR/SWL/SWR/SDL/SDR) — inline fastmem +// read-modify-write codegen. Mirrors x86 master's REC_LOADS/REC_STORES paths: +// the inline path needs only FLUSH_CONSTANT_REGS plus fastmem accesses +// (no C call on the fast path), avoiding a full FLUSH_INTERPRETER eviction. +// ===================================================================================================== + +// Inline LWL/LWR codegen. Mirrors x86 recLWL/recLWR (ix86-32/iR5900LoadStore.cpp). +// +// addr = Rs + imm +// shift8 = (addr & 3) * 8 // kept in a callee-saved temp across the read +// aligned = addr & ~3 +// loaded = mem32(aligned) +// +// LWL: Rt = sign_ext_32_to_64( (Rt & (0xffffff >> shift8)) | (loaded << (24 - shift8)) ) +// LWR: if shift8 == 0: Rt = sign_ext_32_to_64(loaded) +// else : Rt[31:0] = (Rt[31:0] & (0xffffff00 << (24 - shift8))) | (loaded >> shift8) +// Rt[63:32] preserved (see interpreter LWL/LWR in R5900OpcodeImpl.cpp) +// +// Uses fastmem when available (the backpatch thunk spills the live temp around +// its slow-path C call); softmem path's slow-path C call obeys AAPCS so the +// callee-saved temp survives there too. +static void recUnalignedWord(bool is_lwl) +{ + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + // Compute Rs+imm in w9. Mirrors recLoad: load Rs first, then flush, then add imm. + _eeMoveGPRtoR(a64::w9, _Rs_); + iFlushCall(FLUSH_CONSTANT_REGS); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + + // shift8 lives in a callee-saved temp so it survives vtlb's slow-path + // C call (fastmem backpatch thunk OR softmem slow path). + const int shift8 = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + + armAsm->And(armWRegister(shift8), a64::w9, 3); + armAsm->Lsl(armWRegister(shift8), armWRegister(shift8), 3); + armAsm->And(a64::w9, a64::w9, ~3u); + + // 32-bit aligned read; result in w0. + if (useFastmem) + vtlbFastmemRead(9, 0, 32, false); + else + vtlbSoftmemRead(9, 32, false); + + if (!_Rt_) + { + _freeArm64GPR(shift8); + return; + } + + const int rt = _allocArm64GPR(ARM64TYPE_GPR, _Rt_, MODE_READ | MODE_WRITE); + + if (is_lwl) + { + // mask = 0xffffff >> shift8 + armAsm->Mov(RWSCRATCH, 0xffffff); + armAsm->Lsr(RWSCRATCH, RWSCRATCH, armWRegister(shift8)); + armAsm->And(armWRegister(rt), armWRegister(rt), RWSCRATCH); + + // shifted_loaded = loaded << (24 - shift8); reuse RWSCRATCH as shift amount. + armAsm->Mov(RWSCRATCH, 24); + armAsm->Sub(RWSCRATCH, RWSCRATCH, armWRegister(shift8)); + armAsm->Lsl(a64::w0, a64::w0, RWSCRATCH); + + // Merge and sign-extend the 32-bit result into the 64-bit guest reg. + armAsm->Orr(armWRegister(rt), armWRegister(rt), a64::w0); + armAsm->Sxtw(armXRegister(rt), armWRegister(rt)); + } + else + { + a64::Label nomask, done; + armAsm->Cbz(armWRegister(shift8), &nomask); + + // mask = 0xffffff00 << (24 - shift8); held in RSCRATCHADDR.W() since RWSCRATCH carries the shift amount. + armAsm->Mov(RWSCRATCH, 24); + armAsm->Sub(RWSCRATCH, RWSCRATCH, armWRegister(shift8)); + armAsm->Mov(RSCRATCHADDR.W(), 0xffffff00u); + armAsm->Lsl(RSCRATCHADDR.W(), RSCRATCHADDR.W(), RWSCRATCH); + armAsm->And(RWSCRATCH, armWRegister(rt), RSCRATCHADDR.W()); + + armAsm->Lsr(a64::w0, a64::w0, armWRegister(shift8)); + armAsm->Orr(a64::w0, a64::w0, RWSCRATCH); + + // Per interp: when shift8 != 0, only Rt[31:0] changes; upper 32 preserved. + armAsm->Bfi(armXRegister(rt), a64::x0, 0, 32); + armAsm->B(&done); + + // shift8 == 0 (aligned): straight sign-extend, full 64-bit overwrite. + armAsm->Bind(&nomask); + armAsm->Sxtw(armXRegister(rt), a64::w0); + + armAsm->Bind(&done); + } + + _freeArm64GPR(shift8); +} + +// Inline SWL/SWR codegen (32-bit unaligned store, read-modify-write). Mirrors +// x86 recSWL/recSWR (ix86-32/iR5900LoadStore.cpp) and interp R5900OpcodeImpl.cpp SWL/SWR. +// +// addr = Rs + imm ; shift8 = (addr & 3) * 8 ; aligned = addr & ~3 +// mem = mem32(aligned) +// SWL: mem32(aligned) = (Rt >> (24 - shift8)) | (mem & (0xffffff00 << shift8)) +// SWR: mem32(aligned) = (Rt << shift8 ) | (mem & (0x00ffffff >> (24 - shift8))) +// +// aligned and shift8 live in callee-saved temps so they survive the read's +// slow-path C call (fastmem backpatch thunk OR softmem slow path). The Rt load +// and merged value never cross a call. The implementation always performs a +// full RMW (no shift8==24 full-overwrite skip): the aligned word is the same +// one written, so reading it is always safe, and the general shifts already +// collapse to "store Rt" at that alignment. +static void recUnalignedStoreWord(bool is_swl) +{ + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + _eeMoveGPRtoR(a64::w9, _Rs_); + iFlushCall(FLUSH_CONSTANT_REGS); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + + const int addrTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + const int shiftTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + + armAsm->And(armWRegister(addrTemp), a64::w9, ~3u); // aligned addr + armAsm->And(armWRegister(shiftTemp), a64::w9, 3); + armAsm->Lsl(armWRegister(shiftTemp), armWRegister(shiftTemp), 3); // shift8 + + // 32-bit aligned read; mem -> w0. + armAsm->Mov(a64::w9, armWRegister(addrTemp)); + if (useFastmem) + vtlbFastmemRead(9, 0, 32, false); + else + vtlbSoftmemRead(9, 32, false); + + // Load Rt after the read (never crosses a call). Handles Rt==0 -> 0. + _eeMoveGPRtoR(a64::w1, _Rt_); + + if (is_swl) + { + armAsm->Mov(RWSCRATCH, 0xffffff00u); + armAsm->Lsl(RWSCRATCH, RWSCRATCH, armWRegister(shiftTemp)); // 0xffffff00 << shift8 + armAsm->And(a64::w0, a64::w0, RWSCRATCH); // mem & mask + armAsm->Mov(RWSCRATCH, 24); + armAsm->Sub(RWSCRATCH, RWSCRATCH, armWRegister(shiftTemp)); // 24 - shift8 + armAsm->Lsr(a64::w1, a64::w1, RWSCRATCH); // Rt >> (24 - shift8) + } + else + { + armAsm->Mov(RWSCRATCH, 24); + armAsm->Sub(RWSCRATCH, RWSCRATCH, armWRegister(shiftTemp)); // 24 - shift8 + armAsm->Mov(RSCRATCHADDR.W(), 0x00ffffffu); + armAsm->Lsr(RSCRATCHADDR.W(), RSCRATCHADDR.W(), RWSCRATCH); // 0x00ffffff >> (24 - shift8) + armAsm->And(a64::w0, a64::w0, RSCRATCHADDR.W()); // mem & mask + armAsm->Lsl(a64::w1, a64::w1, armWRegister(shiftTemp)); // Rt << shift8 + } + + armAsm->Orr(a64::w0, a64::w0, a64::w1); // merged -> w0 + + armAsm->Mov(a64::w9, armWRegister(addrTemp)); + if (useFastmem) + vtlbFastmemWrite(9, 0, 32); + else + vtlbSoftmemWrite(9, 0, 32); + + _freeArm64GPR(shiftTemp); + _freeArm64GPR(addrTemp); +} + +// Inline LDL/LDR codegen (64-bit unaligned load). Mirrors x86 recLDL/recLDR and +// interp R5900OpcodeImpl.cpp LDL/LDR. +// +// addr = Rs + imm ; s = addr & 7 ; shift8 = s * 8 ; aligned = addr & ~7 +// mem = mem64(aligned) +// LDL: Rt = (Rt & (~0 >> (shift8 + 8))) | (mem << (56 - shift8)) [s==7: Rt = mem] +// LDR: Rt = (Rt & (~0 << (64 - shift8))) | (mem >> shift8) [s==0: Rt = mem] +// +// The degenerate alignment (LDL s==7 / LDR s==0) needs a shift by 64, which the +// AArch64 variable-shift uses mod-64 — so those map to a straight Rt = mem and +// are branched out, exactly like x86's CMOVE/skip. +static void recUnalignedLoadDouble(bool is_ldl) +{ + if (!_Rt_) + return; + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + _eeMoveGPRtoR(a64::w9, _Rs_); + iFlushCall(FLUSH_CONSTANT_REGS); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + + // mem must be parked in a callee-saved temp before the Rt alloc: under + // register pressure the alloc can spill a guest reg or land Rt in x0, + // clobbering x0 between the fastmem read and the merge. s = addr & 7 also + // lives in a callee-saved temp. The store path parks all its operands in + // callee-saved temps for the same reason. + const int sTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + const int memTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + armAsm->And(armWRegister(sTemp), a64::w9, 7); + armAsm->And(a64::w9, a64::w9, ~7u); // aligned + + if (useFastmem) + vtlbFastmemRead(9, 0, 64, false); // mem -> x0 + else + vtlbSoftmemRead(9, 64, false); + + armAsm->Mov(armXRegister(memTemp), a64::x0); // park mem (x0 unsafe across Rt alloc) + + const int rt = _allocArm64GPR(ARM64TYPE_GPR, _Rt_, MODE_READ | MODE_WRITE); + + a64::Label special, done; + armAsm->Cmp(armWRegister(sTemp), is_ldl ? 7 : 0); + armAsm->B(&special, a64::eq); + + armAsm->Lsl(RWSCRATCH, armWRegister(sTemp), 3); // x8 = shift8 (<=56) + + if (is_ldl) + { + // value: mem << (56 - shift8) + armAsm->Mov(RSCRATCHADDR, 56); + armAsm->Sub(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); // 56 - shift8 + armAsm->Lsl(armXRegister(memTemp), armXRegister(memTemp), RSCRATCHADDR); + // mask: Rt & (~0 >> (shift8 + 8)) + armAsm->Add(RXSCRATCH, RXSCRATCH, 8); // shift8 + 8 + armAsm->Mov(RSCRATCHADDR, UINT64_C(0xFFFFFFFFFFFFFFFF)); + armAsm->Lsr(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); + armAsm->And(armXRegister(rt), armXRegister(rt), RSCRATCHADDR); + } + else + { + // mask amount = 64 - shift8 + armAsm->Mov(RSCRATCHADDR, 64); + armAsm->Sub(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); // 64 - shift8 + // value: mem >> shift8 + armAsm->Lsr(armXRegister(memTemp), armXRegister(memTemp), RXSCRATCH); + // mask: Rt & (~0 << (64 - shift8)) + armAsm->Mov(RXSCRATCH, UINT64_C(0xFFFFFFFFFFFFFFFF)); + armAsm->Lsl(RXSCRATCH, RXSCRATCH, RSCRATCHADDR); + armAsm->And(armXRegister(rt), armXRegister(rt), RXSCRATCH); + } + + armAsm->Orr(armXRegister(rt), armXRegister(rt), armXRegister(memTemp)); + armAsm->B(&done); + + armAsm->Bind(&special); + armAsm->Mov(armXRegister(rt), armXRegister(memTemp)); // Rt = mem + + armAsm->Bind(&done); + _freeArm64GPR(memTemp); + _freeArm64GPR(sTemp); +} + +// Inline SDL/SDR codegen (64-bit unaligned store, read-modify-write). Mirrors +// x86 recSDL/recSDR and interp R5900OpcodeImpl.cpp SDL/SDR. +// +// addr = Rs + imm ; s = addr & 7 ; shift8 = s * 8 ; aligned = addr & ~7 +// mem = mem64(aligned) +// SDL: mem64(aligned) = (Rt >> (56 - shift8)) | (mem & (~0 << (shift8 + 8))) [s==7: store Rt] +// SDR: mem64(aligned) = (Rt << shift8 ) | (mem & (~0 >> (64 - shift8))) [s==0: store Rt] +// +// aligned, s and Rt all live in callee-saved temps across the read. The +// degenerate alignment (SDL s==7 / SDR s==0) stores Rt whole and skips the read. +static void recUnalignedStoreDouble(bool is_sdl) +{ + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + + _eeMoveGPRtoR(a64::w9, _Rs_); + iFlushCall(FLUSH_CONSTANT_REGS); + if (_Imm_ != 0) + armAsm->Add(a64::w9, a64::w9, _Imm_); + + const int addrTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + const int sTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + const int valTemp = _allocArm64GPR(ARM64TYPE_TEMP, 0, MODE_CALLEESAVED); + + armAsm->And(armWRegister(addrTemp), a64::w9, ~7u); // aligned + armAsm->And(armWRegister(sTemp), a64::w9, 7); // s + _eeMoveGPRtoR(armXRegister(valTemp), _Rt_); // Rt (64-bit), handles Rt==0 + + a64::Label special, merged; + armAsm->Cmp(armWRegister(sTemp), is_sdl ? 7 : 0); + armAsm->B(&special, a64::eq); + + // General path: read aligned word, merge with Rt. + armAsm->Mov(a64::w9, armWRegister(addrTemp)); + if (useFastmem) + vtlbFastmemRead(9, 0, 64, false); // mem -> x0 + else + vtlbSoftmemRead(9, 64, false); + + armAsm->Lsl(RWSCRATCH, armWRegister(sTemp), 3); // x8 = shift8 + + if (is_sdl) + { + // Rt >> (56 - shift8) + armAsm->Mov(RSCRATCHADDR, 56); + armAsm->Sub(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); // 56 - shift8 + armAsm->Lsr(armXRegister(valTemp), armXRegister(valTemp), RSCRATCHADDR); + // mem & (~0 << (shift8 + 8)) + armAsm->Add(RXSCRATCH, RXSCRATCH, 8); // shift8 + 8 + armAsm->Mov(RSCRATCHADDR, UINT64_C(0xFFFFFFFFFFFFFFFF)); + armAsm->Lsl(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); + armAsm->And(a64::x0, a64::x0, RSCRATCHADDR); + } + else + { + // Rt << shift8 + armAsm->Lsl(armXRegister(valTemp), armXRegister(valTemp), RXSCRATCH); + // mem & (~0 >> (64 - shift8)) + armAsm->Mov(RSCRATCHADDR, 64); + armAsm->Sub(RSCRATCHADDR, RSCRATCHADDR, RXSCRATCH); // 64 - shift8 + armAsm->Mov(RXSCRATCH, UINT64_C(0xFFFFFFFFFFFFFFFF)); + armAsm->Lsr(RXSCRATCH, RXSCRATCH, RSCRATCHADDR); + armAsm->And(a64::x0, a64::x0, RXSCRATCH); + } + + armAsm->Orr(a64::x0, a64::x0, armXRegister(valTemp)); // merged -> x0 + armAsm->B(&merged); + + armAsm->Bind(&special); + armAsm->Mov(a64::x0, armXRegister(valTemp)); // store Rt whole + + armAsm->Bind(&merged); + armAsm->Mov(a64::w9, armWRegister(addrTemp)); + if (useFastmem) + vtlbFastmemWrite(9, 0, 64); + else + vtlbSoftmemWrite(9, 0, 64); + + _freeArm64GPR(valTemp); + _freeArm64GPR(sTemp); + _freeArm64GPR(addrTemp); +} + +void recLWL() { recUnalignedWord(true); } +void recLWR() { recUnalignedWord(false); } +void recLDL() { recUnalignedLoadDouble(true); } +void recLDR() { recUnalignedLoadDouble(false); } +void recSWL() { recUnalignedStoreWord(true); } +void recSWR() { recUnalignedStoreWord(false); } +void recSDL() { recUnalignedStoreDouble(true); } +void recSDR() { recUnalignedStoreDouble(false); } +// ===================================================================================================== +// LQC2 / SQC2 — 128-bit COP2 (VU0) register load/store +// LQC2: VU0.VF[ft] = mem128((rs + imm) & ~0xF) +// SQC2: mem128((rs + imm) & ~0xF) = VU0.VF[ft] +// Same as LQ/SQ but target is VU0.VF[ft] instead of cpuRegs.GPR[rt]. +// Requires vu0Sync() before access. +// ===================================================================================================== + +void recLQC2() +{ + // Sync VU0 before COP2 register access. Gated on EEINST analysis — + // no emit at all when the analysis says no sync is needed (the common + // case for quad-load-heavy code like vertex streaming). Mirrors x86 + // recLQC2 (mVUSyncVU0 / mVUFinishVU0 gating on EEINST_COP2_SYNC_VU0 / + // EEINST_COP2_FINISH_VU0). The helper handles iFlushCall, RECCYCLE + // save/reload, runtime VPU_STAT check, and cycle accounting. + cop2EmitConditionalSync(false, _vu0FinishMicro); + + // addr = (rs + imm) & ~0xF + recComputeAddr(); + armAsm->And(a64::w9, a64::w9, (u32)~0xF); + + // Match recLQ/recSQ: spill constant tracking before the softmem C call + // can fire (vtlb_memRead128 may dispatch through an MMIO handler that + // reads cpuRegs). Redundant when cop2EmitConditionalSync already + // emitted FLUSH_INTERPRETER, harmless otherwise. + iFlushCall(FLUSH_CONSTANT_REGS); + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + if (useFastmem) + vtlbFastmemRead128(9); + else + vtlbSoftmemRead128(9); + + // Store 128-bit result to VU0.VF[rt] (COP2 ft field = rt field) + if (_Rt_) + { + armMoveAddressToReg(RSCRATCHADDR, &VU0.VF[_Rt_].UQ); + armAsm->Str(a64::q0, a64::MemOperand(RSCRATCHADDR)); + } +} + +void recSQC2() +{ + // EEINST-gated VU0 sync — see recLQC2 above. + cop2EmitConditionalSync(false, _vu0FinishMicro); + + // addr = (rs + imm) & ~0xF — allocator-aware (rs may still be live + // in a host reg if the sync above emitted nothing). + recComputeAddr(); + armAsm->And(a64::w9, a64::w9, (u32)~0xF); + + // Flush before the q0 load. iFlushCall unconditionally evicts all NEON + // (iR5900-arm64.cpp:765-769) and writes back any dirty cache; after + // this point q0 is allocator-detached and safe to touch directly. + iFlushCall(FLUSH_CONSTANT_REGS); + + // Load 128-bit VU0.VF[rt] into q0 (COP2 ft field = rt field). MUST be + // after iFlushCall: q0 may otherwise be tracked by the EE NEON allocator + // as caching a live FPREG, in which case this load stomps q0's runtime + // contents while leaving arm64neon[0] still marked dirty. The next + // allocator flush would then write back the stomped value (VU0.VF[rt]) + // to the FPREG's memory slot, corrupting the FPREG. + armLoadPtr(a64::q0, &VU0.VF[_Rt_].UQ); + + const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc); + if (useFastmem) + vtlbFastmemWrite128(9); + else + vtlbSoftmemWrite128(9); +} + +} // namespace OpcodeImpl +} // namespace Dynarec +} // namespace R5900 diff --git a/pcsx2/x86/iR5900Analysis.cpp b/pcsx2/x86/iR5900Analysis.cpp index ea33fa90bb..a17322a3ba 100644 --- a/pcsx2/x86/iR5900Analysis.cpp +++ b/pcsx2/x86/iR5900Analysis.cpp @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: 2002-2026 PCSX2 Dev Team // SPDX-License-Identifier: GPL-3.0+ +#ifdef ARCH_ARM64 +#include "arm64/iR5900Analysis.h" +#else #include "iR5900Analysis.h" +#endif #include "Memory.h" #include "DebugTools/Debug.h"