GE-10 arm64 EE: LWC1 fastmem loads straight into the allocated FPR slot

recLWC1 loaded into w0 and stored to fpr[ft] memory; the next FPU op
paid a Ldr back plus the store→load-forward stall — 2 mem ops per LWC1
on the hottest float-load idiom. x86-master and a reference ARM64 PS2
implementation both pass a dest-alloc callback into the vtlb read emitter
so the load targets the allocated FPR directly.

New vtlbFastmemReadFPR32: inline LDR S<n>, [RFASTMEMBASE, w9, UXTW]
with is_fpr backpatch info. The RecStubs thunk needed NOTHING — its
is_fpr load tail (Fmov S<n>, w0) and is-load-dest save-mask skip were
already in place; this is the first 32-bit user. ft allocates
MODE_WRITE (wholesale overwrite, recMTC1 rule) before the emit so the
dest rides the live-mask snapshot; the value stays resident for the
following arith and flushes at the next seam. Softmem/faulting-PC
fallback keeps the old w0+store shape (per-compile choice, no mixing).

Census (M2, UYA slot 02): 536,366 -> 533,880 EE-block insns with GE-12
(-0.46% for the pair).

Tests: EeRecLoadStore.Lwc1LoadsToResidentSlotThenArith,
Lwc1OverwritesStaleDirtyResidentDest (stale-dirty-slot writeback
hazard), Lwc1FaultPathLandsInResidentSlot (MMIO fault → thunk Fmov tail
+ dirty-NEON survival + faulting-PC witness). 1207/1207.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Brian Degenhardt
2026-07-13 15:55:15 -07:00
co-authored by Claude Fable 5
parent 7736240a8e
commit 0a40619523
2 changed files with 108 additions and 10 deletions
+38 -10
View File
@@ -309,6 +309,25 @@ static void vtlbFastmemRead128(int addr_wreg)
/*size_in_bits*/ 128, /*is_signed*/ false, /*is_load*/ true, /*is_fpr*/ true);
}
// Emit a single 32-bit fastmem load straight into an allocated FPR NEON slot
// (LDR S<n>, [RFASTMEMBASE, w_addr, UXTW]) — GE-10, the x86-master/4248
// dest-alloc-callback model. The backpatch thunk's is_fpr load tail already
// lands the slow-path C-handler result with Fmov S<n>, w0, and its save-mask
// logic skips the load dest (RecStubs.cpp).
static void vtlbFastmemReadFPR32(int addr_wreg, int dest_neonreg)
{
u32 gpr_bitmask, fpr_bitmask;
vtlbGetLiveRegisterMasks(gpr_bitmask, fpr_bitmask);
const u8* codeStart = armGetCurrentCodePointer();
armAsm->Ldr(a64::SRegister(dest_neonreg),
a64::MemOperand(RFASTMEMBASE, armWRegister(addr_wreg), a64::UXTW));
vtlb_AddLoadStoreInfo((uptr)codeStart, 4, pc, gpr_bitmask, fpr_bitmask,
static_cast<u8>(addr_wreg), static_cast<u8>(dest_neonreg),
/*size_in_bits*/ 32, /*is_signed*/ false, /*is_load*/ true, /*is_fpr*/ true);
}
// Emit a single 128-bit fastmem store (STR Q0, [RFASTMEMBASE, w_addr, UXTW]).
// Value in q0. Backpatch thunk extended in RecStubs.cpp.
static void vtlbFastmemWrite128(int addr_wreg)
@@ -903,9 +922,8 @@ void recSQ()
void recLWC1()
{
// On the fast path a single inline LDR off RFASTMEMBASE + backpatch,
// no iFlushCall and no vtlb C call. The result lands in w0 (a plain GPR
// rather than an allocated FPR host reg). Softmem stays as the
// faulting-PC fallback.
// no iFlushCall and no vtlb C call. Softmem stays as the faulting-PC
// fallback.
const bool useFastmem = CHECK_FASTMEM && !vtlb_IsFaultingPC(pc);
// Compute address into w9 from live registers.
@@ -913,19 +931,29 @@ void recLWC1()
if (useFastmem)
{
vtlbFastmemRead(9, 0, 32, false);
// GE-10: the inline LDR targets ft's allocated S register directly —
// no w0 bounce, no fpr-memory store, and ft stays resident for the
// following FPU op (the LWC1→arith idiom). MODE_WRITE only: LWC1
// overwrites fpr[ft] wholesale, so a prior value in the slot is dead
// (same rule as recMTC1); the dirty slot flushes at the next seam.
// Alloc BEFORE the fastmem emit: its eviction writeback must not
// split the LDR from its backpatch record, and the dest must be in
// the live-mask snapshot taken inside the emitter (where the thunk's
// is-load-dest skip excludes it from the save set).
const int ftreg = _allocFPtoNEONreg(_Rt_, MODE_WRITE);
vtlbFastmemReadFPR32(9, ftreg);
}
else
{
iFlushCall(FLUSH_CONSTANT_REGS);
vtlbSoftmemRead(9, 32, false);
}
// fpr[ft] in memory is about to be overwritten; the allocator's slot
// (if any) is now stale and must not flush back over the write.
_deleteFPtoNEONreg(_Rt_, DELETE_REG_FREE_NO_WRITEBACK);
// Store to fpuRegs.fpr[ft]
armStoreEERegPtr(a64::w0, &fpuRegs.fpr[_Rt_].UL);
// fpr[ft] in memory is about to be overwritten; the allocator's slot
// (if any) is now stale and must not flush back over the write.
_deleteFPtoNEONreg(_Rt_, DELETE_REG_FREE_NO_WRITEBACK);
// Store to fpuRegs.fpr[ft]
armStoreEERegPtr(a64::w0, &fpuRegs.fpr[_Rt_].UL);
}
}
void recSWC1()
@@ -1024,3 +1024,73 @@ TEST(EeRecLoadStore, Swc1ResidentValueStoresAndSurvivesFault)
faulted |= vtlb_IsFaultingPC(a);
EXPECT_TRUE(faulted) << "MMIO SWC1 did not take the fastmem-fault/backpatch path";
}
// ---- GE-10: LWC1 loads straight into the allocated FPR slot ------------------
TEST(EeRecLoadStore, Lwc1LoadsToResidentSlotThenArith)
{
// LWC1's fastmem LDR targets ft's allocated S register; the following
// ADD.S must consume the resident slot, and the block-end flush must land
// the loaded value in fpr memory for the post-state diff.
EeRecTestHarness h;
h.EnableCop1();
h.WriteU32(kScratch, 0x40400000u); // 3.0f
h.SetFprBits(3, 0x3F800000u); // 1.0f
h.SetGpr64(reg::a0, kScratch);
h.LoadProgram({
ee::LWC1(1, 0, reg::a0), // f1 = 3.0f
ee::ADD_S(2, 1, 3), // f2 = 4.0f
});
h.Run();
EXPECT_EQ(h.GetFprBitsInterp(1), 0x40400000u);
EXPECT_EQ(h.GetFprBitsInterp(2), 0x40800000u);
}
TEST(EeRecLoadStore, Lwc1OverwritesStaleDirtyResidentDest)
{
// f1 is left resident+dirty by ADD.S, then LWC1 overwrites it. The loaded
// value must win — for the next consumer AND the block-end flush (a
// stale-slot writeback over the load is the classic hazard the old shape
// avoided with DELETE_REG_FREE_NO_WRITEBACK).
EeRecTestHarness h;
h.EnableCop1();
h.WriteU32(kScratch, 0x41200000u); // 10.0f
h.SetFprBits(3, 0x3F800000u); // 1.0f
h.SetFprBits(4, 0x40000000u); // 2.0f
h.SetGpr64(reg::a0, kScratch);
h.LoadProgram({
ee::ADD_S(1, 3, 4), // f1 = 3.0f (resident, dirty)
ee::LWC1(1, 0, reg::a0), // f1 = 10.0f overwrites the slot
ee::ADD_S(2, 1, 3), // f2 = 11.0f
});
h.Run();
EXPECT_EQ(h.GetFprBitsInterp(1), 0x41200000u);
EXPECT_EQ(h.GetFprBitsInterp(2), 0x41300000u);
}
TEST(EeRecLoadStore, Lwc1FaultPathLandsInResidentSlot)
{
// MMIO LWC1 faults; the backpatch thunk's slow path must Fmov the
// C-handler result into the allocated S register (the is_fpr load tail),
// with OTHER live NEON state (dirty f6) saved/restored around the call.
EeRecTestHarness h;
h.EnableCop1();
h.SetFprBits(4, 0x40000000u); // 2.0f
h.SetFprBits(5, 0x3F800000u); // 1.0f
h.SetGpr64(reg::a1, 0x1000F000u); // INTC_STAT (harness state 0 → loads 0)
vtlb_ClearLoadStoreInfo();
h.LoadProgram({
ee::ADD_S(6, 4, 5), // f6 = 3.0f resident+dirty across the fault
ee::LWC1(1, 0, reg::a1), // MMIO load → fault → thunk → f1 = 0.0f
ee::ADD_S(7, 1, 4), // f7 = 2.0f from the faulted-in f1
ee::ADD_S(8, 6, 5), // f8 = 4.0f from the still-resident f6
});
h.Run();
EXPECT_EQ(h.GetFprBitsInterp(1), 0x00000000u);
EXPECT_EQ(h.GetFprBitsInterp(7), 0x40000000u);
EXPECT_EQ(h.GetFprBitsInterp(8), 0x40800000u);
bool faulted = false;
for (u32 a = RecompilerTestEnvironment::kProgramPc; a < RecompilerTestEnvironment::kProgramPc + 0x20; a += 4)
faulted |= vtlb_IsFaultingPC(a);
EXPECT_TRUE(faulted) << "MMIO LWC1 did not take the fastmem-fault/backpatch path";
}