Merge pull request #2814 from Sonicadvance1/aarch64_faster_fastmem

[AArch64] Banish slowmem operations to farcode.

Source/Core/Common/BitSet.h

@@ -155,6 +155,8 @@ public:
 	const Ref operator[](size_t bit) const { return (*const_cast<BitSet*>(this))[bit]; }
 	bool operator==(BitSet other) const { return m_val == other.m_val; }
 	bool operator!=(BitSet other) const { return m_val != other.m_val; }
+	bool operator<(BitSet other) const { return m_val < other.m_val; }
+	bool operator>(BitSet other) const { return m_val > other.m_val; }
 	BitSet operator|(BitSet other) const { return BitSet(m_val | other.m_val); }
 	BitSet operator&(BitSet other) const { return BitSet(m_val & other.m_val); }
 	BitSet operator^(BitSet other) const { return BitSet(m_val ^ other.m_val); }

Source/Core/Common/CodeBlock.h

@@ -22,9 +22,19 @@ private:
 protected:
 	u8 *region;
 	size_t region_size;
+	size_t parent_region_size;
+
+	bool m_has_child;
+	bool m_is_child;
+	CodeBlock* m_child;
 
 public:
-	CodeBlock() : region(nullptr), region_size(0) {}
+	CodeBlock()
+		: region(nullptr), region_size(0), parent_region_size(0),
+		  m_has_child(false), m_is_child(false), m_child(nullptr)
+	{
+	}
+
 	virtual ~CodeBlock() { if (region) FreeCodeSpace(); }
 
 	// Call this before you generate any code.
@@ -49,6 +59,12 @@ public:
 		FreeMemoryPages(region, region_size);
 		region = nullptr;
 		region_size = 0;
+		parent_region_size = 0;
+		if (m_has_child)
+		{
+			m_child->region = nullptr;
+			m_child->region_size = 0;
+		}
 	}
 
 	bool IsInSpace(u8 *ptr) const
@@ -70,7 +86,7 @@ public:
 
 	size_t GetSpaceLeft() const
 	{
-		return region_size - (T::GetCodePtr() - region);
+		return (m_has_child ? parent_region_size : region_size) - (T::GetCodePtr() - region);
 	}
 
 	bool IsAlmostFull() const
@@ -78,5 +94,16 @@ public:
 		// This should be bigger than the biggest block ever.
 		return GetSpaceLeft() < 0x10000;
 	}
+	void AddChildCodeSpace(CodeBlock* child, size_t size)
+	{
+		_assert_msg_(DYNA_REC, !m_has_child, "Already have a child! Can't have another!");
+		m_child = child;
+		m_has_child = true;
+		m_child->m_is_child = true;
+		u8* child_region = region + region_size - size;
+		m_child->region = child_region;
+		m_child->region_size = size;
+		m_child->ResetCodePtr();
+		parent_region_size = region_size - size;
+	}
 };
-

Source/Core/Core/PowerPC/JitArm64/Jit.cpp

@@ -14,10 +14,13 @@
 
 using namespace Arm64Gen;
 
+static const int AARCH64_FARCODE_SIZE = 1024 * 1024 * 16;
+
 void JitArm64::Init()
 {
-	AllocCodeSpace(CODE_SIZE);
-	farcode.Init(SConfig::GetInstance().bMMU ? FARCODE_SIZE_MMU : FARCODE_SIZE);
+	size_t child_code_size = SConfig::GetInstance().bMMU ? FARCODE_SIZE_MMU : AARCH64_FARCODE_SIZE;
+	AllocCodeSpace(CODE_SIZE + child_code_size);
+	AddChildCodeSpace(&farcode, child_code_size);
 	jo.enableBlocklink = true;
 	jo.optimizeGatherPipe = true;
 	UpdateMemoryOptions();
@@ -30,22 +33,23 @@ void JitArm64::Init()
 	code_block.m_stats = &js.st;
 	code_block.m_gpa = &js.gpa;
 	code_block.m_fpa = &js.fpa;
-	InitBackpatch();
 	analyzer.SetOption(PPCAnalyst::PPCAnalyzer::OPTION_CONDITIONAL_CONTINUE);
 }
 
 void JitArm64::ClearCache()
 {
+	m_fault_to_handler.clear();
+	m_handler_to_loc.clear();
+
+	blocks.Clear();
 	ClearCodeSpace();
 	farcode.ClearCodeSpace();
-	blocks.Clear();
 	UpdateMemoryOptions();
 }
 
 void JitArm64::Shutdown()
 {
 	FreeCodeSpace();
-	farcode.Shutdown();
 	blocks.Shutdown();
 	asm_routines.Shutdown();
 }

Source/Core/Core/PowerPC/JitArm64/Jit.h

@@ -18,15 +18,6 @@
 
 #define PPCSTATE_OFF(elem) (offsetof(PowerPC::PowerPCState, elem))
 
-// A place to throw blocks of code we don't want polluting the cache, e.g. rarely taken
-// exception branches.
-class FarCodeCacheArm64 : public Arm64Gen::ARM64CodeBlock
-{
-public:
-	void Init(int size) { AllocCodeSpace(size); }
-	void Shutdown() { FreeCodeSpace(); }
-};
-
 // Some asserts to make sure we will be able to load everything
 static_assert(PPCSTATE_OFF(spr[1023]) <= 16380, "LDR(32bit) can't reach the last SPR");
 static_assert((PPCSTATE_OFF(ps[0][0]) % 8) == 0, "LDR(64bit VFP) requires FPRs to be 8 byte aligned");
@@ -185,6 +176,40 @@ public:
 	void psq_st(UGeckoInstruction inst);
 
 private:
+
+	struct SlowmemHandler
+	{
+		ARM64Reg dest_reg;
+		ARM64Reg addr_reg;
+		BitSet32 gprs;
+		BitSet32 fprs;
+		u32 flags;
+		bool operator< (const SlowmemHandler& rhs) const
+		{
+			if (dest_reg < rhs.dest_reg) return true;
+			if (dest_reg > rhs.dest_reg) return false;
+			if (addr_reg < rhs.addr_reg) return true;
+			if (addr_reg > rhs.addr_reg) return false;
+			if (gprs < rhs.gprs) return true;
+			if (gprs > rhs.gprs) return false;
+			if (fprs < rhs.fprs) return true;
+			if (fprs > rhs.fprs) return false;
+			if (flags < rhs.flags) return true;
+			if (flags > rhs.flags) return false;
+
+			return false;
+		}
+	};
+
+	struct FastmemArea
+	{
+		u32 length;
+		const u8* slowmem_code;
+	};
+
+	// <Fastmem fault location, slowmem handler location>
+	std::map<const u8*, FastmemArea> m_fault_to_handler;
+	std::map<SlowmemHandler, const u8*> m_handler_to_loc;
 	Arm64GPRCache gpr;
 	Arm64FPRCache fpr;
 
@@ -195,7 +220,7 @@ private:
 
 	ARM64FloatEmitter m_float_emit;
 
-	FarCodeCacheArm64 farcode;
+	Arm64Gen::ARM64CodeBlock farcode;
 	u8* nearcode; // Backed up when we switch to far code.
 
 	// Simple functions to switch between near and far code emitting
@@ -214,13 +239,11 @@ private:
 	// Dump a memory range of code
 	void DumpCode(const u8* start, const u8* end);
 
-	// The key is the backpatch flags
-	std::map<u32, BackPatchInfo> m_backpatch_info;
-
 	// Backpatching routines
 	bool DisasmLoadStore(const u8* ptr, u32* flags, Arm64Gen::ARM64Reg* reg);
-	void InitBackpatch();
-	u32 EmitBackpatchRoutine(ARM64XEmitter* emit, u32 flags, bool fastmem, bool do_padding, Arm64Gen::ARM64Reg RS, Arm64Gen::ARM64Reg addr);
+	void EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode,
+		Arm64Gen::ARM64Reg RS, Arm64Gen::ARM64Reg addr,
+		BitSet32 gprs_to_push = BitSet32(0), BitSet32 fprs_to_push = BitSet32(0));
 	// Loadstore routines
 	void SafeLoadToReg(u32 dest, s32 addr, s32 offsetReg, u32 flags, s32 offset, bool update);
 	void SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s32 offset);

Source/Core/Core/PowerPC/JitArm64/JitArm64_LoadStore.cpp

@@ -37,7 +37,6 @@ void JitArm64::SafeLoadToReg(u32 dest, s32 addr, s32 offsetReg, u32 flags, s32 o
 	BitSet32 regs_in_use = gpr.GetCallerSavedUsed();
 	BitSet32 fprs_in_use = fpr.GetCallerSavedUsed();
 	regs_in_use[W0] = 0;
-	regs_in_use[W30] = 0;
 	regs_in_use[dest_reg] = 0;
 
 	ARM64Reg addr_reg = W0;
@@ -148,7 +147,7 @@ void JitArm64::SafeLoadToReg(u32 dest, s32 addr, s32 offsetReg, u32 flags, s32 o
 
 	if (is_immediate && PowerPC::IsOptimizableRAMAddress(imm_addr))
 	{
-		EmitBackpatchRoutine(this, flags, true, false, dest_reg, XA);
+		EmitBackpatchRoutine(flags, true, false, dest_reg, XA, BitSet32(0), BitSet32(0));
 	}
 	else if (mmio_address)
 	{
@@ -158,16 +157,11 @@ void JitArm64::SafeLoadToReg(u32 dest, s32 addr, s32 offsetReg, u32 flags, s32 o
 	}
 	else
 	{
-		// Has a chance of being backpatched which will destroy our state
-		// push and pop everything in this instance
-		ABI_PushRegisters(regs_in_use);
-		m_float_emit.ABI_PushRegisters(fprs_in_use, X30);
-		EmitBackpatchRoutine(this, flags,
+		EmitBackpatchRoutine(flags,
 			jo.fastmem,
 			jo.fastmem,
-			dest_reg, XA);
-		m_float_emit.ABI_PopRegisters(fprs_in_use, X30);
-		ABI_PopRegisters(regs_in_use);
+			dest_reg, XA,
+			regs_in_use, fprs_in_use);
 	}
 
 	gpr.Unlock(W0, W30);
@@ -192,7 +186,6 @@ void JitArm64::SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s
 	BitSet32 fprs_in_use = fpr.GetCallerSavedUsed();
 	regs_in_use[W0] = 0;
 	regs_in_use[W1] = 0;
-	regs_in_use[W30] = 0;
 
 	ARM64Reg addr_reg = W1;
 
@@ -296,7 +289,7 @@ void JitArm64::SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s
 	{
 		MOVI2R(XA, imm_addr);
 
-		EmitBackpatchRoutine(this, flags, true, false, RS, XA);
+		EmitBackpatchRoutine(flags, true, false, RS, XA, BitSet32(0), BitSet32(0));
 	}
 	else if (mmio_address && !(flags & BackPatchInfo::FLAG_REVERSE))
 	{
@@ -309,16 +302,12 @@ void JitArm64::SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s
 		if (is_immediate)
 			MOVI2R(XA, imm_addr);
 
-		// Has a chance of being backpatched which will destroy our state
-		// push and pop everything in this instance
-		ABI_PushRegisters(regs_in_use);
-		m_float_emit.ABI_PushRegisters(fprs_in_use, X30);
-		EmitBackpatchRoutine(this, flags,
+		EmitBackpatchRoutine(flags,
 			jo.fastmem,
 			jo.fastmem,
-			RS, XA);
-		m_float_emit.ABI_PopRegisters(fprs_in_use, X30);
-		ABI_PopRegisters(regs_in_use);
+			RS, XA,
+			regs_in_use,
+			fprs_in_use);
 	}
 
 	gpr.Unlock(W0, W1, W30);

Source/Core/Core/PowerPC/JitArm64/JitArm64_LoadStoreFloating.cpp

@@ -183,26 +183,20 @@ void JitArm64::lfXX(UGeckoInstruction inst)
 	BitSet32 regs_in_use = gpr.GetCallerSavedUsed();
 	BitSet32 fprs_in_use = fpr.GetCallerSavedUsed();
 	regs_in_use[W0] = 0;
-	regs_in_use[W30] = 0;
 	fprs_in_use[0] = 0; // Q0
 	fprs_in_use[VD - Q0] = 0;
 
 	if (is_immediate && PowerPC::IsOptimizableRAMAddress(imm_addr))
 	{
-		EmitBackpatchRoutine(this, flags, true, false, VD, XA);
+		EmitBackpatchRoutine(flags, true, false, VD, XA, BitSet32(0), BitSet32(0));
 	}
 	else
 	{
-		// Has a chance of being backpatched which will destroy our state
-		// push and pop everything in this instance
-		ABI_PushRegisters(regs_in_use);
-		m_float_emit.ABI_PushRegisters(fprs_in_use, X30);
-		EmitBackpatchRoutine(this, flags,
+		EmitBackpatchRoutine(flags,
 			jo.fastmem,
 			jo.fastmem,
-			VD, XA);
-		m_float_emit.ABI_PopRegisters(fprs_in_use, X30);
-		ABI_PopRegisters(regs_in_use);
+			VD, XA,
+			regs_in_use, fprs_in_use);
 	}
 
 	gpr.Unlock(W0, W30);
@@ -383,7 +377,6 @@ void JitArm64::stfXX(UGeckoInstruction inst)
 	BitSet32 fprs_in_use = fpr.GetCallerSavedUsed();
 	regs_in_use[W0] = 0;
 	regs_in_use[W1] = 0;
-	regs_in_use[W30] = 0;
 	fprs_in_use[0] = 0; // Q0
 
 	if (is_immediate)
@@ -437,29 +430,20 @@ void JitArm64::stfXX(UGeckoInstruction inst)
 		}
 		else if (PowerPC::IsOptimizableRAMAddress(imm_addr))
 		{
-			EmitBackpatchRoutine(this, flags, true, false, V0, XA);
+			EmitBackpatchRoutine(flags, true, false, V0, XA, BitSet32(0), BitSet32(0));
 		}
 		else
 		{
-			ABI_PushRegisters(regs_in_use);
-			m_float_emit.ABI_PushRegisters(fprs_in_use, X30);
-			EmitBackpatchRoutine(this, flags, false, false, V0, XA);
-			m_float_emit.ABI_PopRegisters(fprs_in_use, X30);
-			ABI_PopRegisters(regs_in_use);
+			EmitBackpatchRoutine(flags, false, false, V0, XA, regs_in_use, fprs_in_use);
 		}
 	}
 	else
 	{
-		// Has a chance of being backpatched which will destroy our state
-		// push and pop everything in this instance
-		ABI_PushRegisters(regs_in_use);
-		m_float_emit.ABI_PushRegisters(fprs_in_use, X30);
-		EmitBackpatchRoutine(this, flags,
+		EmitBackpatchRoutine(flags,
 			jo.fastmem,
 			jo.fastmem,
-			V0, XA);
-		m_float_emit.ABI_PopRegisters(fprs_in_use, X30);
-		ABI_PopRegisters(regs_in_use);
+			V0, XA,
+			regs_in_use, fprs_in_use);
 	}
 	gpr.Unlock(W0, W1, W30);
 	fpr.Unlock(Q0);

Source/Core/Core/PowerPC/JitArmCommon/BackPatch.h

@@ -34,8 +34,4 @@ struct BackPatchInfo
 			return 64;
 		return 0;
 	}
-
-	u32 m_fastmem_size;
-	u32 m_fastmem_trouble_inst_offset;
-	u32 m_slowmem_size;
 };