Merge pull request #1025 from comex/blr-optimization

Opportunistically predict BLR destinations using RET.

Source/Core/Common/MemoryUtil.cpp

@@ -158,6 +158,25 @@ void FreeAlignedMemory(void* ptr)
 	}
 }
 
+void ReadProtectMemory(void* ptr, size_t size)
+{
+	bool error_occurred = false;
+
+#ifdef _WIN32
+	DWORD oldValue;
+	if (!VirtualProtect(ptr, size, PAGE_NOACCESS, &oldValue))
+		error_occurred = true;
+#else
+	int retval = mprotect(ptr, size, PROT_NONE);
+
+	if (retval != 0)
+		error_occurred = true;
+#endif
+
+	if (error_occurred)
+		PanicAlert("ReadProtectMemory failed!\n%s", GetLastErrorMsg());
+}
+
 void WriteProtectMemory(void* ptr, size_t size, bool allowExecute)
 {
 	bool error_occurred = false;

Source/Core/Common/MemoryUtil.h

@@ -12,8 +12,12 @@ void* AllocateMemoryPages(size_t size);
 void FreeMemoryPages(void* ptr, size_t size);
 void* AllocateAlignedMemory(size_t size,size_t alignment);
 void FreeAlignedMemory(void* ptr);
+void ReadProtectMemory(void* ptr, size_t size);
 void WriteProtectMemory(void* ptr, size_t size, bool executable = false);
 void UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute = false);
 std::string MemUsage();
 
+void GuardMemoryMake(void* ptr, size_t size);
+void GuardMemoryUnmake(void* ptr, size_t size);
+
 inline int GetPageSize() { return 4096; }

Source/Core/Common/x64Emitter.cpp

@@ -1766,6 +1766,8 @@ void XEmitter::ANDN(int bits, X64Reg regOp1, X64Reg regOp2, OpArg arg) {WriteBMI
 void XEmitter::LOCK()  { Write8(0xF0); }
 void XEmitter::REP()   { Write8(0xF3); }
 void XEmitter::REPNE() { Write8(0xF2); }
+void XEmitter::FSOverride() { Write8(0x64); }
+void XEmitter::GSOverride() { Write8(0x65); }
 
 void XEmitter::FWAIT()
 {

Source/Core/Common/x64Emitter.h

@@ -467,6 +467,8 @@ public:
 	void LOCK();
 	void REP();
 	void REPNE();
+	void FSOverride();
+	void GSOverride();
 
 	// x87
 	enum x87StatusWordBits {

Source/Core/Core/ArmMemTools.cpp

@@ -32,9 +32,9 @@ typedef struct ucontext {
 } ucontext_t;
 #endif
 
-void sigsegv_handler(int signal, siginfo_t *info, void *raw_context)
+static void sigsegv_handler(int sig, siginfo_t *info, void *raw_context)
 {
-	if (signal != SIGSEGV)
+	if (sig != SIGSEGV)
 	{
 		// We are not interested in other signals - handle it as usual.
 		return;
@@ -47,33 +47,18 @@ void sigsegv_handler(int signal, siginfo_t *info, void *raw_context)
 		return;
 	}
 
-
 	// Get all the information we can out of the context.
 	mcontext_t *ctx = &context->uc_mcontext;
 
-	void *fault_memory_ptr = (void*)ctx->arm_r10;
-	u8 *fault_instruction_ptr = (u8 *)ctx->arm_pc;
+	// comex says hello, and is most curious whether this is arm_r10 for a
+	// reason as opposed to si_addr like the x64MemTools.cpp version.  Is there
+	// even a need for this file to be architecture specific?
+	uintptr_t fault_memory_ptr = (uintptr_t)ctx->arm_r10;
 
-	if (!JitInterface::IsInCodeSpace(fault_instruction_ptr))
+	if (!JitInterface::HandleFault(fault_memory_ptr, ctx))
 	{
-		// Let's not prevent debugging.
-		return;
-	}
-
-	u64 bad_address = (u64)fault_memory_ptr;
-	u64 memspace_bottom = (u64)Memory::base;
-	if (bad_address < memspace_bottom)
-	{
-		PanicAlertT("Exception handler - access below memory space. %08llx%08llx",
-			bad_address >> 32, bad_address);
-	}
-
-	u32 em_address = (u32)(bad_address - memspace_bottom);
-
-	const u8 *new_rip = jit->BackPatch(fault_instruction_ptr, em_address, ctx);
-	if (new_rip)
-	{
-		ctx->arm_pc = (u32) new_rip;
+		// retry and crash
+		signal(SIGSEGV, SIG_DFL);
 	}
 }
 
@@ -86,4 +71,7 @@ void InstallExceptionHandler()
 	sigemptyset(&sa.sa_mask);
 	sigaction(SIGSEGV, &sa, nullptr);
 }
+
+void UninstallExceptionHandler() {}
+
 }  // namespace

Source/Core/Core/CMakeLists.txt

@@ -195,9 +195,10 @@ if(_M_X86)
 			PowerPC/Jit64/Jit_Paired.cpp
 			PowerPC/Jit64/JitRegCache.cpp
 			PowerPC/Jit64/Jit_SystemRegisters.cpp
-			PowerPC/JitCommon/JitBackpatch.cpp
 			PowerPC/JitCommon/JitAsmCommon.cpp
-			PowerPC/JitCommon/Jit_Util.cpp)
+			PowerPC/JitCommon/JitBackpatch.cpp
+			PowerPC/JitCommon/Jit_Util.cpp
+			PowerPC/JitCommon/TrampolineCache.cpp)
 elseif(_M_ARM_32)
 	set(SRCS ${SRCS}
 			ArmMemTools.cpp

Source/Core/Core/Core.cpp

@@ -277,6 +277,10 @@ static void CpuThread()
 	if (!_CoreParameter.bCPUThread)
 		g_video_backend->Video_Cleanup();
 
+	#if _M_X86_64 || _M_ARM_32
+	EMM::UninstallExceptionHandler();
+	#endif
+
 	return;
 }
 

Source/Core/Core/Core.vcxproj

@@ -229,6 +229,7 @@
     <ClCompile Include="PowerPC\JitCommon\JitBase.cpp" />
     <ClCompile Include="PowerPC\JitCommon\JitCache.cpp" />
     <ClCompile Include="PowerPC\JitCommon\Jit_Util.cpp" />
+    <ClCompile Include="PowerPC\JitCommon\TrampolineCache.cpp" />
     <ClCompile Include="PowerPC\JitInterface.cpp" />
     <ClCompile Include="PowerPC\PowerPC.cpp" />
     <ClCompile Include="PowerPC\PPCAnalyst.cpp" />
@@ -406,6 +407,7 @@
     <ClInclude Include="PowerPC\JitCommon\JitBase.h" />
     <ClInclude Include="PowerPC\JitCommon\JitCache.h" />
     <ClInclude Include="PowerPC\JitCommon\Jit_Util.h" />
+    <ClInclude Include="PowerPC\JitCommon\TrampolineCache.h" />
     <ClInclude Include="PowerPC\JitInterface.h" />
     <ClInclude Include="PowerPC\PowerPC.h" />
     <ClInclude Include="PowerPC\PPCAnalyst.h" />
@@ -464,4 +466,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

Source/Core/Core/Core.vcxproj.filters

@@ -640,6 +640,9 @@
     <ClCompile Include="PowerPC\JitCommon\JitCache.cpp">
       <Filter>PowerPC\JitCommon</Filter>
     </ClCompile>
+    <ClCompile Include="PowerPC\JitCommon\TrampolineCache.cpp">
+      <Filter>PowerPC\JitCommon</Filter>
+    </ClCompile>
     <ClCompile Include="PowerPC\Jit64IL\IR_X86.cpp">
       <Filter>PowerPC\JitIL</Filter>
     </ClCompile>
@@ -1182,6 +1185,9 @@
     <ClInclude Include="PowerPC\JitCommon\JitCache.h">
       <Filter>PowerPC\JitCommon</Filter>
     </ClInclude>
+    <ClInclude Include="PowerPC\JitCommon\TrampolineCache.h">
+      <Filter>PowerPC\JitCommon</Filter>
+    </ClInclude>
     <ClInclude Include="PowerPC\Jit64IL\JitIL.h">
       <Filter>PowerPC\JitIL</Filter>
     </ClInclude>
@@ -1204,4 +1210,4 @@
   <ItemGroup>
     <Text Include="CMakeLists.txt" />
   </ItemGroup>
-</Project>
+</Project>

Source/Core/Core/MemTools.h

@@ -11,4 +11,5 @@ namespace EMM
 {
 	typedef u32 EAddr;
 	void InstallExceptionHandler();
+	void UninstallExceptionHandler();
 }

Source/Core/Core/PowerPC/Jit64/Jit.cpp

@@ -95,6 +95,83 @@ using namespace PowerPC;
     and such, but it's currently limited to integer ops only. This can definitely be made better.
 */
 
+// The BLR optimization is nice, but it means that JITted code can overflow the
+// native stack by repeatedly running BL.  (The chance of this happening in any
+// retail game is close to 0, but correctness is correctness...) Also, the
+// overflow might not happen directly in the JITted code but in a C++ function
+// called from it, so we can't just adjust RSP in the case of a fault.
+// Instead, we have to have extra stack space preallocated under the fault
+// point which allows the code to continue, after wiping the JIT cache so we
+// can reset things at a safe point.  Once this condition trips, the
+// optimization is permanently disabled, under the assumption this will never
+// happen in practice.
+
+// On Unix, we just mark an appropriate region of the stack as PROT_NONE and
+// handle it the same way as fastmem faults.  It's safe to take a fault with a
+// bad RSP, because on Linux we can use sigaltstack and on OS X we're already
+// on a separate thread.
+
+// On Windows, the OS gets upset if RSP doesn't work, and I don't know any
+// equivalent of sigaltstack.  Windows supports guard pages which, when
+// accessed, immediately turn into regular pages but cause a trap... but
+// putting them in the path of RSP just leads to something (in the kernel?)
+// thinking a regular stack extension is required.  So this protection is not
+// supported on Windows yet...  We still use a separate stack for the sake of
+// simplicity.
+
+enum
+{
+	STACK_SIZE = 2 * 1024 * 1024,
+	SAFE_STACK_SIZE = 512 * 1024,
+	GUARD_SIZE = 0x10000, // two guards - bottom (permanent) and middle (see above)
+	GUARD_OFFSET = STACK_SIZE - SAFE_STACK_SIZE - GUARD_SIZE,
+};
+
+void Jit64::AllocStack()
+{
+#if defined(_WIN32)
+	m_stack = (u8*)AllocateMemoryPages(STACK_SIZE);
+	ReadProtectMemory(m_stack, GUARD_SIZE);
+	ReadProtectMemory(m_stack + GUARD_OFFSET, GUARD_SIZE);
+#endif
+}
+
+void Jit64::FreeStack()
+{
+#if defined(_WIN32)
+	if (m_stack)
+	{
+		FreeMemoryPages(m_stack, STACK_SIZE);
+		m_stack = NULL;
+	}
+#endif
+}
+
+bool Jit64::HandleFault(uintptr_t access_address, SContext* ctx)
+{
+	uintptr_t stack = (uintptr_t)m_stack, diff = access_address - stack;
+	// In the trap region?
+	if (stack && diff >= GUARD_OFFSET && diff < GUARD_OFFSET + GUARD_SIZE)
+	{
+		WARN_LOG(POWERPC, "BLR cache disabled due to excessive BL in the emulated program.");
+		m_enable_blr_optimization = false;
+		UnWriteProtectMemory(m_stack + GUARD_OFFSET, GUARD_SIZE);
+		// We're going to need to clear the whole cache to get rid of the bad
+		// CALLs, but we can't yet.  Fake the downcount so we're forced to the
+		// dispatcher (no block linking), and clear the cache so we're sent to
+		// Jit.  Yeah, it's kind of gross.
+		GetBlockCache()->InvalidateICache(0, 0xffffffff);
+		CoreTiming::ForceExceptionCheck(0);
+		m_clear_cache_asap = true;
+
+		return true;
+	}
+
+	return Jitx86Base::HandleFault(access_address, ctx);
+}
+
+
+
 void Jit64::Init()
 {
 	jo.optimizeStack = true;
@@ -130,8 +207,18 @@ void Jit64::Init()
 
 	trampolines.Init();
 	AllocCodeSpace(CODE_SIZE);
+
+	// BLR optimization has the same consequences as block linking, as well as
+	// depending on the fault handler to be safe in the event of excessive BL.
+	m_enable_blr_optimization = jo.enableBlocklink && SConfig::GetInstance().m_LocalCoreStartupParameter.bFastmem;
+	m_clear_cache_asap = false;
+
+	m_stack = nullptr;
+	if (m_enable_blr_optimization)
+		AllocStack();
+
 	blocks.Init();
-	asm_routines.Init();
+	asm_routines.Init(m_stack ? (m_stack + STACK_SIZE) : nullptr);
 
 	// important: do this *after* generating the global asm routines, because we can't use farcode in them.
 	// it'll crash because the farcode functions get cleared on JIT clears.
@@ -155,6 +242,7 @@ void Jit64::ClearCache()
 
 void Jit64::Shutdown()
 {
+	FreeStack();
 	FreeCodeSpace();
 
 	blocks.Shutdown();
@@ -174,7 +262,9 @@ void Jit64::WriteCallInterpreter(UGeckoInstruction inst)
 		MOV(32, PPCSTATE(npc), Imm32(js.compilerPC + 4));
 	}
 	Interpreter::_interpreterInstruction instr = GetInterpreterOp(inst);
+	ABI_PushRegistersAndAdjustStack(0, 0);
 	ABI_CallFunctionC((void*)instr, inst.hex);
+	ABI_PopRegistersAndAdjustStack(0, 0);
 }
 
 void Jit64::unknown_instruction(UGeckoInstruction inst)
@@ -191,7 +281,9 @@ void Jit64::HLEFunction(UGeckoInstruction _inst)
 {
 	gpr.Flush();
 	fpr.Flush();
+	ABI_PushRegistersAndAdjustStack(0, 0);
 	ABI_CallFunctionCC((void*)&HLE::Execute, js.compilerPC, _inst.hex);
+	ABI_PopRegistersAndAdjustStack(0, 0);
 }
 
 void Jit64::DoNothing(UGeckoInstruction _inst)
@@ -223,29 +315,52 @@ static void ImHere()
 	been_here[PC] = 1;
 }
 
-void Jit64::Cleanup()
+bool Jit64::Cleanup()
 {
+	bool did_something = false;
+
 	if (jo.optimizeGatherPipe && js.fifoBytesThisBlock > 0)
 	{
+		ABI_PushRegistersAndAdjustStack(0, 0);
 		ABI_CallFunction((void *)&GPFifo::CheckGatherPipe);
+		ABI_PopRegistersAndAdjustStack(0, 0);
+		did_something = true;
 	}
 
 	// SPEED HACK: MMCR0/MMCR1 should be checked at run-time, not at compile time.
 	if (MMCR0.Hex || MMCR1.Hex)
+	{
 		ABI_CallFunctionCCC((void *)&PowerPC::UpdatePerformanceMonitor, js.downcountAmount, jit->js.numLoadStoreInst, jit->js.numFloatingPointInst);
+		did_something = true;
+	}
+
+	return did_something;
 }
 
-void Jit64::WriteExit(u32 destination)
+void Jit64::WriteExit(u32 destination, bool bl, u32 after)
 {
+	if (!m_enable_blr_optimization)
+		bl = false;
+
 	Cleanup();
 
+	if (bl)
+	{
+		MOV(32, R(RSCRATCH2), Imm32(after));
+		PUSH(RSCRATCH2);
+	}
+
 	SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount));
 
+	JustWriteExit(destination, bl, after);
+}
+
+void Jit64::JustWriteExit(u32 destination, bool bl, u32 after)
+{
 	//If nobody has taken care of this yet (this can be removed when all branches are done)
 	JitBlock *b = js.curBlock;
 	JitBlock::LinkData linkData;
 	linkData.exitAddress = destination;
-	linkData.exitPtrs = GetWritableCodePtr();
 	linkData.linkStatus = false;
 
 	// Link opportunity!
@@ -253,24 +368,76 @@ void Jit64::WriteExit(u32 destination)
 	if (jo.enableBlocklink && (block = blocks.GetBlockNumberFromStartAddress(destination)) >= 0)
 	{
 		// It exists! Joy of joy!
-		JMP(blocks.GetBlock(block)->checkedEntry, true);
+		JitBlock* jb = blocks.GetBlock(block);
+		const u8* addr = jb->checkedEntry;
+		linkData.exitPtrs = GetWritableCodePtr();
+		if (bl)
+			CALL(addr);
+		else
+			JMP(addr, true);
 		linkData.linkStatus = true;
 	}
 	else
 	{
 		MOV(32, PPCSTATE(pc), Imm32(destination));
-		JMP(asm_routines.dispatcher, true);
+		linkData.exitPtrs = GetWritableCodePtr();
+		if (bl)
+			CALL(asm_routines.dispatcher);
+		else
+			JMP(asm_routines.dispatcher, true);
 	}
 
 	b->linkData.push_back(linkData);
+
+	if (bl)
+	{
+		POP(RSCRATCH);
+		JustWriteExit(after, false, 0);
+	}
 }
 
-void Jit64::WriteExitDestInRSCRATCH()
+void Jit64::WriteExitDestInRSCRATCH(bool bl, u32 after)
 {
+	if (!m_enable_blr_optimization)
+		bl = false;
 	MOV(32, PPCSTATE(pc), R(RSCRATCH));
 	Cleanup();
+
+	if (bl)
+	{
+		MOV(32, R(RSCRATCH2), Imm32(after));
+		PUSH(RSCRATCH2);
+	}
+
 	SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount));
-	JMP(asm_routines.dispatcher, true);
+	if (bl)
+	{
+		CALL(asm_routines.dispatcher);
+		POP(RSCRATCH);
+		JustWriteExit(after, false, 0);
+	}
+	else
+	{
+		JMP(asm_routines.dispatcher, true);
+	}
+}
+
+void Jit64::WriteBLRExit()
+{
+	if (!m_enable_blr_optimization)
+	{
+		WriteExitDestInRSCRATCH();
+		return;
+	}
+	MOV(32, PPCSTATE(pc), R(RSCRATCH));
+	bool disturbed = Cleanup();
+	if (disturbed)
+		MOV(32, R(RSCRATCH), PPCSTATE(pc));
+	CMP(64, R(RSCRATCH), MDisp(RSP, 8));
+	MOV(32, R(RSCRATCH), Imm32(js.downcountAmount));
+	J_CC(CC_NE, asm_routines.dispatcherMispredictedBLR);
+	SUB(32, PPCSTATE(downcount), R(RSCRATCH));
+	RET();
 }
 
 void Jit64::WriteRfiExitDestInRSCRATCH()
@@ -278,7 +445,9 @@ void Jit64::WriteRfiExitDestInRSCRATCH()
 	MOV(32, PPCSTATE(pc), R(RSCRATCH));
 	MOV(32, PPCSTATE(npc), R(RSCRATCH));
 	Cleanup();
+	ABI_PushRegistersAndAdjustStack(0, 0);
 	ABI_CallFunction(reinterpret_cast<void *>(&PowerPC::CheckExceptions));
+	ABI_PopRegistersAndAdjustStack(0, 0);
 	SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount));
 	JMP(asm_routines.dispatcher, true);
 }
@@ -288,7 +457,9 @@ void Jit64::WriteExceptionExit()
 	Cleanup();
 	MOV(32, R(RSCRATCH), PPCSTATE(pc));
 	MOV(32, PPCSTATE(npc), R(RSCRATCH));
+	ABI_PushRegistersAndAdjustStack(0, 0);
 	ABI_CallFunction(reinterpret_cast<void *>(&PowerPC::CheckExceptions));
+	ABI_PopRegistersAndAdjustStack(0, 0);
 	SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount));
 	JMP(asm_routines.dispatcher, true);
 }
@@ -298,7 +469,9 @@ void Jit64::WriteExternalExceptionExit()
 	Cleanup();
 	MOV(32, R(RSCRATCH), PPCSTATE(pc));
 	MOV(32, PPCSTATE(npc), R(RSCRATCH));
+	ABI_PushRegistersAndAdjustStack(0, 0);
 	ABI_CallFunction(reinterpret_cast<void *>(&PowerPC::CheckExternalExceptions));
+	ABI_PopRegistersAndAdjustStack(0, 0);
 	SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount));
 	JMP(asm_routines.dispatcher, true);
 }
@@ -340,8 +513,11 @@ void Jit64::Trace()
 
 void STACKALIGN Jit64::Jit(u32 em_address)
 {
-	if (GetSpaceLeft() < 0x10000 || farcode.GetSpaceLeft() < 0x10000 || blocks.IsFull() ||
-		SConfig::GetInstance().m_LocalCoreStartupParameter.bJITNoBlockCache)
+	if (GetSpaceLeft() < 0x10000 ||
+	    farcode.GetSpaceLeft() < 0x10000 ||
+		blocks.IsFull() ||
+		SConfig::GetInstance().m_LocalCoreStartupParameter.bJITNoBlockCache ||
+		m_clear_cache_asap)
 	{
 		ClearCache();
 	}
@@ -395,7 +571,11 @@ const u8* Jit64::DoJit(u32 em_address, PPCAnalyst::CodeBuffer *code_buf, JitBloc
 	b->normalEntry = normalEntry;
 
 	if (ImHereDebug)
+	{
+		ABI_PushRegistersAndAdjustStack(0, 0);
 		ABI_CallFunction((void *)&ImHere); //Used to get a trace of the last few blocks before a crash, sometimes VERY useful
+		ABI_PopRegistersAndAdjustStack(0, 0);
+	}
 
 	// Conditionally add profiling code.
 	if (Profiler::g_ProfileBlocks)
@@ -548,7 +728,9 @@ const u8* Jit64::DoJit(u32 em_address, PPCAnalyst::CodeBuffer *code_buf, JitBloc
 				fpr.Flush();
 
 				MOV(32, PPCSTATE(pc), Imm32(ops[i].address));
+				ABI_PushRegistersAndAdjustStack(0, 0);
 				ABI_CallFunction(reinterpret_cast<void *>(&PowerPC::CheckBreakPoints));
+				ABI_PopRegistersAndAdjustStack(0, 0);
 				TEST(32, M((void*)PowerPC::GetStatePtr()), Imm32(0xFFFFFFFF));
 				FixupBranch noBreakpoint = J_CC(CC_Z);
 

Source/Core/Core/PowerPC/Jit64/Jit.h

@@ -18,6 +18,10 @@
 // ----------
 #pragma once
 
+#ifdef _WIN32
+#include <winnt.h>
+#endif
+
 #include "Common/x64ABI.h"
 #include "Common/x64Analyzer.h"
 #include "Common/x64Emitter.h"
@@ -40,6 +44,9 @@
 class Jit64 : public Jitx86Base
 {
 private:
+	void AllocStack();
+	void FreeStack();
+
 	GPRRegCache gpr;
 	FPURegCache fpr;
 
@@ -48,6 +55,10 @@ private:
 	PPCAnalyst::CodeBuffer code_buffer;
 	Jit64AsmRoutineManager asm_routines;
 
+	bool m_enable_blr_optimization;
+	bool m_clear_cache_asap;
+	u8* m_stack;
+
 public:
 	Jit64() : code_buffer(32000) {}
 	~Jit64() {}
@@ -55,6 +66,8 @@ public:
 	void Init() override;
 	void Shutdown() override;
 
+	bool HandleFault(uintptr_t access_address, SContext* ctx) override;
+
 	// Jit!
 
 	void Jit(u32 em_address) override;
@@ -89,13 +102,15 @@ public:
 
 	// Utilities for use by opcodes
 
-	void WriteExit(u32 destination);
-	void WriteExitDestInRSCRATCH();
+	void WriteExit(u32 destination, bool bl = false, u32 after = 0);
+	void JustWriteExit(u32 destination, bool bl, u32 after);
+	void WriteExitDestInRSCRATCH(bool bl = false, u32 after = 0);
+	void WriteBLRExit();
 	void WriteExceptionExit();
 	void WriteExternalExceptionExit();
 	void WriteRfiExitDestInRSCRATCH();
 	void WriteCallInterpreter(UGeckoInstruction _inst);
-	void Cleanup();
+	bool Cleanup();
 
 	void GenerateConstantOverflow(bool overflow);
 	void GenerateConstantOverflow(s64 val);

Source/Core/Core/PowerPC/Jit64/JitAsm.cpp

@@ -9,6 +9,9 @@
 
 using namespace Gen;
 
+// Not PowerPC state.  Can't put in 'this' because it's out of range...
+static void* s_saved_rsp;
+
 // PLAN: no more block numbers - crazy opcodes just contain offset within
 // dynarec buffer
 // At this offset - 4, there is an int specifying the block number.
@@ -16,7 +19,23 @@ using namespace Gen;
 void Jit64AsmRoutineManager::Generate()
 {
 	enterCode = AlignCode16();
-	ABI_PushRegistersAndAdjustStack(ABI_ALL_CALLEE_SAVED, 8);
+	// We need to own the beginning of RSP, so we do an extra stack adjustment
+	// for the shadow region before calls in this function.  This call will
+	// waste a bit of space for a second shadow, but whatever.
+	ABI_PushRegistersAndAdjustStack(ABI_ALL_CALLEE_SAVED, 8, /*frame*/ 16);
+	if (m_stack_top)
+	{
+		// Pivot the stack to our custom one.
+		MOV(64, R(RSCRATCH), R(RSP));
+		MOV(64, R(RSP), Imm64((u64)m_stack_top - 0x20));
+		MOV(64, MDisp(RSP, 0x18), R(RSCRATCH));
+	}
+	else
+	{
+		MOV(64, M(&s_saved_rsp), R(RSP));
+	}
+	// something that can't pass the BLR test
+	MOV(64, MDisp(RSP, 8), Imm32((u32)-1));
 
 	// Two statically allocated registers.
 	MOV(64, R(RMEM), Imm64((u64)Memory::base));
@@ -24,24 +43,42 @@ void Jit64AsmRoutineManager::Generate()
 	MOV(64, R(RPPCSTATE), Imm64((u64)&PowerPC::ppcState + 0x80));
 
 	const u8* outerLoop = GetCodePtr();
+		ABI_PushRegistersAndAdjustStack(0, 0);
 		ABI_CallFunction(reinterpret_cast<void *>(&CoreTiming::Advance));
+		ABI_PopRegistersAndAdjustStack(0, 0);
 		FixupBranch skipToRealDispatch = J(SConfig::GetInstance().m_LocalCoreStartupParameter.bEnableDebugging); //skip the sync and compare first time
+		dispatcherMispredictedBLR = GetCodePtr();
+
+		#if 0 // debug mispredicts
+		MOV(32, R(ABI_PARAM1), MDisp(RSP, 8)); // guessed_pc
+		ABI_PushRegistersAndAdjustStack(1 << RSCRATCH, 0);
+		CALL(reinterpret_cast<void *>(&ReportMispredict));
+		ABI_PopRegistersAndAdjustStack(1 << RSCRATCH, 0);
+		#endif
+
+		if (m_stack_top)
+			MOV(64, R(RSP), Imm64((u64)m_stack_top - 0x20));
+		else
+			MOV(64, R(RSP), M(&s_saved_rsp));
+
+		SUB(32, PPCSTATE(downcount), R(RSCRATCH));
 
 		dispatcher = GetCodePtr();
 			// The result of slice decrementation should be in flags if somebody jumped here
 			// IMPORTANT - We jump on negative, not carry!!!
 			FixupBranch bail = J_CC(CC_BE, true);
 
+			FixupBranch dbg_exit;
+
 			if (SConfig::GetInstance().m_LocalCoreStartupParameter.bEnableDebugging)
 			{
 				TEST(32, M((void*)PowerPC::GetStatePtr()), Imm32(PowerPC::CPU_STEPPING));
 				FixupBranch notStepping = J_CC(CC_Z);
+				ABI_PushRegistersAndAdjustStack(0, 0);
 				ABI_CallFunction(reinterpret_cast<void *>(&PowerPC::CheckBreakPoints));
+				ABI_PopRegistersAndAdjustStack(0, 0);
 				TEST(32, M((void*)PowerPC::GetStatePtr()), Imm32(0xFFFFFFFF));
-				FixupBranch noBreakpoint = J_CC(CC_Z);
-				ABI_PopRegistersAndAdjustStack(ABI_ALL_CALLEE_SAVED, 8);
-				RET();
-				SetJumpTarget(noBreakpoint);
+				dbg_exit = J_CC(CC_NZ);
 				SetJumpTarget(notStepping);
 			}
 
@@ -106,8 +143,9 @@ void Jit64AsmRoutineManager::Generate()
 			SetJumpTarget(notfound);
 
 			//Ok, no block, let's jit
-			MOV(32, R(ABI_PARAM1), PPCSTATE(pc));
-			CALL((void *)&Jit);
+			ABI_PushRegistersAndAdjustStack(0, 0);
+			ABI_CallFunctionA((void *)&Jit, PPCSTATE(pc));
+			ABI_PopRegistersAndAdjustStack(0, 0);
 
 			JMP(dispatcherNoCheck); // no point in special casing this
 
@@ -119,14 +157,27 @@ void Jit64AsmRoutineManager::Generate()
 		FixupBranch noExtException = J_CC(CC_Z);
 		MOV(32, R(RSCRATCH), PPCSTATE(pc));
 		MOV(32, PPCSTATE(npc), R(RSCRATCH));
+		ABI_PushRegistersAndAdjustStack(0, 0);
 		ABI_CallFunction(reinterpret_cast<void *>(&PowerPC::CheckExternalExceptions));
+		ABI_PopRegistersAndAdjustStack(0, 0);
 		SetJumpTarget(noExtException);
 
 		TEST(32, M((void*)PowerPC::GetStatePtr()), Imm32(0xFFFFFFFF));
 		J_CC(CC_Z, outerLoop);
 
 	//Landing pad for drec space
-	ABI_PopRegistersAndAdjustStack(ABI_ALL_CALLEE_SAVED, 8);
+	if (SConfig::GetInstance().m_LocalCoreStartupParameter.bEnableDebugging)
+		SetJumpTarget(dbg_exit);
+	if (m_stack_top)
+	{
+		MOV(64, R(RSP), Imm64((u64)m_stack_top - 0x8));
+		POP(RSP);
+	}
+	else
+	{
+		MOV(64, R(RSP), M(&s_saved_rsp));
+	}
+	ABI_PopRegistersAndAdjustStack(ABI_ALL_CALLEE_SAVED, 8, 16);
 	RET();
 
 	GenerateCommon();

Source/Core/Core/PowerPC/Jit64/JitAsm.h

@@ -25,10 +25,12 @@ class Jit64AsmRoutineManager : public CommonAsmRoutines
 private:
 	void Generate();
 	void GenerateCommon();
+	u8* m_stack_top;
 
 public:
-	void Init()
+	void Init(u8* stack_top)
 	{
+		m_stack_top = stack_top;
 		AllocCodeSpace(8192);
 		Generate();
 		WriteProtect();

Source/Core/Core/PowerPC/Jit64/Jit_Branch.cpp

@@ -92,7 +92,7 @@ void Jit64::bx(UGeckoInstruction inst)
 		// make idle loops go faster
 		js.downcountAmount += 8;
 	}
-	WriteExit(destination);
+	WriteExit(destination, inst.LK, js.compilerPC + 4);
 }
 
 // TODO - optimize to hell and beyond
@@ -133,7 +133,7 @@ void Jit64::bcx(UGeckoInstruction inst)
 
 	gpr.Flush(FLUSH_MAINTAIN_STATE);
 	fpr.Flush(FLUSH_MAINTAIN_STATE);
-	WriteExit(destination);
+	WriteExit(destination, inst.LK, js.compilerPC + 4);
 
 	if ((inst.BO & BO_DONT_CHECK_CONDITION) == 0)
 		SetJumpTarget( pConditionDontBranch );
@@ -168,7 +168,7 @@ void Jit64::bcctrx(UGeckoInstruction inst)
 		if (inst.LK_3)
 			MOV(32, PPCSTATE_LR, Imm32(js.compilerPC + 4)); // LR = PC + 4;
 		AND(32, R(RSCRATCH), Imm32(0xFFFFFFFC));
-		WriteExitDestInRSCRATCH();
+		WriteExitDestInRSCRATCH(inst.LK_3, js.compilerPC + 4);
 	}
 	else
 	{
@@ -187,7 +187,7 @@ void Jit64::bcctrx(UGeckoInstruction inst)
 
 		gpr.Flush(FLUSH_MAINTAIN_STATE);
 		fpr.Flush(FLUSH_MAINTAIN_STATE);
-		WriteExitDestInRSCRATCH();
+		WriteExitDestInRSCRATCH(inst.LK_3, js.compilerPC + 4);
 		// Would really like to continue the block here, but it ends. TODO.
 		SetJumpTarget(b);
 
@@ -235,7 +235,7 @@ void Jit64::bclrx(UGeckoInstruction inst)
 
 	gpr.Flush(FLUSH_MAINTAIN_STATE);
 	fpr.Flush(FLUSH_MAINTAIN_STATE);
-	WriteExitDestInRSCRATCH();
+	WriteBLRExit();
 
 	if ((inst.BO & BO_DONT_CHECK_CONDITION) == 0)
 		SetJumpTarget( pConditionDontBranch );

Source/Core/Core/PowerPC/Jit64/Jit_Integer.cpp

@@ -312,7 +312,7 @@ void Jit64::DoMergedBranch()
 			destination = SignExt16(js.next_inst.BD << 2);
 		else
 			destination = js.next_compilerPC + SignExt16(js.next_inst.BD << 2);
-		WriteExit(destination);
+		WriteExit(destination, js.next_inst.LK, js.next_compilerPC + 4);
 	}
 	else if ((js.next_inst.OPCD == 19) && (js.next_inst.SUBOP10 == 528)) // bcctrx
 	{
@@ -320,7 +320,7 @@ void Jit64::DoMergedBranch()
 			MOV(32, M(&LR), Imm32(js.next_compilerPC + 4));
 		MOV(32, R(RSCRATCH), M(&CTR));
 		AND(32, R(RSCRATCH), Imm32(0xFFFFFFFC));
-		WriteExitDestInRSCRATCH();
+		WriteExitDestInRSCRATCH(js.next_inst.LK, js.next_compilerPC + 4);
 	}
 	else if ((js.next_inst.OPCD == 19) && (js.next_inst.SUBOP10 == 16)) // bclrx
 	{
@@ -328,7 +328,7 @@ void Jit64::DoMergedBranch()
 		AND(32, R(RSCRATCH), Imm32(0xFFFFFFFC));
 		if (js.next_inst.LK)
 			MOV(32, M(&LR), Imm32(js.next_compilerPC + 4));
-		WriteExitDestInRSCRATCH();
+		WriteExitDestInRSCRATCH(js.next_inst.LK, js.next_compilerPC + 4);
 	}
 	else
 	{

Source/Core/Core/PowerPC/Jit64IL/JitIL.cpp

@@ -272,7 +272,7 @@ void JitIL::Init()
 	trampolines.Init();
 	AllocCodeSpace(CODE_SIZE);
 	blocks.Init();
-	asm_routines.Init();
+	asm_routines.Init(nullptr);
 
 	farcode.Init(js.memcheck ? FARCODE_SIZE_MMU : FARCODE_SIZE);
 

Source/Core/Core/PowerPC/Jit64IL/JitIL.h

@@ -56,6 +56,10 @@ public:
 
 	void Trace();
 
+	JitBlockCache *GetBlockCache() override { return &blocks; }
+
+	bool HandleFault(uintptr_t access_address, SContext* ctx) override { return false; }
+
 	void ClearCache() override;
 	const u8 *GetDispatcher()
 	{
@@ -105,4 +109,5 @@ public:
 	void DynaRunTable31(UGeckoInstruction _inst) override;
 	void DynaRunTable59(UGeckoInstruction _inst) override;
 	void DynaRunTable63(UGeckoInstruction _inst) override;
+
 };

Source/Core/Core/PowerPC/JitArm32/Jit.h

@@ -58,6 +58,8 @@ private:
 	void SetFPException(ArmGen::ARMReg Reg, u32 Exception);
 
 	ArmGen::FixupBranch JumpIfCRFieldBit(int field, int bit, bool jump_if_set);
+
+	bool BackPatch(SContext* ctx);
 public:
 	JitArm() : code_buffer(32000) {}
 	~JitArm() {}
@@ -72,9 +74,7 @@ public:
 
 	JitBaseBlockCache *GetBlockCache() { return &blocks; }
 
-	const u8 *BackPatch(u8 *codePtr, u32 em_address, void *ctx);
-
-	bool IsInCodeSpace(u8 *ptr) { return IsInSpace(ptr); }
+	bool HandleFault(uintptr_t access_address, SContext* ctx) override;
 
 	void Trace();
 

Source/Core/Core/PowerPC/JitArm32/JitArm_BackPatch.cpp

@@ -66,12 +66,23 @@ bool DisamLoadStore(const u32 inst, ARMReg &rD, u8 &accessSize, bool &Store)
 	}
 	return true;
 }
-const u8 *JitArm::BackPatch(u8 *codePtr, u32, void *ctx_void)
+
+bool JitArm::HandleFault(uintptr_t access_address, SContext* ctx)
+{
+	if (access_address < (uintptr_t)Memory::base)
+	{
+		PanicAlertT("Exception handler - access below memory space. %08llx%08llx",
+		            access_address >> 32, access_address);
+	}
+	return BackPatch(ctx);
+}
+
+bool JitArm::BackPatch(SContext* ctx)
 {
 	// TODO: This ctx needs to be filled with our information
-	SContext *ctx = (SContext *)ctx_void;
 
 	// We need to get the destination register before we start
+	u8* codePtr = (u8*)ctx->CTX_PC;
 	u32 Value = *(u32*)codePtr;
 	ARMReg rD;
 	u8 accessSize;
@@ -109,7 +120,7 @@ const u8 *JitArm::BackPatch(u8 *codePtr, u32, void *ctx_void)
 		u32 newPC = ctx->CTX_PC - (ARMREGOFFSET + 4 * 4);
 		ctx->CTX_PC = newPC;
 		emitter.FlushIcache();
-		return (u8*)ctx->CTX_PC;
+		return true;
 	}
 	else
 	{
@@ -135,7 +146,7 @@ const u8 *JitArm::BackPatch(u8 *codePtr, u32, void *ctx_void)
 		emitter.MOV(rD, R14); // 8
 		ctx->CTX_PC -= ARMREGOFFSET + (4 * 4);
 		emitter.FlushIcache();
-		return (u8*)ctx->CTX_PC;
+		return true;
 	}
 	return 0;
 }