Merge pull request #885 from comex/gpu-determinism

GPU determinism (apparently it is ready for merge)

Source/Core/Core/BootManager.cpp

@@ -55,10 +55,24 @@ struct ConfigCache
 	unsigned int framelimit, frameSkip;
 	TEXIDevices m_EXIDevice[MAX_EXI_CHANNELS];
 	std::string strBackend, sBackend;
+	std::string m_strGPUDeterminismMode;
 	bool bSetFramelimit, bSetEXIDevice[MAX_EXI_CHANNELS], bSetVolume, bSetPads[MAX_SI_CHANNELS], bSetWiimoteSource[MAX_BBMOTES], bSetFrameSkip;
 };
 static ConfigCache config_cache;
 
+static GPUDeterminismMode ParseGPUDeterminismMode(const std::string& mode)
+{
+	if (mode == "auto")
+		return GPU_DETERMINISM_AUTO;
+	if (mode == "none")
+		return GPU_DETERMINISM_NONE;
+	if (mode == "fake-completion")
+		return GPU_DETERMINISM_FAKE_COMPLETION;
+
+	NOTICE_LOG(BOOT, "Unknown GPU determinism mode %s", mode.c_str());
+	return GPU_DETERMINISM_AUTO;
+}
+
 // Boot the ISO or file
 bool BootCore(const std::string& _rFilename)
 {
@@ -109,6 +123,7 @@ bool BootCore(const std::string& _rFilename)
 		config_cache.bMergeBlocks = StartUp.bMergeBlocks;
 		config_cache.bDSPHLE = StartUp.bDSPHLE;
 		config_cache.strBackend = StartUp.m_strVideoBackend;
+		config_cache.m_strGPUDeterminismMode = StartUp.m_strGPUDeterminismMode;
 		config_cache.m_EnableJIT = SConfig::GetInstance().m_DSPEnableJIT;
 		config_cache.bDSPThread = StartUp.bDSPThread;
 		config_cache.Volume = SConfig::GetInstance().m_Volume;
@@ -168,6 +183,8 @@ bool BootCore(const std::string& _rFilename)
 		dsp_section->Get("EnableJIT",         &SConfig::GetInstance().m_DSPEnableJIT, SConfig::GetInstance().m_DSPEnableJIT);
 		dsp_section->Get("Backend",           &SConfig::GetInstance().sBackend, SConfig::GetInstance().sBackend);
 		VideoBackend::ActivateBackend(StartUp.m_strVideoBackend);
+		core_section->Get("GPUDeterminismMode", &StartUp.m_strGPUDeterminismMode, StartUp.m_strGPUDeterminismMode);
+		StartUp.m_GPUDeterminismMode = ParseGPUDeterminismMode(StartUp.m_strGPUDeterminismMode);
 
 		for (unsigned int i = 0; i < MAX_SI_CHANNELS; ++i)
 		{
@@ -277,6 +294,7 @@ void Stop()
 		StartUp.bDSPHLE = config_cache.bDSPHLE;
 		StartUp.bDSPThread = config_cache.bDSPThread;
 		StartUp.m_strVideoBackend = config_cache.strBackend;
+		StartUp.m_strGPUDeterminismMode = config_cache.m_strGPUDeterminismMode;
 		VideoBackend::ActivateBackend(StartUp.m_strVideoBackend);
 		StartUp.bHLE_BS2 = config_cache.bHLE_BS2;
 		SConfig::GetInstance().sBackend = config_cache.sBackend;

Source/Core/Core/ConfigManager.cpp

@@ -317,6 +317,7 @@ void SConfig::SaveCoreSettings(IniFile& ini)
 	core->Set("FrameLimit", m_Framelimit);
 	core->Set("FrameSkip", m_FrameSkip);
 	core->Set("GFXBackend", m_LocalCoreStartupParameter.m_strVideoBackend);
+	core->Set("GPUDeterminismMode", m_LocalCoreStartupParameter.m_strGPUDeterminismMode);
 }
 
 void SConfig::SaveMovieSettings(IniFile& ini)
@@ -542,6 +543,7 @@ void SConfig::LoadCoreSettings(IniFile& ini)
 	core->Get("FrameLimit",                &m_Framelimit,                                  1); // auto frame limit by default
 	core->Get("FrameSkip",                 &m_FrameSkip,                                   0);
 	core->Get("GFXBackend",                &m_LocalCoreStartupParameter.m_strVideoBackend, "");
+	core->Get("GPUDeterminismMode",        &m_LocalCoreStartupParameter.m_strGPUDeterminismMode, "auto");
 }
 
 void SConfig::LoadMovieSettings(IniFile& ini)

Source/Core/Core/Core.cpp

@@ -48,6 +48,7 @@
 #include "Core/HW/VideoInterface.h"
 #include "Core/HW/Wiimote.h"
 #include "Core/IPC_HLE/WII_IPC_HLE_Device_usb.h"
+#include "Core/IPC_HLE/WII_Socket.h"
 #include "Core/PowerPC/PowerPC.h"
 
 #ifdef USE_GDBSTUB
@@ -65,6 +66,8 @@ bool g_aspect_wide;
 namespace Core
 {
 
+bool g_want_determinism;
+
 // Declarations and definitions
 static Common::Timer s_timer;
 static volatile u32 s_drawn_frame = 0;
@@ -177,6 +180,8 @@ bool Init()
 		s_emu_thread.join();
 	}
 
+	Core::UpdateWantDeterminism(/*initial*/ true);
+
 	INFO_LOG(OSREPORT, "Starting core = %s mode",
 		_CoreParameter.bWii ? "Wii" : "GameCube");
 	INFO_LOG(OSREPORT, "CPU Thread separate = %s",
@@ -564,6 +569,9 @@ void RequestRefreshInfo()
 
 bool PauseAndLock(bool doLock, bool unpauseOnUnlock)
 {
+	if (!IsRunning())
+		return true;
+
 	// let's support recursive locking to simplify things on the caller's side,
 	// and let's do it at this outer level in case the individual systems don't support it.
 	if (doLock ? s_pause_and_lock_depth++ : --s_pause_and_lock_depth)
@@ -702,4 +710,27 @@ void SetOnStoppedCallback(StoppedCallbackFunc callback)
 	s_on_stopped_callback = callback;
 }
 
+void UpdateWantDeterminism(bool initial)
+{
+	// For now, this value is not itself configurable.  Instead, individual
+	// settings that depend on it, such as GPU determinism mode. should have
+	// override options for testing,
+	bool new_want_determinism =
+		Movie::IsPlayingInput() ||
+		Movie::IsRecordingInput() ||
+		NetPlay::IsNetPlayRunning();
+	if (new_want_determinism != g_want_determinism || initial)
+	{
+		WARN_LOG(COMMON, "Want determinism <- %s", new_want_determinism ? "true" : "false");
+
+		bool was_unpaused = Core::PauseAndLock(true);
+
+		g_want_determinism = new_want_determinism;
+		WiiSockMan::GetInstance().UpdateWantDeterminism(new_want_determinism);
+		g_video_backend->UpdateWantDeterminism(new_want_determinism);
+
+		Core::PauseAndLock(false, was_unpaused);
+	}
+}
+
 } // Core

Source/Core/Core/Core.h

@@ -23,6 +23,8 @@ extern bool g_aspect_wide;
 namespace Core
 {
 
+extern bool g_want_determinism;
+
 bool GetIsFramelimiterTempDisabled();
 void SetIsFramelimiterTempDisabled(bool disable);
 
@@ -79,4 +81,7 @@ bool PauseAndLock(bool doLock, bool unpauseOnUnlock=true);
 typedef void(*StoppedCallbackFunc)(void);
 void SetOnStoppedCallback(StoppedCallbackFunc callback);
 
+// Run on the GUI thread when the factors change.
+void UpdateWantDeterminism(bool initial = false);
+
 }  // namespace

Source/Core/Core/CoreParameter.h

@@ -97,6 +97,15 @@ enum Hotkey
 	NUM_HOTKEYS,
 };
 
+enum GPUDeterminismMode
+{
+	GPU_DETERMINISM_AUTO,
+	GPU_DETERMINISM_NONE,
+	// This is currently the only mode.  There will probably be at least
+	// one more at some point.
+	GPU_DETERMINISM_FAKE_COMPLETION,
+};
+
 struct SCoreStartupParameter
 {
 	// Settings
@@ -200,6 +209,10 @@ struct SCoreStartupParameter
 	EBootType m_BootType;
 
 	std::string m_strVideoBackend;
+	std::string m_strGPUDeterminismMode;
+
+	// set based on the string version
+	GPUDeterminismMode m_GPUDeterminismMode;
 
 	// files
 	std::string m_strFilename;

Source/Core/Core/HW/WiimoteEmu/WiimoteEmu.cpp

@@ -331,7 +331,7 @@ bool Wiimote::Step()
 	m_rumble->controls[0]->control_ref->State(m_rumble_on);
 
 	// when a movie is active, this button status update is disabled (moved), because movies only record data reports.
-	if (!(Movie::IsMovieActive()) || NetPlay::IsNetPlayRunning())
+	if (!Core::g_want_determinism)
 	{
 		UpdateButtonsStatus();
 	}
@@ -385,7 +385,7 @@ void Wiimote::UpdateButtonsStatus()
 void Wiimote::GetCoreData(u8* const data)
 {
 	// when a movie is active, the button update happens here instead of Wiimote::Step, to avoid potential desync issues.
-	if (Movie::IsMovieActive() || NetPlay::IsNetPlayRunning())
+	if (Core::g_want_determinism)
 	{
 		UpdateButtonsStatus();
 	}

Source/Core/Core/IPC_HLE/WII_Socket.cpp

@@ -4,8 +4,7 @@
 
 #include <algorithm>
 
-#include "Core/Movie.h"
-#include "Core/NetPlayProto.h"
+#include "Core/Core.h"
 #include "Core/IPC_HLE/WII_IPC_HLE.h"
 #include "Core/IPC_HLE/WII_IPC_HLE_Device.h"
 #include "Core/IPC_HLE/WII_Socket.h" // No Wii socket support while using NetPlay or TAS
@@ -559,9 +558,7 @@ void WiiSockMan::AddSocket(s32 fd)
 
 s32 WiiSockMan::NewSocket(s32 af, s32 type, s32 protocol)
 {
-	if (NetPlay::IsNetPlayRunning() ||
-	    Movie::IsRecordingInput() ||
-	    Movie::IsPlayingInput())
+	if (Core::g_want_determinism)
 	{
 		return SO_ENOMEM;
 	}
@@ -664,5 +661,12 @@ void WiiSockMan::Convert(sockaddr_in const & from, WiiSockAddrIn& to, s32 addrle
 		to.len = addrlen;
 }
 
+void WiiSockMan::UpdateWantDeterminism(bool want)
+{
+	// If we switched into movie recording, kill existing sockets.
+	if (want)
+		Clean();
+}
+
 #undef ERRORCODE
 #undef EITHER

Source/Core/Core/IPC_HLE/WII_Socket.h

@@ -242,6 +242,8 @@ public:
 		}
 	}
 
+	void UpdateWantDeterminism(bool want);
+
 private:
 	WiiSockMan() = default;
 

Source/Core/Core/Movie.cpp

@@ -437,6 +437,8 @@ bool BeginRecordingInput(int controllers)
 	if (s_playMode != MODE_NONE || controllers == 0)
 		return false;
 
+	bool was_unpaused = Core::PauseAndLock(true);
+
 	s_numPads = controllers;
 	g_currentFrame = g_totalFrames = 0;
 	g_currentLagCount = s_totalLagCount = 0;
@@ -487,6 +489,10 @@ bool BeginRecordingInput(int controllers)
 
 	s_currentByte = s_totalBytes = 0;
 
+	Core::UpdateWantDeterminism();
+
+	Core::PauseAndLock(false, was_unpaused);
+
 	Core::DisplayMessage("Starting movie recording", 2000);
 	return true;
 }
@@ -764,6 +770,8 @@ bool PlayInput(const std::string& filename)
 
 	s_playMode = MODE_PLAYING;
 
+	Core::UpdateWantDeterminism();
+
 	s_totalBytes = g_recordfd.GetSize() - 256;
 	EnsureTmpInputSize((size_t)s_totalBytes);
 	g_recordfd.ReadArray(tmpInput, (size_t)s_totalBytes);
@@ -1097,6 +1105,7 @@ void EndPlayInput(bool cont)
 		s_rerecords = 0;
 		s_currentByte = 0;
 		s_playMode = MODE_NONE;
+		Core::UpdateWantDeterminism();
 		Core::DisplayMessage("Movie End.", 2000);
 		s_bRecordingFromSaveState = false;
 		// we don't clear these things because otherwise we can't resume playback if we load a movie state later

Source/Core/VideoBackends/Software/CPMemLoader.cpp

@@ -13,46 +13,46 @@ void SWLoadCPReg(u32 sub_cmd, u32 value)
 	switch (sub_cmd & 0xF0)
 	{
 	case 0x30:
-		MatrixIndexA.Hex = value;
+		g_main_cp_state.matrix_index_a.Hex = value;
 		break;
 
 	case 0x40:
-		MatrixIndexB.Hex = value;
+		g_main_cp_state.matrix_index_b.Hex = value;
 		break;
 
 	case 0x50:
-		g_VtxDesc.Hex &= ~0x1FFFF;  // keep the Upper bits
-		g_VtxDesc.Hex |= value;
+		g_main_cp_state.vtx_desc.Hex &= ~0x1FFFF;  // keep the Upper bits
+		g_main_cp_state.vtx_desc.Hex |= value;
 		break;
 
 	case 0x60:
-		g_VtxDesc.Hex &= 0x1FFFF;  // keep the lower 17Bits
-		g_VtxDesc.Hex |= (u64)value << 17;
+		g_main_cp_state.vtx_desc.Hex &= 0x1FFFF;  // keep the lower 17Bits
+		g_main_cp_state.vtx_desc.Hex |= (u64)value << 17;
 		break;
 
 	case 0x70:
 		_assert_((sub_cmd & 0x0F) < 8);
-		g_VtxAttr[sub_cmd & 7].g0.Hex = value;
+		g_main_cp_state.vtx_attr[sub_cmd & 7].g0.Hex = value;
 		break;
 
 	case 0x80:
 		_assert_((sub_cmd & 0x0F) < 8);
-		g_VtxAttr[sub_cmd & 7].g1.Hex = value;
+		g_main_cp_state.vtx_attr[sub_cmd & 7].g1.Hex = value;
 		break;
 
 	case 0x90:
 		_assert_((sub_cmd & 0x0F) < 8);
-		g_VtxAttr[sub_cmd & 7].g2.Hex = value;
+		g_main_cp_state.vtx_attr[sub_cmd & 7].g2.Hex = value;
 		break;
 
 	// Pointers to vertex arrays in GC RAM
 	case 0xA0:
-		arraybases[sub_cmd & 0xF] = value;
+		g_main_cp_state.array_bases[sub_cmd & 0xF] = value;
 		cached_arraybases[sub_cmd & 0xF] = Memory::GetPointer(value);
 		break;
 
 	case 0xB0:
-		arraystrides[sub_cmd & 0xF] = value & 0xFF;
+		g_main_cp_state.array_strides[sub_cmd & 0xF] = value & 0xFF;
 		break;
 	}
 }

Source/Core/VideoBackends/Software/OpcodeDecoder.cpp

@@ -57,7 +57,7 @@ static void DecodePrimitiveStream(u32 iBufferSize)
 	{
 		while (streamSize > 0 && iBufferSize >= vertexSize)
 		{
-			g_pVideoData += vertexSize;
+			g_video_buffer_read_ptr += vertexSize;
 			iBufferSize -= vertexSize;
 			streamSize--;
 		}
@@ -94,26 +94,26 @@ static void ReadXFData(u32 iBufferSize)
 
 static void ExecuteDisplayList(u32 addr, u32 count)
 {
-	u8 *videoDataSave = g_pVideoData;
+	u8 *videoDataSave = g_video_buffer_read_ptr;
 
 	u8 *dlStart = Memory::GetPointer(addr);
 
-	g_pVideoData = dlStart;
+	g_video_buffer_read_ptr = dlStart;
 
 	while (OpcodeDecoder::CommandRunnable(count))
 	{
 		OpcodeDecoder::Run(count);
 
 		// if data was read by the opcode decoder then the video data pointer changed
-		u32 readCount = (u32)(g_pVideoData - dlStart);
-		dlStart = g_pVideoData;
+		u32 readCount = (u32)(g_video_buffer_read_ptr - dlStart);
+		dlStart = g_video_buffer_read_ptr;
 
 		_assert_msg_(VIDEO, count >= readCount, "Display list underrun");
 
 		count -= readCount;
 	}
 
-	g_pVideoData = videoDataSave;
+	g_video_buffer_read_ptr = videoDataSave;
 }
 
 static void DecodeStandard(u32 bufferSize)

Source/Core/VideoBackends/Software/SWCommandProcessor.cpp

@@ -57,7 +57,7 @@ void DoState(PointerWrap &p)
 	p.Do(interruptWaiting);
 
 	// Is this right?
-	p.DoArray(g_pVideoData,writePos);
+	p.DoArray(g_video_buffer_read_ptr,writePos);
 }
 
 static void UpdateInterrupts_Wrapper(u64 userdata, int cyclesLate)
@@ -95,7 +95,7 @@ void Init()
 	interruptSet = false;
 	interruptWaiting = false;
 
-	g_pVideoData = nullptr;
+	g_video_buffer_read_ptr = nullptr;
 	g_bSkipCurrentFrame = false;
 }
 
@@ -311,7 +311,7 @@ bool RunBuffer()
 
 	_dbg_assert_(COMMANDPROCESSOR, writePos >= readPos);
 
-	g_pVideoData = &commandBuffer[readPos];
+	g_video_buffer_read_ptr = &commandBuffer[readPos];
 
 	u32 availableBytes = writePos - readPos;
 
@@ -322,7 +322,7 @@ bool RunBuffer()
 		OpcodeDecoder::Run(availableBytes);
 
 		// if data was read by the opcode decoder then the video data pointer changed
-		readPos = (u32)(g_pVideoData - &commandBuffer[0]);
+		readPos = (u32)(g_video_buffer_read_ptr - &commandBuffer[0]);
 		_dbg_assert_(VIDEO, writePos >= readPos);
 		availableBytes = writePos - readPos;
 	}

Source/Core/VideoBackends/Software/SWVertexLoader.cpp

@@ -39,7 +39,7 @@ SWVertexLoader::~SWVertexLoader()
 
 void SWVertexLoader::SetFormat(u8 attributeIndex, u8 primitiveType)
 {
-	m_CurrentVat = &g_VtxAttr[attributeIndex];
+	m_CurrentVat = &g_main_cp_state.vtx_attr[attributeIndex];
 
 	posScale = 1.0f / float(1 << m_CurrentVat->g0.PosFrac);
 	tcScale[0] = 1.0f / float(1 << m_CurrentVat->g0.Tex0Frac);
@@ -53,20 +53,20 @@ void SWVertexLoader::SetFormat(u8 attributeIndex, u8 primitiveType)
 
 	//TexMtx
 	const u64 tmDesc[8] = {
-		g_VtxDesc.Tex0MatIdx, g_VtxDesc.Tex1MatIdx, g_VtxDesc.Tex2MatIdx, g_VtxDesc.Tex3MatIdx,
-		g_VtxDesc.Tex4MatIdx, g_VtxDesc.Tex5MatIdx, g_VtxDesc.Tex6MatIdx, g_VtxDesc.Tex7MatIdx
+		g_main_cp_state.vtx_desc.Tex0MatIdx, g_main_cp_state.vtx_desc.Tex1MatIdx, g_main_cp_state.vtx_desc.Tex2MatIdx, g_main_cp_state.vtx_desc.Tex3MatIdx,
+		g_main_cp_state.vtx_desc.Tex4MatIdx, g_main_cp_state.vtx_desc.Tex5MatIdx, g_main_cp_state.vtx_desc.Tex6MatIdx, g_main_cp_state.vtx_desc.Tex7MatIdx
 	};
 
 	// Colors
-	const u64 colDesc[2] = {g_VtxDesc.Color0, g_VtxDesc.Color1};
+	const u64 colDesc[2] = {g_main_cp_state.vtx_desc.Color0, g_main_cp_state.vtx_desc.Color1};
 	colElements[0] = m_CurrentVat->g0.Color0Elements;
 	colElements[1] = m_CurrentVat->g0.Color1Elements;
 	const u32 colComp[2] = {m_CurrentVat->g0.Color0Comp, m_CurrentVat->g0.Color1Comp};
 
 	// TextureCoord
 	const u64 tcDesc[8] = {
-		g_VtxDesc.Tex0Coord, g_VtxDesc.Tex1Coord, g_VtxDesc.Tex2Coord, g_VtxDesc.Tex3Coord,
-		g_VtxDesc.Tex4Coord, g_VtxDesc.Tex5Coord, g_VtxDesc.Tex6Coord, g_VtxDesc.Tex7Coord
+		g_main_cp_state.vtx_desc.Tex0Coord, g_main_cp_state.vtx_desc.Tex1Coord, g_main_cp_state.vtx_desc.Tex2Coord, g_main_cp_state.vtx_desc.Tex3Coord,
+		g_main_cp_state.vtx_desc.Tex4Coord, g_main_cp_state.vtx_desc.Tex5Coord, g_main_cp_state.vtx_desc.Tex6Coord, g_main_cp_state.vtx_desc.Tex7Coord
 	};
 	const u32 tcElements[8] = {
 		m_CurrentVat->g0.Tex0CoordElements, m_CurrentVat->g1.Tex1CoordElements, m_CurrentVat->g1.Tex2CoordElements,
@@ -89,15 +89,15 @@ void SWVertexLoader::SetFormat(u8 attributeIndex, u8 primitiveType)
 
 	// Reset vertex
 	// matrix index from xf regs or cp memory?
-	if (xfmem.MatrixIndexA.PosNormalMtxIdx != MatrixIndexA.PosNormalMtxIdx ||
-		xfmem.MatrixIndexA.Tex0MtxIdx != MatrixIndexA.Tex0MtxIdx ||
-		xfmem.MatrixIndexA.Tex1MtxIdx != MatrixIndexA.Tex1MtxIdx ||
-		xfmem.MatrixIndexA.Tex2MtxIdx != MatrixIndexA.Tex2MtxIdx ||
-		xfmem.MatrixIndexA.Tex3MtxIdx != MatrixIndexA.Tex3MtxIdx ||
-		xfmem.MatrixIndexB.Tex4MtxIdx != MatrixIndexB.Tex4MtxIdx ||
-		xfmem.MatrixIndexB.Tex5MtxIdx != MatrixIndexB.Tex5MtxIdx ||
-		xfmem.MatrixIndexB.Tex6MtxIdx != MatrixIndexB.Tex6MtxIdx ||
-		xfmem.MatrixIndexB.Tex7MtxIdx != MatrixIndexB.Tex7MtxIdx)
+	if (xfmem.MatrixIndexA.PosNormalMtxIdx != g_main_cp_state.matrix_index_a.PosNormalMtxIdx ||
+		xfmem.MatrixIndexA.Tex0MtxIdx != g_main_cp_state.matrix_index_a.Tex0MtxIdx ||
+		xfmem.MatrixIndexA.Tex1MtxIdx != g_main_cp_state.matrix_index_a.Tex1MtxIdx ||
+		xfmem.MatrixIndexA.Tex2MtxIdx != g_main_cp_state.matrix_index_a.Tex2MtxIdx ||
+		xfmem.MatrixIndexA.Tex3MtxIdx != g_main_cp_state.matrix_index_a.Tex3MtxIdx ||
+		xfmem.MatrixIndexB.Tex4MtxIdx != g_main_cp_state.matrix_index_b.Tex4MtxIdx ||
+		xfmem.MatrixIndexB.Tex5MtxIdx != g_main_cp_state.matrix_index_b.Tex5MtxIdx ||
+		xfmem.MatrixIndexB.Tex6MtxIdx != g_main_cp_state.matrix_index_b.Tex6MtxIdx ||
+		xfmem.MatrixIndexB.Tex7MtxIdx != g_main_cp_state.matrix_index_b.Tex7MtxIdx)
 	{
 		WARN_LOG(VIDEO, "Matrix indices don't match");
 
@@ -118,18 +118,18 @@ void SWVertexLoader::SetFormat(u8 attributeIndex, u8 primitiveType)
 	m_Vertex.texMtx[6] = xfmem.MatrixIndexB.Tex6MtxIdx;
 	m_Vertex.texMtx[7] = xfmem.MatrixIndexB.Tex7MtxIdx;
 #else
-	m_Vertex.posMtx = MatrixIndexA.PosNormalMtxIdx;
-	m_Vertex.texMtx[0] = MatrixIndexA.Tex0MtxIdx;
-	m_Vertex.texMtx[1] = MatrixIndexA.Tex1MtxIdx;
-	m_Vertex.texMtx[2] = MatrixIndexA.Tex2MtxIdx;
-	m_Vertex.texMtx[3] = MatrixIndexA.Tex3MtxIdx;
-	m_Vertex.texMtx[4] = MatrixIndexB.Tex4MtxIdx;
-	m_Vertex.texMtx[5] = MatrixIndexB.Tex5MtxIdx;
-	m_Vertex.texMtx[6] = MatrixIndexB.Tex6MtxIdx;
-	m_Vertex.texMtx[7] = MatrixIndexB.Tex7MtxIdx;
+	m_Vertex.posMtx = g_main_cp_state.matrix_index_a.PosNormalMtxIdx;
+	m_Vertex.texMtx[0] = g_main_cp_state.matrix_index_a.Tex0MtxIdx;
+	m_Vertex.texMtx[1] = g_main_cp_state.matrix_index_a.Tex1MtxIdx;
+	m_Vertex.texMtx[2] = g_main_cp_state.matrix_index_a.Tex2MtxIdx;
+	m_Vertex.texMtx[3] = g_main_cp_state.matrix_index_a.Tex3MtxIdx;
+	m_Vertex.texMtx[4] = g_main_cp_state.matrix_index_b.Tex4MtxIdx;
+	m_Vertex.texMtx[5] = g_main_cp_state.matrix_index_b.Tex5MtxIdx;
+	m_Vertex.texMtx[6] = g_main_cp_state.matrix_index_b.Tex6MtxIdx;
+	m_Vertex.texMtx[7] = g_main_cp_state.matrix_index_b.Tex7MtxIdx;
 #endif
 
-	if (g_VtxDesc.PosMatIdx != NOT_PRESENT)
+	if (g_main_cp_state.vtx_desc.PosMatIdx != NOT_PRESENT)
 	{
 		AddAttributeLoader(LoadPosMtx);
 		m_VertexSize++;
@@ -145,17 +145,17 @@ void SWVertexLoader::SetFormat(u8 attributeIndex, u8 primitiveType)
 	}
 
 	// Write vertex position loader
-	m_positionLoader = VertexLoader_Position::GetFunction(g_VtxDesc.Position, m_CurrentVat->g0.PosFormat, m_CurrentVat->g0.PosElements);
-	m_VertexSize += VertexLoader_Position::GetSize(g_VtxDesc.Position, m_CurrentVat->g0.PosFormat, m_CurrentVat->g0.PosElements);
+	m_positionLoader = VertexLoader_Position::GetFunction(g_main_cp_state.vtx_desc.Position, m_CurrentVat->g0.PosFormat, m_CurrentVat->g0.PosElements);
+	m_VertexSize += VertexLoader_Position::GetSize(g_main_cp_state.vtx_desc.Position, m_CurrentVat->g0.PosFormat, m_CurrentVat->g0.PosElements);
 	AddAttributeLoader(LoadPosition);
 
 	// Normals
-	if (g_VtxDesc.Normal != NOT_PRESENT)
+	if (g_main_cp_state.vtx_desc.Normal != NOT_PRESENT)
 	{
-		m_VertexSize += VertexLoader_Normal::GetSize(g_VtxDesc.Normal,
+		m_VertexSize += VertexLoader_Normal::GetSize(g_main_cp_state.vtx_desc.Normal,
 			m_CurrentVat->g0.NormalFormat, m_CurrentVat->g0.NormalElements, m_CurrentVat->g0.NormalIndex3);
 
-		m_normalLoader = VertexLoader_Normal::GetFunction(g_VtxDesc.Normal,
+		m_normalLoader = VertexLoader_Normal::GetFunction(g_main_cp_state.vtx_desc.Normal,
 			m_CurrentVat->g0.NormalFormat, m_CurrentVat->g0.NormalElements, m_CurrentVat->g0.NormalIndex3);
 
 		if (m_normalLoader == nullptr)
@@ -234,8 +234,8 @@ void SWVertexLoader::SetFormat(u8 attributeIndex, u8 primitiveType)
 
 	// special case if only pos and tex coord 0 and tex coord input is AB11
 	m_TexGenSpecialCase =
-		((g_VtxDesc.Hex & 0x60600L) == g_VtxDesc.Hex) && // only pos and tex coord 0
-		(g_VtxDesc.Tex0Coord != NOT_PRESENT) &&
+		((g_main_cp_state.vtx_desc.Hex & 0x60600L) == g_main_cp_state.vtx_desc.Hex) && // only pos and tex coord 0
+		(g_main_cp_state.vtx_desc.Tex0Coord != NOT_PRESENT) &&
 		(xfmem.texMtxInfo[0].projection == XF_TEXPROJ_ST);
 
 	m_SetupUnit->Init(primitiveType);
@@ -252,7 +252,7 @@ void SWVertexLoader::LoadVertex()
 	// transform input data
 	TransformUnit::TransformPosition(&m_Vertex, outVertex);
 
-	if (g_VtxDesc.Normal != NOT_PRESENT)
+	if (g_main_cp_state.vtx_desc.Normal != NOT_PRESENT)
 	{
 		TransformUnit::TransformNormal(&m_Vertex, m_CurrentVat->g0.NormalElements, outVertex);
 	}

Source/Core/VideoBackends/Software/SWmain.cpp

@@ -116,14 +116,7 @@ void VideoSoftware::DoState(PointerWrap& p)
 	p.DoPOD(swstats);
 
 	// CP Memory
-	p.DoArray(arraybases, 16);
-	p.DoArray(arraystrides, 16);
-	p.Do(MatrixIndexA);
-	p.Do(MatrixIndexB);
-	p.Do(g_VtxDesc.Hex);
-	p.DoArray(g_VtxAttr, 8);
-	p.DoMarker("CP Memory");
-
+	DoCPState(p);
 }
 
 void VideoSoftware::CheckInvalidState()

Source/Core/VideoBackends/Software/XFMemLoader.cpp

@@ -74,7 +74,7 @@ void SWLoadIndexedXF(u32 val, int array)
 	int size = ((val >> 12) & 0xF) + 1;
 	//load stuff from array to address in xf mem
 
-	u32 *pData = (u32*)Memory::GetPointer(arraybases[array] + arraystrides[array]*index);
+	u32 *pData = (u32*)Memory::GetPointer(g_main_cp_state.array_bases[array] + g_main_cp_state.array_strides[array]*index);
 
 	// byteswap data
 	u32 buffer[16];

Source/Core/VideoCommon/BPMemory.h

@@ -1085,5 +1085,6 @@ struct BPMemory
 extern BPMemory bpmem;
 
 void LoadBPReg(u32 value0);
+void LoadBPRegPreprocess(u32 value0);
 
 void GetBPRegInfo(const u8* data, std::string* name, std::string* desc);

Source/Core/VideoCommon/BPStructs.cpp

@@ -173,7 +173,8 @@ static void BPWritten(const BPCmd& bp)
 		switch (bp.newvalue & 0xFF)
 		{
 		case 0x02:
-			PixelEngine::SetFinish(); // may generate interrupt
+			if (!g_use_deterministic_gpu_thread)
+				PixelEngine::SetFinish(); // may generate interrupt
 			DEBUG_LOG(VIDEO, "GXSetDrawDone SetPEFinish (value: 0x%02X)", (bp.newvalue & 0xFFFF));
 			return;
 
@@ -183,11 +184,13 @@ static void BPWritten(const BPCmd& bp)
 		}
 		return;
 	case BPMEM_PE_TOKEN_ID: // Pixel Engine Token ID
-		PixelEngine::SetToken(static_cast<u16>(bp.newvalue & 0xFFFF), false);
+		if (!g_use_deterministic_gpu_thread)
+			PixelEngine::SetToken(static_cast<u16>(bp.newvalue & 0xFFFF), false);
 		DEBUG_LOG(VIDEO, "SetPEToken 0x%04x", (bp.newvalue & 0xFFFF));
 		return;
 	case BPMEM_PE_TOKEN_INT_ID: // Pixel Engine Interrupt Token ID
-		PixelEngine::SetToken(static_cast<u16>(bp.newvalue & 0xFFFF), true);
+		if (!g_use_deterministic_gpu_thread)
+			PixelEngine::SetToken(static_cast<u16>(bp.newvalue & 0xFFFF), true);
 		DEBUG_LOG(VIDEO, "SetPEToken + INT 0x%04x", (bp.newvalue & 0xFFFF));
 		return;
 
@@ -685,6 +688,26 @@ void LoadBPReg(u32 value0)
 	BPWritten(bp);
 }
 
+void LoadBPRegPreprocess(u32 value0)
+{
+	int regNum = value0 >> 24;
+	// masking could hypothetically be a problem
+	u32 newval = value0 & 0xffffff;
+	switch (regNum)
+	{
+	case BPMEM_SETDRAWDONE:
+		if ((newval & 0xff) == 0x02)
+			PixelEngine::SetFinish();
+		break;
+	case BPMEM_PE_TOKEN_ID:
+		PixelEngine::SetToken(newval & 0xffff, false);
+		break;
+	case BPMEM_PE_TOKEN_INT_ID: // Pixel Engine Interrupt Token ID
+		PixelEngine::SetToken(newval & 0xffff, true);
+		break;
+	}
+}
+
 void GetBPRegInfo(const u8* data, std::string* name, std::string* desc)
 {
 	const char* no_yes[2] = { "No", "Yes" };

Source/Core/VideoCommon/BPStructs.h

@@ -7,5 +7,4 @@
 #include "VideoCommon/BPMemory.h"
 
 void BPInit();
-void LoadBPReg(u32 value0);
 void BPReload();

Source/Core/VideoCommon/CPMemory.cpp

@@ -2,17 +2,32 @@
 // Licensed under GPLv2
 // Refer to the license.txt file included.
 
+#include "Common/ChunkFile.h"
 #include "Common/CommonTypes.h"
 #include "VideoCommon/CPMemory.h"
 
 // CP state
 u8 *cached_arraybases[16];
 
-// STATE_TO_SAVE
-u32 arraybases[16];
-u32 arraystrides[16];
-TMatrixIndexA MatrixIndexA;
-TMatrixIndexB MatrixIndexB;
-TVtxDesc g_VtxDesc;
-// Most games only use the first VtxAttr and simply reconfigure it all the time as needed.
-VAT g_VtxAttr[8];
+CPState g_main_cp_state;
+CPState g_preprocess_cp_state;
+
+void DoCPState(PointerWrap& p)
+{
+	// We don't save g_preprocess_cp_state separately because the GPU should be
+	// synced around state save/load.
+	p.DoArray(g_main_cp_state.array_bases, 16);
+	p.DoArray(g_main_cp_state.array_strides, 16);
+	p.Do(g_main_cp_state.matrix_index_a);
+	p.Do(g_main_cp_state.matrix_index_b);
+	p.Do(g_main_cp_state.vtx_desc.Hex);
+	p.DoArray(g_main_cp_state.vtx_attr, 8);
+	p.DoMarker("CP Memory");
+	if (p.mode == PointerWrap::MODE_READ)
+		CopyPreprocessCPStateFromMain();
+}
+
+void CopyPreprocessCPStateFromMain()
+{
+   memcpy(&g_preprocess_cp_state, &g_main_cp_state, sizeof(CPState));
+}

Source/Core/VideoCommon/CPMemory.h

@@ -231,12 +231,6 @@ union TMatrixIndexB
 
 #pragma pack()
 
-extern u32 arraybases[16];
-extern u8 *cached_arraybases[16];
-extern u32 arraystrides[16];
-extern TMatrixIndexA MatrixIndexA;
-extern TMatrixIndexB MatrixIndexB;
-
 struct VAT
 {
 	UVAT_group0 g0;
@@ -244,11 +238,37 @@ struct VAT
 	UVAT_group2 g2;
 };
 
-extern TVtxDesc g_VtxDesc;
-extern VAT g_VtxAttr[8];
+class VertexLoader;
+
+// STATE_TO_SAVE
+struct CPState final
+{
+    u32 array_bases[16];
+    u32 array_strides[16];
+    TMatrixIndexA matrix_index_a;
+    TMatrixIndexB matrix_index_b;
+    TVtxDesc vtx_desc;
+    // Most games only use the first VtxAttr and simply reconfigure it all the time as needed.
+    VAT vtx_attr[8];
+
+	// Attributes that actually belong to VertexLoaderManager:
+	int attr_dirty; // bitfield
+	VertexLoader* vertex_loaders[8];
+};
+
+class PointerWrap;
+
+extern void DoCPState(PointerWrap& p);
+
+extern void CopyPreprocessCPStateFromMain();
+
+extern CPState g_main_cp_state;
+extern CPState g_preprocess_cp_state;
+
+extern u8 *cached_arraybases[16];
 
 // Might move this into its own file later.
-void LoadCPReg(u32 SubCmd, u32 Value);
+void LoadCPReg(u32 SubCmd, u32 Value, bool is_preprocess = false);
 
 // Fills memory with data from CP regs
 void FillCPMemoryArray(u32 *memory);