diff --git a/pcsx2-gsrunner/Main.cpp b/pcsx2-gsrunner/Main.cpp
index 1ef81a3748..142efa5399 100644
--- a/pcsx2-gsrunner/Main.cpp
+++ b/pcsx2-gsrunner/Main.cpp
@@ -899,6 +899,8 @@ int main(int argc, char* argv[])
 	if (!GSRunner::ParseCommandLineArgs(argc, argv, params))
 		return EXIT_FAILURE;
 
+	SysMemory::ReserveMemory();
+
 	if (s_use_window.value_or(true) && !GSRunner::CreatePlatformWindow())
 	{
 		Console.Error("Failed to create window.");
diff --git a/pcsx2-qt/QtHost.cpp b/pcsx2-qt/QtHost.cpp
index 517ff53233..073d4c62a7 100644
--- a/pcsx2-qt/QtHost.cpp
+++ b/pcsx2-qt/QtHost.cpp
@@ -2406,6 +2406,8 @@ int main(int argc, char* argv[])
 	if (!QtHost::ParseCommandLineOptions(app.arguments(), autoboot))
 		return EXIT_FAILURE;
 
+	SysMemory::ReserveMemory();
+
 	// Bail out if we can't find any config.
 	if (!QtHost::InitializeConfig())
 		return EXIT_FAILURE;
diff --git a/pcsx2/Memory.cpp b/pcsx2/Memory.cpp
index a8431bd0bc..c65f19beac 100644
--- a/pcsx2/Memory.cpp
+++ b/pcsx2/Memory.cpp
@@ -185,7 +185,6 @@ bool SysMemory::AllocateMemoryMap()
 	HostMemoryMap::IOPmem = (uptr)(s_data_memory + HostMemoryMap::IOPmemOffset);
 	HostMemoryMap::VUmem = (uptr)(s_data_memory + HostMemoryMap::VUmemOffset);
 
-	DumpMemoryMap();
 	return true;
 }
 
@@ -235,13 +234,21 @@ void SysMemory::ReleaseMemoryMap()
 	}
 }
 
+void SysMemory::ReserveMemory()
+{
+	if (!s_data_memory_file_handle)
+		AllocateMemoryMap();
+}
+
 bool SysMemory::Allocate()
 {
 	DevCon.WriteLn(Color_StrongBlue, "Allocating host memory for virtual systems...");
 
-	if (!AllocateMemoryMap())
+	if (!s_data_memory_file_handle && !AllocateMemoryMap())
 		return false;
 
+	DumpMemoryMap();
+
 	memAllocate();
 	iopMemAlloc();
 	vuMemAllocate();
@@ -273,8 +280,6 @@ void SysMemory::Release()
 	vuMemRelease();
 	iopMemRelease();
 	memRelease();
-
-	ReleaseMemoryMap();
 }
 
 u8* SysMemory::GetDataPtr(size_t offset)
diff --git a/pcsx2/Memory.h b/pcsx2/Memory.h
index 36f5e2ac37..63543ded78 100644
--- a/pcsx2/Memory.h
+++ b/pcsx2/Memory.h
@@ -97,6 +97,8 @@ namespace HostMemoryMap
 
 namespace SysMemory
 {
+	/// Call this as early as possible to maximize chance of getting the address space we want
+	void ReserveMemory();
 	bool Allocate();
 	void Reset();
 	void Release();