LLVM: Slice PPU executable memory

Utilities/JIT.h

@@ -514,8 +514,8 @@ class jit_compiler final
 	atomic_t<usz> m_disk_space = umax;
 
 public:
-	jit_compiler(const std::unordered_map<std::string, u64>& _link, const std::string& _cpu, u32 flags = 0);
-	~jit_compiler();
+	jit_compiler(const std::unordered_map<std::string, u64>& _link, const std::string& _cpu, u32 flags = 0, std::function<u64(const std::string&)> symbols_cement = {}) noexcept;
+	~jit_compiler() noexcept;
 
 	// Get LLVM context
 	auto& get_context()

Utilities/JITLLVM.cpp

@@ -77,8 +77,7 @@ static u64 make_null_function(const std::string& name)
 
 		if (res.ec == std::errc() && res.ptr == name.c_str() + name.size() && addr < 0x8000'0000)
 		{
-			// Point the garbage to reserved, non-executable memory
-			return reinterpret_cast<u64>(vm::g_sudo_addr + addr);
+			fmt::throw_exception("Unhandled symbols cementing! (name='%s'", name);
 		}
 	}
 
@@ -174,18 +173,34 @@ struct JITAnnouncer : llvm::JITEventListener
 struct MemoryManager1 : llvm::RTDyldMemoryManager
 {
 	// 256 MiB for code or data
-	static constexpr u64 c_max_size = 0x20000000 / 2;
+	static constexpr u64 c_max_size = 0x1000'0000;
 
 	// Allocation unit (2M)
 	static constexpr u64 c_page_size = 2 * 1024 * 1024;
 
-	// Reserve 512 MiB
-	u8* const ptr = static_cast<u8*>(utils::memory_reserve(c_max_size * 2));
+	// Reserve 256 MiB blocks
+	void* m_code_mems = nullptr;
+	void* m_data_ro_mems = nullptr;
+	void* m_data_rw_mems = nullptr;
 
 	u64 code_ptr = 0;
-	u64 data_ptr = c_max_size;
+	u64 data_ro_ptr = 0;
+	u64 data_rw_ptr = 0;
 
-	MemoryManager1() = default;
+	// First fallback for non-existing symbols
+	// May be a memory container internally
+	std::function<u64(const std::string&)> m_symbols_cement;
+
+	MemoryManager1(std::function<u64(const std::string&)> symbols_cement = {}) noexcept
+		: m_symbols_cement(std::move(symbols_cement))
+	{
+		auto ptr = reinterpret_cast<u8*>(utils::memory_reserve(c_max_size * 3));
+		m_code_mems = ptr;
+		// ptr += c_max_size;
+		// m_data_ro_mems = ptr;
+		 ptr += c_max_size;
+		m_data_rw_mems = ptr;
+	}
 
 	MemoryManager1(const MemoryManager1&) = delete;
 
@@ -194,13 +209,22 @@ struct MemoryManager1 : llvm::RTDyldMemoryManager
 	~MemoryManager1() override
 	{
 		// Hack: don't release to prevent reuse of address space, see jit_announce
-		utils::memory_decommit(ptr, c_max_size * 2);
+		// constexpr auto how_much = [](u64 pos) { return utils::align(pos, pos < c_page_size ? c_page_size / 4 : c_page_size); };
+		// utils::memory_decommit(m_code_mems, how_much(code_ptr));
+		// utils::memory_decommit(m_data_ro_mems, how_much(data_ro_ptr));
+		// utils::memory_decommit(m_data_rw_mems, how_much(data_rw_ptr));
+		utils::memory_decommit(m_code_mems, c_max_size * 3);
 	}
 
 	llvm::JITSymbol findSymbol(const std::string& name) override
 	{
 		u64 addr = RTDyldMemoryManager::getSymbolAddress(name);
 
+		if (!addr && m_symbols_cement)
+		{
+			addr = m_symbols_cement(name);
+		}
+
 		if (!addr)
 		{
 			addr = make_null_function(name);
@@ -214,45 +238,79 @@ struct MemoryManager1 : llvm::RTDyldMemoryManager
 		return {addr, llvm::JITSymbolFlags::Exported};
 	}
 
-	u8* allocate(u64& oldp, uptr size, uint align, utils::protection prot)
+	u8* allocate(u64& alloc_pos, void* block, uptr size, u64 align, utils::protection prot)
 	{
-		if (align > c_page_size)
+		align = align ? align : 16;
+ 
+		const u64 sizea = utils::align(size, align);
+
+		if (!size || align > c_page_size || sizea > c_max_size || sizea < size)
 		{
-			jit_log.fatal("Unsupported alignment (size=0x%x, align=0x%x)", size, align);
+			jit_log.fatal("Unsupported size/alignment (size=0x%x, align=0x%x)", size, align);
 			return nullptr;
 		}
 
-		const u64 olda = utils::align(oldp, align);
-		const u64 newp = utils::align(olda + size, align);
+		u64 oldp = alloc_pos;
 
-		if ((newp - 1) / c_max_size != oldp / c_max_size)
+		u64 olda = utils::align(oldp, align);
+
+		ensure(olda >= oldp);
+		ensure(olda < ~sizea);
+
+		u64 newp = olda + sizea;
+
+		if ((newp - 1) / c_max_size != (oldp - 1) / c_max_size)
 		{
-			jit_log.fatal("Out of memory (size=0x%x, align=0x%x)", size, align);
-			return nullptr;
+			constexpr usz num_of_allocations = 1;
+
+			if ((newp - 1) / c_max_size > num_of_allocations)
+			{
+				// Allocating more than one region does not work for relocations, needs more robust solution
+				fmt::throw_exception("Out of memory (size=0x%x, align=0x%x)", size, align);
+			}
 		}
 
-		if ((oldp - 1) / c_page_size != (newp - 1) / c_page_size)
+		// Update allocation counter
+		alloc_pos = newp;
+
+		constexpr usz page_quarter = c_page_size / 4;
+
+		// Optimization: split the first allocation to 512 KiB for single-module compilers
+		if (oldp < c_page_size && align < page_quarter && (std::min(newp, c_page_size) - 1) / page_quarter != (oldp - 1) / page_quarter)
+		{
+			const u64 pagea = utils::align(oldp, page_quarter);
+			const u64 psize = utils::align(std::min(newp, c_page_size) - pagea, page_quarter);
+			utils::memory_commit(reinterpret_cast<u8*>(block) + (pagea % c_max_size), psize, prot);
+
+			// Advance
+			oldp = pagea + psize;
+		}
+
+		if ((newp - 1) / c_page_size != (oldp - 1) / c_page_size)
 		{
 			// Allocate pages on demand
 			const u64 pagea = utils::align(oldp, c_page_size);
 			const u64 psize = utils::align(newp - pagea, c_page_size);
-			utils::memory_commit(this->ptr + pagea, psize, prot);
+			utils::memory_commit(reinterpret_cast<u8*>(block) + (pagea % c_max_size), psize, prot);
 		}
 
-		// Update allocation counter
-		oldp = newp;
-
-		return this->ptr + olda;
+		return reinterpret_cast<u8*>(block) + (olda % c_max_size);
 	}
 
 	u8* allocateCodeSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/) override
 	{
-		return allocate(code_ptr, size, align, utils::protection::wx);
+		return allocate(code_ptr, m_code_mems, size, align, utils::protection::wx);
 	}
 
-	u8* allocateDataSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/, bool /*is_ro*/) override
+	u8* allocateDataSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/, bool is_ro) override
 	{
-		return allocate(data_ptr, size, align, utils::protection::rw);
+		if (is_ro)
+		{
+			// Disabled
+			//return allocate(data_ro_ptr, m_data_ro_mems, size, align, utils::protection::rw);
+		}
+
+		return allocate(data_rw_ptr, m_data_rw_mems, size, align, utils::protection::rw);
 	}
 
 	bool finalizeMemory(std::string* = nullptr) override
@@ -272,7 +330,14 @@ struct MemoryManager1 : llvm::RTDyldMemoryManager
 // Simple memory manager
 struct MemoryManager2 : llvm::RTDyldMemoryManager
 {
-	MemoryManager2() = default;
+	// First fallback for non-existing symbols
+	// May be a memory container internally
+	std::function<u64(const std::string&)> m_symbols_cement;
+
+	MemoryManager2(std::function<u64(const std::string&)> symbols_cement = {}) noexcept
+		: m_symbols_cement(std::move(symbols_cement))
+	{
+	}
 
 	~MemoryManager2() override
 	{
@@ -282,6 +347,11 @@ struct MemoryManager2 : llvm::RTDyldMemoryManager
 	{
 		u64 addr = RTDyldMemoryManager::getSymbolAddress(name);
 
+		if (!addr && m_symbols_cement)
+		{
+			addr = m_symbols_cement(name);
+		}
+
 		if (!addr)
 		{
 			addr = make_null_function(name);
@@ -561,7 +631,7 @@ bool jit_compiler::add_sub_disk_space(ssz space)
 	}).second;
 }
 
-jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, const std::string& _cpu, u32 flags)
+jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, const std::string& _cpu, u32 flags, std::function<u64(const std::string&)> symbols_cement) noexcept
 	: m_context(new llvm::LLVMContext)
 	, m_cpu(cpu(_cpu))
 {
@@ -589,17 +659,17 @@ jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, co
 		// Auxiliary JIT (does not use custom memory manager, only writes the objects)
 		if (flags & 0x1)
 		{
-			mem = std::make_unique<MemoryManager1>();
+			mem = std::make_unique<MemoryManager1>(std::move(symbols_cement));
 		}
 		else
 		{
-			mem = std::make_unique<MemoryManager2>();
+			mem = std::make_unique<MemoryManager2>(std::move(symbols_cement));
 			null_mod->setTargetTriple(jit_compiler::triple2());
 		}
 	}
 	else
 	{
-		mem = std::make_unique<MemoryManager1>();
+		mem = std::make_unique<MemoryManager1>(std::move(symbols_cement));
 	}
 
 	{
@@ -648,7 +718,7 @@ jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, co
 	}
 }
 
-jit_compiler::~jit_compiler()
+jit_compiler::~jit_compiler() noexcept
 {
 }
 

Utilities/Thread.h

@@ -769,7 +769,7 @@ public:
 		}
 
 		// Move the context (if movable)
-		new (static_cast<void*>(m_threads + m_count - 1)) Thread(std::string(name) + std::to_string(m_count - 1), std::forward<Context>(f));
+		new (static_cast<void*>(m_threads + m_count - 1)) Thread(std::string(name) + std::to_string(m_count), std::forward<Context>(f));
 	}
 
 	// Constructor with a function performed before adding more threads

rpcs3/Emu/Cell/PPUAnalyser.h

@@ -4,6 +4,7 @@
 #include <map>
 #include <set>
 #include <deque>
+#include <span>
 #include "util/types.hpp"
 #include "util/endian.hpp"
 #include "util/asm.hpp"
@@ -38,7 +39,51 @@ struct ppu_function
 	std::map<u32, u32> blocks{}; // Basic blocks: addr -> size
 	std::set<u32> calls{}; // Set of called functions
 	std::set<u32> callers{};
-	std::string name{}; // Function name
+	mutable std::string name{}; // Function name
+
+	struct iterator
+	{
+		const ppu_function* _this;
+		typename std::map<u32, u32>::const_iterator it;
+		usz index = 0;
+
+		std::pair<const u32, u32> operator*() const
+		{
+			return _this->blocks.empty() ? std::pair<const u32, u32>(_this->addr, _this->size) : *it;
+		}
+
+		iterator& operator++()
+		{
+			index++;
+
+			if (it != _this->blocks.end())
+			{
+				it++;
+			}
+
+			return *this;
+		}
+
+		bool operator==(const iterator& rhs) const noexcept
+		{
+			return it == rhs.it || (rhs.index == index && _this->blocks.empty());
+		}
+
+		bool operator!=(const iterator& rhs) const noexcept
+		{
+			return !operator==(rhs);
+		}
+	};
+
+	iterator begin() const
+	{
+		return iterator{this, blocks.begin()};
+	}
+
+	iterator end() const
+	{
+		return iterator{this, blocks.end(), 1};
+	}
 };
 
 // PPU Relocation Information
@@ -87,18 +132,56 @@ struct ppu_module : public Type
 
 	ppu_module& operator=(ppu_module&&) noexcept = default;
 
-	uchar sha1[20]{};
-	std::string name{};
-	std::string path{};
+	uchar sha1[20]{}; // Hash
+	std::string name{}; // Filename
+	std::string path{}; // Filepath
 	s64 offset = 0; // Offset of file
-	std::string cache{};
-	std::vector<ppu_reloc> relocs{};
-	std::vector<ppu_segment> segs{};
-	std::vector<ppu_segment> secs{};
-	std::vector<ppu_function> funcs{};
-	std::vector<u32> applied_patches;
-	std::deque<std::shared_ptr<void>> allocations;
-	std::map<u32, u32> addr_to_seg_index;
+	mutable bs_t<ppu_attr> attr{}; // Shared module attributes
+	std::string cache{}; // Cache file path
+	std::vector<ppu_reloc> relocs{}; // Relocations
+	std::vector<ppu_segment> segs{}; // Segments
+	std::vector<ppu_segment> secs{}; // Segment sections
+	std::vector<ppu_function> funcs{}; // Function list
+	std::vector<u32> applied_patches; // Patch addresses
+	std::deque<std::shared_ptr<void>> allocations; // Segment memory allocations
+	std::map<u32, u32> addr_to_seg_index; // address->segment ordered translator map
+	ppu_module* parent = nullptr;
+	std::pair<u32, u32> local_bounds{0, u32{umax}}; // Module addresses range
+	std::shared_ptr<std::pair<u32, u32>> jit_bounds; // JIT instance modules addresses range
+
+	template <typename T>
+	auto as_span(T&& arg, bool bound_local, bool bound_jit) const
+	{
+		using unref = std::remove_reference_t<T>;
+		using type = std::conditional_t<std::is_const_v<unref>, std::add_const_t<typename unref::value_type>, typename unref::value_type>;
+
+		if (bound_local || bound_jit)
+		{
+			// Return span bound to specified bounds
+			const auto [min_addr, max_addr] = bound_jit ? *jit_bounds : local_bounds;
+			constexpr auto compare = [](const type& a, u32 addr) { return a.addr < addr; };
+			const auto end = arg.data() + arg.size();
+			const auto start = std::lower_bound(arg.data(), end, min_addr, compare);
+			return std::span<type>{ start, std::lower_bound(start, end, max_addr, compare) };
+		}
+
+		return std::span<type>(arg.data(), arg.size());
+	}
+
+	auto get_funcs(bool bound_local = true, bool bound_jit = false)
+	{
+		return as_span(parent ? parent->funcs : funcs, bound_local, bound_jit);
+	}
+
+	auto get_funcs(bool bound_local = true, bool bound_jit = false) const
+	{
+		return as_span(parent ? parent->funcs : funcs, bound_local, bound_jit);
+	}
+
+	auto get_relocs(bool bound_local = false) const
+	{
+		return as_span(parent ? parent->relocs : relocs, bound_local, false);
+	}
 
 	// Copy info without functions
 	void copy_part(const ppu_module& info)
@@ -106,11 +189,12 @@ struct ppu_module : public Type
 		std::memcpy(sha1, info.sha1, sizeof(sha1));
 		name = info.name;
 		path = info.path;
-		relocs = info.relocs;
 		segs = info.segs;
-		secs = info.secs;
 		allocations = info.allocations;
 		addr_to_seg_index = info.addr_to_seg_index;
+		parent = const_cast<ppu_module*>(&info);
+		attr = info.attr;
+		local_bounds = {u32{umax}, 0}; // Initially empty range
 	}
 
 	bool analyse(u32 lib_toc, u32 entry, u32 end, const std::vector<u32>& applied, const std::vector<u32>& exported_funcs = std::vector<u32>{}, std::function<bool()> check_aborted = {});

rpcs3/Emu/Cell/PPUTranslator.cpp

@@ -114,7 +114,7 @@ PPUTranslator::PPUTranslator(LLVMContext& context, Module* _module, const ppu_mo
 	const auto caddr = m_info.segs[0].addr;
 	const auto cend = caddr + m_info.segs[0].size;
 
-	for (const auto& rel : m_info.relocs)
+	for (const auto& rel : m_info.get_relocs())
 	{
 		if (rel.addr >= caddr && rel.addr < cend)
 		{
@@ -162,7 +162,7 @@ PPUTranslator::PPUTranslator(LLVMContext& context, Module* _module, const ppu_mo
 		}
 	}
 
-	if (!m_info.relocs.empty())
+	if (!m_info.get_relocs().empty())
 	{
 		m_reloc = &m_info.segs[0];
 	}
@@ -196,7 +196,7 @@ Function* PPUTranslator::Translate(const ppu_function& info)
 	// Instruction address is (m_addr + base)
 	const u64 base = m_reloc ? m_reloc->addr : 0;
 	m_addr = info.addr - base;
-	m_attr = info.attr;
+	m_attr = m_info.attr + info.attr;
 
 	// Don't emit check in small blocks without terminator
 	bool need_check = info.size >= 16;
@@ -325,6 +325,9 @@ Function* PPUTranslator::Translate(const ppu_function& info)
 
 Function* PPUTranslator::GetSymbolResolver(const ppu_module<lv2_obj>& info)
 {
+	ensure(m_module->getFunction("__resolve_symbols") == nullptr);
+	ensure(info.jit_bounds);
+
 	m_function = cast<Function>(m_module->getOrInsertFunction("__resolve_symbols", FunctionType::get(get_type<void>(), { get_type<u8*>(), get_type<u64>() }, false)).getCallee());
 
 	IRBuilder<> irb(BasicBlock::Create(m_context, "__entry", m_function));
@@ -351,12 +354,13 @@ Function* PPUTranslator::GetSymbolResolver(const ppu_module<lv2_obj>& info)
 	// This is made in loop instead of inlined because it took tremendous amount of time to compile.
 
 	std::vector<u32> vec_addrs;
-	vec_addrs.reserve(info.funcs.size());
 
 	// Create an array of function pointers
 	std::vector<llvm::Constant*> functions;
 
-	for (const auto& f : info.funcs)
+	const auto [min_addr, max_addr] = *ensure(info.jit_bounds);
+
+	for (const auto& f : info.get_funcs(false, true))
 	{
 		if (!f.size)
 		{
@@ -379,7 +383,7 @@ Function* PPUTranslator::GetSymbolResolver(const ppu_module<lv2_obj>& info)
 	const auto addr_array = new GlobalVariable(*m_module, addr_array_type, false, GlobalValue::PrivateLinkage, ConstantDataArray::get(m_context, vec_addrs));
 
 	// Create an array of function pointers
-	const auto func_table_type = ArrayType::get(ftype->getPointerTo(), info.funcs.size());
+	const auto func_table_type = ArrayType::get(ftype->getPointerTo(), functions.size());
 	const auto init_func_table = ConstantArray::get(func_table_type, functions);
 	const auto func_table = new GlobalVariable(*m_module, func_table_type, false, GlobalVariable::PrivateLinkage, init_func_table);
 

rpcs3/util/vm_native.cpp

@@ -310,6 +310,11 @@ namespace utils
 
 	void memory_commit(void* pointer, usz size, protection prot)
 	{
+		if (!size)
+		{
+			return;
+		}
+
 #ifdef _WIN32
 		ensure(::VirtualAlloc(pointer, size, MEM_COMMIT, +prot));
 #else
@@ -329,6 +334,11 @@ namespace utils
 
 	void memory_decommit(void* pointer, usz size)
 	{
+		if (!size)
+		{
+			return;
+		}
+
 #ifdef _WIN32
 		ensure(::VirtualFree(pointer, size, MEM_DECOMMIT));
 #else
@@ -357,6 +367,11 @@ namespace utils
 
 	void memory_reset(void* pointer, usz size, protection prot)
 	{
+		if (!size)
+		{
+			return;
+		}
+
 #ifdef _WIN32
 		memory_decommit(pointer, size);
 		memory_commit(pointer, size, prot);
@@ -390,6 +405,11 @@ namespace utils
 
 	void memory_release(void* pointer, usz size)
 	{
+		if (!size)
+		{
+			return;
+		}
+
 #ifdef _WIN32
 		unmap_mappping_memory(reinterpret_cast<u64>(pointer), size);
 		ensure(::VirtualFree(pointer, 0, MEM_RELEASE));
@@ -400,6 +420,11 @@ namespace utils
 
 	void memory_protect(void* pointer, usz size, protection prot)
 	{
+		if (!size)
+		{
+			return;
+		}
+
 #ifdef _WIN32
 
 		DWORD old;
@@ -429,6 +454,11 @@ namespace utils
 
 	bool memory_lock(void* pointer, usz size)
 	{
+		if (!size)
+		{
+			return true;
+		}
+
 #ifdef _WIN32
 		return ::VirtualLock(pointer, size);
 #else