diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc index 10a945620..88c2bbf33 100644 --- a/src/xenia/cpu/backend/a64/a64_assembler.cc +++ b/src/xenia/cpu/backend/a64/a64_assembler.cc @@ -92,11 +92,15 @@ bool A64Assembler::Assemble(GuestFunction* function, HIRBuilder* builder, reinterpret_cast(machine_code), code_size); // Install into indirection table. + auto* code_cache = reinterpret_cast(backend_->code_cache()); uint64_t host_address = reinterpret_cast(machine_code); - assert_true((host_address >> 32) == 0); - reinterpret_cast(backend_->code_cache()) - ->AddIndirection(function->address(), - static_cast(host_address)); + if (code_cache->encoded_indirection()) { + code_cache->AddIndirectionEncoded(function->address(), host_address); + } else { + assert_true((host_address >> 32) == 0); + code_cache->AddIndirection(function->address(), + static_cast(host_address)); + } return true; } diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index 3d3e272a1..6af764c75 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -574,6 +574,15 @@ A64Backend::A64Backend() { break; } } + if (!buf) { + // Fixed allocation failed (e.g. macOS). Allocate at any address. + // Trampolines will be outside the code cache; encoded indirection + // handles this via the external table. + buf = memory::AllocFixed(nullptr, + kGuestTrampolineSize * MAX_GUEST_TRAMPOLINES, + xe::memory::AllocationType::kReserveCommit, + xe::memory::PageAccess::kExecuteReadWrite); + } xenia_assert(buf); guest_trampoline_memory_ = reinterpret_cast(buf); guest_trampoline_address_bitmap_.Resize(MAX_GUEST_TRAMPOLINES); @@ -639,8 +648,13 @@ bool A64Backend::Initialize(Processor* processor) { } // Set the indirection table default to point at the resolve thunk. - code_cache_->set_indirection_default( - uint32_t(reinterpret_cast(resolve_function_thunk_))); + if (code_cache_->encoded_indirection()) { + code_cache_->set_indirection_default_encoded( + reinterpret_cast(resolve_function_thunk_)); + } else { + code_cache_->set_indirection_default( + uint32_t(reinterpret_cast(resolve_function_thunk_))); + } // Commit the indirection table range used by guest trampolines so that // CreateGuestTrampoline can call AddIndirection without faulting. @@ -727,6 +741,21 @@ void A64Backend::InitializeBackendContext(void* ctx) { a64_ctx->flags = (1U << kA64BackendNJMOn); // NJM on by default a64_ctx->guest_tick_count = Clock::GetGuestTickCountPointer(); + // Encoded indirection fields (used when fixed-address allocation failed). + if (code_cache_->encoded_indirection()) { + a64_ctx->indirection_table_bias = + reinterpret_cast(code_cache_->indirection_table_base()) - + static_cast(0x80000000); + a64_ctx->code_execute_base = + reinterpret_cast(code_cache_->generated_code_execute_base()); + a64_ctx->external_indirection_table = + reinterpret_cast(code_cache_->external_table()); + } else { + a64_ctx->indirection_table_bias = 0; + a64_ctx->code_execute_base = 0; + a64_ctx->external_indirection_table = 0; + } + // Allocate stackpoints for longjmp detection. if (cvars::a64_enable_host_guest_stack_synchronization) { uint64_t max_stackpoints = cvars::a64_max_stackpoints; @@ -790,9 +819,14 @@ uint32_t A64Backend::CreateGuestTrampoline(GuestTrampolineProc proc, GUEST_TRAMPOLINE_BASE + (static_cast(new_index) * GUEST_TRAMPOLINE_MIN_LEN); - code_cache()->AddIndirection( - indirection_guest_addr, - static_cast(reinterpret_cast(write_pos))); + if (code_cache()->encoded_indirection()) { + code_cache()->AddIndirectionEncoded(indirection_guest_addr, + reinterpret_cast(write_pos)); + } else { + code_cache()->AddIndirection( + indirection_guest_addr, + static_cast(reinterpret_cast(write_pos))); + } return indirection_guest_addr; } diff --git a/src/xenia/cpu/backend/a64/a64_backend.h b/src/xenia/cpu/backend/a64/a64_backend.h index 7f4ea0ed6..cc86b6a22 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.h +++ b/src/xenia/cpu/backend/a64/a64_backend.h @@ -84,6 +84,13 @@ struct A64BackendContext { // bit 1 = got reserve unsigned int flags; unsigned int Ox1000; // constant 0x1000 + + // Encoded indirection support (for platforms where fixed-address allocation + // fails, e.g. macOS ARM64). When encoded_indirection is false these are + // unused; the fast 2-instruction indirection path is emitted instead. + uintptr_t indirection_table_bias; // actual_table_base - 0x80000000 + uintptr_t code_execute_base; // actual code cache base address + uintptr_t external_indirection_table; // pointer to external uint64_t table }; // Default FPCR for FPU mode (round to nearest, no flush to zero). diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 6be6493e2..aa01bc7c2 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -9,6 +9,12 @@ #include "xenia/cpu/backend/a64/a64_code_cache.h" +#include + +#include "xenia/base/clock.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" #include "xenia/base/platform.h" #if XE_PLATFORM_WIN32 #include "xenia/base/platform_win.h" @@ -19,7 +25,158 @@ namespace cpu { namespace backend { namespace a64 { -bool A64CodeCache::Initialize() { return CodeCacheBase::Initialize(); } +A64CodeCache::~A64CodeCache() { + delete[] external_table_; + external_table_ = nullptr; +} + +bool A64CodeCache::Initialize() { + // Try the fast path: fixed-address allocation. + if (CodeCacheBase::Initialize()) { + encoded_indirection_ = false; + return true; + } + + // Fixed allocation failed (e.g. macOS ARM64). Fall back to dynamic. + XELOGI( + "A64CodeCache: fixed-address allocation failed; using encoded " + "indirection fallback"); + encoded_indirection_ = true; + + // Allocate indirection table at any available address. + indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( + nullptr, kIndirectionTableSize, xe::memory::AllocationType::kReserve, + xe::memory::PageAccess::kReadWrite)); + if (!indirection_table_base_) { + XELOGE("A64CodeCache: unable to allocate indirection table (dynamic)"); + return false; + } + + // Create file mapping for the code cache. + file_name_ = fmt::format("xenia_code_cache_{}", Clock::QueryHostTickCount()); + mapping_ = xe::memory::CreateFileMappingHandle( + file_name_, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, + false); + if (mapping_ == xe::memory::kFileMappingHandleInvalid) { + XELOGE("A64CodeCache: unable to create code cache file mapping (dynamic)"); + return false; + } + + // Map execute and write views at OS-chosen addresses. + if (xe::memory::IsWritableExecutableMemoryPreferred()) { + generated_code_execute_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kExecuteReadWrite, 0)); + generated_code_write_base_ = generated_code_execute_base_; + } else { + generated_code_execute_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kExecuteReadOnly, 0)); + generated_code_write_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kReadWrite, 0)); + } + if (!generated_code_execute_base_ || !generated_code_write_base_) { + XELOGE("A64CodeCache: unable to map code cache views (dynamic)"); + return false; + } + + XELOGI("A64CodeCache: indirection table at {:016X}, code cache at {:016X}", + reinterpret_cast(indirection_table_base_), + reinterpret_cast(generated_code_execute_base_)); + + // Allocate external table for trampoline addresses. + external_table_ = new uint64_t[kMaxExternalEntries](); + external_table_count_ = 0; + + generated_code_map_.reserve(kMaximumFunctionCount); + return true; +} + +void A64CodeCache::set_indirection_default(uint32_t default_value) { + indirection_default_value_ = default_value; +} + +void A64CodeCache::set_indirection_default_encoded(uint64_t default_value) { + // The resolve thunk is in the code cache, encode as cache-relative offset. + uint64_t code_base = reinterpret_cast(generated_code_execute_base_); + encoded_default_value_ = EncodeIndirectionTarget(default_value); + // Also set the base class default for CommitExecutableRange to use. + indirection_default_value_ = encoded_default_value_; +} + +uint32_t A64CodeCache::EncodeIndirectionTarget(uint64_t host_address) { + uint64_t code_base = reinterpret_cast(generated_code_execute_base_); + uint64_t code_end = code_base + kGeneratedCodeSize; + if (host_address >= code_base && host_address < code_end) { + uint32_t offset = static_cast(host_address - code_base); + assert_true((offset & 0x80000000) == 0); + return offset; + } + return AllocateExternalSlot(host_address) | 0x80000000; +} + +uint32_t A64CodeCache::AllocateExternalSlot(uint64_t host_address) { + std::lock_guard lock(external_table_mutex_); + // Check if already registered. + for (uint32_t i = 0; i < external_table_count_; i++) { + if (external_table_[i] == host_address) { + return i; + } + } + assert_true(external_table_count_ < kMaxExternalEntries); + uint32_t index = external_table_count_++; + external_table_[index] = host_address; + return index; +} + +void A64CodeCache::AddIndirection(uint32_t guest_address, + uint32_t host_address) { + if (!encoded_indirection_) { + CodeCacheBase::AddIndirection(guest_address, host_address); + return; + } + AddIndirectionEncoded(guest_address, static_cast(host_address)); +} + +void A64CodeCache::AddIndirectionEncoded(uint32_t guest_address, + uint64_t host_address) { + if (!indirection_table_base_) return; + uint32_t* slot = reinterpret_cast( + indirection_table_base_ + (guest_address - kIndirectionTableBase)); + *slot = EncodeIndirectionTarget(host_address); +} + +void A64CodeCache::CommitExecutableRange(uint32_t guest_low, + uint32_t guest_high) { + if (!encoded_indirection_) { + CodeCacheBase::CommitExecutableRange(guest_low, guest_high); + return; + } + if (!indirection_table_base_) return; + + xe::memory::AllocFixed( + indirection_table_base_ + (guest_low - kIndirectionTableBase), + guest_high - guest_low, xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kReadWrite); + uint32_t* p = reinterpret_cast(indirection_table_base_); + for (uint32_t address = guest_low; address < guest_high; address += 4) { + p[(address - kIndirectionTableBase) / 4] = encoded_default_value_; + } +} + +void A64CodeCache::OnPlaceGuestCodeIndirection(uint32_t guest_address, + void* code_execute_address) { + if (!indirection_table_base_ || !guest_address) return; + if (!encoded_indirection_) { + uint32_t* slot = reinterpret_cast( + indirection_table_base_ + (guest_address - kIndirectionTableBase)); + *slot = uint32_t(reinterpret_cast(code_execute_address)); + } else { + AddIndirectionEncoded(guest_address, + reinterpret_cast(code_execute_address)); + } +} void A64CodeCache::FillCode(void* write_address, size_t size) { // Fill with BRK #0 (0xD4200000), 4-byte aligned. diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h index 3239fea85..f4f35f8c2 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.h +++ b/src/xenia/cpu/backend/a64/a64_code_cache.h @@ -10,7 +10,9 @@ #ifndef XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ #define XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ +#include #include +#include #include "xenia/cpu/backend/code_cache_base.h" @@ -21,12 +23,40 @@ namespace a64 { class A64CodeCache : public CodeCacheBase { public: - ~A64CodeCache() override = default; + ~A64CodeCache() override; static std::unique_ptr Create(); virtual bool Initialize(); + // Whether the indirection table uses encoded (relative) entries rather + // than direct absolute 32-bit host addresses. True when fixed-address + // allocation failed (e.g. macOS ARM64). + bool encoded_indirection() const { return encoded_indirection_; } + + // Indirection table operations — shadow the base class to handle encoded + // mode. Callers go through A64CodeCache* so these are found first. + void set_indirection_default(uint32_t default_value); + void set_indirection_default_encoded(uint64_t default_value); + void AddIndirection(uint32_t guest_address, uint32_t host_address); + void AddIndirectionEncoded(uint32_t guest_address, uint64_t host_address); + void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high); + + // CRTP hook: called from CodeCacheBase::PlaceGuestCode to write the + // indirection entry for a newly placed function. + void OnPlaceGuestCodeIndirection(uint32_t guest_address, + void* code_execute_address); + + uintptr_t execute_base_address() const override { + return reinterpret_cast(generated_code_execute_base_); + } + + uint64_t* external_table() const { return external_table_; } + uint8_t* indirection_table_base() const { return indirection_table_base_; } + uint8_t* generated_code_execute_base() const { + return generated_code_execute_base_; + } + void* LookupUnwindInfo(uint64_t host_pc) override { return nullptr; } // CRTP hooks for CodeCacheBase. @@ -44,6 +74,22 @@ class A64CodeCache : public CodeCacheBase { protected: A64CodeCache() = default; + + private: + uint32_t EncodeIndirectionTarget(uint64_t host_address); + uint32_t AllocateExternalSlot(uint64_t host_address); + + bool encoded_indirection_ = false; + + // External table for addresses outside the code cache (trampolines). + static constexpr size_t kMaxExternalEntries = 32768; + uint64_t* external_table_ = nullptr; + uint32_t external_table_count_ = 0; + std::mutex external_table_mutex_; + + // Encoded-mode default indirection value (code-cache-relative offset of + // the resolve thunk). + uint32_t encoded_default_value_ = 0; }; } // namespace a64 diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 3e894db93..2a2d02ffb 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -360,11 +360,10 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { } if (code_cache_->has_indirection_table()) { - // Load host code address from indirection table. mov(w16, function->address()); - ldr(w9, ptr(x16, static_cast(0))); + EmitIndirectionLookup(); } else { - // Fallback: resolve at runtime. + // No indirection table: resolve at runtime. mov(x0, x20); // context mov(x1, static_cast(function->address())); mov(x9, reinterpret_cast(&ResolveFunction)); @@ -405,10 +404,9 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, int reg_index) { // Load host code address from indirection table. if (code_cache_->has_indirection_table()) { mov(w16, target_w); // w16 = guest address (also used by resolve thunk) - ldr(w9, ptr(x16, static_cast( - 0))); // w9 = host code from indirection table + EmitIndirectionLookup(); } else { - // Fallback: resolve at runtime. + // No indirection table: resolve at runtime. mov(w16, target_w); mov(x0, x20); // context mov(x1, x16); // guest address @@ -486,6 +484,40 @@ void A64Emitter::SetReturnAddress(uint64_t value) { str(x0, ptr(sp, static_cast(StackLayout::GUEST_CALL_RET_ADDR))); } +void A64Emitter::EmitIndirectionLookup() { + // w16 already holds the guest address. Load the host code address into x9. + if (!code_cache_->encoded_indirection()) { + // Fast path: table is at the guest address range. + ldr(w9, ptr(x16, static_cast(0))); + } else { + // Encoded path: table is at a dynamic address. Entries are + // code-cache-relative offsets (bit 31 clear) or external table indices + // (bit 31 set). + ldr(x17, ptr(x19, static_cast(offsetof(A64BackendContext, + indirection_table_bias)))); + add(x17, x17, x16); + ldr(w9, ptr(x17, static_cast(0))); + + auto& external_label = NewCachedLabel(); + auto& done_label = NewCachedLabel(); + tbnz(w9, 31, external_label); + + // Internal: offset from code cache base. + ldr(x17, ptr(x19, static_cast( + offsetof(A64BackendContext, code_execute_base)))); + add(x9, x17, x9, UXTW); + b(done_label); + + L(external_label); + and_(w9, w9, 0x7FFFFFFF); + ldr(x17, ptr(x19, static_cast(offsetof( + A64BackendContext, external_indirection_table)))); + ldr(x9, ptr(x17, x9, LSL, 3)); + + L(done_label); + } +} + void A64Emitter::ReloadMembase() { // Reload x21 from context->virtual_membase. ldr(x21, ptr(x20, static_cast( diff --git a/src/xenia/cpu/backend/a64/a64_emitter.h b/src/xenia/cpu/backend/a64/a64_emitter.h index 9007aace9..f9d225ad2 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.h +++ b/src/xenia/cpu/backend/a64/a64_emitter.h @@ -131,6 +131,7 @@ class A64Emitter : public Xbyak_aarch64::CodeGenerator { const Xbyak_aarch64::XReg& GetMembaseReg() const { return x21; } void ReloadMembase(); + void EmitIndirectionLookup(); void PushStackpoint(); void PopStackpoint(); diff --git a/src/xenia/cpu/backend/code_cache_base.h b/src/xenia/cpu/backend/code_cache_base.h index 976030c59..3fe78a0e3 100644 --- a/src/xenia/cpu/backend/code_cache_base.h +++ b/src/xenia/cpu/backend/code_cache_base.h @@ -210,13 +210,9 @@ class CodeCacheBase : public CodeCache { self().OnCodePlaced(guest_address, function_info, code_execute_address, func_info.code_size.total); - // Fix up indirection table. - if (guest_address && indirection_table_base_) { - uint32_t* indirection_slot = reinterpret_cast( - indirection_table_base_ + (guest_address - kIndirectionTableBase)); - *indirection_slot = - uint32_t(reinterpret_cast(code_execute_address)); - } + // Fix up indirection table. Dispatch through the derived class so + // it can use encoded entries when the table is dynamically allocated. + self().OnPlaceGuestCodeIndirection(guest_address, code_execute_address); } uint32_t PlaceData(const void* data, size_t length) { @@ -297,6 +293,7 @@ class CodeCacheBase : public CodeCache { "other system DLL", static_cast(kIndirectionTableBase), kIndirectionTableBase + kIndirectionTableSize); + return false; } file_name_ = @@ -355,6 +352,17 @@ class CodeCacheBase : public CodeCache { void OnCodePlaced(uint32_t guest_address, GuestFunction* function_info, void* code_execute_address, size_t code_size) {} + // Default indirection write: store truncated 32-bit absolute address. + // Overridden by A64CodeCache for encoded indirection mode. + void OnPlaceGuestCodeIndirection(uint32_t guest_address, + void* code_execute_address) { + if (guest_address && indirection_table_base_) { + uint32_t* slot = reinterpret_cast( + indirection_table_base_ + (guest_address - kIndirectionTableBase)); + *slot = uint32_t(reinterpret_cast(code_execute_address)); + } + } + std::filesystem::path file_name_; xe::memory::FileMappingHandle mapping_ = xe::memory::kFileMappingHandleInvalid;