From e2419b51c0fbe36efc90f406fcce3293f75fb2aa Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 29 Apr 2024 16:35:26 -0700 Subject: [PATCH 001/179] [CPU] Add ARM64 backend build target Adds the new `xenia-cpu-backend-a64` build-target with linkage following the x64 backend. --- premake5.lua | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 premake5.lua diff --git a/premake5.lua b/premake5.lua new file mode 100644 index 000000000..de00dc64c --- /dev/null +++ b/premake5.lua @@ -0,0 +1,20 @@ +project_root = "../../../../.." +include(project_root.."/tools/build") + +group("src") +project("xenia-cpu-backend-a64") + uuid("495f3f3e-f5e8-489a-bd0f-289d0495bc08") + filter("architecture:ARM64") + kind("StaticLib") + filter("architecture:not ARM64") + kind("None") + filter({}) + language("C++") + links({ + "fmt", + "xenia-base", + "xenia-cpu", + }) + defines({ + }) + local_platform_files() From a36329b224806b61b31336a37488922bef98d104 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 29 Apr 2024 17:48:40 -0700 Subject: [PATCH 002/179] [a64] Integrate `oaknut` submodule Header-only library for emitting arm64v8 instructions. Enables C++20 only for the a64 backend for now --- premake5.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/premake5.lua b/premake5.lua index de00dc64c..01a55e3c2 100644 --- a/premake5.lua +++ b/premake5.lua @@ -10,6 +10,7 @@ project("xenia-cpu-backend-a64") kind("None") filter({}) language("C++") + cppdialect("C++20") links({ "fmt", "xenia-base", @@ -17,4 +18,8 @@ project("xenia-cpu-backend-a64") }) defines({ }) + + includedirs({ + project_root.."/third_party/oaknut/include", + }) local_platform_files() From d80e3eacb7bd175a4147cb8091232b45804f0c4c Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 1 May 2024 04:12:34 -0700 Subject: [PATCH 003/179] [CPU] Implement ARM64 CPU backend First pass framework that gets emitted ARM code executing. Based on the x64 backend, implements an ARM64 JIT backend. --- a64_assembler.cc | 133 ++ a64_assembler.h | 58 + a64_backend.cc | 548 +++++++++ a64_backend.h | 88 ++ a64_code_cache.cc | 342 ++++++ a64_code_cache.h | 151 +++ a64_code_cache_win.cc | 337 +++++ a64_emitter.cc | 961 +++++++++++++++ a64_emitter.h | 250 ++++ a64_function.cc | 45 + a64_function.h | 44 + a64_op.h | 618 ++++++++++ a64_seq_control.cc | 553 +++++++++ a64_seq_memory.cc | 1155 +++++++++++++++++ a64_seq_vector.cc | 764 ++++++++++++ a64_sequences.cc | 2733 +++++++++++++++++++++++++++++++++++++++++ a64_sequences.h | 51 + a64_stack_layout.h | 135 ++ a64_tracers.cc | 225 ++++ a64_tracers.h | 82 ++ a64_util.h | 25 + premake5.lua | 6 + 22 files changed, 9304 insertions(+) create mode 100644 a64_assembler.cc create mode 100644 a64_assembler.h create mode 100644 a64_backend.cc create mode 100644 a64_backend.h create mode 100644 a64_code_cache.cc create mode 100644 a64_code_cache.h create mode 100644 a64_code_cache_win.cc create mode 100644 a64_emitter.cc create mode 100644 a64_emitter.h create mode 100644 a64_function.cc create mode 100644 a64_function.h create mode 100644 a64_op.h create mode 100644 a64_seq_control.cc create mode 100644 a64_seq_memory.cc create mode 100644 a64_seq_vector.cc create mode 100644 a64_sequences.cc create mode 100644 a64_sequences.h create mode 100644 a64_stack_layout.h create mode 100644 a64_tracers.cc create mode 100644 a64_tracers.h create mode 100644 a64_util.h diff --git a/a64_assembler.cc b/a64_assembler.cc new file mode 100644 index 000000000..4647a4c0a --- /dev/null +++ b/a64_assembler.cc @@ -0,0 +1,133 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_assembler.h" + +#include + +#include "third_party/capstone/include/capstone/capstone.h" +#include "third_party/capstone/include/capstone/x86.h" +#include "xenia/base/profiling.h" +#include "xenia/base/reset_scope.h" +#include "xenia/base/string.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_code_cache.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_function.h" +#include "xenia/cpu/cpu_flags.h" +#include "xenia/cpu/hir/hir_builder.h" +#include "xenia/cpu/hir/label.h" +#include "xenia/cpu/processor.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using xe::cpu::hir::HIRBuilder; + +A64Assembler::A64Assembler(A64Backend* backend) + : Assembler(backend), a64_backend_(backend) {} + +A64Assembler::~A64Assembler() { + // Emitter must be freed before the allocator. + emitter_.reset(); +} + +bool A64Assembler::Initialize() { + if (!Assembler::Initialize()) { + return false; + } + + emitter_.reset(new A64Emitter(a64_backend_)); + + return true; +} + +void A64Assembler::Reset() { + string_buffer_.Reset(); + Assembler::Reset(); +} + +bool A64Assembler::Assemble(GuestFunction* function, HIRBuilder* builder, + uint32_t debug_info_flags, + std::unique_ptr debug_info) { + SCOPE_profile_cpu_f("cpu"); + + // Reset when we leave. + xe::make_reset_scope(this); + + // Lower HIR -> a64. + void* machine_code = nullptr; + size_t code_size = 0; + if (!emitter_->Emit(function, builder, debug_info_flags, debug_info.get(), + &machine_code, &code_size, &function->source_map())) { + return false; + } + + // Stash generated machine code. + if (debug_info_flags & DebugInfoFlags::kDebugInfoDisasmMachineCode) { + DumpMachineCode(machine_code, code_size, function->source_map(), + &string_buffer_); + debug_info->set_machine_code_disasm(xe_strdup(string_buffer_.buffer())); + string_buffer_.Reset(); + } + + function->set_debug_info(std::move(debug_info)); + static_cast(function)->Setup( + reinterpret_cast(machine_code), code_size); + + // Install into indirection table. + const uint64_t host_address = reinterpret_cast(machine_code); + assert_true((host_address >> 32) == 0); + reinterpret_cast(backend_->code_cache()) + ->AddIndirection(function->address(), + static_cast(host_address)); + + return true; +} + +void A64Assembler::DumpMachineCode( + void* machine_code, size_t code_size, + const std::vector& source_map, StringBuffer* str) { + if (source_map.empty()) { + return; + } + auto source_map_index = 0; + uint32_t next_code_offset = source_map[0].code_offset; + + const uint8_t* code_ptr = reinterpret_cast(machine_code); + size_t remaining_code_size = code_size; + uint64_t address = uint64_t(machine_code); + cs_insn insn = {0}; + while (remaining_code_size) { + // Look up source offset. + auto code_offset = + uint32_t(code_ptr - reinterpret_cast(machine_code)); + if (code_offset >= next_code_offset && + source_map_index < source_map.size()) { + auto& source_map_entry = source_map[source_map_index]; + str->AppendFormat("{:08X} ", source_map_entry.guest_address); + ++source_map_index; + next_code_offset = source_map_index < source_map.size() + ? source_map[source_map_index].code_offset + : UINT_MAX; + } else { + str->Append(" "); + } + + str->AppendFormat("{:08X} {:<6} {}\n", uint32_t(insn.address), + insn.mnemonic, insn.op_str); + } +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_assembler.h b/a64_assembler.h new file mode 100644 index 000000000..f85d12339 --- /dev/null +++ b/a64_assembler.h @@ -0,0 +1,58 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ +#define XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ + +#include +#include + +#include "xenia/base/string_buffer.h" +#include "xenia/cpu/backend/assembler.h" +#include "xenia/cpu/function.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Backend; +class A64Emitter; + +class A64Assembler : public Assembler { + public: + explicit A64Assembler(A64Backend* backend); + ~A64Assembler() override; + + bool Initialize() override; + + void Reset() override; + + bool Assemble(GuestFunction* function, hir::HIRBuilder* builder, + uint32_t debug_info_flags, + std::unique_ptr debug_info) override; + + private: + void DumpMachineCode(void* machine_code, size_t code_size, + const std::vector& source_map, + StringBuffer* str); + + private: + A64Backend* a64_backend_; + std::unique_ptr emitter_; + + StringBuffer string_buffer_; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ diff --git a/a64_backend.cc b/a64_backend.cc new file mode 100644 index 000000000..076c1d487 --- /dev/null +++ b/a64_backend.cc @@ -0,0 +1,548 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_backend.h" + +#include + +#include "xenia/base/exception_handler.h" +#include "xenia/base/logging.h" +#include "xenia/cpu/backend/a64/a64_assembler.h" +#include "xenia/cpu/backend/a64/a64_code_cache.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_function.h" +#include "xenia/cpu/backend/a64/a64_sequences.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/breakpoint.h" +#include "xenia/cpu/processor.h" +#include "xenia/cpu/stack_walker.h" + +DEFINE_int32(a64_extension_mask, -1, + "Allow the detection and utilization of specific instruction set " + "features.\n" + " 0 = arm64v8\n" + " -1 = Detect and utilize all possible processor features\n", + "a64"); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using namespace oaknut::util; + +class A64ThunkEmitter : public A64Emitter { + public: + A64ThunkEmitter(A64Backend* backend); + ~A64ThunkEmitter() override; + HostToGuestThunk EmitHostToGuestThunk(); + GuestToHostThunk EmitGuestToHostThunk(); + ResolveFunctionThunk EmitResolveFunctionThunk(); + + private: + // The following four functions provide save/load functionality for registers. + // They assume at least StackLayout::THUNK_STACK_SIZE bytes have been + // allocated on the stack. + void EmitSaveVolatileRegs(); + void EmitLoadVolatileRegs(); + void EmitSaveNonvolatileRegs(); + void EmitLoadNonvolatileRegs(); +}; + +A64Backend::A64Backend() : Backend() {} + +A64Backend::~A64Backend() { + A64Emitter::FreeConstData(emitter_data_); + ExceptionHandler::Uninstall(&ExceptionCallbackThunk, this); +} + +bool A64Backend::Initialize(Processor* processor) { + if (!Backend::Initialize(processor)) { + return false; + } + + auto& gprs = machine_info_.register_sets[0]; + gprs.id = 0; + std::strcpy(gprs.name, "x"); + gprs.types = MachineInfo::RegisterSet::INT_TYPES; + gprs.count = A64Emitter::GPR_COUNT; + + auto& fprs = machine_info_.register_sets[1]; + fprs.id = 1; + std::strcpy(fprs.name, "v"); + fprs.types = MachineInfo::RegisterSet::FLOAT_TYPES | + MachineInfo::RegisterSet::VEC_TYPES; + fprs.count = A64Emitter::XMM_COUNT; + + code_cache_ = A64CodeCache::Create(); + Backend::code_cache_ = code_cache_.get(); + if (!code_cache_->Initialize()) { + return false; + } + + // Generate thunks used to transition between jitted code and host code. + A64ThunkEmitter thunk_emitter(this); + host_to_guest_thunk_ = thunk_emitter.EmitHostToGuestThunk(); + guest_to_host_thunk_ = thunk_emitter.EmitGuestToHostThunk(); + resolve_function_thunk_ = thunk_emitter.EmitResolveFunctionThunk(); + + // Set the code cache to use the ResolveFunction thunk for default + // indirections. + assert_zero(uint64_t(resolve_function_thunk_) & 0xFFFFFFFF00000000ull); + code_cache_->set_indirection_default( + uint32_t(uint64_t(resolve_function_thunk_))); + + // Allocate some special indirections. + code_cache_->CommitExecutableRange(0x9FFF0000, 0x9FFFFFFF); + + // Allocate emitter constant data. + emitter_data_ = A64Emitter::PlaceConstData(); + + // Setup exception callback + ExceptionHandler::Install(&ExceptionCallbackThunk, this); + + return true; +} + +void A64Backend::CommitExecutableRange(uint32_t guest_low, + uint32_t guest_high) { + code_cache_->CommitExecutableRange(guest_low, guest_high); +} + +std::unique_ptr A64Backend::CreateAssembler() { + return std::make_unique(this); +} + +std::unique_ptr A64Backend::CreateGuestFunction( + Module* module, uint32_t address) { + return std::make_unique(module, address); +} +uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, + uint64_t current_pc) { + // TODO(wunkolo): Capstone hookup + return current_pc += 4; +} + +void A64Backend::InstallBreakpoint(Breakpoint* breakpoint) { + breakpoint->ForEachHostAddress([breakpoint](uint64_t host_address) { + auto ptr = reinterpret_cast(host_address); + auto original_bytes = xe::load_and_swap(ptr); + assert_true(original_bytes != 0x0F0B); + xe::store_and_swap(ptr, 0x0F0B); + breakpoint->backend_data().emplace_back(host_address, original_bytes); + }); +} + +void A64Backend::InstallBreakpoint(Breakpoint* breakpoint, Function* fn) { + assert_true(breakpoint->address_type() == Breakpoint::AddressType::kGuest); + assert_true(fn->is_guest()); + auto guest_function = reinterpret_cast(fn); + auto host_address = + guest_function->MapGuestAddressToMachineCode(breakpoint->guest_address()); + if (!host_address) { + assert_always(); + return; + } + + // Assume we haven't already installed a breakpoint in this spot. + auto ptr = reinterpret_cast(host_address); + auto original_bytes = xe::load_and_swap(ptr); + assert_true(original_bytes != 0x0F0B); + xe::store_and_swap(ptr, 0x0F0B); + breakpoint->backend_data().emplace_back(host_address, original_bytes); +} + +void A64Backend::UninstallBreakpoint(Breakpoint* breakpoint) { + for (auto& pair : breakpoint->backend_data()) { + auto ptr = reinterpret_cast(pair.first); + auto instruction_bytes = xe::load_and_swap(ptr); + assert_true(instruction_bytes == 0x0F0B); + xe::store_and_swap(ptr, static_cast(pair.second)); + } + breakpoint->backend_data().clear(); +} + +bool A64Backend::ExceptionCallbackThunk(Exception* ex, void* data) { + auto backend = reinterpret_cast(data); + return backend->ExceptionCallback(ex); +} + +bool A64Backend::ExceptionCallback(Exception* ex) { + if (ex->code() != Exception::Code::kIllegalInstruction) { + // We only care about illegal instructions. Other things will be handled by + // other handlers (probably). If nothing else picks it up we'll be called + // with OnUnhandledException to do real crash handling. + return false; + } + + // Verify an expected illegal instruction. + auto instruction_bytes = + xe::load_and_swap(reinterpret_cast(ex->pc())); + if (instruction_bytes != 0x0F0B) { + // Not our ud2 - not us. + return false; + } + + // Let the processor handle things. + return processor()->OnThreadBreakpointHit(ex); +} + +A64ThunkEmitter::A64ThunkEmitter(A64Backend* backend) : A64Emitter(backend) {} + +A64ThunkEmitter::~A64ThunkEmitter() {} + +HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { + // X0 = target + // X1 = arg0 (context) + // X2 = arg1 (guest return address) + + struct _code_offsets { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + const size_t stack_size = StackLayout::THUNK_STACK_SIZE; + + code_offsets.prolog = offset(); + + // mov(qword[rsp + 8 * 3], r8); + // mov(qword[rsp + 8 * 2], rdx); + // mov(qword[rsp + 8 * 1], rcx); + // sub(rsp, stack_size); + STR(X2, XSP, 8 * 3); + STR(X1, XSP, 8 * 2); + STR(X0, XSP, 8 * 1); + SUB(XSP, XSP, stack_size); + + code_offsets.prolog_stack_alloc = offset(); + code_offsets.body = offset(); + + // Save nonvolatile registers. + EmitSaveNonvolatileRegs(); + + // mov(rax, rcx); + // mov(rsi, rdx); // context + // mov(rcx, r8); // return address + // call(rax); + MOV(X16, X0); + MOV(A64Emitter::GetContextReg(), X1); // context + MOV(X0, X2); // return address + + BLR(X16); + + EmitLoadNonvolatileRegs(); + + code_offsets.epilog = offset(); + + // add(rsp, stack_size); + // mov(rcx, qword[rsp + 8 * 1]); + // mov(rdx, qword[rsp + 8 * 2]); + // mov(r8, qword[rsp + 8 * 3]); + // ret(); + + ADD(XSP, XSP, stack_size); + LDR(X0, XSP, 8 * 1); + LDR(X1, XSP, 8 * 2); + LDR(X2, XSP, 8 * 3); + RET(); + + code_offsets.tail = offset(); + + assert_zero(code_offsets.prolog); + EmitFunctionInfo func_info = {}; + func_info.code_size.total = offset(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = offset() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = stack_size; + + void* fn = Emplace(func_info); + return (HostToGuestThunk)fn; +} + +GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { + // rcx = target function + // rdx = arg0 + // r8 = arg1 + // r9 = arg2 + + struct _code_offsets { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + const size_t stack_size = StackLayout::THUNK_STACK_SIZE; + + code_offsets.prolog = offset(); + + // rsp + 0 = return address + // sub(rsp, stack_size); + SUB(XSP, XSP, stack_size); + + code_offsets.prolog_stack_alloc = offset(); + code_offsets.body = offset(); + + // Save off volatile registers. + EmitSaveVolatileRegs(); + + // mov(rax, rcx); // function + // mov(rcx, GetContextReg()); // context + // call(rax); + MOV(X16, X0); // function + MOV(X0, GetContextReg()); // context + BLR(X16); + + EmitLoadVolatileRegs(); + + code_offsets.epilog = offset(); + + // add(rsp, stack_size); + // ret(); + ADD(XSP, XSP, stack_size); + RET(); + + code_offsets.tail = offset(); + + assert_zero(code_offsets.prolog); + EmitFunctionInfo func_info = {}; + func_info.code_size.total = offset(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = offset() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = stack_size; + + void* fn = Emplace(func_info); + return (GuestToHostThunk)fn; +} + +// A64Emitter handles actually resolving functions. +uint64_t ResolveFunction(void* raw_context, uint64_t target_address); + +ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { + // ebx = target PPC address + // rcx = context + + struct _code_offsets { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + const size_t stack_size = StackLayout::THUNK_STACK_SIZE; + + code_offsets.prolog = offset(); + + // rsp + 0 = return address + // sub(rsp, stack_size); + SUB(XSP, XSP, stack_size); + + code_offsets.prolog_stack_alloc = offset(); + code_offsets.body = offset(); + + // Save volatile registers + EmitSaveVolatileRegs(); + + // mov(rcx, rsi); // context + // mov(rdx, rbx); + // mov(rax, reinterpret_cast(&ResolveFunction)); + // call(rax) + MOV(X0, GetContextReg()); // context + MOV(X1, X1); + MOVP2R(X16, &ResolveFunction); + BLR(X16); + + EmitLoadVolatileRegs(); + + code_offsets.epilog = offset(); + + // add(rsp, stack_size); + // jmp(rax); + ADD(XSP, XSP, stack_size); + BR(X16); + + code_offsets.tail = offset(); + + assert_zero(code_offsets.prolog); + EmitFunctionInfo func_info = {}; + func_info.code_size.total = offset(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = offset() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = stack_size; + + void* fn = Emplace(func_info); + return (ResolveFunctionThunk)fn; +} + +void A64ThunkEmitter::EmitSaveVolatileRegs() { + // Save off volatile registers. + STR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); + STR(X1, XSP, offsetof(StackLayout::Thunk, r[1])); + STR(X2, XSP, offsetof(StackLayout::Thunk, r[2])); + STR(X3, XSP, offsetof(StackLayout::Thunk, r[3])); + STR(X4, XSP, offsetof(StackLayout::Thunk, r[4])); + STR(X5, XSP, offsetof(StackLayout::Thunk, r[5])); + STR(X6, XSP, offsetof(StackLayout::Thunk, r[6])); + STR(X7, XSP, offsetof(StackLayout::Thunk, r[7])); + STR(X8, XSP, offsetof(StackLayout::Thunk, r[8])); + + STR(X9, XSP, offsetof(StackLayout::Thunk, r[9])); + STR(X10, XSP, offsetof(StackLayout::Thunk, r[10])); + STR(X11, XSP, offsetof(StackLayout::Thunk, r[11])); + STR(X12, XSP, offsetof(StackLayout::Thunk, r[12])); + STR(X13, XSP, offsetof(StackLayout::Thunk, r[13])); + STR(X14, XSP, offsetof(StackLayout::Thunk, r[14])); + STR(X15, XSP, offsetof(StackLayout::Thunk, r[15])); + STR(X16, XSP, offsetof(StackLayout::Thunk, r[16])); + STR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); + STR(X17, XSP, offsetof(StackLayout::Thunk, r[18])); + + STR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); + STR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); + STR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); + STR(Q3, XSP, offsetof(StackLayout::Thunk, xmm[3])); + STR(Q4, XSP, offsetof(StackLayout::Thunk, xmm[4])); + STR(Q5, XSP, offsetof(StackLayout::Thunk, xmm[5])); + STR(Q6, XSP, offsetof(StackLayout::Thunk, xmm[6])); + STR(Q7, XSP, offsetof(StackLayout::Thunk, xmm[7])); + + STR(Q8, XSP, offsetof(StackLayout::Thunk, xmm[8])); + STR(Q9, XSP, offsetof(StackLayout::Thunk, xmm[9])); + STR(Q10, XSP, offsetof(StackLayout::Thunk, xmm[10])); + STR(Q11, XSP, offsetof(StackLayout::Thunk, xmm[11])); + STR(Q12, XSP, offsetof(StackLayout::Thunk, xmm[12])); + STR(Q13, XSP, offsetof(StackLayout::Thunk, xmm[13])); + STR(Q14, XSP, offsetof(StackLayout::Thunk, xmm[14])); + STR(Q15, XSP, offsetof(StackLayout::Thunk, xmm[15])); + STR(Q16, XSP, offsetof(StackLayout::Thunk, xmm[16])); + STR(Q17, XSP, offsetof(StackLayout::Thunk, xmm[17])); + STR(Q18, XSP, offsetof(StackLayout::Thunk, xmm[18])); + STR(Q19, XSP, offsetof(StackLayout::Thunk, xmm[19])); + STR(Q20, XSP, offsetof(StackLayout::Thunk, xmm[20])); + STR(Q21, XSP, offsetof(StackLayout::Thunk, xmm[21])); + STR(Q22, XSP, offsetof(StackLayout::Thunk, xmm[22])); + STR(Q23, XSP, offsetof(StackLayout::Thunk, xmm[23])); + STR(Q24, XSP, offsetof(StackLayout::Thunk, xmm[24])); + STR(Q25, XSP, offsetof(StackLayout::Thunk, xmm[25])); + STR(Q26, XSP, offsetof(StackLayout::Thunk, xmm[26])); + STR(Q27, XSP, offsetof(StackLayout::Thunk, xmm[27])); + STR(Q28, XSP, offsetof(StackLayout::Thunk, xmm[28])); + STR(Q29, XSP, offsetof(StackLayout::Thunk, xmm[29])); + STR(Q30, XSP, offsetof(StackLayout::Thunk, xmm[30])); + STR(Q31, XSP, offsetof(StackLayout::Thunk, xmm[31])); +} + +void A64ThunkEmitter::EmitLoadVolatileRegs() { + LDR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); + LDR(X1, XSP, offsetof(StackLayout::Thunk, r[1])); + LDR(X2, XSP, offsetof(StackLayout::Thunk, r[2])); + LDR(X3, XSP, offsetof(StackLayout::Thunk, r[3])); + LDR(X4, XSP, offsetof(StackLayout::Thunk, r[4])); + LDR(X5, XSP, offsetof(StackLayout::Thunk, r[5])); + LDR(X6, XSP, offsetof(StackLayout::Thunk, r[6])); + LDR(X7, XSP, offsetof(StackLayout::Thunk, r[7])); + LDR(X8, XSP, offsetof(StackLayout::Thunk, r[8])); + + LDR(X9, XSP, offsetof(StackLayout::Thunk, r[9])); + LDR(X10, XSP, offsetof(StackLayout::Thunk, r[10])); + LDR(X11, XSP, offsetof(StackLayout::Thunk, r[11])); + LDR(X12, XSP, offsetof(StackLayout::Thunk, r[12])); + LDR(X13, XSP, offsetof(StackLayout::Thunk, r[13])); + LDR(X14, XSP, offsetof(StackLayout::Thunk, r[14])); + LDR(X15, XSP, offsetof(StackLayout::Thunk, r[15])); + LDR(X16, XSP, offsetof(StackLayout::Thunk, r[16])); + LDR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); + LDR(X18, XSP, offsetof(StackLayout::Thunk, r[18])); + + LDR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); + LDR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); + LDR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); + LDR(Q3, XSP, offsetof(StackLayout::Thunk, xmm[3])); + LDR(Q4, XSP, offsetof(StackLayout::Thunk, xmm[4])); + LDR(Q5, XSP, offsetof(StackLayout::Thunk, xmm[5])); + LDR(Q6, XSP, offsetof(StackLayout::Thunk, xmm[6])); + LDR(Q7, XSP, offsetof(StackLayout::Thunk, xmm[7])); + + LDR(Q8, XSP, offsetof(StackLayout::Thunk, xmm[8])); + LDR(Q9, XSP, offsetof(StackLayout::Thunk, xmm[9])); + LDR(Q10, XSP, offsetof(StackLayout::Thunk, xmm[10])); + LDR(Q11, XSP, offsetof(StackLayout::Thunk, xmm[11])); + LDR(Q12, XSP, offsetof(StackLayout::Thunk, xmm[12])); + LDR(Q13, XSP, offsetof(StackLayout::Thunk, xmm[13])); + LDR(Q14, XSP, offsetof(StackLayout::Thunk, xmm[14])); + LDR(Q15, XSP, offsetof(StackLayout::Thunk, xmm[15])); + LDR(Q16, XSP, offsetof(StackLayout::Thunk, xmm[16])); + LDR(Q17, XSP, offsetof(StackLayout::Thunk, xmm[17])); + LDR(Q18, XSP, offsetof(StackLayout::Thunk, xmm[18])); + LDR(Q19, XSP, offsetof(StackLayout::Thunk, xmm[19])); + LDR(Q20, XSP, offsetof(StackLayout::Thunk, xmm[20])); + LDR(Q21, XSP, offsetof(StackLayout::Thunk, xmm[21])); + LDR(Q22, XSP, offsetof(StackLayout::Thunk, xmm[22])); + LDR(Q23, XSP, offsetof(StackLayout::Thunk, xmm[23])); + LDR(Q24, XSP, offsetof(StackLayout::Thunk, xmm[24])); + LDR(Q25, XSP, offsetof(StackLayout::Thunk, xmm[25])); + LDR(Q26, XSP, offsetof(StackLayout::Thunk, xmm[26])); + LDR(Q27, XSP, offsetof(StackLayout::Thunk, xmm[27])); + LDR(Q28, XSP, offsetof(StackLayout::Thunk, xmm[28])); + LDR(Q29, XSP, offsetof(StackLayout::Thunk, xmm[29])); + LDR(Q30, XSP, offsetof(StackLayout::Thunk, xmm[30])); + LDR(Q31, XSP, offsetof(StackLayout::Thunk, xmm[31])); +} + +void A64ThunkEmitter::EmitSaveNonvolatileRegs() { + STR(X19, XSP, offsetof(StackLayout::Thunk, r[0])); + STR(X20, XSP, offsetof(StackLayout::Thunk, r[1])); + STR(X21, XSP, offsetof(StackLayout::Thunk, r[2])); + STR(X22, XSP, offsetof(StackLayout::Thunk, r[3])); + STR(X23, XSP, offsetof(StackLayout::Thunk, r[4])); + STR(X24, XSP, offsetof(StackLayout::Thunk, r[5])); + STR(X25, XSP, offsetof(StackLayout::Thunk, r[6])); + STR(X26, XSP, offsetof(StackLayout::Thunk, r[7])); + STR(X27, XSP, offsetof(StackLayout::Thunk, r[8])); + STR(X28, XSP, offsetof(StackLayout::Thunk, r[9])); + STR(X29, XSP, offsetof(StackLayout::Thunk, r[10])); + STR(X30, XSP, offsetof(StackLayout::Thunk, r[11])); +} + +void A64ThunkEmitter::EmitLoadNonvolatileRegs() { + LDR(X19, XSP, offsetof(StackLayout::Thunk, r[0])); + LDR(X20, XSP, offsetof(StackLayout::Thunk, r[1])); + LDR(X21, XSP, offsetof(StackLayout::Thunk, r[2])); + LDR(X22, XSP, offsetof(StackLayout::Thunk, r[3])); + LDR(X23, XSP, offsetof(StackLayout::Thunk, r[4])); + LDR(X24, XSP, offsetof(StackLayout::Thunk, r[5])); + LDR(X25, XSP, offsetof(StackLayout::Thunk, r[6])); + LDR(X26, XSP, offsetof(StackLayout::Thunk, r[7])); + LDR(X27, XSP, offsetof(StackLayout::Thunk, r[8])); + LDR(X28, XSP, offsetof(StackLayout::Thunk, r[9])); + LDR(X29, XSP, offsetof(StackLayout::Thunk, r[10])); + LDR(X30, XSP, offsetof(StackLayout::Thunk, r[11])); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_backend.h b/a64_backend.h new file mode 100644 index 000000000..85801c592 --- /dev/null +++ b/a64_backend.h @@ -0,0 +1,88 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ +#define XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ + +#include + +#include "xenia/base/cvar.h" +#include "xenia/cpu/backend/backend.h" + +DECLARE_int32(a64_extension_mask); + +namespace xe { +class Exception; +} // namespace xe +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64CodeCache; + +typedef void* (*HostToGuestThunk)(void* target, void* arg0, void* arg1); +typedef void* (*GuestToHostThunk)(void* target, void* arg0, void* arg1); +typedef void (*ResolveFunctionThunk)(); + +class A64Backend : public Backend { + public: + static const uint32_t kForceReturnAddress = 0x9FFF0000u; + + explicit A64Backend(); + ~A64Backend() override; + + A64CodeCache* code_cache() const { return code_cache_.get(); } + uintptr_t emitter_data() const { return emitter_data_; } + + // Call a generated function, saving all stack parameters. + HostToGuestThunk host_to_guest_thunk() const { return host_to_guest_thunk_; } + // Function that guest code can call to transition into host code. + GuestToHostThunk guest_to_host_thunk() const { return guest_to_host_thunk_; } + // Function that thunks to the ResolveFunction in A64Emitter. + ResolveFunctionThunk resolve_function_thunk() const { + return resolve_function_thunk_; + } + + bool Initialize(Processor* processor) override; + + void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high) override; + + std::unique_ptr CreateAssembler() override; + + std::unique_ptr CreateGuestFunction(Module* module, + uint32_t address) override; + + uint64_t CalculateNextHostInstruction(ThreadDebugInfo* thread_info, + uint64_t current_pc) override; + + void InstallBreakpoint(Breakpoint* breakpoint) override; + void InstallBreakpoint(Breakpoint* breakpoint, Function* fn) override; + void UninstallBreakpoint(Breakpoint* breakpoint) override; + + private: + static bool ExceptionCallbackThunk(Exception* ex, void* data); + bool ExceptionCallback(Exception* ex); + + // uintptr_t capstone_handle_ = 0; + + std::unique_ptr code_cache_; + uintptr_t emitter_data_ = 0; + + HostToGuestThunk host_to_guest_thunk_; + GuestToHostThunk guest_to_host_thunk_; + ResolveFunctionThunk resolve_function_thunk_; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ diff --git a/a64_code_cache.cc b/a64_code_cache.cc new file mode 100644 index 000000000..fb16c2ffe --- /dev/null +++ b/a64_code_cache.cc @@ -0,0 +1,342 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_code_cache.h" + +#include +#include + +#include "third_party/fmt/include/fmt/format.h" +#include "xenia/base/assert.h" +#include "xenia/base/clock.h" +#include "xenia/base/literals.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" +#include "xenia/cpu/function.h" +#include "xenia/cpu/module.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using namespace xe::literals; + +A64CodeCache::A64CodeCache() = default; + +A64CodeCache::~A64CodeCache() { + if (indirection_table_base_) { + xe::memory::DeallocFixed(indirection_table_base_, 0, + xe::memory::DeallocationType::kRelease); + } + + // Unmap all views and close mapping. + if (mapping_ != xe::memory::kFileMappingHandleInvalid) { + if (generated_code_write_base_ && + generated_code_write_base_ != generated_code_execute_base_) { + xe::memory::UnmapFileView(mapping_, generated_code_write_base_, + kGeneratedCodeSize); + } + if (generated_code_execute_base_) { + xe::memory::UnmapFileView(mapping_, generated_code_execute_base_, + kGeneratedCodeSize); + } + xe::memory::CloseFileMappingHandle(mapping_, file_name_); + mapping_ = xe::memory::kFileMappingHandleInvalid; + } +} + +bool A64CodeCache::Initialize() { + indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( + reinterpret_cast(kIndirectionTableBase), kIndirectionTableSize, + xe::memory::AllocationType::kReserve, + xe::memory::PageAccess::kReadWrite)); + if (!indirection_table_base_) { + XELOGE("Unable to allocate code cache indirection table"); + XELOGE( + "This is likely because the {:X}-{:X} range is in use by some other " + "system DLL", + static_cast(kIndirectionTableBase), + kIndirectionTableBase + kIndirectionTableSize); + } + + // Create mmap file. This allows us to share the code cache with the debugger. + file_name_ = fmt::format("xenia_code_cache_{}", Clock::QueryHostTickCount()); + mapping_ = xe::memory::CreateFileMappingHandle( + file_name_, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, + false); + if (mapping_ == xe::memory::kFileMappingHandleInvalid) { + XELOGE("Unable to create code cache mmap"); + return false; + } + + // Map generated code region into the file. Pages are committed as required. + if (xe::memory::IsWritableExecutableMemoryPreferred()) { + generated_code_execute_base_ = + reinterpret_cast(xe::memory::MapFileView( + mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), + kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, 0)); + generated_code_write_base_ = generated_code_execute_base_; + if (!generated_code_execute_base_ || !generated_code_write_base_) { + XELOGE("Unable to allocate code cache generated code storage"); + XELOGE( + "This is likely because the {:X}-{:X} range is in use by some other " + "system DLL", + uint64_t(kGeneratedCodeExecuteBase), + uint64_t(kGeneratedCodeExecuteBase + kGeneratedCodeSize)); + return false; + } + } else { + generated_code_execute_base_ = + reinterpret_cast(xe::memory::MapFileView( + mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), + kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); + generated_code_write_base_ = + reinterpret_cast(xe::memory::MapFileView( + mapping_, reinterpret_cast(kGeneratedCodeWriteBase), + kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); + if (!generated_code_execute_base_ || !generated_code_write_base_) { + XELOGE("Unable to allocate code cache generated code storage"); + XELOGE( + "This is likely because the {:X}-{:X} and {:X}-{:X} ranges are in " + "use by some other system DLL", + uint64_t(kGeneratedCodeExecuteBase), + uint64_t(kGeneratedCodeExecuteBase + kGeneratedCodeSize), + uint64_t(kGeneratedCodeWriteBase), + uint64_t(kGeneratedCodeWriteBase + kGeneratedCodeSize)); + return false; + } + } + + // Preallocate the function map to a large, reasonable size. + generated_code_map_.reserve(kMaximumFunctionCount); + + return true; +} + +void A64CodeCache::set_indirection_default(uint32_t default_value) { + indirection_default_value_ = default_value; +} + +void A64CodeCache::AddIndirection(uint32_t guest_address, + uint32_t host_address) { + if (!indirection_table_base_) { + return; + } + + uint32_t* indirection_slot = reinterpret_cast( + indirection_table_base_ + (guest_address - kIndirectionTableBase)); + *indirection_slot = host_address; +} + +void A64CodeCache::CommitExecutableRange(uint32_t guest_low, + uint32_t guest_high) { + if (!indirection_table_base_) { + return; + } + + // Commit the memory. + xe::memory::AllocFixed( + indirection_table_base_ + (guest_low - kIndirectionTableBase), + guest_high - guest_low, xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kReadWrite); + + // Fill memory with the default value. + uint32_t* p = reinterpret_cast(indirection_table_base_); + for (uint32_t address = guest_low; address < guest_high; ++address) { + p[(address - kIndirectionTableBase) / 4] = indirection_default_value_; + } +} + +void A64CodeCache::PlaceHostCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void*& code_execute_address_out, + void*& code_write_address_out) { + // Same for now. We may use different pools or whatnot later on, like when + // we only want to place guest code in a serialized cache on disk. + PlaceGuestCode(guest_address, machine_code, func_info, nullptr, + code_execute_address_out, code_write_address_out); +} + +void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + GuestFunction* function_info, + void*& code_execute_address_out, + void*& code_write_address_out) { + // Hold a lock while we bump the pointers up. This is important as the + // unwind table requires entries AND code to be sorted in order. + size_t low_mark; + size_t high_mark; + uint8_t* code_execute_address; + UnwindReservation unwind_reservation; + { + auto global_lock = global_critical_region_.Acquire(); + + low_mark = generated_code_offset_; + + // Reserve code. + // Always move the code to land on 16b alignment. + code_execute_address = + generated_code_execute_base_ + generated_code_offset_; + code_execute_address_out = code_execute_address; + uint8_t* code_write_address = + generated_code_write_base_ + generated_code_offset_; + code_write_address_out = code_write_address; + generated_code_offset_ += xe::round_up(func_info.code_size.total, 16); + + auto tail_write_address = + generated_code_write_base_ + generated_code_offset_; + + // Reserve unwind info. + // We go on the high size of the unwind info as we don't know how big we + // need it, and a few extra bytes of padding isn't the worst thing. + unwind_reservation = RequestUnwindReservation(generated_code_write_base_ + + generated_code_offset_); + generated_code_offset_ += xe::round_up(unwind_reservation.data_size, 16); + + auto end_write_address = + generated_code_write_base_ + generated_code_offset_; + + high_mark = generated_code_offset_; + + // Store in map. It is maintained in sorted order of host PC dependent on + // us also being append-only. + generated_code_map_.emplace_back( + (uint64_t(code_execute_address - generated_code_execute_base_) << 32) | + generated_code_offset_, + function_info); + + // TODO(DrChat): The following code doesn't really need to be under the + // global lock except for PlaceCode (but it depends on the previous code + // already being ran) + + // If we are going above the high water mark of committed memory, commit + // some more. It's ok if multiple threads do this, as redundant commits + // aren't harmful. + size_t old_commit_mark, new_commit_mark; + do { + old_commit_mark = generated_code_commit_mark_; + if (high_mark <= old_commit_mark) break; + + new_commit_mark = old_commit_mark + 16_MiB; + if (generated_code_execute_base_ == generated_code_write_base_) { + xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kExecuteReadWrite); + } else { + xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kExecuteReadOnly); + xe::memory::AllocFixed(generated_code_write_base_, new_commit_mark, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kReadWrite); + } + } while (generated_code_commit_mark_.compare_exchange_weak( + old_commit_mark, new_commit_mark)); + + // Copy code. + std::memcpy(code_write_address, machine_code, func_info.code_size.total); + + // Fill unused slots with 0xCC + std::memset(tail_write_address, 0xCC, + static_cast(end_write_address - tail_write_address)); + + // Notify subclasses of placed code. + PlaceCode(guest_address, machine_code, func_info, code_execute_address, + unwind_reservation); + } + + // Now that everything is ready, fix up the indirection table. + // Note that we do support code that doesn't have an indirection fixup, so + // ignore those when we see them. + if (guest_address && indirection_table_base_) { + uint32_t* indirection_slot = reinterpret_cast( + indirection_table_base_ + (guest_address - kIndirectionTableBase)); + *indirection_slot = + uint32_t(reinterpret_cast(code_execute_address)); + } +} + +uint32_t A64CodeCache::PlaceData(const void* data, size_t length) { + // Hold a lock while we bump the pointers up. + size_t high_mark; + uint8_t* data_address = nullptr; + { + auto global_lock = global_critical_region_.Acquire(); + + // Reserve code. + // Always move the code to land on 16b alignment. + data_address = generated_code_write_base_ + generated_code_offset_; + generated_code_offset_ += xe::round_up(length, 16); + + high_mark = generated_code_offset_; + } + + // If we are going above the high water mark of committed memory, commit some + // more. It's ok if multiple threads do this, as redundant commits aren't + // harmful. + size_t old_commit_mark, new_commit_mark; + do { + old_commit_mark = generated_code_commit_mark_; + if (high_mark <= old_commit_mark) break; + + new_commit_mark = old_commit_mark + 16_MiB; + if (generated_code_execute_base_ == generated_code_write_base_) { + xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kExecuteReadWrite); + } else { + xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kExecuteReadOnly); + xe::memory::AllocFixed(generated_code_write_base_, new_commit_mark, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kReadWrite); + } + } while (generated_code_commit_mark_.compare_exchange_weak(old_commit_mark, + new_commit_mark)); + + // Copy code. + std::memcpy(data_address, data, length); + + return uint32_t(uintptr_t(data_address)); +} + +GuestFunction* A64CodeCache::LookupFunction(uint64_t host_pc) { + uint32_t key = uint32_t(host_pc - kGeneratedCodeExecuteBase); + void* fn_entry = std::bsearch( + &key, generated_code_map_.data(), generated_code_map_.size() + 1, + sizeof(std::pair), + [](const void* key_ptr, const void* element_ptr) { + auto key = *reinterpret_cast(key_ptr); + auto element = + reinterpret_cast*>( + element_ptr); + if (key < (element->first >> 32)) { + return -1; + } else if (key > uint32_t(element->first)) { + return 1; + } else { + return 0; + } + }); + if (fn_entry) { + return reinterpret_cast*>( + fn_entry) + ->second; + } else { + return nullptr; + } +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_code_cache.h b/a64_code_cache.h new file mode 100644 index 000000000..2bc9ed59f --- /dev/null +++ b/a64_code_cache.h @@ -0,0 +1,151 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ +#define XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "xenia/base/memory.h" +#include "xenia/base/mutex.h" +#include "xenia/cpu/backend/code_cache.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +struct EmitFunctionInfo { + struct _code_size { + size_t prolog; + size_t body; + size_t epilog; + size_t tail; + size_t total; + } code_size; + size_t prolog_stack_alloc_offset; // offset of instruction after stack alloc + size_t stack_size; +}; + +class A64CodeCache : public CodeCache { + public: + ~A64CodeCache() override; + + static std::unique_ptr Create(); + + virtual bool Initialize(); + + const std::filesystem::path& file_name() const override { return file_name_; } + uintptr_t execute_base_address() const override { + return kGeneratedCodeExecuteBase; + } + size_t total_size() const override { return kGeneratedCodeSize; } + + // TODO(benvanik): ELF serialization/etc + // TODO(benvanik): keep track of code blocks + // TODO(benvanik): padding/guards/etc + + bool has_indirection_table() { return indirection_table_base_ != nullptr; } + void set_indirection_default(uint32_t default_value); + void AddIndirection(uint32_t guest_address, uint32_t host_address); + + void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high); + + void PlaceHostCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void*& code_execute_address_out, + void*& code_write_address_out); + void PlaceGuestCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + GuestFunction* function_info, + void*& code_execute_address_out, + void*& code_write_address_out); + uint32_t PlaceData(const void* data, size_t length); + + GuestFunction* LookupFunction(uint64_t host_pc) override; + + protected: + // All executable code falls within 0x80000000 to 0x9FFFFFFF, so we can + // only map enough for lookups within that range. + static const size_t kIndirectionTableSize = 0x1FFFFFFF; + static const uintptr_t kIndirectionTableBase = 0x80000000; + // The code range is 512MB, but we know the total code games will have is + // pretty small (dozens of mb at most) and our expansion is reasonablish + // so 256MB should be more than enough. + static const size_t kGeneratedCodeSize = 0x0FFFFFFF; + static const uintptr_t kGeneratedCodeExecuteBase = 0xA0000000; + // Used for writing when PageAccess::kExecuteReadWrite is not supported. + static const uintptr_t kGeneratedCodeWriteBase = + kGeneratedCodeExecuteBase + kGeneratedCodeSize + 1; + + // This is picked to be high enough to cover whatever we can reasonably + // expect. If we hit issues with this it probably means some corner case + // in analysis triggering. + static const size_t kMaximumFunctionCount = 100000; + + struct UnwindReservation { + size_t data_size = 0; + size_t table_slot = 0; + uint8_t* entry_address = 0; + }; + + A64CodeCache(); + + virtual UnwindReservation RequestUnwindReservation(uint8_t* entry_address) { + return UnwindReservation(); + } + virtual void PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void* code_execute_address, + UnwindReservation unwind_reservation) {} + + std::filesystem::path file_name_; + xe::memory::FileMappingHandle mapping_ = + xe::memory::kFileMappingHandleInvalid; + + // NOTE: the global critical region must be held when manipulating the offsets + // or counts of anything, to keep the tables consistent and ordered. + xe::global_critical_region global_critical_region_; + + // Value that the indirection table will be initialized with upon commit. + uint32_t indirection_default_value_ = 0xFEEDF00D; + + // Fixed at kIndirectionTableBase in host space, holding 4 byte pointers into + // the generated code table that correspond to the PPC functions in guest + // space. + uint8_t* indirection_table_base_ = nullptr; + // Fixed at kGeneratedCodeExecuteBase and holding all generated code, growing + // as needed. + uint8_t* generated_code_execute_base_ = nullptr; + // View of the memory that backs generated_code_execute_base_ when + // PageAccess::kExecuteReadWrite is not supported, for writing the generated + // code. Equals to generated_code_execute_base_ when it's supported. + uint8_t* generated_code_write_base_ = nullptr; + // Current offset to empty space in generated code. + size_t generated_code_offset_ = 0; + // Current high water mark of COMMITTED code. + std::atomic generated_code_commit_mark_ = {0}; + // Sorted map by host PC base offsets to source function info. + // This can be used to bsearch on host PC to find the guest function. + // The key is [start address | end address]. + std::vector> generated_code_map_; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ diff --git a/a64_code_cache_win.cc b/a64_code_cache_win.cc new file mode 100644 index 000000000..a48426260 --- /dev/null +++ b/a64_code_cache_win.cc @@ -0,0 +1,337 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_code_cache.h" + +#include +#include + +#include "xenia/base/assert.h" +#include "xenia/base/clock.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" +#include "xenia/base/platform_win.h" +#include "xenia/cpu/function.h" + +// Function pointer definitions +using FnRtlAddGrowableFunctionTable = decltype(&RtlAddGrowableFunctionTable); +using FnRtlGrowFunctionTable = decltype(&RtlGrowFunctionTable); +using FnRtlDeleteGrowableFunctionTable = + decltype(&RtlDeleteGrowableFunctionTable); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +// https://msdn.microsoft.com/en-us/library/ssa62fwe.aspx +typedef enum _UNWIND_OP_CODES { + UWOP_PUSH_NONVOL = 0, /* info == register number */ + UWOP_ALLOC_LARGE, /* no info, alloc size in next 2 slots */ + UWOP_ALLOC_SMALL, /* info == size of allocation / 8 - 1 */ + UWOP_SET_FPREG, /* no info, FP = RSP + UNWIND_INFO.FPRegOffset*16 */ + UWOP_SAVE_NONVOL, /* info == register number, offset in next slot */ + UWOP_SAVE_NONVOL_FAR, /* info == register number, offset in next 2 slots */ + UWOP_SAVE_XMM128, /* info == XMM reg number, offset in next slot */ + UWOP_SAVE_XMM128_FAR, /* info == XMM reg number, offset in next 2 slots */ + UWOP_PUSH_MACHFRAME /* info == 0: no error-code, 1: error-code */ +} UNWIND_CODE_OPS; +class UNWIND_REGISTER { + public: + enum _ { + RAX = 0, + RCX = 1, + RDX = 2, + RBX = 3, + RSP = 4, + RBP = 5, + RSI = 6, + RDI = 7, + R8 = 8, + R9 = 9, + R10 = 10, + R11 = 11, + R12 = 12, + R13 = 13, + R14 = 14, + R15 = 15, + }; +}; + +typedef union _UNWIND_CODE { + struct { + uint8_t CodeOffset; + uint8_t UnwindOp : 4; + uint8_t OpInfo : 4; + }; + USHORT FrameOffset; +} UNWIND_CODE, *PUNWIND_CODE; + +typedef struct _UNWIND_INFO { + uint8_t Version : 3; + uint8_t Flags : 5; + uint8_t SizeOfProlog; + uint8_t CountOfCodes; + uint8_t FrameRegister : 4; + uint8_t FrameOffset : 4; + UNWIND_CODE UnwindCode[1]; + /* UNWIND_CODE MoreUnwindCode[((CountOfCodes + 1) & ~1) - 1]; + * union { + * OPTIONAL ULONG ExceptionHandler; + * OPTIONAL ULONG FunctionEntry; + * }; + * OPTIONAL ULONG ExceptionData[]; */ +} UNWIND_INFO, *PUNWIND_INFO; + +// Size of unwind info per function. +// TODO(benvanik): move this to emitter. +static const uint32_t kUnwindInfoSize = + sizeof(UNWIND_INFO) + (sizeof(UNWIND_CODE) * (6 - 1)); + +class Win32A64CodeCache : public A64CodeCache { + public: + Win32A64CodeCache(); + ~Win32A64CodeCache() override; + + bool Initialize() override; + + void* LookupUnwindInfo(uint64_t host_pc) override; + + private: + UnwindReservation RequestUnwindReservation(uint8_t* entry_address) override; + void PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, void* code_execute_address, + UnwindReservation unwind_reservation) override; + + void InitializeUnwindEntry(uint8_t* unwind_entry_address, + size_t unwind_table_slot, + void* code_execute_address, + const EmitFunctionInfo& func_info); + + // Growable function table system handle. + void* unwind_table_handle_ = nullptr; + // Actual unwind table entries. + std::vector unwind_table_; + // Current number of entries in the table. + std::atomic unwind_table_count_ = {0}; + // Does this version of Windows support growable funciton tables? + bool supports_growable_table_ = false; + + FnRtlAddGrowableFunctionTable add_growable_table_ = nullptr; + FnRtlDeleteGrowableFunctionTable delete_growable_table_ = nullptr; + FnRtlGrowFunctionTable grow_table_ = nullptr; +}; + +std::unique_ptr A64CodeCache::Create() { + return std::make_unique(); +} + +Win32A64CodeCache::Win32A64CodeCache() = default; + +Win32A64CodeCache::~Win32A64CodeCache() { + if (supports_growable_table_) { + if (unwind_table_handle_) { + delete_growable_table_(unwind_table_handle_); + } + } else { + if (generated_code_execute_base_) { + RtlDeleteFunctionTable(reinterpret_cast( + reinterpret_cast(generated_code_execute_base_) | 0x3)); + } + } +} + +bool Win32A64CodeCache::Initialize() { + if (!A64CodeCache::Initialize()) { + return false; + } + + // Compute total number of unwind entries we should allocate. + // We don't support reallocing right now, so this should be high. + unwind_table_.resize(kMaximumFunctionCount); + + // Check if this version of Windows supports growable function tables. + auto ntdll_handle = GetModuleHandleW(L"ntdll.dll"); + if (!ntdll_handle) { + add_growable_table_ = nullptr; + delete_growable_table_ = nullptr; + grow_table_ = nullptr; + } else { + add_growable_table_ = (FnRtlAddGrowableFunctionTable)GetProcAddress( + ntdll_handle, "RtlAddGrowableFunctionTable"); + delete_growable_table_ = (FnRtlDeleteGrowableFunctionTable)GetProcAddress( + ntdll_handle, "RtlDeleteGrowableFunctionTable"); + grow_table_ = (FnRtlGrowFunctionTable)GetProcAddress( + ntdll_handle, "RtlGrowFunctionTable"); + } + supports_growable_table_ = + add_growable_table_ && delete_growable_table_ && grow_table_; + + // Create table and register with the system. It's empty now, but we'll grow + // it as functions are added. + if (supports_growable_table_) { + if (add_growable_table_( + &unwind_table_handle_, unwind_table_.data(), unwind_table_count_, + DWORD(unwind_table_.size()), + reinterpret_cast(generated_code_execute_base_), + reinterpret_cast(generated_code_execute_base_ + + kGeneratedCodeSize))) { + XELOGE("Unable to create unwind function table"); + return false; + } + } else { + // Install a callback that the debugger will use to lookup unwind info on + // demand. + if (!RtlInstallFunctionTableCallback( + reinterpret_cast(generated_code_execute_base_) | 0x3, + reinterpret_cast(generated_code_execute_base_), + kGeneratedCodeSize, + [](DWORD64 control_pc, PVOID context) { + auto code_cache = reinterpret_cast(context); + return reinterpret_cast( + code_cache->LookupUnwindInfo(control_pc)); + }, + this, nullptr)) { + XELOGE("Unable to install function table callback"); + return false; + } + } + + return true; +} + +Win32A64CodeCache::UnwindReservation +Win32A64CodeCache::RequestUnwindReservation(uint8_t* entry_address) { + assert_false(unwind_table_count_ >= kMaximumFunctionCount); + UnwindReservation unwind_reservation; + unwind_reservation.data_size = xe::round_up(kUnwindInfoSize, 16); + unwind_reservation.table_slot = unwind_table_count_++; + unwind_reservation.entry_address = entry_address; + return unwind_reservation; +} + +void Win32A64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void* code_execute_address, + UnwindReservation unwind_reservation) { + // Add unwind info. + InitializeUnwindEntry(unwind_reservation.entry_address, + unwind_reservation.table_slot, code_execute_address, + func_info); + + if (supports_growable_table_) { + // Notify that the unwind table has grown. + // We do this outside of the lock, but with the latest total count. + grow_table_(unwind_table_handle_, unwind_table_count_); + } + + // This isn't needed on a64 (probably), but is convention. + // On UWP, FlushInstructionCache available starting from 10.0.16299.0. + // https://docs.microsoft.com/en-us/uwp/win32-and-com/win32-apis + FlushInstructionCache(GetCurrentProcess(), code_execute_address, + func_info.code_size.total); +} + +void Win32A64CodeCache::InitializeUnwindEntry( + uint8_t* unwind_entry_address, size_t unwind_table_slot, + void* code_execute_address, const EmitFunctionInfo& func_info) { + auto unwind_info = reinterpret_cast(unwind_entry_address); + UNWIND_CODE* unwind_code = nullptr; + + assert_true(func_info.code_size.prolog < 256); // needs to fit into a uint8_t + auto prolog_size = static_cast(func_info.code_size.prolog); + assert_true(func_info.prolog_stack_alloc_offset < + 256); // needs to fit into a uint8_t + auto prolog_stack_alloc_offset = + static_cast(func_info.prolog_stack_alloc_offset); + + if (!func_info.stack_size) { + // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info + unwind_info->Version = 1; + unwind_info->Flags = 0; + unwind_info->SizeOfProlog = prolog_size; + unwind_info->CountOfCodes = 0; + unwind_info->FrameRegister = 0; + unwind_info->FrameOffset = 0; + } else if (func_info.stack_size <= 128) { + // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info + unwind_info->Version = 1; + unwind_info->Flags = 0; + unwind_info->SizeOfProlog = prolog_size; + unwind_info->CountOfCodes = 0; + unwind_info->FrameRegister = 0; + unwind_info->FrameOffset = 0; + + // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_code + unwind_code = &unwind_info->UnwindCode[unwind_info->CountOfCodes++]; + unwind_code->CodeOffset = prolog_stack_alloc_offset; + unwind_code->UnwindOp = UWOP_ALLOC_SMALL; + unwind_code->OpInfo = (func_info.stack_size / 8) - 1; + } else { + // TODO(benvanik): take as parameters? + + // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info + unwind_info->Version = 1; + unwind_info->Flags = 0; + unwind_info->SizeOfProlog = prolog_size; + unwind_info->CountOfCodes = 0; + unwind_info->FrameRegister = 0; + unwind_info->FrameOffset = 0; + + // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_code + unwind_code = &unwind_info->UnwindCode[unwind_info->CountOfCodes++]; + unwind_code->CodeOffset = prolog_stack_alloc_offset; + unwind_code->UnwindOp = UWOP_ALLOC_LARGE; + unwind_code->OpInfo = 0; // One slot for size + + assert_true((func_info.stack_size / 8) < 65536u); + unwind_code = &unwind_info->UnwindCode[unwind_info->CountOfCodes++]; + unwind_code->FrameOffset = (USHORT)(func_info.stack_size) / 8; + } + + if (unwind_info->CountOfCodes % 1) { + // Count of unwind codes must always be even. + std::memset(&unwind_info->UnwindCode[unwind_info->CountOfCodes + 1], 0, + sizeof(UNWIND_CODE)); + } + + // Add entry. + auto& fn_entry = unwind_table_[unwind_table_slot]; + fn_entry.BeginAddress = + DWORD(reinterpret_cast(code_execute_address) - + generated_code_execute_base_); + fn_entry.FunctionLength = + DWORD(func_info.code_size.total); + fn_entry.UnwindData = + DWORD(unwind_entry_address - generated_code_execute_base_); +} + +void* Win32A64CodeCache::LookupUnwindInfo(uint64_t host_pc) { + return std::bsearch( + &host_pc, unwind_table_.data(), unwind_table_count_, + sizeof(RUNTIME_FUNCTION), + [](const void* key_ptr, const void* element_ptr) { + auto key = *reinterpret_cast(key_ptr) - + kGeneratedCodeExecuteBase; + auto element = reinterpret_cast(element_ptr); + if (key < element->BeginAddress) { + return -1; + } else if (key > (element->BeginAddress + element->FunctionLength)) { + return 1; + } else { + return 0; + } + }); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_emitter.cc b/a64_emitter.cc new file mode 100644 index 000000000..6010d762f --- /dev/null +++ b/a64_emitter.cc @@ -0,0 +1,961 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_emitter.h" + +#include + +#include +#include + +#include "third_party/fmt/include/fmt/format.h" +#include "xenia/base/assert.h" +#include "xenia/base/atomic.h" +#include "xenia/base/debugging.h" +#include "xenia/base/literals.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" +#include "xenia/base/profiling.h" +#include "xenia/base/vec128.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_code_cache.h" +#include "xenia/cpu/backend/a64/a64_function.h" +#include "xenia/cpu/backend/a64/a64_sequences.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/cpu_flags.h" +#include "xenia/cpu/function.h" +#include "xenia/cpu/function_debug_info.h" +#include "xenia/cpu/processor.h" +#include "xenia/cpu/symbol.h" +#include "xenia/cpu/thread_state.h" + +#include "oaknut/feature_detection/cpu_feature.hpp" +#include "oaknut/feature_detection/feature_detection.hpp" + +DEFINE_bool(debugprint_trap_log, false, + "Log debugprint traps to the active debugger", "CPU"); +DEFINE_bool(ignore_undefined_externs, true, + "Don't exit when an undefined extern is called.", "CPU"); +DEFINE_bool(emit_source_annotations, false, + "Add extra movs and nops to make disassembly easier to read.", + "CPU"); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using xe::cpu::hir::HIRBuilder; +using xe::cpu::hir::Instr; +using namespace xe::literals; +using namespace oaknut::util; + +static const size_t kMaxCodeSize = 1_MiB; + +static const size_t kStashOffset = 32; +// static const size_t kStashOffsetHigh = 32 + 32; + +const uint32_t A64Emitter::gpr_reg_map_[A64Emitter::GPR_COUNT] = { + 1, 10, 11, 12, 13, 14, 15, +}; + +const uint32_t A64Emitter::xmm_reg_map_[A64Emitter::XMM_COUNT] = { + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, +}; + +A64Emitter::A64Emitter(A64Backend* backend) + : CodeBlock(4_KiB), + CodeGenerator(CodeBlock::ptr()), + processor_(backend->processor()), + backend_(backend), + code_cache_(backend->code_cache()) { + const oaknut::CpuFeatures cpu_ = oaknut::detect_features(); +#define TEST_EMIT_FEATURE(emit, ext) \ + if ((cvars::a64_extension_mask & emit) == emit) { \ + feature_flags_ |= (cpu_.has(ext) ? emit : 0); \ + } + + // TEST_EMIT_FEATURE(kA64EmitAVX2, oaknut::util::Cpu::tAVX2); + // TEST_EMIT_FEATURE(kA64EmitFMA, oaknut::util::Cpu::tFMA); + // TEST_EMIT_FEATURE(kA64EmitLZCNT, oaknut::util::Cpu::tLZCNT); + // TEST_EMIT_FEATURE(kA64EmitBMI1, oaknut::util::Cpu::tBMI1); + // TEST_EMIT_FEATURE(kA64EmitBMI2, oaknut::util::Cpu::tBMI2); + // TEST_EMIT_FEATURE(kA64EmitF16C, oaknut::util::Cpu::tF16C); + // TEST_EMIT_FEATURE(kA64EmitMovbe, oaknut::util::Cpu::tMOVBE); + // TEST_EMIT_FEATURE(kA64EmitGFNI, oaknut::util::Cpu::tGFNI); + // TEST_EMIT_FEATURE(kA64EmitAVX512F, oaknut::util::Cpu::tAVX512F); + // TEST_EMIT_FEATURE(kA64EmitAVX512VL, oaknut::util::Cpu::tAVX512VL); + // TEST_EMIT_FEATURE(kA64EmitAVX512BW, oaknut::util::Cpu::tAVX512BW); + // TEST_EMIT_FEATURE(kA64EmitAVX512DQ, oaknut::util::Cpu::tAVX512DQ); + // TEST_EMIT_FEATURE(kA64EmitAVX512VBMI, oaknut::util::Cpu::tAVX512_VBMI); + +#undef TEST_EMIT_FEATURE +} + +A64Emitter::~A64Emitter() = default; + +bool A64Emitter::Emit(GuestFunction* function, HIRBuilder* builder, + uint32_t debug_info_flags, FunctionDebugInfo* debug_info, + void** out_code_address, size_t* out_code_size, + std::vector* out_source_map) { + SCOPE_profile_cpu_f("cpu"); + + // Reset. + debug_info_ = debug_info; + debug_info_flags_ = debug_info_flags; + trace_data_ = &function->trace_data(); + source_map_arena_.Reset(); + + // Fill the generator with code. + EmitFunctionInfo func_info = {}; + if (!Emit(builder, func_info)) { + return false; + } + + // Copy the final code to the cache and relocate it. + *out_code_size = offset(); + *out_code_address = Emplace(func_info, function); + + // Stash source map. + source_map_arena_.CloneContents(out_source_map); + + return true; +} + +void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, + GuestFunction* function) { + // To avoid changing xbyak, we do a switcharoo here. + // top_ points to the Xbyak buffer, and since we are in AutoGrow mode + // it has pending relocations. We copy the top_ to our buffer, swap the + // pointer, relocate, then return the original scratch pointer for use. + // top_ is used by Xbyak's ready() as both write base pointer and the absolute + // address base, which would not work on platforms not supporting writable + // executable memory, but Xenia doesn't use absolute label addresses in the + // generated code. + + // uint8_t* old_address = top_; + uint32_t* old_address = CodeBlock::ptr(); + void* new_execute_address; + void* new_write_address; + + // assert_true(func_info.code_size.total == size_); + assert_true(func_info.code_size.total == offset()); + + if (function) { + code_cache_->PlaceGuestCode(function->address(), CodeBlock::ptr(), + func_info, function, new_execute_address, + new_write_address); + } else { + code_cache_->PlaceHostCode(0, CodeBlock::ptr(), func_info, + new_execute_address, new_write_address); + } + // top_ = reinterpret_cast(new_write_address); + // set_wptr(reinterpret_cast(new_write_address)); + + // ready(); + + // top_ = old_address; + set_wptr(reinterpret_cast(old_address)); + // reset(); + + return new_execute_address; +} + +bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { + oaknut::Label epilog_label; + epilog_label_ = &epilog_label; + + // Calculate stack size. We need to align things to their natural sizes. + // This could be much better (sort by type/etc). + auto locals = builder->locals(); + size_t stack_offset = StackLayout::GUEST_STACK_SIZE; + for (auto it = locals.begin(); it != locals.end(); ++it) { + auto slot = *it; + size_t type_size = GetTypeSize(slot->type); + + // Align to natural size. + stack_offset = xe::align(stack_offset, type_size); + slot->set_constant((uint32_t)stack_offset); + stack_offset += type_size; + } + + // Ensure 16b alignment. + stack_offset -= StackLayout::GUEST_STACK_SIZE; + stack_offset = xe::align(stack_offset, static_cast(16)); + + struct _code_offsets { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + code_offsets.prolog = offset(); + + // Function prolog. + // Must be 16b aligned. + // Windows is very strict about the form of this and the epilog: + // https://docs.microsoft.com/en-us/cpp/build/prolog-and-epilog?view=vs-2017 + // IMPORTANT: any changes to the prolog must be kept in sync with + // A64CodeCache, which dynamically generates exception information. + // Adding or changing anything here must be matched! + const size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; + assert_true(stack_size % 16 == 0); + func_info.stack_size = stack_size; + stack_size_ = stack_size; + + // sub(rsp, (uint32_t)stack_size); + SUB(XSP, XSP, stack_size); + + code_offsets.prolog_stack_alloc = offset(); + code_offsets.body = offset(); + + // mov(qword[rsp + StackLayout::GUEST_CTX_HOME], GetContextReg()); + // mov(qword[rsp + StackLayout::GUEST_RET_ADDR], rcx); + // mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], 0); + STR(GetContextReg(), XSP, StackLayout::GUEST_CTX_HOME); + STR(X0, XSP, StackLayout::GUEST_RET_ADDR); + STR(XZR, XSP, StackLayout::GUEST_CALL_RET_ADDR); + + // Safe now to do some tracing. + if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctions) { + //// We require 32-bit addresses. + // assert_true(uint64_t(trace_data_->header()) < UINT_MAX); + // auto trace_header = trace_data_->header(); + + //// Call count. + // lock(); + // inc(qword[low_address(&trace_header->function_call_count)]); + + //// Get call history slot. + // static_assert(FunctionTraceData::kFunctionCallerHistoryCount == 4, + // "bitmask depends on count"); + // mov(rax, qword[low_address(&trace_header->function_call_count)]); + // and_(rax, 0b00000011); + + //// Record call history value into slot (guest addr in RDX). + // mov(dword[Xbyak::RegExp(uint32_t(uint64_t( + // low_address(&trace_header->function_caller_history)))) + + // rax * 4], + // edx); + + //// Calling thread. Load ax with thread ID. + // EmitGetCurrentThreadId(); + // lock(); + // bts(qword[low_address(&trace_header->function_thread_use)], rax); + } + + // Load membase. + // mov(GetMembaseReg(), + // qword[GetContextReg() + offsetof(ppc::PPCContext, virtual_membase)]); + LDR(GetMembaseReg(), GetContextReg(), + offsetof(ppc::PPCContext, virtual_membase)); + + // Body. + auto block = builder->first_block(); + while (block) { + // Mark block labels. + auto label = block->label_head; + while (label) { + // TODO(wunkolo): string-labels? + // L(label->name); + label = label->next; + } + + // Process instructions. + const Instr* instr = block->instr_head; + while (instr) { + const Instr* new_tail = instr; + if (!SelectSequence(this, instr, &new_tail)) { + // No sequence found! + // NOTE: If you encounter this after adding a new instruction, do a full + // rebuild! + assert_always(); + XELOGE("Unable to process HIR opcode {}", instr->opcode->name); + break; + } + instr = new_tail; + } + + block = block->next; + } + + // Function epilog. + l(epilog_label); + epilog_label_ = nullptr; + EmitTraceUserCallReturn(); + // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); + LDR(GetContextReg(), XSP, StackLayout::GUEST_CTX_HOME); + + code_offsets.epilog = offset(); + + // add(rsp, (uint32_t)stack_size); + // ret(); + ADD(XSP, XSP, stack_size); + RET(); + + code_offsets.tail = offset(); + + if (cvars::emit_source_annotations) { + NOP(); + NOP(); + NOP(); + NOP(); + NOP(); + } + + assert_zero(code_offsets.prolog); + func_info.code_size.total = offset(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = offset() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + + return true; +} + +void A64Emitter::MarkSourceOffset(const Instr* i) { + auto entry = source_map_arena_.Alloc(); + entry->guest_address = static_cast(i->src1.offset); + entry->hir_offset = uint32_t(i->block->ordinal << 16) | i->ordinal; + entry->code_offset = static_cast(offset()); + + if (cvars::emit_source_annotations) { + NOP(); + NOP(); + // mov(eax, entry->guest_address); + MOV(X0, entry->guest_address); + NOP(); + NOP(); + } + + if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctionCoverage) { + uint32_t instruction_index = + (entry->guest_address - trace_data_->start_address()) / 4; + // lock(); + // inc(qword[low_address(trace_data_->instruction_execute_counts() + + // instruction_index * 8)]); + } +} + +void A64Emitter::EmitGetCurrentThreadId() { + // rsi must point to context. We could fetch from the stack if needed. + // mov(ax, word[GetContextReg() + offsetof(ppc::PPCContext, thread_id)]); + LDRB(W0, GetContextReg(), offsetof(ppc::PPCContext, thread_id)); +} + +void A64Emitter::EmitTraceUserCallReturn() {} + +void A64Emitter::DebugBreak() { BRK(0xF000); } + +uint64_t TrapDebugPrint(void* raw_context, uint64_t address) { + auto thread_state = *reinterpret_cast(raw_context); + uint32_t str_ptr = uint32_t(thread_state->context()->r[3]); + // uint16_t str_len = uint16_t(thread_state->context()->r[4]); + auto str = thread_state->memory()->TranslateVirtual(str_ptr); + // TODO(benvanik): truncate to length? + XELOGD("(DebugPrint) {}", str); + + if (cvars::debugprint_trap_log) { + debugging::DebugPrint("(DebugPrint) {}", str); + } + + return 0; +} + +uint64_t TrapDebugBreak(void* raw_context, uint64_t address) { + auto thread_state = *reinterpret_cast(raw_context); + XELOGE("tw/td forced trap hit! This should be a crash!"); + if (cvars::break_on_debugbreak) { + xe::debugging::Break(); + } + return 0; +} + +void A64Emitter::Trap(uint16_t trap_type) { + switch (trap_type) { + case 20: + case 26: + // 0x0FE00014 is a 'debug print' where r3 = buffer r4 = length + CallNative(TrapDebugPrint, 0); + break; + case 0: + case 22: + // Always trap? + // TODO(benvanik): post software interrupt to debugger. + CallNative(TrapDebugBreak, 0); + break; + case 25: + // ? + break; + default: + XELOGW("Unknown trap type {}", trap_type); + // db(0xCC); + BRK(0xF000); + break; + } +} + +void A64Emitter::UnimplementedInstr(const hir::Instr* i) { + // TODO(benvanik): notify debugger. + // db(0xCC); + BRK(0xF000); + assert_always(); +} + +// This is used by the A64ThunkEmitter's ResolveFunctionThunk. +uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { + auto thread_state = *reinterpret_cast(raw_context); + + // TODO(benvanik): required? + assert_not_zero(target_address); + + auto fn = thread_state->processor()->ResolveFunction( + static_cast(target_address)); + assert_not_null(fn); + auto a64_fn = static_cast(fn); + uint64_t addr = reinterpret_cast(a64_fn->machine_code()); + + return addr; +} + +void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { + assert_not_null(function); + auto fn = static_cast(function); + // Resolve address to the function to call and store in rax. + if (fn->machine_code()) { + // TODO(benvanik): is it worth it to do this? It removes the need for + // a ResolveFunction call, but makes the table less useful. + assert_zero(uint64_t(fn->machine_code()) & 0xFFFFFFFF00000000); + // mov(eax, uint32_t(uint64_t(fn->machine_code()))); + MOV(X16, uint32_t(uint64_t(fn->machine_code()))); + } else if (code_cache_->has_indirection_table()) { + // Load the pointer to the indirection table maintained in A64CodeCache. + // The target dword will either contain the address of the generated code + // or a thunk to ResolveAddress. + // mov(ebx, function->address()); + // mov(eax, dword[ebx]); + MOV(X16, function->address()); + LDR(W16, X16); + } else { + // Old-style resolve. + // Not too important because indirection table is almost always available. + // TODO: Overwrite the call-site with a straight call. + CallNative(&ResolveFunction, function->address()); + } + + // Actually jump/call to rax. + if (instr->flags & hir::CALL_TAIL) { + // Since we skip the prolog we need to mark the return here. + EmitTraceUserCallReturn(); + + // Pass the callers return address over. + // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); + LDR(X0, XSP, StackLayout::GUEST_RET_ADDR); + + // add(rsp, static_cast(stack_size())); + // jmp(rax); + ADD(XSP, XSP, stack_size()); + BR(X16); + } else { + // Return address is from the previous SET_RETURN_ADDRESS. + // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); + LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + + // call(rax); + BR(X16); + } +} + +void A64Emitter::CallIndirect(const hir::Instr* instr, + const oaknut::XReg& reg) { + // Check if return. + if (instr->flags & hir::CALL_POSSIBLE_RETURN) { + // cmp(reg.cvt32(), dword[rsp + StackLayout::GUEST_RET_ADDR]); + // je(epilog_label(), CodeGenerator::T_NEAR); + LDR(W0, XSP, StackLayout::GUEST_RET_ADDR); + CMP(reg.toW(), W0); + B(oaknut::Cond::EQ, epilog_label()); + } + + // Load the pointer to the indirection table maintained in A64CodeCache. + // The target dword will either contain the address of the generated code + // or a thunk to ResolveAddress. + if (code_cache_->has_indirection_table()) { + if (reg.toW().index() != W1.index()) { + // mov(ebx, reg.cvt32()); + MOV(W1, reg.toW()); + } + // mov(eax, dword[ebx]); + LDR(X16, X1); + } else { + // Old-style resolve. + // Not too important because indirection table is almost always available. + // mov(rcx, GetContextReg()); + // mov(edx, reg.cvt32()); + // + // mov(rax, reinterpret_cast(ResolveFunction)); + // call(rax); + MOV(X0, GetContextReg()); + MOV(W1, reg.toW()); + + ADRP(X16, ResolveFunction); + BLR(X16); + } + + // Actually jump/call to rax. + if (instr->flags & hir::CALL_TAIL) { + // Since we skip the prolog we need to mark the return here. + EmitTraceUserCallReturn(); + + // Pass the callers return address over. + // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); + LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + + // add(rsp, static_cast(stack_size())); + ADD(XSP, XSP, stack_size()); + + // jmp(rax); + BR(X16); + } else { + // Return address is from the previous SET_RETURN_ADDRESS. + // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); + // call(rax); + LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + BLR(X16); + } +} + +uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) { + auto function = reinterpret_cast(function_ptr); + if (!cvars::ignore_undefined_externs) { + xe::FatalError(fmt::format("undefined extern call to {:08X} {}", + function->address(), function->name().c_str())); + } else { + XELOGE("undefined extern call to {:08X} {}", function->address(), + function->name()); + } + return 0; +} +void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { + bool undefined = true; + if (function->behavior() == Function::Behavior::kBuiltin) { + auto builtin_function = static_cast(function); + if (builtin_function->handler()) { + undefined = false; + // x0 = target function + // x1 = arg0 + // x2 = arg1 + // x3 = arg2 + auto thunk = backend()->guest_to_host_thunk(); + // mov(rax, reinterpret_cast(thunk)); + // mov(rcx, reinterpret_cast(builtin_function->handler())); + // mov(rdx, reinterpret_cast(builtin_function->arg0())); + // mov(r8, reinterpret_cast(builtin_function->arg1())); + // call(rax); + MOV(X0, reinterpret_cast(builtin_function->handler())); + MOV(X1, reinterpret_cast(builtin_function->arg0())); + MOV(X2, reinterpret_cast(builtin_function->arg1())); + + MOV(X16, reinterpret_cast(thunk)); + BLR(X16); + // x0 = host return + } + } else if (function->behavior() == Function::Behavior::kExtern) { + auto extern_function = static_cast(function); + if (extern_function->extern_handler()) { + undefined = false; + // x0 = target function + // x1 = arg0 + // x2 = arg1 + // x3 = arg2 + auto thunk = backend()->guest_to_host_thunk(); + // mov(rax, reinterpret_cast(thunk)); + // mov(rcx, + // reinterpret_cast(extern_function->extern_handler())); + // mov(rdx, + // qword[GetContextReg() + offsetof(ppc::PPCContext, kernel_state)]); + // call(rax); + MOV(X0, reinterpret_cast(thunk)); + MOV(X1, reinterpret_cast(extern_function->extern_handler())); + LDR(X2, GetContextReg(), offsetof(ppc::PPCContext, kernel_state)); + + MOV(X16, reinterpret_cast(thunk)); + BLR(X16); + // x0 = host return + } + } + if (undefined) { + CallNative(UndefinedCallExtern, reinterpret_cast(function)); + } +} + +void A64Emitter::CallNative(void* fn) { CallNativeSafe(fn); } + +void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context)) { + CallNativeSafe(reinterpret_cast(fn)); +} + +void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0)) { + CallNativeSafe(reinterpret_cast(fn)); +} + +void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0), + uint64_t arg0) { + // mov(GetNativeParam(0), arg0); + MOV(GetNativeParam(0), arg0); + CallNativeSafe(reinterpret_cast(fn)); +} + +void A64Emitter::CallNativeSafe(void* fn) { + // X0 = target function + // X1 = arg0 + // X2 = arg1 + // X3 = arg2 + auto thunk = backend()->guest_to_host_thunk(); + // mov(rax, reinterpret_cast(thunk)); + // mov(rcx, reinterpret_cast(fn)); + // call(rax); + MOV(X0, reinterpret_cast(fn)); + MOV(X16, reinterpret_cast(thunk)); + BLR(X16); + // X0 = host return +} + +void A64Emitter::SetReturnAddress(uint64_t value) { + // mov(rax, value); + // mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], rax); + MOV(X0, value); + STR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); +} + +oaknut::XReg A64Emitter::GetNativeParam(uint32_t param) { + if (param == 0) + return X1; + else if (param == 1) + return X2; + else if (param == 2) + return X3; + + assert_always(); + return X3; +} + +// Important: If you change these, you must update the thunks in a64_backend.cc! +oaknut::XReg A64Emitter::GetContextReg() { return X14; } +oaknut::XReg A64Emitter::GetMembaseReg() { return X15; } + +void A64Emitter::ReloadContext() { + // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); + LDR(GetContextReg(), XSP, StackLayout::GUEST_CTX_HOME); +} + +void A64Emitter::ReloadMembase() { + // mov(GetMembaseReg(), qword[GetContextReg() + 8]); // membase + LDR(GetMembaseReg(), GetContextReg(), 8); // membase +} + +bool A64Emitter::ConstantFitsIn32Reg(uint64_t v) { + if ((v & ~0x7FFFFFFF) == 0) { + // Fits under 31 bits, so just load using normal mov. + return true; + } else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) { + // Negative number that fits in 32bits. + return true; + } + return false; +} + +void A64Emitter::MovMem64(const oaknut::XRegSp& addr, intptr_t offset, + uint64_t v) { + // if ((v & ~0x7FFFFFFF) == 0) { + // // Fits under 31 bits, so just load using normal mov. + // mov(qword[addr], v); + // } else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) { + // // Negative number that fits in 32bits. + // mov(qword[addr], v); + // } else if (!(v >> 32)) { + // // All high bits are zero. It'd be nice if we had a way to load a 32bit + // // immediate without sign extending! + // // TODO(benvanik): this is super common, find a better way. + // mov(dword[addr], static_cast(v)); + // mov(dword[addr + 4], 0); + // } else + { + // 64bit number that needs double movs. + MOV(X0, v); + STR(X0, addr, offset); + } +} + +static const vec128_t xmm_consts[] = { + /* VZero */ vec128f(0.0f), + /* VOne */ vec128f(1.0f), + /* VOnePD */ vec128d(1.0), + /* VNegativeOne */ vec128f(-1.0f, -1.0f, -1.0f, -1.0f), + /* VFFFF */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu), + /* VMaskX16Y16 */ + vec128i(0x0000FFFFu, 0xFFFF0000u, 0x00000000u, 0x00000000u), + /* VFlipX16Y16 */ + vec128i(0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u), + /* VFixX16Y16 */ vec128f(-32768.0f, 0.0f, 0.0f, 0.0f), + /* VNormalizeX16Y16 */ + vec128f(1.0f / 32767.0f, 1.0f / (32767.0f * 65536.0f), 0.0f, 0.0f), + /* V0001 */ vec128f(0.0f, 0.0f, 0.0f, 1.0f), + /* V3301 */ vec128f(3.0f, 3.0f, 0.0f, 1.0f), + /* V3331 */ vec128f(3.0f, 3.0f, 3.0f, 1.0f), + /* V3333 */ vec128f(3.0f, 3.0f, 3.0f, 3.0f), + /* VSignMaskPS */ + vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u), + /* VSignMaskPD */ + vec128i(0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u), + /* VAbsMaskPS */ + vec128i(0x7FFFFFFFu, 0x7FFFFFFFu, 0x7FFFFFFFu, 0x7FFFFFFFu), + /* VAbsMaskPD */ + vec128i(0xFFFFFFFFu, 0x7FFFFFFFu, 0xFFFFFFFFu, 0x7FFFFFFFu), + /* VByteSwapMask */ + vec128i(0x00010203u, 0x04050607u, 0x08090A0Bu, 0x0C0D0E0Fu), + /* VByteOrderMask */ + vec128i(0x01000302u, 0x05040706u, 0x09080B0Au, 0x0D0C0F0Eu), + /* VPermuteControl15 */ vec128b(15), + /* VPermuteByteMask */ vec128b(0x1F), + /* VPackD3DCOLORSat */ vec128i(0x404000FFu), + /* VPackD3DCOLOR */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0C000408u), + /* VUnpackD3DCOLOR */ + vec128i(0xFFFFFF0Eu, 0xFFFFFF0Du, 0xFFFFFF0Cu, 0xFFFFFF0Fu), + /* VPackFLOAT16_2 */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000302u), + /* VUnpackFLOAT16_2 */ + vec128i(0x0D0C0F0Eu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu), + /* VPackFLOAT16_4 */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000302u, 0x05040706u), + /* VUnpackFLOAT16_4 */ + vec128i(0x09080B0Au, 0x0D0C0F0Eu, 0xFFFFFFFFu, 0xFFFFFFFFu), + /* VPackSHORT_Min */ vec128i(0x403F8001u), + /* VPackSHORT_Max */ vec128i(0x40407FFFu), + /* VPackSHORT_2 */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000504u), + /* VPackSHORT_4 */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000504u, 0x09080D0Cu), + /* VUnpackSHORT_2 */ + vec128i(0xFFFF0F0Eu, 0xFFFF0D0Cu, 0xFFFFFFFFu, 0xFFFFFFFFu), + /* VUnpackSHORT_4 */ + vec128i(0xFFFF0B0Au, 0xFFFF0908u, 0xFFFF0F0Eu, 0xFFFF0D0Cu), + /* VUnpackSHORT_Overflow */ vec128i(0x403F8000u), + /* VPackUINT_2101010_MinUnpacked */ + vec128i(0x403FFE01u, 0x403FFE01u, 0x403FFE01u, 0x40400000u), + /* VPackUINT_2101010_MaxUnpacked */ + vec128i(0x404001FFu, 0x404001FFu, 0x404001FFu, 0x40400003u), + /* VPackUINT_2101010_MaskUnpacked */ + vec128i(0x3FFu, 0x3FFu, 0x3FFu, 0x3u), + /* VPackUINT_2101010_MaskPacked */ + vec128i(0x3FFu, 0x3FFu << 10, 0x3FFu << 20, 0x3u << 30), + /* VPackUINT_2101010_Shift */ vec128i(0, 10, 20, 30), + /* VUnpackUINT_2101010_Overflow */ vec128i(0x403FFE00u), + /* VPackULONG_4202020_MinUnpacked */ + vec128i(0x40380001u, 0x40380001u, 0x40380001u, 0x40400000u), + /* VPackULONG_4202020_MaxUnpacked */ + vec128i(0x4047FFFFu, 0x4047FFFFu, 0x4047FFFFu, 0x4040000Fu), + /* VPackULONG_4202020_MaskUnpacked */ + vec128i(0xFFFFFu, 0xFFFFFu, 0xFFFFFu, 0xFu), + /* VPackULONG_4202020_PermuteXZ */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x0A0908FFu, 0xFF020100u), + /* VPackULONG_4202020_PermuteYW */ + vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x0CFFFF06u, 0x0504FFFFu), + /* VUnpackULONG_4202020_Permute */ + vec128i(0xFF0E0D0Cu, 0xFF0B0A09u, 0xFF080F0Eu, 0xFFFFFF0Bu), + /* VUnpackULONG_4202020_Overflow */ vec128i(0x40380000u), + /* VOneOver255 */ vec128f(1.0f / 255.0f), + /* VMaskEvenPI16 */ + vec128i(0x0000FFFFu, 0x0000FFFFu, 0x0000FFFFu, 0x0000FFFFu), + /* VShiftMaskEvenPI16 */ + vec128i(0x0000000Fu, 0x0000000Fu, 0x0000000Fu, 0x0000000Fu), + /* VShiftMaskPS */ + vec128i(0x0000001Fu, 0x0000001Fu, 0x0000001Fu, 0x0000001Fu), + /* VShiftByteMask */ + vec128i(0x000000FFu, 0x000000FFu, 0x000000FFu, 0x000000FFu), + /* VSwapWordMask */ + vec128i(0x03030303u, 0x03030303u, 0x03030303u, 0x03030303u), + /* VUnsignedDwordMax */ + vec128i(0xFFFFFFFFu, 0x00000000u, 0xFFFFFFFFu, 0x00000000u), + /* V255 */ vec128f(255.0f), + /* VPI32 */ vec128i(32), + /* VSignMaskI8 */ + vec128i(0x80808080u, 0x80808080u, 0x80808080u, 0x80808080u), + /* VSignMaskI16 */ + vec128i(0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u), + /* VSignMaskI32 */ + vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u), + /* VSignMaskF32 */ + vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u), + /* VShortMinPS */ vec128f(SHRT_MIN), + /* VShortMaxPS */ vec128f(SHRT_MAX), + /* VIntMin */ vec128i(INT_MIN), + /* VIntMax */ vec128i(INT_MAX), + /* VIntMaxPD */ vec128d(INT_MAX), + /* VPosIntMinPS */ vec128f((float)0x80000000u), + /* VQNaN */ vec128i(0x7FC00000u), + /* VInt127 */ vec128i(0x7Fu), + /* V2To32 */ vec128f(0x1.0p32f), +}; + +// First location to try and place constants. +static const uintptr_t kConstDataLocation = 0x20000000; +static const uintptr_t kConstDataSize = sizeof(xmm_consts); + +// Increment the location by this amount for every allocation failure. +static const uintptr_t kConstDataIncrement = 0x00001000; + +// This function places constant data that is used by the emitter later on. +// Only called once and used by multiple instances of the emitter. +// +// TODO(DrChat): This should be placed in the code cache with the code, but +// doing so requires RIP-relative addressing, which is difficult to support +// given the current setup. +uintptr_t A64Emitter::PlaceConstData() { + uint8_t* ptr = reinterpret_cast(kConstDataLocation); + void* mem = nullptr; + while (!mem) { + mem = memory::AllocFixed( + ptr, xe::round_up(kConstDataSize, memory::page_size()), + memory::AllocationType::kReserveCommit, memory::PageAccess::kReadWrite); + + ptr += kConstDataIncrement; + } + + // The pointer must not be greater than 31 bits. + assert_zero(reinterpret_cast(mem) & ~0x7FFFFFFF); + std::memcpy(mem, xmm_consts, sizeof(xmm_consts)); + memory::Protect(mem, kConstDataSize, memory::PageAccess::kReadOnly, nullptr); + + return reinterpret_cast(mem); +} + +void A64Emitter::FreeConstData(uintptr_t data) { + memory::DeallocFixed(reinterpret_cast(data), 0, + memory::DeallocationType::kRelease); +} + +std::byte* A64Emitter::GetVConstPtr(VConst id) { + // Load through fixed constant table setup by PlaceConstData. + // It's important that the pointer is not signed, as it will be sign-extended. + return reinterpret_cast(backend_->emitter_data() + + sizeof(vec128_t) * id); +} + +// Implies possible StashV(0, ...)! +void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { + // https://www.agner.org/optimize/optimizing_assembly.pdf + // 13.4 Generating constants + if (!v.low && !v.high) { + // 0000... + EOR(dest.B16(), dest.B16(), dest.B16()); + } + // else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) { + // // 1111... + // vpcmpeqb(dest, dest); + // } + else { + // TODO(benvanik): see what other common values are. + // TODO(benvanik): build constant table - 99% are reused. + MovMem64(XSP, kStashOffset, v.low); + MovMem64(XSP, kStashOffset + 8, v.high); + LDR(dest, XSP, kStashOffset); + } +} + +void A64Emitter::LoadConstantV(oaknut::QReg dest, float v) { + union { + float f; + uint32_t i; + } x = {v}; + if (!x.i) { + // +0.0f (but not -0.0f because it may be used to flip the sign via xor). + EOR(dest.B16(), dest.B16(), dest.B16()); + } + // else if (x.i == ~uint32_t(0)) { + // // 1111... + // vpcmpeqb(dest, dest); + // } + else { + // TODO(benvanik): see what other common values are. + // TODO(benvanik): build constant table - 99% are reused. + MOV(W0, x.i); + MOV(dest.Selem()[0], W0); + } +} + +void A64Emitter::LoadConstantV(oaknut::QReg dest, double v) { + union { + double d; + uint64_t i; + } x = {v}; + if (!x.i) { + // +0.0 (but not -0.0 because it may be used to flip the sign via xor). + EOR(dest.B16(), dest.B16(), dest.B16()); + } + // else if (x.i == ~uint64_t(0)) { + // // 1111... + // vpcmpeqb(dest, dest); + // } + else { + // TODO(benvanik): see what other common values are. + // TODO(benvanik): build constant table - 99% are reused. + MOV(X0, x.i); + MOV(dest.Delem()[0], X0); + } +} + +uintptr_t A64Emitter::StashV(int index, const oaknut::QReg& r) { + // auto addr = ptr[rsp + kStashOffset + (index * 16)]; + // vmovups(addr, r); + const auto addr = kStashOffset + (index * 16); + STR(r, XSP, addr); + return addr; +} + +uintptr_t A64Emitter::StashConstantV(int index, float v) { + union { + float f; + uint32_t i; + } x = {v}; + const auto addr = kStashOffset + (index * 16); + MovMem64(XSP, addr, x.i); + MovMem64(XSP, addr + 8, 0); + return addr; +} + +uintptr_t A64Emitter::StashConstantV(int index, double v) { + union { + double d; + uint64_t i; + } x = {v}; + const auto addr = kStashOffset + (index * 16); + MovMem64(XSP, addr, x.i); + MovMem64(XSP, addr + 8, 0); + return addr; +} + +uintptr_t A64Emitter::StashConstantV(int index, const vec128_t& v) { + const auto addr = kStashOffset + (index * 16); + MovMem64(XSP, addr, v.low); + MovMem64(XSP, addr + 8, v.high); + return addr; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_emitter.h b/a64_emitter.h new file mode 100644 index 000000000..1cbd60e30 --- /dev/null +++ b/a64_emitter.h @@ -0,0 +1,250 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ +#define XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ + +#include + +#include "xenia/base/arena.h" +#include "xenia/cpu/function.h" +#include "xenia/cpu/function_trace_data.h" +#include "xenia/cpu/hir/hir_builder.h" +#include "xenia/cpu/hir/instr.h" +#include "xenia/cpu/hir/value.h" +#include "xenia/memory.h" + +#include "oaknut/code_block.hpp" +#include "oaknut/oaknut.hpp" + +namespace xe { +namespace cpu { +class Processor; +} // namespace cpu +} // namespace xe + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Backend; +class A64CodeCache; + +struct EmitFunctionInfo; + +enum RegisterFlags { + REG_DEST = (1 << 0), + REG_ABCD = (1 << 1), +}; + +enum VConst { + VZero = 0, + VOne, + VOnePD, + VNegativeOne, + VFFFF, + VMaskX16Y16, + VFlipX16Y16, + VFixX16Y16, + VNormalizeX16Y16, + V0001, + V3301, + V3331, + V3333, + VSignMaskPS, + VSignMaskPD, + VAbsMaskPS, + VAbsMaskPD, + VByteSwapMask, + VByteOrderMask, + VPermuteControl15, + VPermuteByteMask, + VPackD3DCOLORSat, + VPackD3DCOLOR, + VUnpackD3DCOLOR, + VPackFLOAT16_2, + VUnpackFLOAT16_2, + VPackFLOAT16_4, + VUnpackFLOAT16_4, + VPackSHORT_Min, + VPackSHORT_Max, + VPackSHORT_2, + VPackSHORT_4, + VUnpackSHORT_2, + VUnpackSHORT_4, + VUnpackSHORT_Overflow, + VPackUINT_2101010_MinUnpacked, + VPackUINT_2101010_MaxUnpacked, + VPackUINT_2101010_MaskUnpacked, + VPackUINT_2101010_MaskPacked, + VPackUINT_2101010_Shift, + VUnpackUINT_2101010_Overflow, + VPackULONG_4202020_MinUnpacked, + VPackULONG_4202020_MaxUnpacked, + VPackULONG_4202020_MaskUnpacked, + VPackULONG_4202020_PermuteXZ, + VPackULONG_4202020_PermuteYW, + VUnpackULONG_4202020_Permute, + VUnpackULONG_4202020_Overflow, + VOneOver255, + VMaskEvenPI16, + VShiftMaskEvenPI16, + VShiftMaskPS, + VShiftByteMask, + VSwapWordMask, + VUnsignedDwordMax, + V255, + VPI32, + VSignMaskI8, + VSignMaskI16, + VSignMaskI32, + VSignMaskF32, + VShortMinPS, + VShortMaxPS, + VIntMin, + VIntMax, + VIntMaxPD, + VPosIntMinPS, + VQNaN, + VInt127, + V2To32, +}; + +enum A64EmitterFeatureFlags {}; + +class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { + public: + A64Emitter(A64Backend* backend); + virtual ~A64Emitter(); + + Processor* processor() const { return processor_; } + A64Backend* backend() const { return backend_; } + + static uintptr_t PlaceConstData(); + static void FreeConstData(uintptr_t data); + + bool Emit(GuestFunction* function, hir::HIRBuilder* builder, + uint32_t debug_info_flags, FunctionDebugInfo* debug_info, + void** out_code_address, size_t* out_code_size, + std::vector* out_source_map); + + public: + // Reserved: XSP, X14, X15 + // Scratch: X0/X1/X2 + // V0-2 + // Available: X1, X10-r15 + // V4-V15 (save to get V3) + static const int GPR_COUNT = 7; + static const int XMM_COUNT = 12; + + static void SetupReg(const hir::Value* v, oaknut::WReg& r) { + const auto idx = gpr_reg_map_[v->reg.index]; + r = oaknut::WReg(idx); + } + static void SetupReg(const hir::Value* v, oaknut::XReg& r) { + const auto idx = gpr_reg_map_[v->reg.index]; + r = oaknut::XReg(idx); + } + static void SetupReg(const hir::Value* v, oaknut::SReg& r) { + const auto idx = xmm_reg_map_[v->reg.index]; + r = oaknut::SReg(idx); + } + static void SetupReg(const hir::Value* v, oaknut::DReg& r) { + const auto idx = xmm_reg_map_[v->reg.index]; + r = oaknut::DReg(idx); + } + static void SetupReg(const hir::Value* v, oaknut::QReg& r) { + const auto idx = xmm_reg_map_[v->reg.index]; + r = oaknut::QReg(idx); + } + + oaknut::Label& epilog_label() { return *epilog_label_; } + + void MarkSourceOffset(const hir::Instr* i); + + void DebugBreak(); + void Trap(uint16_t trap_type = 0); + void UnimplementedInstr(const hir::Instr* i); + + void Call(const hir::Instr* instr, GuestFunction* function); + void CallIndirect(const hir::Instr* instr, const oaknut::XReg& reg); + void CallExtern(const hir::Instr* instr, const Function* function); + void CallNative(void* fn); + void CallNative(uint64_t (*fn)(void* raw_context)); + void CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0)); + void CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0), + uint64_t arg0); + void CallNativeSafe(void* fn); + void SetReturnAddress(uint64_t value); + + static oaknut::XReg GetNativeParam(uint32_t param); + + static oaknut::XReg GetContextReg(); + static oaknut::XReg GetMembaseReg(); + void ReloadContext(); + void ReloadMembase(); + + // Moves a 64bit immediate into memory. + bool ConstantFitsIn32Reg(uint64_t v); + void MovMem64(const oaknut::XRegSp& addr, intptr_t offset, uint64_t v); + + std::byte* GetVConstPtr(VConst id); + void LoadConstantV(oaknut::QReg dest, float v); + void LoadConstantV(oaknut::QReg dest, double v); + void LoadConstantV(oaknut::QReg dest, const vec128_t& v); + + // Returned addresses are relative to XSP + uintptr_t StashV(int index, const oaknut::QReg& r); + uintptr_t StashConstantV(int index, float v); + uintptr_t StashConstantV(int index, double v); + uintptr_t StashConstantV(int index, const vec128_t& v); + + bool IsFeatureEnabled(uint32_t feature_flag) const { + return (feature_flags_ & feature_flag) == feature_flag; + } + + FunctionDebugInfo* debug_info() const { return debug_info_; } + + size_t stack_size() const { return stack_size_; } + + protected: + void* Emplace(const EmitFunctionInfo& func_info, + GuestFunction* function = nullptr); + bool Emit(hir::HIRBuilder* builder, EmitFunctionInfo& func_info); + void EmitGetCurrentThreadId(); + void EmitTraceUserCallReturn(); + + protected: + Processor* processor_ = nullptr; + A64Backend* backend_ = nullptr; + A64CodeCache* code_cache_ = nullptr; + uint32_t feature_flags_ = 0; + + oaknut::Label* epilog_label_ = nullptr; + + hir::Instr* current_instr_ = nullptr; + + FunctionDebugInfo* debug_info_ = nullptr; + uint32_t debug_info_flags_ = 0; + FunctionTraceData* trace_data_ = nullptr; + Arena source_map_arena_; + + size_t stack_size_ = 0; + + static const uint32_t gpr_reg_map_[GPR_COUNT]; + static const uint32_t xmm_reg_map_[XMM_COUNT]; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ diff --git a/a64_function.cc b/a64_function.cc new file mode 100644 index 000000000..9167bde7c --- /dev/null +++ b/a64_function.cc @@ -0,0 +1,45 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_function.h" + +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/processor.h" +#include "xenia/cpu/thread_state.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +A64Function::A64Function(Module* module, uint32_t address) + : GuestFunction(module, address) {} + +A64Function::~A64Function() { + // machine_code_ is freed by code cache. +} + +void A64Function::Setup(uint8_t* machine_code, size_t machine_code_length) { + machine_code_ = machine_code; + machine_code_length_ = machine_code_length; +} + +bool A64Function::CallImpl(ThreadState* thread_state, uint32_t return_address) { + auto backend = + reinterpret_cast(thread_state->processor()->backend()); + auto thunk = backend->host_to_guest_thunk(); + thunk(machine_code_, thread_state->context(), + reinterpret_cast(uintptr_t(return_address))); + return true; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_function.h b/a64_function.h new file mode 100644 index 000000000..d4c568567 --- /dev/null +++ b/a64_function.h @@ -0,0 +1,44 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ +#define XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ + +#include "xenia/cpu/function.h" +#include "xenia/cpu/thread_state.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Function : public GuestFunction { + public: + A64Function(Module* module, uint32_t address); + ~A64Function() override; + + uint8_t* machine_code() const override { return machine_code_; } + size_t machine_code_length() const override { return machine_code_length_; } + + void Setup(uint8_t* machine_code, size_t machine_code_length); + + protected: + bool CallImpl(ThreadState* thread_state, uint32_t return_address) override; + + private: + uint8_t* machine_code_ = nullptr; + size_t machine_code_length_ = 0; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ diff --git a/a64_op.h b/a64_op.h new file mode 100644 index 000000000..2eaea627c --- /dev/null +++ b/a64_op.h @@ -0,0 +1,618 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2018 Xenia Developers. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ +#ifndef XENIA_CPU_BACKEND_A64_A64_OP_H_ +#define XENIA_CPU_BACKEND_A64_A64_OP_H_ + +#include "xenia/cpu/backend/a64/a64_emitter.h" + +#include "xenia/cpu/hir/instr.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +// TODO(benvanik): direct usings. +using namespace xe::cpu; +using namespace xe::cpu::hir; +using namespace oaknut; +using namespace oaknut::util; + +// Selects the right byte/word/etc from a vector. We need to flip logical +// indices (0,1,2,3,4,5,6,7,...) = (3,2,1,0,7,6,5,4,...) +#define VEC128_B(n) ((n) ^ 0x3) +#define VEC128_W(n) ((n) ^ 0x1) +#define VEC128_D(n) (n) +#define VEC128_F(n) (n) + +enum KeyType { + KEY_TYPE_X = OPCODE_SIG_TYPE_X, + KEY_TYPE_L = OPCODE_SIG_TYPE_L, + KEY_TYPE_O = OPCODE_SIG_TYPE_O, + KEY_TYPE_S = OPCODE_SIG_TYPE_S, + KEY_TYPE_V_I8 = OPCODE_SIG_TYPE_V + INT8_TYPE, + KEY_TYPE_V_I16 = OPCODE_SIG_TYPE_V + INT16_TYPE, + KEY_TYPE_V_I32 = OPCODE_SIG_TYPE_V + INT32_TYPE, + KEY_TYPE_V_I64 = OPCODE_SIG_TYPE_V + INT64_TYPE, + KEY_TYPE_V_F32 = OPCODE_SIG_TYPE_V + FLOAT32_TYPE, + KEY_TYPE_V_F64 = OPCODE_SIG_TYPE_V + FLOAT64_TYPE, + KEY_TYPE_V_V128 = OPCODE_SIG_TYPE_V + VEC128_TYPE, +}; + +#pragma pack(push, 1) +union InstrKey { + uint32_t value; + struct { + uint32_t opcode : 8; + uint32_t dest : 5; + uint32_t src1 : 5; + uint32_t src2 : 5; + uint32_t src3 : 5; + uint32_t reserved : 4; + }; + + operator uint32_t() const { return value; } + + InstrKey() : value(0) { static_assert_size(*this, sizeof(value)); } + InstrKey(uint32_t v) : value(v) {} + InstrKey(const Instr* i) : value(0) { + opcode = i->opcode->num; + uint32_t sig = i->opcode->signature; + dest = + GET_OPCODE_SIG_TYPE_DEST(sig) ? OPCODE_SIG_TYPE_V + i->dest->type : 0; + src1 = GET_OPCODE_SIG_TYPE_SRC1(sig); + if (src1 == OPCODE_SIG_TYPE_V) { + src1 += i->src1.value->type; + } + src2 = GET_OPCODE_SIG_TYPE_SRC2(sig); + if (src2 == OPCODE_SIG_TYPE_V) { + src2 += i->src2.value->type; + } + src3 = GET_OPCODE_SIG_TYPE_SRC3(sig); + if (src3 == OPCODE_SIG_TYPE_V) { + src3 += i->src3.value->type; + } + } + + template + struct Construct { + static const uint32_t value = + (OPCODE) | (DEST << 8) | (SRC1 << 13) | (SRC2 << 18) | (SRC3 << 23); + }; +}; +#pragma pack(pop) +static_assert(sizeof(InstrKey) <= 4, "Key must be 4 bytes"); + +template +struct CombinedStruct; +template <> +struct CombinedStruct<> {}; +template +struct CombinedStruct : T, CombinedStruct {}; + +struct OpBase {}; + +template +struct Op : OpBase { + static const KeyType key_type = KEY_TYPE; +}; + +struct VoidOp : Op { + protected: + friend struct Op; + template + friend struct I; + void Load(const Instr::Op& op) {} +}; + +struct OffsetOp : Op { + uint64_t value; + + protected: + friend struct Op; + template + friend struct I; + void Load(const Instr::Op& op) { this->value = op.offset; } +}; + +struct SymbolOp : Op { + Function* value; + + protected: + friend struct Op; + template + friend struct I; + bool Load(const Instr::Op& op) { + this->value = op.symbol; + return true; + } +}; + +struct LabelOp : Op { + hir::Label* value; + + protected: + friend struct Op; + template + friend struct I; + void Load(const Instr::Op& op) { this->value = op.label; } +}; + +template +struct ValueOp : Op, KEY_TYPE> { + typedef REG_TYPE reg_type; + const Value* value; + bool is_constant; + virtual bool ConstantFitsIn32Reg() const { return true; } + const REG_TYPE& reg() const { + assert_true(!is_constant); + return reg_; + } + operator const REG_TYPE&() const { return reg(); } + bool IsEqual(const T& b) const { + if (is_constant && b.is_constant) { + return reinterpret_cast(this)->constant() == b.constant(); + } else if (!is_constant && !b.is_constant) { + return reg_.index() == b.reg_.index(); + } else { + return false; + } + } + bool IsEqual(const oaknut::Reg& b) const { + if (is_constant) { + return false; + } else if (!is_constant) { + return reg_.index() == b.index(); + } else { + return false; + } + } + bool operator==(const T& b) const { return IsEqual(b); } + bool operator!=(const T& b) const { return !IsEqual(b); } + bool operator==(const oaknut::Reg& b) const { return IsEqual(b); } + bool operator!=(const oaknut::Reg& b) const { return !IsEqual(b); } + void Load(const Instr::Op& op) { + value = op.value; + is_constant = value->IsConstant(); + if (!is_constant) { + A64Emitter::SetupReg(value, reg_); + } + } + + protected: + REG_TYPE reg_ = REG_TYPE(0); +}; + +struct I8Op : ValueOp { + typedef ValueOp BASE; + const int8_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i8; + } +}; +struct I16Op : ValueOp { + typedef ValueOp BASE; + const int16_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i16; + } +}; +struct I32Op : ValueOp { + typedef ValueOp BASE; + const int32_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i32; + } +}; +struct I64Op : ValueOp { + typedef ValueOp BASE; + const int64_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i64; + } + bool ConstantFitsIn32Reg() const override { + int64_t v = BASE::value->constant.i64; + if ((v & ~0x7FFFFFFF) == 0) { + // Fits under 31 bits, so just load using normal mov. + return true; + } else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) { + // Negative number that fits in 32bits. + return true; + } + return false; + } +}; +struct F32Op : ValueOp { + typedef ValueOp BASE; + const float constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.f32; + } +}; +struct F64Op : ValueOp { + typedef ValueOp BASE; + const double constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.f64; + } +}; +struct V128Op : ValueOp { + typedef ValueOp BASE; + const vec128_t& constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.v128; + } +}; + +template +struct DestField; +template +struct DestField { + DEST dest; + + protected: + bool LoadDest(const Instr* i) { + Instr::Op op; + op.value = i->dest; + dest.Load(op); + return true; + } +}; +template <> +struct DestField { + protected: + bool LoadDest(const Instr* i) { return true; } +}; + +template +struct I; +template +struct I : DestField { + typedef DestField BASE; + static const hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + const Instr* instr; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i) { + if (InstrKey(i).value == key && BASE::LoadDest(i)) { + instr = i; + return true; + } + return false; + } +}; +template +struct I : DestField { + typedef DestField BASE; + static const hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + static const KeyType src1_type = SRC1::key_type; + const Instr* instr; + SRC1 src1 = {}; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i) { + if (InstrKey(i).value == key && BASE::LoadDest(i)) { + instr = i; + src1.Load(i->src1); + return true; + } + return false; + } +}; +template +struct I : DestField { + typedef DestField BASE; + static const hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + static const KeyType src1_type = SRC1::key_type; + static const KeyType src2_type = SRC2::key_type; + const Instr* instr; + SRC1 src1; + SRC2 src2; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i) { + if (InstrKey(i).value == key && BASE::LoadDest(i)) { + instr = i; + src1.Load(i->src1); + src2.Load(i->src2); + return true; + } + return false; + } +}; +template +struct I : DestField { + typedef DestField BASE; + static const hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + static const KeyType src1_type = SRC1::key_type; + static const KeyType src2_type = SRC2::key_type; + static const KeyType src3_type = SRC3::key_type; + const Instr* instr; + SRC1 src1; + SRC2 src2; + SRC3 src3; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i) { + if (InstrKey(i).value == key && BASE::LoadDest(i)) { + instr = i; + src1.Load(i->src1); + src2.Load(i->src2); + src3.Load(i->src3); + return true; + } + return false; + } +}; + +template +static const T GetTempReg(A64Emitter& e); +template <> +const WReg GetTempReg(A64Emitter& e) { + return W0; +} +template <> +const XReg GetTempReg(A64Emitter& e) { + return X0; +} + +template +struct Sequence { + typedef T EmitArgType; + + static constexpr uint32_t head_key() { return T::key; } + + static bool Select(A64Emitter& e, const Instr* i) { + T args; + if (!args.Load(i)) { + return false; + } + SEQ::Emit(e, args); + return true; + } + + template + static void EmitUnaryOp(A64Emitter& e, const EmitArgType& i, + const REG_FN& reg_fn) { + if (i.src1.is_constant) { + e.MOV(i.dest, i.src1.constant()); + reg_fn(e, i.dest); + } else { + if (i.dest != i.src1) { + e.MOV(i.dest, i.src1); + } + reg_fn(e, i.dest); + } + } + + template + static void EmitCommutativeBinaryOp(A64Emitter& e, const EmitArgType& i, + const REG_REG_FN& reg_reg_fn, + const REG_CONST_FN& reg_const_fn) { + if (i.src1.is_constant) { + if (i.src2.is_constant) { + // Both constants. + if (i.src1.ConstantFitsIn32Reg()) { + e.MOV(i.dest, i.src2.constant()); + reg_const_fn(e, i.dest, static_cast(i.src1.constant())); + } else if (i.src2.ConstantFitsIn32Reg()) { + e.MOV(i.dest, i.src1.constant()); + reg_const_fn(e, i.dest, static_cast(i.src2.constant())); + } else { + e.MOV(i.dest, i.src1.constant()); + auto temp = GetTempReg(e); + e.MOV(temp, i.src2.constant()); + reg_reg_fn(e, i.dest, temp); + } + } else { + // src1 constant. + if (i.dest == i.src2) { + if (i.src1.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.dest, static_cast(i.src1.constant())); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src1.constant()); + reg_reg_fn(e, i.dest, temp); + } + } else { + e.MOV(i.dest, i.src1.constant()); + reg_reg_fn(e, i.dest, i.src2); + } + } + } else if (i.src2.is_constant) { + if (i.dest == i.src1) { + if (i.src2.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.dest, static_cast(i.src2.constant())); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2.constant()); + reg_reg_fn(e, i.dest, temp); + } + } else { + e.MOV(i.dest, i.src2.constant()); + reg_reg_fn(e, i.dest, i.src1); + } + } else { + if (i.dest == i.src1) { + reg_reg_fn(e, i.dest, i.src2); + } else if (i.dest == i.src2) { + reg_reg_fn(e, i.dest, i.src1); + } else { + e.MOV(i.dest, i.src1); + reg_reg_fn(e, i.dest, i.src2); + } + } + } + template + static void EmitAssociativeBinaryOp(A64Emitter& e, const EmitArgType& i, + const REG_REG_FN& reg_reg_fn, + const REG_CONST_FN& reg_const_fn) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + if (i.dest == i.src2) { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2); + e.MOV(i.dest, i.src1.constant()); + reg_reg_fn(e, i.dest, temp); + } else { + e.MOV(i.dest, i.src1.constant()); + reg_reg_fn(e, i.dest, i.src2); + } + } else if (i.src2.is_constant) { + if (i.dest == i.src1) { + if (i.src2.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.dest, static_cast(i.src2.constant())); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2.constant()); + reg_reg_fn(e, i.dest, temp); + } + } else { + e.MOV(i.dest, i.src1); + if (i.src2.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.dest, static_cast(i.src2.constant())); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2.constant()); + reg_reg_fn(e, i.dest, temp); + } + } + } else { + if (i.dest == i.src1) { + reg_reg_fn(e, i.dest, i.src2); + } else if (i.dest == i.src2) { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2); + e.MOV(i.dest, i.src1); + reg_reg_fn(e, i.dest, temp); + } else { + e.MOV(i.dest, i.src1); + reg_reg_fn(e, i.dest, i.src2); + } + } + } + + template + static void EmitCommutativeBinaryVOp(A64Emitter& e, const EmitArgType& i, + const FN& fn) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.LoadConstantV(Q0, i.src1.constant()); + fn(e, i.dest, REG(0), i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.LoadConstantV(Q0, i.src2.constant()); + fn(e, i.dest, i.src1, REG(0)); + } else { + fn(e, i.dest, i.src1, i.src2); + } + } + + template + static void EmitAssociativeBinaryVOp(A64Emitter& e, const EmitArgType& i, + const FN& fn) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.LoadConstantV(Q0, i.src1.constant()); + fn(e, i.dest, REG(0), i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.LoadConstantV(Q0, i.src2.constant()); + fn(e, i.dest, i.src1, REG(0)); + } else { + fn(e, i.dest, i.src1, i.src2); + } + } + + template + static void EmitCommutativeCompareOp(A64Emitter& e, const EmitArgType& i, + const REG_REG_FN& reg_reg_fn, + const REG_CONST_FN& reg_const_fn) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + if (i.src1.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.src2, static_cast(i.src1.constant())); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src1.constant()); + reg_reg_fn(e, i.src2, temp); + } + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + if (i.src2.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.src1, static_cast(i.src2.constant())); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2.constant()); + reg_reg_fn(e, i.src1, temp); + } + } else { + reg_reg_fn(e, i.src1, i.src2); + } + } + template + static void EmitAssociativeCompareOp(A64Emitter& e, const EmitArgType& i, + const REG_REG_FN& reg_reg_fn, + const REG_CONST_FN& reg_const_fn) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + if (i.src1.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.dest, i.src2, static_cast(i.src1.constant()), + true); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src1.constant()); + reg_reg_fn(e, i.dest, i.src2, temp, true); + } + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + if (i.src2.ConstantFitsIn32Reg()) { + reg_const_fn(e, i.dest, i.src1, static_cast(i.src2.constant()), + false); + } else { + auto temp = GetTempReg(e); + e.MOV(temp, i.src2.constant()); + reg_reg_fn(e, i.dest, i.src1, temp, false); + } + } else { + reg_reg_fn(e, i.dest, i.src1, i.src2, false); + } + } +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_OP_H_ diff --git a/a64_seq_control.cc b/a64_seq_control.cc new file mode 100644 index 000000000..082e6b3bf --- /dev/null +++ b/a64_seq_control.cc @@ -0,0 +1,553 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Xenia Developers. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include +#include + +#include "xenia/cpu/backend/a64/a64_op.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +volatile int anchor_control = 0; + +// ============================================================================ +// OPCODE_DEBUG_BREAK +// ============================================================================ +struct DEBUG_BREAK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { e.DebugBreak(); } +}; +EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK, DEBUG_BREAK); + +// ============================================================================ +// OPCODE_DEBUG_BREAK_TRUE +// ============================================================================ +struct DEBUG_BREAK_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.DebugBreak(); + // e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.DebugBreak(); + // e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.DebugBreak(); + // e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.DebugBreak(); + // e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.DebugBreak(); + // e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.DebugBreak(); + // e.L(skip); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK_TRUE, DEBUG_BREAK_TRUE_I8, + DEBUG_BREAK_TRUE_I16, DEBUG_BREAK_TRUE_I32, + DEBUG_BREAK_TRUE_I64, DEBUG_BREAK_TRUE_F32, + DEBUG_BREAK_TRUE_F64); + +// ============================================================================ +// OPCODE_TRAP +// ============================================================================ +struct TRAP : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.Trap(i.instr->flags); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TRAP, TRAP); + +// ============================================================================ +// OPCODE_TRAP_TRUE +// ============================================================================ +struct TRAP_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Trap(i.instr->flags); + // e.L(skip); + } +}; +struct TRAP_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Trap(i.instr->flags); + // e.L(skip); + } +}; +struct TRAP_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Trap(i.instr->flags); + // e.L(skip); + } +}; +struct TRAP_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Trap(i.instr->flags); + // e.L(skip); + } +}; +struct TRAP_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Trap(i.instr->flags); + // e.L(skip); + } +}; +struct TRAP_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Trap(i.instr->flags); + // e.L(skip); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TRAP_TRUE, TRAP_TRUE_I8, TRAP_TRUE_I16, + TRAP_TRUE_I32, TRAP_TRUE_I64, TRAP_TRUE_F32, + TRAP_TRUE_F64); + +// ============================================================================ +// OPCODE_CALL +// ============================================================================ +struct CALL : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src1.value->is_guest()); + e.Call(i.instr, static_cast(i.src1.value)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL, CALL); + +// ============================================================================ +// OPCODE_CALL_TRUE +// ============================================================================ +struct CALL_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Call(i.instr, static_cast(i.src2.value)); + // e.L(skip); + } +}; +struct CALL_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Call(i.instr, static_cast(i.src2.value)); + // e.L(skip); + } +}; +struct CALL_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Call(i.instr, static_cast(i.src2.value)); + // e.L(skip); + } +}; +struct CALL_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Call(i.instr, static_cast(i.src2.value)); + // e.L(skip); + } +}; +struct CALL_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Call(i.instr, static_cast(i.src2.value)); + // e.L(skip); + } +}; +struct CALL_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip); + // e.Call(i.instr, static_cast(i.src2.value)); + // e.L(skip); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_TRUE, CALL_TRUE_I8, CALL_TRUE_I16, + CALL_TRUE_I32, CALL_TRUE_I64, CALL_TRUE_F32, + CALL_TRUE_F64); + +// ============================================================================ +// OPCODE_CALL_INDIRECT +// ============================================================================ +struct CALL_INDIRECT + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CallIndirect(i.instr, i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT, CALL_INDIRECT); + +// ============================================================================ +// OPCODE_CALL_INDIRECT_TRUE +// ============================================================================ +struct CALL_INDIRECT_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip, CodeGenerator::T_NEAR); + // e.CallIndirect(i.instr, i.src2); + // e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip, CodeGenerator::T_NEAR); + // e.CallIndirect(i.instr, i.src2); + // e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip, CodeGenerator::T_NEAR); + // e.CallIndirect(i.instr, i.src2); + // e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip, CodeGenerator::T_NEAR); + // e.CallIndirect(i.instr, i.src2); + // e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip, CodeGenerator::T_NEAR); + // e.CallIndirect(i.instr, i.src2); + // e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // Xbyak::Label skip; + // e.jz(skip, CodeGenerator::T_NEAR); + // e.CallIndirect(i.instr, i.src2); + // e.L(skip); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT_TRUE, CALL_INDIRECT_TRUE_I8, + CALL_INDIRECT_TRUE_I16, CALL_INDIRECT_TRUE_I32, + CALL_INDIRECT_TRUE_I64, CALL_INDIRECT_TRUE_F32, + CALL_INDIRECT_TRUE_F64); + +// ============================================================================ +// OPCODE_CALL_EXTERN +// ============================================================================ +struct CALL_EXTERN + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CallExtern(i.instr, i.src1.value); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_EXTERN, CALL_EXTERN); + +// ============================================================================ +// OPCODE_RETURN +// ============================================================================ +struct RETURN : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // If this is the last instruction in the last block, just let us + // fall through. + if (i.instr->next || i.instr->block->next) { + // e.jmp(e.epilog_label(), CodeGenerator::T_NEAR); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RETURN, RETURN); + +// ============================================================================ +// OPCODE_RETURN_TRUE +// ============================================================================ +struct RETURN_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + } +}; +struct RETURN_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + } +}; +struct RETURN_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + } +}; +struct RETURN_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + } +}; +struct RETURN_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + } +}; +struct RETURN_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RETURN_TRUE, RETURN_TRUE_I8, RETURN_TRUE_I16, + RETURN_TRUE_I32, RETURN_TRUE_I64, RETURN_TRUE_F32, + RETURN_TRUE_F64); + +// ============================================================================ +// OPCODE_SET_RETURN_ADDRESS +// ============================================================================ +struct SET_RETURN_ADDRESS + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SetReturnAddress(i.src1.constant()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SET_RETURN_ADDRESS, SET_RETURN_ADDRESS); + +// ============================================================================ +// OPCODE_BRANCH +// ============================================================================ +struct BRANCH : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.jmp(i.src1.value->name, e.T_NEAR); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH); + +// ============================================================================ +// OPCODE_BRANCH_TRUE +// ============================================================================ +struct BRANCH_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jnz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // e.jnz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // e.jnz(i.src2.value->name, e.T_NEAR); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16, + BRANCH_TRUE_I32, BRANCH_TRUE_I64, BRANCH_TRUE_F32, + BRANCH_TRUE_F64); + +// ============================================================================ +// OPCODE_BRANCH_FALSE +// ============================================================================ +struct BRANCH_FALSE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_FALSE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_FALSE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_FALSE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.test(i.src1, i.src1); + // e.jz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_FALSE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // e.jz(i.src2.value->name, e.T_NEAR); + } +}; +struct BRANCH_FALSE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vptest(i.src1, i.src1); + // e.jz(i.src2.value->name, e.T_NEAR); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BRANCH_FALSE, BRANCH_FALSE_I8, BRANCH_FALSE_I16, + BRANCH_FALSE_I32, BRANCH_FALSE_I64, BRANCH_FALSE_F32, + BRANCH_FALSE_F64); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe \ No newline at end of file diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc new file mode 100644 index 000000000..54ac1177a --- /dev/null +++ b/a64_seq_memory.cc @@ -0,0 +1,1155 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Xenia Developers. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include +#include + +#include "xenia/base/memory.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_tracers.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +volatile int anchor_memory = 0; + +template +XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, + XReg address_register = X0) { + assert_true(offset.is_constant); + int32_t offset_const = static_cast(offset.constant()); + + if (guest.is_constant) { + uint32_t address = static_cast(guest.constant()); + address += offset_const; + if (address < 0x80000000) { + e.ADD(address_register, e.GetMembaseReg(), address); + return address_register; + } else { + if (address >= 0xE0000000 && + xe::memory::allocation_granularity() > 0x1000) { + e.MOV(address_register, address + 0x1000); + } else { + e.MOV(address_register, address); + } + e.ADD(address_register, e.GetMembaseReg(), address_register); + return address_register; + } + } else { + if (xe::memory::allocation_granularity() > 0x1000) { + // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do + // it via memory mapping. + e.CMP(guest.reg().toW(), 0xE0000000 - offset_const); + e.CSET(X0, Cond::HS); + e.LSL(X0, X0, 12); + e.ADD(X0, X0, guest.reg()); + e.MOV(W0, W0); + } else { + // Clear the top 32 bits, as they are likely garbage. + // TODO(benvanik): find a way to avoid doing this. + e.MOV(W0, guest.reg().toW()); + } + e.ADD(address_register, e.GetMembaseReg(), X0); + e.ADD(address_register, address_register, offset_const); + return address_register; + // return e.GetMembaseReg() + e.rax + offset_const; + } +} + +// Note: most *should* be aligned, but needs to be checked! +template +XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, + XReg address_register = X0) { + if (guest.is_constant) { + // TODO(benvanik): figure out how to do this without a temp. + // Since the constant is often 0x8... if we tried to use that as a + // displacement it would be sign extended and mess things up. + uint32_t address = static_cast(guest.constant()); + if (address < 0x80000000) { + e.ADD(address_register, e.GetMembaseReg(), address); + return address_register; + } else { + if (address >= 0xE0000000 && + xe::memory::allocation_granularity() > 0x1000) { + e.MOV(address_register, address + 0x1000); + } else { + e.MOV(address_register, address); + } + e.ADD(address_register, e.GetMembaseReg(), address_register); + return address_register; + } + } else { + if (xe::memory::allocation_granularity() > 0x1000) { + // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do + // it via memory mapping. + e.CMP(guest.reg().toW(), 0xE0000000); + e.CSET(X0, Cond::HS); + e.LSL(X0, X0, 12); + e.ADD(X0, X0, guest); + e.MOV(W0, W0); + } else { + // Clear the top 32 bits, as they are likely garbage. + // TODO(benvanik): find a way to avoid doing this. + e.MOV(W0, guest.reg().toW()); + } + e.ADD(address_register, e.GetMembaseReg(), X0); + return address_register; + // return e.GetMembaseReg() + e.rax; + } +} + +// ============================================================================ +// OPCODE_ATOMIC_EXCHANGE +// ============================================================================ +// Note that the address we use here is a real, host address! +// This is weird, and should be fixed. +template +void EmitAtomicExchangeXX(A64Emitter& e, const ARGS& i) { + if (i.dest == i.src1) { + // e.mov(e.rax, i.src1); + if (i.dest != i.src2) { + if (i.src2.is_constant) { + // e.mov(i.dest, i.src2.constant()); + } else { + // e.mov(i.dest, i.src2); + } + } + // e.lock(); + // e.xchg(e.dword[e.rax], i.dest); + } else { + if (i.dest != i.src2) { + if (i.src2.is_constant) { + // e.mov(i.dest, i.src2.constant()); + } else { + // e.mov(i.dest, i.src2); + } + } + // e.lock(); + // e.xchg(e.dword[i.src1.reg()], i.dest); + } +} +struct ATOMIC_EXCHANGE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAtomicExchangeXX(e, i); + } +}; +struct ATOMIC_EXCHANGE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAtomicExchangeXX(e, i); + } +}; +struct ATOMIC_EXCHANGE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAtomicExchangeXX(e, i); + } +}; +struct ATOMIC_EXCHANGE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAtomicExchangeXX(e, i); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_EXCHANGE, ATOMIC_EXCHANGE_I8, + ATOMIC_EXCHANGE_I16, ATOMIC_EXCHANGE_I32, + ATOMIC_EXCHANGE_I64); + +// ============================================================================ +// OPCODE_ATOMIC_COMPARE_EXCHANGE +// ============================================================================ +struct ATOMIC_COMPARE_EXCHANGE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mov(e.eax, i.src2); + if (xe::memory::allocation_granularity() > 0x1000) { + // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do + // it via memory mapping. + // e.cmp(i.src1.reg().cvt32(), 0xE0000000); + // e.setae(e.cl); + // e.movzx(e.ecx, e.cl); + // e.shl(e.ecx, 12); + // e.add(e.ecx, i.src1.reg().cvt32()); + } else { + // e.mov(e.ecx, i.src1.reg().cvt32()); + } + // e.lock(); + // e.cmpxchg(e.dword[e.GetMembaseReg() + e.rcx], i.src3); + // e.sete(i.dest); + } +}; +struct ATOMIC_COMPARE_EXCHANGE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mov(e.rax, i.src2); + if (xe::memory::allocation_granularity() > 0x1000) { + // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do + // it via memory mapping. + // e.cmp(i.src1.reg().cvt32(), 0xE0000000); + // e.setae(e.cl); + // e.movzx(e.ecx, e.cl); + // e.shl(e.ecx, 12); + // e.add(e.ecx, i.src1.reg().cvt32()); + } else { + // e.mov(e.ecx, i.src1.reg().cvt32()); + } + // e.lock(); + // e.cmpxchg(e.qword[e.GetMembaseReg() + e.rcx], i.src3); + // e.sete(i.dest); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, + ATOMIC_COMPARE_EXCHANGE_I32, ATOMIC_COMPARE_EXCHANGE_I64); + +// ============================================================================ +// OPCODE_LOAD_LOCAL +// ============================================================================ +// Note: all types are always aligned on the stack. +struct LOAD_LOCAL_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mov(i.dest, e.byte[e.rsp + i.src1.constant()]); + // e.TraceLoadI8(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +struct LOAD_LOCAL_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mov(i.dest, e.word[e.rsp + i.src1.constant()]); + // e.TraceLoadI16(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +struct LOAD_LOCAL_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mov(i.dest, e.dword[e.rsp + i.src1.constant()]); + // e.TraceLoadI32(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +struct LOAD_LOCAL_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mov(i.dest, e.qword[e.rsp + i.src1.constant()]); + // e.TraceLoadI64(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +struct LOAD_LOCAL_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vmovss(i.dest, e.dword[e.rsp + i.src1.constant()]); + // e.TraceLoadF32(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +struct LOAD_LOCAL_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vmovsd(i.dest, e.qword[e.rsp + i.src1.constant()]); + // e.TraceLoadF64(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +struct LOAD_LOCAL_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vmovaps(i.dest, e.ptr[e.rsp + i.src1.constant()]); + // e.TraceLoadV128(DATA_LOCAL, i.src1.constant, i.dest); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_LOCAL, LOAD_LOCAL_I8, LOAD_LOCAL_I16, + LOAD_LOCAL_I32, LOAD_LOCAL_I64, LOAD_LOCAL_F32, + LOAD_LOCAL_F64, LOAD_LOCAL_V128); + +// ============================================================================ +// OPCODE_STORE_LOCAL +// ============================================================================ +// Note: all types are always aligned on the stack. +struct STORE_LOCAL_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreI8(DATA_LOCAL, i.src1.constant, i.src2); + // e.mov(e.byte[e.rsp + i.src1.constant()], i.src2); + } +}; +struct STORE_LOCAL_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreI16(DATA_LOCAL, i.src1.constant, i.src2); + // e.mov(e.word[e.rsp + i.src1.constant()], i.src2); + } +}; +struct STORE_LOCAL_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreI32(DATA_LOCAL, i.src1.constant, i.src2); + // e.mov(e.dword[e.rsp + i.src1.constant()], i.src2); + } +}; +struct STORE_LOCAL_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreI64(DATA_LOCAL, i.src1.constant, i.src2); + // e.mov(e.qword[e.rsp + i.src1.constant()], i.src2); + } +}; +struct STORE_LOCAL_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreF32(DATA_LOCAL, i.src1.constant, i.src2); + // e.vmovss(e.dword[e.rsp + i.src1.constant()], i.src2); + } +}; +struct STORE_LOCAL_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreF64(DATA_LOCAL, i.src1.constant, i.src2); + // e.vmovsd(e.qword[e.rsp + i.src1.constant()], i.src2); + } +}; +struct STORE_LOCAL_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.TraceStoreV128(DATA_LOCAL, i.src1.constant, i.src2); + // e.vmovaps(e.ptr[e.rsp + i.src1.constant()], i.src2); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_LOCAL, STORE_LOCAL_I8, STORE_LOCAL_I16, + STORE_LOCAL_I32, STORE_LOCAL_I64, STORE_LOCAL_F32, + STORE_LOCAL_F64, STORE_LOCAL_V128); + +// ============================================================================ +// OPCODE_LOAD_CONTEXT +// ============================================================================ +struct LOAD_CONTEXT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDRB(i.dest, e.GetContextReg(), i.src1.value); + // e.mov(i.dest, e.byte[addr]); + if (IsTracingData()) { + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.mov(e.GetNativeParam(1), e.byte[addr]); + // e.CallNative(reinterpret_cast(TraceContextLoadI8)); + } + } +}; +struct LOAD_CONTEXT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDRH(i.dest, e.GetContextReg(), i.src1.value); + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.word[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextLoadI16)); + } + } +}; +struct LOAD_CONTEXT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDR(i.dest, e.GetContextReg(), i.src1.value); + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.dword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextLoadI32)); + } + } +}; +struct LOAD_CONTEXT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDR(i.dest, e.GetContextReg(), i.src1.value); + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.qword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextLoadI64)); + } + } +}; +struct LOAD_CONTEXT_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ADD(X0, e.GetContextReg(), i.src1.value); + e.LD1(List{i.dest.reg().toQ().Selem()[0]}, X0); + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.dword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextLoadF32)); + } + } +}; +struct LOAD_CONTEXT_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ADD(X0, e.GetContextReg(), i.src1.value); + e.LD1(List{i.dest.reg().toQ().Delem()[0]}, X0); + // e.vmovsd(i.dest, e.qword[addr]); + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.qword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextLoadF64)); + } + } +}; +struct LOAD_CONTEXT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDR(i.dest, e.GetContextReg(), i.src1.value); + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.ptr[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextLoadV128)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_CONTEXT, LOAD_CONTEXT_I8, LOAD_CONTEXT_I16, + LOAD_CONTEXT_I32, LOAD_CONTEXT_I64, LOAD_CONTEXT_F32, + LOAD_CONTEXT_F64, LOAD_CONTEXT_V128); + +// ============================================================================ +// OPCODE_STORE_CONTEXT +// ============================================================================ +// Note: all types are always aligned on the stack. +struct STORE_CONTEXT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.constant()); + e.STRB(W0, e.GetContextReg(), i.src1.value); + } else { + e.STRB(i.src2.reg(), e.GetContextReg(), i.src1.value); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.byte[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreI8)); + } + } +}; +struct STORE_CONTEXT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.constant()); + e.STRH(W0, e.GetContextReg(), i.src1.value); + } else { + e.STRH(i.src2.reg(), e.GetContextReg(), i.src1.value); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.word[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreI16)); + } + } +}; +struct STORE_CONTEXT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.constant()); + e.STR(W0, e.GetContextReg(), i.src1.value); + } else { + e.STR(i.src2.reg(), e.GetContextReg(), i.src1.value); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.dword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreI32)); + } + } +}; +struct STORE_CONTEXT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.MOV(X0, i.src2.constant()); + e.STR(X0, e.GetContextReg(), i.src1.value); + } else { + e.STR(i.src2.reg(), e.GetContextReg(), i.src1.value); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), e.qword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreI64)); + } + } +}; +struct STORE_CONTEXT_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.value->constant.i32); + e.STR(W0, e.GetContextReg(), i.src1.value); + } else { + e.ADD(X0, e.GetContextReg(), i.src1.value); + e.ST1(List{i.src2.reg().toQ().Selem()[0]}, X0); + } + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.dword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreF32)); + } + } +}; +struct STORE_CONTEXT_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + // e.MovMem64(addr, i.src2.value->constant.i64); + } else { + // e.vmovsd(e.qword[addr], i.src2); + } + + if (i.src2.is_constant) { + e.MOV(X0, i.src2.value->constant.i64); + e.STR(X0, e.GetContextReg(), i.src1.value); + } else { + e.ADD(X0, e.GetContextReg(), i.src1.value); + e.ST1(List{i.src2.reg().toQ().Delem()[0]}, X0); + } + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.qword[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreF64)); + } + } +}; +struct STORE_CONTEXT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.LoadConstantV(Q0, i.src2.constant()); + e.STR(Q0, e.GetContextReg(), i.src1.value); + } else { + e.STR(i.src2, e.GetContextReg(), i.src1.value); + } + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.ptr[addr]); + // e.mov(e.GetNativeParam(0), i.src1.value); + // e.CallNative(reinterpret_cast(TraceContextStoreV128)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_CONTEXT, STORE_CONTEXT_I8, STORE_CONTEXT_I16, + STORE_CONTEXT_I32, STORE_CONTEXT_I64, STORE_CONTEXT_F32, + STORE_CONTEXT_F64, STORE_CONTEXT_V128); + +// ============================================================================ +// OPCODE_LOAD_MMIO +// ============================================================================ +// Note: all types are always aligned in the context. +struct LOAD_MMIO_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // uint64_t (context, addr) + auto mmio_range = reinterpret_cast(i.src1.value); + auto read_address = uint32_t(i.src2.value); + // e.mov(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); + // e.mov(e.GetNativeParam(1).cvt32(), read_address); + // e.CallNativeSafe(reinterpret_cast(mmio_range->read)); + // e.bswap(e.eax); + // e.mov(i.dest, e.eax); + if (IsTracingData()) { + // e.mov(e.GetNativeParam(0), i.dest); + // e.mov(e.edx, read_address); + // e.CallNative(reinterpret_cast(TraceContextLoadI32)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_MMIO, LOAD_MMIO_I32); + +// ============================================================================ +// OPCODE_STORE_MMIO +// ============================================================================ +// Note: all types are always aligned on the stack. +struct STORE_MMIO_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // void (context, addr, value) + auto mmio_range = reinterpret_cast(i.src1.value); + auto write_address = uint32_t(i.src2.value); + // e.mov(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); + // e.mov(e.GetNativeParam(1).cvt32(), write_address); + if (i.src3.is_constant) { + // e.mov(e.GetNativeParam(2).cvt32(), xe::byte_swap(i.src3.constant())); + } else { + // e.mov(e.GetNativeParam(2).cvt32(), i.src3); + // e.bswap(e.GetNativeParam(2).cvt32()); + } + // e.CallNativeSafe(reinterpret_cast(mmio_range->write)); + if (IsTracingData()) { + if (i.src3.is_constant) { + // e.mov(e.GetNativeParam(0).cvt32(), i.src3.constant()); + } else { + // e.mov(e.GetNativeParam(0).cvt32(), i.src3); + } + // e.mov(e.edx, write_address); + // e.CallNative(reinterpret_cast(TraceContextStoreI32)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_MMIO, STORE_MMIO_I32); + +// ============================================================================ +// OPCODE_LOAD_OFFSET +// ============================================================================ +struct LOAD_OFFSET_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + e.LDRB(i.dest, addr_reg); + } +}; + +struct LOAD_OFFSET_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.LDRH(i.dest, addr_reg); + e.REV16(i.dest, i.dest); + } else { + e.LDRH(i.dest, addr_reg); + } + } +}; + +struct LOAD_OFFSET_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.LDR(i.dest, addr_reg); + e.REV32(i.dest.reg().toX(), i.dest.reg().toX()); + } else { + e.LDR(i.dest, addr_reg); + } + } +}; + +struct LOAD_OFFSET_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.LDR(i.dest, addr_reg); + e.REV64(i.dest, i.dest); + } else { + e.LDR(i.dest, addr_reg); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_OFFSET, LOAD_OFFSET_I8, LOAD_OFFSET_I16, + LOAD_OFFSET_I32, LOAD_OFFSET_I64); + +// ============================================================================ +// OPCODE_STORE_OFFSET +// ============================================================================ +struct STORE_OFFSET_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.src3.is_constant) { + e.MOV(W0, i.src3.constant()); + e.STRB(W0, addr_reg); + } else { + e.STRB(i.src3, addr_reg); + } + } +}; + +struct STORE_OFFSET_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src3.is_constant); + assert_always("not implemented"); + } else { + if (i.src3.is_constant) { + e.MOV(W0, i.src3.constant()); + e.STRH(W0, addr_reg); + } else { + e.STRH(i.src3, addr_reg); + } + } + } +}; + +struct STORE_OFFSET_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src3.is_constant); + assert_always("not implemented"); + } else { + if (i.src3.is_constant) { + e.MOV(W0, i.src3.constant()); + e.STRH(W0, addr_reg); + } else { + e.STRH(i.src3, addr_reg); + } + } + } +}; + +struct STORE_OFFSET_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src3.is_constant); + assert_always("not implemented"); + } else { + if (i.src3.is_constant) { + e.MovMem64(addr_reg, 0, i.src3.constant()); + } else { + e.STR(i.src3, addr_reg); + } + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_OFFSET, STORE_OFFSET_I8, STORE_OFFSET_I16, + STORE_OFFSET_I32, STORE_OFFSET_I64); + +// ============================================================================ +// OPCODE_LOAD +// ============================================================================ +struct LOAD_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + e.LDRB(i.dest, addr_reg); + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1).cvt8(), i.dest); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadI8)); + } + } +}; +struct LOAD_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.LDRH(i.dest, addr_reg); + e.REV16(i.dest, i.dest); + } else { + e.LDRH(i.dest, addr_reg); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1).cvt16(), i.dest); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadI16)); + } + } +}; +struct LOAD_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.LDR(i.dest, addr_reg); + e.REV32(i.dest.reg().toX(), i.dest.reg().toX()); + } else { + e.LDR(i.dest, addr_reg); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1).cvt32(), i.dest); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadI32)); + } + } +}; +struct LOAD_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.LDR(i.dest, addr_reg); + e.REV64(i.dest, i.dest); + } else { + e.LDR(i.dest, addr_reg); + } + if (IsTracingData()) { + // e.mov(e.GetNativeParam(1), i.dest); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadI64)); + } + } +}; +struct LOAD_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + // e.vmovss(i.dest, e.dword[addr]); + e.LD1(List{i.dest.reg().toQ().Selem()[0]}, addr_reg); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_always("not implemented yet"); + } + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.dword[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadF32)); + } + } +}; +struct LOAD_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + // e.vmovsd(i.dest, e.qword[addr]); + e.LD1(List{i.dest.reg().toQ().Delem()[0]}, addr_reg); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_always("not implemented yet"); + } + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.qword[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadF64)); + } + } +}; +struct LOAD_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + e.LDR(i.dest, addr_reg); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + // Reverse upper and lower 64-bit halfs + e.REV64(i.dest.reg().B16(), i.dest.reg().B16()); + // Reverse the 64-bit halfs themselves + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); + } + if (IsTracingData()) { + // e.lea(e.GetNativeParam(1), e.ptr[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryLoadV128)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD, LOAD_I8, LOAD_I16, LOAD_I32, LOAD_I64, + LOAD_F32, LOAD_F64, LOAD_V128); + +// ============================================================================ +// OPCODE_STORE +// ============================================================================ +// Note: most *should* be aligned, but needs to be checked! +struct STORE_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.MOV(W0, i.src2.constant()); + e.STRB(W0, addr_reg); + } else { + e.STRB(i.src2.reg(), addr_reg); + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.mov(e.GetNativeParam(1).cvt8(), e.byte[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreI8)); + } + } +}; +struct STORE_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src2.is_constant); + assert_always("not implemented"); + } else { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.constant()); + e.STRH(W0, addr_reg); + } else { + e.STRH(i.src2.reg(), addr_reg); + } + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.mov(e.GetNativeParam(1).cvt16(), e.word[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreI16)); + } + } +}; +struct STORE_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src2.is_constant); + assert_always("not implemented"); + } else { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.constant()); + e.STR(W0, addr_reg); + } else { + e.STR(i.src2.reg(), addr_reg); + } + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.mov(e.GetNativeParam(1).cvt32(), e.dword[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreI32)); + } + } +}; +struct STORE_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src2.is_constant); + assert_always("not implemented"); + } else { + if (i.src2.is_constant) { + e.MovMem64(addr_reg, 0, i.src2.constant()); + } else { + e.STR(i.src2.reg(), addr_reg); + } + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.mov(e.GetNativeParam(1), e.qword[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreI64)); + } + } +}; +struct STORE_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src2.is_constant); + assert_always("not yet implemented"); + } else { + if (i.src2.is_constant) { + e.MOV(W0, i.src2.value->constant.i32); + e.STR(W0, addr_reg); + } else { + e.ST1(List{i.src2.reg().toQ().Selem()[0]}, addr_reg); + } + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.lea(e.GetNativeParam(1), e.ptr[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreF32)); + } + } +}; +struct STORE_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src2.is_constant); + assert_always("not yet implemented"); + } else { + if (i.src2.is_constant) { + e.MOV(X0, i.src2.value->constant.i64); + e.STR(X0, addr_reg); + } else { + e.ST1(List{i.src2.reg().toQ().Delem()[0]}, addr_reg); + } + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.lea(e.GetNativeParam(1), e.ptr[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreF64)); + } + } +}; +struct STORE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + assert_false(i.src2.is_constant); + // Reverse upper and lower 64-bit halfs + e.REV64(Q0.B16(), i.src2.reg().B16()); + // Reverse the 64-bit halfs themselves + e.EXT(Q0.B16(), Q0.B16(), Q0.B16(), 8); + e.STR(Q0, addr_reg); + } else { + if (i.src2.is_constant) { + e.LoadConstantV(Q0, i.src2.constant()); + e.STR(Q0, addr_reg); + } else { + e.STR(i.src2, addr_reg); + } + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.lea(e.GetNativeParam(1), e.ptr[addr]); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemoryStoreV128)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE, STORE_I8, STORE_I16, STORE_I32, STORE_I64, + STORE_F32, STORE_F64, STORE_V128); + +// ============================================================================ +// OPCODE_CACHE_CONTROL +// ============================================================================ +struct CACHE_CONTROL + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + bool is_clflush = false, is_prefetch = false; + switch (CacheControlType(i.instr->flags)) { + case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH: + case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH_FOR_STORE: + is_prefetch = true; + break; + case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE: + case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE_AND_FLUSH: + is_clflush = true; + break; + default: + assert_unhandled_case(CacheControlType(i.instr->flags)); + return; + } + size_t cache_line_size = i.src2.value; + + // RegExp addr; + // uint32_t address_constant; + // if (i.src1.is_constant) { + // // TODO(benvanik): figure out how to do this without a temp. + // // Since the constant is often 0x8... if we tried to use that as a + // // displacement it would be sign extended and mess things up. + // address_constant = static_cast(i.src1.constant()); + // if (address_constant < 0x80000000) { + // addr = e.GetMembaseReg() + address_constant; + // } else { + // if (address_constant >= 0xE0000000 && + // xe::memory::allocation_granularity() > 0x1000) { + // // e.mov(e.eax, address_constant + 0x1000); + // } else { + // // e.mov(e.eax, address_constant); + // } + // addr = e.GetMembaseReg() + e.rax; + // } + // } else { + // if (xe::memory::allocation_granularity() > 0x1000) { + // // Emulate the 4 KB physical address offset in 0xE0000000+ when can't + // do + // // it via memory mapping. + // // e.cmp(i.src1.reg().cvt32(), 0xE0000000); + // // e.setae(e.al); + // // e.movzx(e.eax, e.al); + // // e.shl(e.eax, 12); + // // e.add(e.eax, i.src1.reg().cvt32()); + // } else { + // // Clear the top 32 bits, as they are likely garbage. + // // TODO(benvanik): find a way to avoid doing this. + // // e.mov(e.eax, i.src1.reg().cvt32()); + // } + // addr = e.GetMembaseReg() + e.rax; + // } + // if (is_clflush) { + // // e.clflush(e.ptr[addr]); + // } + // if (is_prefetch) { + // // e.prefetcht0(e.ptr[addr]); + // } + + // if (cache_line_size >= 128) { + // // Prefetch the other 64 bytes of the 128-byte cache line. + // if (i.src1.is_constant && address_constant < 0x80000000) { + // addr = e.GetMembaseReg() + (address_constant ^ 64); + // } else { + // // e.xor_(e.eax, 64); + // } + // if (is_clflush) { + // // e.clflush(e.ptr[addr]); + // } + // if (is_prefetch) { + // // e.prefetcht0(e.ptr[addr]); + // } + // assert_true(cache_line_size == 128); + // } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); + +// ============================================================================ +// OPCODE_MEMORY_BARRIER +// ============================================================================ +struct MEMORY_BARRIER + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.mfence(); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MEMORY_BARRIER, MEMORY_BARRIER); + +// ============================================================================ +// OPCODE_MEMSET +// ============================================================================ +struct MEMSET_I64_I8_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + assert_true(i.src3.is_constant); + assert_true(i.src2.constant() == 0); + // e.vpxor(e.xmm0, e.xmm0); + e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); + auto addr_reg = ComputeMemoryAddress(e, i.src1); + switch (i.src3.constant()) { + case 32: + // e.vmovaps(e.ptr[addr + 0 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 1 * 16], e.xmm0); + break; + case 128: + // e.vmovaps(e.ptr[addr + 0 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 1 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 2 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 3 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 4 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 5 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 6 * 16], e.xmm0); + // e.vmovaps(e.ptr[addr + 7 * 16], e.xmm0); + break; + default: + assert_unhandled_case(i.src3.constant()); + break; + } + if (IsTracingData()) { + addr_reg = ComputeMemoryAddress(e, i.src1); + // e.mov(e.GetNativeParam(2), i.src3.constant()); + // e.mov(e.GetNativeParam(1), i.src2.constant()); + // e.lea(e.GetNativeParam(0), e.ptr[addr]); + // e.CallNative(reinterpret_cast(TraceMemset)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MEMSET, MEMSET_I64_I8_I64); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc new file mode 100644 index 000000000..ff313aefb --- /dev/null +++ b/a64_seq_vector.cc @@ -0,0 +1,764 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Xenia Developers. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include +#include + +#include "xenia/cpu/backend/a64/a64_op.h" + +// For OPCODE_PACK/OPCODE_UNPACK +#include "third_party/half/include/half.hpp" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +volatile int anchor_vector = 0; + +// ============================================================================ +// OPCODE_VECTOR_CONVERT_I2F +// ============================================================================ +struct VECTOR_CONVERT_I2F + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // flags = ARITHMETIC_UNSIGNED + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + // Round manually to (1.stored mantissa bits * 2^31) or to 2^32 to the + // nearest even (the only rounding mode used on AltiVec) if the number is + // 0x80000000 or greater, instead of converting src & 0x7FFFFFFF and then + // adding 2147483648.0f, which results in double rounding that can give a + // result larger than needed - see OPCODE_VECTOR_CONVERT_I2F notes. + + // [0x80000000, 0xFFFFFFFF] case: + + // Round to the nearest even, from (0x80000000 | 31 stored mantissa bits) + // to ((-1 << 23) | 23 stored mantissa bits), or to 0 if the result should + // be 4294967296.0f. + // xmm0 = src + 0b01111111 + ((src >> 8) & 1) + // (xmm1 also used to launch reg + mem early and to require it late) + // e.vpaddd(Q1, i.src1, e.GetXmmConstPtr(XMMInt127)); + // e.vpslld(Q0, i.src1, 31 - 8); + // e.vpsrld(Q0, Q0, 31); + // e.vpaddd(Q0, Q0, Q1); + // xmm0 = (0xFF800000 | 23 explicit mantissa bits), or 0 if overflowed + // e.vpsrad(Q0, Q0, 8); + // Calculate the result for the [0x80000000, 0xFFFFFFFF] case - take the + // rounded mantissa, and add -1 or 0 to the exponent of 32, depending on + // whether the number should be (1.stored mantissa bits * 2^31) or 2^32. + // xmm0 = [0x80000000, 0xFFFFFFFF] case result + // e.vpaddd(Q0, Q0, e.GetXmmConstPtr(XMM2To32)); + + // [0x00000000, 0x7FFFFFFF] case + // (during vblendvps reg -> vpaddd reg -> vpaddd mem dependency): + + // Convert from signed integer to float. + // xmm1 = [0x00000000, 0x7FFFFFFF] case result + // e.vcvtdq2ps(Q1, i.src1); + + // Merge the two ways depending on whether the number is >= 0x80000000 + // (has high bit set). + // e.vblendvps(i.dest, Q1, Q0, i.src1); + } else { + // e.vcvtdq2ps(i.dest, i.src1); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_I2F, VECTOR_CONVERT_I2F); + +// ============================================================================ +// OPCODE_VECTOR_CONVERT_F2I +// ============================================================================ +struct VECTOR_CONVERT_F2I + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + // clamp to min 0 + // e.vmaxps(Q0, i.src1, e.GetXmmConstPtr(XMMZero)); + + // xmm1 = mask of values >= (unsigned)INT_MIN + // e.vcmpgeps(Q1, Q0, e.GetXmmConstPtr(XMMPosIntMinPS)); + + // scale any values >= (unsigned)INT_MIN back to [0, ...] + // e.vsubps(e.xmm2, Q0, e.GetXmmConstPtr(XMMPosIntMinPS)); + // e.vblendvps(Q0, Q0, e.xmm2, Q1); + + // xmm0 = [0, INT_MAX] + // this may still contain values > INT_MAX (if src has vals > UINT_MAX) + // e.vcvttps2dq(i.dest, Q0); + + // xmm0 = mask of values that need saturation + // e.vpcmpeqd(Q0, i.dest, e.GetXmmConstPtr(XMMIntMin)); + + // scale values back above [INT_MIN, UINT_MAX] + // e.vpand(Q1, Q1, e.GetXmmConstPtr(XMMIntMin)); + // e.vpaddd(i.dest, i.dest, Q1); + + // saturate values > UINT_MAX + // e.vpor(i.dest, i.dest, Q0); + } else { + // xmm2 = NaN mask + // e.vcmpunordps(e.xmm2, i.src1, i.src1); + + // convert packed floats to packed dwords + // e.vcvttps2dq(Q0, i.src1); + + // (high bit) xmm1 = dest is indeterminate and i.src1 >= 0 + // e.vpcmpeqd(Q1, Q0, e.GetXmmConstPtr(XMMIntMin)); + // e.vpandn(Q1, i.src1, Q1); + + // saturate positive values + // e.vblendvps(i.dest, Q0, e.GetXmmConstPtr(XMMIntMax), Q1); + + // mask NaNs + // e.vpandn(i.dest, e.xmm2, i.dest); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_F2I, VECTOR_CONVERT_F2I); + +// ============================================================================ +// OPCODE_LOAD_VECTOR_SHL +// ============================================================================ +static const vec128_t lvsl_table[16] = { + vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), + vec128b(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), + vec128b(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17), + vec128b(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18), + vec128b(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19), + vec128b(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20), + vec128b(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21), + vec128b(7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22), + vec128b(8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23), + vec128b(9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24), + vec128b(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25), + vec128b(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), + vec128b(12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27), + vec128b(13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28), + vec128b(14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29), + vec128b(15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30), +}; +struct LOAD_VECTOR_SHL_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + auto sh = i.src1.constant(); + assert_true(sh < xe::countof(lvsl_table)); + // e.mov(e.rax, (uintptr_t)&lvsl_table[sh]); + // e.vmovaps(i.dest, e.ptr[e.rax]); + } else { + // TODO(benvanik): find a cheaper way of doing this. + // e.movzx(e.rdx, i.src1); + // e.and_(e.dx, 0xF); + // e.shl(e.dx, 4); + // e.mov(e.rax, (uintptr_t)lvsl_table); + // e.vmovaps(i.dest, e.ptr[e.rax + e.rdx]); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHL, LOAD_VECTOR_SHL_I8); + +// ============================================================================ +// OPCODE_LOAD_VECTOR_SHR +// ============================================================================ +static const vec128_t lvsr_table[16] = { + vec128b(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31), + vec128b(15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30), + vec128b(14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29), + vec128b(13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28), + vec128b(12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27), + vec128b(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), + vec128b(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25), + vec128b(9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24), + vec128b(8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23), + vec128b(7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22), + vec128b(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21), + vec128b(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20), + vec128b(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19), + vec128b(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18), + vec128b(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17), + vec128b(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), +}; +struct LOAD_VECTOR_SHR_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + auto sh = i.src1.constant(); + assert_true(sh < xe::countof(lvsr_table)); + // e.mov(e.rax, (uintptr_t)&lvsr_table[sh]); + // e.vmovaps(i.dest, e.ptr[e.rax]); + } else { + // TODO(benvanik): find a cheaper way of doing this. + // e.movzx(e.rdx, i.src1); + // e.and_(e.dx, 0xF); + // e.shl(e.dx, 4); + // e.mov(e.rax, (uintptr_t)lvsr_table); + // e.vmovaps(i.dest, e.ptr[e.rax + e.rdx]); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHR, LOAD_VECTOR_SHR_I8); + +// ============================================================================ +// OPCODE_VECTOR_MAX +// ============================================================================ +struct VECTOR_MAX + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + uint32_t part_type = i.instr->flags >> 8; + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + switch (part_type) { + case INT8_TYPE: + // e.vpmaxub(dest, src1, src2); + break; + case INT16_TYPE: + // e.vpmaxuw(dest, src1, src2); + break; + case INT32_TYPE: + // e.vpmaxud(dest, src1, src2); + break; + default: + assert_unhandled_case(part_type); + break; + } + } else { + switch (part_type) { + case INT8_TYPE: + // e.vpmaxsb(dest, src1, src2); + break; + case INT16_TYPE: + // e.vpmaxsw(dest, src1, src2); + break; + case INT32_TYPE: + // e.vpmaxsd(dest, src1, src2); + break; + default: + assert_unhandled_case(part_type); + break; + } + } + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MAX, VECTOR_MAX); + +// ============================================================================ +// OPCODE_VECTOR_MIN +// ============================================================================ +struct VECTOR_MIN + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + uint32_t part_type = i.instr->flags >> 8; + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + switch (part_type) { + case INT8_TYPE: + // e.vpminub(dest, src1, src2); + break; + case INT16_TYPE: + // e.vpminuw(dest, src1, src2); + break; + case INT32_TYPE: + // e.vpminud(dest, src1, src2); + break; + default: + assert_unhandled_case(part_type); + break; + } + } else { + switch (part_type) { + case INT8_TYPE: + // e.vpminsb(dest, src1, src2); + break; + case INT16_TYPE: + // e.vpminsw(dest, src1, src2); + break; + case INT32_TYPE: + // e.vpminsd(dest, src1, src2); + break; + default: + assert_unhandled_case(part_type); + break; + } + } + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MIN, VECTOR_MIN); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_EQ +// ============================================================================ +struct VECTOR_COMPARE_EQ_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + switch (i.instr->flags) { + case INT8_TYPE: + // e.vpcmpeqb(dest, src1, src2); + break; + case INT16_TYPE: + // e.vpcmpeqw(dest, src1, src2); + break; + case INT32_TYPE: + // e.vpcmpeqd(dest, src1, src2); + break; + case FLOAT32_TYPE: + // e.vcmpeqps(dest, src1, src2); + break; + } + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_EQ, VECTOR_COMPARE_EQ_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_SGT +// ============================================================================ +struct VECTOR_COMPARE_SGT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAssociativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + switch (i.instr->flags) { + case INT8_TYPE: + // e.vpcmpgtb(dest, src1, src2); + break; + case INT16_TYPE: + // e.vpcmpgtw(dest, src1, src2); + break; + case INT32_TYPE: + // e.vpcmpgtd(dest, src1, src2); + break; + case FLOAT32_TYPE: + // e.vcmpgtps(dest, src1, src2); + break; + } + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGT, VECTOR_COMPARE_SGT_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_SGE +// ============================================================================ +struct VECTOR_COMPARE_SGE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAssociativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + switch (i.instr->flags) { + case INT8_TYPE: + // e.vpcmpeqb(Q0, src1, src2); + // e.vpcmpgtb(dest, src1, src2); + // e.vpor(dest, Q0); + break; + case INT16_TYPE: + // e.vpcmpeqw(Q0, src1, src2); + // e.vpcmpgtw(dest, src1, src2); + // e.vpor(dest, Q0); + break; + case INT32_TYPE: + // e.vpcmpeqd(Q0, src1, src2); + // e.vpcmpgtd(dest, src1, src2); + // e.vpor(dest, Q0); + break; + case FLOAT32_TYPE: + // e.vcmpgeps(dest, src1, src2); + break; + } + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGE, VECTOR_COMPARE_SGE_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_UGT +// ============================================================================ +struct VECTOR_COMPARE_UGT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGT, VECTOR_COMPARE_UGT_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_UGE +// ============================================================================ +struct VECTOR_COMPARE_UGE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGE, VECTOR_COMPARE_UGE_V128); + +// ============================================================================ +// OPCODE_VECTOR_ADD +// ============================================================================ +struct VECTOR_ADD + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ADD, VECTOR_ADD); + +// ============================================================================ +// OPCODE_VECTOR_SUB +// ============================================================================ +struct VECTOR_SUB + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB); + +// ============================================================================ +// OPCODE_VECTOR_SHL +// ============================================================================ + +struct VECTOR_SHL_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case INT8_TYPE: + EmitInt8(e, i); + break; + case INT16_TYPE: + EmitInt16(e, i); + break; + case INT32_TYPE: + EmitInt32(e, i); + break; + default: + assert_always(); + break; + } + } + + static void EmitInt8(A64Emitter& e, const EmitArgType& i) {} + + static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} + + static void EmitInt32(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128); + +// ============================================================================ +// OPCODE_VECTOR_SHR +// ============================================================================ + +struct VECTOR_SHR_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case INT8_TYPE: + EmitInt8(e, i); + break; + case INT16_TYPE: + EmitInt16(e, i); + break; + case INT32_TYPE: + EmitInt32(e, i); + break; + default: + assert_always(); + break; + } + } + + static void EmitInt8(A64Emitter& e, const EmitArgType& i) {} + + static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} + + static void EmitInt32(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHR, VECTOR_SHR_V128); + +// ============================================================================ +// OPCODE_VECTOR_SHA +// ============================================================================ +struct VECTOR_SHA_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case INT8_TYPE: + EmitInt8(e, i); + break; + case INT16_TYPE: + EmitInt16(e, i); + break; + case INT32_TYPE: + EmitInt32(e, i); + break; + default: + assert_always(); + break; + } + } + + static void EmitInt8(A64Emitter& e, const EmitArgType& i) {} + + static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} + + static void EmitInt32(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128); + +// ============================================================================ +// OPCODE_VECTOR_ROTATE_LEFT +// ============================================================================ +struct VECTOR_ROTATE_LEFT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128); + +// ============================================================================ +// OPCODE_VECTOR_AVERAGE +// ============================================================================ + +struct VECTOR_AVERAGE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_AVERAGE, VECTOR_AVERAGE); + +// ============================================================================ +// OPCODE_INSERT +// ============================================================================ +struct INSERT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + // e.vpinsrb(i.dest, i.src3.reg().cvt32(), i.src2.constant() ^ 0x3); + } +}; +struct INSERT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + // e.vpinsrw(i.dest, i.src3.reg().cvt32(), i.src2.constant() ^ 0x1); + } +}; +struct INSERT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + // e.vpinsrd(i.dest, i.src3, i.src2.constant()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_INSERT, INSERT_I8, INSERT_I16, INSERT_I32); + +// ============================================================================ +// OPCODE_EXTRACT +// ============================================================================ +// TODO(benvanik): sequence extract/splat: +// v0.i32 = extract v0.v128, 0 +// v0.v128 = splat v0.i32 +// This can be a single broadcast. +struct EXTRACT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +struct EXTRACT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +struct EXTRACT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); + +// ============================================================================ +// OPCODE_SPLAT +// ============================================================================ +// Copy a value into all elements of a vector +struct SPLAT_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +struct SPLAT_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +struct SPLAT_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +struct SPLAT_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + {} + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SPLAT, SPLAT_I8, SPLAT_I16, SPLAT_I32, SPLAT_F32); + +// ============================================================================ +// OPCODE_PERMUTE +// ============================================================================ +struct PERMUTE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +struct PERMUTE_V128 + : Sequence> { + static void EmitByInt8(A64Emitter& e, const EmitArgType& i) {} + + static void EmitByInt16(A64Emitter& e, const EmitArgType& i) {} + + static void EmitByInt32(A64Emitter& e, const EmitArgType& i) { + assert_always(); + } + + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case INT8_TYPE: + EmitByInt8(e, i); + break; + case INT16_TYPE: + EmitByInt16(e, i); + break; + case INT32_TYPE: + EmitByInt32(e, i); + break; + default: + assert_unhandled_case(i.instr->flags); + return; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_PERMUTE, PERMUTE_I32, PERMUTE_V128); + +// ============================================================================ +// OPCODE_SWIZZLE +// ============================================================================ +struct SWIZZLE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i){}; +}; +EMITTER_OPCODE_TABLE(OPCODE_SWIZZLE, SWIZZLE); + +// ============================================================================ +// OPCODE_PACK +// ============================================================================ +struct PACK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags & PACK_TYPE_MODE) { + case PACK_TYPE_D3DCOLOR: + EmitD3DCOLOR(e, i); + break; + case PACK_TYPE_FLOAT16_2: + EmitFLOAT16_2(e, i); + break; + case PACK_TYPE_FLOAT16_4: + EmitFLOAT16_4(e, i); + break; + case PACK_TYPE_SHORT_2: + EmitSHORT_2(e, i); + break; + case PACK_TYPE_SHORT_4: + EmitSHORT_4(e, i); + break; + case PACK_TYPE_UINT_2101010: + EmitUINT_2101010(e, i); + break; + case PACK_TYPE_ULONG_4202020: + EmitULONG_4202020(e, i); + break; + case PACK_TYPE_8_IN_16: + Emit8_IN_16(e, i, i.instr->flags); + break; + case PACK_TYPE_16_IN_32: + Emit16_IN_32(e, i, i.instr->flags); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } + static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) {} + static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) {} + static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) {} + static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) {} + static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) {} + static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) {} + static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) {} + static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { + } + // Pack 2 32-bit vectors into a 16-bit vector. + static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, + uint32_t flags) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_PACK, PACK); + +// ============================================================================ +// OPCODE_UNPACK +// ============================================================================ +struct UNPACK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags & PACK_TYPE_MODE) { + case PACK_TYPE_D3DCOLOR: + EmitD3DCOLOR(e, i); + break; + case PACK_TYPE_FLOAT16_2: + EmitFLOAT16_2(e, i); + break; + case PACK_TYPE_FLOAT16_4: + EmitFLOAT16_4(e, i); + break; + case PACK_TYPE_SHORT_2: + EmitSHORT_2(e, i); + break; + case PACK_TYPE_SHORT_4: + EmitSHORT_4(e, i); + break; + case PACK_TYPE_UINT_2101010: + EmitUINT_2101010(e, i); + break; + case PACK_TYPE_ULONG_4202020: + EmitULONG_4202020(e, i); + break; + case PACK_TYPE_8_IN_16: + Emit8_IN_16(e, i, i.instr->flags); + break; + case PACK_TYPE_16_IN_32: + Emit16_IN_32(e, i, i.instr->flags); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } + static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) {} + static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) {} + static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) {} + static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) {} + static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) {} + static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) {} + static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) {} + static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { + } + static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, + uint32_t flags) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_UNPACK, UNPACK); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_sequences.cc b/a64_sequences.cc new file mode 100644 index 000000000..a5c20802b --- /dev/null +++ b/a64_sequences.cc @@ -0,0 +1,2733 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +// A note about vectors: +// Xenia represents vectors as xyzw pairs, with indices 0123. +// XMM registers are xyzw pairs with indices 3210, making them more like wzyx. +// This makes things somewhat confusing. It'd be nice to just shuffle the +// registers around on load/store, however certain operations require that +// data be in the right offset. +// Basically, this identity must hold: +// shuffle(vec, b00011011) -> {x,y,z,w} => {x,y,z,w} +// All indices and operations must respect that. +// +// Memory (big endian): +// [00 01 02 03] [04 05 06 07] [08 09 0A 0B] [0C 0D 0E 0F] (x, y, z, w) +// load into xmm register: +// [0F 0E 0D 0C] [0B 0A 09 08] [07 06 05 04] [03 02 01 00] (w, z, y, x) + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include +#include + +#include "xenia/base/assert.h" +#include "xenia/base/clock.h" +#include "xenia/base/logging.h" +#include "xenia/base/string.h" +#include "xenia/base/threading.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_tracers.h" +#include "xenia/cpu/backend/a64/a64_util.h" +#include "xenia/cpu/hir/hir_builder.h" +#include "xenia/cpu/processor.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using namespace oaknut; + +// TODO(benvanik): direct usings. +using namespace xe::cpu; +using namespace xe::cpu::hir; + +using xe::cpu::hir::Instr; + +typedef bool (*SequenceSelectFn)(A64Emitter&, const Instr*); +std::unordered_map sequence_table; + +// ============================================================================ +// OPCODE_COMMENT +// ============================================================================ +struct COMMENT : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (IsTracingInstr()) { + auto str = reinterpret_cast(i.src1.value); + // TODO(benvanik): pass through. + // TODO(benvanik): don't just leak this memory. + auto str_copy = xe_strdup(str); + e.MOV(e.GetNativeParam(0), reinterpret_cast(str_copy)); + e.CallNative(reinterpret_cast(TraceString)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_COMMENT, COMMENT); + +// ============================================================================ +// OPCODE_NOP +// ============================================================================ +struct NOP : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { e.NOP(); } +}; +EMITTER_OPCODE_TABLE(OPCODE_NOP, NOP); + +// ============================================================================ +// OPCODE_SOURCE_OFFSET +// ============================================================================ +struct SOURCE_OFFSET + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MarkSourceOffset(i.instr); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SOURCE_OFFSET, SOURCE_OFFSET); + +// ============================================================================ +// OPCODE_ASSIGN +// ============================================================================ +struct ASSIGN_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest, i.src1); + } +}; +struct ASSIGN_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest, i.src1); + } +}; +struct ASSIGN_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest, i.src1); + } +}; +struct ASSIGN_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest, i.src1); + } +}; +struct ASSIGN_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMOV(i.dest, i.src1); + } +}; +struct ASSIGN_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMOV(i.dest, i.src1); + } +}; +struct ASSIGN_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest.reg().B16(), i.src1.reg().B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ASSIGN, ASSIGN_I8, ASSIGN_I16, ASSIGN_I32, + ASSIGN_I64, ASSIGN_F32, ASSIGN_F64, ASSIGN_V128); + +// ============================================================================ +// OPCODE_CAST +// ============================================================================ +struct CAST_I32_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMOV(i.dest, i.src1); + } +}; +struct CAST_I64_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMOV(i.dest, i.src1); + } +}; +struct CAST_F32_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMOV(i.dest, i.src1); + } +}; +struct CAST_F64_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest.reg().Delem()[0], i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CAST, CAST_I32_F32, CAST_I64_F64, CAST_F32_I32, + CAST_F64_I64); + +// ============================================================================ +// OPCODE_ZERO_EXTEND +// ============================================================================ +struct ZERO_EXTEND_I16_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTB(i.dest, i.src1); + } +}; +struct ZERO_EXTEND_I32_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTB(i.dest, i.src1); + } +}; +struct ZERO_EXTEND_I64_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTB(i.dest.reg().toW(), i.src1); + } +}; +struct ZERO_EXTEND_I32_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTH(i.dest, i.src1); + } +}; +struct ZERO_EXTEND_I64_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTH(i.dest.reg().toW(), i.src1); + } +}; +struct ZERO_EXTEND_I64_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest.reg().toW(), i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ZERO_EXTEND, ZERO_EXTEND_I16_I8, ZERO_EXTEND_I32_I8, + ZERO_EXTEND_I64_I8, ZERO_EXTEND_I32_I16, + ZERO_EXTEND_I64_I16, ZERO_EXTEND_I64_I32); + +// ============================================================================ +// OPCODE_SIGN_EXTEND +// ============================================================================ +struct SIGN_EXTEND_I16_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SXTB(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I32_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SXTB(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I64_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SXTB(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I32_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SXTH(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I64_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SXTH(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I64_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SXTH(i.dest, i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SIGN_EXTEND, SIGN_EXTEND_I16_I8, SIGN_EXTEND_I32_I8, + SIGN_EXTEND_I64_I8, SIGN_EXTEND_I32_I16, + SIGN_EXTEND_I64_I16, SIGN_EXTEND_I64_I32); + +// ============================================================================ +// OPCODE_TRUNCATE +// ============================================================================ +struct TRUNCATE_I8_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTB(i.dest, i.src1); + } +}; +struct TRUNCATE_I8_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTB(i.dest, i.src1); + } +}; +struct TRUNCATE_I8_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTB(i.dest, i.src1.reg().toW()); + } +}; +struct TRUNCATE_I16_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTH(i.dest, i.src1); + } +}; +struct TRUNCATE_I16_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTH(i.dest, i.src1.reg().toW()); + } +}; +struct TRUNCATE_I32_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MOV(i.dest, i.src1.reg().toW()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TRUNCATE, TRUNCATE_I8_I16, TRUNCATE_I8_I32, + TRUNCATE_I8_I64, TRUNCATE_I16_I32, TRUNCATE_I16_I64, + TRUNCATE_I32_I64); + +// ============================================================================ +// OPCODE_CONVERT +// ============================================================================ +struct CONVERT_I32_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): saturation check? cvtt* (trunc?) + if (i.instr->flags == ROUND_TO_ZERO) { + // e.vcvttss2si(i.dest, i.src1); + e.FCVTZS(i.dest, i.src1.reg().toS()); + } else { + // e.vcvtss2si(i.dest, i.src1); + e.FCVTNS(i.dest, i.src1.reg().toS()); + } + } +}; +struct CONVERT_I32_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Intel returns 0x80000000 if the double value does not fit within an int32 + // ARM64 and PPC saturates the value instead. + // e.vminsd(e.xmm0, i.src1, e.GetVConstPtr(XMMIntMaxPD)); + if (i.instr->flags == ROUND_TO_ZERO) { + // e.vcvttsd2si(i.dest, e.xmm0); + e.FCVTZS(i.dest, i.src1.reg().toD()); + } else { + // e.vcvtsd2si(i.dest, e.xmm0); + e.FCVTNS(i.dest, i.src1.reg().toD()); + } + } +}; +struct CONVERT_I64_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags == ROUND_TO_ZERO) { + // e.vcvttsd2si(i.dest, i.src1); + e.FCVTZS(i.dest, i.src1.reg().toD()); + } else { + // e.vcvtsd2si(i.dest, i.src1); + e.FCVTNS(i.dest, i.src1.reg().toD()); + } + } +}; +struct CONVERT_F32_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): saturation check? cvtt* (trunc?) + // e.vcvtsi2ss(i.dest, i.src1); + e.SCVTF(i.dest.reg().toS(), i.src1); + } +}; +struct CONVERT_F32_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): saturation check? cvtt* (trunc?) + // e.vcvtsd2ss(i.dest, i.src1); + e.FCVT(i.dest.reg().toS(), i.src1.reg().toD()); + } +}; +struct CONVERT_F64_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): saturation check? cvtt* (trunc?) + // e.vcvtsi2sd(i.dest, i.src1); + e.SCVTF(i.dest.reg().toD(), i.src1); + } +}; +struct CONVERT_F64_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // e.vcvtss2sd(i.dest, i.src1); + e.FCVT(i.dest.reg().toD(), i.src1.reg().toS()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CONVERT, CONVERT_I32_F32, CONVERT_I32_F64, + CONVERT_I64_F64, CONVERT_F32_I32, CONVERT_F32_F64, + CONVERT_F64_I64, CONVERT_F64_F32); + +// ============================================================================ +// OPCODE_ROUND +// ============================================================================ +struct ROUND_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case ROUND_TO_ZERO: + // e.vroundss(i.dest, i.src1, 0b00000011); + e.FRINTZ(i.dest.reg().toS(), i.src1.reg().toS()); + break; + case ROUND_TO_NEAREST: + // e.vroundss(i.dest, i.src1, 0b00000000); + e.FRINTN(i.dest.reg().toS(), i.src1.reg().toS()); + break; + case ROUND_TO_MINUS_INFINITY: + // e.vroundss(i.dest, i.src1, 0b00000001); + e.FRINTM(i.dest.reg().toS(), i.src1.reg().toS()); + break; + case ROUND_TO_POSITIVE_INFINITY: + // e.vroundss(i.dest, i.src1, 0b00000010); + e.FRINTP(i.dest.reg().toS(), i.src1.reg().toS()); + break; + } + } +}; +struct ROUND_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case ROUND_TO_ZERO: + // e.vroundsd(i.dest, i.src1, 0b00000011); + e.FRINTZ(i.dest, i.src1); + break; + case ROUND_TO_NEAREST: + // e.vroundsd(i.dest, i.src1, 0b00000000); + e.FRINTN(i.dest, i.src1); + break; + case ROUND_TO_MINUS_INFINITY: + // e.vroundsd(i.dest, i.src1, 0b00000001); + e.FRINTM(i.dest, i.src1); + break; + case ROUND_TO_POSITIVE_INFINITY: + // e.vroundsd(i.dest, i.src1, 0b00000010); + e.FRINTP(i.dest, i.src1); + break; + } + } +}; +struct ROUND_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case ROUND_TO_ZERO: + // e.vroundps(i.dest, i.src1, 0b00000011); + e.FRINTZ(i.dest.reg().S4(), i.src1.reg().S4()); + break; + case ROUND_TO_NEAREST: + // e.vroundps(i.dest, i.src1, 0b00000000); + e.FRINTN(i.dest.reg().S4(), i.src1.reg().S4()); + break; + case ROUND_TO_MINUS_INFINITY: + // e.vroundps(i.dest, i.src1, 0b00000001); + e.FRINTM(i.dest.reg().S4(), i.src1.reg().S4()); + break; + case ROUND_TO_POSITIVE_INFINITY: + // e.vroundps(i.dest, i.src1, 0b00000010); + e.FRINTP(i.dest.reg().S4(), i.src1.reg().S4()); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ROUND, ROUND_F32, ROUND_F64, ROUND_V128); + +// ============================================================================ +// OPCODE_LOAD_CLOCK +// ============================================================================ +struct LOAD_CLOCK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // When scaling is disabled and the raw clock source is selected, the code + // in the Clock class is actually just forwarding tick counts after one + // simple multiply and division. In that case we rather bake the scaling in + // here to cut extra function calls with CPU cache misses and stack frame + // overhead. + if (cvars::clock_no_scaling && cvars::clock_source_raw) { + auto ratio = Clock::guest_tick_ratio(); + // The 360 CPU is an in-order CPU, AMD64 usually isn't. Without + // mfence/lfence magic the rdtsc instruction can be executed sooner or + // later in the cache window. Since it's resolution however is much higher + // than the 360's mftb instruction this can safely be ignored. + + // Read time stamp in edx (high part) and eax (low part). + // e.rdtsc(); + // Make it a 64 bit number in rax. + // e.shl(e.rdx, 32); + // e.or_(e.rax, e.rdx); + // Apply tick frequency scaling. + // e.MOV(e.rcx, ratio.first); + // e.mul(e.rcx); + // We actually now have a 128 bit number in rdx:rax. + // e.MOV(e.rcx, ratio.second); + // e.div(e.rcx); + // e.MOV(i.dest, e.rax); + } else { + e.CallNative(LoadClock); + e.MOV(i.dest, X0); + } + } + static uint64_t LoadClock(void* raw_context) { + return Clock::QueryGuestTickCount(); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_CLOCK, LOAD_CLOCK); + +// ============================================================================ +// OPCODE_CONTEXT_BARRIER +// ============================================================================ +struct CONTEXT_BARRIER + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) {} +}; +EMITTER_OPCODE_TABLE(OPCODE_CONTEXT_BARRIER, CONTEXT_BARRIER); + +// ============================================================================ +// OPCODE_MAX +// ============================================================================ +struct MAX_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMAX(i.dest, i.src1, i.src2); + } +}; +struct MAX_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMAX(i.dest, i.src1, i.src2); + } +}; +struct MAX_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FMAX(i.dest.reg().S4(), i.src1.reg().S4(), i.src2.reg().S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MAX, MAX_F32, MAX_F64, MAX_V128); + +// ============================================================================ +// OPCODE_MIN +// ============================================================================ +struct MIN_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.CMP(dest_src, src); + e.CSEL(dest_src, dest_src, src, Cond::LO); + }, + [](A64Emitter& e, WReg dest_src, int32_t constant) { + e.MOV(W0, constant); + e.CMP(dest_src, W0); + e.CSEL(dest_src, dest_src, W0, Cond::LO); + }); + } +}; +struct MIN_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.CMP(dest_src, src); + e.CSEL(dest_src, dest_src, src, Cond::LO); + }, + [](A64Emitter& e, WReg dest_src, int32_t constant) { + e.MOV(W0, constant); + e.CMP(dest_src, W0); + e.CSEL(dest_src, dest_src, W0, Cond::LO); + }); + } +}; +struct MIN_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.CMP(dest_src, src); + e.CSEL(dest_src, dest_src, src, Cond::LO); + }, + [](A64Emitter& e, WReg dest_src, int32_t constant) { + e.MOV(W0, constant); + e.CMP(dest_src, W0); + e.CSEL(dest_src, dest_src, W0, Cond::LO); + }); + } +}; +struct MIN_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, XReg dest_src, XReg src) { + e.CMP(dest_src, src); + e.CSEL(dest_src, dest_src, src, Cond::LO); + }, + [](A64Emitter& e, XReg dest_src, int64_t constant) { + e.MOV(X0, constant); + e.CMP(dest_src, X0); + e.CSEL(dest_src, dest_src, X0, Cond::LO); + }); + } +}; +struct MIN_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FMIN(dest, src1, src2); + }); + } +}; +struct MIN_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FMIN(dest, src1, src2); + }); + } +}; +struct MIN_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMIN(dest.S4(), src1.S4(), src2.S4()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MIN, MIN_I8, MIN_I16, MIN_I32, MIN_I64, MIN_F32, + MIN_F64, MIN_V128); + +// ============================================================================ +// OPCODE_SELECT +// ============================================================================ +// dest = src1 ? src2 : src3 +// TODO(benvanik): match compare + select sequences, as often it's something +// like SELECT(VECTOR_COMPARE_SGE(a, b), a, b) +struct SELECT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg src2(0); + if (i.src2.is_constant) { + src2 = W0; + e.MOV(src2, i.src2.constant()); + } else { + src2 = i.src2; + } + e.CMP(i.src1.reg().toX(), 0); + e.CSEL(i.dest, src2, i.src3, Cond::NE); + } +}; +struct SELECT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg src2(0); + if (i.src2.is_constant) { + src2 = W0; + e.MOV(src2, i.src2.constant()); + } else { + src2 = i.src2; + } + e.CMP(i.src1.reg().toX(), 0); + e.CSEL(i.dest, src2, i.src3, Cond::NE); + } +}; +struct SELECT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg src2(0); + if (i.src2.is_constant) { + src2 = W0; + e.MOV(src2, i.src2.constant()); + } else { + src2 = i.src2; + } + e.CMP(i.src1.reg().toX(), 0); + e.CSEL(i.dest, src2, i.src3, Cond::NE); + } +}; +struct SELECT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + XReg src2(0); + if (i.src2.is_constant) { + src2 = X0; + e.MOV(src2, i.src2.constant()); + } else { + src2 = i.src2; + } + e.CMP(i.src1.reg().toX(), 0); + e.CSEL(i.dest, src2, i.src3, Cond::NE); + } +}; +struct SELECT_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 != 0 ? src2 : src3 + + SReg src2 = i.src2.is_constant ? S2 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2.toQ(), i.src2.constant()); + } + + SReg src3 = i.src3.is_constant ? S2 : i.src3; + if (i.src3.is_constant) { + e.LoadConstantV(src3.toQ(), i.src3.constant()); + } + + e.CMP(i.src1.reg().toX(), 0); + e.FCSEL(i.dest, src2, i.src3, Cond::NE); + } +}; +struct SELECT_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 != 0 ? src2 : src3 + + DReg src2 = i.src2.is_constant ? D2 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2.toQ(), i.src2.constant()); + } + + DReg src3 = i.src3.is_constant ? D2 : i.src3; + if (i.src3.is_constant) { + e.LoadConstantV(src3.toQ(), i.src3.constant()); + } + + e.CMP(i.src1.reg().toX(), 0); + e.FCSEL(i.dest, src2, i.src3, Cond::NE); + } +}; +struct SELECT_V128_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 != 0 ? src2 : src3 + + QReg src2 = i.src2.is_constant ? Q2 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2, i.src2.constant()); + } + + QReg src3 = i.src3.is_constant ? Q2 : i.src3; + if (i.src3.is_constant) { + e.LoadConstantV(src3, i.src3.constant()); + } + + e.CMP(i.src1.reg().toX(), 0); + e.CSETM(W0, Cond::NE); + e.DUP(i.dest.reg().S4(), W0); + e.BSL(i.dest.reg().B16(), src2.B16(), src3.B16()); + } +}; +struct SELECT_V128_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + + const QReg src2 = i.src2.is_constant ? Q1 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2, i.src2.constant()); + } + + const QReg src3 = i.src3.is_constant ? Q2 : i.src3; + if (i.src3.is_constant) { + e.LoadConstantV(src3, i.src3.constant()); + } + + // src1 ? src2 : src3; + e.BSL(src1.B16(), src2.B16(), src3.B16()); + e.MOV(i.dest.reg().B16(), src1.B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SELECT, SELECT_I8, SELECT_I16, SELECT_I32, + SELECT_I64, SELECT_F32, SELECT_F64, SELECT_V128_I8, + SELECT_V128_V128); + +// ============================================================================ +// OPCODE_IS_TRUE +// ============================================================================ +struct IS_TRUE_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::NE); + } +}; +struct IS_TRUE_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::NE); + } +}; +struct IS_TRUE_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::NE); + } +}; +struct IS_TRUE_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::NE); + } +}; +struct IS_TRUE_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::NE); + } +}; +struct IS_TRUE_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::NE); + } +}; +struct IS_TRUE_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UMAXV(Q0.toS(), i.src1.reg().S4()); + e.MOV(W0, Q0.Selem()[0]); + e.CMP(W0, 0); + e.CSET(i.dest, Cond::NE); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_IS_TRUE, IS_TRUE_I8, IS_TRUE_I16, IS_TRUE_I32, + IS_TRUE_I64, IS_TRUE_F32, IS_TRUE_F64, IS_TRUE_V128); + +// ============================================================================ +// OPCODE_IS_FALSE +// ============================================================================ +struct IS_FALSE_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::EQ); + } +}; +struct IS_FALSE_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::EQ); + } +}; +struct IS_FALSE_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::EQ); + } +}; +struct IS_FALSE_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::EQ); + } +}; +struct IS_FALSE_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::EQ); + } +}; +struct IS_FALSE_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1.reg(), 0); + e.CSET(i.dest, Cond::EQ); + } +}; +struct IS_FALSE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UMAXV(Q0.toS(), i.src1.reg().S4()); + e.MOV(W0, Q0.Selem()[0]); + e.CMP(W0, 0); + e.CSET(i.dest, Cond::EQ); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_IS_FALSE, IS_FALSE_I8, IS_FALSE_I16, IS_FALSE_I32, + IS_FALSE_I64, IS_FALSE_F32, IS_FALSE_F64, IS_FALSE_V128); + +// ============================================================================ +// OPCODE_IS_NAN +// ============================================================================ +struct IS_NAN_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1, i.src1); + e.CSET(i.dest, Cond::VS); + } +}; + +struct IS_NAN_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1, i.src1); + e.CSET(i.dest, Cond::VS); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_IS_NAN, IS_NAN_F32, IS_NAN_F64); + +// ============================================================================ +// OPCODE_COMPARE_EQ +// ============================================================================ +struct COMPARE_EQ_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, WReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::EQ); + } +}; +struct COMPARE_EQ_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, WReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::EQ); + } +}; +struct COMPARE_EQ_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, WReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::EQ); + } +}; +struct COMPARE_EQ_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, XReg src1, XReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, XReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::EQ); + } +}; +struct COMPARE_EQ_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, + [&i](A64Emitter& e, I8Op dest, const SReg& src1, const SReg& src2) { + e.FCMP(src1, src2); + }); + e.CSET(i.dest, Cond::EQ); + } +}; +struct COMPARE_EQ_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, + [&i](A64Emitter& e, I8Op dest, const DReg& src1, const DReg& src2) { + e.FCMP(src1, src2); + }); + e.CSET(i.dest, Cond::EQ); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_EQ, COMPARE_EQ_I8, COMPARE_EQ_I16, + COMPARE_EQ_I32, COMPARE_EQ_I64, COMPARE_EQ_F32, + COMPARE_EQ_F64); + +// ============================================================================ +// OPCODE_COMPARE_NE +// ============================================================================ +struct COMPARE_NE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, WReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::NE); + } +}; +struct COMPARE_NE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, WReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::NE); + } +}; +struct COMPARE_NE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, WReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::NE); + } +}; +struct COMPARE_NE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeCompareOp( + e, i, [](A64Emitter& e, XReg src1, XReg src2) { e.CMP(src1, src2); }, + [](A64Emitter& e, XReg src1, int32_t constant) { + e.CMP(src1, constant); + }); + e.CSET(i.dest, Cond::NE); + } +}; +struct COMPARE_NE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1, i.src2); + e.CSET(i.dest, Cond::NE); + } +}; +struct COMPARE_NE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FCMP(i.src1, i.src2); + e.CSET(i.dest, Cond::NE); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_NE, COMPARE_NE_I8, COMPARE_NE_I16, + COMPARE_NE_I32, COMPARE_NE_I64, COMPARE_NE_F32, + COMPARE_NE_F64); + +// ============================================================================ +// OPCODE_COMPARE_* +// ============================================================================ +#define EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, type, \ + reg_type) \ + struct COMPARE_##op##_##type \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + EmitAssociativeCompareOp( \ + e, i, \ + [](A64Emitter& e, WReg dest, const reg_type& src1, \ + const reg_type& src2, bool inverse) { \ + e.CMP(src1, src2); \ + if (!inverse) { \ + e.CSET(dest, cond); \ + } else { \ + e.CSET(dest, inverse_cond); \ + } \ + }, \ + [](A64Emitter& e, WReg dest, const reg_type& src1, int32_t constant, \ + bool inverse) { \ + e.CMP(src1, constant); \ + if (!inverse) { \ + e.CSET(dest, cond); \ + } else { \ + e.CSET(dest, inverse_cond); \ + } \ + }); \ + } \ + }; +#define EMITTER_ASSOCIATIVE_COMPARE_XX(op, cond, inverse_cond) \ + EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I8Op, WReg); \ + EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I16Op, WReg); \ + EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I32Op, WReg); \ + EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I64Op, XReg); \ + EMITTER_OPCODE_TABLE(OPCODE_COMPARE_##op, COMPARE_##op##_I8Op, \ + COMPARE_##op##_I16Op, COMPARE_##op##_I32Op, \ + COMPARE_##op##_I64Op); +EMITTER_ASSOCIATIVE_COMPARE_XX(SLT, Cond::LT, Cond::GT); // setl, setg +EMITTER_ASSOCIATIVE_COMPARE_XX(SLE, Cond::LE, Cond::GE); // setle, setge +EMITTER_ASSOCIATIVE_COMPARE_XX(SGT, Cond::GT, Cond::LT); // setg, setl +EMITTER_ASSOCIATIVE_COMPARE_XX(SGE, Cond::GE, Cond::LE); // setge, setle +EMITTER_ASSOCIATIVE_COMPARE_XX(ULT, Cond::LO, Cond::HI); // setb, seta +EMITTER_ASSOCIATIVE_COMPARE_XX(ULE, Cond::LS, Cond::HS); // setbe, setae +EMITTER_ASSOCIATIVE_COMPARE_XX(UGE, Cond::HS, Cond::LS); // setae, setbe +EMITTER_ASSOCIATIVE_COMPARE_XX(UGT, Cond::HI, Cond::LO); // seta, setb + +// https://web.archive.org/web/20171129015931/https://x86.renejeschke.de/html/file_module_x86_id_288.html +// Original link: https://x86.renejeschke.de/html/file_module_x86_id_288.html +#define EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(op, cond) \ + struct COMPARE_##op##_F32 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + e.FCMP(i.src1, i.src2); \ + e.CSET(i.dest, cond); \ + } \ + }; \ + struct COMPARE_##op##_F64 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.LoadConstantV(Q0, i.src1.constant()); \ + e.FCMP(D0, i.src2); \ + } else if (i.src2.is_constant) { \ + e.LoadConstantV(Q0, i.src2.constant()); \ + e.FCMP(i.src1, D0); \ + } else { \ + e.FCMP(i.src1, i.src2); \ + } \ + e.CSET(i.dest, cond); \ + } \ + }; \ + EMITTER_OPCODE_TABLE(OPCODE_COMPARE_##op##_FLT, COMPARE_##op##_F32, \ + COMPARE_##op##_F64); +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SLT, Cond::LT); // setb +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SLE, Cond::LE); // setbe +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SGT, Cond::GT); // seta +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SGE, Cond::GE); // setae +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(ULT, Cond::LO); // setb +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(ULE, Cond::LS); // setbe +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(UGT, Cond::HI); // seta +EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(UGE, Cond::HS); // setae + +// ============================================================================ +// OPCODE_DID_SATURATE +// ============================================================================ +struct DID_SATURATE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): implement saturation check (VECTOR_ADD, etc). + e.EOR(i.dest, i.dest, i.dest); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DID_SATURATE, DID_SATURATE); + +// ============================================================================ +// OPCODE_ADD +// ============================================================================ +// TODO(benvanik): put dest/src1|2 together. +template +void EmitAddXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + // e.add(dest_src, src); + e.ADD(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int32_t constant) { + // e.add(dest_src, constant); + e.ADD(dest_src, dest_src, constant); + }); +} +struct ADD_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // EmitAddXX(e, i); + EmitAddXX(e, i); + } +}; +struct ADD_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // EmitAddXX(e, i); + EmitAddXX(e, i); + } +}; +struct ADD_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAddXX(e, i); + } +}; +struct ADD_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAddXX(e, i); + } +}; +struct ADD_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + // e.vaddss(dest, src1, src2); + e.FADD(dest, src1, src2); + }); + } +}; +struct ADD_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + // e.vaddsd(dest, src1, src2); + e.FADD(dest, src1, src2); + }); + } +}; +struct ADD_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + // e.vaddps(dest, src1, src2); + e.FADD(dest.S4(), src1.S4(), src2.S4()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ADD, ADD_I8, ADD_I16, ADD_I32, ADD_I64, ADD_F32, + ADD_F64, ADD_V128); + +// ============================================================================ +// OPCODE_ADD_CARRY +// ============================================================================ +// TODO(benvanik): put dest/src1|2 together. +template +void EmitAddCarryXX(A64Emitter& e, const ARGS& i) { + // TODO(benvanik): faster setting? we could probably do some fun math tricks + // here to get the carry flag set. + if (i.src3.is_constant) { + if (i.src3.constant()) { + // Set carry + // This is implicitly "SUBS 0 - 0" + e.CMP(WZR.toW(), 0); + } else { + // Clear carry + e.CMN(WZR.toW(), 0); + } + } else { + // If src3 is non-zero, set the carry flag + e.CMP(i.src3.reg().toW(), 0); + e.CSET(X0, Cond::NE); + + e.MRS(X1, SystemReg::NZCV); + // Assign carry bit + e.BFI(X1, X0, 61, 1); + e.MSR(SystemReg::NZCV, X1); + } + e.ADC(i.dest, i.src1, i.src2); +} +struct ADD_CARRY_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAddCarryXX(e, i); + } +}; +struct ADD_CARRY_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAddCarryXX(e, i); + } +}; +struct ADD_CARRY_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAddCarryXX(e, i); + } +}; +struct ADD_CARRY_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAddCarryXX(e, i); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ADD_CARRY, ADD_CARRY_I8, ADD_CARRY_I16, + ADD_CARRY_I32, ADD_CARRY_I64); + +// ============================================================================ +// OPCODE_SUB +// ============================================================================ +// TODO(benvanik): put dest/src1|2 together. +template +void EmitSubXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + e.SUB(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int32_t constant) { + e.SUB(dest_src, dest_src, constant); + }); +} +struct SUB_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubXX(e, i); + } +}; +struct SUB_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubXX(e, i); + } +}; +struct SUB_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubXX(e, i); + } +}; +struct SUB_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubXX(e, i); + } +}; +struct SUB_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitAssociativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FSUB(dest, src1, src2); + }); + } +}; +struct SUB_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitAssociativeBinaryVOp( + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FSUB(dest, src1, src2); + }); + } +}; +struct SUB_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitAssociativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FSUB(dest.S4(), src1.S4(), src2.S4()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SUB, SUB_I8, SUB_I16, SUB_I32, SUB_I64, SUB_F32, + SUB_F64, SUB_V128); + +// ============================================================================ +// OPCODE_MUL +// ============================================================================ +// Sign doesn't matter here, as we don't use the high bits. +// We exploit mulx here to avoid creating too much register pressure. +struct MUL_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + } +}; +struct MUL_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + } +}; +struct MUL_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + } +}; +struct MUL_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(X0, i.src1.constant()); + e.MUL(i.dest, X0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(X0, i.src2.constant()); + e.MUL(i.dest, i.src1, X0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + } +}; +struct MUL_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FMUL(dest, src1, src2); + }); + } +}; +struct MUL_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FMUL(dest, src1, src2); + }); + } +}; +struct MUL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMUL(dest.S4(), src1.S4(), src2.S4()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL, MUL_I8, MUL_I16, MUL_I32, MUL_I64, MUL_F32, + MUL_F64, MUL_V128); + +// ============================================================================ +// OPCODE_MUL_HI +// ============================================================================ +struct MUL_HI_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + e.UBFX(i.dest, i.dest, 8, 8); + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + e.SBFX(i.dest, i.dest, 8, 8); + } + } +}; +struct MUL_HI_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + e.UBFX(i.dest, i.dest, 16, 16); + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.MUL(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.MUL(i.dest, i.src1, W0); + } else { + e.MUL(i.dest, i.src1, i.src2); + } + e.SBFX(i.dest, i.dest, 16, 16); + } + } +}; +struct MUL_HI_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.UMULL(X0, W0, i.src2); + e.UBFX(X0, X0, 32, 32); + e.MOV(i.dest, X0.toW()); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.UMULL(X0, W0, i.src2); + e.UBFX(X0, X0, 32, 32); + e.MOV(i.dest, X0.toW()); + } else { + e.UMULL(X0, W0, i.src2); + e.UBFX(X0, X0, 32, 32); + e.MOV(i.dest, X0.toW()); + } + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.UMULL(X0, W0, i.src2); + e.UBFX(X0, X0, 32, 32); + e.MOV(i.dest, X0.toW()); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.UMULL(X0, W0, i.src2); + e.UBFX(X0, X0, 32, 32); + e.MOV(i.dest, X0.toW()); + } else { + e.UMULL(X0, W0, i.src2); + e.UBFX(X0, X0, 32, 32); + e.MOV(i.dest, X0.toW()); + } + } + } +}; +struct MUL_HI_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(X0, i.src1.constant()); + e.UMULH(i.dest, X0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(X0, i.src2.constant()); + e.UMULH(i.dest, i.src1, X0); + } else { + e.UMULH(i.dest, i.src1, i.src2); + } + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(X0, i.src1.constant()); + e.UMULH(i.dest, X0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(X0, i.src2.constant()); + e.UMULH(i.dest, i.src1, X0); + } else { + e.UMULH(i.dest, i.src1, i.src2); + } + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL_HI, MUL_HI_I8, MUL_HI_I16, MUL_HI_I32, + MUL_HI_I64); + +// ============================================================================ +// OPCODE_DIV +// ============================================================================ +struct DIV_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.UDIV(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.UDIV(i.dest, i.src1, W0); + } else { + e.UDIV(i.dest, i.src1, i.src2); + } + e.UXTB(i.dest, i.dest); + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.SDIV(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.SDIV(i.dest, i.src1, W0); + } else { + e.SDIV(i.dest, i.src1, i.src2); + } + e.SXTB(i.dest, i.dest); + } + } +}; +struct DIV_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.UDIV(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.UDIV(i.dest, i.src1, W0); + } else { + e.UDIV(i.dest, i.src1, i.src2); + } + e.UXTH(i.dest, i.dest); + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.SDIV(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.SDIV(i.dest, i.src1, W0); + } else { + e.SDIV(i.dest, i.src1, i.src2); + } + e.SXTH(i.dest, i.dest); + } + } +}; +struct DIV_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.UDIV(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.UDIV(i.dest, i.src1, W0); + } else { + e.UDIV(i.dest, i.src1, i.src2); + } + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(W0, i.src1.constant()); + e.SDIV(i.dest, W0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(W0, i.src2.constant()); + e.SDIV(i.dest, i.src1, W0); + } else { + e.SDIV(i.dest, i.src1, i.src2); + } + } + } +}; +struct DIV_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(X0, i.src1.constant()); + e.UDIV(i.dest, X0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(X0, i.src2.constant()); + e.UDIV(i.dest, i.src1, X0); + } else { + e.UDIV(i.dest, i.src1, i.src2); + } + } else { + if (i.src1.is_constant) { + assert_true(!i.src2.is_constant); + e.MOV(X0, i.src1.constant()); + e.SDIV(i.dest, X0, i.src2); + } else if (i.src2.is_constant) { + assert_true(!i.src1.is_constant); + e.MOV(X0, i.src2.constant()); + e.SDIV(i.dest, i.src1, X0); + } else { + e.SDIV(i.dest, i.src1, i.src2); + } + } + } +}; +struct DIV_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitAssociativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FDIV(dest, src1, src2); + }); + } +}; +struct DIV_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitAssociativeBinaryVOp( + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FDIV(dest, src1, src2); + }); + } +}; +struct DIV_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + EmitAssociativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FDIV(dest.S4(), src1.S4(), src2.S4()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DIV, DIV_I8, DIV_I16, DIV_I32, DIV_I64, DIV_F32, + DIV_F64, DIV_V128); + +// ============================================================================ +// OPCODE_MUL_ADD +// ============================================================================ +// d = 1 * 2 + 3 +// $0 = $1x$0 + $2 +struct MUL_ADD_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg src3(1); + if (i.src3.is_constant) { + src3 = S1; + e.LoadConstantV(src3.toQ(), i.src3.constant()); + } else { + // If i.dest == i.src3, back up i.src3 so we don't overwrite it. + src3 = i.src3.reg(); + if (i.dest.reg().index() == i.src3.reg().index()) { + e.FMOV(S1, i.src3); + src3 = S1; + } + } + + // Multiply operation is commutative. + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FMUL(dest, src1, src2); // $0 = $1 * $2 + }); + + e.FADD(i.dest, i.dest, src3); // $0 = $1 + $2 + } +}; +struct MUL_ADD_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg src3(1); + if (i.src3.is_constant) { + src3 = D1; + e.LoadConstantV(src3.toQ(), i.src3.constant()); + } else { + // If i.dest == i.src3, back up i.src3 so we don't overwrite it. + src3 = i.src3.reg(); + if (i.dest.reg().index() == i.src3.reg().index()) { + e.FMOV(D1, i.src3); + src3 = D1; + } + } + + // Multiply operation is commutative. + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FMUL(dest, src1, src2); // $0 = $1 * $2 + }); + + e.FADD(i.dest, i.dest, src3); // $0 = $1 + $2 + } +}; +struct MUL_ADD_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + QReg src3(1); + if (i.src3.is_constant) { + src3 = Q1; + e.LoadConstantV(src3, i.src3.constant()); + } else { + // If i.dest == i.src3, back up i.src3 so we don't overwrite it. + src3 = i.src3; + if (i.dest == i.src3) { + e.MOV(Q1.B16(), i.src3.reg().B16()); + src3 = Q1; + } + } + + // Multiply operation is commutative. + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMUL(dest.S4(), src1.S4(), src2.S4()); // $0 = $1 * $2 + }); + + e.FADD(i.dest.reg().S4(), i.dest.reg().S4(), src3.S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL_ADD, MUL_ADD_F32, MUL_ADD_F64, MUL_ADD_V128); + +// ============================================================================ +// OPCODE_MUL_SUB +// ============================================================================ +// d = 1 * 2 - 3 +// $0 = $2x$0 - $3 +// TODO(benvanik): use other forms (132/213/etc) to avoid register shuffling. +// dest could be src2 or src3 - need to ensure it's not before overwriting dest +// perhaps use other 132/213/etc +// Forms: +// - 132 -> $1 = $1 * $3 - $2 +// - 213 -> $1 = $2 * $1 - $3 +// - 231 -> $1 = $2 * $3 - $1 +struct MUL_SUB_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg src3(1); + if (i.src3.is_constant) { + src3 = S1; + e.LoadConstantV(src3.toQ(), i.src3.constant()); + } else { + // If i.dest == i.src3, back up i.src3 so we don't overwrite it. + src3 = i.src3.reg(); + if (i.dest.reg().index() == i.src3.reg().index()) { + e.FMOV(S1, i.src3); + src3 = S1; + } + } + + // Multiply operation is commutative. + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FMUL(dest, src1, src2); // $0 = $1 * $2 + }); + + e.FSUB(i.dest, i.dest, src3); // $0 = $1 - $2 + } +}; +struct MUL_SUB_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg src3(1); + if (i.src3.is_constant) { + src3 = D1; + e.LoadConstantV(src3.toQ(), i.src3.constant()); + } else { + // If i.dest == i.src3, back up i.src3 so we don't overwrite it. + src3 = i.src3.reg(); + if (i.dest.reg().index() == i.src3.reg().index()) { + e.FMOV(D1, i.src3); + src3 = D1; + } + } + + // Multiply operation is commutative. + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FMUL(dest, src1, src2); // $0 = $1 * $2 + }); + + e.FSUB(i.dest, i.dest, src3); // $0 = $1 + $2 + } +}; +struct MUL_SUB_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + QReg src3(1); + if (i.src3.is_constant) { + src3 = Q1; + e.LoadConstantV(src3, i.src3.constant()); + } else { + // If i.dest == i.src3, back up i.src3 so we don't overwrite it. + src3 = i.src3; + if (i.dest == i.src3) { + e.MOV(Q1.B16(), i.src3.reg().B16()); + src3 = Q1; + } + } + + // Multiply operation is commutative. + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMUL(dest.S4(), src1.S4(), src2.S4()); // $0 = $1 * $2 + }); + + e.FSUB(i.dest.reg().S4(), i.dest.reg().S4(), src3.S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL_SUB, MUL_SUB_F32, MUL_SUB_F64, MUL_SUB_V128); + +// ============================================================================ +// OPCODE_NEG +// ============================================================================ +// TODO(benvanik): put dest/src1 together. +template +void EmitNegXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitUnaryOp( + e, i, [](A64Emitter& e, REG dest_src) { e.NEG(dest_src, dest_src); }); +} +struct NEG_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNegXX(e, i); + } +}; +struct NEG_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNegXX(e, i); + } +}; +struct NEG_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNegXX(e, i); + } +}; +struct NEG_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNegXX(e, i); + } +}; +struct NEG_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FNEG(i.dest, i.src1); + } +}; +struct NEG_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FNEG(i.dest, i.src1); + } +}; +struct NEG_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(!i.instr->flags); + e.FNEG(i.dest.reg().S4(), i.src1.reg().S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_NEG, NEG_I8, NEG_I16, NEG_I32, NEG_I64, NEG_F32, + NEG_F64, NEG_V128); + +// ============================================================================ +// OPCODE_ABS +// ============================================================================ +struct ABS_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FABS(i.dest, i.src1); + } +}; +struct ABS_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FABS(i.dest, i.src1); + } +}; +struct ABS_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FABS(i.dest.reg().S4(), i.src1.reg().S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ABS, ABS_F32, ABS_F64, ABS_V128); + +// ============================================================================ +// OPCODE_SQRT +// ============================================================================ +struct SQRT_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FSQRT(i.dest, i.src1); + } +}; +struct SQRT_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FSQRT(i.dest, i.src1); + } +}; +struct SQRT_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FSQRT(i.dest.reg().S4(), i.src1.reg().S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SQRT, SQRT_F32, SQRT_F64, SQRT_V128); + +// ============================================================================ +// OPCODE_RSQRT +// ============================================================================ +// Altivec guarantees an error of < 1/4096 for vrsqrtefp +struct RSQRT_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FRSQRTE(i.dest, i.src1); + } +}; +struct RSQRT_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FRSQRTE(i.dest, i.src1); + } +}; +struct RSQRT_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FRSQRTE(i.dest.reg().S4(), i.src1.reg().S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RSQRT, RSQRT_F32, RSQRT_F64, RSQRT_V128); + +// ============================================================================ +// OPCODE_RECIP +// ============================================================================ +// Altivec guarantees an error of < 1/4096 for vrefp +struct RECIP_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FRECPE(i.dest, i.src1); + } +}; +struct RECIP_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FRECPE(i.dest, i.src1); + } +}; +struct RECIP_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.FRECPE(i.dest.reg().S4(), i.src1.reg().S4()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RECIP, RECIP_F32, RECIP_F64, RECIP_V128); + +// ============================================================================ +// OPCODE_POW2 +// ============================================================================ +// TODO(benvanik): use approx here: +// https://jrfonseca.blogspot.com/2008/09/fast-sse2-pow-tables-or-polynomials.html +struct POW2_F32 : Sequence> { + static float32x4_t EmulatePow2(void*, float32x4_t src) { + float src_value; + vst1q_lane_f32(&src_value, src, 0); + const float result = std::exp2(src_value); + return vld1q_lane_f32(&result, src, 0); + } + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always(); + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.CallNativeSafe(reinterpret_cast(EmulatePow2)); + e.FMOV(i.dest, S0); + } +}; +struct POW2_F64 : Sequence> { + static float64x2_t EmulatePow2(void*, float64x2_t src) { + double src_value; + vst1q_lane_f64(&src_value, src, 0); + const double result = std::exp2(src_value); + return vld1q_lane_f64(&result, src, 0); + } + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always(); + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.CallNativeSafe(reinterpret_cast(EmulatePow2)); + e.FMOV(i.dest, D0); + } +}; +struct POW2_V128 : Sequence> { + static float32x4_t EmulatePow2(void*, float32x4_t src) { + alignas(16) float values[4]; + vst1q_f32(values, src); + for (size_t i = 0; i < 4; ++i) { + values[i] = std::exp2(values[i]); + } + return vld1q_f32(values); + } + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.CallNativeSafe(reinterpret_cast(EmulatePow2)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_POW2, POW2_F32, POW2_F64, POW2_V128); + +// ============================================================================ +// OPCODE_LOG2 +// ============================================================================ +// TODO(benvanik): use approx here: +// https://jrfonseca.blogspot.com/2008/09/fast-sse2-pow-tables-or-polynomials.html +// TODO(benvanik): this emulated fn destroys all xmm registers! don't do it! +struct LOG2_F32 : Sequence> { + static float32x4_t EmulateLog2(void*, float32x4_t src) { + float src_value; + vst1q_lane_f32(&src_value, src, 0); + float result = std::log2(src_value); + return vld1q_lane_f32(&result, src, 0); + } + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always(); + if (i.src1.is_constant) { + e.LDR(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + } else { + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + } + e.CallNativeSafe(reinterpret_cast(EmulateLog2)); + e.FMOV(i.dest, S0); + } +}; +struct LOG2_F64 : Sequence> { + static float64x2_t EmulateLog2(void*, float64x2_t src) { + double src_value; + vst1q_lane_f64(&src_value, src, 0); + double result = std::log2(src_value); + return vld1q_lane_f64(&result, src, 0); + } + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always(); + if (i.src1.is_constant) { + e.LDR(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + } else { + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + } + e.CallNativeSafe(reinterpret_cast(EmulateLog2)); + e.FMOV(i.dest, D0); + } +}; +struct LOG2_V128 : Sequence> { + static float32x4_t EmulateLog2(void*, float32x4_t src) { + alignas(16) float values[4]; + vst1q_f32(values, src); + for (size_t i = 0; i < 4; ++i) { + values[i] = std::log2(values[i]); + } + return vld1q_f32(values); + } + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.LDR(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + } else { + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + } + e.CallNativeSafe(reinterpret_cast(EmulateLog2)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOG2, LOG2_F32, LOG2_F64, LOG2_V128); + +// ============================================================================ +// OPCODE_DOT_PRODUCT_3 +// ============================================================================ +struct DOT_PRODUCT_3_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // https://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, QReg src1, QReg src2) { + e.FMUL(dest.toQ().S4(), src1.S4(), src2.S4()); + e.MOV(dest.toQ().Selem()[3], WZR); + e.FADDP(dest.toQ().S4(), dest.toQ().S4(), dest.toQ().S4()); + e.FADDP(dest.toS(), dest.toD().S2()); + + // Isolate lower lane + e.MOVI(Q0.D2(), RepImm(0b00'00'00'00)); + e.INS(Q0.Selem()[0], dest.toQ().Selem()[0]); + e.MOV(dest.toQ().B16(), Q0.B16()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_3, DOT_PRODUCT_3_V128); + +// ============================================================================ +// OPCODE_DOT_PRODUCT_4 +// ============================================================================ +struct DOT_PRODUCT_4_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // https://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, QReg src1, QReg src2) { + e.FMUL(dest.toQ().S4(), src1.S4(), src2.S4()); + e.FADDP(dest.toQ().S4(), dest.toQ().S4(), dest.toQ().S4()); + e.FADDP(dest.toS(), dest.toD().S2()); + + // Isolate lower lane + e.MOVI(Q0.D2(), RepImm(0b00'00'00'00)); + e.INS(Q0.Selem()[0], dest.toQ().Selem()[0]); + e.MOV(dest.toQ().B16(), Q0.B16()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_4, DOT_PRODUCT_4_V128); + +// ============================================================================ +// OPCODE_AND +// ============================================================================ +// TODO(benvanik): put dest/src1|2 together. +template +void EmitAndXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + e.AND(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int32_t constant) { + e.AND(dest_src, dest_src, constant); + }); +} +struct AND_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndXX(e, i); + } +}; +struct AND_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndXX(e, i); + } +}; +struct AND_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndXX(e, i); + } +}; +struct AND_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndXX(e, i); + } +}; +struct AND_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.AND(dest.B16(), src1.B16(), src2.B16()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_AND, AND_I8, AND_I16, AND_I32, AND_I64, AND_V128); + +// ============================================================================ +// OPCODE_AND_NOT +// ============================================================================ +template +void EmitAndNotXX(A64Emitter& e, const ARGS& i) { + if (i.src1.is_constant) { + // src1 constant. + auto temp = GetTempReg(e); + e.MOV(temp, i.src1.constant()); + e.BIC(i.dest, temp, i.src2); + } else if (i.src2.is_constant) { + // src2 constant. + if (i.dest.reg().index() == i.src1.reg().index()) { + auto temp = GetTempReg(e); + e.MOV(temp, ~i.src2.constant()); + e.AND(i.dest, i.dest, temp); + } else { + e.MOV(i.dest, i.src1); + auto temp = GetTempReg(e); + e.MOV(temp, ~i.src2.constant()); + e.AND(i.dest, i.dest, temp); + } + } else { + // neither are constant + e.BIC(i.dest, i.src1, i.src2); + } +} +struct AND_NOT_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndNotXX(e, i); + } +}; +struct AND_NOT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndNotXX(e, i); + } +}; +struct AND_NOT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndNotXX(e, i); + } +}; +struct AND_NOT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAndNotXX(e, i); + } +}; +struct AND_NOT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.BIC(dest.B16(), src2.B16(), src1.B16()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_AND_NOT, AND_NOT_I8, AND_NOT_I16, AND_NOT_I32, + AND_NOT_I64, AND_NOT_V128); + +// ============================================================================ +// OPCODE_OR +// ============================================================================ +// TODO(benvanik): put dest/src1|2 together. +template +void EmitOrXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + e.ORR(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int32_t constant) { + e.ORR(dest_src, dest_src, constant); + }); +} +struct OR_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitOrXX(e, i); + } +}; +struct OR_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitOrXX(e, i); + } +}; +struct OR_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitOrXX(e, i); + } +}; +struct OR_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitOrXX(e, i); + } +}; +struct OR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.ORR(dest.B16(), src1.B16(), src2.B16()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_OR, OR_I8, OR_I16, OR_I32, OR_I64, OR_V128); + +// ============================================================================ +// OPCODE_XOR +// ============================================================================ +// TODO(benvanik): put dest/src1|2 together. +template +void EmitXorXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + e.EOR(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int32_t constant) { + e.EOR(dest_src, dest_src, constant); + }); +} +struct XOR_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitXorXX(e, i); + } +}; +struct XOR_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitXorXX(e, i); + } +}; +struct XOR_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitXorXX(e, i); + } +}; +struct XOR_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitXorXX(e, i); + } +}; +struct XOR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.EOR(dest.B16(), src1.B16(), src2.B16()); + }); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_XOR, XOR_I8, XOR_I16, XOR_I32, XOR_I64, XOR_V128); + +// ============================================================================ +// OPCODE_NOT +// ============================================================================ +// TODO(benvanik): put dest/src1 together. +template +void EmitNotXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitUnaryOp( + e, i, [](A64Emitter& e, REG dest_src) { e.MVN(dest_src, dest_src); }); +} +struct NOT_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNotXX(e, i); + } +}; +struct NOT_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNotXX(e, i); + } +}; +struct NOT_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNotXX(e, i); + } +}; +struct NOT_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitNotXX(e, i); + } +}; +struct NOT_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.NOT(i.dest.reg().B16(), i.src1.reg().B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_NOT, NOT_I8, NOT_I16, NOT_I32, NOT_I64, NOT_V128); + +// ============================================================================ +// OPCODE_SHL +// ============================================================================ +// TODO(benvanik): optimize common shifts. +template +void EmitShlXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, WReg src) { + e.LSL(dest_src, dest_src, REG(src.index())); + }, + [](A64Emitter& e, REG dest_src, int8_t constant) { + e.LSL(dest_src, dest_src, constant); + }); +} +struct SHL_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShlXX(e, i); + } +}; +struct SHL_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShlXX(e, i); + } +}; +struct SHL_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShlXX(e, i); + } +}; +struct SHL_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShlXX(e, i); + } +}; +struct SHL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): native version (with shift magic). + if (i.src2.is_constant) { + e.MOV(e.GetNativeParam(1), i.src2.constant()); + } else { + e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); + } + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateShlV128)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static float32x4_t EmulateShlV128(void*, float32x4_t src1, uint8_t src2) { + // Almost all instances are shamt = 1, but non-constant. + // shamt is [0,7] + uint8_t shamt = src2 & 0x7; + alignas(16) vec128_t value; + vst1q_f32(reinterpret_cast(&value), src1); + for (int i = 0; i < 15; ++i) { + value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] << shamt) | + (value.u8[(i + 1) ^ 0x3] >> (8 - shamt)); + } + value.u8[15 ^ 0x3] = value.u8[15 ^ 0x3] << shamt; + return vld1q_f32(reinterpret_cast(&value)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SHL, SHL_I8, SHL_I16, SHL_I32, SHL_I64, SHL_V128); + +// ============================================================================ +// OPCODE_SHR +// ============================================================================ +// TODO(benvanik): optimize common shifts. +template +void EmitShrXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + e.LSR(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int8_t constant) { + e.LSR(dest_src, dest_src, constant); + }); +} +struct SHR_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShrXX(e, i); + } +}; +struct SHR_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShrXX(e, i); + } +}; +struct SHR_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShrXX(e, i); + } +}; +struct SHR_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitShrXX(e, i); + } +}; +struct SHR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(benvanik): native version (with shift magic). + if (i.src2.is_constant) { + e.MOV(e.GetNativeParam(1), i.src2.constant()); + } else { + e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); + } + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateShrV128)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static float32x4_t EmulateShrV128(void*, float32x4_t src1, uint8_t src2) { + // Almost all instances are shamt = 1, but non-constant. + // shamt is [0,7] + uint8_t shamt = src2 & 0x7; + alignas(16) vec128_t value; + vst1q_f32(reinterpret_cast(&value), src1); + for (int i = 15; i > 0; --i) { + value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] >> shamt) | + (value.u8[(i - 1) ^ 0x3] << (8 - shamt)); + } + value.u8[0 ^ 0x3] = value.u8[0 ^ 0x3] >> shamt; + return vld1q_f32(reinterpret_cast(&value)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SHR, SHR_I8, SHR_I16, SHR_I32, SHR_I64, SHR_V128); + +// ============================================================================ +// OPCODE_SHA +// ============================================================================ +// TODO(benvanik): optimize common shifts. +template +void EmitSarXX(A64Emitter& e, const ARGS& i) { + SEQ::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, REG dest_src, REG src) { + e.ASR(dest_src, dest_src, src); + }, + [](A64Emitter& e, REG dest_src, int8_t constant) { + e.ASR(dest_src, dest_src, constant); + }); +} +struct SHA_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSarXX(e, i); + } +}; +struct SHA_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSarXX(e, i); + } +}; +struct SHA_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSarXX(e, i); + } +}; +struct SHA_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSarXX(e, i); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SHA, SHA_I8, SHA_I16, SHA_I32, SHA_I64); + +// ============================================================================ +// OPCODE_ROTATE_LEFT +// ============================================================================ +// TODO(benvanik): put dest/src1 together, src2 in cl. +template +void EmitRotateLeftXX(A64Emitter& e, const ARGS& i) { + // ; rotate r1 left by r2, producing r0 + // ; (destroys r2) + // ; r1 = ABCDEFGH + // lslv r0, r1, r2 ; r0 = EFGH0000 + // mvn r2, r2 ; r2 = leftover bits + // lsrv r2, r1, r2 ; r2 = 0000ABCD + // orr r0, r0, r2 ; r0 = EFGHABCD + if (i.src1.is_constant) { + e.MOV(REG(0), i.src1.constant()); + } else { + e.MOV(REG(0), i.src1.reg()); + } + + if (i.src2.is_constant) { + e.MOV(REG(1), i.src2.constant()); + } else { + e.MOV(W0, i.src2.reg().toW()); + } + + e.LSLV(i.dest, REG(0), REG(1)); + e.MVN(REG(1), REG(1)); + e.LSRV(REG(1), REG(0), REG(1)); + e.ORR(i.dest, i.dest, REG(1)); +} +struct ROTATE_LEFT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitRotateLeftXX(e, i); + } +}; +struct ROTATE_LEFT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitRotateLeftXX(e, i); + } +}; +struct ROTATE_LEFT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitRotateLeftXX(e, i); + } +}; +struct ROTATE_LEFT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitRotateLeftXX(e, i); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ROTATE_LEFT, ROTATE_LEFT_I8, ROTATE_LEFT_I16, + ROTATE_LEFT_I32, ROTATE_LEFT_I64); + +// ============================================================================ +// OPCODE_BYTE_SWAP +// ============================================================================ +// TODO(benvanik): put dest/src1 together. +struct BYTE_SWAP_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitUnaryOp(e, i, [](A64Emitter& e, WReg dest_src) { + e.REV16(dest_src, dest_src); + }); + } +}; +struct BYTE_SWAP_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitUnaryOp(e, i, [](A64Emitter& e, WReg dest_src) { + e.REV32(dest_src.toX(), dest_src.toX()); + }); + } +}; +struct BYTE_SWAP_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitUnaryOp(e, i, [](A64Emitter& e, XReg dest_src) { + e.REV64(dest_src, dest_src); + }); + } +}; +struct BYTE_SWAP_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Reverse upper and lower 64-bit halfs + e.REV64(i.dest.reg().B16(), i.src1.reg().B16()); + // Reverse the 64-bit halfs themselves + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BYTE_SWAP, BYTE_SWAP_I16, BYTE_SWAP_I32, + BYTE_SWAP_I64, BYTE_SWAP_V128); + +// ============================================================================ +// OPCODE_CNTLZ +// Count leading zeroes +// ============================================================================ +struct CNTLZ_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // No 8bit lzcnt, so do 16 and sub 8. + e.UXTB(i.dest, i.src1); + e.CLZ(i.dest, i.dest); + e.SUB(i.dest.reg(), i.dest.reg(), 8); + } +}; +struct CNTLZ_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.UXTH(i.dest, i.src1); + e.CLZ(i.dest, i.dest); + } +}; +struct CNTLZ_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CLZ(i.dest, i.src1); + } +}; +struct CNTLZ_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CLZ(i.dest.reg().toX(), i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CNTLZ, CNTLZ_I8, CNTLZ_I16, CNTLZ_I32, CNTLZ_I64); + +// ============================================================================ +// OPCODE_SET_ROUNDING_MODE +// ============================================================================ +// Input: FPSCR (PPC format) +// Convert from PPC rounding mode to ARM +// PPC | ARM | +// 00 | 00 | nearest +// 01 | 11 | toward zero +// 10 | 01 | toward +infinity +// 11 | 10 | toward -infinity +static const uint8_t fpcr_table[] = { + 0b0'00, // |--|nearest + 0b0'11, // |--|toward zero + 0b0'01, // |--|toward +infinity + 0b0'10, // |--|toward -infinity + 0b1'00, // |FZ|nearest + 0b1'11, // |FZ|toward zero + 0b1'01, // |FZ|toward +infinity + 0b1'10, // |FZ|toward -infinity +}; +struct SET_ROUNDING_MODE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Low 3 bits are |Non-IEEE:1|RoundingMode:2| + // Non-IEEE bit is flush-to-zero + e.AND(W1, i.src1, 0b111); + + // Use the low 3 bits as an index into a LUT + e.ADRL(X0, fpcr_table); + e.LDRB(W0, X0, W1); + + // Replace FPCR bits with new value + e.MRS(X1, SystemReg::FPCR); + e.BFI(X1, X0, 54, 3); + e.MSR(SystemReg::FPCR, X1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SET_ROUNDING_MODE, SET_ROUNDING_MODE_I32); + +// Include anchors to other sequence sources so they get included in the build. +extern volatile int anchor_control; +static int anchor_control_dest = anchor_control; + +extern volatile int anchor_memory; +static int anchor_memory_dest = anchor_memory; + +extern volatile int anchor_vector; +static int anchor_vector_dest = anchor_vector; + +bool SelectSequence(A64Emitter* e, const Instr* i, const Instr** new_tail) { + const InstrKey key(i); + auto it = sequence_table.find(key); + if (it != sequence_table.end()) { + if (it->second(*e, i)) { + *new_tail = i->next; + return true; + } + } + XELOGE("No sequence match for variant {}", i->opcode->name); + return false; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_sequences.h b/a64_sequences.h new file mode 100644 index 000000000..b47382633 --- /dev/null +++ b/a64_sequences.h @@ -0,0 +1,51 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ +#define XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ + +#include "xenia/cpu/hir/instr.h" + +#include + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Emitter; + +typedef bool (*SequenceSelectFn)(A64Emitter&, const hir::Instr*); +extern std::unordered_map sequence_table; + +template +bool Register() { + sequence_table.insert({T::head_key(), T::Select}); + return true; +} + +template +static bool Register() { + bool b = true; + b = b && Register(); // Call the above function + b = b && Register(); // Call ourself again (recursively) + return b; +} +#define EMITTER_OPCODE_TABLE(name, ...) \ + const auto A64_INSTR_##name = Register<__VA_ARGS__>(); + +bool SelectSequence(A64Emitter* e, const hir::Instr* i, + const hir::Instr** new_tail); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ diff --git a/a64_stack_layout.h b/a64_stack_layout.h new file mode 100644 index 000000000..87dc3379c --- /dev/null +++ b/a64_stack_layout.h @@ -0,0 +1,135 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ +#define XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ + +#include "xenia/base/vec128.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class StackLayout { + public: + /** + * Stack Layout + * ---------------------------- + * NOTE: stack must always be 16b aligned. + * + * Thunk stack: + * Non-Volatile Volatile + * +------------------+------------------+ + * | arg temp, 3 * 8 | arg temp, 3 * 8 | rsp + 0x000 + * | | | + * | | | + * +------------------+------------------+ + * | rbx | (unused) | rsp + 0x018 + * +------------------+------------------+ + * | rbp | rcx | rsp + 0x020 + * +------------------+------------------+ + * | rcx (Win32) | rdx | rsp + 0x028 + * +------------------+------------------+ + * | rsi (Win32) | rsi (Linux) | rsp + 0x030 + * +------------------+------------------+ + * | rdi (Win32) | rdi (Linux) | rsp + 0x038 + * +------------------+------------------+ + * | r12 | r8 | rsp + 0x040 + * +------------------+------------------+ + * | r13 | r9 | rsp + 0x048 + * +------------------+------------------+ + * | r14 | r10 | rsp + 0x050 + * +------------------+------------------+ + * | r15 | r11 | rsp + 0x058 + * +------------------+------------------+ + * | xmm6 (Win32) | (unused) | rsp + 0x060 + * | | | + * +------------------+------------------+ + * | xmm7 (Win32) | xmm1 | rsp + 0x070 + * | | | + * +------------------+------------------+ + * | xmm8 (Win32) | xmm2 | rsp + 0x080 + * | | | + * +------------------+------------------+ + * | xmm9 (Win32) | xmm3 | rsp + 0x090 + * | | | + * +------------------+------------------+ + * | xmm10 (Win32) | xmm4 | rsp + 0x0A0 + * | | | + * +------------------+------------------+ + * | xmm11 (Win32) | xmm5 | rsp + 0x0B0 + * | | | + * +------------------+------------------+ + * | xmm12 (Win32) | (unused) | rsp + 0x0C0 + * | | | + * +------------------+------------------+ + * | xmm13 (Win32) | (unused) | rsp + 0x0D0 + * | | | + * +------------------+------------------+ + * | xmm14 (Win32) | (unused) | rsp + 0x0E0 + * | | | + * +------------------+------------------+ + * | xmm15 (Win32) | (unused) | rsp + 0x0F0 + * | | | + * +------------------+------------------+ + * | (return address) | (return address) | rsp + 0x100 + * +------------------+------------------+ + * | (rcx home) | (rcx home) | rsp + 0x108 + * +------------------+------------------+ + * | (rdx home) | (rdx home) | rsp + 0x110 + * +------------------+------------------+ + */ + XEPACKEDSTRUCT(Thunk, { + uint64_t arg_temp[3]; + uint64_t r[19]; + vec128_t xmm[31]; + }); + static_assert(sizeof(Thunk) % 16 == 0, + "sizeof(Thunk) must be a multiple of 16!"); + static const size_t THUNK_STACK_SIZE = sizeof(Thunk); + + /** + * + * + * Guest stack: + * +------------------+ + * | arg temp, 3 * 8 | rsp + 0 + * | | + * | | + * +------------------+ + * | scratch, 48b | rsp + 32 + * | | + * +------------------+ + * | rcx / context | rsp + 80 + * +------------------+ + * | guest ret addr | rsp + 88 + * +------------------+ + * | call ret addr | rsp + 96 + * +------------------+ + * ... locals ... + * +------------------+ + * | (return address) | + * +------------------+ + * + */ + static const size_t GUEST_STACK_SIZE = 96; + static const size_t GUEST_CTX_HOME = 80; + static const size_t GUEST_RET_ADDR = 88; + static const size_t GUEST_CALL_RET_ADDR = 96; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ diff --git a/a64_tracers.cc b/a64_tracers.cc new file mode 100644 index 000000000..146f50982 --- /dev/null +++ b/a64_tracers.cc @@ -0,0 +1,225 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_tracers.h" + +#include + +#include "xenia/base/logging.h" +#include "xenia/base/vec128.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/processor.h" +#include "xenia/cpu/thread_state.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +#define ITRACE 0 +#define DTRACE 0 + +#define TARGET_THREAD 0 + +bool trace_enabled = true; + +#define THREAD_MATCH \ + (!TARGET_THREAD || thread_state->thread_id() == TARGET_THREAD) +#define IFLUSH() +#define IPRINT(s) \ + if (trace_enabled && THREAD_MATCH) \ + xe::logging::AppendLogLine(xe::LogLevel::Debug, 't', s) +#define DFLUSH() +#define DPRINT(...) \ + if (trace_enabled && THREAD_MATCH) \ + xe::logging::AppendLogLineFormat(xe::LogLevel::Debug, 't', __VA_ARGS__) + +uint32_t GetTracingMode() { + uint32_t mode = 0; +#if ITRACE + mode |= TRACING_INSTR; +#endif // ITRACE +#if DTRACE + mode |= TRACING_DATA; +#endif // DTRACE + return mode; +} + +void TraceString(void* raw_context, const char* str) { + auto thread_state = *reinterpret_cast(raw_context); + IPRINT(str); + IFLUSH(); +} + +void TraceContextLoadI8(void* raw_context, uint64_t offset, uint8_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i8 +{}\n", (int8_t)value, value, offset); +} +void TraceContextLoadI16(void* raw_context, uint64_t offset, uint16_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i16 +{}\n", (int16_t)value, value, offset); +} +void TraceContextLoadI32(void* raw_context, uint64_t offset, uint32_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i32 +{}\n", (int32_t)value, value, offset); +} +void TraceContextLoadI64(void* raw_context, uint64_t offset, uint64_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i64 +{}\n", (int64_t)value, value, offset); +} +void TraceContextLoadF32(void* raw_context, uint64_t offset, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx f32 +{}\n", xe::m128_f32<0>(value), + xe::m128_i32<0>(value), offset); +} +void TraceContextLoadF64(void* raw_context, uint64_t offset, + const double* value) { + auto thread_state = *reinterpret_cast(raw_context); + // auto v = _mm_loadu_pd(value); + auto v = vld1q_f64(value); + DPRINT("{} ({:X}) = ctx f64 +{}\n", xe::m128_f64<0>(v), xe::m128_i64<0>(v), + offset); +} +void TraceContextLoadV128(void* raw_context, uint64_t offset, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("[{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}] = ctx v128 +{}\n", + xe::m128_f32<0>(value), xe::m128_f32<1>(value), xe::m128_f32<2>(value), + xe::m128_f32<3>(value), xe::m128_i32<0>(value), xe::m128_i32<1>(value), + xe::m128_i32<2>(value), xe::m128_i32<3>(value), offset); +} + +void TraceContextStoreI8(void* raw_context, uint64_t offset, uint8_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("ctx i8 +{} = {} ({:X})\n", offset, (int8_t)value, value); +} +void TraceContextStoreI16(void* raw_context, uint64_t offset, uint16_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("ctx i16 +{} = {} ({:X})\n", offset, (int16_t)value, value); +} +void TraceContextStoreI32(void* raw_context, uint64_t offset, uint32_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("ctx i32 +{} = {} ({:X})\n", offset, (int32_t)value, value); +} +void TraceContextStoreI64(void* raw_context, uint64_t offset, uint64_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("ctx i64 +{} = {} ({:X})\n", offset, (int64_t)value, value); +} +void TraceContextStoreF32(void* raw_context, uint64_t offset, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("ctx f32 +{} = {} ({:X})\n", offset, xe::m128_f32<0>(value), + xe::m128_i32<0>(value)); +} +void TraceContextStoreF64(void* raw_context, uint64_t offset, + const double* value) { + auto thread_state = *reinterpret_cast(raw_context); + // auto v = _mm_loadu_pd(value); + auto v = vld1q_f64(value); + DPRINT("ctx f64 +{} = {} ({:X})\n", offset, xe::m128_f64<0>(v), + xe::m128_i64<0>(v)); +} +void TraceContextStoreV128(void* raw_context, uint64_t offset, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("ctx v128 +{} = [{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}]\n", + offset, xe::m128_f32<0>(value), xe::m128_f32<1>(value), + xe::m128_f32<2>(value), xe::m128_f32<3>(value), xe::m128_i32<0>(value), + xe::m128_i32<1>(value), xe::m128_i32<2>(value), + xe::m128_i32<3>(value)); +} + +void TraceMemoryLoadI8(void* raw_context, uint32_t address, uint8_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i8 {:08X}\n", (int8_t)value, value, address); +} +void TraceMemoryLoadI16(void* raw_context, uint32_t address, uint16_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i16 {:08X}\n", (int16_t)value, value, address); +} +void TraceMemoryLoadI32(void* raw_context, uint32_t address, uint32_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i32 {:08X}\n", (int32_t)value, value, address); +} +void TraceMemoryLoadI64(void* raw_context, uint32_t address, uint64_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i64 {:08X}\n", (int64_t)value, value, address); +} +void TraceMemoryLoadF32(void* raw_context, uint32_t address, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.f32 {:08X}\n", xe::m128_f32<0>(value), + xe::m128_i32<0>(value), address); +} +void TraceMemoryLoadF64(void* raw_context, uint32_t address, + float64x2_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.f64 {:08X}\n", xe::m128_f64<0>(value), + xe::m128_i64<0>(value), address); +} +void TraceMemoryLoadV128(void* raw_context, uint32_t address, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT( + "[{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}] = load.v128 {:08X}\n", + xe::m128_f32<0>(value), xe::m128_f32<1>(value), xe::m128_f32<2>(value), + xe::m128_f32<3>(value), xe::m128_i32<0>(value), xe::m128_i32<1>(value), + xe::m128_i32<2>(value), xe::m128_i32<3>(value), address); +} + +void TraceMemoryStoreI8(void* raw_context, uint32_t address, uint8_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("store.i8 {:08X} = {} ({:X})\n", address, (int8_t)value, value); +} +void TraceMemoryStoreI16(void* raw_context, uint32_t address, uint16_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("store.i16 {:08X} = {} ({:X})\n", address, (int16_t)value, value); +} +void TraceMemoryStoreI32(void* raw_context, uint32_t address, uint32_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("store.i32 {:08X} = {} ({:X})\n", address, (int32_t)value, value); +} +void TraceMemoryStoreI64(void* raw_context, uint32_t address, uint64_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("store.i64 {:08X} = {} ({:X})\n", address, (int64_t)value, value); +} +void TraceMemoryStoreF32(void* raw_context, uint32_t address, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("store.f32 {:08X} = {} ({:X})\n", address, xe::m128_f32<0>(value), + xe::m128_i32<0>(value)); +} +void TraceMemoryStoreF64(void* raw_context, uint32_t address, + float64x2_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("store.f64 {:08X} = {} ({:X})\n", address, xe::m128_f64<0>(value), + xe::m128_i64<0>(value)); +} +void TraceMemoryStoreV128(void* raw_context, uint32_t address, + float32x4_t value) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT( + "store.v128 {:08X} = [{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}]\n", + address, xe::m128_f32<0>(value), xe::m128_f32<1>(value), + xe::m128_f32<2>(value), xe::m128_f32<3>(value), xe::m128_i32<0>(value), + xe::m128_i32<1>(value), xe::m128_i32<2>(value), xe::m128_i32<3>(value)); +} + +void TraceMemset(void* raw_context, uint32_t address, uint8_t value, + uint32_t length) { + auto thread_state = *reinterpret_cast(raw_context); + DPRINT("memset {:08X}-{:08X} ({}) = {:02X}", address, address + length, + length, value); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/a64_tracers.h b/a64_tracers.h new file mode 100644 index 000000000..62b740356 --- /dev/null +++ b/a64_tracers.h @@ -0,0 +1,82 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ +#define XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ + +#include +#include + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { +class A64Emitter; + +enum TracingMode { + TRACING_INSTR = (1 << 1), + TRACING_DATA = (1 << 2), +}; + +uint32_t GetTracingMode(); +inline bool IsTracingInstr() { return (GetTracingMode() & TRACING_INSTR) != 0; } +inline bool IsTracingData() { return (GetTracingMode() & TRACING_DATA) != 0; } + +void TraceString(void* raw_context, const char* str); + +void TraceContextLoadI8(void* raw_context, uint64_t offset, uint8_t value); +void TraceContextLoadI16(void* raw_context, uint64_t offset, uint16_t value); +void TraceContextLoadI32(void* raw_context, uint64_t offset, uint32_t value); +void TraceContextLoadI64(void* raw_context, uint64_t offset, uint64_t value); +void TraceContextLoadF32(void* raw_context, uint64_t offset, float32x4_t value); +void TraceContextLoadF64(void* raw_context, uint64_t offset, + const double* value); +void TraceContextLoadV128(void* raw_context, uint64_t offset, + float32x4_t value); + +void TraceContextStoreI8(void* raw_context, uint64_t offset, uint8_t value); +void TraceContextStoreI16(void* raw_context, uint64_t offset, uint16_t value); +void TraceContextStoreI32(void* raw_context, uint64_t offset, uint32_t value); +void TraceContextStoreI64(void* raw_context, uint64_t offset, uint64_t value); +void TraceContextStoreF32(void* raw_context, uint64_t offset, + float32x4_t value); +void TraceContextStoreF64(void* raw_context, uint64_t offset, + const double* value); +void TraceContextStoreV128(void* raw_context, uint64_t offset, + float32x4_t value); + +void TraceMemoryLoadI8(void* raw_context, uint32_t address, uint8_t value); +void TraceMemoryLoadI16(void* raw_context, uint32_t address, uint16_t value); +void TraceMemoryLoadI32(void* raw_context, uint32_t address, uint32_t value); +void TraceMemoryLoadI64(void* raw_context, uint32_t address, uint64_t value); +void TraceMemoryLoadF32(void* raw_context, uint32_t address, float32x4_t value); +void TraceMemoryLoadF64(void* raw_context, uint32_t address, float64x2_t value); +void TraceMemoryLoadV128(void* raw_context, uint32_t address, + float32x4_t value); + +void TraceMemoryStoreI8(void* raw_context, uint32_t address, uint8_t value); +void TraceMemoryStoreI16(void* raw_context, uint32_t address, uint16_t value); +void TraceMemoryStoreI32(void* raw_context, uint32_t address, uint32_t value); +void TraceMemoryStoreI64(void* raw_context, uint32_t address, uint64_t value); +void TraceMemoryStoreF32(void* raw_context, uint32_t address, + float32x4_t value); +void TraceMemoryStoreF64(void* raw_context, uint32_t address, + float64x2_t value); +void TraceMemoryStoreV128(void* raw_context, uint32_t address, + float32x4_t value); + +void TraceMemset(void* raw_context, uint32_t address, uint8_t value, + uint32_t length); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ diff --git a/a64_util.h b/a64_util.h new file mode 100644 index 000000000..e3a34ac00 --- /dev/null +++ b/a64_util.h @@ -0,0 +1,25 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2024 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_UTIL_H_ +#define XENIA_CPU_BACKEND_A64_A64_UTIL_H_ + +#include "xenia/base/vec128.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 {} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_UTIL_H_ diff --git a/premake5.lua b/premake5.lua index 01a55e3c2..32b2d51a0 100644 --- a/premake5.lua +++ b/premake5.lua @@ -19,6 +19,12 @@ project("xenia-cpu-backend-a64") defines({ }) + disablewarnings({ + -- Silence errors in oaknut + "4146", -- unary minus operator applied to unsigned type, result still unsigned + "4267" -- 'initializing': conversion from 'size_t' to 'uint32_t', possible loss of data + }) + includedirs({ project_root.."/third_party/oaknut/include", }) From d961eb299364802453f9aea58123b8fe7375bb2a Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 14:26:29 -0700 Subject: [PATCH 004/179] [a64] Fix `BYTE_SWAP_V128` This just reverses the bytes of 32-bit values, not reverse the whole vector. --- a64_sequences.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index a5c20802b..21fcde1de 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2626,9 +2626,7 @@ struct BYTE_SWAP_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // Reverse upper and lower 64-bit halfs - e.REV64(i.dest.reg().B16(), i.src1.reg().B16()); - // Reverse the 64-bit halfs themselves - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); + e.REV32(i.dest.reg().B16(), i.src1.reg().B16()); } }; EMITTER_OPCODE_TABLE(OPCODE_BYTE_SWAP, BYTE_SWAP_I16, BYTE_SWAP_I32, From a741404a961698fb8fb97e17a2be99caecb41322 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 15:49:14 -0700 Subject: [PATCH 005/179] [a64] Implement `OPCODE_EXTRACT` --- a64_seq_vector.cc | 69 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index ff313aefb..a1c69ccfe 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -575,15 +575,78 @@ EMITTER_OPCODE_TABLE(OPCODE_INSERT, INSERT_I8, INSERT_I16, INSERT_I32); // This can be a single broadcast. struct EXTRACT_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.UMOV(i.dest, i.src1.reg().Belem()[VEC128_B(i.src2.constant())]); + } else { + // Fixup index + e.EOR(W0, i.src2, 0b11); + e.AND(W0, W0, 0x1F); + e.DUP(Q0.B16(), W0); + // Byte-table lookup + e.TBL(Q0.B16(), List{i.src1.reg().B16()}, Q0.B16()); + // Get lowest element + e.UMOV(i.dest, Q0.Belem()[0]); + } + } }; struct EXTRACT_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.UMOV(i.dest, i.src1.reg().Helem()[VEC128_W(i.src2.constant())]); + } else { + // Fixup index + e.EOR(W0, i.src2, 0b01); + e.LSL(W0, W0, 1); + + // Replicate index as byte + e.MOV(W1, 0x01'01); + e.MUL(W0, W0, W1); + + // Byte indices + e.ADD(W0, W0, 0x01'00); + e.UXTH(W0, W0); + + // Replicate byte indices + e.DUP(Q0.H8(), W0); + // Byte-table lookup + e.TBL(Q0.B16(), List{i.src1.reg().B16()}, Q0.B16()); + // Get lowest element + e.UMOV(i.dest, Q0.Helem()[0]); + } + } }; struct EXTRACT_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + static const vec128_t extract_table_32[4] = { + vec128b(3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + vec128b(7, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + vec128b(11, 10, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + vec128b(15, 14, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + }; + if (i.src2.is_constant) { + e.UMOV(i.dest, i.src1.reg().Selem()[VEC128_D(i.src2.constant())]); + } else { + QReg src1 = i.src1.reg(); + if (i.src1.is_constant) { + src1 = Q1; + e.LoadConstantV(src1, i.src1.constant()); + } + + e.AND(X0, i.src2.reg().toX(), 0b11); + e.LSL(X0, X0, 4); + + e.MOVP2R(X1, extract_table_32); + e.LDR(Q0, X1, X0); + + // Byte-table lookup + e.TBL(Q0.B16(), List{src1.B16()}, Q0.B16()); + // Get lowest element + e.UMOV(i.dest, Q0.Selem()[0]); + } + } }; EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); From 43ded872779cc3f6d4ab43d3c97983d873ce8c25 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 15:54:09 -0700 Subject: [PATCH 006/179] [a64] Implement `OPCODE_SPLAT` --- a64_seq_vector.cc | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index a1c69ccfe..3a918ff43 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -655,17 +655,43 @@ EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); // ============================================================================ // Copy a value into all elements of a vector struct SPLAT_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.MOV(W0, i.src1.constant()); + e.DUP(Q0.B16(), W0); + } else { + e.DUP(Q0.B16(), i.src1); + } + } }; struct SPLAT_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.MOV(W0, i.src1.constant()); + e.DUP(Q0.H8(), W0); + } else { + e.DUP(Q0.H8(), i.src1); + } + } }; struct SPLAT_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.MOV(W0, i.src1.constant()); + e.DUP(Q0.S4(), W0); + } else { + e.DUP(Q0.S4(), i.src1); + } + } }; struct SPLAT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - {} + if (i.src1.is_constant) { + e.MOV(W0, i.src1.value->constant.i32); + e.DUP(Q0.S4(), W0); + } else { + e.DUP(Q0.S4(), i.src1.reg().toQ().Selem()[0]); + } } }; EMITTER_OPCODE_TABLE(OPCODE_SPLAT, SPLAT_I8, SPLAT_I16, SPLAT_I32, SPLAT_F32); From 34138478b216e84b485b46866b6c1966eddf01db Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 15:59:31 -0700 Subject: [PATCH 007/179] [a64] Implement `OPCODE_INSERT` --- a64_seq_vector.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 3a918ff43..9317e233a 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -547,21 +547,21 @@ struct INSERT_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.is_constant); - // e.vpinsrb(i.dest, i.src3.reg().cvt32(), i.src2.constant() ^ 0x3); + e.MOV(i.dest.reg().Belem()[i.src2.constant() ^ 0x3], i.src3.reg()); } }; struct INSERT_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.is_constant); - // e.vpinsrw(i.dest, i.src3.reg().cvt32(), i.src2.constant() ^ 0x1); + e.MOV(i.dest.reg().Helem()[i.src2.constant() ^ 0x1], i.src3.reg()); } }; struct INSERT_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.is_constant); - // e.vpinsrd(i.dest, i.src3, i.src2.constant()); + e.MOV(i.dest.reg().Selem()[i.src2.constant()], i.src3.reg()); } }; EMITTER_OPCODE_TABLE(OPCODE_INSERT, INSERT_I8, INSERT_I16, INSERT_I32); From c6f360271386ef4bde3d7a09a2ec9ceb49b2dbe8 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 16:04:39 -0700 Subject: [PATCH 008/179] [a64] Implement `OPCODE_LOAD_VECTOR_SHL` --- a64_seq_vector.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 9317e233a..02300072c 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -154,15 +154,12 @@ struct LOAD_VECTOR_SHL_I8 if (i.src1.is_constant) { auto sh = i.src1.constant(); assert_true(sh < xe::countof(lvsl_table)); - // e.mov(e.rax, (uintptr_t)&lvsl_table[sh]); - // e.vmovaps(i.dest, e.ptr[e.rax]); + e.MOVP2R(X0, &lvsl_table[sh]); + e.LDR(i.dest, X0); } else { - // TODO(benvanik): find a cheaper way of doing this. - // e.movzx(e.rdx, i.src1); - // e.and_(e.dx, 0xF); - // e.shl(e.dx, 4); - // e.mov(e.rax, (uintptr_t)lvsl_table); - // e.vmovaps(i.dest, e.ptr[e.rax + e.rdx]); + e.MOVP2R(X0, lvsl_table); + e.AND(X1, i.src1.reg().toX(), 0xf); + e.LDR(i.dest, X0, X1, IndexExt::LSL, 4); } } }; From 2c565e2d65aac558432b7769728ddf399acbde40 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 16:05:47 -0700 Subject: [PATCH 009/179] [a64] Implement `OPCODE_LOAD_VECTOR_SHR` --- a64_seq_vector.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 02300072c..90bd3db33 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -192,15 +192,12 @@ struct LOAD_VECTOR_SHR_I8 if (i.src1.is_constant) { auto sh = i.src1.constant(); assert_true(sh < xe::countof(lvsr_table)); - // e.mov(e.rax, (uintptr_t)&lvsr_table[sh]); - // e.vmovaps(i.dest, e.ptr[e.rax]); + e.MOVP2R(X0, &lvsr_table[sh]); + e.LDR(i.dest, X0); } else { - // TODO(benvanik): find a cheaper way of doing this. - // e.movzx(e.rdx, i.src1); - // e.and_(e.dx, 0xF); - // e.shl(e.dx, 4); - // e.mov(e.rax, (uintptr_t)lvsr_table); - // e.vmovaps(i.dest, e.ptr[e.rax + e.rdx]); + e.MOVP2R(X0, lvsr_table); + e.AND(X1, i.src1.reg().toX(), 0xf); + e.LDR(i.dest, X0, X1, IndexExt::LSL, 4); } } }; From 041e6a1286e8bb5c67273121c3a1a40b66074eae Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 16:13:50 -0700 Subject: [PATCH 010/179] [a64] Implement `OPCODE_PACK`(D3DCOLOR) --- a64_seq_vector.cc | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 90bd3db33..69c0e3e21 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -774,7 +774,32 @@ struct PACK : Sequence> { break; } } - static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) {} + static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + QReg src = i.src1; + if (i.src1.is_constant) { + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } + // Saturate to [3,3....] so that only values between 3...[00] and 3...[FF] + // are valid - max before min to pack NaN as zero (5454082B is heavily + // affected by the order - packs 0xFFFFFFFF in matrix code to get a 0 + // constant). + e.MOVP2R(X0, e.GetVConstPtr(V3333)); + e.LDR(Q0, X0); + e.FMAX(i.dest.reg().S4(), i.dest.reg().S4(), Q0.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VPackD3DCOLORSat)); + e.LDR(Q0, X0); + e.FMIN(i.dest.reg().S4(), src.S4(), Q0.S4()); + // Extract bytes. + // RGBA (XYZW) -> ARGB (WXYZ) + // w = ((src1.uw & 0xFF) << 24) | ((src1.ux & 0xFF) << 16) | + // ((src1.uy & 0xFF) << 8) | (src1.uz & 0xFF) + e.MOVP2R(X0, e.GetVConstPtr(VPackD3DCOLOR)); + e.LDR(Q0, X0); + e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q0.B16()); + } static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) {} static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) {} static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) {} From 7c02579e5661c28dfef95329addacc063d85ff3d Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 17:55:28 -0700 Subject: [PATCH 011/179] [a64] Implement `OPCODE_VECTOR_SHA` --- a64_seq_vector.cc | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 69c0e3e21..fb13ad044 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -486,6 +486,22 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHR, VECTOR_SHR_V128); // ============================================================================ // OPCODE_VECTOR_SHA // ============================================================================ +template ::value, int> = 0> +static uint8x16_t EmulateVectorShr(void*, uint8x16_t src1, uint8x16_t src2) { + alignas(16) T value[16 / sizeof(T)]; + alignas(16) T shamt[16 / sizeof(T)]; + + // Load SSE registers into a C array. + vst1q_u8(reinterpret_cast(value), src1); + vst1q_u8(reinterpret_cast(shamt), src2); + + for (size_t i = 0; i < (16 / sizeof(T)); ++i) { + value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1)); + } + + // Store result and return it. + return vld1q_f32(value); +} struct VECTOR_SHA_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { @@ -505,7 +521,29 @@ struct VECTOR_SHA_V128 } } - static void EmitInt8(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt8(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 16 - n; ++n) { + if (shamt.u8[n] != shamt.u8[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use SSHR + e.SSHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); + return; + } + e.LDR(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.LDR(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} From 6a557459e84efcd258ff8971ab0449ed535f2ac0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 17:57:49 -0700 Subject: [PATCH 012/179] [a64] Implement `OPCODE_{SHR,SHA}` --- a64_sequences.cc | 107 +++++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 36 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 21fcde1de..2f7cb96e3 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2440,36 +2440,52 @@ EMITTER_OPCODE_TABLE(OPCODE_SHL, SHL_I8, SHL_I16, SHL_I32, SHL_I64, SHL_V128); // ============================================================================ // OPCODE_SHR // ============================================================================ -// TODO(benvanik): optimize common shifts. -template -void EmitShrXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.LSR(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int8_t constant) { - e.LSR(dest_src, dest_src, constant); - }); -} struct SHR_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShrXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.LSR(dest_src, dest_src, src); + }, + [](A64Emitter& e, WReg dest_src, int8_t constant) { + e.LSR(dest_src, dest_src, constant); + }); } }; -struct SHR_I16 : Sequence> { +struct SHR_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShrXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.LSR(dest_src, dest_src, src); + }, + [](A64Emitter& e, WReg dest_src, int8_t constant) { + e.LSR(dest_src, dest_src, constant); + }); } }; -struct SHR_I32 : Sequence> { +struct SHR_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShrXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.LSR(dest_src, dest_src, src); + }, + [](A64Emitter& e, WReg dest_src, int8_t constant) { + e.LSR(dest_src, dest_src, constant); + }); } }; -struct SHR_I64 : Sequence> { +struct SHR_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShrXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, XReg dest_src, WReg src) { + e.LSR(dest_src, dest_src, src.toX()); + }, + [](A64Emitter& e, XReg dest_src, int8_t constant) { + e.LSR(dest_src, dest_src, constant); + }); } }; struct SHR_V128 : Sequence> { @@ -2503,36 +2519,55 @@ EMITTER_OPCODE_TABLE(OPCODE_SHR, SHR_I8, SHR_I16, SHR_I32, SHR_I64, SHR_V128); // ============================================================================ // OPCODE_SHA // ============================================================================ -// TODO(benvanik): optimize common shifts. -template -void EmitSarXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.ASR(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int8_t constant) { - e.ASR(dest_src, dest_src, constant); - }); -} struct SHA_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSarXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.SXTB(dest_src, dest_src); + e.ASR(dest_src, dest_src, src); + }, + [](A64Emitter& e, WReg dest_src, int8_t constant) { + e.SXTB(dest_src, dest_src); + e.ASR(dest_src, dest_src, constant); + }); } }; struct SHA_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSarXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.SXTH(dest_src, dest_src); + e.ASR(dest_src, dest_src, src); + }, + [](A64Emitter& e, WReg dest_src, int8_t constant) { + e.ASR(dest_src, dest_src, constant); + }); } }; struct SHA_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSarXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, WReg dest_src, WReg src) { + e.ASR(dest_src, dest_src, src); + }, + [](A64Emitter& e, WReg dest_src, int8_t constant) { + e.ASR(dest_src, dest_src, constant); + }); } }; -struct SHA_I64 : Sequence> { +struct SHA_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSarXX(e, i); + Sequence::EmitAssociativeBinaryOp( + e, i, + [](A64Emitter& e, XReg dest_src, WReg src) { + e.ASR(dest_src, dest_src, src.toX()); + }, + [](A64Emitter& e, XReg dest_src, int8_t constant) { + e.ASR(dest_src, dest_src, constant); + }); } }; EMITTER_OPCODE_TABLE(OPCODE_SHA, SHA_I8, SHA_I16, SHA_I32, SHA_I64); From a3b5ea2124bcfa720209015b1640b0ca5e2073cd Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 18:07:01 -0700 Subject: [PATCH 013/179] [a64] Fix StackLayout Wrong register index and vector-register size --- a64_backend.cc | 2 +- a64_stack_layout.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 076c1d487..d1780980b 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -418,7 +418,7 @@ void A64ThunkEmitter::EmitSaveVolatileRegs() { STR(X15, XSP, offsetof(StackLayout::Thunk, r[15])); STR(X16, XSP, offsetof(StackLayout::Thunk, r[16])); STR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); - STR(X17, XSP, offsetof(StackLayout::Thunk, r[18])); + STR(X18, XSP, offsetof(StackLayout::Thunk, r[18])); STR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); STR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); diff --git a/a64_stack_layout.h b/a64_stack_layout.h index 87dc3379c..02db19db2 100644 --- a/a64_stack_layout.h +++ b/a64_stack_layout.h @@ -91,7 +91,7 @@ class StackLayout { XEPACKEDSTRUCT(Thunk, { uint64_t arg_temp[3]; uint64_t r[19]; - vec128_t xmm[31]; + vec128_t xmm[32]; }); static_assert(sizeof(Thunk) % 16 == 0, "sizeof(Thunk) must be a multiple of 16!"); From f63eb72dc3742824a67c57199a5a48751c9be42f Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 19:00:05 -0700 Subject: [PATCH 014/179] [a64] Fix Guest-To-Host native calls These calls need to preserve and restore the `lr` register. Unit tests all run now! --- a64_backend.cc | 12 ++++++++---- a64_emitter.cc | 8 ++++++++ a64_stack_layout.h | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index d1780980b..6320cf4d7 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -273,10 +273,10 @@ HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { } GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { - // rcx = target function - // rdx = arg0 - // r8 = arg1 - // r9 = arg2 + // X0 = target function + // X1 = arg0 + // X2 = arg1 + // X3 = arg2 struct _code_offsets { size_t prolog; @@ -420,6 +420,8 @@ void A64ThunkEmitter::EmitSaveVolatileRegs() { STR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); STR(X18, XSP, offsetof(StackLayout::Thunk, r[18])); + STR(X30, XSP, offsetof(StackLayout::Thunk, r[19])); + STR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); STR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); STR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); @@ -477,6 +479,8 @@ void A64ThunkEmitter::EmitLoadVolatileRegs() { LDR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); LDR(X18, XSP, offsetof(StackLayout::Thunk, r[18])); + LDR(X30, XSP, offsetof(StackLayout::Thunk, r[19])); + LDR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); LDR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); LDR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); diff --git a/a64_emitter.cc b/a64_emitter.cc index 6010d762f..5a905d9b8 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -628,7 +628,15 @@ void A64Emitter::CallNativeSafe(void* fn) { // call(rax); MOV(X0, reinterpret_cast(fn)); MOV(X16, reinterpret_cast(thunk)); + + // Preserve frame and link register + STP(X29, X30, XSP, POST_INDEXED, -16); + BLR(X16); + + // Restore frame and link register + LDP(X29, X30, XSP, PRE_INDEXED, 16); + // X0 = host return } diff --git a/a64_stack_layout.h b/a64_stack_layout.h index 02db19db2..64deece50 100644 --- a/a64_stack_layout.h +++ b/a64_stack_layout.h @@ -90,7 +90,7 @@ class StackLayout { */ XEPACKEDSTRUCT(Thunk, { uint64_t arg_temp[3]; - uint64_t r[19]; + uint64_t r[20]; vec128_t xmm[32]; }); static_assert(sizeof(Thunk) % 16 == 0, From bf941bffe12aef941da497bee1d8602a3a5254f1 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 20:41:10 -0700 Subject: [PATCH 015/179] [a64] Fix memory address generation --- a64_seq_memory.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 54ac1177a..12b625c01 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -49,7 +49,8 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.CMP(guest.reg().toW(), 0xE0000000 - offset_const); + e.MOV(W1, 0xE0000000 - offset_const); + e.CMP(guest.reg().toW(), W1); e.CSET(X0, Cond::HS); e.LSL(X0, X0, 12); e.ADD(X0, X0, guest.reg()); @@ -59,8 +60,9 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, // TODO(benvanik): find a way to avoid doing this. e.MOV(W0, guest.reg().toW()); } - e.ADD(address_register, e.GetMembaseReg(), X0); - e.ADD(address_register, address_register, offset_const); + e.MOV(address_register, offset_const); + e.ADD(address_register, X0, address_register); + e.ADD(address_register, address_register, e.GetMembaseReg()); return address_register; // return e.GetMembaseReg() + e.rax + offset_const; } @@ -92,7 +94,8 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.CMP(guest.reg().toW(), 0xE0000000); + e.MOV(W1, 0xE0000000); + e.CMP(guest.reg().toW(), W1); e.CSET(X0, Cond::HS); e.LSL(X0, X0, 12); e.ADD(X0, X0, guest); From ea7475eb6ccf3f34eac265ac90b86967e0d9afb0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 3 May 2024 21:45:46 -0700 Subject: [PATCH 016/179] [a64] Fix indirect and external calls --- a64_emitter.cc | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/a64_emitter.cc b/a64_emitter.cc index 5a905d9b8..83854ed2d 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -510,7 +510,14 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, MOV(W1, reg.toW()); ADRP(X16, ResolveFunction); + + // Preserve frame and link register + STP(X29, X30, XSP, POST_INDEXED, -16); + BLR(X16); + + // Restore frame and link register + LDP(X29, X30, XSP, PRE_INDEXED, 16); } // Actually jump/call to rax. @@ -532,7 +539,14 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); // call(rax); LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + + // Preserve frame and link register + STP(X29, X30, XSP, POST_INDEXED, -16); + BLR(X16); + + // Restore frame and link register + LDP(X29, X30, XSP, PRE_INDEXED, 16); } } @@ -568,7 +582,15 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { MOV(X2, reinterpret_cast(builtin_function->arg1())); MOV(X16, reinterpret_cast(thunk)); + + // Preserve frame and link register + STP(X29, X30, XSP, POST_INDEXED, -16); + BLR(X16); + + // Restore frame and link register + LDP(X29, X30, XSP, PRE_INDEXED, 16); + // x0 = host return } } else if (function->behavior() == Function::Behavior::kExtern) { @@ -591,7 +613,15 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { LDR(X2, GetContextReg(), offsetof(ppc::PPCContext, kernel_state)); MOV(X16, reinterpret_cast(thunk)); + + // Preserve frame and link register + STP(X29, X30, XSP, POST_INDEXED, -16); + BLR(X16); + + // Restore frame and link register + LDP(X29, X30, XSP, PRE_INDEXED, 16); + // x0 = host return } } From 8ef4697c08e9b762ce7dd017f98ad348d361f44e Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 11:22:37 -0700 Subject: [PATCH 017/179] [a64] Fix overwriting of return-value registers These are stomping over X0 and Q0 which is returning input argument registers as return values. Fixes some guest-to-host calls. --- a64_backend.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 6320cf4d7..35bba8f46 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -458,7 +458,9 @@ void A64ThunkEmitter::EmitSaveVolatileRegs() { } void A64ThunkEmitter::EmitLoadVolatileRegs() { - LDR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); + // Preserve arguments passed to and returned from a subroutine + // LDR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); + LDR(X1, XSP, offsetof(StackLayout::Thunk, r[1])); LDR(X2, XSP, offsetof(StackLayout::Thunk, r[2])); LDR(X3, XSP, offsetof(StackLayout::Thunk, r[3])); @@ -481,7 +483,8 @@ void A64ThunkEmitter::EmitLoadVolatileRegs() { LDR(X30, XSP, offsetof(StackLayout::Thunk, r[19])); - LDR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); + // Preserve arguments passed to and returned from a subroutine + // LDR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); LDR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); LDR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); LDR(Q3, XSP, offsetof(StackLayout::Thunk, xmm[3])); From 84681ae6d6a886ee213a76e98c7e434597c69db4 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 11:23:45 -0700 Subject: [PATCH 018/179] [a64] Implement `OPCODE_VECTOR_SHL` Vector registers are passed as pointers rather than directly in the `Qn` registers. So these functions should be taking pointer-type arguments rather than vector-register types directly. Fixes `OPCODE_VECTOR_SHL` and passes unit tests. --- a64_seq_vector.cc | 90 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index fb13ad044..c64134fe6 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -424,7 +424,23 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB); // ============================================================================ // OPCODE_VECTOR_SHL // ============================================================================ +template ::value, int> = 0> +static uint8x16_t EmulateVectorShl(void*, std::byte src1[16], + std::byte src2[16]) { + alignas(16) T value[16 / sizeof(T)]; + alignas(16) T shamt[16 / sizeof(T)]; + // Load NEON registers into a C array. + vst1q_u8(reinterpret_cast(value), vld1q_u8(src1)); + vst1q_u8(reinterpret_cast(shamt), vld1q_u8(src2)); + + for (size_t i = 0; i < (16 / sizeof(T)); ++i) { + value[i] = value[i] << (shamt[i] & ((sizeof(T) * 8) - 1)); + } + + // Store result and return it. + return vld1q_u8(value); +} struct VECTOR_SHL_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { @@ -444,11 +460,77 @@ struct VECTOR_SHL_V128 } } - static void EmitInt8(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt8(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 16 - n; ++n) { + if (shamt.u8[n] != shamt.u8[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use SHL + e.SHL(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } - static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt16(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 16 - n; ++n) { + if (shamt.u8[n] != shamt.u8[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use SHL + e.SHL(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u8[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } - static void EmitInt32(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt32(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 16 - n; ++n) { + if (shamt.u8[n] != shamt.u8[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use SHL + e.SHL(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u8[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128); @@ -491,7 +573,7 @@ static uint8x16_t EmulateVectorShr(void*, uint8x16_t src1, uint8x16_t src2) { alignas(16) T value[16 / sizeof(T)]; alignas(16) T shamt[16 / sizeof(T)]; - // Load SSE registers into a C array. + // Load NEON registers into a C array. vst1q_u8(reinterpret_cast(value), src1); vst1q_u8(reinterpret_cast(shamt), src2); From 07a6f7a8118d1d86152be05599aa8d2ddc0eb73d Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 11:52:09 -0700 Subject: [PATCH 019/179] [a64] Remove volatile storing of X0/Q0 We dont load it back so no need to store it --- a64_backend.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 35bba8f46..287aec925 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -399,7 +399,8 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { void A64ThunkEmitter::EmitSaveVolatileRegs() { // Save off volatile registers. - STR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); + // Preserve arguments passed to and returned from a subroutine + // STR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); STR(X1, XSP, offsetof(StackLayout::Thunk, r[1])); STR(X2, XSP, offsetof(StackLayout::Thunk, r[2])); STR(X3, XSP, offsetof(StackLayout::Thunk, r[3])); @@ -422,7 +423,8 @@ void A64ThunkEmitter::EmitSaveVolatileRegs() { STR(X30, XSP, offsetof(StackLayout::Thunk, r[19])); - STR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); + // Preserve arguments passed to and returned from a subroutine + // STR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); STR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); STR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); STR(Q3, XSP, offsetof(StackLayout::Thunk, xmm[3])); From 720023c6f3a7ec6dd7fe711484a08f7f93d73df3 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 11:52:56 -0700 Subject: [PATCH 020/179] [a64] Implement `OPCODE_VECTOR_{SHR,SHA}` Passes all unit tests --- a64_seq_vector.cc | 158 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 134 insertions(+), 24 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index c64134fe6..19345a294 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -537,7 +537,23 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128); // ============================================================================ // OPCODE_VECTOR_SHR // ============================================================================ +template ::value, int> = 0> +static uint8x16_t EmulateVectorShr(void*, std::byte src1[16], + std::byte src2[16]) { + alignas(16) T value[16 / sizeof(T)]; + alignas(16) T shamt[16 / sizeof(T)]; + // Load NEON registers into a C array. + vst1q_u8(reinterpret_cast(value), vld1q_u8(src1)); + vst1q_u8(reinterpret_cast(shamt), vld1q_u8(src2)); + + for (size_t i = 0; i < (16 / sizeof(T)); ++i) { + value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1)); + } + + // Store result and return it. + return vld1q_u8(value); +} struct VECTOR_SHR_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { @@ -557,33 +573,83 @@ struct VECTOR_SHR_V128 } } - static void EmitInt8(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt8(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 16 - n; ++n) { + if (shamt.u8[n] != shamt.u8[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use USHR + e.USHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } - static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt16(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 8 - n; ++n) { + if (shamt.u16[n] != shamt.u16[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use USHR + e.USHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } - static void EmitInt32(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt32(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 4 - n; ++n) { + if (shamt.u32[n] != shamt.u32[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use USHR + e.USHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHR, VECTOR_SHR_V128); // ============================================================================ // OPCODE_VECTOR_SHA // ============================================================================ -template ::value, int> = 0> -static uint8x16_t EmulateVectorShr(void*, uint8x16_t src1, uint8x16_t src2) { - alignas(16) T value[16 / sizeof(T)]; - alignas(16) T shamt[16 / sizeof(T)]; - - // Load NEON registers into a C array. - vst1q_u8(reinterpret_cast(value), src1); - vst1q_u8(reinterpret_cast(shamt), src2); - - for (size_t i = 0; i < (16 / sizeof(T)); ++i) { - value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1)); - } - - // Store result and return it. - return vld1q_f32(value); -} struct VECTOR_SHA_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { @@ -618,18 +684,62 @@ struct VECTOR_SHA_V128 e.SSHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); return; } - e.LDR(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); } else { - e.LDR(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); } - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } - static void EmitInt16(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt16(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 8 - n; ++n) { + if (shamt.u16[n] != shamt.u16[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use SSHR + e.SSHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } - static void EmitInt32(A64Emitter& e, const EmitArgType& i) {} + static void EmitInt32(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + const auto& shamt = i.src2.constant(); + bool all_same = true; + for (size_t n = 0; n < 4 - n; ++n) { + if (shamt.u32[n] != shamt.u32[n + 1]) { + all_same = false; + break; + } + } + if (all_same) { + // Every count is the same, so we can use SSHR + e.SSHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0]); + return; + } + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128); From f5b5dc96b1c799237d3cec23f9ccd22233a3c07a Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 11:59:30 -0700 Subject: [PATCH 021/179] [a64] Implement `OPCODE_VECTOR_ROTATE_LEFT` Uses the emulated fallback for now. Will have to come back to this later. Passes unit tests. --- a64_seq_vector.cc | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 19345a294..27da95f86 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -746,10 +746,52 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128); // ============================================================================ // OPCODE_VECTOR_ROTATE_LEFT // ============================================================================ +template ::value, int> = 0> +static uint8x16_t EmulateVectorRotateLeft(void*, std::byte src1[16], + std::byte src2[16]) { + alignas(16) T value[16 / sizeof(T)]; + alignas(16) T shamt[16 / sizeof(T)]; + + // Load NEON registers into a C array. + vst1q_u8(reinterpret_cast(value), vld1q_u8(src1)); + vst1q_u8(reinterpret_cast(shamt), vld1q_u8(src2)); + + for (size_t i = 0; i < (16 / sizeof(T)); ++i) { + value[i] = xe::rotate_left(value[i], shamt[i] & ((sizeof(T) * 8) - 1)); + } + + // Store result and return it. + return vld1q_u8(value); +} struct VECTOR_ROTATE_LEFT_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + } else { + e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + } + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + switch (i.instr->flags) { + case INT8_TYPE: + e.CallNativeSafe( + reinterpret_cast(EmulateVectorRotateLeft)); + break; + case INT16_TYPE: + e.CallNativeSafe( + reinterpret_cast(EmulateVectorRotateLeft)); + break; + case INT32_TYPE: + e.CallNativeSafe( + reinterpret_cast(EmulateVectorRotateLeft)); + break; + default: + assert_always(); + break; + } + e.MOV(i.dest.reg().B16(), Q0.B16()); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128); From 373786e463f9c1b7db537760869009a0ee3e77ae Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 12:01:57 -0700 Subject: [PATCH 022/179] [a64] Implement `OPCODE_VECTOR_MIN` Passes unit tests --- a64_seq_vector.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 27da95f86..6b0b6dba9 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -260,13 +260,13 @@ struct VECTOR_MIN if (i.instr->flags & ARITHMETIC_UNSIGNED) { switch (part_type) { case INT8_TYPE: - // e.vpminub(dest, src1, src2); + e.UMIN(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpminuw(dest, src1, src2); + e.UMIN(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpminud(dest, src1, src2); + e.UMIN(dest.S4(), src1.S4(), src2.S4()); break; default: assert_unhandled_case(part_type); @@ -275,13 +275,13 @@ struct VECTOR_MIN } else { switch (part_type) { case INT8_TYPE: - // e.vpminsb(dest, src1, src2); + e.SMIN(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpminsw(dest, src1, src2); + e.SMIN(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpminsd(dest, src1, src2); + e.SMIN(dest.S4(), src1.S4(), src2.S4()); break; default: assert_unhandled_case(part_type); From 1bda067aab1a74332a807ddd1a459c0f101e67d2 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 12:03:10 -0700 Subject: [PATCH 023/179] [a64] Implement `OPCODE_VECTOR_MAX` Passes unit tests --- a64_seq_vector.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 6b0b6dba9..e41ef71d3 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -215,13 +215,13 @@ struct VECTOR_MAX if (i.instr->flags & ARITHMETIC_UNSIGNED) { switch (part_type) { case INT8_TYPE: - // e.vpmaxub(dest, src1, src2); + e.UMAX(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpmaxuw(dest, src1, src2); + e.UMAX(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpmaxud(dest, src1, src2); + e.UMAX(dest.S4(), src1.S4(), src2.S4()); break; default: assert_unhandled_case(part_type); @@ -230,13 +230,13 @@ struct VECTOR_MAX } else { switch (part_type) { case INT8_TYPE: - // e.vpmaxsb(dest, src1, src2); + e.SMAX(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpmaxsw(dest, src1, src2); + e.SMAX(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpmaxsd(dest, src1, src2); + e.SMAX(dest.S4(), src1.S4(), src2.S4()); break; default: assert_unhandled_case(part_type); From 79ee20fd65fc90ee42d5ec8598f87dd2ed02ec73 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 12:12:34 -0700 Subject: [PATCH 024/179] [a64] Implement `OPCODE_VECTOR_ADD` There is quite literally an instruction for each and every one of these cases. Passes unit tests --- a64_seq_vector.cc | 54 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index e41ef71d3..a9a4c4f99 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -408,7 +408,59 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGE, VECTOR_COMPARE_UGE_V128); // ============================================================================ struct VECTOR_ADD : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, const QReg& dest, QReg src1, QReg src2) { + const TypeName part_type = + static_cast(i.instr->flags & 0xFF); + const uint32_t arithmetic_flags = i.instr->flags >> 8; + bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED); + bool saturate = !!(arithmetic_flags & ARITHMETIC_SATURATE); + switch (part_type) { + case INT8_TYPE: + if (saturate) { + if (is_unsigned) { + e.UQADD(dest.B16(), src1.B16(), src2.B16()); + } else { + e.SQADD(dest.B16(), src1.B16(), src2.B16()); + } + } else { + e.ADD(dest.B16(), src1.B16(), src2.B16()); + } + break; + case INT16_TYPE: + if (saturate) { + if (is_unsigned) { + e.UQADD(dest.H8(), src1.H8(), src2.H8()); + } else { + e.SQADD(dest.H8(), src1.H8(), src2.H8()); + } + } else { + e.ADD(dest.H8(), src1.H8(), src2.H8()); + } + break; + case INT32_TYPE: + if (saturate) { + if (is_unsigned) { + e.UQADD(dest.S4(), src1.S4(), src2.S4()); + } else { + e.SQADD(dest.S4(), src1.S4(), src2.S4()); + } + } else { + e.ADD(dest.S4(), src1.S4(), src2.S4()); + } + break; + case FLOAT32_TYPE: + assert_false(is_unsigned); + assert_false(saturate); + e.FADD(dest.S4(), src1.S4(), src2.S4()); + break; + default: + assert_unhandled_case(part_type); + break; + } + }); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ADD, VECTOR_ADD); From 1de4a36ce5cdd6fc4c737aedf569537534133cd0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 12:21:17 -0700 Subject: [PATCH 025/179] [a64] Fix native vector calls Arguments need to be pointers stored in X0, X1, X2, ... rather than bassed directly in Q0, Q1 etc. There are no unit tests for these functions in particular. --- a64_sequences.cc | 62 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 2f7cb96e3..ff13a0c7f 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1987,44 +1987,44 @@ EMITTER_OPCODE_TABLE(OPCODE_RECIP, RECIP_F32, RECIP_F64, RECIP_V128); // TODO(benvanik): use approx here: // https://jrfonseca.blogspot.com/2008/09/fast-sse2-pow-tables-or-polynomials.html struct POW2_F32 : Sequence> { - static float32x4_t EmulatePow2(void*, float32x4_t src) { + static float32x4_t EmulatePow2(void*, std::byte src[16]) { float src_value; - vst1q_lane_f32(&src_value, src, 0); + vst1q_lane_f32(&src_value, vld1q_u8(src), 0); const float result = std::exp2(src_value); - return vld1q_lane_f32(&result, src, 0); + return vld1q_lane_f32(&result, vld1q_u8(src), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.FMOV(i.dest, S0); } }; struct POW2_F64 : Sequence> { - static float64x2_t EmulatePow2(void*, float64x2_t src) { + static float64x2_t EmulatePow2(void*, std::byte src[16]) { double src_value; - vst1q_lane_f64(&src_value, src, 0); + vst1q_lane_f64(&src_value, vld1q_u8(src), 0); const double result = std::exp2(src_value); - return vld1q_lane_f64(&result, src, 0); + return vld1q_lane_f64(&result, vld1q_u8(src), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.FMOV(i.dest, D0); } }; struct POW2_V128 : Sequence> { - static float32x4_t EmulatePow2(void*, float32x4_t src) { + static float32x4_t EmulatePow2(void*, std::byte src[16]) { alignas(16) float values[4]; - vst1q_f32(values, src); + vst1q_f32(values, vld1q_u8(src)); for (size_t i = 0; i < 4; ++i) { values[i] = std::exp2(values[i]); } return vld1q_f32(values); } static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -2038,45 +2038,45 @@ EMITTER_OPCODE_TABLE(OPCODE_POW2, POW2_F32, POW2_F64, POW2_V128); // https://jrfonseca.blogspot.com/2008/09/fast-sse2-pow-tables-or-polynomials.html // TODO(benvanik): this emulated fn destroys all xmm registers! don't do it! struct LOG2_F32 : Sequence> { - static float32x4_t EmulateLog2(void*, float32x4_t src) { + static float32x4_t EmulateLog2(void*, std::byte src[16]) { float src_value; - vst1q_lane_f32(&src_value, src, 0); + vst1q_lane_f32(&src_value, vld1q_u8(src), 0); float result = std::log2(src_value); - return vld1q_lane_f32(&result, src, 0); + return vld1q_lane_f32(&result, vld1q_u8(src), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); if (i.src1.is_constant) { - e.LDR(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); } else { - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); } e.CallNativeSafe(reinterpret_cast(EmulateLog2)); e.FMOV(i.dest, S0); } }; struct LOG2_F64 : Sequence> { - static float64x2_t EmulateLog2(void*, float64x2_t src) { + static float64x2_t EmulateLog2(void*, std::byte src[16]) { double src_value; - vst1q_lane_f64(&src_value, src, 0); + vst1q_lane_f64(&src_value, vld1q_u8(src), 0); double result = std::log2(src_value); - return vld1q_lane_f64(&result, src, 0); + return vld1q_lane_f64(&result, vld1q_u8(src), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); if (i.src1.is_constant) { - e.LDR(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); } else { - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); } e.CallNativeSafe(reinterpret_cast(EmulateLog2)); e.FMOV(i.dest, D0); } }; struct LOG2_V128 : Sequence> { - static float32x4_t EmulateLog2(void*, float32x4_t src) { + static float32x4_t EmulateLog2(void*, std::byte src[16]) { alignas(16) float values[4]; - vst1q_f32(values, src); + vst1q_f32(values, vld1q_u8(src)); for (size_t i = 0; i < 4; ++i) { values[i] = std::log2(values[i]); } @@ -2084,9 +2084,9 @@ struct LOG2_V128 : Sequence> { } static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - e.LDR(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); } else { - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); } e.CallNativeSafe(reinterpret_cast(EmulateLog2)); e.MOV(i.dest.reg().B16(), Q0.B16()); @@ -2417,16 +2417,16 @@ struct SHL_V128 : Sequence> { } else { e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); } - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateShlV128)); e.MOV(i.dest.reg().B16(), Q0.B16()); } - static float32x4_t EmulateShlV128(void*, float32x4_t src1, uint8_t src2) { + static float32x4_t EmulateShlV128(void*, std::byte src1[16], uint8_t src2) { // Almost all instances are shamt = 1, but non-constant. // shamt is [0,7] uint8_t shamt = src2 & 0x7; alignas(16) vec128_t value; - vst1q_f32(reinterpret_cast(&value), src1); + vst1q_f32(reinterpret_cast(&value), vld1q_u8(src1)); for (int i = 0; i < 15; ++i) { value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] << shamt) | (value.u8[(i + 1) ^ 0x3] >> (8 - shamt)); @@ -2496,16 +2496,16 @@ struct SHR_V128 : Sequence> { } else { e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); } - e.LDR(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateShrV128)); e.MOV(i.dest.reg().B16(), Q0.B16()); } - static float32x4_t EmulateShrV128(void*, float32x4_t src1, uint8_t src2) { + static float32x4_t EmulateShrV128(void*, std::byte src1[16], uint8_t src2) { // Almost all instances are shamt = 1, but non-constant. // shamt is [0,7] uint8_t shamt = src2 & 0x7; alignas(16) vec128_t value; - vst1q_f32(reinterpret_cast(&value), src1); + vst1q_f32(reinterpret_cast(&value), vld1q_u8(src1)); for (int i = 15; i > 0; --i) { value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] >> shamt) | (value.u8[(i - 1) ^ 0x3] << (8 - shamt)); From 9d722bdbd8f3743758e597cdb1bffcb0437d380d Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 12:31:56 -0700 Subject: [PATCH 026/179] [a64] Implement `OPCODE_PACK`(FLOAT16) Fails the unit tests due to subtle rounding errors --- a64_seq_vector.cc | 53 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index a9a4c4f99..dab01ee68 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1124,8 +1124,57 @@ struct PACK : Sequence> { e.LDR(Q0, X0); e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q0.B16()); } - static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) {} - static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) {} + static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { + alignas(16) float a[4]; + alignas(16) uint16_t b[8]; + vst1q_u8(a, vld1q_u8(src1)); + std::memset(b, 0, sizeof(b)); + + for (int i = 0; i < 2; i++) { + b[7 - i] = half_float::detail::float2half(a[i]); + } + + return vld1q_u8(b); + } + static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx + // dest = [(src1.x | src1.y), 0, 0, 0] + + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + } + e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_2)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static uint8x16_t EmulateFLOAT16_4(void*, std::byte src1[16]) { + alignas(16) float a[4]; + alignas(16) uint16_t b[8]; + vst1q_u8(a, vld1q_u8(src1)); + std::memset(b, 0, sizeof(b)); + + for (int i = 0; i < 4; i++) { + b[7 - (i ^ 2)] = + half_float::detail::float2half(a[i]); + } + + return vld1q_u8(b); + } + static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx + // dest = [(src1.z | src1.w), (src1.x | src1.y), 0, 0] + + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + } + e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_4)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) {} static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) {} static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) {} From 986e9981406309cd1992cc438a2f7e126bd7896b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 12:59:01 -0700 Subject: [PATCH 027/179] [a64] Implement `OPCODE_PACK`(SHORT) Fails unit tests due to subtle rounding errors `SHORT_4` unit-test is missing but implementation is the same as `SHORT_4` --- a64_seq_vector.cc | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index dab01ee68..188b8dfdb 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1175,8 +1175,26 @@ struct PACK : Sequence> { e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_4)); e.MOV(i.dest.reg().B16(), Q0.B16()); } - static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) {} - static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) {} + static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { + QReg src = i.src1; + if (i.src1.is_constant) { + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } + e.SQSHRN(i.dest.reg().toD().H4(), src.S4(), 8); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 4); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + } + static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { + QReg src = i.src1; + if (i.src1.is_constant) { + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } + e.SQSHRN(i.dest.reg().toD().H4(), src.S4(), 8); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 4); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + } static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) {} static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) {} static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { From b6a8b228364c1881022731fd6f94df9810729d95 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 15:46:18 -0700 Subject: [PATCH 028/179] [a64] Implement HIR Branch labeling Adds support for HIR labels to create actual oaknut labels --- a64_emitter.cc | 3 +-- a64_emitter.h | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 83854ed2d..513a489ad 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -265,8 +265,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // Mark block labels. auto label = block->label_head; while (label) { - // TODO(wunkolo): string-labels? - // L(label->name); + l(label_lookup_[label->name]); label = label->next; } diff --git a/a64_emitter.h b/a64_emitter.h index 1cbd60e30..a0f4a88c6 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -10,6 +10,7 @@ #ifndef XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ #define XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ +#include #include #include "xenia/base/arena.h" @@ -165,6 +166,11 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { r = oaknut::QReg(idx); } + // Gets(and possibly create) an HIR label with the specified name + oaknut::Label* lookup_label(const char* label_name) { + return &label_lookup_[label_name]; + } + oaknut::Label& epilog_label() { return *epilog_label_; } void MarkSourceOffset(const hir::Instr* i); @@ -229,6 +235,9 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { oaknut::Label* epilog_label_ = nullptr; + // Convert from plain-text label-names into oaknut-labels + std::unordered_map label_lookup_; + hir::Instr* current_instr_ = nullptr; FunctionDebugInfo* debug_info_ = nullptr; From 70dda5eeb918a01dc2ffe8990b6066dfb47d5ed0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 15:47:19 -0700 Subject: [PATCH 029/179] [a64] Implement control sequences Implements control sequences such as conditional branching, breaking, and trapping --- a64_seq_control.cc | 314 ++++++++++++++++++++++----------------------- 1 file changed, 156 insertions(+), 158 deletions(-) diff --git a/a64_seq_control.cc b/a64_seq_control.cc index 082e6b3bf..1fffcb27a 100644 --- a/a64_seq_control.cc +++ b/a64_seq_control.cc @@ -35,66 +35,62 @@ EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK, DEBUG_BREAK); struct DEBUG_BREAK_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.DebugBreak(); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.DebugBreak(); + e.l(skip); } }; struct DEBUG_BREAK_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.DebugBreak(); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.DebugBreak(); + e.l(skip); } }; struct DEBUG_BREAK_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.DebugBreak(); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.DebugBreak(); + e.l(skip); } }; struct DEBUG_BREAK_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.DebugBreak(); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.DebugBreak(); + e.l(skip); } }; struct DEBUG_BREAK_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.DebugBreak(); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.DebugBreak(); + e.l(skip); } }; struct DEBUG_BREAK_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.DebugBreak(); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.DebugBreak(); + e.l(skip); } }; EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK_TRUE, DEBUG_BREAK_TRUE_I8, @@ -118,61 +114,57 @@ EMITTER_OPCODE_TABLE(OPCODE_TRAP, TRAP); struct TRAP_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Trap(i.instr->flags); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Trap(i.instr->flags); + e.l(skip); } }; struct TRAP_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Trap(i.instr->flags); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Trap(i.instr->flags); + e.l(skip); } }; struct TRAP_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Trap(i.instr->flags); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Trap(i.instr->flags); + e.l(skip); } }; struct TRAP_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Trap(i.instr->flags); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Trap(i.instr->flags); + e.l(skip); } }; struct TRAP_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Trap(i.instr->flags); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.Trap(i.instr->flags); + e.l(skip); } }; struct TRAP_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Trap(i.instr->flags); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.Trap(i.instr->flags); + e.l(skip); } }; EMITTER_OPCODE_TABLE(OPCODE_TRAP_TRUE, TRAP_TRUE_I8, TRAP_TRUE_I16, @@ -197,66 +189,62 @@ struct CALL_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->is_guest()); - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Call(i.instr, static_cast(i.src2.value)); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.l(skip); } }; struct CALL_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->is_guest()); - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Call(i.instr, static_cast(i.src2.value)); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.l(skip); } }; struct CALL_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->is_guest()); - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Call(i.instr, static_cast(i.src2.value)); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.l(skip); } }; struct CALL_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->is_guest()); - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Call(i.instr, static_cast(i.src2.value)); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.l(skip); } }; struct CALL_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->is_guest()); - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Call(i.instr, static_cast(i.src2.value)); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.l(skip); } }; struct CALL_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->is_guest()); - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip); - // e.Call(i.instr, static_cast(i.src2.value)); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.l(skip); } }; EMITTER_OPCODE_TABLE(OPCODE_CALL_TRUE, CALL_TRUE_I8, CALL_TRUE_I16, @@ -281,66 +269,62 @@ struct CALL_INDIRECT_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip, CodeGenerator::T_NEAR); - // e.CallIndirect(i.instr, i.src2); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.CallIndirect(i.instr, i.src2); + e.l(skip); } }; struct CALL_INDIRECT_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip, CodeGenerator::T_NEAR); - // e.CallIndirect(i.instr, i.src2); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.CallIndirect(i.instr, i.src2); + e.l(skip); } }; struct CALL_INDIRECT_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip, CodeGenerator::T_NEAR); - // e.CallIndirect(i.instr, i.src2); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.CallIndirect(i.instr, i.src2); + e.l(skip); } }; struct CALL_INDIRECT_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip, CodeGenerator::T_NEAR); - // e.CallIndirect(i.instr, i.src2); - // e.L(skip); + oaknut::Label skip; + e.CBZ(i.src1, skip); + e.CallIndirect(i.instr, i.src2); + e.l(skip); } }; struct CALL_INDIRECT_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip, CodeGenerator::T_NEAR); - // e.CallIndirect(i.instr, i.src2); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.CallIndirect(i.instr, i.src2); + e.l(skip); } }; struct CALL_INDIRECT_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // Xbyak::Label skip; - // e.jz(skip, CodeGenerator::T_NEAR); - // e.CallIndirect(i.instr, i.src2); - // e.L(skip); + oaknut::Label skip; + e.FCMP(i.src1, 0); + e.B(Cond::EQ, skip); + e.CallIndirect(i.instr, i.src2); + e.l(skip); } }; EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT_TRUE, CALL_INDIRECT_TRUE_I8, @@ -367,7 +351,7 @@ struct RETURN : Sequence> { // If this is the last instruction in the last block, just let us // fall through. if (i.instr->next || i.instr->block->next) { - // e.jmp(e.epilog_label(), CodeGenerator::T_NEAR); + e.B(e.epilog_label()); } } }; @@ -379,43 +363,39 @@ EMITTER_OPCODE_TABLE(OPCODE_RETURN, RETURN); struct RETURN_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + e.CBNZ(i.src1, e.epilog_label()); } }; struct RETURN_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + e.CBNZ(i.src1, e.epilog_label()); } }; struct RETURN_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + e.CBNZ(i.src1, e.epilog_label()); } }; struct RETURN_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + e.CBNZ(i.src1, e.epilog_label()); } }; struct RETURN_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + e.FCMP(i.src1, 0); + e.B(Cond::NE, e.epilog_label()); } }; struct RETURN_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // e.jnz(e.epilog_label(), CodeGenerator::T_NEAR); + e.FCMP(i.src1, 0); + e.B(Cond::NE, e.epilog_label()); } }; EMITTER_OPCODE_TABLE(OPCODE_RETURN_TRUE, RETURN_TRUE_I8, RETURN_TRUE_I16, @@ -439,7 +419,9 @@ EMITTER_OPCODE_TABLE(OPCODE_SET_RETURN_ADDRESS, SET_RETURN_ADDRESS); // ============================================================================ struct BRANCH : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.jmp(i.src1.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src1.value->name); + assert_not_null(label); + e.B(*label); } }; EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH); @@ -450,43 +432,51 @@ EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH); struct BRANCH_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBNZ(i.src1, *label); } }; struct BRANCH_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBNZ(i.src1, *label); } }; struct BRANCH_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBNZ(i.src1, *label); } }; struct BRANCH_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jnz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBNZ(i.src1, *label); } }; struct BRANCH_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // e.jnz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.FCMP(i.src1, 0); + e.B(Cond::NE, *label); } }; struct BRANCH_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // e.jnz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.FCMP(i.src1, 0); + e.B(Cond::NE, *label); } }; EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16, @@ -499,48 +489,56 @@ EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16, struct BRANCH_FALSE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBZ(i.src1, *label); } }; struct BRANCH_FALSE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBZ(i.src1, *label); } }; struct BRANCH_FALSE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBZ(i.src1, *label); } }; struct BRANCH_FALSE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.test(i.src1, i.src1); - // e.jz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.CBZ(i.src1, *label); } }; struct BRANCH_FALSE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // e.jz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.FCMP(i.src1, 0); + e.B(Cond::NE, *label); } }; struct BRANCH_FALSE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vptest(i.src1, i.src1); - // e.jz(i.src2.value->name, e.T_NEAR); + oaknut::Label* label = e.lookup_label(i.src2.value->name); + assert_not_null(label); + e.FCMP(i.src1, 0); + e.B(Cond::NE, *label); } }; EMITTER_OPCODE_TABLE(OPCODE_BRANCH_FALSE, BRANCH_FALSE_I8, BRANCH_FALSE_I16, From c55e04928743b5032f3f4ba7c1b536263346689e Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 16:12:02 -0700 Subject: [PATCH 030/179] [a64] Fix ResolveFunction thunk Register was getting stomped over --- a64_backend.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 287aec925..4be244847 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -337,8 +337,12 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { uint64_t ResolveFunction(void* raw_context, uint64_t target_address); ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { - // ebx = target PPC address - // rcx = context + // Entry: + // X0 = target PPC address + + // Resolve Function: + // X0 = context + // X1 = target PPC address struct _code_offsets { size_t prolog; @@ -366,8 +370,8 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // mov(rdx, rbx); // mov(rax, reinterpret_cast(&ResolveFunction)); // call(rax) + MOV(X1, X0); MOV(X0, GetContextReg()); // context - MOV(X1, X1); MOVP2R(X16, &ResolveFunction); BLR(X16); From c2a3845c7770de1cd8a7aa19f400acff9b3683a4 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 16:12:48 -0700 Subject: [PATCH 031/179] [a64] Fix resetting of labels during Emplace On the x64 side, this is the same as the `reset()` function resetting the label-manager --- a64_emitter.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 513a489ad..1f84a0e8f 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -163,7 +163,9 @@ void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, // top_ = old_address; set_wptr(reinterpret_cast(old_address)); + // reset(); + label_lookup_.clear(); return new_execute_address; } @@ -399,7 +401,6 @@ void A64Emitter::Trap(uint16_t trap_type) { break; default: XELOGW("Unknown trap type {}", trap_type); - // db(0xCC); BRK(0xF000); break; } @@ -407,7 +408,6 @@ void A64Emitter::Trap(uint16_t trap_type) { void A64Emitter::UnimplementedInstr(const hir::Instr* i) { // TODO(benvanik): notify debugger. - // db(0xCC); BRK(0xF000); assert_always(); } From a17ff39d52b8148a0d1632aa4cad8f10b06ecf00 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 16:18:43 -0700 Subject: [PATCH 032/179] [a64] Fix ResolveFunctionThunk call Resolving the function puts it into X0 and should be called immediately after. We were just calling ResolveFunction on ResolveFunction recursively --- a64_backend.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_backend.cc b/a64_backend.cc index 4be244847..800c4f539 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -382,7 +382,7 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // add(rsp, stack_size); // jmp(rax); ADD(XSP, XSP, stack_size); - BR(X16); + BR(X0); code_offsets.tail = offset(); From f2a19bf374e9da4f37693c02dfeed5bdd659b2ed Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 4 May 2024 21:21:56 -0700 Subject: [PATCH 033/179] [a64] Pad code cache with `0x00` bytes --- a64_code_cache.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_code_cache.cc b/a64_code_cache.cc index fb16c2ffe..f484967ac 100644 --- a/a64_code_cache.cc +++ b/a64_code_cache.cc @@ -244,8 +244,8 @@ void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, // Copy code. std::memcpy(code_write_address, machine_code, func_info.code_size.total); - // Fill unused slots with 0xCC - std::memset(tail_write_address, 0xCC, + // Fill unused slots with 0x00 + std::memset(tail_write_address, 0x00, static_cast(end_write_address - tail_write_address)); // Notify subclasses of placed code. From 25a39dabad3048c01dbedfa86af9f7df1ce076bc Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 19:01:39 -0700 Subject: [PATCH 034/179] [a64] Draft Windows-ARM64 stack unwinding data Things still get weird at the thunks, but this allows for callstacks between-to-guest calls --- a64_code_cache_win.cc | 210 +++++++++++++++++++----------------------- 1 file changed, 96 insertions(+), 114 deletions(-) diff --git a/a64_code_cache_win.cc b/a64_code_cache_win.cc index a48426260..93310af9d 100644 --- a/a64_code_cache_win.cc +++ b/a64_code_cache_win.cc @@ -31,69 +31,39 @@ namespace cpu { namespace backend { namespace a64 { -// https://msdn.microsoft.com/en-us/library/ssa62fwe.aspx +// ARM64 unwind-op codes +// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes +// https://www.corsix.org/content/windows-arm64-unwind-codes typedef enum _UNWIND_OP_CODES { - UWOP_PUSH_NONVOL = 0, /* info == register number */ - UWOP_ALLOC_LARGE, /* no info, alloc size in next 2 slots */ - UWOP_ALLOC_SMALL, /* info == size of allocation / 8 - 1 */ - UWOP_SET_FPREG, /* no info, FP = RSP + UNWIND_INFO.FPRegOffset*16 */ - UWOP_SAVE_NONVOL, /* info == register number, offset in next slot */ - UWOP_SAVE_NONVOL_FAR, /* info == register number, offset in next 2 slots */ - UWOP_SAVE_XMM128, /* info == XMM reg number, offset in next slot */ - UWOP_SAVE_XMM128_FAR, /* info == XMM reg number, offset in next 2 slots */ - UWOP_PUSH_MACHFRAME /* info == 0: no error-code, 1: error-code */ + UWOP_NOP = 0xE3, + UWOP_ALLOC_S = 0x00, // sub sp, sp, i*16 + UWOP_ALLOC_L = 0xE0'00'00'00, // sub sp, sp, i*16 + UWOP_SAVE_FPLR = 0x40, // stp fp, lr, [sp+i*8] + UWOP_SAVE_FPLRX = 0x80, // stp fp, lr, [sp-(i+1)*8]! + UWOP_SET_FP = 0xE1, // mov fp, sp + UWOP_END = 0xE4, } UNWIND_CODE_OPS; -class UNWIND_REGISTER { - public: - enum _ { - RAX = 0, - RCX = 1, - RDX = 2, - RBX = 3, - RSP = 4, - RBP = 5, - RSI = 6, - RDI = 7, - R8 = 8, - R9 = 9, - R10 = 10, - R11 = 11, - R12 = 12, - R13 = 13, - R14 = 14, - R15 = 15, - }; -}; -typedef union _UNWIND_CODE { - struct { - uint8_t CodeOffset; - uint8_t UnwindOp : 4; - uint8_t OpInfo : 4; - }; - USHORT FrameOffset; -} UNWIND_CODE, *PUNWIND_CODE; +using UNWIND_CODE = uint32_t; +static_assert(sizeof(UNWIND_CODE) == sizeof(uint32_t)); + +// UNWIND_INFO defines the static part (first 32-bit) of the .xdata record typedef struct _UNWIND_INFO { - uint8_t Version : 3; - uint8_t Flags : 5; - uint8_t SizeOfProlog; - uint8_t CountOfCodes; - uint8_t FrameRegister : 4; - uint8_t FrameOffset : 4; - UNWIND_CODE UnwindCode[1]; - /* UNWIND_CODE MoreUnwindCode[((CountOfCodes + 1) & ~1) - 1]; - * union { - * OPTIONAL ULONG ExceptionHandler; - * OPTIONAL ULONG FunctionEntry; - * }; - * OPTIONAL ULONG ExceptionData[]; */ + uint32_t FunctionLength : 18; + uint32_t Version : 2; + uint32_t X : 1; + uint32_t E : 1; + uint32_t EpilogCount : 5; + uint32_t CodeWords : 5; + UNWIND_CODE UnwindCodes[2]; } UNWIND_INFO, *PUNWIND_INFO; +static_assert(offsetof(UNWIND_INFO, UnwindCodes[0]) == 4); +static_assert(offsetof(UNWIND_INFO, UnwindCodes[1]) == 8); + // Size of unwind info per function. -// TODO(benvanik): move this to emitter. -static const uint32_t kUnwindInfoSize = - sizeof(UNWIND_INFO) + (sizeof(UNWIND_CODE) * (6 - 1)); +static const uint32_t kUnwindInfoSize = sizeof(UNWIND_INFO); class Win32A64CodeCache : public A64CodeCache { public: @@ -232,83 +202,95 @@ void Win32A64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, grow_table_(unwind_table_handle_, unwind_table_count_); } - // This isn't needed on a64 (probably), but is convention. - // On UWP, FlushInstructionCache available starting from 10.0.16299.0. // https://docs.microsoft.com/en-us/uwp/win32-and-com/win32-apis FlushInstructionCache(GetCurrentProcess(), code_execute_address, func_info.code_size.total); } +constexpr UNWIND_CODE UnwindOpWord(uint8_t code0 = UWOP_NOP, + uint8_t code1 = UWOP_NOP, + uint8_t code2 = UWOP_NOP, + uint8_t code3 = UWOP_NOP) { + return static_cast(code0) | (static_cast(code1) << 8) | + (static_cast(code2) << 16) | + (static_cast(code3) << 24); +} + +// 8-byte unwind code for "stp fp, lr, [sp, #-16]! +// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes +static uint8_t OpSaveFpLrX(int16_t pre_index_offset) { + assert_true(pre_index_offset <= -8); + assert_true(pre_index_offset >= -512); + // 16-byte aligned + constexpr int IndexShift = 3; + constexpr int IndexMask = (1 << IndexShift) - 1; + assert_true((pre_index_offset & IndexMask) == 0); + const uint32_t encoded_value = (-pre_index_offset >> IndexShift) - 1; + return UWOP_SAVE_FPLRX | encoded_value; +} + +// Ensure a 16-byte aligned stack +static constexpr size_t StackAlignShift = 4; // n / 16 +static constexpr size_t StackAlignMask = (1 << StackAlignShift) - 1; // n % 16 + +// 8-byte unwind code for up to +512-byte "sub sp, sp, #stack_space" +// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes +static uint8_t OpAllocS(int16_t stack_space) { + assert_true(stack_space >= 0); + assert_true(stack_space < 512); + assert_true((stack_space & StackAlignMask) == 0); + return UWOP_ALLOC_S | (stack_space >> StackAlignShift); +} + +// 4-byte unwind code for +256MiB "sub sp, sp, #stack_space" +// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes +uint32_t OpAllocL(int32_t stack_space) { + assert_true(stack_space >= 0); + assert_true(stack_space < (0xFFFFFF * 16)); + assert_true((stack_space & StackAlignMask) == 0); + return xe::byte_swap(UWOP_ALLOC_L | + ((stack_space >> StackAlignShift) & 0xFF'FF'FF)); +} + void Win32A64CodeCache::InitializeUnwindEntry( uint8_t* unwind_entry_address, size_t unwind_table_slot, void* code_execute_address, const EmitFunctionInfo& func_info) { auto unwind_info = reinterpret_cast(unwind_entry_address); - UNWIND_CODE* unwind_code = nullptr; - assert_true(func_info.code_size.prolog < 256); // needs to fit into a uint8_t - auto prolog_size = static_cast(func_info.code_size.prolog); - assert_true(func_info.prolog_stack_alloc_offset < - 256); // needs to fit into a uint8_t - auto prolog_stack_alloc_offset = - static_cast(func_info.prolog_stack_alloc_offset); + *unwind_info = {}; + // ARM64 instructions are always multiples of 4 bytes + // Windows ignores the bottom 2 bits + unwind_info->FunctionLength = func_info.code_size.total / 4; + unwind_info->CodeWords = 2; - if (!func_info.stack_size) { - // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info - unwind_info->Version = 1; - unwind_info->Flags = 0; - unwind_info->SizeOfProlog = prolog_size; - unwind_info->CountOfCodes = 0; - unwind_info->FrameRegister = 0; - unwind_info->FrameOffset = 0; - } else if (func_info.stack_size <= 128) { - // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info - unwind_info->Version = 1; - unwind_info->Flags = 0; - unwind_info->SizeOfProlog = prolog_size; - unwind_info->CountOfCodes = 0; - unwind_info->FrameRegister = 0; - unwind_info->FrameOffset = 0; + // https://learn.microsoft.com/en-us/cpp/build/arm64-exception-handling?view=msvc-170#unwind-codes + // The array of unwind codes is a pool of sequences that describe exactly how + // to undo the effects of the prolog. They're stored in the same order the + // operations need to be undone. The unwind codes can be thought of as a small + // instruction set, encoded as a string of bytes. When execution is complete, + // the return address to the calling function is in the lr register. And, all + // non-volatile registers are restored to their values at the time the + // function was called. - // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_code - unwind_code = &unwind_info->UnwindCode[unwind_info->CountOfCodes++]; - unwind_code->CodeOffset = prolog_stack_alloc_offset; - unwind_code->UnwindOp = UWOP_ALLOC_SMALL; - unwind_code->OpInfo = (func_info.stack_size / 8) - 1; - } else { - // TODO(benvanik): take as parameters? + // Function frames are generally: + // STP(X29, X30, SP, PRE_INDEXED, -32); + // MOV(X29, XSP); + // SUB(XSP, XSP, stack_size); + // ... function body ... + // ADD(XSP, XSP, stack_size); + // MOV(XSP, X29); + // LDP(X29, X30, SP, POST_INDEXED, 32); - // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info - unwind_info->Version = 1; - unwind_info->Flags = 0; - unwind_info->SizeOfProlog = prolog_size; - unwind_info->CountOfCodes = 0; - unwind_info->FrameRegister = 0; - unwind_info->FrameOffset = 0; - - // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_code - unwind_code = &unwind_info->UnwindCode[unwind_info->CountOfCodes++]; - unwind_code->CodeOffset = prolog_stack_alloc_offset; - unwind_code->UnwindOp = UWOP_ALLOC_LARGE; - unwind_code->OpInfo = 0; // One slot for size - - assert_true((func_info.stack_size / 8) < 65536u); - unwind_code = &unwind_info->UnwindCode[unwind_info->CountOfCodes++]; - unwind_code->FrameOffset = (USHORT)(func_info.stack_size) / 8; - } - - if (unwind_info->CountOfCodes % 1) { - // Count of unwind codes must always be even. - std::memset(&unwind_info->UnwindCode[unwind_info->CountOfCodes + 1], 0, - sizeof(UNWIND_CODE)); - } + // These opcodes must undo the epilog and put the return address within lr + unwind_info->UnwindCodes[0] = OpAllocL(func_info.stack_size); + unwind_info->UnwindCodes[1] = + UnwindOpWord(UWOP_SET_FP, OpSaveFpLrX(-32), UWOP_END); // Add entry. - auto& fn_entry = unwind_table_[unwind_table_slot]; + RUNTIME_FUNCTION& fn_entry = unwind_table_[unwind_table_slot]; fn_entry.BeginAddress = DWORD(reinterpret_cast(code_execute_address) - generated_code_execute_base_); - fn_entry.FunctionLength = - DWORD(func_info.code_size.total); fn_entry.UnwindData = DWORD(unwind_entry_address - generated_code_execute_base_); } From 3546106683507e0287dbd2a794092c3b978ea6e9 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 19:14:51 -0700 Subject: [PATCH 035/179] [a64] Use `X4` for address-generation veneer --- a64_seq_memory.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 12b625c01..4b86aa4cf 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -25,7 +25,7 @@ volatile int anchor_memory = 0; template XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, - XReg address_register = X0) { + XReg address_register = X4) { assert_true(offset.is_constant); int32_t offset_const = static_cast(offset.constant()); @@ -71,7 +71,7 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, // Note: most *should* be aligned, but needs to be checked! template XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, - XReg address_register = X0) { + XReg address_register = X4) { if (guest.is_constant) { // TODO(benvanik): figure out how to do this without a temp. // Since the constant is often 0x8... if we tried to use that as a From e5a45c196770df9f7d12318be4c613616ec9aa47 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 19:18:07 -0700 Subject: [PATCH 036/179] [a64] Optimize Volatile/NonVolatile push/pop --- a64_backend.cc | 211 ++++++++++++++++++------------------------------- 1 file changed, 75 insertions(+), 136 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 800c4f539..f40b80d93 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -358,7 +358,9 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // rsp + 0 = return address // sub(rsp, stack_size); - SUB(XSP, XSP, stack_size); + STP(X29, X30, SP, POST_INDEXED, -32); + MOV(X29, SP); + SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); @@ -401,158 +403,95 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { return (ResolveFunctionThunk)fn; } +// Caller saved: +// x0-x15, x30 | d0-d7 and d16-v31 void A64ThunkEmitter::EmitSaveVolatileRegs() { // Save off volatile registers. // Preserve arguments passed to and returned from a subroutine - // STR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); - STR(X1, XSP, offsetof(StackLayout::Thunk, r[1])); - STR(X2, XSP, offsetof(StackLayout::Thunk, r[2])); - STR(X3, XSP, offsetof(StackLayout::Thunk, r[3])); - STR(X4, XSP, offsetof(StackLayout::Thunk, r[4])); - STR(X5, XSP, offsetof(StackLayout::Thunk, r[5])); - STR(X6, XSP, offsetof(StackLayout::Thunk, r[6])); - STR(X7, XSP, offsetof(StackLayout::Thunk, r[7])); - STR(X8, XSP, offsetof(StackLayout::Thunk, r[8])); - - STR(X9, XSP, offsetof(StackLayout::Thunk, r[9])); - STR(X10, XSP, offsetof(StackLayout::Thunk, r[10])); - STR(X11, XSP, offsetof(StackLayout::Thunk, r[11])); - STR(X12, XSP, offsetof(StackLayout::Thunk, r[12])); - STR(X13, XSP, offsetof(StackLayout::Thunk, r[13])); - STR(X14, XSP, offsetof(StackLayout::Thunk, r[14])); - STR(X15, XSP, offsetof(StackLayout::Thunk, r[15])); - STR(X16, XSP, offsetof(StackLayout::Thunk, r[16])); - STR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); - STR(X18, XSP, offsetof(StackLayout::Thunk, r[18])); - - STR(X30, XSP, offsetof(StackLayout::Thunk, r[19])); + // STR(X0, SP, offsetof(StackLayout::Thunk, r[0])); + STP(X1, X2, SP, offsetof(StackLayout::Thunk, r[0])); + STP(X3, X4, SP, offsetof(StackLayout::Thunk, r[2])); + STP(X5, X6, SP, offsetof(StackLayout::Thunk, r[4])); + STP(X7, X8, SP, offsetof(StackLayout::Thunk, r[6])); + STP(X9, X10, SP, offsetof(StackLayout::Thunk, r[8])); + STP(X11, X12, SP, offsetof(StackLayout::Thunk, r[10])); + STP(X13, X14, SP, offsetof(StackLayout::Thunk, r[12])); + STP(X15, X30, SP, offsetof(StackLayout::Thunk, r[14])); // Preserve arguments passed to and returned from a subroutine - // STR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); - STR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); - STR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); - STR(Q3, XSP, offsetof(StackLayout::Thunk, xmm[3])); - STR(Q4, XSP, offsetof(StackLayout::Thunk, xmm[4])); - STR(Q5, XSP, offsetof(StackLayout::Thunk, xmm[5])); - STR(Q6, XSP, offsetof(StackLayout::Thunk, xmm[6])); - STR(Q7, XSP, offsetof(StackLayout::Thunk, xmm[7])); - - STR(Q8, XSP, offsetof(StackLayout::Thunk, xmm[8])); - STR(Q9, XSP, offsetof(StackLayout::Thunk, xmm[9])); - STR(Q10, XSP, offsetof(StackLayout::Thunk, xmm[10])); - STR(Q11, XSP, offsetof(StackLayout::Thunk, xmm[11])); - STR(Q12, XSP, offsetof(StackLayout::Thunk, xmm[12])); - STR(Q13, XSP, offsetof(StackLayout::Thunk, xmm[13])); - STR(Q14, XSP, offsetof(StackLayout::Thunk, xmm[14])); - STR(Q15, XSP, offsetof(StackLayout::Thunk, xmm[15])); - STR(Q16, XSP, offsetof(StackLayout::Thunk, xmm[16])); - STR(Q17, XSP, offsetof(StackLayout::Thunk, xmm[17])); - STR(Q18, XSP, offsetof(StackLayout::Thunk, xmm[18])); - STR(Q19, XSP, offsetof(StackLayout::Thunk, xmm[19])); - STR(Q20, XSP, offsetof(StackLayout::Thunk, xmm[20])); - STR(Q21, XSP, offsetof(StackLayout::Thunk, xmm[21])); - STR(Q22, XSP, offsetof(StackLayout::Thunk, xmm[22])); - STR(Q23, XSP, offsetof(StackLayout::Thunk, xmm[23])); - STR(Q24, XSP, offsetof(StackLayout::Thunk, xmm[24])); - STR(Q25, XSP, offsetof(StackLayout::Thunk, xmm[25])); - STR(Q26, XSP, offsetof(StackLayout::Thunk, xmm[26])); - STR(Q27, XSP, offsetof(StackLayout::Thunk, xmm[27])); - STR(Q28, XSP, offsetof(StackLayout::Thunk, xmm[28])); - STR(Q29, XSP, offsetof(StackLayout::Thunk, xmm[29])); - STR(Q30, XSP, offsetof(StackLayout::Thunk, xmm[30])); - STR(Q31, XSP, offsetof(StackLayout::Thunk, xmm[31])); + // STR(Q0, SP, offsetof(StackLayout::Thunk, xmm[0])); + STP(Q1, Q2, SP, offsetof(StackLayout::Thunk, xmm[0])); + STP(Q3, Q4, SP, offsetof(StackLayout::Thunk, xmm[2])); + STP(Q5, Q6, SP, offsetof(StackLayout::Thunk, xmm[4])); + STP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); + STP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); + STP(Q17, Q18, SP, offsetof(StackLayout::Thunk, xmm[8])); + STP(Q19, Q20, SP, offsetof(StackLayout::Thunk, xmm[10])); + STP(Q21, Q22, SP, offsetof(StackLayout::Thunk, xmm[12])); + STP(Q23, Q24, SP, offsetof(StackLayout::Thunk, xmm[14])); + STP(Q25, Q26, SP, offsetof(StackLayout::Thunk, xmm[16])); + STP(Q27, Q28, SP, offsetof(StackLayout::Thunk, xmm[18])); + STP(Q29, Q30, SP, offsetof(StackLayout::Thunk, xmm[20])); + STR(Q31, SP, offsetof(StackLayout::Thunk, xmm[21])); } void A64ThunkEmitter::EmitLoadVolatileRegs() { // Preserve arguments passed to and returned from a subroutine - // LDR(X0, XSP, offsetof(StackLayout::Thunk, r[0])); - - LDR(X1, XSP, offsetof(StackLayout::Thunk, r[1])); - LDR(X2, XSP, offsetof(StackLayout::Thunk, r[2])); - LDR(X3, XSP, offsetof(StackLayout::Thunk, r[3])); - LDR(X4, XSP, offsetof(StackLayout::Thunk, r[4])); - LDR(X5, XSP, offsetof(StackLayout::Thunk, r[5])); - LDR(X6, XSP, offsetof(StackLayout::Thunk, r[6])); - LDR(X7, XSP, offsetof(StackLayout::Thunk, r[7])); - LDR(X8, XSP, offsetof(StackLayout::Thunk, r[8])); - - LDR(X9, XSP, offsetof(StackLayout::Thunk, r[9])); - LDR(X10, XSP, offsetof(StackLayout::Thunk, r[10])); - LDR(X11, XSP, offsetof(StackLayout::Thunk, r[11])); - LDR(X12, XSP, offsetof(StackLayout::Thunk, r[12])); - LDR(X13, XSP, offsetof(StackLayout::Thunk, r[13])); - LDR(X14, XSP, offsetof(StackLayout::Thunk, r[14])); - LDR(X15, XSP, offsetof(StackLayout::Thunk, r[15])); - LDR(X16, XSP, offsetof(StackLayout::Thunk, r[16])); - LDR(X17, XSP, offsetof(StackLayout::Thunk, r[17])); - LDR(X18, XSP, offsetof(StackLayout::Thunk, r[18])); - - LDR(X30, XSP, offsetof(StackLayout::Thunk, r[19])); + // LDR(X0, SP, offsetof(StackLayout::Thunk, r[0])); + LDP(X1, X2, SP, offsetof(StackLayout::Thunk, r[0])); + LDP(X3, X4, SP, offsetof(StackLayout::Thunk, r[2])); + LDP(X5, X6, SP, offsetof(StackLayout::Thunk, r[4])); + LDP(X7, X8, SP, offsetof(StackLayout::Thunk, r[6])); + LDP(X9, X10, SP, offsetof(StackLayout::Thunk, r[8])); + LDP(X11, X12, SP, offsetof(StackLayout::Thunk, r[10])); + LDP(X13, X14, SP, offsetof(StackLayout::Thunk, r[12])); + LDP(X15, X30, SP, offsetof(StackLayout::Thunk, r[14])); // Preserve arguments passed to and returned from a subroutine - // LDR(Q0, XSP, offsetof(StackLayout::Thunk, xmm[0])); - LDR(Q1, XSP, offsetof(StackLayout::Thunk, xmm[1])); - LDR(Q2, XSP, offsetof(StackLayout::Thunk, xmm[2])); - LDR(Q3, XSP, offsetof(StackLayout::Thunk, xmm[3])); - LDR(Q4, XSP, offsetof(StackLayout::Thunk, xmm[4])); - LDR(Q5, XSP, offsetof(StackLayout::Thunk, xmm[5])); - LDR(Q6, XSP, offsetof(StackLayout::Thunk, xmm[6])); - LDR(Q7, XSP, offsetof(StackLayout::Thunk, xmm[7])); - - LDR(Q8, XSP, offsetof(StackLayout::Thunk, xmm[8])); - LDR(Q9, XSP, offsetof(StackLayout::Thunk, xmm[9])); - LDR(Q10, XSP, offsetof(StackLayout::Thunk, xmm[10])); - LDR(Q11, XSP, offsetof(StackLayout::Thunk, xmm[11])); - LDR(Q12, XSP, offsetof(StackLayout::Thunk, xmm[12])); - LDR(Q13, XSP, offsetof(StackLayout::Thunk, xmm[13])); - LDR(Q14, XSP, offsetof(StackLayout::Thunk, xmm[14])); - LDR(Q15, XSP, offsetof(StackLayout::Thunk, xmm[15])); - LDR(Q16, XSP, offsetof(StackLayout::Thunk, xmm[16])); - LDR(Q17, XSP, offsetof(StackLayout::Thunk, xmm[17])); - LDR(Q18, XSP, offsetof(StackLayout::Thunk, xmm[18])); - LDR(Q19, XSP, offsetof(StackLayout::Thunk, xmm[19])); - LDR(Q20, XSP, offsetof(StackLayout::Thunk, xmm[20])); - LDR(Q21, XSP, offsetof(StackLayout::Thunk, xmm[21])); - LDR(Q22, XSP, offsetof(StackLayout::Thunk, xmm[22])); - LDR(Q23, XSP, offsetof(StackLayout::Thunk, xmm[23])); - LDR(Q24, XSP, offsetof(StackLayout::Thunk, xmm[24])); - LDR(Q25, XSP, offsetof(StackLayout::Thunk, xmm[25])); - LDR(Q26, XSP, offsetof(StackLayout::Thunk, xmm[26])); - LDR(Q27, XSP, offsetof(StackLayout::Thunk, xmm[27])); - LDR(Q28, XSP, offsetof(StackLayout::Thunk, xmm[28])); - LDR(Q29, XSP, offsetof(StackLayout::Thunk, xmm[29])); - LDR(Q30, XSP, offsetof(StackLayout::Thunk, xmm[30])); - LDR(Q31, XSP, offsetof(StackLayout::Thunk, xmm[31])); + // LDR(Q0, SP, offsetof(StackLayout::Thunk, xmm[0])); + LDP(Q1, Q2, SP, offsetof(StackLayout::Thunk, xmm[0])); + LDP(Q3, Q4, SP, offsetof(StackLayout::Thunk, xmm[2])); + LDP(Q5, Q6, SP, offsetof(StackLayout::Thunk, xmm[4])); + LDP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); + LDP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); + LDP(Q17, Q18, SP, offsetof(StackLayout::Thunk, xmm[8])); + LDP(Q19, Q20, SP, offsetof(StackLayout::Thunk, xmm[10])); + LDP(Q21, Q22, SP, offsetof(StackLayout::Thunk, xmm[12])); + LDP(Q23, Q24, SP, offsetof(StackLayout::Thunk, xmm[14])); + LDP(Q25, Q26, SP, offsetof(StackLayout::Thunk, xmm[16])); + LDP(Q27, Q28, SP, offsetof(StackLayout::Thunk, xmm[18])); + LDP(Q29, Q30, SP, offsetof(StackLayout::Thunk, xmm[20])); + LDR(Q31, SP, offsetof(StackLayout::Thunk, xmm[21])); } +// Callee saved: +// x19-x30 | d8-d15 void A64ThunkEmitter::EmitSaveNonvolatileRegs() { - STR(X19, XSP, offsetof(StackLayout::Thunk, r[0])); - STR(X20, XSP, offsetof(StackLayout::Thunk, r[1])); - STR(X21, XSP, offsetof(StackLayout::Thunk, r[2])); - STR(X22, XSP, offsetof(StackLayout::Thunk, r[3])); - STR(X23, XSP, offsetof(StackLayout::Thunk, r[4])); - STR(X24, XSP, offsetof(StackLayout::Thunk, r[5])); - STR(X25, XSP, offsetof(StackLayout::Thunk, r[6])); - STR(X26, XSP, offsetof(StackLayout::Thunk, r[7])); - STR(X27, XSP, offsetof(StackLayout::Thunk, r[8])); - STR(X28, XSP, offsetof(StackLayout::Thunk, r[9])); - STR(X29, XSP, offsetof(StackLayout::Thunk, r[10])); - STR(X30, XSP, offsetof(StackLayout::Thunk, r[11])); + STP(X19, X20, SP, offsetof(StackLayout::Thunk, r[0])); + STP(X21, X22, SP, offsetof(StackLayout::Thunk, r[2])); + STP(X23, X24, SP, offsetof(StackLayout::Thunk, r[4])); + STP(X25, X26, SP, offsetof(StackLayout::Thunk, r[6])); + STP(X27, X28, SP, offsetof(StackLayout::Thunk, r[8])); + STP(X29, X30, SP, offsetof(StackLayout::Thunk, r[10])); + + STP(Q8, Q9, SP, offsetof(StackLayout::Thunk, xmm[0])); + STP(Q10, Q11, SP, offsetof(StackLayout::Thunk, xmm[2])); + STP(Q12, Q13, SP, offsetof(StackLayout::Thunk, xmm[4])); + STP(Q14, Q15, SP, offsetof(StackLayout::Thunk, xmm[6])); } void A64ThunkEmitter::EmitLoadNonvolatileRegs() { - LDR(X19, XSP, offsetof(StackLayout::Thunk, r[0])); - LDR(X20, XSP, offsetof(StackLayout::Thunk, r[1])); - LDR(X21, XSP, offsetof(StackLayout::Thunk, r[2])); - LDR(X22, XSP, offsetof(StackLayout::Thunk, r[3])); - LDR(X23, XSP, offsetof(StackLayout::Thunk, r[4])); - LDR(X24, XSP, offsetof(StackLayout::Thunk, r[5])); - LDR(X25, XSP, offsetof(StackLayout::Thunk, r[6])); - LDR(X26, XSP, offsetof(StackLayout::Thunk, r[7])); - LDR(X27, XSP, offsetof(StackLayout::Thunk, r[8])); - LDR(X28, XSP, offsetof(StackLayout::Thunk, r[9])); - LDR(X29, XSP, offsetof(StackLayout::Thunk, r[10])); - LDR(X30, XSP, offsetof(StackLayout::Thunk, r[11])); + LDP(X19, X20, SP, offsetof(StackLayout::Thunk, r[0])); + LDP(X21, X22, SP, offsetof(StackLayout::Thunk, r[2])); + LDP(X23, X24, SP, offsetof(StackLayout::Thunk, r[4])); + LDP(X25, X26, SP, offsetof(StackLayout::Thunk, r[6])); + LDP(X27, X28, SP, offsetof(StackLayout::Thunk, r[8])); + LDP(X29, X30, SP, offsetof(StackLayout::Thunk, r[10])); + + LDP(Q8, Q9, SP, offsetof(StackLayout::Thunk, xmm[0])); + LDP(Q10, Q11, SP, offsetof(StackLayout::Thunk, xmm[2])); + LDP(Q12, Q13, SP, offsetof(StackLayout::Thunk, xmm[4])); + LDP(Q14, Q15, SP, offsetof(StackLayout::Thunk, xmm[6])); } } // namespace a64 From 4962ed8bc1cfc64f64d53b049189f1dc882140cd Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 21:14:52 -0700 Subject: [PATCH 037/179] [a64] Refactor thunk prolog/epilog --- a64_backend.cc | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index f40b80d93..211ddc97d 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -214,14 +214,18 @@ HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { code_offsets.prolog = offset(); - // mov(qword[rsp + 8 * 3], r8); - // mov(qword[rsp + 8 * 2], rdx); - // mov(qword[rsp + 8 * 1], rcx); - // sub(rsp, stack_size); - STR(X2, XSP, 8 * 3); - STR(X1, XSP, 8 * 2); - STR(X0, XSP, 8 * 1); - SUB(XSP, XSP, stack_size); + // mov(qword[rsp + 8 * 3], r8); + // mov(qword[rsp + 8 * 2], rdx); + // mov(qword[rsp + 8 * 1], rcx); + // sub(rsp, stack_size); + + STP(X29, X30, SP, PRE_INDEXED, -32); + MOV(X29, SP); + + STR(X2, SP, 8 * 3); + STR(X1, SP, 8 * 2); + STR(X0, SP, 8 * 1); + SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); @@ -249,10 +253,14 @@ HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { // mov(r8, qword[rsp + 8 * 3]); // ret(); - ADD(XSP, XSP, stack_size); - LDR(X0, XSP, 8 * 1); - LDR(X1, XSP, 8 * 2); - LDR(X2, XSP, 8 * 3); + ADD(SP, SP, stack_size); + LDR(X0, SP, 8 * 1); + LDR(X1, SP, 8 * 2); + LDR(X2, SP, 8 * 3); + + MOV(SP, X29); + LDP(X29, X30, SP, POST_INDEXED, 32); + RET(); code_offsets.tail = offset(); @@ -290,15 +298,18 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { code_offsets.prolog = offset(); + STP(X29, X30, SP, PRE_INDEXED, -32); + MOV(X29, SP); // rsp + 0 = return address // sub(rsp, stack_size); - SUB(XSP, XSP, stack_size); + SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); // Save off volatile registers. EmitSaveVolatileRegs(); + MOV(X29, SP); // mov(rax, rcx); // function // mov(rcx, GetContextReg()); // context @@ -307,13 +318,17 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { MOV(X0, GetContextReg()); // context BLR(X16); + MOV(SP, X29); + EmitLoadVolatileRegs(); code_offsets.epilog = offset(); // add(rsp, stack_size); // ret(); - ADD(XSP, XSP, stack_size); + ADD(SP, SP, stack_size); + MOV(SP, X29); + LDP(X29, X30, SP, POST_INDEXED, 32); RET(); code_offsets.tail = offset(); @@ -383,7 +398,9 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // add(rsp, stack_size); // jmp(rax); - ADD(XSP, XSP, stack_size); + ADD(SP, SP, stack_size); + MOV(SP, X29); + LDP(X29, X30, SP, POST_INDEXED, 32); BR(X0); code_offsets.tail = offset(); From 2471941d0ab611463bb61671b07789fc24c2c98f Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 21:15:49 -0700 Subject: [PATCH 038/179] [a64] Update Membase and Context register --- a64_emitter.cc | 4 ++-- a64_emitter.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 1f84a0e8f..8c06eaabc 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -689,8 +689,8 @@ oaknut::XReg A64Emitter::GetNativeParam(uint32_t param) { } // Important: If you change these, you must update the thunks in a64_backend.cc! -oaknut::XReg A64Emitter::GetContextReg() { return X14; } -oaknut::XReg A64Emitter::GetMembaseReg() { return X15; } +oaknut::XReg A64Emitter::GetContextReg() { return X19; } +oaknut::XReg A64Emitter::GetMembaseReg() { return X20; } void A64Emitter::ReloadContext() { // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); diff --git a/a64_emitter.h b/a64_emitter.h index a0f4a88c6..d44700847 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -137,7 +137,7 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { std::vector* out_source_map); public: - // Reserved: XSP, X14, X15 + // Reserved: XSP, X19, X20 // Scratch: X0/X1/X2 // V0-2 // Available: X1, X10-r15 From 214fcb6c12fe8b7adb0c431f2b9f9b4101b08f06 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 21:17:57 -0700 Subject: [PATCH 039/179] [a64] Fix emitted function prolog/epilog --- a64_emitter.cc | 19 +++++++++----- a64_stack_layout.h | 64 +++++++++++++++++++++++----------------------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 8c06eaabc..ff7883cd9 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -214,8 +214,11 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { func_info.stack_size = stack_size; stack_size_ = stack_size; + STP(X29, X30, SP, PRE_INDEXED, -32); + MOV(X29, SP); + // sub(rsp, (uint32_t)stack_size); - SUB(XSP, XSP, stack_size); + SUB(SP, SP, (uint32_t)stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); @@ -223,9 +226,9 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // mov(qword[rsp + StackLayout::GUEST_CTX_HOME], GetContextReg()); // mov(qword[rsp + StackLayout::GUEST_RET_ADDR], rcx); // mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], 0); - STR(GetContextReg(), XSP, StackLayout::GUEST_CTX_HOME); - STR(X0, XSP, StackLayout::GUEST_RET_ADDR); - STR(XZR, XSP, StackLayout::GUEST_CALL_RET_ADDR); + STR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); + STR(X0, SP, StackLayout::GUEST_RET_ADDR); + STR(XZR, SP, StackLayout::GUEST_CALL_RET_ADDR); // Safe now to do some tracing. if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctions) { @@ -294,13 +297,17 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { epilog_label_ = nullptr; EmitTraceUserCallReturn(); // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); - LDR(GetContextReg(), XSP, StackLayout::GUEST_CTX_HOME); + LDR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); code_offsets.epilog = offset(); // add(rsp, (uint32_t)stack_size); // ret(); - ADD(XSP, XSP, stack_size); + ADD(SP, SP, (uint32_t)stack_size); + + MOV(SP, X29); + LDP(X29, X30, SP, POST_INDEXED, 32); + RET(); code_offsets.tail = offset(); diff --git a/a64_stack_layout.h b/a64_stack_layout.h index 64deece50..72ded80dc 100644 --- a/a64_stack_layout.h +++ b/a64_stack_layout.h @@ -29,91 +29,91 @@ class StackLayout { * Thunk stack: * Non-Volatile Volatile * +------------------+------------------+ - * | arg temp, 3 * 8 | arg temp, 3 * 8 | rsp + 0x000 + * | arg temp, 3 * 8 | arg temp, 3 * 8 | xsp + 0x000 * | | | * | | | * +------------------+------------------+ - * | rbx | (unused) | rsp + 0x018 + * | rbx | (unused) | xsp + 0x018 * +------------------+------------------+ - * | rbp | rcx | rsp + 0x020 + * | rbp | X1 | xsp + 0x020 * +------------------+------------------+ - * | rcx (Win32) | rdx | rsp + 0x028 + * | rcx (Win32) | X2 | xsp + 0x028 * +------------------+------------------+ - * | rsi (Win32) | rsi (Linux) | rsp + 0x030 + * | rsi (Win32) | X3 | xsp + 0x030 * +------------------+------------------+ - * | rdi (Win32) | rdi (Linux) | rsp + 0x038 + * | rdi (Win32) | X4 | xsp + 0x038 * +------------------+------------------+ - * | r12 | r8 | rsp + 0x040 + * | r12 | X5 | xsp + 0x040 * +------------------+------------------+ - * | r13 | r9 | rsp + 0x048 + * | r13 | X6 | xsp + 0x048 * +------------------+------------------+ - * | r14 | r10 | rsp + 0x050 + * | r14 | X7 | xsp + 0x050 * +------------------+------------------+ - * | r15 | r11 | rsp + 0x058 + * | r15 | X8 | xsp + 0x058 * +------------------+------------------+ - * | xmm6 (Win32) | (unused) | rsp + 0x060 + * | xmm6 (Win32) | X9 | xsp + 0x060 * | | | * +------------------+------------------+ - * | xmm7 (Win32) | xmm1 | rsp + 0x070 + * | xmm7 (Win32) | X10 | xsp + 0x070 * | | | * +------------------+------------------+ - * | xmm8 (Win32) | xmm2 | rsp + 0x080 + * | xmm8 (Win32) | X11 | xsp + 0x080 * | | | * +------------------+------------------+ - * | xmm9 (Win32) | xmm3 | rsp + 0x090 + * | xmm9 (Win32) | X12 | xsp + 0x090 * | | | * +------------------+------------------+ - * | xmm10 (Win32) | xmm4 | rsp + 0x0A0 + * | xmm10 (Win32) | X13 | xsp + 0x0A0 * | | | * +------------------+------------------+ - * | xmm11 (Win32) | xmm5 | rsp + 0x0B0 + * | xmm11 (Win32) | X14 | xsp + 0x0B0 * | | | * +------------------+------------------+ - * | xmm12 (Win32) | (unused) | rsp + 0x0C0 + * | xmm12 (Win32) | X15 | xsp + 0x0C0 * | | | * +------------------+------------------+ - * | xmm13 (Win32) | (unused) | rsp + 0x0D0 + * | xmm13 (Win32) | X16 | xsp + 0x0D0 * | | | * +------------------+------------------+ - * | xmm14 (Win32) | (unused) | rsp + 0x0E0 + * | xmm14 (Win32) | X17 | xsp + 0x0E0 * | | | * +------------------+------------------+ - * | xmm15 (Win32) | (unused) | rsp + 0x0F0 + * | xmm15 (Win32) | X18 | xsp + 0x0F0 * | | | * +------------------+------------------+ - * | (return address) | (return address) | rsp + 0x100 + * | (return address) | (return address) | xsp + 0x100 * +------------------+------------------+ - * | (rcx home) | (rcx home) | rsp + 0x108 + * | (rcx home) | (rcx home) | xsp + 0x108 * +------------------+------------------+ - * | (rdx home) | (rdx home) | rsp + 0x110 + * | (rdx home) | (rdx home) | xsp + 0x110 * +------------------+------------------+ */ XEPACKEDSTRUCT(Thunk, { uint64_t arg_temp[3]; - uint64_t r[20]; - vec128_t xmm[32]; + uint64_t r[17]; + vec128_t xmm[22]; }); static_assert(sizeof(Thunk) % 16 == 0, "sizeof(Thunk) must be a multiple of 16!"); - static const size_t THUNK_STACK_SIZE = sizeof(Thunk); + static const size_t THUNK_STACK_SIZE = sizeof(Thunk) + 16; /** * * * Guest stack: * +------------------+ - * | arg temp, 3 * 8 | rsp + 0 + * | arg temp, 3 * 8 | xsp + 0 * | | * | | * +------------------+ - * | scratch, 48b | rsp + 32 + * | scratch, 48b | xsp + 32 * | | * +------------------+ - * | rcx / context | rsp + 80 + * | X0 / context | xsp + 80 * +------------------+ - * | guest ret addr | rsp + 88 + * | guest ret addr | xsp + 88 * +------------------+ - * | call ret addr | rsp + 96 + * | call ret addr | xsp + 96 * +------------------+ * ... locals ... * +------------------+ @@ -121,7 +121,7 @@ class StackLayout { * +------------------+ * */ - static const size_t GUEST_STACK_SIZE = 96; + static const size_t GUEST_STACK_SIZE = 96 + 16; static const size_t GUEST_CTX_HOME = 80; static const size_t GUEST_RET_ADDR = 88; static const size_t GUEST_CALL_RET_ADDR = 96; From 4199c581322243656830018284e4ac05b001445b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 5 May 2024 21:18:09 -0700 Subject: [PATCH 040/179] [a64] Refactor XSP to SP --- a64_emitter.cc | 95 +++++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 63 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index ff7883cd9..e36490ea5 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -438,7 +438,7 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { assert_not_null(function); auto fn = static_cast(function); - // Resolve address to the function to call and store in rax. + // Resolve address to the function to call and store in X16. if (fn->machine_code()) { // TODO(benvanik): is it worth it to do this? It removes the need for // a ResolveFunction call, but makes the table less useful. @@ -451,35 +451,36 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { // or a thunk to ResolveAddress. // mov(ebx, function->address()); // mov(eax, dword[ebx]); - MOV(X16, function->address()); - LDR(W16, X16); + MOV(W1, function->address()); + LDR(W16, X1); } else { // Old-style resolve. // Not too important because indirection table is almost always available. // TODO: Overwrite the call-site with a straight call. CallNative(&ResolveFunction, function->address()); + MOV(X16, X0); } - // Actually jump/call to rax. + // Actually jump/call to X16. if (instr->flags & hir::CALL_TAIL) { // Since we skip the prolog we need to mark the return here. EmitTraceUserCallReturn(); // Pass the callers return address over. // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); - LDR(X0, XSP, StackLayout::GUEST_RET_ADDR); + LDR(X0, SP, StackLayout::GUEST_RET_ADDR); // add(rsp, static_cast(stack_size())); // jmp(rax); - ADD(XSP, XSP, stack_size()); + ADD(SP, SP, stack_size()); BR(X16); } else { // Return address is from the previous SET_RETURN_ADDRESS. // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); - LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); // call(rax); - BR(X16); + BLR(X16); } } @@ -489,8 +490,8 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, if (instr->flags & hir::CALL_POSSIBLE_RETURN) { // cmp(reg.cvt32(), dword[rsp + StackLayout::GUEST_RET_ADDR]); // je(epilog_label(), CodeGenerator::T_NEAR); - LDR(W0, XSP, StackLayout::GUEST_RET_ADDR); - CMP(reg.toW(), W0); + LDR(W16, SP, StackLayout::GUEST_RET_ADDR); + CMP(reg.toW(), W16); B(oaknut::Cond::EQ, epilog_label()); } @@ -503,7 +504,7 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, MOV(W1, reg.toW()); } // mov(eax, dword[ebx]); - LDR(X16, X1); + LDR(W16, X1); } else { // Old-style resolve. // Not too important because indirection table is almost always available. @@ -516,27 +517,21 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, MOV(W1, reg.toW()); ADRP(X16, ResolveFunction); - - // Preserve frame and link register - STP(X29, X30, XSP, POST_INDEXED, -16); - BLR(X16); - - // Restore frame and link register - LDP(X29, X30, XSP, PRE_INDEXED, 16); + MOV(X16, X0); } - // Actually jump/call to rax. + // Actually jump/call to X16. if (instr->flags & hir::CALL_TAIL) { // Since we skip the prolog we need to mark the return here. EmitTraceUserCallReturn(); // Pass the callers return address over. // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); - LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); // add(rsp, static_cast(stack_size())); - ADD(XSP, XSP, stack_size()); + ADD(SP, SP, static_cast(stack_size())); // jmp(rax); BR(X16); @@ -544,15 +539,9 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, // Return address is from the previous SET_RETURN_ADDRESS. // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); // call(rax); - LDR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); - - // Preserve frame and link register - STP(X29, X30, XSP, POST_INDEXED, -16); + LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); BLR(X16); - - // Restore frame and link register - LDP(X29, X30, XSP, PRE_INDEXED, 16); } } @@ -577,7 +566,6 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { // x1 = arg0 // x2 = arg1 // x3 = arg2 - auto thunk = backend()->guest_to_host_thunk(); // mov(rax, reinterpret_cast(thunk)); // mov(rcx, reinterpret_cast(builtin_function->handler())); // mov(rdx, reinterpret_cast(builtin_function->arg0())); @@ -587,16 +575,11 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { MOV(X1, reinterpret_cast(builtin_function->arg0())); MOV(X2, reinterpret_cast(builtin_function->arg1())); + auto thunk = backend()->guest_to_host_thunk(); MOV(X16, reinterpret_cast(thunk)); - // Preserve frame and link register - STP(X29, X30, XSP, POST_INDEXED, -16); - BLR(X16); - // Restore frame and link register - LDP(X29, X30, XSP, PRE_INDEXED, 16); - // x0 = host return } } else if (function->behavior() == Function::Behavior::kExtern) { @@ -620,14 +603,8 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { MOV(X16, reinterpret_cast(thunk)); - // Preserve frame and link register - STP(X29, X30, XSP, POST_INDEXED, -16); - BLR(X16); - // Restore frame and link register - LDP(X29, X30, XSP, PRE_INDEXED, 16); - // x0 = host return } } @@ -659,20 +636,12 @@ void A64Emitter::CallNativeSafe(void* fn) { // X2 = arg1 // X3 = arg2 auto thunk = backend()->guest_to_host_thunk(); - // mov(rax, reinterpret_cast(thunk)); - // mov(rcx, reinterpret_cast(fn)); - // call(rax); + MOV(X0, reinterpret_cast(fn)); + MOV(X16, reinterpret_cast(thunk)); - - // Preserve frame and link register - STP(X29, X30, XSP, POST_INDEXED, -16); - BLR(X16); - // Restore frame and link register - LDP(X29, X30, XSP, PRE_INDEXED, 16); - // X0 = host return } @@ -680,7 +649,7 @@ void A64Emitter::SetReturnAddress(uint64_t value) { // mov(rax, value); // mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], rax); MOV(X0, value); - STR(X0, XSP, StackLayout::GUEST_CALL_RET_ADDR); + STR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); } oaknut::XReg A64Emitter::GetNativeParam(uint32_t param) { @@ -701,7 +670,7 @@ oaknut::XReg A64Emitter::GetMembaseReg() { return X20; } void A64Emitter::ReloadContext() { // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); - LDR(GetContextReg(), XSP, StackLayout::GUEST_CTX_HOME); + LDR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); } void A64Emitter::ReloadMembase() { @@ -914,9 +883,9 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. - MovMem64(XSP, kStashOffset, v.low); - MovMem64(XSP, kStashOffset + 8, v.high); - LDR(dest, XSP, kStashOffset); + MovMem64(SP, kStashOffset, v.low); + MovMem64(SP, kStashOffset + 8, v.high); + LDR(dest, SP, kStashOffset); } } @@ -966,7 +935,7 @@ uintptr_t A64Emitter::StashV(int index, const oaknut::QReg& r) { // auto addr = ptr[rsp + kStashOffset + (index * 16)]; // vmovups(addr, r); const auto addr = kStashOffset + (index * 16); - STR(r, XSP, addr); + STR(r, SP, addr); return addr; } @@ -976,8 +945,8 @@ uintptr_t A64Emitter::StashConstantV(int index, float v) { uint32_t i; } x = {v}; const auto addr = kStashOffset + (index * 16); - MovMem64(XSP, addr, x.i); - MovMem64(XSP, addr + 8, 0); + MovMem64(SP, addr, x.i); + MovMem64(SP, addr + 8, 0); return addr; } @@ -987,15 +956,15 @@ uintptr_t A64Emitter::StashConstantV(int index, double v) { uint64_t i; } x = {v}; const auto addr = kStashOffset + (index * 16); - MovMem64(XSP, addr, x.i); - MovMem64(XSP, addr + 8, 0); + MovMem64(SP, addr, x.i); + MovMem64(SP, addr + 8, 0); return addr; } uintptr_t A64Emitter::StashConstantV(int index, const vec128_t& v) { const auto addr = kStashOffset + (index * 16); - MovMem64(XSP, addr, v.low); - MovMem64(XSP, addr + 8, v.high); + MovMem64(SP, addr, v.low); + MovMem64(SP, addr + 8, v.high); return addr; } From ff68e1b3048924a35ae811001bcb0dfed73e69b3 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 12:50:47 -0700 Subject: [PATCH 041/179] [a64] Implement `OPCODE_{LOAD,STORE}_MMIO` --- a64_seq_memory.cc | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 4b86aa4cf..06169f130 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -566,17 +566,16 @@ struct LOAD_MMIO_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // uint64_t (context, addr) - auto mmio_range = reinterpret_cast(i.src1.value); - auto read_address = uint32_t(i.src2.value); - // e.mov(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); - // e.mov(e.GetNativeParam(1).cvt32(), read_address); - // e.CallNativeSafe(reinterpret_cast(mmio_range->read)); - // e.bswap(e.eax); - // e.mov(i.dest, e.eax); + const auto mmio_range = reinterpret_cast(i.src1.value); + const auto read_address = uint32_t(i.src2.value); + e.MOV(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); + e.MOV(e.GetNativeParam(1).toW(), read_address); + e.CallNativeSafe(reinterpret_cast(mmio_range->read)); + e.REV(i.dest, W0); if (IsTracingData()) { - // e.mov(e.GetNativeParam(0), i.dest); - // e.mov(e.edx, read_address); - // e.CallNative(reinterpret_cast(TraceContextLoadI32)); + e.MOV(e.GetNativeParam(0).toW(), i.dest); + e.MOV(X1, read_address); + e.CallNative(reinterpret_cast(TraceContextLoadI32)); } } }; @@ -591,25 +590,24 @@ struct STORE_MMIO_I32 I> { static void Emit(A64Emitter& e, const EmitArgType& i) { // void (context, addr, value) - auto mmio_range = reinterpret_cast(i.src1.value); - auto write_address = uint32_t(i.src2.value); - // e.mov(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); - // e.mov(e.GetNativeParam(1).cvt32(), write_address); + const auto mmio_range = reinterpret_cast(i.src1.value); + const auto write_address = uint32_t(i.src2.value); + e.MOV(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); + e.MOV(e.GetNativeParam(1).toW(), write_address); if (i.src3.is_constant) { - // e.mov(e.GetNativeParam(2).cvt32(), xe::byte_swap(i.src3.constant())); + e.MOV(e.GetNativeParam(2).toW(), xe::byte_swap(i.src3.constant())); } else { - // e.mov(e.GetNativeParam(2).cvt32(), i.src3); - // e.bswap(e.GetNativeParam(2).cvt32()); + e.REV(e.GetNativeParam(2).toW(), i.src3); } - // e.CallNativeSafe(reinterpret_cast(mmio_range->write)); + e.CallNativeSafe(reinterpret_cast(mmio_range->write)); if (IsTracingData()) { if (i.src3.is_constant) { - // e.mov(e.GetNativeParam(0).cvt32(), i.src3.constant()); + e.MOV(e.GetNativeParam(0).toW(), i.src3.constant()); } else { - // e.mov(e.GetNativeParam(0).cvt32(), i.src3); + e.MOV(e.GetNativeParam(0).toW(), i.src3); } - // e.mov(e.edx, write_address); - // e.CallNative(reinterpret_cast(TraceContextStoreI32)); + e.MOV(X1, write_address); + e.CallNative(reinterpret_cast(TraceContextStoreI32)); } } }; From d43219ea737f615329276c34eb5802afcd0e406b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 12:51:59 -0700 Subject: [PATCH 042/179] [a64] Remove redundant zero-extension during address computation Also changes the register to X3 by default --- a64_seq_memory.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 06169f130..63ae0d810 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -25,7 +25,7 @@ volatile int anchor_memory = 0; template XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, - XReg address_register = X4) { + XReg address_register = X3) { assert_true(offset.is_constant); int32_t offset_const = static_cast(offset.constant()); @@ -51,27 +51,26 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, // it via memory mapping. e.MOV(W1, 0xE0000000 - offset_const); e.CMP(guest.reg().toW(), W1); - e.CSET(X0, Cond::HS); - e.LSL(X0, X0, 12); - e.ADD(X0, X0, guest.reg()); - e.MOV(W0, W0); + e.CSET(W0, Cond::HS); + e.LSL(W0, W0, 12); + e.ADD(W0, W0, guest.reg().toW()); } else { // Clear the top 32 bits, as they are likely garbage. // TODO(benvanik): find a way to avoid doing this. e.MOV(W0, guest.reg().toW()); } - e.MOV(address_register, offset_const); - e.ADD(address_register, X0, address_register); - e.ADD(address_register, address_register, e.GetMembaseReg()); + e.ADD(address_register, e.GetMembaseReg(), X0); + + e.MOV(X0, offset_const); + e.ADD(address_register, address_register, X0); return address_register; - // return e.GetMembaseReg() + e.rax + offset_const; } } // Note: most *should* be aligned, but needs to be checked! template XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, - XReg address_register = X4) { + XReg address_register = X3) { if (guest.is_constant) { // TODO(benvanik): figure out how to do this without a temp. // Since the constant is often 0x8... if we tried to use that as a From b4414e393f9c907fb3de017bd1b7a973464474ef Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 12:53:59 -0700 Subject: [PATCH 043/179] [a64] Fix `CallIndirect` return address Should be `GUEST_RET_ADDR` not `GUEST_CALL_RET_ADDR`. --- a64_emitter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index e36490ea5..744e2548a 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -528,7 +528,7 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, // Pass the callers return address over. // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); - LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); + LDR(X0, SP, StackLayout::GUEST_RET_ADDR); // add(rsp, static_cast(stack_size())); ADD(SP, SP, static_cast(stack_size())); From 4144f2ce6494060943c9b8e937b0aa2e936b910b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 12:57:16 -0700 Subject: [PATCH 044/179] [a64] Refactor `REV{32,64}` to `REV` Let the register type determine the reverse-size REV32 was also the wrong instruction to use. --- a64_seq_memory.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 63ae0d810..9c8f2535d 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -642,7 +642,7 @@ struct LOAD_OFFSET_I32 auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { e.LDR(i.dest, addr_reg); - e.REV32(i.dest.reg().toX(), i.dest.reg().toX()); + e.REV(i.dest.reg().toX(), i.dest.reg().toX()); } else { e.LDR(i.dest, addr_reg); } @@ -655,7 +655,7 @@ struct LOAD_OFFSET_I64 auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { e.LDR(i.dest, addr_reg); - e.REV64(i.dest, i.dest); + e.REV(i.dest, i.dest); } else { e.LDR(i.dest, addr_reg); } @@ -774,7 +774,7 @@ struct LOAD_I32 : Sequence> { auto addr_reg = ComputeMemoryAddress(e, i.src1); if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { e.LDR(i.dest, addr_reg); - e.REV32(i.dest.reg().toX(), i.dest.reg().toX()); + e.REV(i.dest, i.dest); } else { e.LDR(i.dest, addr_reg); } From 7ffbc3a084ffa14faeb689209b9c33e37ccb7fd7 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 12:59:30 -0700 Subject: [PATCH 045/179] [a64] Implement `OPCODE_MEMSET` --- a64_seq_memory.cc | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 9c8f2535d..28bd3c414 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -1116,23 +1116,22 @@ struct MEMSET_I64_I8_I64 assert_true(i.src2.is_constant); assert_true(i.src3.is_constant); assert_true(i.src2.constant() == 0); - // e.vpxor(e.xmm0, e.xmm0); e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); auto addr_reg = ComputeMemoryAddress(e, i.src1); switch (i.src3.constant()) { case 32: - // e.vmovaps(e.ptr[addr + 0 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 1 * 16], e.xmm0); + e.STR(Q0, addr_reg, 0 * 16); + e.STR(Q0, addr_reg, 1 * 16); break; case 128: - // e.vmovaps(e.ptr[addr + 0 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 1 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 2 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 3 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 4 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 5 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 6 * 16], e.xmm0); - // e.vmovaps(e.ptr[addr + 7 * 16], e.xmm0); + e.STR(Q0, addr_reg, 0 * 16); + e.STR(Q0, addr_reg, 1 * 16); + e.STR(Q0, addr_reg, 2 * 16); + e.STR(Q0, addr_reg, 3 * 16); + e.STR(Q0, addr_reg, 4 * 16); + e.STR(Q0, addr_reg, 5 * 16); + e.STR(Q0, addr_reg, 6 * 16); + e.STR(Q0, addr_reg, 7 * 16); break; default: assert_unhandled_case(i.src3.constant()); @@ -1140,10 +1139,10 @@ struct MEMSET_I64_I8_I64 } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.mov(e.GetNativeParam(2), i.src3.constant()); - // e.mov(e.GetNativeParam(1), i.src2.constant()); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemset)); + e.MOV(e.GetNativeParam(2), i.src3.constant()); + e.MOV(e.GetNativeParam(1), i.src2.constant()); + e.LDR(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemset)); } } }; From a7498d5ede7d21833ddab5df4b1838732441b0b0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 13:13:05 -0700 Subject: [PATCH 046/179] [a64] Implement `OPCODE_MEMORY_BARRIER` --- a64_seq_memory.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 28bd3c414..671470eba 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -1101,7 +1101,11 @@ EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); struct MEMORY_BARRIER : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + // mfence on x64 flushes all writes before any later instructions // e.mfence(); + + // This is equivalent to DMB SY + e.DMB(BarrierOp::SY); } }; EMITTER_OPCODE_TABLE(OPCODE_MEMORY_BARRIER, MEMORY_BARRIER); From 8b17a6ec01b7be582840f08b3bdb0d727914a0e3 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 13:22:09 -0700 Subject: [PATCH 047/179] [a64] Implement `OPCODE_{LOAD,STORE}_LOCAL` --- a64_seq_memory.cc | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 671470eba..e12bb9fd7 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -227,6 +227,7 @@ EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, struct LOAD_LOCAL_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + e.LDRB(i.dest, SP, i.src1.constant()); // e.mov(i.dest, e.byte[e.rsp + i.src1.constant()]); // e.TraceLoadI8(DATA_LOCAL, i.src1.constant, i.dest); } @@ -234,42 +235,42 @@ struct LOAD_LOCAL_I8 struct LOAD_LOCAL_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.mov(i.dest, e.word[e.rsp + i.src1.constant()]); + e.LDRH(i.dest, SP, i.src1.constant()); // e.TraceLoadI16(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.mov(i.dest, e.dword[e.rsp + i.src1.constant()]); + e.LDR(i.dest, SP, i.src1.constant()); // e.TraceLoadI32(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.mov(i.dest, e.qword[e.rsp + i.src1.constant()]); + e.LDR(i.dest, SP, i.src1.constant()); // e.TraceLoadI64(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vmovss(i.dest, e.dword[e.rsp + i.src1.constant()]); + e.LDR(i.dest, SP, i.src1.constant()); // e.TraceLoadF32(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vmovsd(i.dest, e.qword[e.rsp + i.src1.constant()]); + e.LDR(i.dest, SP, i.src1.constant()); // e.TraceLoadF64(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vmovaps(i.dest, e.ptr[e.rsp + i.src1.constant()]); + e.LDR(i.dest, SP, i.src1.constant()); // e.TraceLoadV128(DATA_LOCAL, i.src1.constant, i.dest); } }; @@ -285,49 +286,49 @@ struct STORE_LOCAL_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI8(DATA_LOCAL, i.src1.constant, i.src2); - // e.mov(e.byte[e.rsp + i.src1.constant()], i.src2); + e.STRB(i.src2, SP, i.src1.constant()); } }; struct STORE_LOCAL_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI16(DATA_LOCAL, i.src1.constant, i.src2); - // e.mov(e.word[e.rsp + i.src1.constant()], i.src2); + e.STRH(i.src2, SP, i.src1.constant()); } }; struct STORE_LOCAL_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI32(DATA_LOCAL, i.src1.constant, i.src2); - // e.mov(e.dword[e.rsp + i.src1.constant()], i.src2); + e.STR(i.src2, SP, i.src1.constant()); } }; struct STORE_LOCAL_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI64(DATA_LOCAL, i.src1.constant, i.src2); - // e.mov(e.qword[e.rsp + i.src1.constant()], i.src2); + e.STR(i.src2, SP, i.src1.constant()); } }; struct STORE_LOCAL_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreF32(DATA_LOCAL, i.src1.constant, i.src2); - // e.vmovss(e.dword[e.rsp + i.src1.constant()], i.src2); + e.STR(i.src2, SP, i.src1.constant()); } }; struct STORE_LOCAL_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreF64(DATA_LOCAL, i.src1.constant, i.src2); - // e.vmovsd(e.qword[e.rsp + i.src1.constant()], i.src2); + e.STR(i.src2, SP, i.src1.constant()); } }; struct STORE_LOCAL_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreV128(DATA_LOCAL, i.src1.constant, i.src2); - // e.vmovaps(e.ptr[e.rsp + i.src1.constant()], i.src2); + e.STR(i.src2, SP, i.src1.constant()); } }; EMITTER_OPCODE_TABLE(OPCODE_STORE_LOCAL, STORE_LOCAL_I8, STORE_LOCAL_I16, From b16ef6830d03226e5b2fb7a1392994b25a02701a Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 15:47:33 -0700 Subject: [PATCH 048/179] [a64] Implement `OPCODE_ATOMIC_EXCHANGE` --- a64_seq_memory.cc | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index e12bb9fd7..46003ab7c 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -115,57 +115,63 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, // ============================================================================ // Note that the address we use here is a real, host address! // This is weird, and should be fixed. -template -void EmitAtomicExchangeXX(A64Emitter& e, const ARGS& i) { +template +void EmitAtomicExchangeXX(A64Emitter& e, const ARGS& i, const FN& fn) { if (i.dest == i.src1) { - // e.mov(e.rax, i.src1); + e.MOV(X0, i.src1); if (i.dest != i.src2) { if (i.src2.is_constant) { - // e.mov(i.dest, i.src2.constant()); + e.MOV(i.dest, i.src2.constant()); } else { - // e.mov(i.dest, i.src2); + e.MOV(i.dest, i.src2); } } - // e.lock(); - // e.xchg(e.dword[e.rax], i.dest); + fn(e, i.dest, X0); } else { if (i.dest != i.src2) { if (i.src2.is_constant) { - // e.mov(i.dest, i.src2.constant()); + e.MOV(i.dest, i.src2.constant()); } else { - // e.mov(i.dest, i.src2); + e.MOV(i.dest, i.src2); } } - // e.lock(); - // e.xchg(e.dword[i.src1.reg()], i.dest); + fn(e, i.dest, i.src1); } } struct ATOMIC_EXCHANGE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX(e, i); + EmitAtomicExchangeXX( + e, i, + [](A64Emitter& e, WReg dest, XReg src) { e.SWPALB(dest, dest, src); }); } }; struct ATOMIC_EXCHANGE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX(e, i); + EmitAtomicExchangeXX( + e, i, + [](A64Emitter& e, WReg dest, XReg src) { e.SWPALH(dest, dest, src); }); } }; struct ATOMIC_EXCHANGE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX(e, i); + EmitAtomicExchangeXX( + e, i, + [](A64Emitter& e, WReg dest, XReg src) { e.SWPAL(dest, dest, src); }); } }; struct ATOMIC_EXCHANGE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX(e, i); + EmitAtomicExchangeXX( + e, i, + [](A64Emitter& e, XReg dest, XReg src) { e.SWPAL(dest, dest, src); }); } }; EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_EXCHANGE, ATOMIC_EXCHANGE_I8, From 05777046319a08f8e8cf4bc42979ddf025f53cd6 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 16:48:20 -0700 Subject: [PATCH 049/179] [a64] Implement `OPCODE_ATOMIC_COMPARE_EXCHANGE` --- a64_seq_memory.cc | 50 ++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 46003ab7c..63b31e510 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -185,42 +185,52 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.mov(e.eax, i.src2); + e.MOV(W0, i.src2); if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - // e.cmp(i.src1.reg().cvt32(), 0xE0000000); - // e.setae(e.cl); - // e.movzx(e.ecx, e.cl); - // e.shl(e.ecx, 12); - // e.add(e.ecx, i.src1.reg().cvt32()); + e.CMP(i.src1.reg(), 0xE0, LSL, 24); + e.CSET(W1, Cond::HS); + e.LSL(W1, W1, 12); + e.ADD(W1, W1, i.src1.reg().toW()); } else { - // e.mov(e.ecx, i.src1.reg().cvt32()); + e.MOV(W1, i.src1.reg().toW()); } - // e.lock(); - // e.cmpxchg(e.dword[e.GetMembaseReg() + e.rcx], i.src3); - // e.sete(i.dest); + e.ADD(W1, e.GetMembaseReg().toW(), W1); + + // if([C] == A) [C] = B + // else A = [C] + e.CASAL(W0, i.src3, X1); + + // Set dest to 1 in the case of a successful exchange + e.CMP(W0, i.src2); + e.CSET(i.dest, Cond::EQ); } }; struct ATOMIC_COMPARE_EXCHANGE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.mov(e.rax, i.src2); + e.MOV(X0, i.src2); if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - // e.cmp(i.src1.reg().cvt32(), 0xE0000000); - // e.setae(e.cl); - // e.movzx(e.ecx, e.cl); - // e.shl(e.ecx, 12); - // e.add(e.ecx, i.src1.reg().cvt32()); + e.CMP(i.src1.reg(), 0xE0, LSL, 24); + e.CSET(W1, Cond::HS); + e.LSL(W1, W1, 12); + e.ADD(W1, W1, i.src1.reg().toW()); } else { - // e.mov(e.ecx, i.src1.reg().cvt32()); + e.MOV(W1, i.src1.reg().toW()); } - // e.lock(); - // e.cmpxchg(e.qword[e.GetMembaseReg() + e.rcx], i.src3); - // e.sete(i.dest); + e.ADD(W1, e.GetMembaseReg().toW(), W1); + + // if([C] == A) [C] = B + // else A = [C] + e.CASAL(X0, i.src3, X1); + + // Set dest to 1 in the case of a successful exchange + e.CMP(X0, i.src2); + e.CSET(i.dest, Cond::EQ); } }; EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, From 38ed3a5892059b7be6dad8f9b29bc941bd88e794 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 20:10:14 -0700 Subject: [PATCH 050/179] [a64] Fix `ComputeMemoryAddress{Offset}` register stomp `W1` is a possible HIR register allocation and using W1 here was stomping over it. Don't use W1, use the provided "scratch" register. --- a64_seq_memory.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 63b31e510..a23b708eb 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -49,8 +49,8 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.MOV(W1, 0xE0000000 - offset_const); - e.CMP(guest.reg().toW(), W1); + e.MOV(address_register.toW(), 0xE0000000 - offset_const); + e.CMP(guest.reg().toW(), address_register.toW()); e.CSET(W0, Cond::HS); e.LSL(W0, W0, 12); e.ADD(W0, W0, guest.reg().toW()); @@ -93,8 +93,8 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.MOV(W1, 0xE0000000); - e.CMP(guest.reg().toW(), W1); + e.MOV(address_register.toW(), 0xE0000000); + e.CMP(guest.reg().toW(), address_register.toW()); e.CSET(X0, Cond::HS); e.LSL(X0, X0, 12); e.ADD(X0, X0, guest); From 3636ac795085ea326e16e3fb73c689dd1f46e757 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 20:27:54 -0700 Subject: [PATCH 051/179] [a64] Refactor `REV{16,32}` to `REV` Derive the reversal-size from the register-size. REV32 is also the wrong one to be using here since it will reverse the bytes of upper and lower 32-bit words. --- a64_sequences.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index ff13a0c7f..8dbab3797 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2645,7 +2645,7 @@ struct BYTE_SWAP_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitUnaryOp(e, i, [](A64Emitter& e, WReg dest_src) { - e.REV32(dest_src.toX(), dest_src.toX()); + e.REV(dest_src, dest_src); }); } }; @@ -2653,7 +2653,7 @@ struct BYTE_SWAP_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitUnaryOp(e, i, [](A64Emitter& e, XReg dest_src) { - e.REV64(dest_src, dest_src); + e.REV(dest_src, dest_src); }); } }; From 3f6af52652eee70770c1352a6d36e253b04ac9a0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 22:37:37 -0700 Subject: [PATCH 052/179] [a64] Reorganize guest register allocation Share a somewhat similar calling convention as ARM64 --- a64_backend.cc | 11 ++++++++++- a64_emitter.cc | 13 +++++++------ a64_emitter.h | 20 ++++++++++---------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 211ddc97d..e6ebc045d 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -49,8 +49,17 @@ class A64ThunkEmitter : public A64Emitter { // The following four functions provide save/load functionality for registers. // They assume at least StackLayout::THUNK_STACK_SIZE bytes have been // allocated on the stack. + + // Caller saved: + // Dont assume these registers will survive a subroutine call + // x0, v0 is not saved/preserved since this is used to return values from + // subroutines x1-x15, x30 | d0-d7 and d16-v31 void EmitSaveVolatileRegs(); void EmitLoadVolatileRegs(); + + // Callee saved: + // Subroutines must preserve these registers if they intend to use them + // x19-x30 | d8-d15 void EmitSaveNonvolatileRegs(); void EmitLoadNonvolatileRegs(); }; @@ -78,7 +87,7 @@ bool A64Backend::Initialize(Processor* processor) { std::strcpy(fprs.name, "v"); fprs.types = MachineInfo::RegisterSet::FLOAT_TYPES | MachineInfo::RegisterSet::VEC_TYPES; - fprs.count = A64Emitter::XMM_COUNT; + fprs.count = A64Emitter::FPR_COUNT; code_cache_ = A64CodeCache::Create(); Backend::code_cache_ = code_cache_.get(); diff --git a/a64_emitter.cc b/a64_emitter.cc index 744e2548a..9325e5f3c 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -62,12 +62,13 @@ static const size_t kMaxCodeSize = 1_MiB; static const size_t kStashOffset = 32; // static const size_t kStashOffsetHigh = 32 + 32; -const uint32_t A64Emitter::gpr_reg_map_[A64Emitter::GPR_COUNT] = { - 1, 10, 11, 12, 13, 14, 15, +// Register indices that the HIR is allowed to use for operands +const uint8_t A64Emitter::gpr_reg_map_[A64Emitter::GPR_COUNT] = { + 19, 20, 21, 22, 23, 24, 25, 26, }; -const uint32_t A64Emitter::xmm_reg_map_[A64Emitter::XMM_COUNT] = { - 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, +const uint8_t A64Emitter::fpr_reg_map_[A64Emitter::FPR_COUNT] = { + 8, 9, 10, 11, 12, 13, 14, 15, }; A64Emitter::A64Emitter(A64Backend* backend) @@ -665,8 +666,8 @@ oaknut::XReg A64Emitter::GetNativeParam(uint32_t param) { } // Important: If you change these, you must update the thunks in a64_backend.cc! -oaknut::XReg A64Emitter::GetContextReg() { return X19; } -oaknut::XReg A64Emitter::GetMembaseReg() { return X20; } +oaknut::XReg A64Emitter::GetContextReg() { return X27; } +oaknut::XReg A64Emitter::GetMembaseReg() { return X28; } void A64Emitter::ReloadContext() { // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); diff --git a/a64_emitter.h b/a64_emitter.h index d44700847..60641fdba 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -137,13 +137,13 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { std::vector* out_source_map); public: - // Reserved: XSP, X19, X20 - // Scratch: X0/X1/X2 + // Reserved: XSP, X27, X28 + // Scratch: X1-X15, X30 | V0-v7 and V16-V31 // V0-2 - // Available: X1, X10-r15 + // Available: X19-X26 // V4-V15 (save to get V3) - static const int GPR_COUNT = 7; - static const int XMM_COUNT = 12; + static const size_t GPR_COUNT = 8; + static const size_t FPR_COUNT = 8; static void SetupReg(const hir::Value* v, oaknut::WReg& r) { const auto idx = gpr_reg_map_[v->reg.index]; @@ -154,15 +154,15 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { r = oaknut::XReg(idx); } static void SetupReg(const hir::Value* v, oaknut::SReg& r) { - const auto idx = xmm_reg_map_[v->reg.index]; + const auto idx = fpr_reg_map_[v->reg.index]; r = oaknut::SReg(idx); } static void SetupReg(const hir::Value* v, oaknut::DReg& r) { - const auto idx = xmm_reg_map_[v->reg.index]; + const auto idx = fpr_reg_map_[v->reg.index]; r = oaknut::DReg(idx); } static void SetupReg(const hir::Value* v, oaknut::QReg& r) { - const auto idx = xmm_reg_map_[v->reg.index]; + const auto idx = fpr_reg_map_[v->reg.index]; r = oaknut::QReg(idx); } @@ -247,8 +247,8 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { size_t stack_size_ = 0; - static const uint32_t gpr_reg_map_[GPR_COUNT]; - static const uint32_t xmm_reg_map_[XMM_COUNT]; + static const uint8_t gpr_reg_map_[GPR_COUNT]; + static const uint8_t fpr_reg_map_[FPR_COUNT]; }; } // namespace a64 From c2475a39ac21465709c3a322e4e12cb42ef5313e Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 6 May 2024 22:45:52 -0700 Subject: [PATCH 053/179] [a64] Remove standard prolog/epilog from thunks Fixes callstacks!!!! --- a64_backend.cc | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index e6ebc045d..db332ec87 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -228,8 +228,6 @@ HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { // mov(qword[rsp + 8 * 1], rcx); // sub(rsp, stack_size); - STP(X29, X30, SP, PRE_INDEXED, -32); - MOV(X29, SP); STR(X2, SP, 8 * 3); STR(X1, SP, 8 * 2); @@ -267,9 +265,6 @@ HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { LDR(X1, SP, 8 * 2); LDR(X2, SP, 8 * 3); - MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 32); - RET(); code_offsets.tail = offset(); @@ -307,8 +302,6 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { code_offsets.prolog = offset(); - STP(X29, X30, SP, PRE_INDEXED, -32); - MOV(X29, SP); // rsp + 0 = return address // sub(rsp, stack_size); SUB(SP, SP, stack_size); @@ -318,7 +311,6 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { // Save off volatile registers. EmitSaveVolatileRegs(); - MOV(X29, SP); // mov(rax, rcx); // function // mov(rcx, GetContextReg()); // context @@ -327,8 +319,6 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { MOV(X0, GetContextReg()); // context BLR(X16); - MOV(SP, X29); - EmitLoadVolatileRegs(); code_offsets.epilog = offset(); @@ -336,8 +326,6 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { // add(rsp, stack_size); // ret(); ADD(SP, SP, stack_size); - MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 32); RET(); code_offsets.tail = offset(); @@ -382,8 +370,6 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // rsp + 0 = return address // sub(rsp, stack_size); - STP(X29, X30, SP, POST_INDEXED, -32); - MOV(X29, SP); SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); @@ -408,8 +394,6 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // add(rsp, stack_size); // jmp(rax); ADD(SP, SP, stack_size); - MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 32); BR(X0); code_offsets.tail = offset(); @@ -429,8 +413,6 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { return (ResolveFunctionThunk)fn; } -// Caller saved: -// x0-x15, x30 | d0-d7 and d16-v31 void A64ThunkEmitter::EmitSaveVolatileRegs() { // Save off volatile registers. // Preserve arguments passed to and returned from a subroutine @@ -490,8 +472,6 @@ void A64ThunkEmitter::EmitLoadVolatileRegs() { LDR(Q31, SP, offsetof(StackLayout::Thunk, xmm[21])); } -// Callee saved: -// x19-x30 | d8-d15 void A64ThunkEmitter::EmitSaveNonvolatileRegs() { STP(X19, X20, SP, offsetof(StackLayout::Thunk, r[0])); STP(X21, X22, SP, offsetof(StackLayout::Thunk, r[2])); From aa2a824386227559f50e180c742414ccb71342d4 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 7 May 2024 06:34:50 -0700 Subject: [PATCH 054/179] [a64] Fix `EmitGetCurrentThreadId` type 16-bit word rather than 8-bit --- a64_emitter.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 9325e5f3c..6dc702d17 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -359,8 +359,7 @@ void A64Emitter::MarkSourceOffset(const Instr* i) { void A64Emitter::EmitGetCurrentThreadId() { // rsi must point to context. We could fetch from the stack if needed. - // mov(ax, word[GetContextReg() + offsetof(ppc::PPCContext, thread_id)]); - LDRB(W0, GetContextReg(), offsetof(ppc::PPCContext, thread_id)); + LDRH(W0, GetContextReg(), offsetof(ppc::PPCContext, thread_id)); } void A64Emitter::EmitTraceUserCallReturn() {} From fa74d1041ec2dfeeedd0b767ffe16ec6ee3f4410 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 7 May 2024 06:49:25 -0700 Subject: [PATCH 055/179] [a64] Fix immediates being too large These instructions need to use an extra register to generate their constants if they are too large --- a64_sequences.cc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 8dbab3797..45c2711d9 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -894,7 +894,8 @@ struct COMPARE_EQ_I32 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, WReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(W1, constant); + e.CMP(src1, W1); }); e.CSET(i.dest, Cond::EQ); } @@ -905,7 +906,8 @@ struct COMPARE_EQ_I64 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, XReg src1, XReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, XReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(X1, constant); + e.CMP(src1, X1); }); e.CSET(i.dest, Cond::EQ); } @@ -1023,7 +1025,8 @@ EMITTER_OPCODE_TABLE(OPCODE_COMPARE_NE, COMPARE_NE_I8, COMPARE_NE_I16, }, \ [](A64Emitter& e, WReg dest, const reg_type& src1, int32_t constant, \ bool inverse) { \ - e.CMP(src1, constant); \ + e.MOV(reg_type(1), constant); \ + e.CMP(src1, reg_type(1)); \ if (!inverse) { \ e.CSET(dest, cond); \ } else { \ @@ -1113,7 +1116,8 @@ void EmitAddXX(A64Emitter& e, const ARGS& i) { }, [](A64Emitter& e, REG dest_src, int32_t constant) { // e.add(dest_src, constant); - e.ADD(dest_src, dest_src, constant); + e.MOV(REG(1), constant); + e.ADD(dest_src, dest_src, REG(1)); }); } struct ADD_I8 : Sequence> { @@ -2644,17 +2648,15 @@ struct BYTE_SWAP_I16 struct BYTE_SWAP_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitUnaryOp(e, i, [](A64Emitter& e, WReg dest_src) { - e.REV(dest_src, dest_src); - }); + EmitUnaryOp( + e, i, [](A64Emitter& e, WReg dest_src) { e.REV(dest_src, dest_src); }); } }; struct BYTE_SWAP_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitUnaryOp(e, i, [](A64Emitter& e, XReg dest_src) { - e.REV(dest_src, dest_src); - }); + EmitUnaryOp( + e, i, [](A64Emitter& e, XReg dest_src) { e.REV(dest_src, dest_src); }); } }; struct BYTE_SWAP_V128 From 6ae01f66575563c914525159e397ce6e6246bfe5 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 7 May 2024 06:50:21 -0700 Subject: [PATCH 056/179] [a64] Increase function code size to 1MiB --- a64_emitter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 6dc702d17..4dd5cfdee 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -72,7 +72,7 @@ const uint8_t A64Emitter::fpr_reg_map_[A64Emitter::FPR_COUNT] = { }; A64Emitter::A64Emitter(A64Backend* backend) - : CodeBlock(4_KiB), + : CodeBlock(kMaxCodeSize), CodeGenerator(CodeBlock::ptr()), processor_(backend->processor()), backend_(backend), From 561a02580d189a41978bb27c98f7c07ad719211f Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 7 May 2024 06:52:05 -0700 Subject: [PATCH 057/179] [a64] Fix external function call arguments `x0` was loading the thunk rather than using `xip` Fixes lots of init bugs! --- a64_emitter.cc | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 4dd5cfdee..dfcc0c7e9 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -504,7 +504,6 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, MOV(W1, reg.toW()); } // mov(eax, dword[ebx]); - LDR(W16, X1); } else { // Old-style resolve. // Not too important because indirection table is almost always available. @@ -566,11 +565,6 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { // x1 = arg0 // x2 = arg1 // x3 = arg2 - // mov(rax, reinterpret_cast(thunk)); - // mov(rcx, reinterpret_cast(builtin_function->handler())); - // mov(rdx, reinterpret_cast(builtin_function->arg0())); - // mov(r8, reinterpret_cast(builtin_function->arg1())); - // call(rax); MOV(X0, reinterpret_cast(builtin_function->handler())); MOV(X1, reinterpret_cast(builtin_function->arg0())); MOV(X2, reinterpret_cast(builtin_function->arg1())); @@ -590,17 +584,10 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { // x1 = arg0 // x2 = arg1 // x3 = arg2 - auto thunk = backend()->guest_to_host_thunk(); - // mov(rax, reinterpret_cast(thunk)); - // mov(rcx, - // reinterpret_cast(extern_function->extern_handler())); - // mov(rdx, - // qword[GetContextReg() + offsetof(ppc::PPCContext, kernel_state)]); - // call(rax); - MOV(X0, reinterpret_cast(thunk)); - MOV(X1, reinterpret_cast(extern_function->extern_handler())); - LDR(X2, GetContextReg(), offsetof(ppc::PPCContext, kernel_state)); + MOV(X0, reinterpret_cast(extern_function->extern_handler())); + LDR(X1, GetContextReg(), offsetof(ppc::PPCContext, kernel_state)); + auto thunk = backend()->guest_to_host_thunk(); MOV(X16, reinterpret_cast(thunk)); BLR(X16); From 69b2427c74ace9597313311eab2ad09d9ada2e42 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 8 May 2024 09:24:11 -0700 Subject: [PATCH 058/179] [a64] Fix up-casting zero/sign extensions --- a64_sequences.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 45c2711d9..5e67a744d 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -96,12 +96,12 @@ EMITTER_OPCODE_TABLE(OPCODE_SOURCE_OFFSET, SOURCE_OFFSET); // ============================================================================ struct ASSIGN_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest, i.src1); + e.UXTB(i.dest, i.src1); } }; struct ASSIGN_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest, i.src1); + e.UXTH(i.dest, i.src1); } }; struct ASSIGN_I32 : Sequence> { @@ -152,7 +152,7 @@ struct CAST_F32_I32 : Sequence> { }; struct CAST_F64_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest.reg().Delem()[0], i.src1); + e.FMOV(i.dest, i.src1); } }; EMITTER_OPCODE_TABLE(OPCODE_CAST, CAST_I32_F32, CAST_I64_F64, CAST_F32_I32, @@ -237,7 +237,7 @@ struct SIGN_EXTEND_I64_I16 struct SIGN_EXTEND_I64_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTH(i.dest, i.src1); + e.SXTW(i.dest, i.src1.reg()); } }; EMITTER_OPCODE_TABLE(OPCODE_SIGN_EXTEND, SIGN_EXTEND_I16_I8, SIGN_EXTEND_I32_I8, From 80c7f1b3c4e1cf07735b21f02487b967cd9fdb08 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 8 May 2024 09:26:47 -0700 Subject: [PATCH 059/179] [a64] Compute memory offsets as 32-bit registers Additionally fixes some instruction forms to use the more general `STR` instruction with an offset --- a64_seq_memory.cc | 83 ++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index a23b708eb..f1d6af164 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -25,7 +25,7 @@ volatile int anchor_memory = 0; template XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, - XReg address_register = X3) { + WReg address_register = W3) { assert_true(offset.is_constant); int32_t offset_const = static_cast(offset.constant()); @@ -33,24 +33,24 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, uint32_t address = static_cast(guest.constant()); address += offset_const; if (address < 0x80000000) { - e.ADD(address_register, e.GetMembaseReg(), address); - return address_register; + e.ADD(address_register.toX(), e.GetMembaseReg(), address); + return address_register.toX(); } else { if (address >= 0xE0000000 && xe::memory::allocation_granularity() > 0x1000) { - e.MOV(address_register, address + 0x1000); + e.MOV(W0, address + 0x1000); } else { - e.MOV(address_register, address); + e.MOV(W0, address); } - e.ADD(address_register, e.GetMembaseReg(), address_register); - return address_register; + e.ADD(address_register.toX(), e.GetMembaseReg(), X0); + return address_register.toX(); } } else { if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.MOV(address_register.toW(), 0xE0000000 - offset_const); - e.CMP(guest.reg().toW(), address_register.toW()); + e.MOV(W0, 0xE0000000 - offset_const); + e.CMP(guest.reg().toW(), W0); e.CSET(W0, Cond::HS); e.LSL(W0, W0, 12); e.ADD(W0, W0, guest.reg().toW()); @@ -59,53 +59,52 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, // TODO(benvanik): find a way to avoid doing this. e.MOV(W0, guest.reg().toW()); } - e.ADD(address_register, e.GetMembaseReg(), X0); + e.ADD(address_register.toX(), e.GetMembaseReg(), X0); e.MOV(X0, offset_const); - e.ADD(address_register, address_register, X0); - return address_register; + e.ADD(address_register.toX(), address_register.toX(), X0); + return address_register.toX(); } } // Note: most *should* be aligned, but needs to be checked! template XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, - XReg address_register = X3) { + WReg address_register = W3) { if (guest.is_constant) { // TODO(benvanik): figure out how to do this without a temp. // Since the constant is often 0x8... if we tried to use that as a // displacement it would be sign extended and mess things up. uint32_t address = static_cast(guest.constant()); if (address < 0x80000000) { - e.ADD(address_register, e.GetMembaseReg(), address); - return address_register; + e.ADD(address_register.toX(), e.GetMembaseReg(), address); + return address_register.toX(); } else { if (address >= 0xE0000000 && xe::memory::allocation_granularity() > 0x1000) { - e.MOV(address_register, address + 0x1000); + e.MOV(W0, address + 0x1000u); } else { - e.MOV(address_register, address); + e.MOV(W0, address); } - e.ADD(address_register, e.GetMembaseReg(), address_register); - return address_register; + e.ADD(address_register.toX(), e.GetMembaseReg(), X0); + return address_register.toX(); } } else { if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.MOV(address_register.toW(), 0xE0000000); - e.CMP(guest.reg().toW(), address_register.toW()); - e.CSET(X0, Cond::HS); - e.LSL(X0, X0, 12); - e.ADD(X0, X0, guest); - e.MOV(W0, W0); + e.MOV(W0, 0xE0000000); + e.CMP(guest.reg().toW(), W0); + e.CSET(W0, Cond::HS); + e.LSL(W0, W0, 12); + e.ADD(W0, W0, guest.reg().toW()); } else { // Clear the top 32 bits, as they are likely garbage. // TODO(benvanik): find a way to avoid doing this. e.MOV(W0, guest.reg().toW()); } - e.ADD(address_register, e.GetMembaseReg(), X0); - return address_register; + e.ADD(address_register.toX(), e.GetMembaseReg(), X0); + return address_register.toX(); // return e.GetMembaseReg() + e.rax; } } @@ -402,8 +401,7 @@ struct LOAD_CONTEXT_I64 struct LOAD_CONTEXT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.ADD(X0, e.GetContextReg(), i.src1.value); - e.LD1(List{i.dest.reg().toQ().Selem()[0]}, X0); + e.LDR(i.dest, e.GetContextReg(), i.src1.value); if (IsTracingData()) { // e.lea(e.GetNativeParam(1), e.dword[addr]); // e.mov(e.GetNativeParam(0), i.src1.value); @@ -414,8 +412,7 @@ struct LOAD_CONTEXT_F32 struct LOAD_CONTEXT_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.ADD(X0, e.GetContextReg(), i.src1.value); - e.LD1(List{i.dest.reg().toQ().Delem()[0]}, X0); + e.LDR(i.dest, e.GetContextReg(), i.src1.value); // e.vmovsd(i.dest, e.qword[addr]); if (IsTracingData()) { // e.lea(e.GetNativeParam(1), e.qword[addr]); @@ -519,8 +516,7 @@ struct STORE_CONTEXT_F32 e.MOV(W0, i.src2.value->constant.i32); e.STR(W0, e.GetContextReg(), i.src1.value); } else { - e.ADD(X0, e.GetContextReg(), i.src1.value); - e.ST1(List{i.src2.reg().toQ().Selem()[0]}, X0); + e.STR(i.src2, e.GetContextReg(), i.src1.value); } if (IsTracingData()) { // e.lea(e.GetNativeParam(1), e.dword[addr]); @@ -533,18 +529,11 @@ struct STORE_CONTEXT_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - // e.MovMem64(addr, i.src2.value->constant.i64); - } else { - // e.vmovsd(e.qword[addr], i.src2); - } - if (i.src2.is_constant) { e.MOV(X0, i.src2.value->constant.i64); e.STR(X0, e.GetContextReg(), i.src1.value); } else { - e.ADD(X0, e.GetContextReg(), i.src1.value); - e.ST1(List{i.src2.reg().toQ().Delem()[0]}, X0); + e.STR(i.src2, e.GetContextReg(), i.src1.value); } if (IsTracingData()) { // e.lea(e.GetNativeParam(1), e.qword[addr]); @@ -659,7 +648,7 @@ struct LOAD_OFFSET_I32 auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { e.LDR(i.dest, addr_reg); - e.REV(i.dest.reg().toX(), i.dest.reg().toX()); + e.REV(i.dest, i.dest); } else { e.LDR(i.dest, addr_reg); } @@ -821,8 +810,7 @@ struct LOAD_I64 : Sequence> { struct LOAD_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { auto addr_reg = ComputeMemoryAddress(e, i.src1); - // e.vmovss(i.dest, e.dword[addr]); - e.LD1(List{i.dest.reg().toQ().Selem()[0]}, addr_reg); + e.LDR(i.dest, addr_reg); if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { assert_always("not implemented yet"); } @@ -836,8 +824,7 @@ struct LOAD_F32 : Sequence> { struct LOAD_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { auto addr_reg = ComputeMemoryAddress(e, i.src1); - // e.vmovsd(i.dest, e.qword[addr]); - e.LD1(List{i.dest.reg().toQ().Delem()[0]}, addr_reg); + e.LDR(i.dest, addr_reg); if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { assert_always("not implemented yet"); } @@ -965,7 +952,7 @@ struct STORE_F32 : Sequence> { e.MOV(W0, i.src2.value->constant.i32); e.STR(W0, addr_reg); } else { - e.ST1(List{i.src2.reg().toQ().Selem()[0]}, addr_reg); + e.STR(i.src2, addr_reg); } } if (IsTracingData()) { @@ -987,7 +974,7 @@ struct STORE_F64 : Sequence> { e.MOV(X0, i.src2.value->constant.i64); e.STR(X0, addr_reg); } else { - e.ST1(List{i.src2.reg().toQ().Delem()[0]}, addr_reg); + e.STR(i.src2, addr_reg); } } if (IsTracingData()) { From ea86ac3ce1b58539b3701b5a51db6e7aba007749 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 8 May 2024 09:29:03 -0700 Subject: [PATCH 060/179] [a64] Use `offsetof` to reload membase --- a64_emitter.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index dfcc0c7e9..ab7ea125e 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -661,8 +661,8 @@ void A64Emitter::ReloadContext() { } void A64Emitter::ReloadMembase() { - // mov(GetMembaseReg(), qword[GetContextReg() + 8]); // membase - LDR(GetMembaseReg(), GetContextReg(), 8); // membase + LDR(GetMembaseReg(), GetContextReg(), + offsetof(ppc::PPCContext, virtual_membase)); } bool A64Emitter::ConstantFitsIn32Reg(uint64_t v) { From 871adfb768751d8bbafdd204a37471496b7e53ee Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 8 May 2024 11:08:21 -0700 Subject: [PATCH 061/179] [a64] Fix 32-bit store You wouldn't believe how much time this bug costed me --- a64_seq_memory.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index f1d6af164..0bf67e8c9 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -717,9 +717,9 @@ struct STORE_OFFSET_I32 } else { if (i.src3.is_constant) { e.MOV(W0, i.src3.constant()); - e.STRH(W0, addr_reg); + e.STR(W0, addr_reg); } else { - e.STRH(i.src3, addr_reg); + e.STR(i.src3, addr_reg); } } } From cbda1acafb9ffc70bac45cd0574b37aca81f5c59 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 8 May 2024 11:34:26 -0700 Subject: [PATCH 062/179] [a64] Update guest calling conventions Guest-function calls will use W17 for indirect calls --- a64_backend.cc | 73 ++++++++++++---------------------------------- a64_emitter.cc | 58 +++++++++++------------------------- a64_seq_memory.cc | 3 +- a64_sequences.cc | 22 ++++++++++---- a64_stack_layout.h | 58 +++++++++++++++++------------------- 5 files changed, 81 insertions(+), 133 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index db332ec87..60e6c6236 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -52,8 +52,8 @@ class A64ThunkEmitter : public A64Emitter { // Caller saved: // Dont assume these registers will survive a subroutine call - // x0, v0 is not saved/preserved since this is used to return values from - // subroutines x1-x15, x30 | d0-d7 and d16-v31 + // x0, v0 is not saved for use as arg0/return + // x1-x15, x30 | v0-v7 and v16-v31 void EmitSaveVolatileRegs(); void EmitLoadVolatileRegs(); @@ -223,47 +223,23 @@ HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { code_offsets.prolog = offset(); - // mov(qword[rsp + 8 * 3], r8); - // mov(qword[rsp + 8 * 2], rdx); - // mov(qword[rsp + 8 * 1], rcx); - // sub(rsp, stack_size); - - - STR(X2, SP, 8 * 3); - STR(X1, SP, 8 * 2); - STR(X0, SP, 8 * 1); SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); - // Save nonvolatile registers. EmitSaveNonvolatileRegs(); - // mov(rax, rcx); - // mov(rsi, rdx); // context - // mov(rcx, r8); // return address - // call(rax); MOV(X16, X0); - MOV(A64Emitter::GetContextReg(), X1); // context - MOV(X0, X2); // return address - + MOV(GetContextReg(), X1); // context + MOV(X0, X2); // return address BLR(X16); EmitLoadNonvolatileRegs(); code_offsets.epilog = offset(); - // add(rsp, stack_size); - // mov(rcx, qword[rsp + 8 * 1]); - // mov(rdx, qword[rsp + 8 * 2]); - // mov(r8, qword[rsp + 8 * 3]); - // ret(); - ADD(SP, SP, stack_size); - LDR(X0, SP, 8 * 1); - LDR(X1, SP, 8 * 2); - LDR(X2, SP, 8 * 3); RET(); @@ -302,19 +278,13 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { code_offsets.prolog = offset(); - // rsp + 0 = return address - // sub(rsp, stack_size); SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); - // Save off volatile registers. EmitSaveVolatileRegs(); - // mov(rax, rcx); // function - // mov(rcx, GetContextReg()); // context - // call(rax); MOV(X16, X0); // function MOV(X0, GetContextReg()); // context BLR(X16); @@ -323,8 +293,6 @@ GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { code_offsets.epilog = offset(); - // add(rsp, stack_size); - // ret(); ADD(SP, SP, stack_size); RET(); @@ -350,11 +318,8 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address); ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // Entry: - // X0 = target PPC address - - // Resolve Function: + // W17 = target PPC address // X0 = context - // X1 = target PPC address struct _code_offsets { size_t prolog; @@ -369,22 +334,20 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { code_offsets.prolog = offset(); // rsp + 0 = return address - // sub(rsp, stack_size); SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); - // Save volatile registers EmitSaveVolatileRegs(); // mov(rcx, rsi); // context // mov(rdx, rbx); // mov(rax, reinterpret_cast(&ResolveFunction)); // call(rax) - MOV(X1, X0); MOV(X0, GetContextReg()); // context - MOVP2R(X16, &ResolveFunction); + MOV(W1, W17); + MOV(X16, reinterpret_cast(&ResolveFunction)); BLR(X16); EmitLoadVolatileRegs(); @@ -432,7 +395,6 @@ void A64ThunkEmitter::EmitSaveVolatileRegs() { STP(Q3, Q4, SP, offsetof(StackLayout::Thunk, xmm[2])); STP(Q5, Q6, SP, offsetof(StackLayout::Thunk, xmm[4])); STP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); - STP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); STP(Q17, Q18, SP, offsetof(StackLayout::Thunk, xmm[8])); STP(Q19, Q20, SP, offsetof(StackLayout::Thunk, xmm[10])); STP(Q21, Q22, SP, offsetof(StackLayout::Thunk, xmm[12])); @@ -461,7 +423,6 @@ void A64ThunkEmitter::EmitLoadVolatileRegs() { LDP(Q3, Q4, SP, offsetof(StackLayout::Thunk, xmm[2])); LDP(Q5, Q6, SP, offsetof(StackLayout::Thunk, xmm[4])); LDP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); - LDP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); LDP(Q17, Q18, SP, offsetof(StackLayout::Thunk, xmm[8])); LDP(Q19, Q20, SP, offsetof(StackLayout::Thunk, xmm[10])); LDP(Q21, Q22, SP, offsetof(StackLayout::Thunk, xmm[12])); @@ -480,10 +441,12 @@ void A64ThunkEmitter::EmitSaveNonvolatileRegs() { STP(X27, X28, SP, offsetof(StackLayout::Thunk, r[8])); STP(X29, X30, SP, offsetof(StackLayout::Thunk, r[10])); - STP(Q8, Q9, SP, offsetof(StackLayout::Thunk, xmm[0])); - STP(Q10, Q11, SP, offsetof(StackLayout::Thunk, xmm[2])); - STP(Q12, Q13, SP, offsetof(StackLayout::Thunk, xmm[4])); - STP(Q14, Q15, SP, offsetof(StackLayout::Thunk, xmm[6])); + STR(X17, SP, offsetof(StackLayout::Thunk, r[12])); + + STP(D8, D9, SP, offsetof(StackLayout::Thunk, xmm[0])); + STP(D10, D11, SP, offsetof(StackLayout::Thunk, xmm[1])); + STP(D12, D13, SP, offsetof(StackLayout::Thunk, xmm[2])); + STP(D14, D15, SP, offsetof(StackLayout::Thunk, xmm[3])); } void A64ThunkEmitter::EmitLoadNonvolatileRegs() { @@ -494,10 +457,12 @@ void A64ThunkEmitter::EmitLoadNonvolatileRegs() { LDP(X27, X28, SP, offsetof(StackLayout::Thunk, r[8])); LDP(X29, X30, SP, offsetof(StackLayout::Thunk, r[10])); - LDP(Q8, Q9, SP, offsetof(StackLayout::Thunk, xmm[0])); - LDP(Q10, Q11, SP, offsetof(StackLayout::Thunk, xmm[2])); - LDP(Q12, Q13, SP, offsetof(StackLayout::Thunk, xmm[4])); - LDP(Q14, Q15, SP, offsetof(StackLayout::Thunk, xmm[6])); + LDR(X17, SP, offsetof(StackLayout::Thunk, r[12])); + + LDP(D8, D9, SP, offsetof(StackLayout::Thunk, xmm[0])); + LDP(D10, D11, SP, offsetof(StackLayout::Thunk, xmm[1])); + LDP(D12, D13, SP, offsetof(StackLayout::Thunk, xmm[2])); + LDP(D14, D15, SP, offsetof(StackLayout::Thunk, xmm[3])); } } // namespace a64 diff --git a/a64_emitter.cc b/a64_emitter.cc index ab7ea125e..d57bae253 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -83,20 +83,6 @@ A64Emitter::A64Emitter(A64Backend* backend) feature_flags_ |= (cpu_.has(ext) ? emit : 0); \ } - // TEST_EMIT_FEATURE(kA64EmitAVX2, oaknut::util::Cpu::tAVX2); - // TEST_EMIT_FEATURE(kA64EmitFMA, oaknut::util::Cpu::tFMA); - // TEST_EMIT_FEATURE(kA64EmitLZCNT, oaknut::util::Cpu::tLZCNT); - // TEST_EMIT_FEATURE(kA64EmitBMI1, oaknut::util::Cpu::tBMI1); - // TEST_EMIT_FEATURE(kA64EmitBMI2, oaknut::util::Cpu::tBMI2); - // TEST_EMIT_FEATURE(kA64EmitF16C, oaknut::util::Cpu::tF16C); - // TEST_EMIT_FEATURE(kA64EmitMovbe, oaknut::util::Cpu::tMOVBE); - // TEST_EMIT_FEATURE(kA64EmitGFNI, oaknut::util::Cpu::tGFNI); - // TEST_EMIT_FEATURE(kA64EmitAVX512F, oaknut::util::Cpu::tAVX512F); - // TEST_EMIT_FEATURE(kA64EmitAVX512VL, oaknut::util::Cpu::tAVX512VL); - // TEST_EMIT_FEATURE(kA64EmitAVX512BW, oaknut::util::Cpu::tAVX512BW); - // TEST_EMIT_FEATURE(kA64EmitAVX512DQ, oaknut::util::Cpu::tAVX512DQ); - // TEST_EMIT_FEATURE(kA64EmitAVX512VBMI, oaknut::util::Cpu::tAVX512_VBMI); - #undef TEST_EMIT_FEATURE } @@ -218,15 +204,11 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { STP(X29, X30, SP, PRE_INDEXED, -32); MOV(X29, SP); - // sub(rsp, (uint32_t)stack_size); SUB(SP, SP, (uint32_t)stack_size); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); - // mov(qword[rsp + StackLayout::GUEST_CTX_HOME], GetContextReg()); - // mov(qword[rsp + StackLayout::GUEST_RET_ADDR], rcx); - // mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], 0); STR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); STR(X0, SP, StackLayout::GUEST_RET_ADDR); STR(XZR, SP, StackLayout::GUEST_CALL_RET_ADDR); @@ -260,8 +242,6 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { } // Load membase. - // mov(GetMembaseReg(), - // qword[GetContextReg() + offsetof(ppc::PPCContext, virtual_membase)]); LDR(GetMembaseReg(), GetContextReg(), offsetof(ppc::PPCContext, virtual_membase)); @@ -297,13 +277,10 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { l(epilog_label); epilog_label_ = nullptr; EmitTraceUserCallReturn(); - // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); LDR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); code_offsets.epilog = offset(); - // add(rsp, (uint32_t)stack_size); - // ret(); ADD(SP, SP, (uint32_t)stack_size); MOV(SP, X29); @@ -342,7 +319,6 @@ void A64Emitter::MarkSourceOffset(const Instr* i) { if (cvars::emit_source_annotations) { NOP(); NOP(); - // mov(eax, entry->guest_address); MOV(X0, entry->guest_address); NOP(); NOP(); @@ -451,8 +427,8 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { // or a thunk to ResolveAddress. // mov(ebx, function->address()); // mov(eax, dword[ebx]); - MOV(W1, function->address()); - LDR(W16, X1); + MOV(W17, function->address()); + LDR(W16, X17); } else { // Old-style resolve. // Not too important because indirection table is almost always available. @@ -472,7 +448,11 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { // add(rsp, static_cast(stack_size())); // jmp(rax); - ADD(SP, SP, stack_size()); + ADD(SP, SP, static_cast(stack_size())); + + MOV(SP, X29); + LDP(X29, X30, SP, POST_INDEXED, 32); + BR(X16); } else { // Return address is from the previous SET_RETURN_ADDRESS. @@ -499,10 +479,11 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, // The target dword will either contain the address of the generated code // or a thunk to ResolveAddress. if (code_cache_->has_indirection_table()) { - if (reg.toW().index() != W1.index()) { + if (reg.toW().index() != W17.index()) { // mov(ebx, reg.cvt32()); - MOV(W1, reg.toW()); + MOV(W17, reg.toW()); } + LDR(W16, X17); // mov(eax, dword[ebx]); } else { // Old-style resolve. @@ -515,7 +496,7 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, MOV(X0, GetContextReg()); MOV(W1, reg.toW()); - ADRP(X16, ResolveFunction); + MOV(X16, reinterpret_cast(ResolveFunction)); BLR(X16); MOV(X16, X0); } @@ -526,18 +507,16 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, EmitTraceUserCallReturn(); // Pass the callers return address over. - // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); LDR(X0, SP, StackLayout::GUEST_RET_ADDR); - // add(rsp, static_cast(stack_size())); ADD(SP, SP, static_cast(stack_size())); - // jmp(rax); + MOV(SP, X29); + LDP(X29, X30, SP, POST_INDEXED, 32); + BR(X16); } else { // Return address is from the previous SET_RETURN_ADDRESS. - // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); - // call(rax); LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); BLR(X16); @@ -571,7 +550,6 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { auto thunk = backend()->guest_to_host_thunk(); MOV(X16, reinterpret_cast(thunk)); - BLR(X16); // x0 = host return @@ -589,7 +567,6 @@ void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { auto thunk = backend()->guest_to_host_thunk(); MOV(X16, reinterpret_cast(thunk)); - BLR(X16); // x0 = host return @@ -612,7 +589,6 @@ void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0)) { void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0), uint64_t arg0) { - // mov(GetNativeParam(0), arg0); MOV(GetNativeParam(0), arg0); CallNativeSafe(reinterpret_cast(fn)); } @@ -698,7 +674,7 @@ void A64Emitter::MovMem64(const oaknut::XRegSp& addr, intptr_t offset, } } -static const vec128_t xmm_consts[] = { +static const vec128_t v_consts[] = { /* VZero */ vec128f(0.0f), /* VOne */ vec128f(1.0f), /* VOnePD */ vec128d(1.0), @@ -813,7 +789,7 @@ static const vec128_t xmm_consts[] = { // First location to try and place constants. static const uintptr_t kConstDataLocation = 0x20000000; -static const uintptr_t kConstDataSize = sizeof(xmm_consts); +static const uintptr_t kConstDataSize = sizeof(v_consts); // Increment the location by this amount for every allocation failure. static const uintptr_t kConstDataIncrement = 0x00001000; @@ -837,7 +813,7 @@ uintptr_t A64Emitter::PlaceConstData() { // The pointer must not be greater than 31 bits. assert_zero(reinterpret_cast(mem) & ~0x7FFFFFFF); - std::memcpy(mem, xmm_consts, sizeof(xmm_consts)); + std::memcpy(mem, v_consts, sizeof(v_consts)); memory::Protect(mem, kConstDataSize, memory::PageAccess::kReadOnly, nullptr); return reinterpret_cast(mem); diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 0bf67e8c9..cf31e42b5 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -33,7 +33,8 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, uint32_t address = static_cast(guest.constant()); address += offset_const; if (address < 0x80000000) { - e.ADD(address_register.toX(), e.GetMembaseReg(), address); + e.MOV(address_register.toX(), address); + e.ADD(address_register.toX(), e.GetMembaseReg(), address_register.toX()); return address_register.toX(); } else { if (address >= 0xE0000000 && diff --git a/a64_sequences.cc b/a64_sequences.cc index 5e67a744d..4125d10fa 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1199,7 +1199,15 @@ void EmitAddCarryXX(A64Emitter& e, const ARGS& i) { e.BFI(X1, X0, 61, 1); e.MSR(SystemReg::NZCV, X1); } - e.ADC(i.dest, i.src1, i.src2); + SEQ::EmitCommutativeBinaryOp( + e, i, + [](A64Emitter& e, const REG& dest_src, const REG& src) { + e.ADC(dest_src, dest_src, src); + }, + [](A64Emitter& e, const REG& dest_src, int32_t constant) { + e.MOV(REG(1), constant); + e.ADC(dest_src, dest_src, REG(1)); + }); } struct ADD_CARRY_I8 : Sequence> { @@ -1240,7 +1248,8 @@ void EmitSubXX(A64Emitter& e, const ARGS& i) { e.SUB(dest_src, dest_src, src); }, [](A64Emitter& e, REG dest_src, int32_t constant) { - e.SUB(dest_src, dest_src, constant); + e.MOV(REG(1), constant); + e.SUB(dest_src, dest_src, REG(1)); }); } struct SUB_I8 : Sequence> { @@ -2157,7 +2166,8 @@ void EmitAndXX(A64Emitter& e, const ARGS& i) { e.AND(dest_src, dest_src, src); }, [](A64Emitter& e, REG dest_src, int32_t constant) { - e.AND(dest_src, dest_src, constant); + e.MOV(REG(1), constant); + e.AND(dest_src, dest_src, REG(1)); }); } struct AND_I8 : Sequence> { @@ -2264,7 +2274,8 @@ void EmitOrXX(A64Emitter& e, const ARGS& i) { e.ORR(dest_src, dest_src, src); }, [](A64Emitter& e, REG dest_src, int32_t constant) { - e.ORR(dest_src, dest_src, constant); + e.MOV(REG(1), constant); + e.ORR(dest_src, dest_src, REG(1)); }); } struct OR_I8 : Sequence> { @@ -2309,7 +2320,8 @@ void EmitXorXX(A64Emitter& e, const ARGS& i) { e.EOR(dest_src, dest_src, src); }, [](A64Emitter& e, REG dest_src, int32_t constant) { - e.EOR(dest_src, dest_src, constant); + e.MOV(REG(1), constant); + e.EOR(dest_src, dest_src, REG(1)); }); } struct XOR_I8 : Sequence> { diff --git a/a64_stack_layout.h b/a64_stack_layout.h index 72ded80dc..ee8cbcfac 100644 --- a/a64_stack_layout.h +++ b/a64_stack_layout.h @@ -29,64 +29,58 @@ class StackLayout { * Thunk stack: * Non-Volatile Volatile * +------------------+------------------+ - * | arg temp, 3 * 8 | arg temp, 3 * 8 | xsp + 0x000 + * | arg temp, 3 * 8 | arg temp, 3 * 8 | sp + 0x000 * | | | * | | | * +------------------+------------------+ - * | rbx | (unused) | xsp + 0x018 + * | rbx | (unused) | sp + 0x018 * +------------------+------------------+ - * | rbp | X1 | xsp + 0x020 + * | rbp | X1 | sp + 0x020 * +------------------+------------------+ - * | rcx (Win32) | X2 | xsp + 0x028 + * | rcx (Win32) | X2 | sp + 0x028 * +------------------+------------------+ - * | rsi (Win32) | X3 | xsp + 0x030 + * | rsi (Win32) | X3 | sp + 0x030 * +------------------+------------------+ - * | rdi (Win32) | X4 | xsp + 0x038 + * | rdi (Win32) | X4 | sp + 0x038 * +------------------+------------------+ - * | r12 | X5 | xsp + 0x040 + * | r12 | X5 | sp + 0x040 * +------------------+------------------+ - * | r13 | X6 | xsp + 0x048 + * | r13 | X6 | sp + 0x048 * +------------------+------------------+ - * | r14 | X7 | xsp + 0x050 + * | r14 | X7 | sp + 0x050 * +------------------+------------------+ - * | r15 | X8 | xsp + 0x058 + * | r15 | X8 | sp + 0x058 * +------------------+------------------+ - * | xmm6 (Win32) | X9 | xsp + 0x060 + * | xmm6 (Win32) | X9 | sp + 0x060 * | | | * +------------------+------------------+ - * | xmm7 (Win32) | X10 | xsp + 0x070 + * | xmm7 (Win32) | X10 | sp + 0x070 * | | | * +------------------+------------------+ - * | xmm8 (Win32) | X11 | xsp + 0x080 + * | xmm8 (Win32) | X11 | sp + 0x080 * | | | * +------------------+------------------+ - * | xmm9 (Win32) | X12 | xsp + 0x090 + * | xmm9 (Win32) | X12 | sp + 0x090 * | | | * +------------------+------------------+ - * | xmm10 (Win32) | X13 | xsp + 0x0A0 + * | xmm10 (Win32) | X13 | sp + 0x0A0 * | | | * +------------------+------------------+ - * | xmm11 (Win32) | X14 | xsp + 0x0B0 + * | xmm11 (Win32) | X14 | sp + 0x0B0 * | | | * +------------------+------------------+ - * | xmm12 (Win32) | X15 | xsp + 0x0C0 + * | xmm12 (Win32) | X15 | sp + 0x0C0 * | | | * +------------------+------------------+ - * | xmm13 (Win32) | X16 | xsp + 0x0D0 + * | xmm13 (Win32) | X16 | sp + 0x0D0 * | | | * +------------------+------------------+ - * | xmm14 (Win32) | X17 | xsp + 0x0E0 + * | xmm14 (Win32) | X17 | sp + 0x0E0 * | | | * +------------------+------------------+ - * | xmm15 (Win32) | X18 | xsp + 0x0F0 + * | xmm15 (Win32) | X18 | sp + 0x0F0 * | | | * +------------------+------------------+ - * | (return address) | (return address) | xsp + 0x100 - * +------------------+------------------+ - * | (rcx home) | (rcx home) | xsp + 0x108 - * +------------------+------------------+ - * | (rdx home) | (rdx home) | xsp + 0x110 - * +------------------+------------------+ */ XEPACKEDSTRUCT(Thunk, { uint64_t arg_temp[3]; @@ -95,25 +89,25 @@ class StackLayout { }); static_assert(sizeof(Thunk) % 16 == 0, "sizeof(Thunk) must be a multiple of 16!"); - static const size_t THUNK_STACK_SIZE = sizeof(Thunk) + 16; + static const size_t THUNK_STACK_SIZE = sizeof(Thunk); /** * * * Guest stack: * +------------------+ - * | arg temp, 3 * 8 | xsp + 0 + * | arg temp, 3 * 8 | sp + 0 * | | * | | * +------------------+ - * | scratch, 48b | xsp + 32 + * | scratch, 48b | sp + 32(kStashOffset) * | | * +------------------+ - * | X0 / context | xsp + 80 + * | X0 / context | sp + 80 * +------------------+ - * | guest ret addr | xsp + 88 + * | guest ret addr | sp + 88 * +------------------+ - * | call ret addr | xsp + 96 + * | call ret addr | sp + 96 * +------------------+ * ... locals ... * +------------------+ From 5734e295ca624ab7bc76870c52cc9d0468439d0b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 9 May 2024 06:29:55 -0700 Subject: [PATCH 063/179] [a64] Fix instruction constant generation Fixes some offset generation as well --- a64_seq_memory.cc | 16 ++++++++------- a64_sequences.cc | 50 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index cf31e42b5..7a09bac8f 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -60,10 +60,10 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, // TODO(benvanik): find a way to avoid doing this. e.MOV(W0, guest.reg().toW()); } - e.ADD(address_register.toX(), e.GetMembaseReg(), X0); + e.MOV(X1, offset_const); + e.ADD(X0, X0, X1); - e.MOV(X0, offset_const); - e.ADD(address_register.toX(), address_register.toX(), X0); + e.ADD(address_register.toX(), e.GetMembaseReg(), X0); return address_register.toX(); } } @@ -189,14 +189,15 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.CMP(i.src1.reg(), 0xE0, LSL, 24); + e.MOV(W3, 0xE0000000); + e.CMP(i.src1.reg(), X3); e.CSET(W1, Cond::HS); e.LSL(W1, W1, 12); e.ADD(W1, W1, i.src1.reg().toW()); } else { e.MOV(W1, i.src1.reg().toW()); } - e.ADD(W1, e.GetMembaseReg().toW(), W1); + e.ADD(X1, e.GetMembaseReg(), X1); // if([C] == A) [C] = B // else A = [C] @@ -215,14 +216,15 @@ struct ATOMIC_COMPARE_EXCHANGE_I64 if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. - e.CMP(i.src1.reg(), 0xE0, LSL, 24); + e.MOV(W3, 0xE0000000); + e.CMP(i.src1.reg(), X3); e.CSET(W1, Cond::HS); e.LSL(W1, W1, 12); e.ADD(W1, W1, i.src1.reg().toW()); } else { e.MOV(W1, i.src1.reg().toW()); } - e.ADD(W1, e.GetMembaseReg().toW(), W1); + e.ADD(X1, e.GetMembaseReg(), X1); // if([C] == A) [C] = B // else A = [C] diff --git a/a64_sequences.cc b/a64_sequences.cc index 4125d10fa..bf1820083 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -872,7 +872,8 @@ struct COMPARE_EQ_I8 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, WReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(W1, constant); + e.CMP(src1, W1); }); e.CSET(i.dest, Cond::EQ); } @@ -883,7 +884,8 @@ struct COMPARE_EQ_I16 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, WReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(W1, constant); + e.CMP(src1, W1); }); e.CSET(i.dest, Cond::EQ); } @@ -947,7 +949,8 @@ struct COMPARE_NE_I8 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, WReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(W1, constant); + e.CMP(src1, W1); }); e.CSET(i.dest, Cond::NE); } @@ -958,7 +961,8 @@ struct COMPARE_NE_I16 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, WReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(W1, constant); + e.CMP(src1, W1); }); e.CSET(i.dest, Cond::NE); } @@ -969,7 +973,8 @@ struct COMPARE_NE_I32 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, WReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(W1, constant); + e.CMP(src1, W1); }); e.CSET(i.dest, Cond::NE); } @@ -980,7 +985,8 @@ struct COMPARE_NE_I64 EmitCommutativeCompareOp( e, i, [](A64Emitter& e, XReg src1, XReg src2) { e.CMP(src1, src2); }, [](A64Emitter& e, XReg src1, int32_t constant) { - e.CMP(src1, constant); + e.MOV(X1, constant); + e.CMP(src1, X1); }); e.CSET(i.dest, Cond::NE); } @@ -2610,7 +2616,7 @@ void EmitRotateLeftXX(A64Emitter& e, const ARGS& i) { if (i.src2.is_constant) { e.MOV(REG(1), i.src2.constant()); } else { - e.MOV(W0, i.src2.reg().toW()); + e.MOV(W1, i.src2.reg().toW()); } e.LSLV(i.dest, REG(0), REG(1)); @@ -2633,13 +2639,39 @@ struct ROTATE_LEFT_I16 struct ROTATE_LEFT_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitRotateLeftXX(e, i); + if (i.src1.is_constant) { + e.MOV(W0, i.src1.constant()); + } else { + e.MOV(W0, i.src1.reg()); + } + + if (i.src2.is_constant) { + e.MOV(W1, i.src2.constant()); + } else { + e.SXTB(W1, i.src2.reg()); + } + e.NEG(W1, W1); + + e.ROR(i.dest, W0, W1); } }; struct ROTATE_LEFT_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitRotateLeftXX(e, i); + if (i.src1.is_constant) { + e.MOV(X0, i.src1.constant()); + } else { + e.MOV(X0, i.src1.reg()); + } + + if (i.src2.is_constant) { + e.MOV(X1, i.src2.constant()); + } else { + e.SXTB(X1, i.src2.reg().toW()); + } + e.NEG(X1, X1); + + e.ROR(i.dest, X0, X1); } }; EMITTER_OPCODE_TABLE(OPCODE_ROTATE_LEFT, ROTATE_LEFT_I8, ROTATE_LEFT_I16, From 03d1fc6bd62d6b5572e4ea77e1f1752c76968ba3 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 9 May 2024 07:24:35 -0700 Subject: [PATCH 064/179] [a64] Implement multi-arch capstone support --- a64_assembler.cc | 19 ++++++-- a64_assembler.h | 1 + a64_backend.cc | 110 +++++++++++++++++++++++++++++++++++++++++++++-- a64_backend.h | 2 +- 4 files changed, 125 insertions(+), 7 deletions(-) diff --git a/a64_assembler.cc b/a64_assembler.cc index 4647a4c0a..280b82468 100644 --- a/a64_assembler.cc +++ b/a64_assembler.cc @@ -11,8 +11,8 @@ #include +#include "third_party/capstone/include/capstone/arm64.h" #include "third_party/capstone/include/capstone/capstone.h" -#include "third_party/capstone/include/capstone/x86.h" #include "xenia/base/profiling.h" #include "xenia/base/reset_scope.h" #include "xenia/base/string.h" @@ -33,11 +33,22 @@ namespace a64 { using xe::cpu::hir::HIRBuilder; A64Assembler::A64Assembler(A64Backend* backend) - : Assembler(backend), a64_backend_(backend) {} + : Assembler(backend), a64_backend_(backend), capstone_handle_(0) { + if (cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != + CS_ERR_OK) { + assert_always("Failed to initialize capstone"); + } + cs_option(capstone_handle_, CS_OPT_SYNTAX, CS_OPT_SYNTAX_INTEL); + cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_OFF); +} A64Assembler::~A64Assembler() { // Emitter must be freed before the allocator. emitter_.reset(); + + if (capstone_handle_) { + cs_close(&capstone_handle_); + } } bool A64Assembler::Initialize() { @@ -106,7 +117,9 @@ void A64Assembler::DumpMachineCode( size_t remaining_code_size = code_size; uint64_t address = uint64_t(machine_code); cs_insn insn = {0}; - while (remaining_code_size) { + while (remaining_code_size && + cs_disasm_iter(capstone_handle_, &code_ptr, &remaining_code_size, + &address, &insn)) { // Look up source offset. auto code_offset = uint32_t(code_ptr - reinterpret_cast(machine_code)); diff --git a/a64_assembler.h b/a64_assembler.h index f85d12339..95e0a6f1e 100644 --- a/a64_assembler.h +++ b/a64_assembler.h @@ -46,6 +46,7 @@ class A64Assembler : public Assembler { private: A64Backend* a64_backend_; std::unique_ptr emitter_; + uintptr_t capstone_handle_; StringBuffer string_buffer_; }; diff --git a/a64_backend.cc b/a64_backend.cc index 60e6c6236..f95ae43d5 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -11,6 +11,9 @@ #include +#include "third_party/capstone/include/capstone/arm64.h" +#include "third_party/capstone/include/capstone/capstone.h" + #include "xenia/base/exception_handler.h" #include "xenia/base/logging.h" #include "xenia/cpu/backend/a64/a64_assembler.h" @@ -64,9 +67,21 @@ class A64ThunkEmitter : public A64Emitter { void EmitLoadNonvolatileRegs(); }; -A64Backend::A64Backend() : Backend() {} +A64Backend::A64Backend() : Backend(), code_cache_(nullptr) { + if (cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != + CS_ERR_OK) { + assert_always("Failed to initialize capstone"); + } + cs_option(capstone_handle_, CS_OPT_SYNTAX, CS_OPT_SYNTAX_INTEL); + cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_ON); + cs_option(capstone_handle_, CS_OPT_SKIPDATA, CS_OPT_OFF); +} A64Backend::~A64Backend() { + if (capstone_handle_) { + cs_close(&capstone_handle_); + } + A64Emitter::FreeConstData(emitter_data_); ExceptionHandler::Uninstall(&ExceptionCallbackThunk, this); } @@ -132,10 +147,99 @@ std::unique_ptr A64Backend::CreateGuestFunction( Module* module, uint32_t address) { return std::make_unique(module, address); } + +uint64_t ReadCapstoneReg(HostThreadContext* context, arm64_reg reg) { + switch (reg) { + case ARM64_REG_X0: + return context->x[0]; + case ARM64_REG_X1: + return context->x[1]; + case ARM64_REG_X2: + return context->x[2]; + case ARM64_REG_X3: + return context->x[3]; + case ARM64_REG_X4: + return context->x[4]; + case ARM64_REG_X5: + return context->x[5]; + case ARM64_REG_X6: + return context->x[6]; + case ARM64_REG_X7: + return context->x[7]; + case ARM64_REG_X8: + return context->x[8]; + case ARM64_REG_X9: + return context->x[9]; + case ARM64_REG_X10: + return context->x[10]; + case ARM64_REG_X11: + return context->x[11]; + case ARM64_REG_X12: + return context->x[12]; + case ARM64_REG_X13: + return context->x[13]; + case ARM64_REG_X14: + return context->x[14]; + case ARM64_REG_X15: + return context->x[15]; + case ARM64_REG_X16: + return context->x[16]; + case ARM64_REG_X17: + return context->x[17]; + case ARM64_REG_X18: + return context->x[18]; + case ARM64_REG_X19: + return context->x[19]; + case ARM64_REG_X20: + return context->x[20]; + case ARM64_REG_X21: + return context->x[21]; + case ARM64_REG_X22: + return context->x[22]; + case ARM64_REG_X23: + return context->x[23]; + case ARM64_REG_X24: + return context->x[24]; + case ARM64_REG_X25: + return context->x[25]; + case ARM64_REG_X26: + return context->x[26]; + case ARM64_REG_X27: + return context->x[27]; + case ARM64_REG_X28: + return context->x[28]; + case ARM64_REG_X29: + return context->x[29]; + case ARM64_REG_X30: + return context->x[30]; + default: + assert_unhandled_case(reg); + return 0; + } +} + uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, uint64_t current_pc) { - // TODO(wunkolo): Capstone hookup - return current_pc += 4; + auto machine_code_ptr = reinterpret_cast(current_pc); + size_t remaining_machine_code_size = 64; + uint64_t host_address = current_pc; + cs_insn insn = {0}; + cs_detail all_detail = {0}; + insn.detail = &all_detail; + cs_disasm_iter(capstone_handle_, &machine_code_ptr, + &remaining_machine_code_size, &host_address, &insn); + auto& detail = all_detail.x86; + switch (insn.id) { + case ARM64_INS_B: + case ARM64_INS_BL: + case ARM64_INS_BLR: + case ARM64_INS_BR: + case ARM64_INS_RET: + // todo(wunkolo): determine next instruction + default: + // Not a branching instruction - just move over it. + return current_pc + insn.size; + } } void A64Backend::InstallBreakpoint(Breakpoint* breakpoint) { diff --git a/a64_backend.h b/a64_backend.h index 85801c592..57557414c 100644 --- a/a64_backend.h +++ b/a64_backend.h @@ -70,7 +70,7 @@ class A64Backend : public Backend { static bool ExceptionCallbackThunk(Exception* ex, void* data); bool ExceptionCallback(Exception* ex); - // uintptr_t capstone_handle_ = 0; + uintptr_t capstone_handle_ = 0; std::unique_ptr code_cache_; uintptr_t emitter_data_ = 0; From f794f70b99dc165b1aa9d2706c21a250e8ba5136 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 9 May 2024 07:35:54 -0700 Subject: [PATCH 065/179] [a64] Protect address-generation from imm-overflow --- a64_seq_memory.cc | 3 ++- a64_sequences.cc | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 7a09bac8f..71c5c982c 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -78,7 +78,8 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, // displacement it would be sign extended and mess things up. uint32_t address = static_cast(guest.constant()); if (address < 0x80000000) { - e.ADD(address_register.toX(), e.GetMembaseReg(), address); + e.MOV(W0, address); + e.ADD(address_register.toX(), e.GetMembaseReg(), X0); return address_register.toX(); } else { if (address >= 0xE0000000 && diff --git a/a64_sequences.cc b/a64_sequences.cc index bf1820083..f1538b870 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1187,13 +1187,14 @@ void EmitAddCarryXX(A64Emitter& e, const ARGS& i) { // TODO(benvanik): faster setting? we could probably do some fun math tricks // here to get the carry flag set. if (i.src3.is_constant) { + e.MOV(W0, WZR); if (i.src3.constant()) { // Set carry // This is implicitly "SUBS 0 - 0" - e.CMP(WZR.toW(), 0); + e.CMP(W0, 0); } else { // Clear carry - e.CMN(WZR.toW(), 0); + e.CMN(W0, 0); } } else { // If src3 is non-zero, set the carry flag From 3a1756e87be489cb6572ea24a119745f2d6def3d Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 08:05:28 -0700 Subject: [PATCH 066/179] [a64] Preserve X0 when resolving functions Fixes indirect branches --- a64_backend.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index f95ae43d5..ee24766af 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -437,7 +437,7 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { code_offsets.prolog = offset(); - // rsp + 0 = return address + STP(ZR, X0, SP, PRE_INDEXED, -16); SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); @@ -453,6 +453,7 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { MOV(W1, W17); MOV(X16, reinterpret_cast(&ResolveFunction)); BLR(X16); + MOV(X16, X0); EmitLoadVolatileRegs(); @@ -461,7 +462,8 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // add(rsp, stack_size); // jmp(rax); ADD(SP, SP, stack_size); - BR(X0); + LDP(ZR, X0, SP, POST_INDEXED, 16); + BR(X16); code_offsets.tail = offset(); From afd864ebc04f4cfc8e472412907fb8dcbf8ac7ab Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 08:08:25 -0700 Subject: [PATCH 067/179] [a64] Fix ADDC carry-bit assignment --- a64_sequences.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index f1538b870..6d3e045c2 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1203,7 +1203,7 @@ void EmitAddCarryXX(A64Emitter& e, const ARGS& i) { e.MRS(X1, SystemReg::NZCV); // Assign carry bit - e.BFI(X1, X0, 61, 1); + e.BFI(X1, X0, 29, 1); e.MSR(SystemReg::NZCV, X1); } SEQ::EmitCommutativeBinaryOp( From 890e8e6c6122b2940f1874e6e24c936dc3fc8aba Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 08:20:30 -0700 Subject: [PATCH 068/179] [a64] Fix signed MUL_HI --- a64_sequences.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 6d3e045c2..7379f4509 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1495,17 +1495,17 @@ struct MUL_HI_I32 if (i.src1.is_constant) { assert_true(!i.src2.is_constant); e.MOV(W0, i.src1.constant()); - e.UMULL(X0, W0, i.src2); + e.SMULL(X0, W0, i.src2); e.UBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else if (i.src2.is_constant) { assert_true(!i.src1.is_constant); e.MOV(W0, i.src2.constant()); - e.UMULL(X0, W0, i.src2); + e.SMULL(X0, W0, i.src2); e.UBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else { - e.UMULL(X0, W0, i.src2); + e.SMULL(X0, W0, i.src2); e.UBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } @@ -1531,13 +1531,13 @@ struct MUL_HI_I64 if (i.src1.is_constant) { assert_true(!i.src2.is_constant); e.MOV(X0, i.src1.constant()); - e.UMULH(i.dest, X0, i.src2); + e.SMULH(i.dest, X0, i.src2); } else if (i.src2.is_constant) { assert_true(!i.src1.is_constant); e.MOV(X0, i.src2.constant()); - e.UMULH(i.dest, i.src1, X0); + e.SMULH(i.dest, i.src1, X0); } else { - e.UMULH(i.dest, i.src1, i.src2); + e.SMULH(i.dest, i.src1, i.src2); } } } From 13cb3ed892aa44f4121bc0cc8e3cee7e92da1da5 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 08:36:31 -0700 Subject: [PATCH 069/179] [a64] Fix non-const MUL_I32 Was picking up `W0` rather than src1 --- a64_sequences.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 7379f4509..53e4a3836 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1487,7 +1487,7 @@ struct MUL_HI_I32 e.UBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else { - e.UMULL(X0, W0, i.src2); + e.UMULL(X0, i.src1, i.src2); e.UBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } @@ -1496,17 +1496,17 @@ struct MUL_HI_I32 assert_true(!i.src2.is_constant); e.MOV(W0, i.src1.constant()); e.SMULL(X0, W0, i.src2); - e.UBFX(X0, X0, 32, 32); + e.SBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else if (i.src2.is_constant) { assert_true(!i.src1.is_constant); e.MOV(W0, i.src2.constant()); e.SMULL(X0, W0, i.src2); - e.UBFX(X0, X0, 32, 32); + e.SBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else { - e.SMULL(X0, W0, i.src2); - e.UBFX(X0, X0, 32, 32); + e.SMULL(X0, i.src1, i.src2); + e.SBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } } From aaa6aefb514c537af9d87673a982644a3a29bde4 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 09:23:13 -0700 Subject: [PATCH 070/179] [a64] Implement `PERMUTE_V128`(int8) --- a64_seq_vector.cc | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 188b8dfdb..d7b37fc28 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1019,12 +1019,54 @@ EMITTER_OPCODE_TABLE(OPCODE_SPLAT, SPLAT_I8, SPLAT_I16, SPLAT_I32, SPLAT_F32); // ============================================================================ struct PERMUTE_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.instr->flags == INT32_TYPE); + } }; struct PERMUTE_V128 : Sequence> { - static void EmitByInt8(A64Emitter& e, const EmitArgType& i) {} + static void EmitByInt8(A64Emitter& e, const EmitArgType& i) { + // Permute bytes between src2 and src3. + // src1 is an array of indices corresponding to positions within src2 and + // src3. + if (i.src3.value->IsConstantZero()) { + if (i.src2.value->IsConstantZero()) { + // src2 & src3 are zero, so result will always be zero. + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); + return; + } + } + + const QReg indices = Q0; + if (i.src1.is_constant) { + e.LoadConstantV(indices, i.src1.constant()); + } else { + e.MOV(indices.B16(), i.src1.reg().B16()); + } + + // Indices must be endian-swapped + e.MOVP2R(X0, e.GetVConstPtr(VSwapWordMask)); + e.LDR(Q1, X0); + e.EOR(Q0.B16(), Q0.B16(), Q1.B16()); + + // Table-registers must be sequential indices + const QReg table0 = Q2; + if (i.src2.is_constant) { + e.LoadConstantV(table0, i.src2.constant()); + } else { + e.MOV(table0.B16(), i.src2.reg().B16()); + } + + const QReg table1 = Q3; + if (i.src3.is_constant) { + e.LoadConstantV(table1, i.src3.constant()); + } else { + e.MOV(table1.B16(), i.src3.reg().B16()); + } + + e.TBL(i.dest.reg().B16(), List{table0.B16(), table1.B16()}, indices.B16()); + } static void EmitByInt16(A64Emitter& e, const EmitArgType& i) {} From f9ffbea3265407d93e1fed2d4732acc2d200b7bc Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 10:06:24 -0700 Subject: [PATCH 071/179] [a64] Implement `PERMUTE_I32` --- a64_seq_vector.cc | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index d7b37fc28..925e3b644 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1021,6 +1021,47 @@ struct PERMUTE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(i.instr->flags == INT32_TYPE); + // Permute words between src2 and src3. + if (i.src1.is_constant) { + // Each byte is a word-index + const uint32_t control = i.src1.constant(); + const QReg indices = Q0; + + // Word to byte index + e.MOV(W0, control * 4); + e.MOV(indices.Selem()[0], W0); + + // Widen int8 to int16 + e.ZIP1(indices.B16(), indices.B16(), indices.B16()); + // Widen int16 to int32 + e.ZIP1(indices.B16(), indices.B16(), indices.B16()); + + // Convert to byte-indices + e.MOV(W0, 0x03'02'01'00); + e.DUP(Q1.S4(), W0); + e.ADD(indices.S4(), indices.S4(), Q1.S4()); + + // Table-registers must be sequential indices + const QReg table0 = Q2; + if (i.src2.is_constant) { + e.LoadConstantV(table0, i.src2.constant()); + } else { + e.MOV(table0.B16(), i.src2.reg().B16()); + } + + const QReg table1 = Q3; + if (i.src3.is_constant) { + e.LoadConstantV(table1, i.src3.constant()); + } else { + e.MOV(table1.B16(), i.src3.reg().B16()); + } + + e.TBL(i.dest.reg().B16(), List{table0.B16(), table1.B16()}, + indices.B16()); + } else { + // Permute by non-constant. + assert_always(); + } } }; struct PERMUTE_V128 From f6aa0c747fbbc930276e00c19c6bb428188ac132 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 10:19:30 -0700 Subject: [PATCH 072/179] [a64] Implement `OPCODE_SWIZZLE` --- a64_seq_vector.cc | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 925e3b644..35557eed6 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1139,7 +1139,40 @@ EMITTER_OPCODE_TABLE(OPCODE_PERMUTE, PERMUTE_I32, PERMUTE_V128); // ============================================================================ struct SWIZZLE : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i){}; + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto element_type = i.instr->flags; + if (element_type == INT8_TYPE) { + assert_always(); + } else if (element_type == INT16_TYPE) { + assert_always(); + } else if (element_type == INT32_TYPE || element_type == FLOAT32_TYPE) { + // Four 2-bit word-indices packed into one 8-bit value + const uint8_t swizzle_mask = static_cast(i.src2.value); + + // Convert to byte-indices + const vec128_t indice_vec = + vec128i(((swizzle_mask >> 0) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, + ((swizzle_mask >> 2) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, + ((swizzle_mask >> 4) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, + ((swizzle_mask >> 6) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00); + + const QReg indices = Q0; + e.LoadConstantV(indices, indice_vec); + + QReg table0 = Q0; + if (i.src1.is_constant) { + e.LoadConstantV(table0, i.src1.constant()); + } else { + table0 = i.src1; + } + + e.TBL(i.dest.reg().B16(), List{table0.B16()}, indices.B16()); + } else if (element_type == INT64_TYPE || element_type == FLOAT64_TYPE) { + assert_always(); + } else { + assert_always(); + } + }; }; EMITTER_OPCODE_TABLE(OPCODE_SWIZZLE, SWIZZLE); From dc024a1b75db2aee027bf7a7ea517670b64e0d91 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 10:37:36 -0700 Subject: [PATCH 073/179] [a64] Fix `SELECT` register usage --- a64_sequences.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 53e4a3836..0a4324bf8 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -667,13 +667,13 @@ struct SELECT_F32 e.LoadConstantV(src2.toQ(), i.src2.constant()); } - SReg src3 = i.src3.is_constant ? S2 : i.src3; + SReg src3 = i.src3.is_constant ? S3 : i.src3; if (i.src3.is_constant) { e.LoadConstantV(src3.toQ(), i.src3.constant()); } e.CMP(i.src1.reg().toX(), 0); - e.FCSEL(i.dest, src2, i.src3, Cond::NE); + e.FCSEL(i.dest, src2, src3, Cond::NE); } }; struct SELECT_F64 @@ -686,13 +686,13 @@ struct SELECT_F64 e.LoadConstantV(src2.toQ(), i.src2.constant()); } - DReg src3 = i.src3.is_constant ? D2 : i.src3; + DReg src3 = i.src3.is_constant ? D3 : i.src3; if (i.src3.is_constant) { e.LoadConstantV(src3.toQ(), i.src3.constant()); } e.CMP(i.src1.reg().toX(), 0); - e.FCSEL(i.dest, src2, i.src3, Cond::NE); + e.FCSEL(i.dest, src2, src3, Cond::NE); } }; struct SELECT_V128_I8 @@ -705,7 +705,7 @@ struct SELECT_V128_I8 e.LoadConstantV(src2, i.src2.constant()); } - QReg src3 = i.src3.is_constant ? Q2 : i.src3; + QReg src3 = i.src3.is_constant ? Q3 : i.src3; if (i.src3.is_constant) { e.LoadConstantV(src3, i.src3.constant()); } From eb8104e024d352bb421a1cd542eb09f84db6e530 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 10:38:13 -0700 Subject: [PATCH 074/179] [a64] Fix `SET_ROUNDING_MODE_I32` exception --- a64_sequences.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 0a4324bf8..0f957e8ee 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2773,8 +2773,8 @@ struct SET_ROUNDING_MODE_I32 e.AND(W1, i.src1, 0b111); // Use the low 3 bits as an index into a LUT - e.ADRL(X0, fpcr_table); - e.LDRB(W0, X0, W1); + e.MOVP2R(X0, fpcr_table); + e.LDRB(W0, X0, X1); // Replace FPCR bits with new value e.MRS(X1, SystemReg::FPCR); From bf1224c61f8d3fca49436fbe1f883162d4eb2d83 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 16:00:12 -0700 Subject: [PATCH 075/179] [a64] Implement `OPCODE_VECTOR_SUB` --- a64_seq_vector.cc | 54 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 35557eed6..664dcad39 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -469,7 +469,59 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ADD, VECTOR_ADD); // ============================================================================ struct VECTOR_SUB : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, [&i](A64Emitter& e, const QReg& dest, QReg src1, QReg src2) { + const TypeName part_type = + static_cast(i.instr->flags & 0xFF); + const uint32_t arithmetic_flags = i.instr->flags >> 8; + bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED); + bool saturate = !!(arithmetic_flags & ARITHMETIC_SATURATE); + switch (part_type) { + case INT8_TYPE: + if (saturate) { + if (is_unsigned) { + e.UQSUB(dest.B16(), src1.B16(), src2.B16()); + } else { + e.SQSUB(dest.B16(), src1.B16(), src2.B16()); + } + } else { + e.SUB(dest.B16(), src1.B16(), src2.B16()); + } + break; + case INT16_TYPE: + if (saturate) { + if (is_unsigned) { + e.UQSUB(dest.H8(), src1.H8(), src2.H8()); + } else { + e.SQSUB(dest.H8(), src1.H8(), src2.H8()); + } + } else { + e.SUB(dest.H8(), src1.H8(), src2.H8()); + } + break; + case INT32_TYPE: + if (saturate) { + if (is_unsigned) { + e.UQSUB(dest.S4(), src1.S4(), src2.S4()); + } else { + e.SQSUB(dest.S4(), src1.S4(), src2.S4()); + } + } else { + e.SUB(dest.S4(), src1.S4(), src2.S4()); + } + break; + case FLOAT32_TYPE: + assert_false(is_unsigned); + assert_false(saturate); + e.FSUB(dest.S4(), src1.S4(), src2.S4()); + break; + default: + assert_unhandled_case(part_type); + break; + } + }); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB); From 94b4c38c4894e8a3e45779167521f5b8f3aca5aa Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 17:29:56 -0700 Subject: [PATCH 076/179] [a64] Fix `PERMUTE_V128` out-of-index case --- a64_seq_vector.cc | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 664dcad39..4ae805772 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1139,26 +1139,33 @@ struct PERMUTE_V128 } // Indices must be endian-swapped - e.MOVP2R(X0, e.GetVConstPtr(VSwapWordMask)); - e.LDR(Q1, X0); - e.EOR(Q0.B16(), Q0.B16(), Q1.B16()); + e.MOV(W0, 0b11); + e.DUP(Q1.B16(), W0); + e.EOR(indices.B16(), indices.B16(), Q1.B16()); + + + // Modulo 32 the indices + e.MOV(W0, 0b0001'1111); + e.DUP(Q1.B16(), W0); + e.AND(indices.B16(), indices.B16(), Q1.B16()); // Table-registers must be sequential indices - const QReg table0 = Q2; + const QReg table_lo = Q2; if (i.src2.is_constant) { - e.LoadConstantV(table0, i.src2.constant()); + e.LoadConstantV(table_lo, i.src2.constant()); } else { - e.MOV(table0.B16(), i.src2.reg().B16()); + e.MOV(table_lo.B16(), i.src2.reg().B16()); } - const QReg table1 = Q3; + const QReg table_hi = Q3; if (i.src3.is_constant) { - e.LoadConstantV(table1, i.src3.constant()); + e.LoadConstantV(table_hi, i.src3.constant()); } else { - e.MOV(table1.B16(), i.src3.reg().B16()); + e.MOV(table_hi.B16(), i.src3.reg().B16()); } - e.TBL(i.dest.reg().B16(), List{table0.B16(), table1.B16()}, indices.B16()); + e.TBL(i.dest.reg().B16(), List{table_lo.B16(), table_hi.B16()}, + indices.B16()); } static void EmitByInt16(A64Emitter& e, const EmitArgType& i) {} From fcaf78cd5cebafd18163159b366a34afac444331 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 19:57:38 -0700 Subject: [PATCH 077/179] [a64] Fix `AND_NOT_V128` Operand order is wrong. --- a64_sequences.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 0f957e8ee..2a2f64a35 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2262,7 +2262,7 @@ struct AND_NOT_V128 static void Emit(A64Emitter& e, const EmitArgType& i) { EmitCommutativeBinaryVOp( e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.BIC(dest.B16(), src2.B16(), src1.B16()); + e.BIC(dest.B16(), src1.B16(), src2.B16()); }); } }; From c0a708e73d870e0959671ae94cfe74ab8c70e6e8 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 20:14:05 -0700 Subject: [PATCH 078/179] [a64] Implement `VECTOR_COMPARE_{EQ,UGT,UGE,SGT,SGE}_V128` --- a64_seq_vector.cc | 77 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 4ae805772..030d846e1 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -300,20 +300,20 @@ struct VECTOR_COMPARE_EQ_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( + EmitAssociativeBinaryVOp( e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { switch (i.instr->flags) { case INT8_TYPE: - // e.vpcmpeqb(dest, src1, src2); + e.CMEQ(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpcmpeqw(dest, src1, src2); + e.CMEQ(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpcmpeqd(dest, src1, src2); + e.CMEQ(dest.S4(), src1.S4(), src2.S4()); break; case FLOAT32_TYPE: - // e.vcmpeqps(dest, src1, src2); + e.FCMEQ(dest.S4(), src1.S4(), src2.S4()); break; } }); @@ -332,16 +332,16 @@ struct VECTOR_COMPARE_SGT_V128 e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { switch (i.instr->flags) { case INT8_TYPE: - // e.vpcmpgtb(dest, src1, src2); + e.CMGT(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpcmpgtw(dest, src1, src2); + e.CMGT(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpcmpgtd(dest, src1, src2); + e.CMGT(dest.S4(), src1.S4(), src2.S4()); break; case FLOAT32_TYPE: - // e.vcmpgtps(dest, src1, src2); + e.FCMGT(dest.S4(), src1.S4(), src2.S4()); break; } }); @@ -360,22 +360,16 @@ struct VECTOR_COMPARE_SGE_V128 e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { switch (i.instr->flags) { case INT8_TYPE: - // e.vpcmpeqb(Q0, src1, src2); - // e.vpcmpgtb(dest, src1, src2); - // e.vpor(dest, Q0); + e.CMGE(dest.B16(), src1.B16(), src2.B16()); break; case INT16_TYPE: - // e.vpcmpeqw(Q0, src1, src2); - // e.vpcmpgtw(dest, src1, src2); - // e.vpor(dest, Q0); + e.CMGE(dest.H8(), src1.H8(), src2.H8()); break; case INT32_TYPE: - // e.vpcmpeqd(Q0, src1, src2); - // e.vpcmpgtd(dest, src1, src2); - // e.vpor(dest, Q0); + e.CMGE(dest.S4(), src1.S4(), src2.S4()); break; case FLOAT32_TYPE: - // e.vcmpgeps(dest, src1, src2); + e.FCMGE(dest.S4(), src1.S4(), src2.S4()); break; } }); @@ -389,7 +383,27 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGE, VECTOR_COMPARE_SGE_V128); struct VECTOR_COMPARE_UGT_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAssociativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + switch (i.instr->flags) { + case INT8_TYPE: + e.CMHI(dest.B16(), src1.B16(), src2.B16()); + break; + case INT16_TYPE: + e.CMHI(dest.H8(), src1.H8(), src2.H8()); + break; + case INT32_TYPE: + e.CMHI(dest.S4(), src1.S4(), src2.S4()); + break; + case FLOAT32_TYPE: + e.FABS(Q0.S4(), src1.S4()); + e.FABS(Q1.S4(), src2.S4()); + e.FCMGT(dest.S4(), Q0.S4(), Q1.S4()); + break; + } + }); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGT, VECTOR_COMPARE_UGT_V128); @@ -399,7 +413,27 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGT, VECTOR_COMPARE_UGT_V128); struct VECTOR_COMPARE_UGE_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitAssociativeBinaryVOp( + e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + switch (i.instr->flags) { + case INT8_TYPE: + e.CMHS(dest.B16(), src1.B16(), src2.B16()); + break; + case INT16_TYPE: + e.CMHS(dest.H8(), src1.H8(), src2.H8()); + break; + case INT32_TYPE: + e.CMHS(dest.S4(), src1.S4(), src2.S4()); + break; + case FLOAT32_TYPE: + e.FABS(Q0.S4(), src1.S4()); + e.FABS(Q1.S4(), src2.S4()); + e.FCMGE(dest.S4(), Q0.S4(), Q1.S4()); + break; + } + }); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGE, VECTOR_COMPARE_UGE_V128); @@ -1143,7 +1177,6 @@ struct PERMUTE_V128 e.DUP(Q1.B16(), W0); e.EOR(indices.B16(), indices.B16(), Q1.B16()); - // Modulo 32 the indices e.MOV(W0, 0b0001'1111); e.DUP(Q1.B16(), W0); From 849f8c442e03e6f6fca195c0a7677da0ef54267c Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 20:49:42 -0700 Subject: [PATCH 079/179] [a64] Fix `OPCODE_SPLAT` Writing to the wrong register! --- a64_seq_vector.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 030d846e1..4e90efe7d 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1062,9 +1062,9 @@ struct SPLAT_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { e.MOV(W0, i.src1.constant()); - e.DUP(Q0.B16(), W0); + e.DUP(i.dest.reg().B16(), W0); } else { - e.DUP(Q0.B16(), i.src1); + e.DUP(i.dest.reg().B16(), i.src1); } } }; @@ -1072,9 +1072,9 @@ struct SPLAT_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { e.MOV(W0, i.src1.constant()); - e.DUP(Q0.H8(), W0); + e.DUP(i.dest.reg().H8(), W0); } else { - e.DUP(Q0.H8(), i.src1); + e.DUP(i.dest.reg().H8(), i.src1); } } }; @@ -1082,9 +1082,9 @@ struct SPLAT_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { e.MOV(W0, i.src1.constant()); - e.DUP(Q0.S4(), W0); + e.DUP(i.dest.reg().S4(), W0); } else { - e.DUP(Q0.S4(), i.src1); + e.DUP(i.dest.reg().S4(), i.src1); } } }; @@ -1092,9 +1092,9 @@ struct SPLAT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { e.MOV(W0, i.src1.value->constant.i32); - e.DUP(Q0.S4(), W0); + e.DUP(i.dest.reg().S4(), W0); } else { - e.DUP(Q0.S4(), i.src1.reg().toQ().Selem()[0]); + e.DUP(i.dest.reg().S4(), i.src1.reg().toQ().Selem()[0]); } } }; From 93929ab4d0ef5b1547e2bbce8a126cf7cc0b7d99 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 21:57:05 -0700 Subject: [PATCH 080/179] [a64] Fix `SELECT_V128_V128` Potential input-register stomping and operand order is seemingly wrong. Passes generated unit tests. --- a64_sequences.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 2a2f64a35..df11306d1 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -720,9 +720,11 @@ struct SELECT_V128_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + const QReg src1 = Q0; if (i.src1.is_constant) { e.LoadConstantV(src1, i.src1.constant()); + } else { + e.MOV(src1.B16(), i.src1.reg().B16()); } const QReg src2 = i.src2.is_constant ? Q1 : i.src2; @@ -736,7 +738,7 @@ struct SELECT_V128_V128 } // src1 ? src2 : src3; - e.BSL(src1.B16(), src2.B16(), src3.B16()); + e.BSL(src1.B16(), src3.B16(), src2.B16()); e.MOV(i.dest.reg().B16(), src1.B16()); } }; From 60d23c461ed4d8b90e8186b59c2c03e300102412 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 10 May 2024 22:07:50 -0700 Subject: [PATCH 081/179] [a64] Implement `OPCODE_VECTOR_AVERAGE` Passes generated unit tests --- a64_seq_vector.cc | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 4e90efe7d..5f5e519c7 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -936,11 +936,47 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128); // ============================================================================ // OPCODE_VECTOR_AVERAGE // ============================================================================ - struct VECTOR_AVERAGE : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitCommutativeBinaryVOp( + e, i, + [&i](A64Emitter& e, const QReg& dest, const QReg& src1, + const QReg& src2) { + const TypeName part_type = + static_cast(i.instr->flags & 0xFF); + const uint32_t arithmetic_flags = i.instr->flags >> 8; + bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED); + switch (part_type) { + case INT8_TYPE: + if (is_unsigned) { + e.URHADD(dest.B16(), src1.B16(), src2.B16()); + } else { + e.SRHADD(dest.B16(), src1.B16(), src2.B16()); + assert_always(); + } + break; + case INT16_TYPE: + if (is_unsigned) { + e.URHADD(dest.H8(), src1.H8(), src2.H8()); + } else { + e.SRHADD(dest.H8(), src1.H8(), src2.H8()); + } + break; + case INT32_TYPE: + if (is_unsigned) { + e.URHADD(dest.S4(), src1.S4(), src2.S4()); + } else { + e.SRHADD(dest.S4(), src1.S4(), src2.S4()); + } + break; + default: + assert_unhandled_case(part_type); + break; + } + }); + } }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_AVERAGE, VECTOR_AVERAGE); From c8fd32f113670526fd567cc84d253ce3ceb76321 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 11 May 2024 10:45:46 -0700 Subject: [PATCH 082/179] [a64 Fix floating-point `BRANCH_FALSE` --- a64_seq_control.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_seq_control.cc b/a64_seq_control.cc index 1fffcb27a..e68d2955b 100644 --- a/a64_seq_control.cc +++ b/a64_seq_control.cc @@ -528,7 +528,7 @@ struct BRANCH_FALSE_F32 oaknut::Label* label = e.lookup_label(i.src2.value->name); assert_not_null(label); e.FCMP(i.src1, 0); - e.B(Cond::NE, *label); + e.B(Cond::EQ, *label); } }; struct BRANCH_FALSE_F64 @@ -538,7 +538,7 @@ struct BRANCH_FALSE_F64 oaknut::Label* label = e.lookup_label(i.src2.value->name); assert_not_null(label); e.FCMP(i.src1, 0); - e.B(Cond::NE, *label); + e.B(Cond::EQ, *label); } }; EMITTER_OPCODE_TABLE(OPCODE_BRANCH_FALSE, BRANCH_FALSE_I8, BRANCH_FALSE_I16, From cb6a0808e7fd82b9478050edd46561430d5d2949 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 11 May 2024 10:47:11 -0700 Subject: [PATCH 083/179] [a64] Refactor `OPCODE_ATOMIC_COMPARE_EXCHANGE` Much more explicit arguments while trying to debug a deadlock --- a64_seq_memory.cc | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 71c5c982c..e9c64f10a 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -186,7 +186,6 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(W0, i.src2); if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. @@ -200,12 +199,15 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 } e.ADD(X1, e.GetMembaseReg(), X1); + const WReg expected = i.src2; + const WReg desired = i.src3; + const WReg status = W0; + e.MOV(status, expected); + // if([C] == A) [C] = B // else A = [C] - e.CASAL(W0, i.src3, X1); - - // Set dest to 1 in the case of a successful exchange - e.CMP(W0, i.src2); + e.CASAL(status, desired, X1); + e.CMP(status, expected); e.CSET(i.dest, Cond::EQ); } }; @@ -213,7 +215,6 @@ struct ATOMIC_COMPARE_EXCHANGE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(X0, i.src2); if (xe::memory::allocation_granularity() > 0x1000) { // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. @@ -227,12 +228,15 @@ struct ATOMIC_COMPARE_EXCHANGE_I64 } e.ADD(X1, e.GetMembaseReg(), X1); + const XReg expected = i.src2; + const XReg desired = i.src3; + const XReg status = X0; + e.MOV(status, expected); + // if([C] == A) [C] = B // else A = [C] - e.CASAL(X0, i.src3, X1); - - // Set dest to 1 in the case of a successful exchange - e.CMP(X0, i.src2); + e.CASAL(status, desired, X1); + e.CMP(status, expected); e.CSET(i.dest, Cond::EQ); } }; From ca06a9630f3513f6cbe7a626f11ce383a8a7bb8b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 09:02:51 -0700 Subject: [PATCH 084/179] [a64] Fix `OPCODE_MAX` Was not handling constant arguments properly --- a64_sequences.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index df11306d1..5a8578e79 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -490,17 +490,26 @@ EMITTER_OPCODE_TABLE(OPCODE_CONTEXT_BARRIER, CONTEXT_BARRIER); // ============================================================================ struct MAX_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMAX(i.dest, i.src1, i.src2); + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e.FMAX(dest, src1, src2); + }); } }; struct MAX_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMAX(i.dest, i.src1, i.src2); + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e.FMAX(dest, src1, src2); + }); } }; struct MAX_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMAX(i.dest.reg().S4(), i.src1.reg().S4(), i.src2.reg().S4()); + EmitCommutativeBinaryVOp( + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMAX(dest.S4(), src1.S4(), src2.S4()); + }); } }; EMITTER_OPCODE_TABLE(OPCODE_MAX, MAX_F32, MAX_F64, MAX_V128); From fa4752e78db135a62abe58b9ac355daaaf6ead08 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 09:08:18 -0700 Subject: [PATCH 085/179] [a64] Fix `MUL_HI_I32` operands --- a64_sequences.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 5a8578e79..cd42d70bb 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1494,7 +1494,7 @@ struct MUL_HI_I32 } else if (i.src2.is_constant) { assert_true(!i.src1.is_constant); e.MOV(W0, i.src2.constant()); - e.UMULL(X0, W0, i.src2); + e.UMULL(X0, W0, i.src1); e.UBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else { @@ -1512,7 +1512,7 @@ struct MUL_HI_I32 } else if (i.src2.is_constant) { assert_true(!i.src1.is_constant); e.MOV(W0, i.src2.constant()); - e.SMULL(X0, W0, i.src2); + e.SMULL(X0, W0, i.src1); e.SBFX(X0, X0, 32, 32); e.MOV(i.dest, X0.toW()); } else { From bac0b0cd03ac8db3335070393d43293cb3e33cea Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 09:48:38 -0700 Subject: [PATCH 086/179] [a64] Fix `OPCODE_VECTOR_SHA`(constant) Values should be modulo-element-size --- a64_seq_vector.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 5f5e519c7..e5cd4e1aa 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -819,7 +819,7 @@ struct VECTOR_SHA_V128 } if (all_same) { // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); + e.SSHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); return; } e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); @@ -843,7 +843,7 @@ struct VECTOR_SHA_V128 } if (all_same) { // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0]); + e.SSHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0] & 0xF); return; } e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); @@ -867,7 +867,7 @@ struct VECTOR_SHA_V128 } if (all_same) { // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0]); + e.SSHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0] & 0x1F); return; } e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); From 8a14e14798d4748a6e5fc7addead89e711a60c72 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 10:02:49 -0700 Subject: [PATCH 087/179] [a64] Implement `VECTOR_CONVERT_{F2I,I2F}` --- a64_seq_vector.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index e5cd4e1aa..44e476dab 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -68,7 +68,9 @@ struct VECTOR_CONVERT_I2F // Merge the two ways depending on whether the number is >= 0x80000000 // (has high bit set). // e.vblendvps(i.dest, Q1, Q0, i.src1); + e.FCVTNU(i.dest.reg().S4(), i.src1.reg().S4()); } else { + e.FCVTNS(i.dest.reg().S4(), i.src1.reg().S4()); // e.vcvtdq2ps(i.dest, i.src1); } } @@ -106,6 +108,7 @@ struct VECTOR_CONVERT_F2I // saturate values > UINT_MAX // e.vpor(i.dest, i.dest, Q0); + e.UCVTF(i.dest.reg().S4(), i.src1.reg().S4()); } else { // xmm2 = NaN mask // e.vcmpunordps(e.xmm2, i.src1, i.src1); @@ -122,6 +125,7 @@ struct VECTOR_CONVERT_F2I // mask NaNs // e.vpandn(i.dest, e.xmm2, i.dest); + e.SCVTF(i.dest.reg().S4(), i.src1.reg().S4()); } } }; From dda69bbbc1e359bf5cbfb363980e8732170fafb9 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 18:26:28 -0700 Subject: [PATCH 088/179] [a64] Fix `OPCODE_VECTOR_CONVERT_{I2F,F2I}` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 😳 --- a64_seq_vector.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 44e476dab..f3abc16c3 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -68,10 +68,11 @@ struct VECTOR_CONVERT_I2F // Merge the two ways depending on whether the number is >= 0x80000000 // (has high bit set). // e.vblendvps(i.dest, Q1, Q0, i.src1); - e.FCVTNU(i.dest.reg().S4(), i.src1.reg().S4()); + e.UCVTF(i.dest.reg().S4(), i.src1.reg().S4()); + } else { - e.FCVTNS(i.dest.reg().S4(), i.src1.reg().S4()); // e.vcvtdq2ps(i.dest, i.src1); + e.SCVTF(i.dest.reg().S4(), i.src1.reg().S4()); } } }; @@ -108,7 +109,8 @@ struct VECTOR_CONVERT_F2I // saturate values > UINT_MAX // e.vpor(i.dest, i.dest, Q0); - e.UCVTF(i.dest.reg().S4(), i.src1.reg().S4()); + e.FCVTNU(i.dest.reg().S4(), i.src1.reg().S4()); + } else { // xmm2 = NaN mask // e.vcmpunordps(e.xmm2, i.src1, i.src1); @@ -125,7 +127,7 @@ struct VECTOR_CONVERT_F2I // mask NaNs // e.vpandn(i.dest, e.xmm2, i.dest); - e.SCVTF(i.dest.reg().S4(), i.src1.reg().S4()); + e.FCVTNS(i.dest.reg().S4(), i.src1.reg().S4()); } } }; From ef76460a43093c3e2a07edace0a2e32b22cddc5c Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 18:27:07 -0700 Subject: [PATCH 089/179] [a64] Fix `FPCR` starting bit index --- a64_sequences.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index cd42d70bb..b156b720b 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2789,7 +2789,7 @@ struct SET_ROUNDING_MODE_I32 // Replace FPCR bits with new value e.MRS(X1, SystemReg::FPCR); - e.BFI(X1, X0, 54, 3); + e.BFI(X1, X0, 23, 3); e.MSR(SystemReg::FPCR, X1); } }; From 227142488eb87cb0841723c7b573cb854e7ba967 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 18:34:09 -0700 Subject: [PATCH 090/179] [a64] Fix `VECTOR_CONVERT_F2I` rounding ``` 4.2.2.4 Floating-Point Rounding and Conversion Instructions ... Floating-point conversions to integers (vctuxs, vctsxs) use round-toward-zero (truncate). ... ``` This passes all of the `vctuxs` and `vctsxs` unit tests --- a64_seq_vector.cc | 81 ++--------------------------------------------- 1 file changed, 2 insertions(+), 79 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index f3abc16c3..8f6f81f6a 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -31,47 +31,9 @@ struct VECTOR_CONVERT_I2F : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // flags = ARITHMETIC_UNSIGNED if (i.instr->flags & ARITHMETIC_UNSIGNED) { - // Round manually to (1.stored mantissa bits * 2^31) or to 2^32 to the - // nearest even (the only rounding mode used on AltiVec) if the number is - // 0x80000000 or greater, instead of converting src & 0x7FFFFFFF and then - // adding 2147483648.0f, which results in double rounding that can give a - // result larger than needed - see OPCODE_VECTOR_CONVERT_I2F notes. - - // [0x80000000, 0xFFFFFFFF] case: - - // Round to the nearest even, from (0x80000000 | 31 stored mantissa bits) - // to ((-1 << 23) | 23 stored mantissa bits), or to 0 if the result should - // be 4294967296.0f. - // xmm0 = src + 0b01111111 + ((src >> 8) & 1) - // (xmm1 also used to launch reg + mem early and to require it late) - // e.vpaddd(Q1, i.src1, e.GetXmmConstPtr(XMMInt127)); - // e.vpslld(Q0, i.src1, 31 - 8); - // e.vpsrld(Q0, Q0, 31); - // e.vpaddd(Q0, Q0, Q1); - // xmm0 = (0xFF800000 | 23 explicit mantissa bits), or 0 if overflowed - // e.vpsrad(Q0, Q0, 8); - // Calculate the result for the [0x80000000, 0xFFFFFFFF] case - take the - // rounded mantissa, and add -1 or 0 to the exponent of 32, depending on - // whether the number should be (1.stored mantissa bits * 2^31) or 2^32. - // xmm0 = [0x80000000, 0xFFFFFFFF] case result - // e.vpaddd(Q0, Q0, e.GetXmmConstPtr(XMM2To32)); - - // [0x00000000, 0x7FFFFFFF] case - // (during vblendvps reg -> vpaddd reg -> vpaddd mem dependency): - - // Convert from signed integer to float. - // xmm1 = [0x00000000, 0x7FFFFFFF] case result - // e.vcvtdq2ps(Q1, i.src1); - - // Merge the two ways depending on whether the number is >= 0x80000000 - // (has high bit set). - // e.vblendvps(i.dest, Q1, Q0, i.src1); e.UCVTF(i.dest.reg().S4(), i.src1.reg().S4()); - } else { - // e.vcvtdq2ps(i.dest, i.src1); e.SCVTF(i.dest.reg().S4(), i.src1.reg().S4()); } } @@ -86,48 +48,9 @@ struct VECTOR_CONVERT_F2I I> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.instr->flags & ARITHMETIC_UNSIGNED) { - // clamp to min 0 - // e.vmaxps(Q0, i.src1, e.GetXmmConstPtr(XMMZero)); - - // xmm1 = mask of values >= (unsigned)INT_MIN - // e.vcmpgeps(Q1, Q0, e.GetXmmConstPtr(XMMPosIntMinPS)); - - // scale any values >= (unsigned)INT_MIN back to [0, ...] - // e.vsubps(e.xmm2, Q0, e.GetXmmConstPtr(XMMPosIntMinPS)); - // e.vblendvps(Q0, Q0, e.xmm2, Q1); - - // xmm0 = [0, INT_MAX] - // this may still contain values > INT_MAX (if src has vals > UINT_MAX) - // e.vcvttps2dq(i.dest, Q0); - - // xmm0 = mask of values that need saturation - // e.vpcmpeqd(Q0, i.dest, e.GetXmmConstPtr(XMMIntMin)); - - // scale values back above [INT_MIN, UINT_MAX] - // e.vpand(Q1, Q1, e.GetXmmConstPtr(XMMIntMin)); - // e.vpaddd(i.dest, i.dest, Q1); - - // saturate values > UINT_MAX - // e.vpor(i.dest, i.dest, Q0); - e.FCVTNU(i.dest.reg().S4(), i.src1.reg().S4()); - + e.FCVTZU(i.dest.reg().S4(), i.src1.reg().S4()); } else { - // xmm2 = NaN mask - // e.vcmpunordps(e.xmm2, i.src1, i.src1); - - // convert packed floats to packed dwords - // e.vcvttps2dq(Q0, i.src1); - - // (high bit) xmm1 = dest is indeterminate and i.src1 >= 0 - // e.vpcmpeqd(Q1, Q0, e.GetXmmConstPtr(XMMIntMin)); - // e.vpandn(Q1, i.src1, Q1); - - // saturate positive values - // e.vblendvps(i.dest, Q0, e.GetXmmConstPtr(XMMIntMax), Q1); - - // mask NaNs - // e.vpandn(i.dest, e.xmm2, i.dest); - e.FCVTNS(i.dest.reg().S4(), i.src1.reg().S4()); + e.FCVTZS(i.dest.reg().S4(), i.src1.reg().S4()); } } }; From fde9c75127e9d65e082164c968c4514460245a2b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 19:04:58 -0700 Subject: [PATCH 091/179] [a64] Implement `PERMUTE_V128`(int16) Passes 'vmrghh' and `vmrglh` unit-tests --- a64_seq_vector.cc | 57 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 8f6f81f6a..9db5b999f 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1166,7 +1166,62 @@ struct PERMUTE_V128 indices.B16()); } - static void EmitByInt16(A64Emitter& e, const EmitArgType& i) {} + static void EmitByInt16(A64Emitter& e, const EmitArgType& i) { + // Permute bytes between src2 and src3. + // src1 is an array of indices corresponding to positions within src2 and + // src3. + if (i.src3.value->IsConstantZero()) { + if (i.src2.value->IsConstantZero()) { + // src2 & src3 are zero, so result will always be zero. + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); + return; + } + } + + const QReg indices = Q0; + if (i.src1.is_constant) { + e.LoadConstantV(indices, i.src1.constant()); + } else { + e.MOV(indices.B16(), i.src1.reg().B16()); + } + + // Indices must be endian-swapped + e.MOV(W0, 0b1); + e.DUP(Q1.H8(), W0); + e.EOR(indices.B16(), indices.B16(), Q1.B16()); + + // Modulo-16 the indices + e.MOV(W0, 0b0000'1111); + e.DUP(Q1.H8(), W0); + e.AND(indices.B16(), indices.B16(), Q1.B16()); + + // Convert int16 indices into int8 + e.MOV(W0, 0x02'02); + e.DUP(Q1.H8(), W0); + e.MUL(indices.H8(), indices.H8(), Q1.H8()); + + e.MOV(W0, 0x01'00); + e.DUP(Q1.H8(), W0); + e.ADD(indices.H8(), indices.H8(), Q1.H8()); + + // Table-registers must be sequential indices + const QReg table_lo = Q2; + if (i.src2.is_constant) { + e.LoadConstantV(table_lo, i.src2.constant()); + } else { + e.MOV(table_lo.B16(), i.src2.reg().B16()); + } + + const QReg table_hi = Q3; + if (i.src3.is_constant) { + e.LoadConstantV(table_hi, i.src3.constant()); + } else { + e.MOV(table_hi.B16(), i.src3.reg().B16()); + } + + e.TBL(i.dest.reg().B16(), List{table_lo.B16(), table_hi.B16()}, + indices.B16()); + } static void EmitByInt32(A64Emitter& e, const EmitArgType& i) { assert_always(); From 264f37fea4dd8d1eb5ae3945267b0b20784cbc5e Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 20:04:47 -0700 Subject: [PATCH 092/179] [a64] Optimize `OPCODE_MUL_ADD` Use `FMADD` and `FMLA` Tests are the same, though now it should run a bit faster. The tests that fail are primarily denormals and other subtle precision issues it seems. Ex: ``` i> 00002358 - vmaddfp_7298_GEN !> 00002358 Register v4 assert failed: !> 00002358 Expected: v4 == [00000000, 00000000, 00000000, 00000000] !> 00002358 Actual: v4 == [000D000E, 00138014, 000E4CDC, 0018B34D] !> 00002358 TEST FAILED ``` Host-To-Guest and Guest-To-Host thunks should probably restore/preserve the FPCR to maintain these roundings. --- a64_sequences.cc | 89 ++++++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index b156b720b..9b58a8409 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1717,77 +1717,86 @@ EMITTER_OPCODE_TABLE(OPCODE_DIV, DIV_I8, DIV_I16, DIV_I32, DIV_I64, DIV_F32, struct MUL_ADD_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - SReg src3(1); + SReg src3 = S3; if (i.src3.is_constant) { - src3 = S1; e.LoadConstantV(src3.toQ(), i.src3.constant()); } else { - // If i.dest == i.src3, back up i.src3 so we don't overwrite it. src3 = i.src3.reg(); - if (i.dest.reg().index() == i.src3.reg().index()) { - e.FMOV(S1, i.src3); - src3 = S1; - } } - // Multiply operation is commutative. - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FMUL(dest, src1, src2); // $0 = $1 * $2 - }); + SReg src2 = S2; + if (i.src2.is_constant) { + e.LoadConstantV(src2.toQ(), i.src2.constant()); + } else { + src2 = i.src2.reg(); + } - e.FADD(i.dest, i.dest, src3); // $0 = $1 + $2 + SReg src1 = S1; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + + e.FMADD(i.dest, src1, src2, src3); } }; struct MUL_ADD_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - DReg src3(1); + DReg src3 = D3; if (i.src3.is_constant) { - src3 = D1; e.LoadConstantV(src3.toQ(), i.src3.constant()); } else { - // If i.dest == i.src3, back up i.src3 so we don't overwrite it. src3 = i.src3.reg(); - if (i.dest.reg().index() == i.src3.reg().index()) { - e.FMOV(D1, i.src3); - src3 = D1; - } } - // Multiply operation is commutative. - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FMUL(dest, src1, src2); // $0 = $1 * $2 - }); + DReg src2 = D2; + if (i.src2.is_constant) { + e.LoadConstantV(src2.toQ(), i.src2.constant()); + } else { + src2 = i.src2.reg(); + } - e.FADD(i.dest, i.dest, src3); // $0 = $1 + $2 + DReg src1 = D1; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + + e.FMADD(i.dest, src1, src2, src3); } }; struct MUL_ADD_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - QReg src3(1); + const QReg dest = i.dest.reg(); if (i.src3.is_constant) { - src3 = Q1; - e.LoadConstantV(src3, i.src3.constant()); + e.LoadConstantV(dest.toQ(), i.src3.constant()); } else { - // If i.dest == i.src3, back up i.src3 so we don't overwrite it. - src3 = i.src3; - if (i.dest == i.src3) { - e.MOV(Q1.B16(), i.src3.reg().B16()); - src3 = Q1; + // If i.dest != i.src3, move the addition-term into dest for FMLA + if (i.dest != i.src3) { + e.MOV(dest.B16(), i.src3.reg().B16()); } } - // Multiply operation is commutative. - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMUL(dest.S4(), src1.S4(), src2.S4()); // $0 = $1 * $2 - }); + QReg src2 = Q2; + if (i.src2.is_constant) { + e.LoadConstantV(src2.toQ(), i.src2.constant()); + } else { + src2 = i.src2.reg(); + } - e.FADD(i.dest.reg().S4(), i.dest.reg().S4(), src3.S4()); + QReg src1 = Q1; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + + e.FMLA(dest.S4(), src1.S4(), src2.S4()); } }; EMITTER_OPCODE_TABLE(OPCODE_MUL_ADD, MUL_ADD_F32, MUL_ADD_F64, MUL_ADD_V128); From 91fdab4f89de688d490d02ee8b4267bf5cf1c138 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 12 May 2024 21:15:33 -0700 Subject: [PATCH 093/179] [a64] Fix `OPCODE_CNTLZ` 8 and 16 bit CNTLZ needs its bit-count fixed to its original element-type --- a64_sequences.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 9b58a8409..e33e7ef5c 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2740,16 +2740,18 @@ EMITTER_OPCODE_TABLE(OPCODE_BYTE_SWAP, BYTE_SWAP_I16, BYTE_SWAP_I32, // ============================================================================ struct CNTLZ_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // No 8bit lzcnt, so do 16 and sub 8. + // No 8bit lzcnt, so do 32 and sub 24. e.UXTB(i.dest, i.src1); e.CLZ(i.dest, i.dest); - e.SUB(i.dest.reg(), i.dest.reg(), 8); + e.SUB(i.dest.reg(), i.dest.reg(), 24); } }; struct CNTLZ_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + // No 16bit lzcnt, so do 32 and sub 16. e.UXTH(i.dest, i.src1); e.CLZ(i.dest, i.dest); + e.SUB(i.dest.reg(), i.dest.reg(), 16); } }; struct CNTLZ_I32 : Sequence> { From 617d9ddcd6517a491028f241e0c9c08eaae89eff Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 13 May 2024 09:05:51 -0700 Subject: [PATCH 094/179] [a64] Implement `kDebugInfoTraceFunctions` and `kDebugInfoTraceFunctionCoverage` Relies on armv8.1-a atomic features --- a64_emitter.cc | 52 +++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index d57bae253..b44361772 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -215,30 +215,33 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // Safe now to do some tracing. if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctions) { - //// We require 32-bit addresses. - // assert_true(uint64_t(trace_data_->header()) < UINT_MAX); - // auto trace_header = trace_data_->header(); + // We require 32-bit addresses. + assert_true(uint64_t(trace_data_->header()) < UINT_MAX); + auto trace_header = trace_data_->header(); - //// Call count. - // lock(); - // inc(qword[low_address(&trace_header->function_call_count)]); + // Call count. + MOV(W0, 1); + MOVP2R(X5, low_address(&trace_header->function_call_count)); + LDADDAL(X0, X0, X5); - //// Get call history slot. - // static_assert(FunctionTraceData::kFunctionCallerHistoryCount == 4, - // "bitmask depends on count"); - // mov(rax, qword[low_address(&trace_header->function_call_count)]); - // and_(rax, 0b00000011); + // Get call history slot. + static_assert(FunctionTraceData::kFunctionCallerHistoryCount == 4, + "bitmask depends on count"); + LDR(X0, X5); + AND(W0, W0, 0b00000011); - //// Record call history value into slot (guest addr in RDX). - // mov(dword[Xbyak::RegExp(uint32_t(uint64_t( - // low_address(&trace_header->function_caller_history)))) + - // rax * 4], - // edx); + // Record call history value into slot (guest addr in W1). + MOV(X5, uint32_t( + uint64_t(low_address(&trace_header->function_caller_history)))); + STR(W1, X5, X0, oaknut::IndexExt::LSL, 2); - //// Calling thread. Load ax with thread ID. - // EmitGetCurrentThreadId(); - // lock(); - // bts(qword[low_address(&trace_header->function_thread_use)], rax); + // Calling thread. Load X0 with thread ID. + EmitGetCurrentThreadId(); + MOV(W5, 1); + LSL(W0, W5, W0); + + MOVP2R(X5, low_address(&trace_header->function_thread_use)); + LDSET(W0, WZR, X5); } // Load membase. @@ -325,11 +328,12 @@ void A64Emitter::MarkSourceOffset(const Instr* i) { } if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctionCoverage) { - uint32_t instruction_index = + const uint32_t instruction_index = (entry->guest_address - trace_data_->start_address()) / 4; - // lock(); - // inc(qword[low_address(trace_data_->instruction_execute_counts() + - // instruction_index * 8)]); + MOV(X0, 1); + MOVP2R(X1, low_address(trace_data_->instruction_execute_counts() + + instruction_index * 8)); + LDADDAL(X0, ZR, X1); } } From ab2d6a6457584525bd24385a4c90253671df7c79 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 13 May 2024 09:46:41 -0700 Subject: [PATCH 095/179] [a64] Fix `ATOMIC_COMPARE_EXCHANGE_I32` comparison type This fixes 32-bit atomic-compare-exchanges. The upper-half of the input register _must_ be clipped off. This fixes a deadlock in some games. --- a64_seq_memory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index e9c64f10a..c7ed1a654 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -190,7 +190,7 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do // it via memory mapping. e.MOV(W3, 0xE0000000); - e.CMP(i.src1.reg(), X3); + e.CMP(i.src1.reg().toW(), W3); e.CSET(W1, Cond::HS); e.LSL(W1, W1, 12); e.ADD(W1, W1, i.src1.reg().toW()); From 3d456df55e8dc50a9fde75cec8efdd80f25ddbb6 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 13 May 2024 11:14:36 -0700 Subject: [PATCH 096/179] [a64] Implement memory tracing --- a64_seq_memory.cc | 169 +++++++++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 85 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index c7ed1a654..773537f1a 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -367,9 +367,9 @@ struct LOAD_CONTEXT_I8 e.LDRB(i.dest, e.GetContextReg(), i.src1.value); // e.mov(i.dest, e.byte[addr]); if (IsTracingData()) { - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.mov(e.GetNativeParam(1), e.byte[addr]); - // e.CallNative(reinterpret_cast(TraceContextLoadI8)); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.LDRB(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadI8)); } } }; @@ -378,9 +378,9 @@ struct LOAD_CONTEXT_I16 static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDRH(i.dest, e.GetContextReg(), i.src1.value); if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.word[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextLoadI16)); + e.LDRH(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadI16)); } } }; @@ -389,9 +389,9 @@ struct LOAD_CONTEXT_I32 static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDR(i.dest, e.GetContextReg(), i.src1.value); if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.dword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextLoadI32)); + e.LDR(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadI32)); } } }; @@ -400,9 +400,9 @@ struct LOAD_CONTEXT_I64 static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDR(i.dest, e.GetContextReg(), i.src1.value); if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.qword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextLoadI64)); + e.LDR(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadI64)); } } }; @@ -411,9 +411,9 @@ struct LOAD_CONTEXT_F32 static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDR(i.dest, e.GetContextReg(), i.src1.value); if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.dword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextLoadF32)); + e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadF32)); } } }; @@ -421,11 +421,10 @@ struct LOAD_CONTEXT_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDR(i.dest, e.GetContextReg(), i.src1.value); - // e.vmovsd(i.dest, e.qword[addr]); if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.qword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextLoadF64)); + e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadF64)); } } }; @@ -434,9 +433,9 @@ struct LOAD_CONTEXT_V128 static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDR(i.dest, e.GetContextReg(), i.src1.value); if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.ptr[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextLoadV128)); + e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextLoadV128)); } } }; @@ -459,9 +458,9 @@ struct STORE_CONTEXT_I8 e.STRB(i.src2.reg(), e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.byte[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreI8)); + e.LDRB(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreI8)); } } }; @@ -476,9 +475,9 @@ struct STORE_CONTEXT_I16 e.STRH(i.src2.reg(), e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.word[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreI16)); + e.LDRH(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreI16)); } } }; @@ -493,9 +492,9 @@ struct STORE_CONTEXT_I32 e.STR(i.src2.reg(), e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.dword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreI32)); + e.LDR(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreI32)); } } }; @@ -510,9 +509,9 @@ struct STORE_CONTEXT_I64 e.STR(i.src2.reg(), e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), e.qword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreI64)); + e.LDR(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreI64)); } } }; @@ -527,9 +526,9 @@ struct STORE_CONTEXT_F32 e.STR(i.src2, e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.dword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreF32)); + e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreF32)); } } }; @@ -544,9 +543,9 @@ struct STORE_CONTEXT_F64 e.STR(i.src2, e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.qword[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreF64)); + e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreF64)); } } }; @@ -561,9 +560,9 @@ struct STORE_CONTEXT_V128 e.STR(i.src2, e.GetContextReg(), i.src1.value); } if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.ptr[addr]); - // e.mov(e.GetNativeParam(0), i.src1.value); - // e.CallNative(reinterpret_cast(TraceContextStoreV128)); + e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); + e.MOV(e.GetNativeParam(0), i.src1.value); + e.CallNative(reinterpret_cast(TraceContextStoreV128)); } } }; @@ -761,9 +760,9 @@ struct LOAD_I8 : Sequence> { auto addr_reg = ComputeMemoryAddress(e, i.src1); e.LDRB(i.dest, addr_reg); if (IsTracingData()) { - // e.mov(e.GetNativeParam(1).cvt8(), i.dest); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadI8)); + e.MOV(e.GetNativeParam(1).toW(), i.dest); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadI8)); } } }; @@ -777,9 +776,9 @@ struct LOAD_I16 : Sequence> { e.LDRH(i.dest, addr_reg); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1).cvt16(), i.dest); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadI16)); + e.MOV(e.GetNativeParam(1).toW(), i.dest); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadI16)); } } }; @@ -793,9 +792,9 @@ struct LOAD_I32 : Sequence> { e.LDR(i.dest, addr_reg); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1).cvt32(), i.dest); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadI32)); + e.MOV(e.GetNativeParam(1).toW(), i.dest); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadI32)); } } }; @@ -809,9 +808,9 @@ struct LOAD_I64 : Sequence> { e.LDR(i.dest, addr_reg); } if (IsTracingData()) { - // e.mov(e.GetNativeParam(1), i.dest); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadI64)); + e.MOV(e.GetNativeParam(1), i.dest); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadI64)); } } }; @@ -823,9 +822,9 @@ struct LOAD_F32 : Sequence> { assert_always("not implemented yet"); } if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.dword[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadF32)); + e.MOV(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadF32)); } } }; @@ -837,9 +836,9 @@ struct LOAD_F64 : Sequence> { assert_always("not implemented yet"); } if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.qword[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadF64)); + e.MOV(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadF64)); } } }; @@ -854,9 +853,9 @@ struct LOAD_V128 : Sequence> { e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } if (IsTracingData()) { - // e.lea(e.GetNativeParam(1), e.ptr[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryLoadV128)); + e.MOV(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryLoadV128)); } } }; @@ -878,9 +877,9 @@ struct STORE_I8 : Sequence> { } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.mov(e.GetNativeParam(1).cvt8(), e.byte[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreI8)); + e.LDRB(e.GetNativeParam(1).toW(), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreI8)); } } }; @@ -900,9 +899,9 @@ struct STORE_I16 : Sequence> { } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.mov(e.GetNativeParam(1).cvt16(), e.word[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreI16)); + e.LDRH(e.GetNativeParam(1).toW(), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreI16)); } } }; @@ -922,9 +921,9 @@ struct STORE_I32 : Sequence> { } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.mov(e.GetNativeParam(1).cvt32(), e.dword[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreI32)); + e.LDR(e.GetNativeParam(1).toW(), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreI32)); } } }; @@ -943,9 +942,9 @@ struct STORE_I64 : Sequence> { } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.mov(e.GetNativeParam(1), e.qword[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreI64)); + e.LDR(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreI64)); } } }; @@ -965,9 +964,9 @@ struct STORE_F32 : Sequence> { } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.lea(e.GetNativeParam(1), e.ptr[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreF32)); + e.MOV(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreF32)); } } }; @@ -987,9 +986,9 @@ struct STORE_F64 : Sequence> { } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.lea(e.GetNativeParam(1), e.ptr[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreF64)); + e.MOV(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreF64)); } } }; @@ -1014,9 +1013,9 @@ struct STORE_V128 } if (IsTracingData()) { addr_reg = ComputeMemoryAddress(e, i.src1); - // e.lea(e.GetNativeParam(1), e.ptr[addr]); - // e.lea(e.GetNativeParam(0), e.ptr[addr]); - // e.CallNative(reinterpret_cast(TraceMemoryStoreV128)); + e.MOV(e.GetNativeParam(1), addr_reg); + e.MOV(e.GetNativeParam(0), addr_reg); + e.CallNative(reinterpret_cast(TraceMemoryStoreV128)); } } }; From 4002266e13566546b5ed0bdd04b5225f1a4e87de Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 16 May 2024 09:22:28 -0700 Subject: [PATCH 097/179] [a64] Reduce function prolog/epilog to 16 bytes Just need to store `fp` and `lr` --- a64_backend.cc | 4 ++++ a64_code_cache_win.cc | 6 +++--- a64_emitter.cc | 18 ++++-------------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index ee24766af..c16961a64 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -437,7 +437,9 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { code_offsets.prolog = offset(); + // Preserve context register STP(ZR, X0, SP, PRE_INDEXED, -16); + SUB(SP, SP, stack_size); code_offsets.prolog_stack_alloc = offset(); @@ -462,6 +464,8 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // add(rsp, stack_size); // jmp(rax); ADD(SP, SP, stack_size); + + // Reload context register LDP(ZR, X0, SP, POST_INDEXED, 16); BR(X16); diff --git a/a64_code_cache_win.cc b/a64_code_cache_win.cc index 93310af9d..21a87e9f2 100644 --- a/a64_code_cache_win.cc +++ b/a64_code_cache_win.cc @@ -273,18 +273,18 @@ void Win32A64CodeCache::InitializeUnwindEntry( // function was called. // Function frames are generally: - // STP(X29, X30, SP, PRE_INDEXED, -32); + // STP(X29, X30, SP, PRE_INDEXED, -16); // MOV(X29, XSP); // SUB(XSP, XSP, stack_size); // ... function body ... // ADD(XSP, XSP, stack_size); // MOV(XSP, X29); - // LDP(X29, X30, SP, POST_INDEXED, 32); + // LDP(X29, X30, SP, POST_INDEXED, 16); // These opcodes must undo the epilog and put the return address within lr unwind_info->UnwindCodes[0] = OpAllocL(func_info.stack_size); unwind_info->UnwindCodes[1] = - UnwindOpWord(UWOP_SET_FP, OpSaveFpLrX(-32), UWOP_END); + UnwindOpWord(UWOP_SET_FP, OpSaveFpLrX(-16), UWOP_END); // Add entry. RUNTIME_FUNCTION& fn_entry = unwind_table_[unwind_table_slot]; diff --git a/a64_emitter.cc b/a64_emitter.cc index b44361772..c1d64cb4c 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -201,7 +201,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { func_info.stack_size = stack_size; stack_size_ = stack_size; - STP(X29, X30, SP, PRE_INDEXED, -32); + STP(X29, X30, SP, PRE_INDEXED, -16); MOV(X29, SP); SUB(SP, SP, (uint32_t)stack_size); @@ -287,7 +287,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { ADD(SP, SP, (uint32_t)stack_size); MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 32); + LDP(X29, X30, SP, POST_INDEXED, 16); RET(); @@ -447,15 +447,12 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { EmitTraceUserCallReturn(); // Pass the callers return address over. - // mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]); LDR(X0, SP, StackLayout::GUEST_RET_ADDR); - // add(rsp, static_cast(stack_size())); - // jmp(rax); ADD(SP, SP, static_cast(stack_size())); MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 32); + LDP(X29, X30, SP, POST_INDEXED, 16); BR(X16); } else { @@ -484,19 +481,12 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, // or a thunk to ResolveAddress. if (code_cache_->has_indirection_table()) { if (reg.toW().index() != W17.index()) { - // mov(ebx, reg.cvt32()); MOV(W17, reg.toW()); } LDR(W16, X17); - // mov(eax, dword[ebx]); } else { // Old-style resolve. // Not too important because indirection table is almost always available. - // mov(rcx, GetContextReg()); - // mov(edx, reg.cvt32()); - // - // mov(rax, reinterpret_cast(ResolveFunction)); - // call(rax); MOV(X0, GetContextReg()); MOV(W1, reg.toW()); @@ -516,7 +506,7 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, ADD(SP, SP, static_cast(stack_size())); MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 32); + LDP(X29, X30, SP, POST_INDEXED, 16); BR(X16); } else { From 4b6e6d0e76ff98fc3bc6dd6f3396dcb1cb7ab875 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 16 May 2024 10:58:19 -0700 Subject: [PATCH 098/179] [a64] Implement instruction stepping. Uses `0x0000'dead` as an instructon-stepping sentinel value. Support for basic jumping instructions like `b`, `bl`, `br`, and `blr`. --- a64_backend.cc | 51 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index c16961a64..79d57424a 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -228,26 +228,41 @@ uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, insn.detail = &all_detail; cs_disasm_iter(capstone_handle_, &machine_code_ptr, &remaining_machine_code_size, &host_address, &insn); - auto& detail = all_detail.x86; + const auto& detail = all_detail.arm64; switch (insn.id) { case ARM64_INS_B: - case ARM64_INS_BL: + case ARM64_INS_BL: { + assert_true(detail.operands[0].type == ARM64_OP_IMM); + uint64_t target_pc = static_cast(detail.operands[0].imm); + return current_pc + target_pc; + } break; case ARM64_INS_BLR: - case ARM64_INS_BR: - case ARM64_INS_RET: - // todo(wunkolo): determine next instruction - default: + case ARM64_INS_BR: { + assert_true(detail.operands[0].type == ARM64_OP_REG); + uint64_t target_pc = + ReadCapstoneReg(&thread_info->host_context, detail.operands[0].reg); + return target_pc; + } break; + case ARM64_INS_RET: { + assert_zero(detail.op_count); + // Jump to link register + return thread_info->host_context.x[30]; + } break; + case ARM64_INS_CBNZ: + case ARM64_INS_CBZ: + default: { // Not a branching instruction - just move over it. return current_pc + insn.size; + } break; } } void A64Backend::InstallBreakpoint(Breakpoint* breakpoint) { breakpoint->ForEachHostAddress([breakpoint](uint64_t host_address) { auto ptr = reinterpret_cast(host_address); - auto original_bytes = xe::load_and_swap(ptr); - assert_true(original_bytes != 0x0F0B); - xe::store_and_swap(ptr, 0x0F0B); + auto original_bytes = xe::load_and_swap(ptr); + assert_true(original_bytes != 0x0000'dead); + xe::store_and_swap(ptr, 0x0000'dead); breakpoint->backend_data().emplace_back(host_address, original_bytes); }); } @@ -265,18 +280,18 @@ void A64Backend::InstallBreakpoint(Breakpoint* breakpoint, Function* fn) { // Assume we haven't already installed a breakpoint in this spot. auto ptr = reinterpret_cast(host_address); - auto original_bytes = xe::load_and_swap(ptr); - assert_true(original_bytes != 0x0F0B); - xe::store_and_swap(ptr, 0x0F0B); + auto original_bytes = xe::load_and_swap(ptr); + assert_true(original_bytes != 0x0000'dead); + xe::store_and_swap(ptr, 0x0000'dead); breakpoint->backend_data().emplace_back(host_address, original_bytes); } void A64Backend::UninstallBreakpoint(Breakpoint* breakpoint) { for (auto& pair : breakpoint->backend_data()) { auto ptr = reinterpret_cast(pair.first); - auto instruction_bytes = xe::load_and_swap(ptr); - assert_true(instruction_bytes == 0x0F0B); - xe::store_and_swap(ptr, static_cast(pair.second)); + auto instruction_bytes = xe::load_and_swap(ptr); + assert_true(instruction_bytes == 0x0000'dead); + xe::store_and_swap(ptr, static_cast(pair.second)); } breakpoint->backend_data().clear(); } @@ -296,9 +311,9 @@ bool A64Backend::ExceptionCallback(Exception* ex) { // Verify an expected illegal instruction. auto instruction_bytes = - xe::load_and_swap(reinterpret_cast(ex->pc())); - if (instruction_bytes != 0x0F0B) { - // Not our ud2 - not us. + xe::load_and_swap(reinterpret_cast(ex->pc())); + if (instruction_bytes != 0x0000'dead) { + // Not our `udf #0xdead` - not us. return false; } From 1343ef353307353a6d8c064b142683d55135facb Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 17 May 2024 08:10:24 -0700 Subject: [PATCH 099/179] [a64] Implement `b` `bl` `br` `blr` `cbnz` `cbz` instruction-stepping --- a64_backend.cc | 157 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 147 insertions(+), 10 deletions(-) diff --git a/a64_backend.cc b/a64_backend.cc index 79d57424a..3955598dc 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -212,12 +212,120 @@ uint64_t ReadCapstoneReg(HostThreadContext* context, arm64_reg reg) { return context->x[29]; case ARM64_REG_X30: return context->x[30]; + case ARM64_REG_W0: + return uint32_t(context->x[0]); + case ARM64_REG_W1: + return uint32_t(context->x[1]); + case ARM64_REG_W2: + return uint32_t(context->x[2]); + case ARM64_REG_W3: + return uint32_t(context->x[3]); + case ARM64_REG_W4: + return uint32_t(context->x[4]); + case ARM64_REG_W5: + return uint32_t(context->x[5]); + case ARM64_REG_W6: + return uint32_t(context->x[6]); + case ARM64_REG_W7: + return uint32_t(context->x[7]); + case ARM64_REG_W8: + return uint32_t(context->x[8]); + case ARM64_REG_W9: + return uint32_t(context->x[9]); + case ARM64_REG_W10: + return uint32_t(context->x[10]); + case ARM64_REG_W11: + return uint32_t(context->x[11]); + case ARM64_REG_W12: + return uint32_t(context->x[12]); + case ARM64_REG_W13: + return uint32_t(context->x[13]); + case ARM64_REG_W14: + return uint32_t(context->x[14]); + case ARM64_REG_W15: + return uint32_t(context->x[15]); + case ARM64_REG_W16: + return uint32_t(context->x[16]); + case ARM64_REG_W17: + return uint32_t(context->x[17]); + case ARM64_REG_W18: + return uint32_t(context->x[18]); + case ARM64_REG_W19: + return uint32_t(context->x[19]); + case ARM64_REG_W20: + return uint32_t(context->x[20]); + case ARM64_REG_W21: + return uint32_t(context->x[21]); + case ARM64_REG_W22: + return uint32_t(context->x[22]); + case ARM64_REG_W23: + return uint32_t(context->x[23]); + case ARM64_REG_W24: + return uint32_t(context->x[24]); + case ARM64_REG_W25: + return uint32_t(context->x[25]); + case ARM64_REG_W26: + return uint32_t(context->x[26]); + case ARM64_REG_W27: + return uint32_t(context->x[27]); + case ARM64_REG_W28: + return uint32_t(context->x[28]); + case ARM64_REG_W29: + return uint32_t(context->x[29]); + case ARM64_REG_W30: + return uint32_t(context->x[30]); default: assert_unhandled_case(reg); return 0; } } +bool TestCapstonePstate(arm64_cc cond, uint32_t pstate) { + // https://devblogs.microsoft.com/oldnewthing/20220815-00/?p=106975 + // Upper 4 bits of pstate are NZCV + const bool N = !!(pstate & 0x80000000); + const bool Z = !!(pstate & 0x40000000); + const bool C = !!(pstate & 0x20000000); + const bool V = !!(pstate & 0x10000000); + switch (cond) { + case ARM64_CC_EQ: + return (Z == true); + case ARM64_CC_NE: + return (Z == false); + case ARM64_CC_HS: + return (C == true); + case ARM64_CC_LO: + return (C == false); + case ARM64_CC_MI: + return (N == true); + case ARM64_CC_PL: + return (N == false); + case ARM64_CC_VS: + return (V == true); + case ARM64_CC_VC: + return (V == false); + case ARM64_CC_HI: + return ((C == true) && (Z == false)); + case ARM64_CC_LS: + return ((C == false) || (Z == true)); + case ARM64_CC_GE: + return (N == V); + case ARM64_CC_LT: + return (N != V); + case ARM64_CC_GT: + return ((Z == false) && (N == V)); + case ARM64_CC_LE: + return ((Z == true) || (N != V)); + case ARM64_CC_AL: + return true; + case ARM64_CC_NV: + return false; + default: + assert_unhandled_case(cond); + return false; + } +} + uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, uint64_t current_pc) { auto machine_code_ptr = reinterpret_cast(current_pc); @@ -233,23 +341,52 @@ uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, case ARM64_INS_B: case ARM64_INS_BL: { assert_true(detail.operands[0].type == ARM64_OP_IMM); - uint64_t target_pc = static_cast(detail.operands[0].imm); - return current_pc + target_pc; + const int64_t pc_offset = static_cast(detail.operands[0].imm); + const bool test_passed = + TestCapstonePstate(detail.cc, thread_info->host_context.cpsr); + if (test_passed) { + return current_pc + pc_offset; + } else { + return current_pc + insn.size; + } } break; - case ARM64_INS_BLR: - case ARM64_INS_BR: { + case ARM64_INS_BR: + case ARM64_INS_BLR: { assert_true(detail.operands[0].type == ARM64_OP_REG); - uint64_t target_pc = + const uint64_t target_pc = ReadCapstoneReg(&thread_info->host_context, detail.operands[0].reg); return target_pc; } break; case ARM64_INS_RET: { - assert_zero(detail.op_count); - // Jump to link register - return thread_info->host_context.x[30]; + assert_true(detail.operands[0].type == ARM64_OP_REG); + const uint64_t target_pc = + ReadCapstoneReg(&thread_info->host_context, detail.operands[0].reg); + return target_pc; + } break; + case ARM64_INS_CBNZ: { + assert_true(detail.operands[0].type == ARM64_OP_REG); + assert_true(detail.operands[1].type == ARM64_OP_IMM); + const int64_t pc_offset = static_cast(detail.operands[1].imm); + const bool test_passed = (0 != ReadCapstoneReg(&thread_info->host_context, + detail.operands[0].reg)); + if (test_passed) { + return current_pc + pc_offset; + } else { + return current_pc + insn.size; + } + } break; + case ARM64_INS_CBZ: { + assert_true(detail.operands[0].type == ARM64_OP_REG); + assert_true(detail.operands[1].type == ARM64_OP_IMM); + const int64_t pc_offset = static_cast(detail.operands[1].imm); + const bool test_passed = (0 == ReadCapstoneReg(&thread_info->host_context, + detail.operands[0].reg)); + if (test_passed) { + return current_pc + pc_offset; + } else { + return current_pc + insn.size; + } } break; - case ARM64_INS_CBNZ: - case ARM64_INS_CBZ: default: { // Not a branching instruction - just move over it. return current_pc + insn.size; From 1602344093b73cbd11a09dadbd4f6e6134fce9d3 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 21 May 2024 09:31:06 -0700 Subject: [PATCH 100/179] [a64] Optimize vector-constant generation Uses MOVI to optimize some cases of constants rather than EOR. MOVI is a register-renaming idiom on many architectures. --- a64_emitter.cc | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index c1d64cb4c..559f66f3f 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -827,17 +827,15 @@ std::byte* A64Emitter::GetVConstPtr(VConst id) { // Implies possible StashV(0, ...)! void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { - // https://www.agner.org/optimize/optimizing_assembly.pdf - // 13.4 Generating constants if (!v.low && !v.high) { // 0000... - EOR(dest.B16(), dest.B16(), dest.B16()); - } - // else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) { - // // 1111... - // vpcmpeqb(dest, dest); - // } - else { + // MOVI is implemented as a register-rename while EOR(x, x, x) is not + // https://dougallj.github.io/applecpu/firestorm.html + MOVI(dest.B16(), 0); + } else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) { + // 1111... + MOVI(dest.B16(), 0xFF); + } else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. MovMem64(SP, kStashOffset, v.low); @@ -853,13 +851,11 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, float v) { } x = {v}; if (!x.i) { // +0.0f (but not -0.0f because it may be used to flip the sign via xor). - EOR(dest.B16(), dest.B16(), dest.B16()); - } - // else if (x.i == ~uint32_t(0)) { - // // 1111... - // vpcmpeqb(dest, dest); - // } - else { + MOVI(dest.B16(), 0); + } else if (x.i == ~uint32_t(0)) { + // 1111... + MOVI(dest.B16(), 0xFF); + } else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. MOV(W0, x.i); @@ -874,13 +870,11 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, double v) { } x = {v}; if (!x.i) { // +0.0 (but not -0.0 because it may be used to flip the sign via xor). - EOR(dest.B16(), dest.B16(), dest.B16()); - } - // else if (x.i == ~uint64_t(0)) { - // // 1111... - // vpcmpeqb(dest, dest); - // } - else { + MOVI(dest.toD(), oaknut::RepImm(0)); + } else if (x.i == ~uint64_t(0)) { + // 1111... + MOVI(dest.toD(), oaknut::RepImm(0xFF)); + } else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. MOV(X0, x.i); From 4fa8a3d3317dc77b2a9c74dc278322339ce9898b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 21 May 2024 09:53:16 -0700 Subject: [PATCH 101/179] [a64] Optimize memory-address calculation The LSL can be embedded into the ADD to remove an additional instruction. What was `cset`+`lsl`+`add` should now just be `cset`+`add ... LSL 12` --- a64_seq_memory.cc | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 773537f1a..8b66c1c68 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -27,7 +27,7 @@ template XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, WReg address_register = W3) { assert_true(offset.is_constant); - int32_t offset_const = static_cast(offset.constant()); + const int32_t offset_const = static_cast(offset.constant()); if (guest.is_constant) { uint32_t address = static_cast(guest.constant()); @@ -53,8 +53,7 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, e.MOV(W0, 0xE0000000 - offset_const); e.CMP(guest.reg().toW(), W0); e.CSET(W0, Cond::HS); - e.LSL(W0, W0, 12); - e.ADD(W0, W0, guest.reg().toW()); + e.ADD(W0, guest.reg().toW(), W0, LSL, 12); } else { // Clear the top 32 bits, as they are likely garbage. // TODO(benvanik): find a way to avoid doing this. @@ -76,7 +75,7 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, // TODO(benvanik): figure out how to do this without a temp. // Since the constant is often 0x8... if we tried to use that as a // displacement it would be sign extended and mess things up. - uint32_t address = static_cast(guest.constant()); + const uint32_t address = static_cast(guest.constant()); if (address < 0x80000000) { e.MOV(W0, address); e.ADD(address_register.toX(), e.GetMembaseReg(), X0); @@ -98,8 +97,7 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, e.MOV(W0, 0xE0000000); e.CMP(guest.reg().toW(), W0); e.CSET(W0, Cond::HS); - e.LSL(W0, W0, 12); - e.ADD(W0, W0, guest.reg().toW()); + e.ADD(W0, guest.reg().toW(), W0, LSL, 12); } else { // Clear the top 32 bits, as they are likely garbage. // TODO(benvanik): find a way to avoid doing this. @@ -107,7 +105,6 @@ XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, } e.ADD(address_register.toX(), e.GetMembaseReg(), X0); return address_register.toX(); - // return e.GetMembaseReg() + e.rax; } } @@ -192,8 +189,7 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 e.MOV(W3, 0xE0000000); e.CMP(i.src1.reg().toW(), W3); e.CSET(W1, Cond::HS); - e.LSL(W1, W1, 12); - e.ADD(W1, W1, i.src1.reg().toW()); + e.ADD(W1, i.src1.reg().toW(), W1, LSL, 12); } else { e.MOV(W1, i.src1.reg().toW()); } @@ -221,8 +217,7 @@ struct ATOMIC_COMPARE_EXCHANGE_I64 e.MOV(W3, 0xE0000000); e.CMP(i.src1.reg(), X3); e.CSET(W1, Cond::HS); - e.LSL(W1, W1, 12); - e.ADD(W1, W1, i.src1.reg().toW()); + e.ADD(W1, i.src1.reg().toW(), W1, LSL, 12); } else { e.MOV(W1, i.src1.reg().toW()); } From 2feb17ed29347022a09b717c559199f13ec56cbf Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 21 May 2024 10:00:57 -0700 Subject: [PATCH 102/179] [a64] Optimize `OPCODE_MEMSET` Use pair-stores rather than singular-stores to write 32-bytes of data at a time. --- a64_seq_memory.cc | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 8b66c1c68..dc9cfca7e 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -1126,22 +1126,17 @@ struct MEMSET_I64_I8_I64 assert_true(i.src2.is_constant); assert_true(i.src3.is_constant); assert_true(i.src2.constant() == 0); - e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); + e.MOVI(Q0.B16(), 0); auto addr_reg = ComputeMemoryAddress(e, i.src1); switch (i.src3.constant()) { case 32: - e.STR(Q0, addr_reg, 0 * 16); - e.STR(Q0, addr_reg, 1 * 16); + e.STP(Q0, Q0, addr_reg, 0 * 16); break; case 128: - e.STR(Q0, addr_reg, 0 * 16); - e.STR(Q0, addr_reg, 1 * 16); - e.STR(Q0, addr_reg, 2 * 16); - e.STR(Q0, addr_reg, 3 * 16); - e.STR(Q0, addr_reg, 4 * 16); - e.STR(Q0, addr_reg, 5 * 16); - e.STR(Q0, addr_reg, 6 * 16); - e.STR(Q0, addr_reg, 7 * 16); + e.STP(Q0, Q0, addr_reg, 0 * 16); + e.STP(Q0, Q0, addr_reg, 2 * 16); + e.STP(Q0, Q0, addr_reg, 4 * 16); + e.STP(Q0, Q0, addr_reg, 6 * 16); break; default: assert_unhandled_case(i.src3.constant()); From 6951cbeccef5571beee19c7f6b9ecdd199456b16 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 21 May 2024 10:20:28 -0700 Subject: [PATCH 103/179] [a64] Implement `OPCODE_LOAD_CLOCk` `clock_source_raw` Uses the `CNTVCT_EL0`-register and applies frequency scaling --- a64_sequences.cc | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index e33e7ef5c..94cb8fcd4 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -448,23 +448,17 @@ struct LOAD_CLOCK : Sequence> { // overhead. if (cvars::clock_no_scaling && cvars::clock_source_raw) { auto ratio = Clock::guest_tick_ratio(); - // The 360 CPU is an in-order CPU, AMD64 usually isn't. Without - // mfence/lfence magic the rdtsc instruction can be executed sooner or - // later in the cache window. Since it's resolution however is much higher - // than the 360's mftb instruction this can safely be ignored. + // The 360 CPU is an in-order CPU, ARM64 usually isn't. Since it's + // resolution however is much higher than the 360's mftb instruction this + // can safely be ignored. - // Read time stamp in edx (high part) and eax (low part). - // e.rdtsc(); - // Make it a 64 bit number in rax. - // e.shl(e.rdx, 32); - // e.or_(e.rax, e.rdx); + // Read clock cycle count + e.MRS(i.dest, SystemReg::CNTVCT_EL0); // Apply tick frequency scaling. - // e.MOV(e.rcx, ratio.first); - // e.mul(e.rcx); - // We actually now have a 128 bit number in rdx:rax. - // e.MOV(e.rcx, ratio.second); - // e.div(e.rcx); - // e.MOV(i.dest, e.rax); + e.MOV(X0, ratio.first); + e.MUL(i.dest, i.dest, X0); + e.MOV(X0, ratio.second); + e.UDIV(i.dest, i.dest, X0); } else { e.CallNative(LoadClock); e.MOV(i.dest, X0); From ab1988d64ebcf2f7fd996a4f0d01f23aa59fe28a Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 23 May 2024 09:39:49 -0700 Subject: [PATCH 104/179] [a64] Implement `OPCODE_PACK`(2101010, 4202020, 8-in-16, 16-in-32) --- a64_seq_vector.cc | 230 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 227 insertions(+), 3 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 9db5b999f..2e49fc92f 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1423,13 +1423,237 @@ struct PACK : Sequence> { e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 4); e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); } - static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) {} - static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) {} + static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { + // https://www.opengl.org/registry/specs/ARB/vertex_type_2_10_10_10_rev.txt + // XYZ are 10 bits, signed and saturated. + // W is 2 bits, unsigned and saturated. + const QReg src = i.dest; + if (i.src1.is_constant) { + e.LoadConstantV(src, i.src1.constant()); + } + + // Saturate. + e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MinUnpacked)); + e.LDR(Q1, X0); + e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MaxUnpacked)); + e.LDR(Q1, X0); + e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Remove the unneeded bits of the floats. + e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MaskUnpacked)); + e.LDR(Q1, X0); + e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); + + // Shift the components up. + e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_Shift)); + e.LDR(Q1, X0); + e.USHL(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Combine the components. + e.LoadConstantV(Q1, vec128i(0x03'02'01'00 + 0x04'04'04'04 * 2, + 0x03'02'01'00 + 0x04'04'04'04 * 3, + 0x03'02'01'00 + 0x04'04'04'04 * 0, + 0x03'02'01'00 + 0x04'04'04'04 * 1)); + e.TBL(Q0.B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); + + e.LoadConstantV(Q1, vec128i(0x03'02'01'00 + 0x04'04'04'04 * 1, + 0x03'02'01'00 + 0x04'04'04'04 * 0, + 0x03'02'01'00 + 0x04'04'04'04 * 3, + 0x03'02'01'00 + 0x04'04'04'04 * 2)); + e.TBL(Q0.B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); + } + static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { + // XYZ are 20 bits, signed and saturated. + // W is 4 bits, unsigned and saturated. + QReg src = i.src1; + if (i.src1.is_constant) { + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } + // Saturate. + e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_MinUnpacked)); + e.LDR(Q1, X0); + e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_MaxUnpacked)); + e.LDR(Q1, X0); + e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Remove the unneeded bits of the floats (so excess nibbles will also be + // cleared). + e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_MaskUnpacked)); + e.LDR(Q1, X0); + e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); + + // Store Y and W shifted left by 4 so vpshufb can be used with them. + e.SHL(Q0.S4(), i.dest.reg().S4(), 4); + + // Place XZ where they're supposed to be. + e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_PermuteXZ)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + // Place YW. + e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_PermuteYW)); + e.LDR(Q1, X0); + e.TBL(Q0.B16(), oaknut::List{Q0.B16()}, Q1.B16()); + // Merge XZ and YW. + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); + } static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { + if (IsPackInUnsigned(flags)) { + if (IsPackOutUnsigned(flags)) { + if (IsPackOutSaturate(flags)) { + // unsigned -> unsigned + saturate + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + + const QReg src2 = i.src2.is_constant ? Q1 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2, i.src2.constant()); + } + e.UQXTN(i.dest.reg().toD().B8(), src1.H8()); + e.UQXTN2(i.dest.reg().B16(), src2.H8()); + + e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); + e.LDR(Q0, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + } else { + // unsigned -> unsigned + e.XTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); + e.XTN2(i.dest.reg().B16(), i.src2.reg().H8()); + + e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); + e.LDR(Q0, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + } + } else { + if (IsPackOutSaturate(flags)) { + // unsigned -> signed + saturate + assert_always(); + } else { + // unsigned -> signed + assert_always(); + } + } + } else { + if (IsPackOutUnsigned(flags)) { + if (IsPackOutSaturate(flags)) { + // signed -> unsigned + saturate + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + + const QReg src2 = i.src2.is_constant ? Q1 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2, i.src2.constant()); + } + + e.UQXTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); + e.UQXTN2(i.dest.reg().B16(), src2.H8()); + + e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); + e.LDR(Q0, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + } else { + // signed -> unsigned + assert_always(); + } + } else { + if (IsPackOutSaturate(flags)) { + // signed -> signed + saturate + e.SQXTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); + e.SQXTN2(i.dest.reg().B16(), i.src2.reg().H8()); + + e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); + e.LDR(Q0, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + } else { + // signed -> signed + assert_always(); + } + } + } } // Pack 2 32-bit vectors into a 16-bit vector. static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, - uint32_t flags) {} + uint32_t flags) { + // TODO(benvanik): handle src2 (or src1) being constant zero + if (IsPackInUnsigned(flags)) { + if (IsPackOutUnsigned(flags)) { + if (IsPackOutSaturate(flags)) { + // unsigned -> unsigned + saturate + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + + const QReg src2 = i.src2.is_constant ? Q1 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2, i.src2.constant()); + } + + e.UQXTN(i.dest.reg().toD().H4(), src1.S4()); + e.UQXTN2(i.dest.reg().H8(), src2.S4()); + + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + } else { + // unsigned -> unsigned + e.XTN(i.dest.reg().toD().H4(), i.src1.reg().S4()); + e.XTN2(i.dest.reg().H8(), i.src2.reg().S4()); + + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + } + } else { + if (IsPackOutSaturate(flags)) { + // unsigned -> signed + saturate + assert_always(); + } else { + // unsigned -> signed + assert_always(); + } + } + } else { + if (IsPackOutUnsigned(flags)) { + if (IsPackOutSaturate(flags)) { + // signed -> unsigned + saturate + e.UQXTN(i.dest.reg().toD().H4(), i.src1.reg().S4()); + e.UQXTN2(i.dest.reg().H8(), i.src2.reg().S4()); + + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + } else { + // signed -> unsigned + assert_always(); + } + } else { + if (IsPackOutSaturate(flags)) { + // signed -> signed + saturate + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + + const QReg src2 = i.src2.is_constant ? Q1 : i.src2; + if (i.src2.is_constant) { + e.LoadConstantV(src2, i.src2.constant()); + } + e.SQXTN(i.dest.reg().toD().H4(), src1.S4()); + e.SQXTN2(i.dest.reg().H8(), src2.S4()); + + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + } else { + // signed -> signed + assert_always(); + } + } + } + } }; EMITTER_OPCODE_TABLE(OPCODE_PACK, PACK); From 15cf9a3fe901d75a56e523d55eee8ac4b1a81a0a Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 23 May 2024 14:11:42 -0700 Subject: [PATCH 105/179] [a64] Fix `OPCODE_PACK` saturation edge-cases Passes cpu-ppc-tests --- a64_seq_vector.cc | 66 ++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 2e49fc92f..a816e4210 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1368,11 +1368,12 @@ struct PACK : Sequence> { assert_true(i.src2.value->IsConstantZero()); // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx // dest = [(src1.x | src1.y), 0, 0, 0] + // TODO(wunkolo): FP16 + FCVTN if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); } else { - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); } e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_2)); e.MOV(i.dest.reg().B16(), Q0.B16()); @@ -1394,11 +1395,12 @@ struct PACK : Sequence> { assert_true(i.src2.value->IsConstantZero()); // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx // dest = [(src1.z | src1.w), (src1.x | src1.y), 0, 0] + // TODO(wunkolo): FP16 + FCVTN if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); } else { - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); } e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_4)); e.MOV(i.dest.reg().B16(), Q0.B16()); @@ -1517,20 +1519,18 @@ struct PACK : Sequence> { if (i.src2.is_constant) { e.LoadConstantV(src2, i.src2.constant()); } - e.UQXTN(i.dest.reg().toD().B8(), src1.H8()); - e.UQXTN2(i.dest.reg().B16(), src2.H8()); + e.UQXTN(i.dest.reg().toD().B8(), src2.H8()); + e.UQXTN2(i.dest.reg().B16(), src1.H8()); - e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); - e.LDR(Q0, X0); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } else { // unsigned -> unsigned - e.XTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); - e.XTN2(i.dest.reg().B16(), i.src2.reg().H8()); + e.XTN(i.dest.reg().toD().B8(), i.src2.reg().H8()); + e.XTN2(i.dest.reg().B16(), i.src1.reg().H8()); - e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); - e.LDR(Q0, X0); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } } else { if (IsPackOutSaturate(flags)) { @@ -1555,12 +1555,11 @@ struct PACK : Sequence> { e.LoadConstantV(src2, i.src2.constant()); } - e.UQXTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); - e.UQXTN2(i.dest.reg().B16(), src2.H8()); + e.SQXTUN(i.dest.reg().toD().B8(), src2.H8()); + e.SQXTUN2(i.dest.reg().B16(), src1.H8()); - e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); - e.LDR(Q0, X0); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } else { // signed -> unsigned assert_always(); @@ -1568,12 +1567,11 @@ struct PACK : Sequence> { } else { if (IsPackOutSaturate(flags)) { // signed -> signed + saturate - e.SQXTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); - e.SQXTN2(i.dest.reg().B16(), i.src2.reg().H8()); + e.SQXTN(i.dest.reg().toD().B8(), i.src2.reg().H8()); + e.SQXTN2(i.dest.reg().B16(), i.src1.reg().H8()); - e.MOVP2R(X0, e.GetVConstPtr(VByteOrderMask)); - e.LDR(Q0, X0); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q0.B16()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } else { // signed -> signed assert_always(); @@ -1599,16 +1597,18 @@ struct PACK : Sequence> { e.LoadConstantV(src2, i.src2.constant()); } - e.UQXTN(i.dest.reg().toD().H4(), src1.S4()); - e.UQXTN2(i.dest.reg().H8(), src2.S4()); + e.UQXTN(i.dest.reg().toD().H4(), src2.S4()); + e.UQXTN2(i.dest.reg().H8(), src1.S4()); e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } else { // unsigned -> unsigned - e.XTN(i.dest.reg().toD().H4(), i.src1.reg().S4()); - e.XTN2(i.dest.reg().H8(), i.src2.reg().S4()); + e.XTN(i.dest.reg().toD().H4(), i.src2.reg().S4()); + e.XTN2(i.dest.reg().H8(), i.src1.reg().S4()); e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } } else { if (IsPackOutSaturate(flags)) { @@ -1623,10 +1623,11 @@ struct PACK : Sequence> { if (IsPackOutUnsigned(flags)) { if (IsPackOutSaturate(flags)) { // signed -> unsigned + saturate - e.UQXTN(i.dest.reg().toD().H4(), i.src1.reg().S4()); - e.UQXTN2(i.dest.reg().H8(), i.src2.reg().S4()); + e.SQXTUN(i.dest.reg().toD().H4(), i.src2.reg().S4()); + e.SQXTUN2(i.dest.reg().H8(), i.src1.reg().S4()); e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } else { // signed -> unsigned assert_always(); @@ -1643,10 +1644,11 @@ struct PACK : Sequence> { if (i.src2.is_constant) { e.LoadConstantV(src2, i.src2.constant()); } - e.SQXTN(i.dest.reg().toD().H4(), src1.S4()); - e.SQXTN2(i.dest.reg().H8(), src2.S4()); + e.SQXTN(i.dest.reg().toD().H4(), src2.S4()); + e.SQXTN2(i.dest.reg().H8(), src1.S4()); e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); } else { // signed -> signed assert_always(); From f9495dd96e8c206539fcd8c853e48ff19edca25d Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 23 May 2024 15:44:19 -0700 Subject: [PATCH 106/179] [a64] Implement `OPCODE_UNPACK` This is a very literal translation from the x64 code into ARM and may not be very optimized. Passes unit test save for a couple off-by-one errors. --- a64_seq_vector.cc | 370 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 362 insertions(+), 8 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index a816e4210..fa44e511b 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1697,17 +1697,371 @@ struct UNPACK : Sequence> { break; } } - static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) {} - static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) {} - static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) {} - static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) {} - static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) {} - static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) {} - static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) {} + static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { + // ARGB (WXYZ) -> RGBA (XYZW) + QReg src(0); + if (i.src1.is_constant) { + if (i.src1.value->IsConstantZero()) { + e.MOVP2R(X0, e.GetVConstPtr(VOne)); + e.LDR(i.dest.reg(), X0); + return; + } + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + // src = ZZYYXXWW + // Unpack to 000000ZZ,000000YY,000000XX,000000WW + e.MOVP2R(X0, e.GetVConstPtr(VUnpackD3DCOLOR)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + // Add 1.0f to each. + e.MOVP2R(X0, e.GetVConstPtr(VOne)); + e.LDR(Q1, X0); + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); + // To convert to 0 to 1, games multiply by 0x47008081 and add 0xC7008081. + } + static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { + alignas(16) uint16_t a[4]; + alignas(16) float b[8]; + vst1q_u8(a, vld1q_u8(src1)); + std::memset(b, 0, sizeof(b)); + + for (int i = 0; i < 2; i++) { + b[i] = half_float::detail::half2float(a[VEC128_W(6 + i)]); + } + + // Constants, or something + b[2] = 0.f; + b[3] = 1.f; + + return vld1q_u8(b); + } + static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { + // 1 bit sign, 5 bit exponent, 10 bit mantissa + // D3D10 half float format + // TODO(wunkolo): FP16 + FCVTL + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + } + e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_2)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static uint8x16_t EmulateFLOAT16_4(void*, std::byte src1[16]) { + alignas(16) uint16_t a[4]; + alignas(16) float b[8]; + vst1q_u8(a, vld1q_u8(src1)); + + for (int i = 0; i < 4; i++) { + b[i] = half_float::detail::half2float(a[VEC128_W(4 + i)]); + } + + return vld1q_u8(b); + } + static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { + // src = [(dest.x | dest.y), (dest.z | dest.w), 0, 0] + // TODO(wunkolo): FP16 + FCVTN + + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + } + e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_4)); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { + // (VD.x) = 3.0 + (VB.x>>16)*2^-22 + // (VD.y) = 3.0 + (VB.x)*2^-22 + // (VD.z) = 0.0 + // (VD.w) = 1.0 (games splat W after unpacking to get vectors of 1.0f) + // src is (xx,xx,xx,VALUE) + QReg src(0); + if (i.src1.is_constant) { + if (i.src1.value->IsConstantZero()) { + src = i.dest; + e.MOVP2R(X0, e.GetVConstPtr(V3301)); + e.LDR(i.dest, X0); + return; + } + // TODO(benvanik): check other common constants/perform shuffle/or here. + src = i.src1; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + // Shuffle bytes. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_2)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + + // If negative, make smaller than 3 - sign extend before adding. + e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 16); + e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 16); + + // Add 3,3,0,1. + e.MOVP2R(X0, e.GetVConstPtr(V3301)); + e.LDR(Q1, X0); + e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Return quiet NaNs in case of negative overflow. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_Overflow)); + e.LDR(Q1, X0); + e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); + e.LDR(Q1, X0); + e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { + // (VD.x) = 3.0 + (VB.x>>16)*2^-22 + // (VD.y) = 3.0 + (VB.x)*2^-22 + // (VD.z) = 3.0 + (VB.y>>16)*2^-22 + // (VD.w) = 3.0 + (VB.y)*2^-22 + // src is (xx,xx,VALUE,VALUE) + QReg src(0); + if (i.src1.is_constant) { + if (i.src1.value->IsConstantZero()) { + e.MOVP2R(X0, e.GetVConstPtr(V3333)); + e.LDR(i.dest, X0); + return; + } + // TODO(benvanik): check other common constants/perform shuffle/or here. + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + // Shuffle bytes. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_4)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + + // If negative, make smaller than 3 - sign extend before adding. + e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 16); + e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 16); + + // Add 3,3,3,3. + e.MOVP2R(X0, e.GetVConstPtr(V3333)); + e.LDR(Q1, X0); + e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Return quiet NaNs in case of negative overflow. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_Overflow)); + e.LDR(Q1, X0); + e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); + e.LDR(Q1, X0); + e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } + static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { + QReg src(0); + if (i.src1.is_constant) { + if (i.src1.value->IsConstantZero()) { + e.MOVP2R(X0, e.GetVConstPtr(V3331)); + e.LDR(i.dest, X0); + return; + } + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + + // Splat W. + e.DUP(i.dest.reg().S4(), src.Selem()[3]); + // Keep only the needed components. + // Red in 0-9 now, green in 10-19, blue in 20-29, alpha in 30-31. + e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MaskPacked)); + e.LDR(Q1, X0); + e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); + + // Shift the components down. + e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_Shift)); + e.LDR(Q1, X0); + e.NEG(Q1.S4(), Q1.S4()); + e.USHL(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + // If XYZ are negative, make smaller than 3 - sign extend XYZ before adding. + // W is unsigned. + e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 22); + e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 22); + // Add 3,3,3,1. + e.MOVP2R(X0, e.GetVConstPtr(V3331)); + e.LDR(Q1, X0); + e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + // Return quiet NaNs in case of negative overflow. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackUINT_2101010_Overflow)); + e.LDR(Q1, X0); + e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); + e.LDR(Q1, X0); + e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); + e.MOV(i.dest.reg().B16(), Q0.B16()); + // To convert XYZ to -1 to 1, games multiply by 0x46004020 & sub 0x46C06030. + // For W to 0 to 1, they multiply by and subtract 0x4A2AAAAB.} + } + static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { + QReg src(0); + if (i.src1.is_constant) { + if (i.src1.value->IsConstantZero()) { + e.MOVP2R(X0, e.GetVConstPtr(V3331)); + e.LDR(i.dest, X0); + return; + } + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + // Extract pairs of nibbles to XZYW. XZ will have excess 4 upper bits, YW + // will have excess 4 lower bits. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackULONG_4202020_Permute)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + + // Drop the excess nibble of YW. + e.USHR(Q0.S4(), i.dest.reg().S4(), 4); + // Merge XZ and YW now both starting at offset 0. + e.LoadConstantV(Q1, vec128i(3 * 0x04'04'04'04 + 0x03'02'01'00, + 2 * 0x04'04'04'04 + 0x03'02'01'00, + 1 * 0x04'04'04'04 + 0x03'02'01'00, + 0 * 0x04'04'04'04 + 0x03'02'01'00)); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16(), Q0.B16()}, + Q1.B16()); + + // Reorder as XYZW. + e.LoadConstantV(Q1, vec128i(3 * 0x04'04'04'04 + 0x03'02'01'00, + 1 * 0x04'04'04'04 + 0x03'02'01'00, + 2 * 0x04'04'04'04 + 0x03'02'01'00, + 0 * 0x04'04'04'04 + 0x03'02'01'00)); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16(), Q0.B16()}, + Q1.B16()); + // Drop the excess upper nibble in XZ and sign-extend XYZ. + e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 12); + e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 12); + // Add 3,3,3,1. + e.MOVP2R(X0, e.GetVConstPtr(V3331)); + e.LDR(Q1, X0); + e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + // Return quiet NaNs in case of negative overflow. + e.MOVP2R(X0, e.GetVConstPtr(VUnpackULONG_4202020_Overflow)); + e.LDR(Q1, X0); + e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); + e.LDR(Q1, X0); + e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); + e.MOV(i.dest.reg().B16(), Q0.B16()); + } static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { + assert_false(IsPackOutSaturate(flags)); + QReg src(0); + if (i.src1.is_constant) { + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + if (IsPackToLo(flags)) { + // Unpack to LO. + if (IsPackInUnsigned(flags)) { + if (IsPackOutUnsigned(flags)) { + // unsigned -> unsigned + assert_always(); + } else { + // unsigned -> signed + assert_always(); + } + } else { + if (IsPackOutUnsigned(flags)) { + // signed -> unsigned + assert_always(); + } else { + // signed -> signed + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.SXTL2(i.dest.reg().H8(), i.dest.reg().B16()); + } + } + } else { + // Unpack to HI. + if (IsPackInUnsigned(flags)) { + if (IsPackOutUnsigned(flags)) { + // unsigned -> unsigned + assert_always(); + } else { + // unsigned -> signed + assert_always(); + } + } else { + if (IsPackOutUnsigned(flags)) { + // signed -> unsigned + assert_always(); + } else { + // signed -> signed + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.SXTL(i.dest.reg().H8(), i.dest.reg().toD().B8()); + } + } + } } static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, - uint32_t flags) {} + uint32_t flags) { + assert_false(IsPackOutSaturate(flags)); + QReg src(0); + if (i.src1.is_constant) { + src = i.dest; + e.LoadConstantV(src, i.src1.constant()); + } else { + src = i.src1; + } + if (IsPackToLo(flags)) { + // Unpack to LO. + if (IsPackInUnsigned(flags)) { + if (IsPackOutUnsigned(flags)) { + // unsigned -> unsigned + assert_always(); + } else { + // unsigned -> signed + assert_always(); + } + } else { + if (IsPackOutUnsigned(flags)) { + // signed -> unsigned + assert_always(); + } else { + // signed -> signed + e.SXTL2(i.dest.reg().S4(), src.H8()); + } + } + } else { + // Unpack to HI. + if (IsPackInUnsigned(flags)) { + if (IsPackOutUnsigned(flags)) { + // unsigned -> unsigned + assert_always(); + } else { + // unsigned -> signed + assert_always(); + } + } else { + if (IsPackOutUnsigned(flags)) { + // signed -> unsigned + assert_always(); + } else { + // signed -> signed + e.SXTL(i.dest.reg().S4(), src.toD().H4()); + } + } + } + e.REV64(i.dest.reg().S4(), i.dest.reg().S4()); + } }; EMITTER_OPCODE_TABLE(OPCODE_UNPACK, UNPACK); From 8db83acdf43d9a322ef7653a5cdbb19941ad49a0 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 25 May 2024 13:46:18 -0700 Subject: [PATCH 107/179] [a64] Implement `LSE` and `FP16C` detection Adds two new flags for allowing the use of LSE and FP16C --- a64_emitter.cc | 21 ++++++++++++++++++++- a64_emitter.h | 5 ++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 559f66f3f..8c4482200 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -38,6 +38,7 @@ #include "oaknut/feature_detection/cpu_feature.hpp" #include "oaknut/feature_detection/feature_detection.hpp" +#include "oaknut/feature_detection/feature_detection_idregs.hpp" DEFINE_bool(debugprint_trap_log, false, "Log debugprint traps to the active debugger", "CPU"); @@ -77,12 +78,30 @@ A64Emitter::A64Emitter(A64Backend* backend) processor_(backend->processor()), backend_(backend), code_cache_(backend->code_cache()) { - const oaknut::CpuFeatures cpu_ = oaknut::detect_features(); + oaknut::CpuFeatures cpu_ = oaknut::detect_features(); + + // Combine with id register detection +#if OAKNUT_SUPPORTS_READING_ID_REGISTERS > 0 +#if OAKNUT_SUPPORTS_READING_ID_REGISTERS == 1 + const std::optional id_registers = + oaknut::read_id_registers(); +#elif OAKNUT_SUPPORTS_READING_ID_REGISTERS == 2 + const std::optional id_registers = + oaknut::read_id_registers(0); +#endif + if (id_registers.has_value()) { + cpu_ = cpu_ | oaknut::detect_features_via_id_registers(*id_registers); + } +#endif + #define TEST_EMIT_FEATURE(emit, ext) \ if ((cvars::a64_extension_mask & emit) == emit) { \ feature_flags_ |= (cpu_.has(ext) ? emit : 0); \ } + TEST_EMIT_FEATURE(kA64EmitLSE, oaknut::CpuFeature::LSE); + TEST_EMIT_FEATURE(kA64EmitF16C, oaknut::CpuFeature::FP16Conv); + #undef TEST_EMIT_FEATURE } diff --git a/a64_emitter.h b/a64_emitter.h index 60641fdba..463064ef1 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -118,7 +118,10 @@ enum VConst { V2To32, }; -enum A64EmitterFeatureFlags {}; +enum A64EmitterFeatureFlags { + kA64EmitLSE = 1 << 0, + kA64EmitF16C = 1 << 1, +}; class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { public: From cd9679c953cd914b6f91914921a34255935f82e7 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 25 May 2024 13:57:52 -0700 Subject: [PATCH 108/179] [a64] Optimize `OPCODE_{UN}PACK`(float16) with `F16C` --- a64_seq_vector.cc | 57 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index fa44e511b..6ade08eb2 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1368,7 +1368,18 @@ struct PACK : Sequence> { assert_true(i.src2.value->IsConstantZero()); // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx // dest = [(src1.x | src1.y), 0, 0, 0] - // TODO(wunkolo): FP16 + FCVTN + + if (e.IsFeatureEnabled(kA64EmitF16C)) { + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + e.FCVTN(i.dest.reg().toD().H4(), src1.S4()); + e.MOVI(Q0.B16(), 0); + e.EXT(i.dest.reg().B16(), Q0.B16(), i.dest.reg().B16(), 4); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + return; + } if (i.src1.is_constant) { e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); @@ -1395,7 +1406,17 @@ struct PACK : Sequence> { assert_true(i.src2.value->IsConstantZero()); // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx // dest = [(src1.z | src1.w), (src1.x | src1.y), 0, 0] - // TODO(wunkolo): FP16 + FCVTN + + if (e.IsFeatureEnabled(kA64EmitF16C)) { + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + e.FCVTN(i.dest.reg().toD().H4(), src1.S4()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + return; + } if (i.src1.is_constant) { e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); @@ -1741,7 +1762,26 @@ struct UNPACK : Sequence> { static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { // 1 bit sign, 5 bit exponent, 10 bit mantissa // D3D10 half float format - // TODO(wunkolo): FP16 + FCVTL + + if (e.IsFeatureEnabled(kA64EmitF16C)) { + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + + // Move the upper 4 bytes to the lower 4 bytes, zero the rest + e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16(), 12); + + e.FCVTL(i.dest.reg().S4(), i.dest.reg().toD().H4()); + e.REV64(i.dest.reg().S4(), i.dest.reg().S4()); + + // Write 1.0 to element 3 + e.FMOV(S0, oaknut::FImm8(0, 7, 0)); + e.MOV(i.dest.reg().Selem()[3], Q0.Selem()[0]); + return; + } + if (i.src1.is_constant) { e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); } else { @@ -1763,7 +1803,16 @@ struct UNPACK : Sequence> { } static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { // src = [(dest.x | dest.y), (dest.z | dest.w), 0, 0] - // TODO(wunkolo): FP16 + FCVTN + if (e.IsFeatureEnabled(kA64EmitF16C)) { + const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.src1.reg().B16(), 8); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + e.FCVTL(i.dest.reg().S4(), i.dest.reg().toD().H4()); + return; + } if (i.src1.is_constant) { e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); From 52d45b53521a575399e453c19d18650b3aba73f5 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 25 May 2024 14:31:45 -0700 Subject: [PATCH 109/179] [a64] Fix `OPCODE_PACK`(short) Narrow-saturation instructions causes off-by-one rounding errors. Using the min+max+shuffle passes more unit tests --- a64_seq_vector.cc | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 6ade08eb2..c92312fed 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1427,24 +1427,46 @@ struct PACK : Sequence> { e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); QReg src = i.src1; if (i.src1.is_constant) { src = i.dest; e.LoadConstantV(src, i.src1.constant()); } - e.SQSHRN(i.dest.reg().toD().H4(), src.S4(), 8); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 4); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + // Saturate + e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Min)); + e.LDR(Q1, X0); + e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Max)); + e.LDR(Q1, X0); + e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Pack + e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_2)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); } static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); QReg src = i.src1; if (i.src1.is_constant) { src = i.dest; e.LoadConstantV(src, i.src1.constant()); } - e.SQSHRN(i.dest.reg().toD().H4(), src.S4(), 8); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 4); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); + // Saturate + e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Min)); + e.LDR(Q1, X0); + e.FMAXNM(i.dest.reg().S4(), src.S4(), Q1.S4()); + + e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Max)); + e.LDR(Q1, X0); + e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Pack + e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_4)); + e.LDR(Q1, X0); + e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); } static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { // https://www.opengl.org/registry/specs/ARB/vertex_type_2_10_10_10_rev.txt From ec1413fd8204f08a9cbb0d8eeda92034467614ad Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 25 May 2024 15:29:28 -0700 Subject: [PATCH 110/179] [a64] Optimize bulk VConst access with relative addressing Load the pointer to the VConst table once, and use offsets from this base address from the underlying enum value. Reduces the amount of instructions for each VConst memory load. --- a64_emitter.cc | 9 +- a64_emitter.h | 10 ++- a64_seq_vector.cc | 224 ++++++++++++++++++++++------------------------ 3 files changed, 120 insertions(+), 123 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 8c4482200..a3cccc231 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -837,11 +837,14 @@ void A64Emitter::FreeConstData(uintptr_t data) { memory::DeallocationType::kRelease); } -std::byte* A64Emitter::GetVConstPtr(VConst id) { +std::byte* A64Emitter::GetVConstPtr() const { + return reinterpret_cast(backend_->emitter_data()); +} + +std::byte* A64Emitter::GetVConstPtr(VConst id) const { // Load through fixed constant table setup by PlaceConstData. // It's important that the pointer is not signed, as it will be sign-extended. - return reinterpret_cast(backend_->emitter_data() + - sizeof(vec128_t) * id); + return GetVConstPtr() + GetVConstOffset(id); } // Implies possible StashV(0, ...)! diff --git a/a64_emitter.h b/a64_emitter.h index 463064ef1..3e0b35f36 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -119,8 +119,8 @@ enum VConst { }; enum A64EmitterFeatureFlags { - kA64EmitLSE = 1 << 0, - kA64EmitF16C = 1 << 1, + kA64EmitLSE = 1 << 0, + kA64EmitF16C = 1 << 1, }; class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { @@ -204,7 +204,11 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { bool ConstantFitsIn32Reg(uint64_t v); void MovMem64(const oaknut::XRegSp& addr, intptr_t offset, uint64_t v); - std::byte* GetVConstPtr(VConst id); + std::byte* GetVConstPtr() const; + std::byte* GetVConstPtr(VConst id) const; + constexpr uintptr_t GetVConstOffset(VConst id) const { + return sizeof(vec128_t) * id; + } void LoadConstantV(oaknut::QReg dest, float v); void LoadConstantV(oaknut::QReg dest, double v); void LoadConstantV(oaknut::QReg dest, const vec128_t& v); diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index c92312fed..4f2b3bd95 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -542,11 +542,11 @@ struct VECTOR_SHL_V128 e.SHL(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -566,11 +566,11 @@ struct VECTOR_SHL_V128 e.SHL(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u8[0]); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -590,11 +590,11 @@ struct VECTOR_SHL_V128 e.SHL(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u8[0]); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -655,11 +655,11 @@ struct VECTOR_SHR_V128 e.USHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -679,11 +679,11 @@ struct VECTOR_SHR_V128 e.USHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0]); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -703,11 +703,11 @@ struct VECTOR_SHR_V128 e.USHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0]); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -751,11 +751,11 @@ struct VECTOR_SHA_V128 e.SSHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -775,11 +775,11 @@ struct VECTOR_SHA_V128 e.SSHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0] & 0xF); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -799,11 +799,11 @@ struct VECTOR_SHA_V128 e.SSHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0] & 0x1F); return; } - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -835,11 +835,11 @@ struct VECTOR_ROTATE_LEFT_V128 I> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src2.is_constant) { - e.ADD(e.GetNativeParam(1), XSP, e.StashConstantV(1, i.src2.constant())); + e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { - e.ADD(e.GetNativeParam(1), XSP, e.StashV(1, i.src2)); + e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); switch (i.instr->flags) { case INT8_TYPE: e.CallNativeSafe( @@ -1333,23 +1333,24 @@ struct PACK : Sequence> { src = i.dest; e.LoadConstantV(src, i.src1.constant()); } + + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + // Saturate to [3,3....] so that only values between 3...[00] and 3...[FF] // are valid - max before min to pack NaN as zero (5454082B is heavily // affected by the order - packs 0xFFFFFFFF in matrix code to get a 0 // constant). - e.MOVP2R(X0, e.GetVConstPtr(V3333)); - e.LDR(Q0, X0); + e.LDR(Q0, VConstData, e.GetVConstOffset(V3333)); e.FMAX(i.dest.reg().S4(), i.dest.reg().S4(), Q0.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VPackD3DCOLORSat)); - e.LDR(Q0, X0); + e.LDR(Q0, VConstData, e.GetVConstOffset(VPackD3DCOLORSat)); e.FMIN(i.dest.reg().S4(), src.S4(), Q0.S4()); // Extract bytes. // RGBA (XYZW) -> ARGB (WXYZ) // w = ((src1.uw & 0xFF) << 24) | ((src1.ux & 0xFF) << 16) | // ((src1.uy & 0xFF) << 8) | (src1.uz & 0xFF) - e.MOVP2R(X0, e.GetVConstPtr(VPackD3DCOLOR)); - e.LDR(Q0, X0); + e.LDR(Q0, VConstData, e.GetVConstOffset(VPackD3DCOLOR)); e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q0.B16()); } static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { @@ -1433,18 +1434,18 @@ struct PACK : Sequence> { src = i.dest; e.LoadConstantV(src, i.src1.constant()); } + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + // Saturate - e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Min)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Max)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Max)); e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Pack - e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_2)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_2)); e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); } static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { @@ -1454,18 +1455,18 @@ struct PACK : Sequence> { src = i.dest; e.LoadConstantV(src, i.src1.constant()); } - // Saturate - e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Min)); - e.LDR(Q1, X0); - e.FMAXNM(i.dest.reg().S4(), src.S4(), Q1.S4()); + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); - e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_Max)); - e.LDR(Q1, X0); - e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + // Saturate + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); + e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); + + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Max)); + e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Pack - e.MOVP2R(X0, e.GetVConstPtr(VPackSHORT_4)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_4)); e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); } static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { @@ -1476,24 +1477,22 @@ struct PACK : Sequence> { if (i.src1.is_constant) { e.LoadConstantV(src, i.src1.constant()); } + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); // Saturate. - e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MinUnpacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MinUnpacked)); e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MaxUnpacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MaxUnpacked)); e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Remove the unneeded bits of the floats. - e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MaskUnpacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MaskUnpacked)); e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); // Shift the components up. - e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_Shift)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_Shift)); e.USHL(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Combine the components. @@ -1519,31 +1518,29 @@ struct PACK : Sequence> { src = i.dest; e.LoadConstantV(src, i.src1.constant()); } + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + // Saturate. - e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_MinUnpacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MinUnpacked)); e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_MaxUnpacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MaxUnpacked)); e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Remove the unneeded bits of the floats (so excess nibbles will also be // cleared). - e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_MaskUnpacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MaskUnpacked)); e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); // Store Y and W shifted left by 4 so vpshufb can be used with them. e.SHL(Q0.S4(), i.dest.reg().S4(), 4); // Place XZ where they're supposed to be. - e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_PermuteXZ)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_PermuteXZ)); e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); // Place YW. - e.MOVP2R(X0, e.GetVConstPtr(VPackULONG_4202020_PermuteYW)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_PermuteYW)); e.TBL(Q0.B16(), oaknut::List{Q0.B16()}, Q1.B16()); // Merge XZ and YW. e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); @@ -1742,11 +1739,14 @@ struct UNPACK : Sequence> { } static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { // ARGB (WXYZ) -> RGBA (XYZW) + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + QReg src(0); + if (i.src1.is_constant) { if (i.src1.value->IsConstantZero()) { - e.MOVP2R(X0, e.GetVConstPtr(VOne)); - e.LDR(i.dest.reg(), X0); + e.LDR(i.dest.reg(), VConstData, e.GetVConstOffset(VOne)); return; } src = i.dest; @@ -1756,12 +1756,10 @@ struct UNPACK : Sequence> { } // src = ZZYYXXWW // Unpack to 000000ZZ,000000YY,000000XX,000000WW - e.MOVP2R(X0, e.GetVConstPtr(VUnpackD3DCOLOR)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackD3DCOLOR)); e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); // Add 1.0f to each. - e.MOVP2R(X0, e.GetVConstPtr(VOne)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VOne)); e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); // To convert to 0 to 1, games multiply by 0x47008081 and add 0xC7008081. } @@ -1850,12 +1848,14 @@ struct UNPACK : Sequence> { // (VD.z) = 0.0 // (VD.w) = 1.0 (games splat W after unpacking to get vectors of 1.0f) // src is (xx,xx,xx,VALUE) + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + QReg src(0); if (i.src1.is_constant) { if (i.src1.value->IsConstantZero()) { src = i.dest; - e.MOVP2R(X0, e.GetVConstPtr(V3301)); - e.LDR(i.dest, X0); + e.LDR(i.dest, VConstData, e.GetVConstOffset(V3301)); return; } // TODO(benvanik): check other common constants/perform shuffle/or here. @@ -1865,8 +1865,7 @@ struct UNPACK : Sequence> { src = i.src1; } // Shuffle bytes. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_2)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_2)); e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); // If negative, make smaller than 3 - sign extend before adding. @@ -1874,17 +1873,14 @@ struct UNPACK : Sequence> { e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 16); // Add 3,3,0,1. - e.MOVP2R(X0, e.GetVConstPtr(V3301)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(V3301)); e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Return quiet NaNs in case of negative overflow. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_Overflow)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_Overflow)); e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -1894,11 +1890,14 @@ struct UNPACK : Sequence> { // (VD.z) = 3.0 + (VB.y>>16)*2^-22 // (VD.w) = 3.0 + (VB.y)*2^-22 // src is (xx,xx,VALUE,VALUE) + + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + QReg src(0); if (i.src1.is_constant) { if (i.src1.value->IsConstantZero()) { - e.MOVP2R(X0, e.GetVConstPtr(V3333)); - e.LDR(i.dest, X0); + e.LDR(i.dest, VConstData, e.GetVConstOffset(V3333)); return; } // TODO(benvanik): check other common constants/perform shuffle/or here. @@ -1908,8 +1907,7 @@ struct UNPACK : Sequence> { src = i.src1; } // Shuffle bytes. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_4)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_4)); e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); // If negative, make smaller than 3 - sign extend before adding. @@ -1917,26 +1915,25 @@ struct UNPACK : Sequence> { e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 16); // Add 3,3,3,3. - e.MOVP2R(X0, e.GetVConstPtr(V3333)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(V3333)); e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Return quiet NaNs in case of negative overflow. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackSHORT_Overflow)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_Overflow)); e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + QReg src(0); if (i.src1.is_constant) { if (i.src1.value->IsConstantZero()) { - e.MOVP2R(X0, e.GetVConstPtr(V3331)); - e.LDR(i.dest, X0); + e.LDR(i.dest, VConstData, e.GetVConstOffset(V3331)); return; } src = i.dest; @@ -1949,13 +1946,11 @@ struct UNPACK : Sequence> { e.DUP(i.dest.reg().S4(), src.Selem()[3]); // Keep only the needed components. // Red in 0-9 now, green in 10-19, blue in 20-29, alpha in 30-31. - e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_MaskPacked)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MaskPacked)); e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); // Shift the components down. - e.MOVP2R(X0, e.GetVConstPtr(VPackUINT_2101010_Shift)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_Shift)); e.NEG(Q1.S4(), Q1.S4()); e.USHL(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // If XYZ are negative, make smaller than 3 - sign extend XYZ before adding. @@ -1963,27 +1958,26 @@ struct UNPACK : Sequence> { e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 22); e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 22); // Add 3,3,3,1. - e.MOVP2R(X0, e.GetVConstPtr(V3331)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(V3331)); e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Return quiet NaNs in case of negative overflow. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackUINT_2101010_Overflow)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackUINT_2101010_Overflow)); e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); e.MOV(i.dest.reg().B16(), Q0.B16()); // To convert XYZ to -1 to 1, games multiply by 0x46004020 & sub 0x46C06030. // For W to 0 to 1, they multiply by and subtract 0x4A2AAAAB.} } static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { + const XReg VConstData = X3; + e.MOVP2R(VConstData, e.GetVConstPtr()); + QReg src(0); if (i.src1.is_constant) { if (i.src1.value->IsConstantZero()) { - e.MOVP2R(X0, e.GetVConstPtr(V3331)); - e.LDR(i.dest, X0); + e.LDR(i.dest, VConstData, e.GetVConstOffset(V3331)); return; } src = i.dest; @@ -1993,8 +1987,7 @@ struct UNPACK : Sequence> { } // Extract pairs of nibbles to XZYW. XZ will have excess 4 upper bits, YW // will have excess 4 lower bits. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackULONG_4202020_Permute)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackULONG_4202020_Permute)); e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); // Drop the excess nibble of YW. @@ -2018,16 +2011,13 @@ struct UNPACK : Sequence> { e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 12); e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 12); // Add 3,3,3,1. - e.MOVP2R(X0, e.GetVConstPtr(V3331)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(V3331)); e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); // Return quiet NaNs in case of negative overflow. - e.MOVP2R(X0, e.GetVConstPtr(VUnpackULONG_4202020_Overflow)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackULONG_4202020_Overflow)); e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - e.MOVP2R(X0, e.GetVConstPtr(VQNaN)); - e.LDR(Q1, X0); + e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); e.MOV(i.dest.reg().B16(), Q0.B16()); } From e4bef5e4b81637e8652368bc7157e1255855433b Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 25 May 2024 15:47:18 -0700 Subject: [PATCH 111/179] [a64] Optimize constant vector byte-splats Detect when all bytes are repeating and use `MOVI` when applicable --- a64_emitter.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/a64_emitter.cc b/a64_emitter.cc index a3cccc231..8913da000 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -857,6 +857,10 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { } else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) { // 1111... MOVI(dest.B16(), 0xFF); + } else if (std::adjacent_find(std::cbegin(v.u8), std::cend(v.u8), + std::not_equal_to<>()) == std::cend(v.u8)) { + // 0xXX, 0xXX, 0xXX... + MOVI(dest.B16(), v.u8[0]); } else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. From a5c1f79024f7770f2067f16f8b2d940cb41330c3 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 26 May 2024 22:24:03 -0700 Subject: [PATCH 112/179] [a64] Fix `OPCODE_SWIZZLE` register-aliasing Indices and non-const tables were using the same scratch-register --- a64_seq_vector.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 4f2b3bd95..2ba9bd2ea 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1268,7 +1268,7 @@ struct SWIZZLE ((swizzle_mask >> 4) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, ((swizzle_mask >> 6) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00); - const QReg indices = Q0; + const QReg indices = Q1; e.LoadConstantV(indices, indice_vec); QReg table0 = Q0; From 3ca54a717978dc7c29a8aaef67f9d9223c0c3ae8 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 27 May 2024 13:13:17 -0700 Subject: [PATCH 113/179] [a64] Remove `VOne` constant in favor of `FMOV` --- a64_emitter.cc | 1 - a64_emitter.h | 5 ++--- a64_seq_vector.cc | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 8913da000..58ab1e20b 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -689,7 +689,6 @@ void A64Emitter::MovMem64(const oaknut::XRegSp& addr, intptr_t offset, static const vec128_t v_consts[] = { /* VZero */ vec128f(0.0f), - /* VOne */ vec128f(1.0f), /* VOnePD */ vec128d(1.0), /* VNegativeOne */ vec128f(-1.0f, -1.0f, -1.0f, -1.0f), /* VFFFF */ diff --git a/a64_emitter.h b/a64_emitter.h index 3e0b35f36..fef334dce 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -47,7 +47,6 @@ enum RegisterFlags { enum VConst { VZero = 0, - VOne, VOnePD, VNegativeOne, VFFFF, @@ -201,12 +200,12 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { void ReloadMembase(); // Moves a 64bit immediate into memory. - bool ConstantFitsIn32Reg(uint64_t v); + static bool ConstantFitsIn32Reg(uint64_t v); void MovMem64(const oaknut::XRegSp& addr, intptr_t offset, uint64_t v); std::byte* GetVConstPtr() const; std::byte* GetVConstPtr(VConst id) const; - constexpr uintptr_t GetVConstOffset(VConst id) const { + static constexpr uintptr_t GetVConstOffset(VConst id){ return sizeof(vec128_t) * id; } void LoadConstantV(oaknut::QReg dest, float v); diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 2ba9bd2ea..71d608f82 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1746,7 +1746,7 @@ struct UNPACK : Sequence> { if (i.src1.is_constant) { if (i.src1.value->IsConstantZero()) { - e.LDR(i.dest.reg(), VConstData, e.GetVConstOffset(VOne)); + e.FMOV(i.dest.reg().S4(), FImm8(0, 7, 0)); return; } src = i.dest; @@ -1759,7 +1759,7 @@ struct UNPACK : Sequence> { e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackD3DCOLOR)); e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); // Add 1.0f to each. - e.LDR(Q1, VConstData, e.GetVConstOffset(VOne)); + e.FMOV(Q1.S4(), FImm8(0, 7, 0)); e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); // To convert to 0 to 1, games multiply by 0x47008081 and add 0xC7008081. } From 977b2c0250459f7e90070e0c8447d210e6222df7 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 27 May 2024 13:16:32 -0700 Subject: [PATCH 114/179] [a64] Add arch-agnostic documentation configurations Misses some during the first pass. Now the config files with mention a64 differences. --- a64_backend.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/a64_backend.cc b/a64_backend.cc index 3955598dc..8b3f3a6f7 100644 --- a/a64_backend.cc +++ b/a64_backend.cc @@ -29,7 +29,9 @@ DEFINE_int32(a64_extension_mask, -1, "Allow the detection and utilization of specific instruction set " "features.\n" - " 0 = arm64v8\n" + " 0 = armv8.0\n" + " 1 = LSE\n" + " 2 = F16C\n" " -1 = Detect and utilize all possible processor features\n", "a64"); From 356b6f2a1e71f667daf5f745c8c77828f30f9b1e Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 27 May 2024 13:17:44 -0700 Subject: [PATCH 115/179] [a64] Optimize zero MovMem64 Read direction from the ZR in the case that we are just storing a 64 or 32 bit zero --- a64_emitter.cc | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 58ab1e20b..6b6182fbd 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -645,7 +645,6 @@ oaknut::XReg A64Emitter::GetContextReg() { return X27; } oaknut::XReg A64Emitter::GetMembaseReg() { return X28; } void A64Emitter::ReloadContext() { - // mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]); LDR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); } @@ -667,20 +666,13 @@ bool A64Emitter::ConstantFitsIn32Reg(uint64_t v) { void A64Emitter::MovMem64(const oaknut::XRegSp& addr, intptr_t offset, uint64_t v) { - // if ((v & ~0x7FFFFFFF) == 0) { - // // Fits under 31 bits, so just load using normal mov. - // mov(qword[addr], v); - // } else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) { - // // Negative number that fits in 32bits. - // mov(qword[addr], v); - // } else if (!(v >> 32)) { - // // All high bits are zero. It'd be nice if we had a way to load a 32bit - // // immediate without sign extending! - // // TODO(benvanik): this is super common, find a better way. - // mov(dword[addr], static_cast(v)); - // mov(dword[addr + 4], 0); - // } else - { + if (v == 0) { + STR(XZR, addr, offset); + } else if (!(v >> 32)) { + // All high bits are zero, 32-bit MOV + MOV(W0, static_cast(v)); + STR(X0, addr, offset); + } else { // 64bit number that needs double movs. MOV(X0, v); STR(X0, addr, offset); From 3a11508aa3359a2c1369f3a7161083e90e30926c Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 27 May 2024 13:29:19 -0700 Subject: [PATCH 116/179] [a64] Implement `OPCODE_DID_SATURATE` This directly maps to the QC bit in the FPSR. Just have to make sure that the saturated instruction is the very last instruction(which is currently the case for stuff like VECTOR_ADD and such). --- a64_sequences.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index 94cb8fcd4..c3c0741e8 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -1107,8 +1107,9 @@ EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(UGE, Cond::HS); // setae struct DID_SATURATE : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): implement saturation check (VECTOR_ADD, etc). - e.EOR(i.dest, i.dest, i.dest); + // Bit 27 in the FPSR is the QC bit + e.MRS(X0, SystemReg::FPSR); + e.UBFX(i.dest, W0, 27, 1); } }; EMITTER_OPCODE_TABLE(OPCODE_DID_SATURATE, DID_SATURATE); From 0570445694eb1286f03a27cfa4e1cbdc8a944835 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 28 May 2024 14:39:10 -0700 Subject: [PATCH 117/179] [a64] Detect `MOVI` utilizations for vector-element splats(u8,u16,u32) The 64-bit cases uses a particular Replicated 8-bit immediate so something else will have to handle that This cases a lot of cases without having to touch memory. Does not catch cases of `1.0`(0x3f800000). --- a64_emitter.cc | 58 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 6b6182fbd..1c6fc7cf9 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -848,11 +848,61 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { } else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) { // 1111... MOVI(dest.B16(), 0xFF); - } else if (std::adjacent_find(std::cbegin(v.u8), std::cend(v.u8), - std::not_equal_to<>()) == std::cend(v.u8)) { - // 0xXX, 0xXX, 0xXX... - MOVI(dest.B16(), v.u8[0]); } else { + // Try to figure out some common splat-patterns to utilize MOVI rather than + // stashing to memory. + const bool all_same_u8 = + std::adjacent_find(std::cbegin(v.u8), std::cend(v.u8), + std::not_equal_to<>()) == std::cend(v.u8); + + if (all_same_u8) { + // 0xXX, 0xXX, 0xXX... + MOVI(dest.B16(), v.u8[0]); + return; + } + + const bool all_same_u16 = + std::adjacent_find(std::cbegin(v.u16), std::cend(v.u16), + std::not_equal_to<>()) == std::cend(v.u16); + + if (all_same_u16) { + if ((v.u16[0] & 0xFF00) == 0) { + // 0x00XX, 0x00XX, 0x00XX... + MOVI(dest.H8(), uint8_t(v.u16[0])); + return; + } else if ((v.u16[0] & 0x00FF) == 0) { + // 0xXX00, 0xXX00, 0xXX00... + MOVI(dest.H8(), uint8_t(v.u16[0] >> 8), oaknut::util::LSL, 8); + return; + } + } + + const bool all_same_u32 = + std::adjacent_find(std::cbegin(v.u32), std::cend(v.u32), + std::not_equal_to<>()) == std::cend(v.u32); + + if (all_same_u32) { + if ((v.u32[0] & 0x00FFFFFF) == 0) { + // This is used a lot for certain float-splats and should be checked + // first before the others + // 0xXX000000, 0xXX000000, 0xXX000000... + MOVI(dest.S4(), uint8_t(v.u32[0] >> 24), oaknut::util::LSL, 24); + return; + } else if ((v.u32[0] & 0xFFFFFF00) == 0) { + // 0x000000XX, 0x000000XX, 0x000000XX... + MOVI(dest.S4(), uint8_t(v.u32[0])); + return; + } else if ((v.u32[0] & 0xFFFF00FF) == 0) { + // 0x0000XX00, 0x0000XX00, 0x0000XX00... + MOVI(dest.S4(), uint8_t(v.u32[0] >> 8), oaknut::util::LSL, 8); + return; + } else if ((v.u32[0] & 0xFF00FFFF) == 0) { + // 0x00XX0000, 0x00XX0000, 0x00XX0000... + MOVI(dest.S4(), uint8_t(v.u32[0] >> 16), oaknut::util::LSL, 16); + return; + } + } + // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. MovMem64(SP, kStashOffset, v.low); From 184acc139ef4dfb6c53e9988572924827f6df4e8 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Tue, 28 May 2024 16:15:18 -0700 Subject: [PATCH 118/179] [a64] Optimize constant-loads with `FMOV` `FMOV` encodes an 8-bit floating point immediate that can be used to accelerate the loading of certain constant floating point values between -31.0 and 32.0. A lot of immediates such as -1.0, 1.0, 0.5, etc fall within this range and this code gets lots of hits in my testing. This is much more optimal than trying to load a 32/64-bit value in W0/X0 and moving it into an FP register. --- a64_emitter.cc | 95 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 1c6fc7cf9..0a83495d2 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -838,6 +838,74 @@ std::byte* A64Emitter::GetVConstPtr(VConst id) const { return GetVConstPtr() + GetVConstOffset(id); } +// Attempts to convert an fp32 bit-value into an fp8-immediate value for FMOV +// returns false if the value cannot be represented +// C2.2.3 Modified immediate constants in A64 floating-point instructions +// abcdefgh +// V +// aBbbbbbc defgh000 00000000 00000000 +// B = NOT(b) +static bool f32_to_fimm8(uint32_t u32, oaknut::FImm8& fp8) { + const uint32_t sign = (u32 >> 31) & 1; + int32_t exp = ((u32 >> 23) & 0xff) - 127; + int64_t mantissa = u32 & 0x7fffff; + + // Too many mantissa bits + if (mantissa & 0x7ffff) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 19; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + // exp = (NOT(b):c:d) - 3 + exp = ((exp + 3) & 0b111) ^ 0b100; + + fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); + return true; +} + +// Attempts to convert an fp64 bit-value into an fp8-immediate value for FMOV +// returns false if the value cannot be represented +// C2.2.3 Modified immediate constants in A64 floating-point instructions +// abcdefgh +// V +// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 +// B = NOT(b) +static bool f64_to_fimm8(uint64_t u64, oaknut::FImm8& fp8) { + const uint32_t sign = (u64 >> 63) & 1; + int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; + int64_t mantissa = u64 & 0xfffffffffffffULL; + + // Too many mantissa bits + if (mantissa & 0xffffffffffffULL) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 48; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + // exp = (NOT(b):c:d) - 3 + exp = ((exp + 3) & 0b111) ^ 0b100; + + fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); + return true; +} + // Implies possible StashV(0, ...)! void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { if (!v.low && !v.high) { @@ -901,6 +969,13 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { MOVI(dest.S4(), uint8_t(v.u32[0] >> 16), oaknut::util::LSL, 16); return; } + + // Try to utilize FMOV if possible + oaknut::FImm8 fp8(0); + if (f32_to_fimm8(v.u32[0], fp8)) { + FMOV(dest.S4(), fp8); + return; + } } // TODO(benvanik): see what other common values are. @@ -925,8 +1000,16 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, float v) { } else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. + + // Try to utilize FMOV if possible + oaknut::FImm8 fp8(0); + if (f32_to_fimm8(x.i, fp8)) { + FMOV(dest.toS(), fp8); + return; + } + MOV(W0, x.i); - MOV(dest.Selem()[0], W0); + FMOV(dest.toS(), W0); } } @@ -944,8 +1027,16 @@ void A64Emitter::LoadConstantV(oaknut::QReg dest, double v) { } else { // TODO(benvanik): see what other common values are. // TODO(benvanik): build constant table - 99% are reused. + + // Try to utilize FMOV if possible + oaknut::FImm8 fp8(0); + if (f64_to_fimm8(x.i, fp8)) { + FMOV(dest.toD(), fp8); + return; + } + MOV(X0, x.i); - MOV(dest.Delem()[0], X0); + FMOV(dest.toD(), X0); } } From c6fa69c1a9d2ac4ea26e48ab2ccc69075b4c537c Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 29 May 2024 10:54:31 -0700 Subject: [PATCH 119/179] [a64] Implement armv8.0 atomic operations Uses LSE when available, but provides an armv8.0 baseline implementation. --- a64_seq_memory.cc | 64 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index dc9cfca7e..be9baa4fc 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -195,15 +195,37 @@ struct ATOMIC_COMPARE_EXCHANGE_I32 } e.ADD(X1, e.GetMembaseReg(), X1); + const XReg address = X1; const WReg expected = i.src2; const WReg desired = i.src3; const WReg status = W0; - e.MOV(status, expected); - // if([C] == A) [C] = B - // else A = [C] - e.CASAL(status, desired, X1); - e.CMP(status, expected); + if (e.IsFeatureEnabled(kA64EmitLSE)) { + e.MOV(status, expected); + + // if([C] == A) [C] = B + // else A = [C] + e.CASAL(status, desired, address); + e.CMP(status, expected); + e.CSET(i.dest, Cond::EQ); + return; + } + + oaknut::Label success, fail, retry; + + e.l(retry); + e.LDAXR(W4, address); + e.CMP(W4, expected); + e.B(Cond::NE, fail); + + e.STLXR(status.toW(), desired, address); + e.CBNZ(status, retry); + e.B(success); + + e.l(fail); + e.CLREX(); + + e.l(success); e.CSET(i.dest, Cond::EQ); } }; @@ -223,15 +245,37 @@ struct ATOMIC_COMPARE_EXCHANGE_I64 } e.ADD(X1, e.GetMembaseReg(), X1); + const XReg address = X1; const XReg expected = i.src2; const XReg desired = i.src3; const XReg status = X0; - e.MOV(status, expected); - // if([C] == A) [C] = B - // else A = [C] - e.CASAL(status, desired, X1); - e.CMP(status, expected); + if (e.IsFeatureEnabled(kA64EmitLSE)) { + e.MOV(status, expected); + + // if([C] == A) [C] = B + // else A = [C] + e.CASAL(status, desired, address); + e.CMP(status, expected); + e.CSET(i.dest, Cond::EQ); + return; + } + + oaknut::Label success, fail, retry; + + e.l(retry); + e.LDAXR(X4, address); + e.CMP(X4, expected); + e.B(Cond::NE, fail); + + e.STLXR(status.toW(), desired, address); + e.CBNZ(status, retry); + e.B(success); + + e.l(fail); + e.CLREX(); + + e.l(success); e.CSET(i.dest, Cond::EQ); } }; From 6bb5753a7223625f1d7738a706c732c7f523d5a2 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Fri, 31 May 2024 16:00:10 -0700 Subject: [PATCH 120/179] [a64] Remove x64 reference implementations Removes all comments relating to x64 implementation details --- a64_emitter.cc | 31 +++------------------ a64_emitter.h | 2 +- a64_op.h | 2 +- a64_seq_memory.cc | 64 +------------------------------------------- a64_sequences.cc | 68 +++++++++++++---------------------------------- 5 files changed, 24 insertions(+), 143 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 0a83495d2..aba1fdd9a 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -137,21 +137,11 @@ bool A64Emitter::Emit(GuestFunction* function, HIRBuilder* builder, void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, GuestFunction* function) { - // To avoid changing xbyak, we do a switcharoo here. - // top_ points to the Xbyak buffer, and since we are in AutoGrow mode - // it has pending relocations. We copy the top_ to our buffer, swap the - // pointer, relocate, then return the original scratch pointer for use. - // top_ is used by Xbyak's ready() as both write base pointer and the absolute - // address base, which would not work on platforms not supporting writable - // executable memory, but Xenia doesn't use absolute label addresses in the - // generated code. - - // uint8_t* old_address = top_; + // Copy the current oaknut instruction-buffer into the code-cache uint32_t* old_address = CodeBlock::ptr(); void* new_execute_address; void* new_write_address; - // assert_true(func_info.code_size.total == size_); assert_true(func_info.code_size.total == offset()); if (function) { @@ -162,15 +152,9 @@ void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, code_cache_->PlaceHostCode(0, CodeBlock::ptr(), func_info, new_execute_address, new_write_address); } - // top_ = reinterpret_cast(new_write_address); - // set_wptr(reinterpret_cast(new_write_address)); - // ready(); - - // top_ = old_address; + // Reset the oaknut instruction-buffer set_wptr(reinterpret_cast(old_address)); - - // reset(); label_lookup_.clear(); return new_execute_address; @@ -357,7 +341,7 @@ void A64Emitter::MarkSourceOffset(const Instr* i) { } void A64Emitter::EmitGetCurrentThreadId() { - // rsi must point to context. We could fetch from the stack if needed. + // X27 must point to context. We could fetch from the stack if needed. LDRH(W0, GetContextReg(), offsetof(ppc::PPCContext, thread_id)); } @@ -442,14 +426,11 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { // TODO(benvanik): is it worth it to do this? It removes the need for // a ResolveFunction call, but makes the table less useful. assert_zero(uint64_t(fn->machine_code()) & 0xFFFFFFFF00000000); - // mov(eax, uint32_t(uint64_t(fn->machine_code()))); MOV(X16, uint32_t(uint64_t(fn->machine_code()))); } else if (code_cache_->has_indirection_table()) { // Load the pointer to the indirection table maintained in A64CodeCache. // The target dword will either contain the address of the generated code // or a thunk to ResolveAddress. - // mov(ebx, function->address()); - // mov(eax, dword[ebx]); MOV(W17, function->address()); LDR(W16, X17); } else { @@ -476,10 +457,8 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { BR(X16); } else { // Return address is from the previous SET_RETURN_ADDRESS. - // mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]); LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); - // call(rax); BLR(X16); } } @@ -488,8 +467,6 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, const oaknut::XReg& reg) { // Check if return. if (instr->flags & hir::CALL_POSSIBLE_RETURN) { - // cmp(reg.cvt32(), dword[rsp + StackLayout::GUEST_RET_ADDR]); - // je(epilog_label(), CodeGenerator::T_NEAR); LDR(W16, SP, StackLayout::GUEST_RET_ADDR); CMP(reg.toW(), W16); B(oaknut::Cond::EQ, epilog_label()); @@ -622,8 +599,6 @@ void A64Emitter::CallNativeSafe(void* fn) { } void A64Emitter::SetReturnAddress(uint64_t value) { - // mov(rax, value); - // mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], rax); MOV(X0, value); STR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); } diff --git a/a64_emitter.h b/a64_emitter.h index fef334dce..6c75e56ec 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -205,7 +205,7 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { std::byte* GetVConstPtr() const; std::byte* GetVConstPtr(VConst id) const; - static constexpr uintptr_t GetVConstOffset(VConst id){ + static constexpr uintptr_t GetVConstOffset(VConst id) { return sizeof(vec128_t) * id; } void LoadConstantV(oaknut::QReg dest, float v); diff --git a/a64_op.h b/a64_op.h index 2eaea627c..2b2f58932 100644 --- a/a64_op.h +++ b/a64_op.h @@ -2,7 +2,7 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2018 Xenia Developers. All rights reserved. * + * Copyright 2024 Xenia Developers. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index be9baa4fc..8892227b2 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -290,7 +290,6 @@ struct LOAD_LOCAL_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDRB(i.dest, SP, i.src1.constant()); - // e.mov(i.dest, e.byte[e.rsp + i.src1.constant()]); // e.TraceLoadI8(DATA_LOCAL, i.src1.constant, i.dest); } }; @@ -404,7 +403,6 @@ struct LOAD_CONTEXT_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { e.LDRB(i.dest, e.GetContextReg(), i.src1.value); - // e.mov(i.dest, e.byte[addr]); if (IsTracingData()) { e.MOV(e.GetNativeParam(0), i.src1.value); e.LDRB(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); @@ -1084,63 +1082,7 @@ struct CACHE_CONTROL } size_t cache_line_size = i.src2.value; - // RegExp addr; - // uint32_t address_constant; - // if (i.src1.is_constant) { - // // TODO(benvanik): figure out how to do this without a temp. - // // Since the constant is often 0x8... if we tried to use that as a - // // displacement it would be sign extended and mess things up. - // address_constant = static_cast(i.src1.constant()); - // if (address_constant < 0x80000000) { - // addr = e.GetMembaseReg() + address_constant; - // } else { - // if (address_constant >= 0xE0000000 && - // xe::memory::allocation_granularity() > 0x1000) { - // // e.mov(e.eax, address_constant + 0x1000); - // } else { - // // e.mov(e.eax, address_constant); - // } - // addr = e.GetMembaseReg() + e.rax; - // } - // } else { - // if (xe::memory::allocation_granularity() > 0x1000) { - // // Emulate the 4 KB physical address offset in 0xE0000000+ when can't - // do - // // it via memory mapping. - // // e.cmp(i.src1.reg().cvt32(), 0xE0000000); - // // e.setae(e.al); - // // e.movzx(e.eax, e.al); - // // e.shl(e.eax, 12); - // // e.add(e.eax, i.src1.reg().cvt32()); - // } else { - // // Clear the top 32 bits, as they are likely garbage. - // // TODO(benvanik): find a way to avoid doing this. - // // e.mov(e.eax, i.src1.reg().cvt32()); - // } - // addr = e.GetMembaseReg() + e.rax; - // } - // if (is_clflush) { - // // e.clflush(e.ptr[addr]); - // } - // if (is_prefetch) { - // // e.prefetcht0(e.ptr[addr]); - // } - - // if (cache_line_size >= 128) { - // // Prefetch the other 64 bytes of the 128-byte cache line. - // if (i.src1.is_constant && address_constant < 0x80000000) { - // addr = e.GetMembaseReg() + (address_constant ^ 64); - // } else { - // // e.xor_(e.eax, 64); - // } - // if (is_clflush) { - // // e.clflush(e.ptr[addr]); - // } - // if (is_prefetch) { - // // e.prefetcht0(e.ptr[addr]); - // } - // assert_true(cache_line_size == 128); - // } + // TODO(wunkolo): Arm64 cache-control } }; EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); @@ -1151,10 +1093,6 @@ EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); struct MEMORY_BARRIER : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // mfence on x64 flushes all writes before any later instructions - // e.mfence(); - - // This is equivalent to DMB SY e.DMB(BarrierOp::SY); } }; diff --git a/a64_sequences.cc b/a64_sequences.cc index c3c0741e8..db334602b 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -295,10 +295,8 @@ struct CONVERT_I32_F32 static void Emit(A64Emitter& e, const EmitArgType& i) { // TODO(benvanik): saturation check? cvtt* (trunc?) if (i.instr->flags == ROUND_TO_ZERO) { - // e.vcvttss2si(i.dest, i.src1); e.FCVTZS(i.dest, i.src1.reg().toS()); } else { - // e.vcvtss2si(i.dest, i.src1); e.FCVTNS(i.dest, i.src1.reg().toS()); } } @@ -307,13 +305,10 @@ struct CONVERT_I32_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // Intel returns 0x80000000 if the double value does not fit within an int32 - // ARM64 and PPC saturates the value instead. - // e.vminsd(e.xmm0, i.src1, e.GetVConstPtr(XMMIntMaxPD)); + // ARM64 and PPC saturates the value instead if (i.instr->flags == ROUND_TO_ZERO) { - // e.vcvttsd2si(i.dest, e.xmm0); e.FCVTZS(i.dest, i.src1.reg().toD()); } else { - // e.vcvtsd2si(i.dest, e.xmm0); e.FCVTNS(i.dest, i.src1.reg().toD()); } } @@ -322,10 +317,8 @@ struct CONVERT_I64_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.instr->flags == ROUND_TO_ZERO) { - // e.vcvttsd2si(i.dest, i.src1); e.FCVTZS(i.dest, i.src1.reg().toD()); } else { - // e.vcvtsd2si(i.dest, i.src1); e.FCVTNS(i.dest, i.src1.reg().toD()); } } @@ -333,24 +326,18 @@ struct CONVERT_I64_F64 struct CONVERT_F32_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): saturation check? cvtt* (trunc?) - // e.vcvtsi2ss(i.dest, i.src1); e.SCVTF(i.dest.reg().toS(), i.src1); } }; struct CONVERT_F32_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): saturation check? cvtt* (trunc?) - // e.vcvtsd2ss(i.dest, i.src1); e.FCVT(i.dest.reg().toS(), i.src1.reg().toD()); } }; struct CONVERT_F64_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): saturation check? cvtt* (trunc?) - // e.vcvtsi2sd(i.dest, i.src1); e.SCVTF(i.dest.reg().toD(), i.src1); } }; @@ -372,19 +359,15 @@ struct ROUND_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { switch (i.instr->flags) { case ROUND_TO_ZERO: - // e.vroundss(i.dest, i.src1, 0b00000011); e.FRINTZ(i.dest.reg().toS(), i.src1.reg().toS()); break; case ROUND_TO_NEAREST: - // e.vroundss(i.dest, i.src1, 0b00000000); e.FRINTN(i.dest.reg().toS(), i.src1.reg().toS()); break; case ROUND_TO_MINUS_INFINITY: - // e.vroundss(i.dest, i.src1, 0b00000001); e.FRINTM(i.dest.reg().toS(), i.src1.reg().toS()); break; case ROUND_TO_POSITIVE_INFINITY: - // e.vroundss(i.dest, i.src1, 0b00000010); e.FRINTP(i.dest.reg().toS(), i.src1.reg().toS()); break; } @@ -394,19 +377,15 @@ struct ROUND_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { switch (i.instr->flags) { case ROUND_TO_ZERO: - // e.vroundsd(i.dest, i.src1, 0b00000011); e.FRINTZ(i.dest, i.src1); break; case ROUND_TO_NEAREST: - // e.vroundsd(i.dest, i.src1, 0b00000000); e.FRINTN(i.dest, i.src1); break; case ROUND_TO_MINUS_INFINITY: - // e.vroundsd(i.dest, i.src1, 0b00000001); e.FRINTM(i.dest, i.src1); break; case ROUND_TO_POSITIVE_INFINITY: - // e.vroundsd(i.dest, i.src1, 0b00000010); e.FRINTP(i.dest, i.src1); break; } @@ -416,19 +395,15 @@ struct ROUND_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { switch (i.instr->flags) { case ROUND_TO_ZERO: - // e.vroundps(i.dest, i.src1, 0b00000011); e.FRINTZ(i.dest.reg().S4(), i.src1.reg().S4()); break; case ROUND_TO_NEAREST: - // e.vroundps(i.dest, i.src1, 0b00000000); e.FRINTN(i.dest.reg().S4(), i.src1.reg().S4()); break; case ROUND_TO_MINUS_INFINITY: - // e.vroundps(i.dest, i.src1, 0b00000001); e.FRINTM(i.dest.reg().S4(), i.src1.reg().S4()); break; case ROUND_TO_POSITIVE_INFINITY: - // e.vroundps(i.dest, i.src1, 0b00000010); e.FRINTP(i.dest.reg().S4(), i.src1.reg().S4()); break; } @@ -684,12 +659,12 @@ struct SELECT_F64 static void Emit(A64Emitter& e, const EmitArgType& i) { // dest = src1 != 0 ? src2 : src3 - DReg src2 = i.src2.is_constant ? D2 : i.src2; + const DReg src2 = i.src2.is_constant ? D2 : i.src2; if (i.src2.is_constant) { e.LoadConstantV(src2.toQ(), i.src2.constant()); } - DReg src3 = i.src3.is_constant ? D3 : i.src3; + const DReg src3 = i.src3.is_constant ? D3 : i.src3; if (i.src3.is_constant) { e.LoadConstantV(src3.toQ(), i.src3.constant()); } @@ -703,12 +678,12 @@ struct SELECT_V128_I8 static void Emit(A64Emitter& e, const EmitArgType& i) { // dest = src1 != 0 ? src2 : src3 - QReg src2 = i.src2.is_constant ? Q2 : i.src2; + const QReg src2 = i.src2.is_constant ? Q2 : i.src2; if (i.src2.is_constant) { e.LoadConstantV(src2, i.src2.constant()); } - QReg src3 = i.src3.is_constant ? Q3 : i.src3; + const QReg src3 = i.src3.is_constant ? Q3 : i.src3; if (i.src3.is_constant) { e.LoadConstantV(src3, i.src3.constant()); } @@ -730,12 +705,12 @@ struct SELECT_V128_V128 e.MOV(src1.B16(), i.src1.reg().B16()); } - const QReg src2 = i.src2.is_constant ? Q1 : i.src2; + const QReg src2 = i.src2.is_constant ? Q2 : i.src2; if (i.src2.is_constant) { e.LoadConstantV(src2, i.src2.constant()); } - const QReg src3 = i.src3.is_constant ? Q2 : i.src3; + const QReg src3 = i.src3.is_constant ? Q3 : i.src3; if (i.src3.is_constant) { e.LoadConstantV(src3, i.src3.constant()); } @@ -1123,24 +1098,20 @@ void EmitAddXX(A64Emitter& e, const ARGS& i) { SEQ::EmitCommutativeBinaryOp( e, i, [](A64Emitter& e, REG dest_src, REG src) { - // e.add(dest_src, src); e.ADD(dest_src, dest_src, src); }, [](A64Emitter& e, REG dest_src, int32_t constant) { - // e.add(dest_src, constant); e.MOV(REG(1), constant); e.ADD(dest_src, dest_src, REG(1)); }); } struct ADD_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // EmitAddXX(e, i); EmitAddXX(e, i); } }; struct ADD_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // EmitAddXX(e, i); EmitAddXX(e, i); } }; @@ -1158,7 +1129,6 @@ struct ADD_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitCommutativeBinaryVOp( e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - // e.vaddss(dest, src1, src2); e.FADD(dest, src1, src2); }); } @@ -1167,7 +1137,6 @@ struct ADD_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitCommutativeBinaryVOp( e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - // e.vaddsd(dest, src1, src2); e.FADD(dest, src1, src2); }); } @@ -1176,7 +1145,6 @@ struct ADD_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitCommutativeBinaryVOp( e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - // e.vaddps(dest, src1, src2); e.FADD(dest.S4(), src1.S4(), src2.S4()); }); } @@ -2030,7 +1998,7 @@ struct POW2_F32 : Sequence> { } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.FMOV(i.dest, S0); } @@ -2044,7 +2012,7 @@ struct POW2_F64 : Sequence> { } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.FMOV(i.dest, D0); } @@ -2059,7 +2027,7 @@ struct POW2_V128 : Sequence> { return vld1q_f32(values); } static void Emit(A64Emitter& e, const EmitArgType& i) { - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -2082,9 +2050,9 @@ struct LOG2_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); } else { - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); } e.CallNativeSafe(reinterpret_cast(EmulateLog2)); e.FMOV(i.dest, S0); @@ -2100,9 +2068,9 @@ struct LOG2_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); } else { - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); } e.CallNativeSafe(reinterpret_cast(EmulateLog2)); e.FMOV(i.dest, D0); @@ -2119,9 +2087,9 @@ struct LOG2_V128 : Sequence> { } static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), XSP, e.StashConstantV(0, i.src1.constant())); + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); } else { - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1.reg().toQ())); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); } e.CallNativeSafe(reinterpret_cast(EmulateLog2)); e.MOV(i.dest.reg().B16(), Q0.B16()); @@ -2455,7 +2423,7 @@ struct SHL_V128 : Sequence> { } else { e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateShlV128)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -2534,7 +2502,7 @@ struct SHR_V128 : Sequence> { } else { e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); } - e.ADD(e.GetNativeParam(0), XSP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); e.CallNativeSafe(reinterpret_cast(EmulateShrV128)); e.MOV(i.dest.reg().B16(), Q0.B16()); } From ab7f517629dbbd9c11f387ad68c95f9e1c4f9459 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 2 Jun 2024 12:01:48 -0700 Subject: [PATCH 121/179] [a64] Implement `OPCODE_CACHE_CONTROL` `dc civac` causes an illegal-instruciton on Windows-ARM. This is likely as a security measure against cache-attacks. On Linux this instruction is trapped into an EL1 kernel function. Windows does not seem to have any user-mode cache-maintenance instructions available for data-cache(only instruction-cache via `FlushInstructionCache`). The closest thing we can do for now is a full data memory-barrier with `dsb ish`. Prefetches are implemented using `prfm pldl1keep, ...`. --- a64_seq_memory.cc | 68 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/a64_seq_memory.cc b/a64_seq_memory.cc index 8892227b2..d7d66a14d 100644 --- a/a64_seq_memory.cc +++ b/a64_seq_memory.cc @@ -1082,7 +1082,73 @@ struct CACHE_CONTROL } size_t cache_line_size = i.src2.value; - // TODO(wunkolo): Arm64 cache-control + XReg addr = X0; + uint32_t address_constant; + if (i.src1.is_constant) { + // TODO(benvanik): figure out how to do this without a temp. + // Since the constant is often 0x8... if we tried to use that as a + // displacement it would be sign extended and mess things up. + address_constant = static_cast(i.src1.constant()); + if (address_constant < 0x80000000) { + e.ADD(addr, e.GetMembaseReg(), address_constant); + } else { + if (address_constant >= 0xE0000000 && + xe::memory::allocation_granularity() > 0x1000) { + e.MOV(X1, address_constant + 0x1000); + } else { + e.MOV(X1, address_constant); + } + e.ADD(addr, e.GetMembaseReg(), X1); + } + } else { + if (xe::memory::allocation_granularity() > 0x1000) { + // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do + // it via memory mapping. + e.MOV(X1, 0xE0000000); + e.CMP(i.src1.reg(), X1); + e.CSET(X1, Cond::HS); + e.ADD(X1, i.src1.reg(), X1, LSL, 12); + } else { + // Clear the top 32 bits, as they are likely garbage. + e.MOV(W1, i.src1.reg().toW()); + } + e.ADD(addr, e.GetMembaseReg(), X1); + } + + if (is_clflush) { + // TODO(wunkolo): These kind of cache-maintenance instructions cause an + // illegal-instruction on windows, but is trapped to proper EL1 code on + // Linux. Need a way to do cache-maintenance on Windows-Arm + // e.DC(DcOp::CIVAC, addr); + + // Full data sync + e.DSB(BarrierOp::ISH); + } + if (is_prefetch) { + e.PRFM(PrfOp::PLDL1KEEP, addr); + } + + if (cache_line_size >= 128) { + // Prefetch the other 64 bytes of the 128-byte cache line. + if (i.src1.is_constant && address_constant < 0x80000000) { + e.ADD(addr, e.GetMembaseReg(), address_constant ^ 64); + } else { + e.EOR(X1, X1, 64); + } + if (is_clflush) { + // TODO(wunkolo): These kind of cache-maintenance instructions cause an + // illegal-instruction on windows, but is trapped to proper EL1 code on + // Linux. Need a way to do cache-maintenance on Windows-Arm + // e.DC(DcOp::CIVAC, addr); + + // Full data sync + e.DSB(BarrierOp::ISH); + } + if (is_prefetch) { + e.PRFM(PrfOp::PLDL1KEEP, addr); + } + assert_true(cache_line_size == 128); + } } }; EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); From 6d995517a495fd6f905165bd13d4320a7443bdba Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Mon, 3 Jun 2024 07:55:09 -0700 Subject: [PATCH 122/179] [a64] Fix out-of-bounds `OPCODE_VECTOR_SHL`(all-same) case Out-of-bound shift-values are handled as modulo-element-size --- a64_seq_vector.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 71d608f82..b19ea6b3e 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -539,7 +539,7 @@ struct VECTOR_SHL_V128 } if (all_same) { // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); + e.SHL(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); @@ -563,7 +563,7 @@ struct VECTOR_SHL_V128 } if (all_same) { // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u8[0]); + e.SHL(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u8[0] & 0xF); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); @@ -587,7 +587,7 @@ struct VECTOR_SHL_V128 } if (all_same) { // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u8[0]); + e.SHL(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u8[0] & 0x1F); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); From 9aa9c2d9e02eb63ab6bd6ac6679c0cd833dcf0a1 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sat, 8 Jun 2024 14:23:59 -0700 Subject: [PATCH 123/179] [a64] Use VectorCodeGenerator rather than CodeBlock+CodeGenerator The emitter doesn't actually hold onto executable code, but just generates the assembly-data into a buffer for the currently-resolving function before placing it into a code-cache. When code gets pushed into the code-cache, it can just be copied from an `std::vector` and reset. The code-cache itself maintains the actual executable memory and stack-unwinding code and such. This also fixes a bunch of errornous relative-addressing glitches where relative addresses were calculated based on the address of the unused CodeBlock rather than being position-independent. `MOVP2R` in particular was generating different instructions depending on its distance from the code block when it should always just use `MOV` and not do any relative-address calculations since we can't predict where the actual instruction's offset will be(we cannot predict what the program counter will be). Oaknut probably needs a "position independent" policy or mode or something so that it avoids PC-relative instructions. --- a64_emitter.cc | 33 +++++++++++++++------------------ a64_emitter.h | 8 +++++--- a64_seq_vector.cc | 30 +++++++++++++++--------------- a64_sequences.cc | 2 +- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index aba1fdd9a..925e8bb9f 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -58,8 +58,6 @@ using xe::cpu::hir::Instr; using namespace xe::literals; using namespace oaknut::util; -static const size_t kMaxCodeSize = 1_MiB; - static const size_t kStashOffset = 32; // static const size_t kStashOffsetHigh = 32 + 32; @@ -73,8 +71,7 @@ const uint8_t A64Emitter::fpr_reg_map_[A64Emitter::FPR_COUNT] = { }; A64Emitter::A64Emitter(A64Backend* backend) - : CodeBlock(kMaxCodeSize), - CodeGenerator(CodeBlock::ptr()), + : VectorCodeGenerator(assembly_buffer), processor_(backend->processor()), backend_(backend), code_cache_(backend->code_cache()) { @@ -138,23 +135,22 @@ bool A64Emitter::Emit(GuestFunction* function, HIRBuilder* builder, void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, GuestFunction* function) { // Copy the current oaknut instruction-buffer into the code-cache - uint32_t* old_address = CodeBlock::ptr(); void* new_execute_address; void* new_write_address; assert_true(func_info.code_size.total == offset()); if (function) { - code_cache_->PlaceGuestCode(function->address(), CodeBlock::ptr(), + code_cache_->PlaceGuestCode(function->address(), assembly_buffer.data(), func_info, function, new_execute_address, new_write_address); } else { - code_cache_->PlaceHostCode(0, CodeBlock::ptr(), func_info, + code_cache_->PlaceHostCode(0, assembly_buffer.data(), func_info, new_execute_address, new_write_address); } // Reset the oaknut instruction-buffer - set_wptr(reinterpret_cast(old_address)); + assembly_buffer.clear(); label_lookup_.clear(); return new_execute_address; @@ -224,7 +220,8 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // Call count. MOV(W0, 1); - MOVP2R(X5, low_address(&trace_header->function_call_count)); + MOV(X5, reinterpret_cast( + low_address(&trace_header->function_call_count))); LDADDAL(X0, X0, X5); // Get call history slot. @@ -234,8 +231,8 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { AND(W0, W0, 0b00000011); // Record call history value into slot (guest addr in W1). - MOV(X5, uint32_t( - uint64_t(low_address(&trace_header->function_caller_history)))); + MOV(X5, reinterpret_cast( + low_address(&trace_header->function_caller_history))); STR(W1, X5, X0, oaknut::IndexExt::LSL, 2); // Calling thread. Load X0 with thread ID. @@ -243,7 +240,8 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { MOV(W5, 1); LSL(W0, W5, W0); - MOVP2R(X5, low_address(&trace_header->function_thread_use)); + MOV(X5, reinterpret_cast( + low_address(&trace_header->function_thread_use))); LDSET(W0, WZR, X5); } @@ -334,8 +332,9 @@ void A64Emitter::MarkSourceOffset(const Instr* i) { const uint32_t instruction_index = (entry->guest_address - trace_data_->start_address()) / 4; MOV(X0, 1); - MOVP2R(X1, low_address(trace_data_->instruction_execute_counts() + - instruction_index * 8)); + MOV(X1, reinterpret_cast( + low_address(trace_data_->instruction_execute_counts() + + instruction_index * 8))); LDADDAL(X0, ZR, X1); } } @@ -803,11 +802,9 @@ void A64Emitter::FreeConstData(uintptr_t data) { memory::DeallocationType::kRelease); } -std::byte* A64Emitter::GetVConstPtr() const { - return reinterpret_cast(backend_->emitter_data()); -} +uintptr_t A64Emitter::GetVConstPtr() const { return backend_->emitter_data(); } -std::byte* A64Emitter::GetVConstPtr(VConst id) const { +uintptr_t A64Emitter::GetVConstPtr(VConst id) const { // Load through fixed constant table setup by PlaceConstData. // It's important that the pointer is not signed, as it will be sign-extended. return GetVConstPtr() + GetVConstOffset(id); diff --git a/a64_emitter.h b/a64_emitter.h index 6c75e56ec..629c67a4b 100644 --- a/a64_emitter.h +++ b/a64_emitter.h @@ -122,7 +122,7 @@ enum A64EmitterFeatureFlags { kA64EmitF16C = 1 << 1, }; -class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { +class A64Emitter : public oaknut::VectorCodeGenerator { public: A64Emitter(A64Backend* backend); virtual ~A64Emitter(); @@ -203,8 +203,8 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { static bool ConstantFitsIn32Reg(uint64_t v); void MovMem64(const oaknut::XRegSp& addr, intptr_t offset, uint64_t v); - std::byte* GetVConstPtr() const; - std::byte* GetVConstPtr(VConst id) const; + uintptr_t GetVConstPtr() const; + uintptr_t GetVConstPtr(VConst id) const; static constexpr uintptr_t GetVConstOffset(VConst id) { return sizeof(vec128_t) * id; } @@ -239,6 +239,8 @@ class A64Emitter : public oaknut::CodeBlock, public oaknut::CodeGenerator { A64CodeCache* code_cache_ = nullptr; uint32_t feature_flags_ = 0; + std::vector assembly_buffer; + oaknut::Label* epilog_label_ = nullptr; // Convert from plain-text label-names into oaknut-labels diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index b19ea6b3e..a309fcc02 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -83,10 +83,10 @@ struct LOAD_VECTOR_SHL_I8 if (i.src1.is_constant) { auto sh = i.src1.constant(); assert_true(sh < xe::countof(lvsl_table)); - e.MOVP2R(X0, &lvsl_table[sh]); + e.MOV(X0, reinterpret_cast(&lvsl_table[sh])); e.LDR(i.dest, X0); } else { - e.MOVP2R(X0, lvsl_table); + e.MOV(X0, reinterpret_cast(lvsl_table)); e.AND(X1, i.src1.reg().toX(), 0xf); e.LDR(i.dest, X0, X1, IndexExt::LSL, 4); } @@ -121,10 +121,10 @@ struct LOAD_VECTOR_SHR_I8 if (i.src1.is_constant) { auto sh = i.src1.constant(); assert_true(sh < xe::countof(lvsr_table)); - e.MOVP2R(X0, &lvsr_table[sh]); + e.MOV(X0, reinterpret_cast(&lvsr_table[sh])); e.LDR(i.dest, X0); } else { - e.MOVP2R(X0, lvsr_table); + e.MOV(X0, reinterpret_cast(lvsr_table)); e.AND(X1, i.src1.reg().toX(), 0xf); e.LDR(i.dest, X0, X1, IndexExt::LSL, 4); } @@ -1007,7 +1007,7 @@ struct EXTRACT_I32 e.AND(X0, i.src2.reg().toX(), 0b11); e.LSL(X0, X0, 4); - e.MOVP2R(X1, extract_table_32); + e.MOV(X1, reinterpret_cast(extract_table_32)); e.LDR(Q0, X1, X0); // Byte-table lookup @@ -1335,7 +1335,7 @@ struct PACK : Sequence> { } const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); // Saturate to [3,3....] so that only values between 3...[00] and 3...[FF] // are valid - max before min to pack NaN as zero (5454082B is heavily @@ -1435,7 +1435,7 @@ struct PACK : Sequence> { e.LoadConstantV(src, i.src1.constant()); } const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); // Saturate e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); @@ -1456,7 +1456,7 @@ struct PACK : Sequence> { e.LoadConstantV(src, i.src1.constant()); } const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); // Saturate e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); @@ -1478,7 +1478,7 @@ struct PACK : Sequence> { e.LoadConstantV(src, i.src1.constant()); } const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); // Saturate. e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MinUnpacked)); @@ -1519,7 +1519,7 @@ struct PACK : Sequence> { e.LoadConstantV(src, i.src1.constant()); } const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); // Saturate. e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MinUnpacked)); @@ -1740,7 +1740,7 @@ struct UNPACK : Sequence> { static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { // ARGB (WXYZ) -> RGBA (XYZW) const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); QReg src(0); @@ -1849,7 +1849,7 @@ struct UNPACK : Sequence> { // (VD.w) = 1.0 (games splat W after unpacking to get vectors of 1.0f) // src is (xx,xx,xx,VALUE) const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); QReg src(0); if (i.src1.is_constant) { @@ -1892,7 +1892,7 @@ struct UNPACK : Sequence> { // src is (xx,xx,VALUE,VALUE) const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); QReg src(0); if (i.src1.is_constant) { @@ -1928,7 +1928,7 @@ struct UNPACK : Sequence> { } static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); QReg src(0); if (i.src1.is_constant) { @@ -1972,7 +1972,7 @@ struct UNPACK : Sequence> { } static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { const XReg VConstData = X3; - e.MOVP2R(VConstData, e.GetVConstPtr()); + e.MOV(VConstData, e.GetVConstPtr()); QReg src(0); if (i.src1.is_constant) { diff --git a/a64_sequences.cc b/a64_sequences.cc index db334602b..c88e201e5 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2758,7 +2758,7 @@ struct SET_ROUNDING_MODE_I32 e.AND(W1, i.src1, 0b111); // Use the low 3 bits as an index into a LUT - e.MOVP2R(X0, fpcr_table); + e.MOV(X0, reinterpret_cast(fpcr_table)); e.LDRB(W0, X0, X1); // Replace FPCR bits with new value From 8741c3fd4bed5f63dc6ed2874516d428649bcacd Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 12 Jun 2024 14:32:36 -0700 Subject: [PATCH 124/179] [a64] Replace instances of `MOV`+`DUP-splats to `MOVI` These `MOV`->`DUP` splats can just be a singular `MOVI` instruction --- a64_seq_vector.cc | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index a309fcc02..43c420b10 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1138,13 +1138,11 @@ struct PERMUTE_V128 } // Indices must be endian-swapped - e.MOV(W0, 0b11); - e.DUP(Q1.B16(), W0); + e.MOVI(Q1.B16(), 0b11); e.EOR(indices.B16(), indices.B16(), Q1.B16()); // Modulo 32 the indices - e.MOV(W0, 0b0001'1111); - e.DUP(Q1.B16(), W0); + e.MOVI(Q1.B16(), 0b0001'1111); e.AND(indices.B16(), indices.B16(), Q1.B16()); // Table-registers must be sequential indices @@ -1186,22 +1184,18 @@ struct PERMUTE_V128 } // Indices must be endian-swapped - e.MOV(W0, 0b1); - e.DUP(Q1.H8(), W0); + e.MOVI(Q1.H8(), 0b1); e.EOR(indices.B16(), indices.B16(), Q1.B16()); // Modulo-16 the indices - e.MOV(W0, 0b0000'1111); - e.DUP(Q1.H8(), W0); + e.MOVI(Q1.H8(), 0b0000'1111); e.AND(indices.B16(), indices.B16(), Q1.B16()); // Convert int16 indices into int8 - e.MOV(W0, 0x02'02); - e.DUP(Q1.H8(), W0); + e.MOVI(Q1.B16(), 0x02); e.MUL(indices.H8(), indices.H8(), Q1.H8()); - e.MOV(W0, 0x01'00); - e.DUP(Q1.H8(), W0); + e.MOVI(Q1.H8(), 0x01, LSL, 8); e.ADD(indices.H8(), indices.H8(), Q1.H8()); // Table-registers must be sequential indices From e5066c84e6051e584d0707af07b731b49140e0b4 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Wed, 12 Jun 2024 14:41:34 -0700 Subject: [PATCH 125/179] [a64] Optimize `OPCODE_SPLAT` byte-constants Byte-sized constants can utilize the `MOVI` instructions. This makes many cases such as zero-splats much faster since this encodes as just a register-rename(similar to `xor` on x64). --- a64_seq_vector.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 43c420b10..682843723 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -1026,6 +1026,10 @@ EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); struct SPLAT_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { + if (i.src1.constant() <= 0xFF) { + e.MOVI(i.dest.reg().B16(), i.src1.constant()); + return; + } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().B16(), W0); } else { @@ -1036,6 +1040,10 @@ struct SPLAT_I8 : Sequence> { struct SPLAT_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { + if (i.src1.constant() <= 0xFF) { + e.MOVI(i.dest.reg().H8(), i.src1.constant()); + return; + } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().H8(), W0); } else { @@ -1046,6 +1054,10 @@ struct SPLAT_I16 : Sequence> { struct SPLAT_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { + if (i.src1.constant() <= 0xFF) { + e.MOVI(i.dest.reg().S4(), i.src1.constant()); + return; + } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().S4(), W0); } else { @@ -1056,6 +1068,10 @@ struct SPLAT_I32 : Sequence> { struct SPLAT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { + if (i.src1.value->constant.i32 <= 0xFF) { + e.MOVI(i.dest.reg().S4(), i.src1.value->constant.i32); + return; + } e.MOV(W0, i.src1.value->constant.i32); e.DUP(i.dest.reg().S4(), W0); } else { From 28729736aa925926e747883d9d587bf19c9791b2 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Thu, 13 Jun 2024 14:07:01 -0700 Subject: [PATCH 126/179] [a64] Optimize `OPCODE_SPLAT` with `MOVI`/`FMOV` Moves the `FMOV` constant functions into `a64_util` so it is available to other translation units. Optimize constant-splats with conditional use of `MOVI` and `FMOV`. --- a64_emitter.cc | 69 +-------------------------------------------- a64_seq_vector.cc | 48 ++++++++++++++++++++++++------- a64_util.h | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 79 deletions(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index 925e8bb9f..e835f4aff 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -8,6 +8,7 @@ */ #include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_util.h" #include @@ -810,74 +811,6 @@ uintptr_t A64Emitter::GetVConstPtr(VConst id) const { return GetVConstPtr() + GetVConstOffset(id); } -// Attempts to convert an fp32 bit-value into an fp8-immediate value for FMOV -// returns false if the value cannot be represented -// C2.2.3 Modified immediate constants in A64 floating-point instructions -// abcdefgh -// V -// aBbbbbbc defgh000 00000000 00000000 -// B = NOT(b) -static bool f32_to_fimm8(uint32_t u32, oaknut::FImm8& fp8) { - const uint32_t sign = (u32 >> 31) & 1; - int32_t exp = ((u32 >> 23) & 0xff) - 127; - int64_t mantissa = u32 & 0x7fffff; - - // Too many mantissa bits - if (mantissa & 0x7ffff) { - return false; - } - // Too many exp bits - if (exp < -3 || exp > 4) { - return false; - } - - // mantissa = (16 + e:f:g:h) / 16. - mantissa >>= 19; - if ((mantissa & 0b1111) != mantissa) { - return false; - } - - // exp = (NOT(b):c:d) - 3 - exp = ((exp + 3) & 0b111) ^ 0b100; - - fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); - return true; -} - -// Attempts to convert an fp64 bit-value into an fp8-immediate value for FMOV -// returns false if the value cannot be represented -// C2.2.3 Modified immediate constants in A64 floating-point instructions -// abcdefgh -// V -// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 -// B = NOT(b) -static bool f64_to_fimm8(uint64_t u64, oaknut::FImm8& fp8) { - const uint32_t sign = (u64 >> 63) & 1; - int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; - int64_t mantissa = u64 & 0xfffffffffffffULL; - - // Too many mantissa bits - if (mantissa & 0xffffffffffffULL) { - return false; - } - // Too many exp bits - if (exp < -3 || exp > 4) { - return false; - } - - // mantissa = (16 + e:f:g:h) / 16. - mantissa >>= 48; - if ((mantissa & 0b1111) != mantissa) { - return false; - } - - // exp = (NOT(b):c:d) - 3 - exp = ((exp + 3) & 0b111) ^ 0b100; - - fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); - return true; -} - // Implies possible StashV(0, ...)! void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { if (!v.low && !v.high) { diff --git a/a64_seq_vector.cc b/a64_seq_vector.cc index 682843723..abc4688ac 100644 --- a/a64_seq_vector.cc +++ b/a64_seq_vector.cc @@ -8,6 +8,7 @@ */ #include "xenia/cpu/backend/a64/a64_sequences.h" +#include "xenia/cpu/backend/a64/a64_util.h" #include #include @@ -1026,12 +1027,7 @@ EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); struct SPLAT_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - if (i.src1.constant() <= 0xFF) { - e.MOVI(i.dest.reg().B16(), i.src1.constant()); - return; - } - e.MOV(W0, i.src1.constant()); - e.DUP(i.dest.reg().B16(), W0); + e.MOVI(i.dest.reg().B16(), i.src1.constant()); } else { e.DUP(i.dest.reg().B16(), i.src1); } @@ -1040,9 +1036,12 @@ struct SPLAT_I8 : Sequence> { struct SPLAT_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - if (i.src1.constant() <= 0xFF) { + if ((i.src1.constant() & 0xFF'00) == 0) { e.MOVI(i.dest.reg().H8(), i.src1.constant()); return; + } else if ((i.src1.constant() & 0x00'FF) == 0) { + e.MOVI(i.dest.reg().H8(), i.src1.constant(), oaknut::util::LSL, 8); + return; } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().H8(), W0); @@ -1054,9 +1053,22 @@ struct SPLAT_I16 : Sequence> { struct SPLAT_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - if (i.src1.constant() <= 0xFF) { + oaknut::FImm8 fp8(0); + if (f32_to_fimm8(i.src1.value->constant.u32, fp8)) { + e.FMOV(i.dest.reg().S4(), fp8); + return; + } else if ((i.src1.constant() & 0xFF'FF'FF'00) == 0) { e.MOVI(i.dest.reg().S4(), i.src1.constant()); return; + } else if ((i.src1.constant() & 0xFF'FF'00'FF) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.constant(), oaknut::util::LSL, 8); + return; + } else if ((i.src1.constant() & 0xFF'00'FF'FF) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.constant(), oaknut::util::LSL, 16); + return; + } else if ((i.src1.constant() & 0x00'FF'FF'FF) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.constant(), oaknut::util::LSL, 24); + return; } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().S4(), W0); @@ -1068,8 +1080,24 @@ struct SPLAT_I32 : Sequence> { struct SPLAT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - if (i.src1.value->constant.i32 <= 0xFF) { - e.MOVI(i.dest.reg().S4(), i.src1.value->constant.i32); + oaknut::FImm8 fp8(0); + if (f32_to_fimm8(i.src1.value->constant.u32, fp8)) { + e.FMOV(i.dest.reg().S4(), fp8); + return; + } else if ((i.src1.value->constant.u32 & 0xFF'FF'FF'00) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32); + return; + } else if ((i.src1.value->constant.u32 & 0xFF'FF'00'FF) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32, oaknut::util::LSL, + 8); + return; + } else if ((i.src1.value->constant.u32 & 0xFF'00'FF'FF) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32, oaknut::util::LSL, + 16); + return; + } else if ((i.src1.value->constant.u32 & 0x00'FF'FF'FF) == 0) { + e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32, oaknut::util::LSL, + 24); return; } e.MOV(W0, i.src1.value->constant.i32); diff --git a/a64_util.h b/a64_util.h index e3a34ac00..0b950b8ae 100644 --- a/a64_util.h +++ b/a64_util.h @@ -17,7 +17,77 @@ namespace xe { namespace cpu { namespace backend { -namespace a64 {} // namespace a64 +namespace a64 { + +// Attempts to convert an fp32 bit-value into an fp8-immediate value for FMOV +// returns false if the value cannot be represented +// C2.2.3 Modified immediate constants in A64 ing-point instructions +// abcdefgh +// V +// aBbbbbbc defgh000 00000000 00000000 +// B = NOT(b) +constexpr bool f32_to_fimm8(uint32_t u32, oaknut::FImm8& fp8) { + const uint32_t sign = (u32 >> 31) & 1; + int32_t exp = ((u32 >> 23) & 0xff) - 127; + int64_t mantissa = u32 & 0x7fffff; + + // Too many mantissa bits + if (mantissa & 0x7ffff) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 19; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + // exp = (NOT(b):c:d) - 3 + exp = ((exp + 3) & 0b111) ^ 0b100; + + fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); + return true; +} + +// Attempts to convert an fp64 bit-value into an fp8-immediate value for FMOV +// returns false if the value cannot be represented +// C2.2.3 Modified immediate constants in A64 floating-point instructions +// abcdefgh +// V +// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 +// B = NOT(b) +constexpr bool f64_to_fimm8(uint64_t u64, oaknut::FImm8& fp8) { + const uint32_t sign = (u64 >> 63) & 1; + int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; + int64_t mantissa = u64 & 0xfffffffffffffULL; + + // Too many mantissa bits + if (mantissa & 0xffffffffffffULL) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 48; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + // exp = (NOT(b):c:d) - 3 + exp = ((exp + 3) & 0b111) ^ 0b100; + + fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); + return true; +} + +} // namespace a64 } // namespace backend } // namespace cpu } // namespace xe From 45e2dc010be46f4531800b3b4748cd9923b39749 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 16 Jun 2024 15:00:03 -0700 Subject: [PATCH 127/179] [a64] Remove redundant `OPCODE_DOT_PRODUCT_{3,4}` lane-isolation The last `FADDP` writes into an `S` register, which automatically masks all the other lanes to zero. --- a64_sequences.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/a64_sequences.cc b/a64_sequences.cc index c88e201e5..3eb60510e 100644 --- a/a64_sequences.cc +++ b/a64_sequences.cc @@ -2111,11 +2111,6 @@ struct DOT_PRODUCT_3_V128 e.MOV(dest.toQ().Selem()[3], WZR); e.FADDP(dest.toQ().S4(), dest.toQ().S4(), dest.toQ().S4()); e.FADDP(dest.toS(), dest.toD().S2()); - - // Isolate lower lane - e.MOVI(Q0.D2(), RepImm(0b00'00'00'00)); - e.INS(Q0.Selem()[0], dest.toQ().Selem()[0]); - e.MOV(dest.toQ().B16(), Q0.B16()); }); } }; @@ -2134,11 +2129,6 @@ struct DOT_PRODUCT_4_V128 e.FMUL(dest.toQ().S4(), src1.S4(), src2.S4()); e.FADDP(dest.toQ().S4(), dest.toQ().S4(), dest.toQ().S4()); e.FADDP(dest.toS(), dest.toD().S2()); - - // Isolate lower lane - e.MOVI(Q0.D2(), RepImm(0b00'00'00'00)); - e.INS(Q0.Selem()[0], dest.toQ().Selem()[0]); - e.MOV(dest.toQ().B16(), Q0.B16()); }); } }; From 8b47bd8b76edadd67f54ba8af60f2e388fde6419 Mon Sep 17 00:00:00 2001 From: Wunkolo Date: Sun, 23 Jun 2024 14:38:06 -0700 Subject: [PATCH 128/179] [a64] Implement support for large stack sizes The `SUB` instruction can only encode immediates in the form of `0xFFF` or `0xFFF000`. In the case that the stack size is greater than `0xFFF`, then just align the stack-size by `0x1000` to keep the bottom 12 bits clear. --- a64_emitter.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/a64_emitter.cc b/a64_emitter.cc index e835f4aff..6ae853ff3 100644 --- a/a64_emitter.cc +++ b/a64_emitter.cc @@ -196,7 +196,14 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // IMPORTANT: any changes to the prolog must be kept in sync with // A64CodeCache, which dynamically generates exception information. // Adding or changing anything here must be matched! - const size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; + size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; + + // The SUB instruction can only encode immediates withi 0xFFF or 0xFFF000 + // If the stack size is greater than 0xFFF, then just align it to 0x1000 + if (stack_size > 0xFFF) { + stack_size = xe::align(stack_size, static_cast(0x1000)); + } + assert_true(stack_size % 16 == 0); func_info.stack_size = stack_size; stack_size_ = stack_size; From fc38ebb985f4b31d49a07da58dce6703f396dff5 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 00:55:40 +0900 Subject: [PATCH 129/179] [A64/CodeCache] Add POSIX code cache and JIT-safe copy hook - Add CopyMachineCode virtual to centralize code writes - Use MAP_JIT write-protect toggles around JIT writes on macOS ARM64 - Add POSIX A64 code cache with unwind table + icache flush --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 25 ++- src/xenia/cpu/backend/a64/a64_code_cache.h | 5 + .../cpu/backend/a64/a64_code_cache_posix.cc | 201 ++++++++++++++++++ 3 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 src/xenia/cpu/backend/a64/a64_code_cache_posix.cc diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index f484967ac..72d80aa47 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -241,12 +241,25 @@ void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, } while (generated_code_commit_mark_.compare_exchange_weak( old_commit_mark, new_commit_mark)); - // Copy code. - std::memcpy(code_write_address, machine_code, func_info.code_size.total); - - // Fill unused slots with 0x00 - std::memset(tail_write_address, 0x00, - static_cast(end_write_address - tail_write_address)); + // Copy code and fill padding while in write mode on MAP_JIT. +#if XE_PLATFORM_MAC && defined(__aarch64__) + const bool jit_write = + (generated_code_execute_base_ == generated_code_write_base_); + if (jit_write) { + pthread_jit_write_protect_np(0); + } +#endif + CopyMachineCode(code_write_address, machine_code, + func_info.code_size.total); + if (end_write_address > tail_write_address) { + std::memset(tail_write_address, 0x00, + static_cast(end_write_address - tail_write_address)); + } +#if XE_PLATFORM_MAC && defined(__aarch64__) + if (jit_write) { + pthread_jit_write_protect_np(1); + } +#endif // Notify subclasses of placed code. PlaceCode(guest_address, machine_code, func_info, code_execute_address, diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h index 2bc9ed59f..0ff678c9a 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.h +++ b/src/xenia/cpu/backend/a64/a64_code_cache.h @@ -111,6 +111,11 @@ class A64CodeCache : public CodeCache { void* code_execute_address, UnwindReservation unwind_reservation) {} + // Platform-specific code copying with JIT protection handling + virtual void CopyMachineCode(void* dest, const void* src, size_t size) { + std::memcpy(dest, src, size); + } + std::filesystem::path file_name_; xe::memory::FileMappingHandle mapping_ = xe::memory::kFileMappingHandleInvalid; diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc new file mode 100644 index 000000000..c2c8b5df7 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc @@ -0,0 +1,201 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2025 Ben Vanik. All rights reserved. + * Released under the BSD license - see LICENSE in the root for more details. + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_code_cache.h" + +#include +#include +#include +#include + +#ifdef XE_PLATFORM_MAC +#include +#include +#endif + +#include "xenia/base/assert.h" +#include "xenia/base/clock.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" +#include "xenia/cpu/function.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +// ARM64 unwind-op codes for POSIX (simplified) +typedef enum _UNWIND_OP_CODES_POSIX { + UWOP_POSIX_NOP = 0x00, + UWOP_POSIX_ALLOC_STACK = 0x01, + UWOP_POSIX_SAVE_FP_LR = 0x02, + UWOP_POSIX_SET_FP = 0x03, + UWOP_POSIX_END = 0xFF, +} UNWIND_CODE_OPS_POSIX; + +using UNWIND_CODE_POSIX = uint8_t; + +// Size of unwind info per function. +static const size_t kUnwindInfoSize = 16; + +class PosixA64CodeCache : public A64CodeCache { + public: + PosixA64CodeCache(); + ~PosixA64CodeCache() override; + + bool Initialize() override; + + void* LookupUnwindInfo(uint64_t host_pc) override; + + protected: + void CopyMachineCode(void* dest, const void* src, size_t size) override; + + private: + struct UnwindInfo { + uint64_t begin_address; + uint64_t end_address; + // Additional unwind information can be added here + }; + + UnwindReservation RequestUnwindReservation(uint8_t* entry_address) override; + void PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, void* code_execute_address, + UnwindReservation unwind_reservation) override; + + void InitializeUnwindEntry(uint8_t* unwind_entry_address, + size_t unwind_table_slot, + void* code_execute_address, + const EmitFunctionInfo& func_info); + + // Unwind table entries. + std::vector unwind_table_; + // Current number of entries in the table. + std::atomic unwind_table_count_ = {0}; +}; + +std::unique_ptr A64CodeCache::Create() { + return std::make_unique(); +} + +PosixA64CodeCache::PosixA64CodeCache() = default; + +PosixA64CodeCache::~PosixA64CodeCache() { + // Cleanup if necessary +} + +bool PosixA64CodeCache::Initialize() { + if (!A64CodeCache::Initialize()) { + return false; + } + + // Resize (not reserve) space for unwind table entries to ensure vector has + // actual elements + unwind_table_.resize(kMaximumFunctionCount); + + // Additional POSIX-specific initialization can be done here + + return true; +} + +void PosixA64CodeCache::CopyMachineCode(void* dest, const void* src, + size_t size) { + std::memcpy(dest, src, size); +} + +PosixA64CodeCache::UnwindReservation +PosixA64CodeCache::RequestUnwindReservation(uint8_t* entry_address) { + uint32_t current_count = unwind_table_count_.fetch_add(1); + assert_false(current_count >= kMaximumFunctionCount); + UnwindReservation unwind_reservation; + unwind_reservation.data_size = xe::round_up(kUnwindInfoSize, 16); + unwind_reservation.table_slot = current_count; + unwind_reservation.entry_address = entry_address; + return unwind_reservation; +} + +void PosixA64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void* code_execute_address, + UnwindReservation unwind_reservation) { + // Add unwind info. + InitializeUnwindEntry( + reinterpret_cast(unwind_reservation.entry_address), + unwind_reservation.table_slot, code_execute_address, func_info); + + // Add entry to unwind table at the reserved slot only + UnwindInfo unwind_info; + unwind_info.begin_address = reinterpret_cast(code_execute_address); + unwind_info.end_address = + unwind_info.begin_address + func_info.code_size.total; + + // Store in the reserved slot + unwind_table_[unwind_reservation.table_slot] = unwind_info; + + // Validate address alignment before cache flushing + if ((uintptr_t)code_execute_address % 4 != 0) { + XELOGW( + "PosixA64CodeCache::PlaceCode: WARNING - code address 0x{:016X} is not " + "4-byte aligned", + (uintptr_t)code_execute_address); + } + + if (func_info.code_size.total % 4 != 0) { + XELOGW( + "PosixA64CodeCache::PlaceCode: WARNING - code size {} is not 4-byte " + "aligned", + func_info.code_size.total); + } + + // Flush instruction cache +#ifdef XE_PLATFORM_MAC + // On macOS, use sys_icache_invalidate + sys_icache_invalidate(code_execute_address, func_info.code_size.total); +#else + // On Linux and other POSIX systems, use GCC builtin + __builtin___clear_cache( + static_cast(code_execute_address), + static_cast(code_execute_address) + func_info.code_size.total); +#endif +} + +void PosixA64CodeCache::InitializeUnwindEntry( + uint8_t* unwind_entry_address, size_t unwind_table_slot, + void* code_execute_address, const EmitFunctionInfo& func_info) { + // Initialize unwind information for POSIX (simplified example) + // In practice, you would populate this with proper unwind info + // based on the function prologue and epilogue. + + // NOTE: Unwind info is already stored in PlaceCode, so we don't store it + // again here to avoid the double-storage bug that was causing memory + // corruption. +} + +void* PosixA64CodeCache::LookupUnwindInfo(uint64_t host_pc) { + // Binary search the unwind table for the given program counter + size_t left = 0; + size_t right = unwind_table_count_.load(); + while (left < right) { + size_t mid = left + (right - left) / 2; + const UnwindInfo& info = unwind_table_[mid]; + if (host_pc < info.begin_address) { + right = mid; + } else if (host_pc >= info.end_address) { + left = mid + 1; + } else { + return &unwind_table_[mid]; + } + } + return nullptr; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe From db039b12951afb6c9f36559d3cfc578081190f8d Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 01:30:38 +0900 Subject: [PATCH 130/179] [A64/CodeCache] Add indirection table config and logging hooks - Introduce XE_A64_INDIRECTION_64BIT and 64-bit indirection entry sizing - Track actual/bias base pointers for ARM64 table placement - Add indirection table logging cvars + helper and static base defs --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 33 +++++++++++ src/xenia/cpu/backend/a64/a64_code_cache.h | 61 ++++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 72d80aa47..05dc2513e 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -9,12 +9,14 @@ #include "xenia/cpu/backend/a64/a64_code_cache.h" +#include #include #include #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" #include "xenia/base/clock.h" +#include "xenia/base/cvar.h" #include "xenia/base/literals.h" #include "xenia/base/logging.h" #include "xenia/base/math.h" @@ -22,6 +24,11 @@ #include "xenia/cpu/function.h" #include "xenia/cpu/module.h" +DEFINE_bool(a64_indirection_table_log, false, + "Log A64 indirection table mapping and updates.", "CPU"); +DEFINE_int32(a64_indirection_table_log_limit, 32, + "Maximum number of A64 indirection table log entries.", "CPU"); + namespace xe { namespace cpu { namespace backend { @@ -29,6 +36,32 @@ namespace a64 { using namespace xe::literals; +namespace { + +bool ShouldLogIndirectionTable() { + if (!cvars::a64_indirection_table_log) { + return false; + } + const int32_t limit = cvars::a64_indirection_table_log_limit; + if (limit <= 0) { + return false; + } + static std::atomic log_count{0}; + const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); + return count < limit; +} + +} // namespace + +// Define static constants for linking +const size_t A64CodeCache::kIndirectionTableSize; +#if XE_A64_INDIRECTION_64BIT +// On ARM64 platforms, this will be set dynamically during initialization +uintptr_t A64CodeCache::kIndirectionTableBase = 0x80000000; +#else +const uintptr_t A64CodeCache::kIndirectionTableBase; +#endif + A64CodeCache::A64CodeCache() = default; A64CodeCache::~A64CodeCache() { diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h index 0ff678c9a..4e1d07f61 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.h +++ b/src/xenia/cpu/backend/a64/a64_code_cache.h @@ -27,6 +27,12 @@ namespace cpu { namespace backend { namespace a64 { +#if XE_ARCH_ARM64 +#define XE_A64_INDIRECTION_64BIT 1 +#else +#define XE_A64_INDIRECTION_64BIT 0 +#endif + struct EmitFunctionInfo { struct _code_size { size_t prolog; @@ -76,11 +82,39 @@ class A64CodeCache : public CodeCache { GuestFunction* LookupFunction(uint64_t host_pc) override; - protected: + // Access to indirection table base for emitter + uint8_t* indirection_table_base() const { return indirection_table_base_; } + + // Returns the actual base address used for indirection table + uintptr_t indirection_table_base_address() const { + return indirection_table_actual_base_; + } +#if XE_A64_INDIRECTION_64BIT + uintptr_t indirection_table_base_bias() const { + return indirection_table_base_bias_; + } +#endif + + public: // All executable code falls within 0x80000000 to 0x9FFFFFFF, so we can // only map enough for lookups within that range. - static const size_t kIndirectionTableSize = 0x1FFFFFFF; + // Size of the indirection table in bytes. + // On ARM64 platforms we store 64-bit entries (8 bytes) per 4-byte guest slot + // for the 0x2000_0000-byte guest executable range (0x8000_0000..0xA000_0000), + // so we need 0x4000_0000 bytes to cover the full space. +#if XE_A64_INDIRECTION_64BIT + static const size_t kIndirectionTableSize = 0x40000000; // 1 GiB +#else + static const size_t kIndirectionTableSize = + 0x20000000 - 1; // 512 MiB - 1 (legacy) +#endif +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, the base address is determined dynamically at runtime + // based on where the OS allows us to allocate memory + static uintptr_t kIndirectionTableBase; +#else static const uintptr_t kIndirectionTableBase = 0x80000000; +#endif // The code range is 512MB, but we know the total code games will have is // pretty small (dozens of mb at most) and our expansion is reasonablish // so 256MB should be more than enough. @@ -125,12 +159,33 @@ class A64CodeCache : public CodeCache { xe::global_critical_region global_critical_region_; // Value that the indirection table will be initialized with upon commit. +#if XE_A64_INDIRECTION_64BIT + uint64_t indirection_default_value_ = 0xFEEDF00D; +#else uint32_t indirection_default_value_ = 0xFEEDF00D; +#endif - // Fixed at kIndirectionTableBase in host space, holding 4 byte pointers into +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, we use 64-bit pointers in the indirection table to + // handle high addresses that can't fit in 32-bit values. + using indirection_entry_t = uint64_t; + static constexpr size_t kIndirectionEntrySize = 8; +#else + // Other platforms use 32-bit pointers + using indirection_entry_t = uint32_t; + static constexpr size_t kIndirectionEntrySize = 4; +#endif + + // Fixed at kIndirectionTableBase in host space, holding pointers into // the generated code table that correspond to the PPC functions in guest // space. uint8_t* indirection_table_base_ = nullptr; + // Actual base address of the indirection table (may differ from + // kIndirectionTableBase on systems where fixed address allocation fails) + uintptr_t indirection_table_actual_base_ = 0; +#if XE_A64_INDIRECTION_64BIT + uintptr_t indirection_table_base_bias_ = 0; +#endif // Fixed at kGeneratedCodeExecuteBase and holding all generated code, growing // as needed. uint8_t* generated_code_execute_base_ = nullptr; From ddd36bbab67b984bc77a09f8585f7a9a94866e6b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 01:34:36 +0900 Subject: [PATCH 131/179] [A64/CodeCache] Add 64-bit indirection update paths - Add 64-bit default/indirection APIs and delegate on ARM64 - Commit executable ranges with 64-bit entry sizing and bounds checks - Log indirection add/commit/place updates when enabled --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 178 +++++++++++++++++++- src/xenia/cpu/backend/a64/a64_code_cache.h | 6 + 2 files changed, 175 insertions(+), 9 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 05dc2513e..1c9295037 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -155,11 +155,26 @@ bool A64CodeCache::Initialize() { } void A64CodeCache::set_indirection_default(uint32_t default_value) { +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, we extend 32-bit values to 64-bit + indirection_default_value_ = default_value; +#else + indirection_default_value_ = default_value; +#endif +} + +#if XE_A64_INDIRECTION_64BIT +void A64CodeCache::set_indirection_default_64(uint64_t default_value) { indirection_default_value_ = default_value; } +#endif void A64CodeCache::AddIndirection(uint32_t guest_address, uint32_t host_address) { +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, delegate to the 64-bit version + AddIndirection64(guest_address, host_address); +#else if (!indirection_table_base_) { return; } @@ -167,25 +182,121 @@ void A64CodeCache::AddIndirection(uint32_t guest_address, uint32_t* indirection_slot = reinterpret_cast( indirection_table_base_ + (guest_address - kIndirectionTableBase)); *indirection_slot = host_address; +#endif } -void A64CodeCache::CommitExecutableRange(uint32_t guest_low, - uint32_t guest_high) { +#if XE_A64_INDIRECTION_64BIT +void A64CodeCache::AddIndirection64(uint32_t guest_address, + uint64_t host_address) { if (!indirection_table_base_) { return; } - // Commit the memory. - xe::memory::AllocFixed( - indirection_table_base_ + (guest_low - kIndirectionTableBase), - guest_high - guest_low, xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kReadWrite); + if (guest_address < kIndirectionTableBase) { + XELOGE( + "A64CodeCache::AddIndirection64: guest_address 0x{:08X} below base " + "0x{:08X}", + guest_address, static_cast(kIndirectionTableBase)); + return; + } + + const uint64_t guest_delta = guest_address - kIndirectionTableBase; + if (guest_delta & 0x3) { + XELOGW( + "A64CodeCache::AddIndirection64: guest_address 0x{:08X} not 4-byte " + "aligned (delta=0x{:X})", + guest_address, guest_delta); + } + + // Calculate offset from the logical base (0x80000000), not from actual table + // address. + const uint64_t guest_offset = (guest_delta >> 2) * kIndirectionEntrySize; + if (guest_offset + kIndirectionEntrySize > kIndirectionTableSize) { + XELOGE( + "A64CodeCache::AddIndirection64: guest_address 0x{:08X} offset 0x{:X} " + "exceeds table size 0x{:X}", + guest_address, guest_offset, + static_cast(kIndirectionTableSize)); + return; + } + + uint64_t* indirection_slot = + reinterpret_cast(indirection_table_base_ + guest_offset); + *indirection_slot = host_address; + + if (ShouldLogIndirectionTable()) { + XELOGI( + "A64 indirection add: guest=0x{:08X} delta=0x{:X} offset=0x{:X} " + "slot=0x{:016X} host=0x{:016X}", + guest_address, guest_delta, guest_offset, + reinterpret_cast(indirection_slot), host_address); + } +} +#endif + +void A64CodeCache::CommitExecutableRange(uint32_t guest_low, + uint32_t guest_high) { + if (!indirection_table_base_) { + XELOGE("CommitExecutableRange: indirection_table_base_ is null!"); + return; + } + +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms: use offset-based addressing from guest base + // (0x80000000) + static const uintptr_t kGuestAddressBase = 0x80000000; + + // Calculate offsets from the guest address base, not the table base + if (guest_low < kGuestAddressBase) { + XELOGE( + "CommitExecutableRange: guest_low 0x{:08X} is below guest base " + "0x{:08X}", + guest_low, kGuestAddressBase); + return; + } + + uint32_t start_offset = + ((guest_low - kGuestAddressBase) >> 2) * kIndirectionEntrySize; + uint32_t size = ((guest_high - guest_low) >> 2) * kIndirectionEntrySize; + + // Sanity check bounds; the table should fully cover the XEX guest range now. + if (start_offset + size > kIndirectionTableSize) { + XELOGE( + "CommitExecutableRange: range [0x{:08X}, 0x{:08X}) exceeds table (size " + "0x{:X})", + guest_low, guest_high, (unsigned)kIndirectionTableSize); + return; + } + + // The memory should already be allocated, just fill with default value + void* target_memory = indirection_table_base_ + start_offset; + uint64_t* p = reinterpret_cast(target_memory); + uint32_t entry_count = size / kIndirectionEntrySize; + for (uint32_t i = 0; i < entry_count; i++) { + p[i] = indirection_default_value_; + } + + if (ShouldLogIndirectionTable()) { + XELOGI( + "A64 indirection commit: guest=[0x{:08X},0x{:08X}) " + "offset=0x{:X} size=0x{:X} entries={} base=0x{:016X}", + guest_low, guest_high, start_offset, size, entry_count, + static_cast(indirection_table_actual_base_)); + } +#else + // Other platforms: use 32-bit entries + uint32_t start_offset = (guest_low - kIndirectionTableBase); + uint32_t size = (guest_high - guest_low); + + xe::memory::AllocFixed(indirection_table_base_ + start_offset, size, + xe::memory::AllocationType::kCommit, + xe::memory::PageAccess::kReadWrite); - // Fill memory with the default value. uint32_t* p = reinterpret_cast(indirection_table_base_); - for (uint32_t address = guest_low; address < guest_high; ++address) { + for (uint32_t address = guest_low; address < guest_high; address += 4) { p[(address - kIndirectionTableBase) / 4] = indirection_default_value_; } +#endif } void A64CodeCache::PlaceHostCode(uint32_t guest_address, void* machine_code, @@ -303,10 +414,59 @@ void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, // Note that we do support code that doesn't have an indirection fixup, so // ignore those when we see them. if (guest_address && indirection_table_base_) { +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, map guest addresses to table offsets using logical + // base kIndirectionTableBase remains 0x80000000 for calculation purposes + + // Calculate offset from the logical guest base (0x80000000) + if (guest_address < kIndirectionTableBase) { + XELOGE( + "A64CodeCache::PlaceGuestCode: ERROR - guest_address 0x{:08X} is " + "below logical base 0x{:08X}!", + guest_address, static_cast(kIndirectionTableBase)); + return; + } + + uintptr_t guest_diff = guest_address - kIndirectionTableBase; + if (guest_diff & 0x3) { + XELOGW( + "A64CodeCache::PlaceGuestCode: guest_address 0x{:08X} not 4-byte " + "aligned (delta=0x{:X})", + guest_address, guest_diff); + } + uintptr_t guest_offset = + (guest_diff >> 2) * kIndirectionEntrySize; // 8-byte entries + uintptr_t slot_address = + reinterpret_cast(indirection_table_base_) + guest_offset; + uint64_t* indirection_slot = reinterpret_cast(slot_address); + + // Check if the slot address is within bounds + uintptr_t table_end = reinterpret_cast(indirection_table_base_) + + kIndirectionTableSize; + if (slot_address >= table_end) { + XELOGE( + "A64CodeCache::PlaceGuestCode: slot 0x{:016X} beyond table end " + "0x{:016X}", + slot_address, table_end); + return; + } + + *indirection_slot = reinterpret_cast(code_execute_address); + + if (ShouldLogIndirectionTable()) { + XELOGI( + "A64 indirection place: guest=0x{:08X} diff=0x{:X} offset=0x{:X} " + "slot=0x{:016X} host=0x{:016X}", + guest_address, guest_diff, guest_offset, slot_address, + static_cast( + reinterpret_cast(code_execute_address))); + } +#else uint32_t* indirection_slot = reinterpret_cast( indirection_table_base_ + (guest_address - kIndirectionTableBase)); *indirection_slot = uint32_t(reinterpret_cast(code_execute_address)); +#endif } } diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h index 4e1d07f61..8d0d50b8a 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.h +++ b/src/xenia/cpu/backend/a64/a64_code_cache.h @@ -65,7 +65,13 @@ class A64CodeCache : public CodeCache { bool has_indirection_table() { return indirection_table_base_ != nullptr; } void set_indirection_default(uint32_t default_value); +#if XE_A64_INDIRECTION_64BIT + void set_indirection_default_64(uint64_t default_value); +#endif void AddIndirection(uint32_t guest_address, uint32_t host_address); +#if XE_A64_INDIRECTION_64BIT + void AddIndirection64(uint32_t guest_address, uint64_t host_address); +#endif void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high); From 9b6af467440392bc3613d98f97c0531d7caea321 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 01:50:23 +0900 Subject: [PATCH 132/179] [A64/CodeCache] Allocate indirection table with ARM64 fallback - Use OS-chosen allocation on ARM64 and record actual base/bias - Fall back to OS-chosen base when fixed mapping fails - Log indirection table mapping details when enabled --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 59 +++++++++++++++++++-- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 1c9295037..2ee0bc7a2 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -87,17 +87,66 @@ A64CodeCache::~A64CodeCache() { } bool A64CodeCache::Initialize() { +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, allocate the indirection table wherever the OS allows, + // then update our base address to match. This gives us the same direct + // access pattern as x64 without needing complex offset calculations. + + indirection_table_base_ = reinterpret_cast( + xe::memory::AllocFixed(nullptr, kIndirectionTableSize, + xe::memory::AllocationType::kReserveCommit, + xe::memory::PageAccess::kReadWrite)); + + if (!indirection_table_base_) { + XELOGE("Unable to allocate indirection table at any address"); + return false; + } + + // Keep kIndirectionTableBase as 0x80000000 for offset calculations + // Store the actual allocated address separately + indirection_table_actual_base_ = + reinterpret_cast(indirection_table_base_); +#if XE_A64_INDIRECTION_64BIT + indirection_table_base_bias_ = + indirection_table_actual_base_ - + (static_cast(kIndirectionTableBase) * 2); +#endif +#else + // Other platforms: try to allocate at the preferred address first. indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( reinterpret_cast(kIndirectionTableBase), kIndirectionTableSize, xe::memory::AllocationType::kReserve, xe::memory::PageAccess::kReadWrite)); + if (!indirection_table_base_) { + XELOGW("Preferred indirection table base unavailable; falling back"); + indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( + nullptr, kIndirectionTableSize, xe::memory::AllocationType::kReserve, + xe::memory::PageAccess::kReadWrite)); + } if (!indirection_table_base_) { XELOGE("Unable to allocate code cache indirection table"); - XELOGE( - "This is likely because the {:X}-{:X} range is in use by some other " - "system DLL", - static_cast(kIndirectionTableBase), - kIndirectionTableBase + kIndirectionTableSize); + XELOGE("Tried preferred range {:X}-{:X} with fallback to OS-chosen", + static_cast(kIndirectionTableBase), + kIndirectionTableBase + kIndirectionTableSize); + return false; + } + indirection_table_actual_base_ = + reinterpret_cast(indirection_table_base_); +#if XE_A64_INDIRECTION_64BIT + indirection_table_base_bias_ = + indirection_table_actual_base_ - + (static_cast(kIndirectionTableBase) * 2); +#endif +#endif + + if (ShouldLogIndirectionTable()) { + XELOGI( + "A64 indirection table: guest_base=0x{:08X} table_base=0x{:016X} " + "size=0x{:X} entry_bytes={}", + static_cast(kIndirectionTableBase), + static_cast(indirection_table_actual_base_), + static_cast(kIndirectionTableSize), + static_cast(kIndirectionEntrySize)); } // Create mmap file. This allows us to share the code cache with the debugger. From 0cb9dfba6b72e3fdcd8c7905c62bc05ac81b9328 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 03:42:25 +0900 Subject: [PATCH 133/179] [A64/CodeCache] Fix indirection table dealloc size --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 2ee0bc7a2..46d30e231 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -66,7 +66,7 @@ A64CodeCache::A64CodeCache() = default; A64CodeCache::~A64CodeCache() { if (indirection_table_base_) { - xe::memory::DeallocFixed(indirection_table_base_, 0, + xe::memory::DeallocFixed(indirection_table_base_, kIndirectionTableSize, xe::memory::DeallocationType::kRelease); } From 107a1484600d673f20515a65b56bc86c6ace7f9e Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 05:37:34 +0900 Subject: [PATCH 134/179] [A64/CodeCache] Update code cache mapping and JIT writes - Use MAP_JIT allocations on macOS ARM64 and dealloc via AllocFixed - Fall back to OS-chosen mappings when fixed bases fail - Protect JIT writes on macOS and use dynamic execute base for lookups --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 81 ++++++++++++++++++++- src/xenia/cpu/backend/a64/a64_code_cache.h | 4 +- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 46d30e231..a8885d827 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -13,6 +13,10 @@ #include #include +#if XE_PLATFORM_MAC +#include +#endif + #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" #include "xenia/base/clock.h" @@ -72,6 +76,15 @@ A64CodeCache::~A64CodeCache() { // Unmap all views and close mapping. if (mapping_ != xe::memory::kFileMappingHandleInvalid) { +#if XE_PLATFORM_MAC && XE_ARCH_ARM64 + // On macOS ARM64, we used AllocFixed instead of MapFileView, so use + // DeallocFixed + if (generated_code_execute_base_) { + xe::memory::DeallocFixed(generated_code_execute_base_, kGeneratedCodeSize, + xe::memory::DeallocationType::kRelease); + } +#else + // Other platforms use MapFileView/UnmapFileView if (generated_code_write_base_ && generated_code_write_base_ != generated_code_execute_base_) { xe::memory::UnmapFileView(mapping_, generated_code_write_base_, @@ -81,6 +94,7 @@ A64CodeCache::~A64CodeCache() { xe::memory::UnmapFileView(mapping_, generated_code_execute_base_, kGeneratedCodeSize); } +#endif xe::memory::CloseFileMappingHandle(mapping_, file_name_); mapping_ = xe::memory::kFileMappingHandleInvalid; } @@ -161,10 +175,32 @@ bool A64CodeCache::Initialize() { // Map generated code region into the file. Pages are committed as required. if (xe::memory::IsWritableExecutableMemoryPreferred()) { +#if XE_PLATFORM_MAC && XE_ARCH_ARM64 + // On macOS ARM64, always use OS-chosen MAP_JIT memory. + generated_code_execute_base_ = reinterpret_cast( + xe::memory::AllocFixed(nullptr, kGeneratedCodeSize, + xe::memory::AllocationType::kReserveCommit, + xe::memory::PageAccess::kExecuteReadWrite)); + generated_code_write_base_ = generated_code_execute_base_; + if (!generated_code_execute_base_ || !generated_code_write_base_) { + XELOGE("Unable to allocate code cache generated code storage"); + return false; + } + // On macOS ARM64, verify the memory is properly allocated for MAP_JIT +#else generated_code_execute_base_ = reinterpret_cast(xe::memory::MapFileView( mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, 0)); + if (!generated_code_execute_base_) { + XELOGW( + "Fixed address mapping for generated code failed, trying OS-chosen " + "address"); + generated_code_execute_base_ = + reinterpret_cast(xe::memory::MapFileView( + mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kExecuteReadWrite, 0)); + } generated_code_write_base_ = generated_code_execute_base_; if (!generated_code_execute_base_ || !generated_code_write_base_) { XELOGE("Unable to allocate code cache generated code storage"); @@ -175,15 +211,45 @@ bool A64CodeCache::Initialize() { uint64_t(kGeneratedCodeExecuteBase + kGeneratedCodeSize)); return false; } +#endif } else { +#if XE_PLATFORM_MAC && XE_ARCH_ARM64 + // On macOS ARM64, always use OS-chosen addresses for the views. + generated_code_execute_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kExecuteReadOnly, 0)); + generated_code_write_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kReadWrite, 0)); + if (!generated_code_execute_base_ || !generated_code_write_base_) { + XELOGE("Unable to allocate code cache generated code storage"); + return false; + } +#else generated_code_execute_base_ = reinterpret_cast(xe::memory::MapFileView( mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); + if (!generated_code_execute_base_) { + XELOGW( + "Fixed address mapping for execute code failed, trying OS-chosen " + "address"); + generated_code_execute_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kExecuteReadOnly, 0)); + } generated_code_write_base_ = reinterpret_cast(xe::memory::MapFileView( mapping_, reinterpret_cast(kGeneratedCodeWriteBase), kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); + if (!generated_code_write_base_) { + XELOGW( + "Fixed address mapping for write code failed, trying OS-chosen " + "address"); + generated_code_write_base_ = reinterpret_cast( + xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, + xe::memory::PageAccess::kReadWrite, 0)); + } if (!generated_code_execute_base_ || !generated_code_write_base_) { XELOGE("Unable to allocate code cache generated code storage"); XELOGE( @@ -195,6 +261,7 @@ bool A64CodeCache::Initialize() { uint64_t(kGeneratedCodeWriteBase + kGeneratedCodeSize)); return false; } +#endif } // Preallocate the function map to a large, reasonable size. @@ -558,14 +625,24 @@ uint32_t A64CodeCache::PlaceData(const void* data, size_t length) { } while (generated_code_commit_mark_.compare_exchange_weak(old_commit_mark, new_commit_mark)); - // Copy code. + // Copy data. +#if XE_PLATFORM_MAC && defined(__aarch64__) + if (generated_code_execute_base_ == generated_code_write_base_) { + pthread_jit_write_protect_np(0); + std::memcpy(data_address, data, length); + pthread_jit_write_protect_np(1); + } else { + std::memcpy(data_address, data, length); + } +#else std::memcpy(data_address, data, length); +#endif return uint32_t(uintptr_t(data_address)); } GuestFunction* A64CodeCache::LookupFunction(uint64_t host_pc) { - uint32_t key = uint32_t(host_pc - kGeneratedCodeExecuteBase); + uint32_t key = uint32_t(host_pc - execute_base_address()); void* fn_entry = std::bsearch( &key, generated_code_map_.data(), generated_code_map_.size() + 1, sizeof(std::pair), diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h index 8d0d50b8a..1da3f27ba 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.h +++ b/src/xenia/cpu/backend/a64/a64_code_cache.h @@ -55,7 +55,9 @@ class A64CodeCache : public CodeCache { const std::filesystem::path& file_name() const override { return file_name_; } uintptr_t execute_base_address() const override { - return kGeneratedCodeExecuteBase; + return generated_code_execute_base_ + ? reinterpret_cast(generated_code_execute_base_) + : kGeneratedCodeExecuteBase; } size_t total_size() const override { return kGeneratedCodeSize; } From 24db01fe759c4699a456f7eadb2c0f259259c177 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 06:02:46 +0900 Subject: [PATCH 135/179] [A64/CodeCache] Use stable code cache mapping name --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index a8885d827..30b4df498 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -164,7 +164,7 @@ bool A64CodeCache::Initialize() { } // Create mmap file. This allows us to share the code cache with the debugger. - file_name_ = fmt::format("xenia_code_cache_{}", Clock::QueryHostTickCount()); + file_name_ = fmt::format("xenia_code_cache"); mapping_ = xe::memory::CreateFileMappingHandle( file_name_, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, false); From 4f3f14a3d98d57211fdb06f01c5d28ac5bccca16 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 06:33:54 +0900 Subject: [PATCH 136/179] [A64/CodeCache] Silence unused low_mark warning --- src/xenia/cpu/backend/a64/a64_code_cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 30b4df498..1bc937cf0 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -432,7 +432,7 @@ void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, void*& code_write_address_out) { // Hold a lock while we bump the pointers up. This is important as the // unwind table requires entries AND code to be sorted in order. - size_t low_mark; + [[maybe_unused]] size_t low_mark; size_t high_mark; uint8_t* code_execute_address; UnwindReservation unwind_reservation; From eb15c34234902ac41703fa587da8387ac87ff606 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:20:09 +0900 Subject: [PATCH 137/179] [A64/Tracers] Use standard ARM NEON include --- src/xenia/cpu/backend/a64/a64_tracers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_tracers.h b/src/xenia/cpu/backend/a64/a64_tracers.h index 62b740356..d004aef6c 100644 --- a/src/xenia/cpu/backend/a64/a64_tracers.h +++ b/src/xenia/cpu/backend/a64/a64_tracers.h @@ -10,7 +10,7 @@ #ifndef XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ #define XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ -#include +#include #include namespace xe { From b18442e19769c33ae3b7ea2ebefd0bd5ca5708e5 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:20:31 +0900 Subject: [PATCH 138/179] [A64/Function] Initialize macOS JIT threads before thunk - Add ARM64 macOS thread-local JIT init using pthread_jit_write_protect_np - Ensure CallImpl enables execute mode before host->guest thunk --- src/xenia/cpu/backend/a64/a64_function.cc | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/xenia/cpu/backend/a64/a64_function.cc b/src/xenia/cpu/backend/a64/a64_function.cc index 9167bde7c..0738a1bc4 100644 --- a/src/xenia/cpu/backend/a64/a64_function.cc +++ b/src/xenia/cpu/backend/a64/a64_function.cc @@ -9,10 +9,31 @@ #include "xenia/cpu/backend/a64/a64_function.h" +#ifdef XE_PLATFORM_MAC +#include +#include +#include +#include +#endif + +#include "xenia/base/logging.h" #include "xenia/cpu/backend/a64/a64_backend.h" #include "xenia/cpu/processor.h" #include "xenia/cpu/thread_state.h" +#if XE_PLATFORM_MAC && defined(__aarch64__) +thread_local bool jit_thread_initialized = false; + +// Initialize JIT execution for the current thread +static void InitializeJITThread() { + if (!jit_thread_initialized) { + // Ensure this thread can execute JIT code by setting execute mode + pthread_jit_write_protect_np(1); // Enable execute, disable write + jit_thread_initialized = true; + } +} +#endif + namespace xe { namespace cpu { namespace backend { @@ -31,11 +52,20 @@ void A64Function::Setup(uint8_t* machine_code, size_t machine_code_length) { } bool A64Function::CallImpl(ThreadState* thread_state, uint32_t return_address) { +#if XE_PLATFORM_MAC && defined(__aarch64__) + // Initialize JIT execution for this thread + // This ensures pthread_jit_write_protect_np is set correctly for execution + InitializeJITThread(); +#endif + auto backend = reinterpret_cast(thread_state->processor()->backend()); auto thunk = backend->host_to_guest_thunk(); + + // Make the actual thunk call thunk(machine_code_, thread_state->context(), reinterpret_cast(uintptr_t(return_address))); + return true; } From f5d02886316b348fd8e5a1ed158880ce51b63ea5 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:26:29 +0900 Subject: [PATCH 139/179] [A64/Backend] Improve capstone init error reporting - Capture cs_open() error code and print it before asserting --- src/xenia/cpu/backend/a64/a64_backend.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index 8b3f3a6f7..6f18034a5 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -70,10 +70,15 @@ class A64ThunkEmitter : public A64Emitter { }; A64Backend::A64Backend() : Backend(), code_cache_(nullptr) { - if (cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != - CS_ERR_OK) { + cs_err err = cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_); + if (err) { + printf("Failed on cs_open() with error returned: %u\n", err); assert_always("Failed to initialize capstone"); } + // if (cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != + // CS_ERR_OK) { + // assert_always("Failed to initialize capstone"); + // } cs_option(capstone_handle_, CS_OPT_SYNTAX, CS_OPT_SYNTAX_INTEL); cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_ON); cs_option(capstone_handle_, CS_OPT_SKIPDATA, CS_OPT_OFF); From 03335d56e84337ba158e3854e52e02a421085d3d Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:28:10 +0900 Subject: [PATCH 140/179] [A64/Backend] Use 64-bit default indirection on ARM64 - Route ResolveFunction thunk into 64-bit indirection entries when enabled --- src/xenia/cpu/backend/a64/a64_backend.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index 6f18034a5..b102594cd 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -123,11 +123,16 @@ bool A64Backend::Initialize(Processor* processor) { guest_to_host_thunk_ = thunk_emitter.EmitGuestToHostThunk(); resolve_function_thunk_ = thunk_emitter.EmitResolveFunctionThunk(); - // Set the code cache to use the ResolveFunction thunk for default - // indirections. +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, we use 64-bit addresses and the indirection table now + // supports 64-bit entries, so we can store the actual thunk address directly. + static_cast(code_cache_.get()) + ->set_indirection_default_64(uint64_t(resolve_function_thunk_)); +#else assert_zero(uint64_t(resolve_function_thunk_) & 0xFFFFFFFF00000000ull); code_cache_->set_indirection_default( uint32_t(uint64_t(resolve_function_thunk_))); +#endif // Allocate some special indirections. code_cache_->CommitExecutableRange(0x9FFF0000, 0x9FFFFFFF); From 74180887eed4eb88b7c12612e67be6178a72d84e Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:42:08 +0900 Subject: [PATCH 141/179] [A64/Backend] Use brace-init for capstone structs --- src/xenia/cpu/backend/a64/a64_backend.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index b102594cd..7e3d3f512 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -343,8 +343,8 @@ uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, auto machine_code_ptr = reinterpret_cast(current_pc); size_t remaining_machine_code_size = 64; uint64_t host_address = current_pc; - cs_insn insn = {0}; - cs_detail all_detail = {0}; + cs_insn insn = {}; + cs_detail all_detail = {}; insn.detail = &all_detail; cs_disasm_iter(capstone_handle_, &machine_code_ptr, &remaining_machine_code_size, &host_address, &insn); From ad7a10edc545c56b72788c041e8e0ee2ce16ac3f Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:43:16 +0900 Subject: [PATCH 142/179] [A64/Backend] Log access violations with guest offsets - Derive guest PC from machine code offsets for AV diagnostics - Include fault address and ARM64 register context when available --- src/xenia/cpu/backend/a64/a64_backend.cc | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index 7e3d3f512..759410320 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -451,6 +451,39 @@ bool A64Backend::ExceptionCallbackThunk(Exception* ex, void* data) { } bool A64Backend::ExceptionCallback(Exception* ex) { + if (ex->code() == Exception::Code::kAccessViolation) { + const uint64_t host_pc = ex->pc(); + const uint64_t fault_address = ex->fault_address(); + uint64_t guest_pc = 0; + uint32_t host_offset = 0; + auto function = code_cache_->LookupFunction(host_pc); + if (function && function->machine_code()) { + const uint64_t function_pc = + reinterpret_cast(function->machine_code()); + host_offset = static_cast(host_pc - function_pc); + if (const auto* entry = function->LookupMachineCodeOffset(host_offset)) { + guest_pc = entry->guest_address; + } + } +#if XE_ARCH_ARM64 + auto* thread_context = ex->thread_context(); + XELOGE( + "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " + "fault=0x{:016X} op={} x21=0x{:016X} x27=0x{:016X} x28=0x{:016X}", + host_pc, guest_pc, host_offset, fault_address, + static_cast(ex->access_violation_operation()), + thread_context ? thread_context->x[21] : 0, + thread_context ? thread_context->x[27] : 0, + thread_context ? thread_context->x[28] : 0); +#else + XELOGE( + "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " + "fault=0x{:016X} op={}", + host_pc, guest_pc, host_offset, fault_address, + static_cast(ex->access_violation_operation())); +#endif + return false; + } if (ex->code() != Exception::Code::kIllegalInstruction) { // We only care about illegal instructions. Other things will be handled by // other handlers (probably). If nothing else picks it up we'll be called From dba51ae0b2d01d50b9b423990dd79faa49426dba Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:44:08 +0900 Subject: [PATCH 143/179] [A64/Backend] Guard resolve thunk against null targets - Return early if the resolved address is null --- src/xenia/cpu/backend/a64/a64_backend.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index 759410320..46b882023 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -664,7 +664,11 @@ ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { // Reload context register LDP(ZR, X0, SP, POST_INDEXED, 16); + oaknut::Label resolve_failed; + CBZ(X16, resolve_failed); BR(X16); + l(resolve_failed); + RET(); code_offsets.tail = offset(); From ed319bccec324c16f9a84a38adb5f89a09b72dda Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:56:12 +0900 Subject: [PATCH 144/179] [A64/Emitter] Move FP immediate helpers into emitter - Relocate f32/f64 fimm8 conversion helpers from a64_util.h - Keep helpers local to emitter translation code --- src/xenia/cpu/backend/a64/a64_emitter.cc | 69 ++++++++++++++++++++++- src/xenia/cpu/backend/a64/a64_util.h | 72 +----------------------- 2 files changed, 69 insertions(+), 72 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 6ae853ff3..411b0a664 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -8,7 +8,6 @@ */ #include "xenia/cpu/backend/a64/a64_emitter.h" -#include "xenia/cpu/backend/a64/a64_util.h" #include @@ -818,6 +817,74 @@ uintptr_t A64Emitter::GetVConstPtr(VConst id) const { return GetVConstPtr() + GetVConstOffset(id); } +// Attempts to convert an fp32 bit-value into an fp8-immediate value for FMOV +// returns false if the value cannot be represented +// C2.2.3 Modified immediate constants in A64 floating-point instructions +// abcdefgh +// V +// aBbbbbbc defgh000 00000000 00000000 +// B = NOT(b) +static bool f32_to_fimm8(uint32_t u32, oaknut::FImm8& fp8) { + const uint32_t sign = (u32 >> 31) & 1; + int32_t exp = ((u32 >> 23) & 0xff) - 127; + int64_t mantissa = u32 & 0x7fffff; + + // Too many mantissa bits + if (mantissa & 0x7ffff) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 19; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + // exp = (NOT(b):c:d) - 3 + exp = ((exp + 3) & 0b111) ^ 0b100; + + fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); + return true; +} + +// Attempts to convert an fp64 bit-value into an fp8-immediate value for FMOV +// returns false if the value cannot be represented +// C2.2.3 Modified immediate constants in A64 floating-point instructions +// abcdefgh +// V +// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 +// B = NOT(b) +static bool f64_to_fimm8(uint64_t u64, oaknut::FImm8& fp8) { + const uint32_t sign = (u64 >> 63) & 1; + int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; + int64_t mantissa = u64 & 0xfffffffffffffULL; + + // Too many mantissa bits + if (mantissa & 0xffffffffffffULL) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 48; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + // exp = (NOT(b):c:d) - 3 + exp = ((exp + 3) & 0b111) ^ 0b100; + + fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); + return true; +} + // Implies possible StashV(0, ...)! void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { if (!v.low && !v.high) { diff --git a/src/xenia/cpu/backend/a64/a64_util.h b/src/xenia/cpu/backend/a64/a64_util.h index 0b950b8ae..e3a34ac00 100644 --- a/src/xenia/cpu/backend/a64/a64_util.h +++ b/src/xenia/cpu/backend/a64/a64_util.h @@ -17,77 +17,7 @@ namespace xe { namespace cpu { namespace backend { -namespace a64 { - -// Attempts to convert an fp32 bit-value into an fp8-immediate value for FMOV -// returns false if the value cannot be represented -// C2.2.3 Modified immediate constants in A64 ing-point instructions -// abcdefgh -// V -// aBbbbbbc defgh000 00000000 00000000 -// B = NOT(b) -constexpr bool f32_to_fimm8(uint32_t u32, oaknut::FImm8& fp8) { - const uint32_t sign = (u32 >> 31) & 1; - int32_t exp = ((u32 >> 23) & 0xff) - 127; - int64_t mantissa = u32 & 0x7fffff; - - // Too many mantissa bits - if (mantissa & 0x7ffff) { - return false; - } - // Too many exp bits - if (exp < -3 || exp > 4) { - return false; - } - - // mantissa = (16 + e:f:g:h) / 16. - mantissa >>= 19; - if ((mantissa & 0b1111) != mantissa) { - return false; - } - - // exp = (NOT(b):c:d) - 3 - exp = ((exp + 3) & 0b111) ^ 0b100; - - fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); - return true; -} - -// Attempts to convert an fp64 bit-value into an fp8-immediate value for FMOV -// returns false if the value cannot be represented -// C2.2.3 Modified immediate constants in A64 floating-point instructions -// abcdefgh -// V -// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 -// B = NOT(b) -constexpr bool f64_to_fimm8(uint64_t u64, oaknut::FImm8& fp8) { - const uint32_t sign = (u64 >> 63) & 1; - int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; - int64_t mantissa = u64 & 0xfffffffffffffULL; - - // Too many mantissa bits - if (mantissa & 0xffffffffffffULL) { - return false; - } - // Too many exp bits - if (exp < -3 || exp > 4) { - return false; - } - - // mantissa = (16 + e:f:g:h) / 16. - mantissa >>= 48; - if ((mantissa & 0b1111) != mantissa) { - return false; - } - - // exp = (NOT(b):c:d) - 3 - exp = ((exp + 3) & 0b111) ^ 0b100; - - fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); - return true; -} - -} // namespace a64 +namespace a64 {} // namespace a64 } // namespace backend } // namespace cpu } // namespace xe From a6bacef4d3ab37da5c363dccf2e5be6ff6c9bf15 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:58:04 +0900 Subject: [PATCH 145/179] [A64/Op] Fix enum arithmetic in key encoding - Use explicit static_casts when combining opcode signature types --- src/xenia/cpu/backend/a64/a64_op.h | 32 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_op.h b/src/xenia/cpu/backend/a64/a64_op.h index 2b2f58932..371f1a034 100644 --- a/src/xenia/cpu/backend/a64/a64_op.h +++ b/src/xenia/cpu/backend/a64/a64_op.h @@ -36,13 +36,20 @@ enum KeyType { KEY_TYPE_L = OPCODE_SIG_TYPE_L, KEY_TYPE_O = OPCODE_SIG_TYPE_O, KEY_TYPE_S = OPCODE_SIG_TYPE_S, - KEY_TYPE_V_I8 = OPCODE_SIG_TYPE_V + INT8_TYPE, - KEY_TYPE_V_I16 = OPCODE_SIG_TYPE_V + INT16_TYPE, - KEY_TYPE_V_I32 = OPCODE_SIG_TYPE_V + INT32_TYPE, - KEY_TYPE_V_I64 = OPCODE_SIG_TYPE_V + INT64_TYPE, - KEY_TYPE_V_F32 = OPCODE_SIG_TYPE_V + FLOAT32_TYPE, - KEY_TYPE_V_F64 = OPCODE_SIG_TYPE_V + FLOAT64_TYPE, - KEY_TYPE_V_V128 = OPCODE_SIG_TYPE_V + VEC128_TYPE, + KEY_TYPE_V_I8 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT8_TYPE), + KEY_TYPE_V_I16 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT16_TYPE), + KEY_TYPE_V_I32 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT32_TYPE), + KEY_TYPE_V_I64 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT64_TYPE), + KEY_TYPE_V_F32 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(FLOAT32_TYPE), + KEY_TYPE_V_F64 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(FLOAT64_TYPE), + KEY_TYPE_V_V128 = + static_cast(OPCODE_SIG_TYPE_V) + static_cast(VEC128_TYPE), }; #pragma pack(push, 1) @@ -64,19 +71,20 @@ union InstrKey { InstrKey(const Instr* i) : value(0) { opcode = i->opcode->num; uint32_t sig = i->opcode->signature; - dest = - GET_OPCODE_SIG_TYPE_DEST(sig) ? OPCODE_SIG_TYPE_V + i->dest->type : 0; + dest = GET_OPCODE_SIG_TYPE_DEST(sig) ? static_cast(OPCODE_SIG_TYPE_V) + + static_cast(i->dest->type) + : 0; src1 = GET_OPCODE_SIG_TYPE_SRC1(sig); if (src1 == OPCODE_SIG_TYPE_V) { - src1 += i->src1.value->type; + src1 += static_cast(i->src1.value->type); } src2 = GET_OPCODE_SIG_TYPE_SRC2(sig); if (src2 == OPCODE_SIG_TYPE_V) { - src2 += i->src2.value->type; + src2 += static_cast(i->src2.value->type); } src3 = GET_OPCODE_SIG_TYPE_SRC3(sig); if (src3 == OPCODE_SIG_TYPE_V) { - src3 += i->src3.value->type; + src3 += static_cast(i->src3.value->type); } } From c8487d641a74072056cc6bc7821c73aa68959aa0 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 09:59:18 +0900 Subject: [PATCH 146/179] [A64/Op] Relax constant accessors and mark temp regs - Drop redundant const on constant() return values - Mark temp reg helpers maybe_unused to avoid warnings --- src/xenia/cpu/backend/a64/a64_op.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_op.h b/src/xenia/cpu/backend/a64/a64_op.h index 371f1a034..d50bd0bb3 100644 --- a/src/xenia/cpu/backend/a64/a64_op.h +++ b/src/xenia/cpu/backend/a64/a64_op.h @@ -200,28 +200,28 @@ struct ValueOp : Op, KEY_TYPE> { struct I8Op : ValueOp { typedef ValueOp BASE; - const int8_t constant() const { + int8_t constant() const { assert_true(BASE::is_constant); return BASE::value->constant.i8; } }; struct I16Op : ValueOp { typedef ValueOp BASE; - const int16_t constant() const { + int16_t constant() const { assert_true(BASE::is_constant); return BASE::value->constant.i16; } }; struct I32Op : ValueOp { typedef ValueOp BASE; - const int32_t constant() const { + int32_t constant() const { assert_true(BASE::is_constant); return BASE::value->constant.i32; } }; struct I64Op : ValueOp { typedef ValueOp BASE; - const int64_t constant() const { + int64_t constant() const { assert_true(BASE::is_constant); return BASE::value->constant.i64; } @@ -239,14 +239,14 @@ struct I64Op : ValueOp { }; struct F32Op : ValueOp { typedef ValueOp BASE; - const float constant() const { + float constant() const { assert_true(BASE::is_constant); return BASE::value->constant.f32; } }; struct F64Op : ValueOp { typedef ValueOp BASE; - const double constant() const { + double constant() const { assert_true(BASE::is_constant); return BASE::value->constant.f64; } @@ -386,11 +386,11 @@ struct I : DestField { template static const T GetTempReg(A64Emitter& e); template <> -const WReg GetTempReg(A64Emitter& e) { +[[maybe_unused]] const WReg GetTempReg(A64Emitter& e) { return W0; } template <> -const XReg GetTempReg(A64Emitter& e) { +[[maybe_unused]] const XReg GetTempReg(A64Emitter& e) { return X0; } From b8ae35085c822c95d858d349ccb6066cae5f4958 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:00:51 +0900 Subject: [PATCH 147/179] [A64/Assembler] Tidy capstone disassembly setup - Drop Intel syntax option for ARM64 disassembly - Use brace-init for cs_insn --- src/xenia/cpu/backend/a64/a64_assembler.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc index 280b82468..45e89270c 100644 --- a/src/xenia/cpu/backend/a64/a64_assembler.cc +++ b/src/xenia/cpu/backend/a64/a64_assembler.cc @@ -38,7 +38,7 @@ A64Assembler::A64Assembler(A64Backend* backend) CS_ERR_OK) { assert_always("Failed to initialize capstone"); } - cs_option(capstone_handle_, CS_OPT_SYNTAX, CS_OPT_SYNTAX_INTEL); + // Remove Intel syntax option as it's not applicable to ARM64 cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_OFF); } @@ -116,7 +116,7 @@ void A64Assembler::DumpMachineCode( const uint8_t* code_ptr = reinterpret_cast(machine_code); size_t remaining_code_size = code_size; uint64_t address = uint64_t(machine_code); - cs_insn insn = {0}; + cs_insn insn = {}; while (remaining_code_size && cs_disasm_iter(capstone_handle_, &code_ptr, &remaining_code_size, &address, &insn)) { From 652eeac197066f5a1269ae417311745a814916b5 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:01:53 +0900 Subject: [PATCH 148/179] [A64/Assembler] Add 64-bit indirection on ARM64 - Use AddIndirection64 when host code addresses exceed 32-bit --- src/xenia/cpu/backend/a64/a64_assembler.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc index 45e89270c..4ee965616 100644 --- a/src/xenia/cpu/backend/a64/a64_assembler.cc +++ b/src/xenia/cpu/backend/a64/a64_assembler.cc @@ -93,13 +93,19 @@ bool A64Assembler::Assemble(GuestFunction* function, HIRBuilder* builder, function->set_debug_info(std::move(debug_info)); static_cast(function)->Setup( reinterpret_cast(machine_code), code_size); - // Install into indirection table. const uint64_t host_address = reinterpret_cast(machine_code); +#if XE_A64_INDIRECTION_64BIT + // On ARM64 platforms, machine code might be allocated in high address space. + // Use the 64-bit version of AddIndirection to store the full address. + reinterpret_cast(backend_->code_cache()) + ->AddIndirection64(function->address(), host_address); +#else assert_true((host_address >> 32) == 0); reinterpret_cast(backend_->code_cache()) ->AddIndirection(function->address(), static_cast(host_address)); +#endif return true; } From f0b87fa1c7fe8a2c877133108854e7e4aa3fa24c Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:02:41 +0900 Subject: [PATCH 149/179] [A64/Assembler] Fix source map index comparisons - Cast source_map size to avoid signed/unsigned warnings --- src/xenia/cpu/backend/a64/a64_assembler.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc index 4ee965616..262de2849 100644 --- a/src/xenia/cpu/backend/a64/a64_assembler.cc +++ b/src/xenia/cpu/backend/a64/a64_assembler.cc @@ -130,11 +130,11 @@ void A64Assembler::DumpMachineCode( auto code_offset = uint32_t(code_ptr - reinterpret_cast(machine_code)); if (code_offset >= next_code_offset && - source_map_index < source_map.size()) { + source_map_index < static_cast(source_map.size())) { auto& source_map_entry = source_map[source_map_index]; str->AppendFormat("{:08X} ", source_map_entry.guest_address); ++source_map_index; - next_code_offset = source_map_index < source_map.size() + next_code_offset = source_map_index < static_cast(source_map.size()) ? source_map[source_map_index].code_offset : UINT_MAX; } else { From b6b45f9977662a440088ce28aeee3717a8eb1107 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:10:45 +0900 Subject: [PATCH 150/179] [A64/Emitter] Emit BTI and robust stack pointer adjusts - Add EmitBtiJc and place BTI in function prologs - Use AdjustStackPointer for large stack sizes in prolog/epilog - Reuse AdjustStackPointer for tail-call stack restoration --- src/xenia/cpu/backend/a64/a64_emitter.cc | 48 ++++++++++++++++++------ src/xenia/cpu/backend/a64/a64_emitter.h | 2 + 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 411b0a664..3e8a7a547 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -58,6 +58,33 @@ using xe::cpu::hir::Instr; using namespace xe::literals; using namespace oaknut::util; +namespace { + +void AdjustStackPointer(A64Emitter& emitter, size_t stack_size, bool add) { + if (!stack_size) { + return; + } + const uint64_t size_u64 = static_cast(stack_size); + const bool imm_valid = size_u64 <= 0xFFF || + ((size_u64 & 0xFFF) == 0 && (size_u64 >> 12) <= 0xFFF); + if (imm_valid) { + if (add) { + emitter.ADD(SP, SP, size_u64); + } else { + emitter.SUB(SP, SP, size_u64); + } + return; + } + emitter.MOV(X15, size_u64); + if (add) { + emitter.ADD(SP, SP, X15); + } else { + emitter.SUB(SP, SP, X15); + } +} + +} // namespace + static const size_t kStashOffset = 32; // static const size_t kStashOffsetHigh = 32 + 32; @@ -156,6 +183,8 @@ void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, return new_execute_address; } +void A64Emitter::EmitBtiJc() { dw(0xD503241F); } + bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { oaknut::Label epilog_label; epilog_label_ = &epilog_label; @@ -195,22 +224,17 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // IMPORTANT: any changes to the prolog must be kept in sync with // A64CodeCache, which dynamically generates exception information. // Adding or changing anything here must be matched! - size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; - - // The SUB instruction can only encode immediates withi 0xFFF or 0xFFF000 - // If the stack size is greater than 0xFFF, then just align it to 0x1000 - if (stack_size > 0xFFF) { - stack_size = xe::align(stack_size, static_cast(0x1000)); - } - + const size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; assert_true(stack_size % 16 == 0); func_info.stack_size = stack_size; stack_size_ = stack_size; + EmitBtiJc(); + STP(X29, X30, SP, PRE_INDEXED, -16); MOV(X29, SP); - SUB(SP, SP, (uint32_t)stack_size); + AdjustStackPointer(*this, stack_size, false); code_offsets.prolog_stack_alloc = offset(); code_offsets.body = offset(); @@ -292,7 +316,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { code_offsets.epilog = offset(); - ADD(SP, SP, (uint32_t)stack_size); + AdjustStackPointer(*this, stack_size, true); MOV(SP, X29); LDP(X29, X30, SP, POST_INDEXED, 16); @@ -455,7 +479,7 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { // Pass the callers return address over. LDR(X0, SP, StackLayout::GUEST_RET_ADDR); - ADD(SP, SP, static_cast(stack_size())); + AdjustStackPointer(*this, stack_size(), true); MOV(SP, X29); LDP(X29, X30, SP, POST_INDEXED, 16); @@ -505,7 +529,7 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, // Pass the callers return address over. LDR(X0, SP, StackLayout::GUEST_RET_ADDR); - ADD(SP, SP, static_cast(stack_size())); + AdjustStackPointer(*this, stack_size(), true); MOV(SP, X29); LDP(X29, X30, SP, POST_INDEXED, 16); diff --git a/src/xenia/cpu/backend/a64/a64_emitter.h b/src/xenia/cpu/backend/a64/a64_emitter.h index 629c67a4b..9330b41dc 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.h +++ b/src/xenia/cpu/backend/a64/a64_emitter.h @@ -168,6 +168,8 @@ class A64Emitter : public oaknut::VectorCodeGenerator { r = oaknut::QReg(idx); } + void EmitBtiJc(); + // Gets(and possibly create) an HIR label with the specified name oaknut::Label* lookup_label(const char* label_name) { return &label_lookup_[label_name]; From bdf24adeb7657365887625da5524c96e3c24024a Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:23:16 +0900 Subject: [PATCH 151/179] [A64/Emitter] Add TraceOnInstruction trap logging - Add TrapLogRegs to dump registers, memory windows, and disassembly - Wire trap type 27 to the TraceOnInstruction logger - Add byte-order and disassembler helpers for trace output --- src/xenia/cpu/backend/a64/a64_emitter.cc | 316 ++++++++++++++++++++++- 1 file changed, 315 insertions(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 3e8a7a547..a859ddc5a 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -11,12 +11,14 @@ #include +#include #include #include #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" #include "xenia/base/atomic.h" +#include "xenia/base/byte_order.h" #include "xenia/base/debugging.h" #include "xenia/base/literals.h" #include "xenia/base/logging.h" @@ -32,6 +34,7 @@ #include "xenia/cpu/cpu_flags.h" #include "xenia/cpu/function.h" #include "xenia/cpu/function_debug_info.h" +#include "xenia/cpu/ppc/ppc_opcode_info.h" #include "xenia/cpu/processor.h" #include "xenia/cpu/symbol.h" #include "xenia/cpu/thread_state.h" @@ -394,8 +397,316 @@ uint64_t TrapDebugPrint(void* raw_context, uint64_t address) { return 0; } +uint64_t TrapLogRegs(void* raw_context, uint64_t address) { + static volatile int32_t log_count = 0; + if (xe::atomic_inc(&log_count) > 8) { + return 0; + } + auto guest_context = reinterpret_cast(raw_context); + if (!guest_context) { + return 0; + } + auto thread_state = guest_context->thread_state; + XELOGI( + "TraceOnInstruction 0x{:08X}: r3=0x{:016X} r4=0x{:016X} r11=0x{:016X} " + "r30=0x{:016X} r31=0x{:016X} lr=0x{:016X} ctr=0x{:016X}", + static_cast(cvars::break_on_instruction), guest_context->r[3], + guest_context->r[4], guest_context->r[11], guest_context->r[30], + guest_context->r[31], guest_context->lr, guest_context->ctr); + if (thread_state) { + auto memory = thread_state->memory(); + if (memory) { + auto page_access_to_string = [](xe::memory::PageAccess access) { + switch (access) { + case xe::memory::PageAccess::kNoAccess: + return "no-access"; + case xe::memory::PageAccess::kReadOnly: + return "read-only"; + case xe::memory::PageAccess::kReadWrite: + return "read-write"; + case xe::memory::PageAccess::kExecuteReadOnly: + return "exec-read"; + case xe::memory::PageAccess::kExecuteReadWrite: + return "exec-read-write"; + } + return "unknown"; + }; + auto heap_type_to_string = [](HeapType type) { + switch (type) { + case HeapType::kGuestVirtual: + return "guest-virtual"; + case HeapType::kGuestXex: + return "guest-xex"; + case HeapType::kGuestPhysical: + return "guest-physical"; + case HeapType::kHostPhysical: + return "host-physical"; + } + return "unknown"; + }; + auto can_read_guest = [&](uint32_t addr) -> bool { + if (!addr) { + return false; + } + auto* heap = memory->LookupHeap(addr); + if (!heap) { + return false; + } + return heap->QueryRangeAccess(addr, addr) != + xe::memory::PageAccess::kNoAccess; + }; + auto log_guest_bytes = [&](uint32_t addr, const char* label) { + if (!can_read_guest(addr)) { + auto* heap = memory->LookupHeap(addr); + XELOGI( + "TraceOnInstruction {}: addr=0x{:08X} unreadable heap={} " + "access={}", + label, addr, + heap ? heap_type_to_string(heap->heap_type()) : "none", + heap ? page_access_to_string(heap->QueryRangeAccess(addr, addr)) + : "no-access"); + return; + } + const auto* heap = memory->LookupHeap(addr); + const uint8_t* host_ptr = nullptr; + if (heap && heap->heap_type() == HeapType::kGuestPhysical) { + uint32_t physical_address = memory->GetPhysicalAddress(addr); + host_ptr = memory->TranslatePhysical(physical_address); + } else { + host_ptr = memory->TranslateVirtual(addr); + } + if (!host_ptr) { + XELOGI("TraceOnInstruction {}: addr=0x{:08X} null", label, addr); + return; + } + uint8_t bytes[16] = {}; + std::memcpy(bytes, host_ptr, sizeof(bytes)); + char ascii[sizeof(bytes) + 1] = {}; + for (size_t i = 0; i < sizeof(bytes); ++i) { + uint8_t ch = bytes[i]; + ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; + } + XELOGI( + "TraceOnInstruction {}: addr=0x{:08X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", + label, addr, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], + bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); + }; + auto log_guest_string = [&](uint32_t addr, const char* label) { + if (!can_read_guest(addr)) { + return; + } + const uint8_t* ptr = memory->TranslateVirtual(addr); + if (!ptr) { + return; + } + char buffer[129] = {}; + size_t len = 0; + for (; len < sizeof(buffer) - 1; ++len) { + char ch = static_cast(ptr[len]); + if (!ch) { + break; + } + if (!std::isprint(static_cast(ch))) { + return; + } + buffer[len] = ch; + } + if (len > 0) { + XELOGI("TraceOnInstruction {}: {}", label, buffer); + } + }; + const uint32_t guest_address = static_cast(guest_context->r[4]); + const auto* heap = memory->LookupHeap(guest_address); + if (heap) { + const uint8_t* host_ptr = nullptr; + if (heap->heap_type() == HeapType::kGuestPhysical) { + uint32_t physical_address = memory->GetPhysicalAddress(guest_address); + host_ptr = memory->TranslatePhysical(physical_address); + } else { + host_ptr = memory->TranslateVirtual(guest_address); + } + if (host_ptr) { + uint8_t bytes[16] = {}; + std::memcpy(bytes, host_ptr, sizeof(bytes)); + char ascii[sizeof(bytes) + 1] = {}; + for (size_t i = 0; i < sizeof(bytes); ++i) { + uint8_t ch = bytes[i]; + ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; + } + XELOGI( + "TraceOnInstruction mem[r4]=0x{:08X}: {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", + guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], + bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); + } + } + + auto read_u32 = [&](uint32_t addr, uint32_t* out) -> bool { + if (!can_read_guest(addr)) { + return false; + } + const auto* heap = memory->LookupHeap(addr); + if (heap->heap_type() == HeapType::kGuestPhysical) { + uint32_t physical_address = memory->GetPhysicalAddress(addr); + auto ptr = + memory->TranslatePhysical(physical_address); + if (!ptr) { + return false; + } + *out = xe::load_and_swap(ptr); + return true; + } + auto ptr = memory->TranslateVirtual(addr); + if (!ptr) { + return false; + } + *out = xe::load_and_swap(ptr); + return true; + }; + + const uint32_t trace_pc = + static_cast(cvars::break_on_instruction); + auto log_trace_instr = [&](uint32_t pc, const char* label) { + if (!can_read_guest(pc)) { + auto* heap = memory->LookupHeap(pc); + XELOGI( + "TraceOnInstruction {}: pc=0x{:08X} unreadable heap={} " + "access={}", + label, pc, heap ? heap_type_to_string(heap->heap_type()) : "none", + heap ? page_access_to_string(heap->QueryRangeAccess(pc, pc)) + : "no-access"); + return; + } + uint32_t instr = 0; + if (!read_u32(pc, &instr)) { + XELOGI("TraceOnInstruction {}: pc=0x{:08X} unreadable", label, pc); + return; + } + xe::StringBuffer disasm; + if (cpu::ppc::DisasmPPC(pc, instr, &disasm)) { + XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X} {}", label, + pc, instr, disasm.to_string_view()); + } else { + XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X}", label, pc, + instr); + } + }; + if (trace_pc) { + log_trace_instr(trace_pc - 4, "target-4"); + log_trace_instr(trace_pc, "target"); + log_trace_instr(trace_pc + 4, "target+4"); + } + if (memory->LookupHeap(trace_pc)) { + for (int offset = -4; offset <= 4; ++offset) { + uint32_t pc = trace_pc + offset * 4; + if (!memory->LookupHeap(pc)) { + continue; + } + uint32_t instr = 0; + if (!read_u32(pc, &instr)) { + XELOGI("TraceOnInstruction window: pc=0x{:08X} unreadable", pc); + continue; + } + xe::StringBuffer disasm_window; + if (cpu::ppc::DisasmPPC(pc, instr, &disasm_window)) { + XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X} {}", + pc, instr, disasm_window.to_string_view()); + } else { + XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X}", pc, + instr); + } + } + } + + const uint32_t obj_address = static_cast(guest_context->r[3]); + if (!obj_address) { + XELOGI("TraceOnInstruction r3 fields: base=0x00000000"); + } else { + const auto* obj_heap = memory->LookupHeap(obj_address); + if (obj_heap) { + uint32_t value_4 = 0; + uint32_t value_8 = 0; + uint32_t value_c = 0; + uint32_t value_20 = 0; + uint32_t value_470 = 0; + bool have_any = false; + have_any |= read_u32(obj_address + 0x4, &value_4); + have_any |= read_u32(obj_address + 0x8, &value_8); + have_any |= read_u32(obj_address + 0xC, &value_c); + have_any |= read_u32(obj_address + 0x20, &value_20); + have_any |= read_u32(obj_address + 0x470, &value_470); + if (have_any) { + XELOGI( + "TraceOnInstruction r3 fields: base=0x{:08X} +0x4=0x{:08X} " + "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", + obj_address, value_4, value_8, value_c, value_20, value_470); + if (value_20) { + log_guest_bytes(value_20, "r3+0x20"); + } + if (value_470) { + log_guest_bytes(value_470, "r3+0x470"); + } + } else { + XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} unmapped", + obj_address); + } + } else { + XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} heap=null", + obj_address); + } + log_guest_string(obj_address, "r3 string"); + } + + const uint32_t r31_address = static_cast(guest_context->r[31]); + if (!r31_address) { + XELOGI("TraceOnInstruction r31 fields: base=0x00000000"); + } else { + const auto* r31_heap = memory->LookupHeap(r31_address); + if (r31_heap) { + uint32_t value_4 = 0; + uint32_t value_8 = 0; + uint32_t value_c = 0; + uint32_t value_20 = 0; + uint32_t value_470 = 0; + bool have_any = false; + have_any |= read_u32(r31_address + 0x4, &value_4); + have_any |= read_u32(r31_address + 0x8, &value_8); + have_any |= read_u32(r31_address + 0xC, &value_c); + have_any |= read_u32(r31_address + 0x20, &value_20); + have_any |= read_u32(r31_address + 0x470, &value_470); + if (have_any) { + XELOGI( + "TraceOnInstruction r31 fields: base=0x{:08X} +0x4=0x{:08X} " + "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", + r31_address, value_4, value_8, value_c, value_20, value_470); + if (value_20) { + log_guest_bytes(value_20, "r31+0x20"); + } + if (value_470) { + log_guest_bytes(value_470, "r31+0x470"); + } + } else { + XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} unmapped", + r31_address); + } + } else { + XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} heap=null", + r31_address); + } + } + } + } + return 0; +} + uint64_t TrapDebugBreak(void* raw_context, uint64_t address) { - auto thread_state = *reinterpret_cast(raw_context); + [[maybe_unused]] auto thread_state = + *reinterpret_cast(raw_context); XELOGE("tw/td forced trap hit! This should be a crash!"); if (cvars::break_on_debugbreak) { xe::debugging::Break(); @@ -410,6 +721,9 @@ void A64Emitter::Trap(uint16_t trap_type) { // 0x0FE00014 is a 'debug print' where r3 = buffer r4 = length CallNative(TrapDebugPrint, 0); break; + case 27: + CallNative(TrapLogRegs, 0); + break; case 0: case 22: // Always trap? From 2e82935dcc213b4468a56b9213e244899c1bc49b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:30:04 +0900 Subject: [PATCH 152/179] [A64/Emitter] Expand ResolveFunction diagnostics - Handle 64-bit host targets and map them back to guest addresses - Add failure logging with module coverage and guest byte dumps - Introduce resolve logging cvars (plus undefined-extern arg toggle) --- src/xenia/cpu/backend/a64/a64_emitter.cc | 160 ++++++++++++++++++++++- 1 file changed, 155 insertions(+), 5 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index a859ddc5a..cf9f72a0d 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -9,6 +9,8 @@ #include "xenia/cpu/backend/a64/a64_emitter.h" +#include +#include #include #include @@ -47,9 +49,15 @@ DEFINE_bool(debugprint_trap_log, false, "Log debugprint traps to the active debugger", "CPU"); DEFINE_bool(ignore_undefined_externs, true, "Don't exit when an undefined extern is called.", "CPU"); +DEFINE_bool(log_undefined_extern_args, false, + "Log PPC args for undefined externs (once per function).", "CPU"); DEFINE_bool(emit_source_annotations, false, "Add extra movs and nops to make disassembly easier to read.", "CPU"); +DEFINE_bool(a64_resolve_function_log, false, + "Log A64 ResolveFunction failures with module ranges.", "CPU"); +DEFINE_int32(a64_resolve_function_log_limit, 8, + "Maximum ResolveFunction failure logs.", "CPU"); namespace xe { namespace cpu { @@ -63,6 +71,19 @@ using namespace oaknut::util; namespace { +bool ShouldLogResolveFailure() { + if (!cvars::a64_resolve_function_log) { + return false; + } + const int32_t limit = cvars::a64_resolve_function_log_limit; + if (limit <= 0) { + return false; + } + static std::atomic log_count{0}; + const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); + return count < limit; +} + void AdjustStackPointer(A64Emitter& emitter, size_t stack_size, bool add) { if (!stack_size) { return; @@ -749,15 +770,144 @@ void A64Emitter::UnimplementedInstr(const hir::Instr* i) { // This is used by the A64ThunkEmitter's ResolveFunctionThunk. uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { auto thread_state = *reinterpret_cast(raw_context); + auto guest_context = thread_state->context(); - // TODO(benvanik): required? - assert_not_zero(target_address); + uint32_t guest_address; + + // Check if this is a 64-bit host address that needs to be mapped back to + // guest address + if (target_address > 0xFFFFFFFF) { + // Precise guard: if target_address is within the PPCContext, this is a bug. + auto ctx_ptr = reinterpret_cast(thread_state->context()); + if (target_address >= ctx_ptr && + target_address < ctx_ptr + sizeof(ppc::PPCContext)) { + XELOGE( + "ResolveFunction: target_address 0x{:016X} is within PPCContext " + "[0x{:016X}, 0x{:016X})", + target_address, ctx_ptr, ctx_ptr + sizeof(ppc::PPCContext)); + XELOGE( + "ResolveFunction: The target register contains a context pointer " + "instead of a function address"); + return 0; + } + + // Try to find a function that contains this host address + auto code_cache = static_cast( + thread_state->processor()->backend()->code_cache()); + auto guest_function = code_cache->LookupFunction(target_address); + if (guest_function) { + guest_address = + guest_function->MapMachineCodeToGuestAddress(target_address); + } else { + // This might be a guest memory address stored in 64-bit form + // Extract the lower 32 bits as the potential guest address + uint32_t potential_guest = static_cast(target_address); + guest_address = potential_guest; + } + } else { + // Normal 32-bit guest address + guest_address = static_cast(target_address); + } + + // Xbox 360 guest addresses can be in these ranges: + // 0x00000000-0x3FFFFFFF: v00000000 heap (virtual) + // 0x40000000-0x7EFFFFFF: v40000000 heap (virtual) + // 0x70000000-0x7F000000: Thread stacks + // 0x80000000-0x8FFFFFFF: v80000000 heap (XEX) + // 0x90000000-0x9FFFFFFF: v90000000 heap (XEX) + // 0xA0000000-0xBFFFFFFF: vA0000000 heap (physical) + // 0xC0000000-0xDFFFFFFF: vC0000000 heap (physical) + // 0xE0000000-0xFFCFFFFF: vE0000000 heap (physical) + + // Most executable code should be in XEX ranges (0x80000000-0x9FFFFFFF) + // but allow other ranges as they may contain valid code + if (guest_address == 0) { + XELOGE("ResolveFunction: guest_address is 0! This should not happen"); + return 0; + } + + auto fn = thread_state->processor()->ResolveFunction(guest_address); + if (!fn) { + XELOGE( + "ResolveFunction: Failed to resolve function at guest address 0x{:08X}", + guest_address); + XELOGE("ResolveFunction: Original target_address was 0x{:016X}", + target_address); + if (ShouldLogResolveFailure()) { + const uint32_t lr_guest = static_cast(guest_context->lr); + XELOGI( + "ResolveFunction: lr=0x{:016X} ctr=0x{:016X} thread_id={} " + "target_is_host={} guest_address=0x{:08X}", + guest_context->lr, guest_context->ctr, guest_context->thread_id, + target_address > 0xFFFFFFFF, guest_address); + auto log_modules_for_address = [&](uint32_t address, const char* label) { + bool found = false; + for (auto* module : thread_state->processor()->GetModules()) { + if (!module) { + continue; + } + if (module->ContainsAddress(address)) { + XELOGI("ResolveFunction: {} module '{}' contains 0x{:08X}", label, + module->name(), address); + found = true; + } + } + if (!found) { + XELOGI("ResolveFunction: {} no module contains 0x{:08X}", label, + address); + } + }; + log_modules_for_address(lr_guest, "lr"); + log_modules_for_address(guest_address, "guest"); + + auto lr_functions = + thread_state->processor()->FindFunctionsWithAddress(lr_guest); + if (lr_functions.empty()) { + XELOGI("ResolveFunction: no resolved function covers LR 0x{:08X}", + lr_guest); + } else { + const auto* fn = lr_functions.front(); + XELOGI("ResolveFunction: LR function {} [0x{:08X},0x{:08X}) name='{}'", + lr_functions.size(), fn->address(), fn->end_address(), + fn->name()); + } + + auto* memory = thread_state->memory(); + if (!memory) { + XELOGI("ResolveFunction: no Memory available for guest dump"); + } else if (!memory->LookupHeap(guest_address)) { + XELOGI( + "ResolveFunction: guest_address 0x{:08X} not in any heap for dump", + guest_address); + } else { + const uint8_t* data = + memory->TranslateVirtual(guest_address); + std::array bytes = {}; + std::memcpy(bytes.data(), data, bytes.size()); + XELOGI( + "ResolveFunction: guest[0x{:08X}] = {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X}", + guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], + bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]); + } + } + + // This can happen if the target address doesn't point to a valid function + // Return 0 to indicate failure - the calling code should handle this + return 0; + } - auto fn = thread_state->processor()->ResolveFunction( - static_cast(target_address)); - assert_not_null(fn); auto a64_fn = static_cast(fn); uint64_t addr = reinterpret_cast(a64_fn->machine_code()); + if (!a64_fn->machine_code()) { + XELOGE( + "ResolveFunction: Function at guest address 0x{:08X} has no machine " + "code", + guest_address); + return 0; + } return addr; } From a05dfe70f20ef9a7dd3ff7fba926af17362b26e5 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:32:21 +0900 Subject: [PATCH 153/179] [A64/Emitter] Log undefined extern arguments - Gate PPC arg dumps behind log_undefined_extern_args - Log XeKeysConsolePrivateKeySign once per run --- src/xenia/cpu/backend/a64/a64_emitter.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index cf9f72a0d..9abc0382d 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -1009,6 +1009,19 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) { auto function = reinterpret_cast(function_ptr); + if (cvars::log_undefined_extern_args && + function->name() == "XeKeysConsolePrivateKeySign") { + static std::atomic logged{false}; + if (!logged.exchange(true)) { + auto* context = reinterpret_cast(raw_context); + XELOGI( + "Undefined extern {} args: r3={:016X} r4={:016X} r5={:016X} " + "r6={:016X} r7={:016X} r8={:016X} r9={:016X} r10={:016X}", + function->name(), context->r[3], context->r[4], context->r[5], + context->r[6], context->r[7], context->r[8], context->r[9], + context->r[10]); + } + } if (!cvars::ignore_undefined_externs) { xe::FatalError(fmt::format("undefined extern call to {:08X} {}", function->address(), function->name().c_str())); From 4df1d22cd1217b37aafb477b1faaefb0e1fad53b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:34:25 +0900 Subject: [PATCH 154/179] [A64/Emitter] Update indirection table loads - Handle 64-bit indirection entries with bias/offset math - Remove ResolveFunction fallback when the table is missing --- src/xenia/cpu/backend/a64/a64_emitter.cc | 63 ++++++++++++++++++------ 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 9abc0382d..c1c946aa2 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -919,20 +919,40 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { if (fn->machine_code()) { // TODO(benvanik): is it worth it to do this? It removes the need for // a ResolveFunction call, but makes the table less useful. +#if XE_A64_INDIRECTION_64BIT + MOV(X16, reinterpret_cast(fn->machine_code())); +#else assert_zero(uint64_t(fn->machine_code()) & 0xFFFFFFFF00000000); MOV(X16, uint32_t(uint64_t(fn->machine_code()))); +#endif } else if (code_cache_->has_indirection_table()) { // Load the pointer to the indirection table maintained in A64CodeCache. // The target dword will either contain the address of the generated code // or a thunk to ResolveAddress. MOV(W17, function->address()); - LDR(W16, X17); +#if XE_A64_INDIRECTION_64BIT + // ARM64 platforms with 64-bit indirection entries must compute an offset. + MOV(X16, code_cache_->indirection_table_base_bias()); + LSL(X15, X17, 1); + ADD(X16, X16, X15); + LDR(X16, X16); +#else + // Other platforms use 32-bit addresses mapped at guest address space. + if (code_cache_->indirection_table_base_address() == + A64CodeCache::kIndirectionTableBase) { + LDR(W16, X17); + } else { + MOV(W16, static_cast(A64CodeCache::kIndirectionTableBase)); + SUB(W17, W17, W16); + MOV(X16, code_cache_->indirection_table_base_address()); + ADD(X16, X16, W17, UXTW); + LDR(W16, X16); + } +#endif } else { - // Old-style resolve. - // Not too important because indirection table is almost always available. - // TODO: Overwrite the call-site with a straight call. - CallNative(&ResolveFunction, function->address()); - MOV(X16, X0); + XELOGE("A64 indirection table missing; ResolveFunction fallback removed"); + BRK(0xF000); + B(epilog_label()); } // Actually jump/call to X16. @@ -973,16 +993,29 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, if (reg.toW().index() != W17.index()) { MOV(W17, reg.toW()); } - LDR(W16, X17); +#if XE_A64_INDIRECTION_64BIT + // ARM64 platforms with 64-bit indirection entries must compute an offset. + MOV(X16, code_cache_->indirection_table_base_bias()); + LSL(X15, X17, 1); + ADD(X16, X16, X15); + LDR(X16, X16); +#else + // Other platforms use 32-bit addresses mapped at guest address space. + if (code_cache_->indirection_table_base_address() == + A64CodeCache::kIndirectionTableBase) { + LDR(W16, X17); + } else { + MOV(W16, static_cast(A64CodeCache::kIndirectionTableBase)); + SUB(W17, W17, W16); + MOV(X16, code_cache_->indirection_table_base_address()); + ADD(X16, X16, W17, UXTW); + LDR(W16, X16); + } +#endif } else { - // Old-style resolve. - // Not too important because indirection table is almost always available. - MOV(X0, GetContextReg()); - MOV(W1, reg.toW()); - - MOV(X16, reinterpret_cast(ResolveFunction)); - BLR(X16); - MOV(X16, X0); + XELOGE("A64 indirection table missing; ResolveFunction fallback removed"); + BRK(0xF000); + B(epilog_label()); } // Actually jump/call to X16. From 7bc540d07a0334936e3cf4f7deb8e1f4b3128327 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:36:06 +0900 Subject: [PATCH 155/179] [A64/Emitter] Fix code size assert cast - Compare code_size.total to size_t offset() --- src/xenia/cpu/backend/a64/a64_emitter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index c1c946aa2..114e4cc3c 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -189,7 +189,7 @@ void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, void* new_execute_address; void* new_write_address; - assert_true(func_info.code_size.total == offset()); + assert_true(func_info.code_size.total == static_cast(offset())); if (function) { code_cache_->PlaceGuestCode(function->address(), assembly_buffer.data(), From 8a915746abc64abdc2b22cb11818dc857b56cfb9 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:37:39 +0900 Subject: [PATCH 156/179] [A64/Emitter] Mark Emit counters unused - Silence unused block/instruction counters in debug paths --- src/xenia/cpu/backend/a64/a64_emitter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 114e4cc3c..a12ba2543 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -306,6 +306,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // Body. auto block = builder->first_block(); + [[maybe_unused]] int block_count = 0; while (block) { // Mark block labels. auto label = block->label_head; @@ -316,6 +317,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // Process instructions. const Instr* instr = block->instr_head; + [[maybe_unused]] int instr_count = 0; while (instr) { const Instr* new_tail = instr; if (!SelectSequence(this, instr, &new_tail)) { From 5c1d4b68519563e7f81cfcb65e05c45f45aabd0b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:39:04 +0900 Subject: [PATCH 157/179] [A64/Emitter] Fix ConstantFitsIn32Reg mask - Use unsigned long mask for 32-bit negative constants --- src/xenia/cpu/backend/a64/a64_emitter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index a12ba2543..56bfd07dc 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -1174,7 +1174,7 @@ bool A64Emitter::ConstantFitsIn32Reg(uint64_t v) { if ((v & ~0x7FFFFFFF) == 0) { // Fits under 31 bits, so just load using normal mov. return true; - } else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) { + } else if ((v & ~0x7FFFFFFFUL) == ~0x7FFFFFFFUL) { // Negative number that fits in 32bits. return true; } From 9d331d99af090088a6c110f231e9f3d3b4e423c8 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:40:21 +0900 Subject: [PATCH 158/179] [A64/Emitter] Document VPackD3DCOLOR mask semantics - Note PSHUFB vs TBL 0xFF behavior --- src/xenia/cpu/backend/a64/a64_emitter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index 56bfd07dc..ac439dba8 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -1229,6 +1229,8 @@ static const vec128_t v_consts[] = { /* VPermuteByteMask */ vec128b(0x1F), /* VPackD3DCOLORSat */ vec128i(0x404000FFu), /* VPackD3DCOLOR */ + // Note: x86 PSHUFB uses 0xFF to zero bytes, ARM TBL uses indices >= 16 + // Keep original 0xFF for consistency, handle in implementation vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0C000408u), /* VUnpackD3DCOLOR */ vec128i(0xFFFFFF0Eu, 0xFFFFFF0Du, 0xFFFFFF0Cu, 0xFFFFFF0Fu), From e3a2b1f9defa36fe18ba0fa21808945f437e9db7 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:41:27 +0900 Subject: [PATCH 159/179] [A64/Emitter] Relax const data placement on macOS - Allow OS-chosen mappings on macOS ARM64 PAGEZERO - Mark fixed-location constants unused on platforms that skip them - Warn on high-address allocations instead of asserting --- src/xenia/cpu/backend/a64/a64_emitter.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index ac439dba8..a07de29f1 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -1311,11 +1311,11 @@ static const vec128_t v_consts[] = { }; // First location to try and place constants. -static const uintptr_t kConstDataLocation = 0x20000000; +[[maybe_unused]] static const uintptr_t kConstDataLocation = 0x20000000; static const uintptr_t kConstDataSize = sizeof(v_consts); // Increment the location by this amount for every allocation failure. -static const uintptr_t kConstDataIncrement = 0x00001000; +[[maybe_unused]] static const uintptr_t kConstDataIncrement = 0x00001000; // This function places constant data that is used by the emitter later on. // Only called once and used by multiple instances of the emitter. @@ -1324,8 +1324,14 @@ static const uintptr_t kConstDataIncrement = 0x00001000; // doing so requires RIP-relative addressing, which is difficult to support // given the current setup. uintptr_t A64Emitter::PlaceConstData() { - uint8_t* ptr = reinterpret_cast(kConstDataLocation); void* mem = nullptr; +#if XE_PLATFORM_MAC && XE_ARCH_ARM64 + // macOS ARM64 PAGEZERO blocks low fixed mappings; use OS-chosen addresses. + mem = memory::AllocFixed( + nullptr, xe::round_up(kConstDataSize, memory::page_size()), + memory::AllocationType::kReserveCommit, memory::PageAccess::kReadWrite); +#else + uint8_t* ptr = reinterpret_cast(kConstDataLocation); while (!mem) { mem = memory::AllocFixed( ptr, xe::round_up(kConstDataSize, memory::page_size()), @@ -1333,9 +1339,21 @@ uintptr_t A64Emitter::PlaceConstData() { ptr += kConstDataIncrement; } +#endif +#if XE_PLATFORM_MAC && XE_ARCH_ARM64 + // On macOS ARM64, memory is often allocated in high address space + if (reinterpret_cast(mem) & ~0x7FFFFFFF) { + XELOGD( + "Const data allocated at high address {:#x}, may cause compatibility " + "issues", + reinterpret_cast(mem)); + // Continue anyway since we'll handle it later + } +#else // The pointer must not be greater than 31 bits. assert_zero(reinterpret_cast(mem) & ~0x7FFFFFFF); +#endif std::memcpy(mem, v_consts, sizeof(v_consts)); memory::Protect(mem, kConstDataSize, memory::PageAccess::kReadOnly, nullptr); From 683cc221c8c0eeeb64fd33cb1e4071c7935a06eb Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 10:57:09 +0900 Subject: [PATCH 160/179] [A64/Sequences] Centralize sequence registration - Replace global table with a singleton accessor - Switch registration to fold expressions with duplicate warnings - Update SelectSequence signature and entrypoint lookup --- src/xenia/cpu/backend/a64/a64_sequences.cc | 12 ++++--- src/xenia/cpu/backend/a64/a64_sequences.h | 37 ++++++++++++++-------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc index 3eb60510e..63269dbee 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -1,4 +1,4 @@ -/** +/** ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** @@ -53,7 +53,7 @@ using namespace xe::cpu::hir; using xe::cpu::hir::Instr; typedef bool (*SequenceSelectFn)(A64Emitter&, const Instr*); -std::unordered_map sequence_table; +// std::unordered_map sequence_table; Removed // ============================================================================ // OPCODE_COMMENT @@ -2769,10 +2769,12 @@ static int anchor_memory_dest = anchor_memory; extern volatile int anchor_vector; static int anchor_vector_dest = anchor_vector; -bool SelectSequence(A64Emitter* e, const Instr* i, const Instr** new_tail) { +bool SelectSequence(A64Emitter* e, const hir::Instr* i, + const hir::Instr** new_tail) { const InstrKey key(i); - auto it = sequence_table.find(key); - if (it != sequence_table.end()) { + auto& table = GetSequenceTable(); // Use the singleton accessor + auto it = table.find(key); + if (it != table.end()) { if (it->second(*e, i)) { *new_tail = i->next; return true; diff --git a/src/xenia/cpu/backend/a64/a64_sequences.h b/src/xenia/cpu/backend/a64/a64_sequences.h index b47382633..65d4adad3 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.h +++ b/src/xenia/cpu/backend/a64/a64_sequences.h @@ -10,9 +10,10 @@ #ifndef XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ #define XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ -#include "xenia/cpu/hir/instr.h" - +#include +#include // For logging #include +#include "xenia/cpu/hir/instr.h" namespace xe { namespace cpu { @@ -22,24 +23,34 @@ namespace a64 { class A64Emitter; typedef bool (*SequenceSelectFn)(A64Emitter&, const hir::Instr*); -extern std::unordered_map sequence_table; +// Singleton accessor for sequence_table +inline std::unordered_map& GetSequenceTable() { + static std::unordered_map sequence_table; + return sequence_table; +} + +// Registration Functions template -bool Register() { - sequence_table.insert({T::head_key(), T::Select}); - return true; +bool RegisterSingle() { + bool inserted = GetSequenceTable().emplace(T::head_key(), T::Select).second; + if (!inserted) { + std::cerr << "Warning: Duplicate head_key detected for key " + << T::head_key() << std::endl; + } + return inserted; } -template -static bool Register() { - bool b = true; - b = b && Register(); // Call the above function - b = b && Register(); // Call ourself again (recursively) - return b; +template +bool RegisterAll() { + return (RegisterSingle() && ...); // Fold expression (C++17) } + +// Macro for Registration #define EMITTER_OPCODE_TABLE(name, ...) \ - const auto A64_INSTR_##name = Register<__VA_ARGS__>(); + static const bool A64_INSTR_##name = RegisterAll<__VA_ARGS__>(); +// Function to Select Sequence bool SelectSequence(A64Emitter* e, const hir::Instr* i, const hir::Instr** new_tail); From 310a2e8bea4de4ae72408b3842d752cc3424397e Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:03:26 +0900 Subject: [PATCH 161/179] [A64/Sequences] Drop unused lambda captures - Remove unused "i" captures in compare and mul-sub sequences --- src/xenia/cpu/backend/a64/a64_sequences.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc index 63269dbee..4297314ad 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -898,8 +898,7 @@ struct COMPARE_EQ_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitCommutativeBinaryVOp( - e, i, - [&i](A64Emitter& e, I8Op dest, const SReg& src1, const SReg& src2) { + e, i, [](A64Emitter& e, I8Op dest, const SReg& src1, const SReg& src2) { e.FCMP(src1, src2); }); e.CSET(i.dest, Cond::EQ); @@ -909,8 +908,7 @@ struct COMPARE_EQ_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { EmitCommutativeBinaryVOp( - e, i, - [&i](A64Emitter& e, I8Op dest, const DReg& src1, const DReg& src2) { + e, i, [](A64Emitter& e, I8Op dest, const DReg& src1, const DReg& src2) { e.FCMP(src1, src2); }); e.CSET(i.dest, Cond::EQ); @@ -1794,7 +1792,7 @@ struct MUL_SUB_F32 // Multiply operation is commutative. EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, SReg dest, SReg src1, SReg src2) { + e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { e.FMUL(dest, src1, src2); // $0 = $1 * $2 }); @@ -1819,7 +1817,7 @@ struct MUL_SUB_F64 // Multiply operation is commutative. EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, DReg dest, DReg src1, DReg src2) { + e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { e.FMUL(dest, src1, src2); // $0 = $1 * $2 }); @@ -1845,7 +1843,7 @@ struct MUL_SUB_V128 // Multiply operation is commutative. EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { e.FMUL(dest.S4(), src1.S4(), src2.S4()); // $0 = $1 * $2 }); From 9fe9f2ab2a09546bf67bf008cc7a0fb1363b0dce Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:04:55 +0900 Subject: [PATCH 162/179] [A64/Sequences] Fix MUL_ADD_V128 temp handling - Always use temporaries to avoid source/dest aliasing - Replace FMLA with FMUL+FADD for clearer ordering --- src/xenia/cpu/backend/a64/a64_sequences.cc | 41 ++++++++++++---------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc index 4297314ad..a41acad4c 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -1734,30 +1734,35 @@ struct MUL_ADD_V128 I> { static void Emit(A64Emitter& e, const EmitArgType& i) { const QReg dest = i.dest.reg(); - if (i.src3.is_constant) { - e.LoadConstantV(dest.toQ(), i.src3.constant()); - } else { - // If i.dest != i.src3, move the addition-term into dest for FMLA - if (i.dest != i.src3) { - e.MOV(dest.B16(), i.src3.reg().B16()); - } - } - QReg src2 = Q2; - if (i.src2.is_constant) { - e.LoadConstantV(src2.toQ(), i.src2.constant()); - } else { - src2 = i.src2.reg(); - } + // Always use temporary registers to avoid conflicts + QReg temp1 = Q0; + QReg temp2 = Q1; + QReg temp3 = Q2; - QReg src1 = Q1; if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); + e.LoadConstantV(temp1, i.src1.constant()); } else { - src1 = i.src1.reg(); + e.MOV(temp1.B16(), i.src1.reg().B16()); } - e.FMLA(dest.S4(), src1.S4(), src2.S4()); + if (i.src2.is_constant) { + e.LoadConstantV(temp2, i.src2.constant()); + } else { + e.MOV(temp2.B16(), i.src2.reg().B16()); + } + + if (i.src3.is_constant) { + e.LoadConstantV(temp3, i.src3.constant()); + } else { + e.MOV(temp3.B16(), i.src3.reg().B16()); + } + + // First multiply: dest = temp1 * temp2 + e.FMUL(dest.S4(), temp1.S4(), temp2.S4()); + + // Then add: dest = dest + temp3 + e.FADD(dest.S4(), dest.S4(), temp3.S4()); } }; EMITTER_OPCODE_TABLE(OPCODE_MUL_ADD, MUL_ADD_F32, MUL_ADD_F64, MUL_ADD_V128); From a83bd019a4456bb8bd5a7a842501edd64236295f Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:11:54 +0900 Subject: [PATCH 163/179] [A64/Sequences] Fix NEON byte loads - Cast std::byte buffers to uint8_t* in pow2/log2 emulators - Update vector shift/rotate helpers to use byte-wise loads --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 32 +++++++++------ src/xenia/cpu/backend/a64/a64_sequences.cc | 45 ++++++++++++++------- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index abc4688ac..3f640a875 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -499,15 +499,17 @@ static uint8x16_t EmulateVectorShl(void*, std::byte src1[16], alignas(16) T shamt[16 / sizeof(T)]; // Load NEON registers into a C array. - vst1q_u8(reinterpret_cast(value), vld1q_u8(src1)); - vst1q_u8(reinterpret_cast(shamt), vld1q_u8(src2)); + vst1q_u8(reinterpret_cast(value), + vld1q_u8(reinterpret_cast(src1))); + vst1q_u8(reinterpret_cast(shamt), + vld1q_u8(reinterpret_cast(src2))); for (size_t i = 0; i < (16 / sizeof(T)); ++i) { value[i] = value[i] << (shamt[i] & ((sizeof(T) * 8) - 1)); } // Store result and return it. - return vld1q_u8(value); + return vld1q_u8(reinterpret_cast(value)); } struct VECTOR_SHL_V128 : Sequence> { @@ -611,16 +613,18 @@ static uint8x16_t EmulateVectorShr(void*, std::byte src1[16], alignas(16) T value[16 / sizeof(T)]; alignas(16) T shamt[16 / sizeof(T)]; - // Load NEON registers into a C array. - vst1q_u8(reinterpret_cast(value), vld1q_u8(src1)); - vst1q_u8(reinterpret_cast(shamt), vld1q_u8(src2)); + // Load NEON registers into a C array by casting to uint8_t* + vst1q_u8(reinterpret_cast(value), + vld1q_u8(reinterpret_cast(src1))); + vst1q_u8(reinterpret_cast(shamt), + vld1q_u8(reinterpret_cast(src2))); for (size_t i = 0; i < (16 / sizeof(T)); ++i) { value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1)); } - // Store result and return it. - return vld1q_u8(value); + // Store result and return it by casting to uint8_t* + return vld1q_u8(reinterpret_cast(value)); } struct VECTOR_SHR_V128 : Sequence> { @@ -820,16 +824,18 @@ static uint8x16_t EmulateVectorRotateLeft(void*, std::byte src1[16], alignas(16) T value[16 / sizeof(T)]; alignas(16) T shamt[16 / sizeof(T)]; - // Load NEON registers into a C array. - vst1q_u8(reinterpret_cast(value), vld1q_u8(src1)); - vst1q_u8(reinterpret_cast(shamt), vld1q_u8(src2)); + // Load NEON registers into a C array by casting to uint8_t* + vst1q_u8(reinterpret_cast(value), + vld1q_u8(reinterpret_cast(src1))); + vst1q_u8(reinterpret_cast(shamt), + vld1q_u8(reinterpret_cast(src2))); for (size_t i = 0; i < (16 / sizeof(T)); ++i) { value[i] = xe::rotate_left(value[i], shamt[i] & ((sizeof(T) * 8) - 1)); } - // Store result and return it. - return vld1q_u8(value); + // Store result and return it by casting to uint8_t* + return vld1q_u8(reinterpret_cast(value)); } struct VECTOR_ROTATE_LEFT_V128 : Sequence> { static float32x4_t EmulatePow2(void*, std::byte src[16]) { float src_value; - vst1q_lane_f32(&src_value, vld1q_u8(src), 0); + vst1q_lane_f32(&src_value, vld1q_u8(reinterpret_cast(src)), + 0); const float result = std::exp2(src_value); - return vld1q_lane_f32(&result, vld1q_u8(src), 0); + return vld1q_lane_f32(&result, + vld1q_u8(reinterpret_cast(src)), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); @@ -2009,9 +2011,11 @@ struct POW2_F32 : Sequence> { struct POW2_F64 : Sequence> { static float64x2_t EmulatePow2(void*, std::byte src[16]) { double src_value; - vst1q_lane_f64(&src_value, vld1q_u8(src), 0); + vst1q_lane_f64(&src_value, vld1q_u8(reinterpret_cast(src)), + 0); const double result = std::exp2(src_value); - return vld1q_lane_f64(&result, vld1q_u8(src), 0); + return vld1q_lane_f64(&result, + vld1q_u8(reinterpret_cast(src)), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); @@ -2023,7 +2027,7 @@ struct POW2_F64 : Sequence> { struct POW2_V128 : Sequence> { static float32x4_t EmulatePow2(void*, std::byte src[16]) { alignas(16) float values[4]; - vst1q_f32(values, vld1q_u8(src)); + vst1q_f32(values, vld1q_u8(reinterpret_cast(src))); for (size_t i = 0; i < 4; ++i) { values[i] = std::exp2(values[i]); } @@ -2046,9 +2050,11 @@ EMITTER_OPCODE_TABLE(OPCODE_POW2, POW2_F32, POW2_F64, POW2_V128); struct LOG2_F32 : Sequence> { static float32x4_t EmulateLog2(void*, std::byte src[16]) { float src_value; - vst1q_lane_f32(&src_value, vld1q_u8(src), 0); + vst1q_lane_f32(&src_value, vld1q_u8(reinterpret_cast(src)), + 0); float result = std::log2(src_value); - return vld1q_lane_f32(&result, vld1q_u8(src), 0); + return vld1q_lane_f32(&result, + vld1q_u8(reinterpret_cast(src)), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); @@ -2064,9 +2070,11 @@ struct LOG2_F32 : Sequence> { struct LOG2_F64 : Sequence> { static float64x2_t EmulateLog2(void*, std::byte src[16]) { double src_value; - vst1q_lane_f64(&src_value, vld1q_u8(src), 0); + vst1q_lane_f64(&src_value, vld1q_u8(reinterpret_cast(src)), + 0); double result = std::log2(src_value); - return vld1q_lane_f64(&result, vld1q_u8(src), 0); + return vld1q_lane_f64(&result, + vld1q_u8(reinterpret_cast(src)), 0); } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); @@ -2082,7 +2090,7 @@ struct LOG2_F64 : Sequence> { struct LOG2_V128 : Sequence> { static float32x4_t EmulateLog2(void*, std::byte src[16]) { alignas(16) float values[4]; - vst1q_f32(values, vld1q_u8(src)); + vst1q_f32(values, vld1q_u8(reinterpret_cast(src))); for (size_t i = 0; i < 4; ++i) { values[i] = std::log2(values[i]); } @@ -2424,14 +2432,18 @@ struct SHL_V128 : Sequence> { // Almost all instances are shamt = 1, but non-constant. // shamt is [0,7] uint8_t shamt = src2 & 0x7; + + // Load `src1` as a byte vector (uint8x16_t) then work on byte shifting + uint8x16_t byte_vec = vld1q_u8(reinterpret_cast(src1)); alignas(16) vec128_t value; - vst1q_f32(reinterpret_cast(&value), vld1q_u8(src1)); + vst1q_u8(reinterpret_cast(&value), byte_vec); for (int i = 0; i < 15; ++i) { value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] << shamt) | (value.u8[(i + 1) ^ 0x3] >> (8 - shamt)); } value.u8[15 ^ 0x3] = value.u8[15 ^ 0x3] << shamt; - return vld1q_f32(reinterpret_cast(&value)); + return vreinterpretq_f32_u8( + vld1q_u8(reinterpret_cast(&value))); } }; EMITTER_OPCODE_TABLE(OPCODE_SHL, SHL_I8, SHL_I16, SHL_I32, SHL_I64, SHL_V128); @@ -2503,14 +2515,19 @@ struct SHR_V128 : Sequence> { // Almost all instances are shamt = 1, but non-constant. // shamt is [0,7] uint8_t shamt = src2 & 0x7; + // Load `src1` as a byte vector (uint8x16_t) and store it into `value` + uint8x16_t byte_vec = vld1q_u8(reinterpret_cast(src1)); alignas(16) vec128_t value; - vst1q_f32(reinterpret_cast(&value), vld1q_u8(src1)); + vst1q_u8(reinterpret_cast(&value), byte_vec); + for (int i = 15; i > 0; --i) { value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] >> shamt) | (value.u8[(i - 1) ^ 0x3] << (8 - shamt)); } value.u8[0 ^ 0x3] = value.u8[0 ^ 0x3] >> shamt; - return vld1q_f32(reinterpret_cast(&value)); + // Convert to float32x4_t by reinterpreting the processed `value` + return vreinterpretq_f32_u8( + vld1q_u8(reinterpret_cast(&value))); } }; EMITTER_OPCODE_TABLE(OPCODE_SHR, SHR_I8, SHR_I16, SHR_I32, SHR_I64, SHR_V128); From 5f1da79eb06e0d30ee218ff3a737b259220c137e Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:15:18 +0900 Subject: [PATCH 164/179] [A64/Vector] Handle constant converts and shamt types - Load constant vectors before convert ops - Use size_t comparisons for lvsl/lvsr tables --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 3f640a875..28561419c 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -32,10 +32,14 @@ struct VECTOR_CONVERT_I2F : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + const QReg src = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src, i.src1.constant()); + } if (i.instr->flags & ARITHMETIC_UNSIGNED) { - e.UCVTF(i.dest.reg().S4(), i.src1.reg().S4()); + e.UCVTF(i.dest.reg().S4(), src.S4()); } else { - e.SCVTF(i.dest.reg().S4(), i.src1.reg().S4()); + e.SCVTF(i.dest.reg().S4(), src.S4()); } } }; @@ -48,10 +52,14 @@ struct VECTOR_CONVERT_F2I : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + const QReg src = i.src1.is_constant ? Q0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src, i.src1.constant()); + } if (i.instr->flags & ARITHMETIC_UNSIGNED) { - e.FCVTZU(i.dest.reg().S4(), i.src1.reg().S4()); + e.FCVTZU(i.dest.reg().S4(), src.S4()); } else { - e.FCVTZS(i.dest.reg().S4(), i.src1.reg().S4()); + e.FCVTZS(i.dest.reg().S4(), src.S4()); } } }; @@ -83,7 +91,7 @@ struct LOAD_VECTOR_SHL_I8 static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { auto sh = i.src1.constant(); - assert_true(sh < xe::countof(lvsl_table)); + assert_true(static_cast(sh) < xe::countof(lvsl_table)); e.MOV(X0, reinterpret_cast(&lvsl_table[sh])); e.LDR(i.dest, X0); } else { @@ -121,7 +129,7 @@ struct LOAD_VECTOR_SHR_I8 static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { auto sh = i.src1.constant(); - assert_true(sh < xe::countof(lvsr_table)); + assert_true(static_cast(sh) < xe::countof(lvsr_table)); e.MOV(X0, reinterpret_cast(&lvsr_table[sh])); e.LDR(i.dest, X0); } else { From 5520cda456186587d95cd6a51f13689266b92dc8 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:44:39 +0900 Subject: [PATCH 165/179] [A64/Vector] Simplify SPLAT constant handling - Drop fimm8 and shifted MOVI paths for constant splats - Use only <= 0xFF MOVI fast path before DUP - Remove a64_util include now that fimm8 helpers are unused --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 46 ++++----------------- 1 file changed, 7 insertions(+), 39 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 28561419c..ae0743621 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -2,13 +2,11 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2024 Xenia Developers. All rights reserved. * + * Copyright 2025 Xenia Developers. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ - #include "xenia/cpu/backend/a64/a64_sequences.h" -#include "xenia/cpu/backend/a64/a64_util.h" #include #include @@ -1041,6 +1039,8 @@ EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); struct SPLAT_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { + // int8_t is always <= 0xFF, so this is always true + // Keep the MOVI path for constants e.MOVI(i.dest.reg().B16(), i.src1.constant()); } else { e.DUP(i.dest.reg().B16(), i.src1); @@ -1050,12 +1050,9 @@ struct SPLAT_I8 : Sequence> { struct SPLAT_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - if ((i.src1.constant() & 0xFF'00) == 0) { + if (i.src1.constant() <= 0xFF) { e.MOVI(i.dest.reg().H8(), i.src1.constant()); return; - } else if ((i.src1.constant() & 0x00'FF) == 0) { - e.MOVI(i.dest.reg().H8(), i.src1.constant(), oaknut::util::LSL, 8); - return; } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().H8(), W0); @@ -1067,22 +1064,9 @@ struct SPLAT_I16 : Sequence> { struct SPLAT_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - oaknut::FImm8 fp8(0); - if (f32_to_fimm8(i.src1.value->constant.u32, fp8)) { - e.FMOV(i.dest.reg().S4(), fp8); - return; - } else if ((i.src1.constant() & 0xFF'FF'FF'00) == 0) { + if (i.src1.constant() <= 0xFF) { e.MOVI(i.dest.reg().S4(), i.src1.constant()); return; - } else if ((i.src1.constant() & 0xFF'FF'00'FF) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.constant(), oaknut::util::LSL, 8); - return; - } else if ((i.src1.constant() & 0xFF'00'FF'FF) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.constant(), oaknut::util::LSL, 16); - return; - } else if ((i.src1.constant() & 0x00'FF'FF'FF) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.constant(), oaknut::util::LSL, 24); - return; } e.MOV(W0, i.src1.constant()); e.DUP(i.dest.reg().S4(), W0); @@ -1094,24 +1078,8 @@ struct SPLAT_I32 : Sequence> { struct SPLAT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { if (i.src1.is_constant) { - oaknut::FImm8 fp8(0); - if (f32_to_fimm8(i.src1.value->constant.u32, fp8)) { - e.FMOV(i.dest.reg().S4(), fp8); - return; - } else if ((i.src1.value->constant.u32 & 0xFF'FF'FF'00) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32); - return; - } else if ((i.src1.value->constant.u32 & 0xFF'FF'00'FF) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32, oaknut::util::LSL, - 8); - return; - } else if ((i.src1.value->constant.u32 & 0xFF'00'FF'FF) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32, oaknut::util::LSL, - 16); - return; - } else if ((i.src1.value->constant.u32 & 0x00'FF'FF'FF) == 0) { - e.MOVI(i.dest.reg().S4(), i.src1.value->constant.u32, oaknut::util::LSL, - 24); + if (i.src1.value->constant.i32 <= 0xFF) { + e.MOVI(i.dest.reg().S4(), i.src1.value->constant.i32); return; } e.MOV(W0, i.src1.value->constant.i32); From 1de36083c8ad6e4ad16ddd8e047433a87b36c266 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:46:21 +0900 Subject: [PATCH 166/179] [A64/Vector] Rewrite PERMUTE_V128 int16 constants - Require constant indices and build byte shuffle/blend masks - Shuffle src2/src3 separately and blend with AND/BIC/ORR - Align the implementation with the x64 backend logic --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 70 ++++++++++++--------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index ae0743621..e1cf3abef 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1191,9 +1191,9 @@ struct PERMUTE_V128 } static void EmitByInt16(A64Emitter& e, const EmitArgType& i) { - // Permute bytes between src2 and src3. - // src1 is an array of indices corresponding to positions within src2 and - // src3. + // Permute 16-bit halfwords between src2 and src3. + // src1 is an array of 16-bit indices corresponding to positions within src2 + // and src3. if (i.src3.value->IsConstantZero()) { if (i.src2.value->IsConstantZero()) { // src2 & src3 are zero, so result will always be zero. @@ -1202,45 +1202,55 @@ struct PERMUTE_V128 } } - const QReg indices = Q0; - if (i.src1.is_constant) { - e.LoadConstantV(indices, i.src1.constant()); - } else { - e.MOV(indices.B16(), i.src1.reg().B16()); + // This implementation is adapted from the x64 backend + assert_true(i.src1.is_constant); + + // Process the constant indices to create byte shuffle masks and blend + // control + vec128_t perm = (i.src1.constant() & vec128s(0xF)) ^ vec128s(0x1); + vec128_t perm_bytes = vec128b(0); + vec128_t blend_mask = vec128b(0); + + for (int idx = 0; idx < 8; idx++) { + bool from_src3 = perm.i16[idx] > 7; + + // Create byte indices for this 16-bit element + uint8_t base_byte_idx = uint8_t(perm.u16[idx] & 7) * 2; + perm_bytes.u8[idx * 2] = base_byte_idx; + perm_bytes.u8[idx * 2 + 1] = base_byte_idx + 1; + + // Create blend mask (0xFF means take from src3, 0x00 means take from + // src2) + blend_mask.u8[idx * 2] = from_src3 ? 0xFF : 0x00; + blend_mask.u8[idx * 2 + 1] = from_src3 ? 0xFF : 0x00; } - // Indices must be endian-swapped - e.MOVI(Q1.H8(), 0b1); - e.EOR(indices.B16(), indices.B16(), Q1.B16()); + // Load the byte shuffle mask + e.LoadConstantV(Q0, perm_bytes); - // Modulo-16 the indices - e.MOVI(Q1.H8(), 0b0000'1111); - e.AND(indices.B16(), indices.B16(), Q1.B16()); + // Load the blend mask + e.LoadConstantV(Q1, blend_mask); - // Convert int16 indices into int8 - e.MOVI(Q1.B16(), 0x02); - e.MUL(indices.H8(), indices.H8(), Q1.H8()); - - e.MOVI(Q1.H8(), 0x01, LSL, 8); - e.ADD(indices.H8(), indices.H8(), Q1.H8()); - - // Table-registers must be sequential indices - const QReg table_lo = Q2; + // Load src2 and shuffle it if (i.src2.is_constant) { - e.LoadConstantV(table_lo, i.src2.constant()); + e.LoadConstantV(Q2, i.src2.constant()); } else { - e.MOV(table_lo.B16(), i.src2.reg().B16()); + e.MOV(Q2.B16(), i.src2.reg().B16()); } + e.TBL(Q2.B16(), List{Q2.B16()}, Q0.B16()); - const QReg table_hi = Q3; + // Load src3 and shuffle it if (i.src3.is_constant) { - e.LoadConstantV(table_hi, i.src3.constant()); + e.LoadConstantV(Q3, i.src3.constant()); } else { - e.MOV(table_hi.B16(), i.src3.reg().B16()); + e.MOV(Q3.B16(), i.src3.reg().B16()); } + e.TBL(Q3.B16(), List{Q3.B16()}, Q0.B16()); - e.TBL(i.dest.reg().B16(), List{table_lo.B16(), table_hi.B16()}, - indices.B16()); + // Blend the results: dest = (Q3 & Q1) | (Q2 & ~Q1) + e.AND(Q3.B16(), Q3.B16(), Q1.B16()); // Q3 & mask + e.BIC(Q2.B16(), Q2.B16(), Q1.B16()); // Q2 & ~mask + e.ORR(i.dest.reg().B16(), Q3.B16(), Q2.B16()); // combine } static void EmitByInt32(A64Emitter& e, const EmitArgType& i) { From 283f3d7db79e961dd8c77514a8ca548534ee0bbb Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:47:25 +0900 Subject: [PATCH 167/179] [A64/Vector] Use numeric min/max for D3DCOLOR pack - Prefer FMAXNM/FMINNM to avoid NaN propagation differences - Match x86 and Windows ARM64 saturation behavior --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index e1cf3abef..85bb5251f 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1371,11 +1371,15 @@ struct PACK : Sequence> { // are valid - max before min to pack NaN as zero (5454082B is heavily // affected by the order - packs 0xFFFFFFFF in matrix code to get a 0 // constant). + // Use FMAXNM/FMINNM (numeric max/min) instead of FMAX/FMIN to handle NaN + // correctly. FMAXNM returns the non-NaN value when one operand is NaN, + // matching x86 behavior and Windows ARM64 expectations. e.LDR(Q0, VConstData, e.GetVConstOffset(V3333)); - e.FMAX(i.dest.reg().S4(), i.dest.reg().S4(), Q0.S4()); + e.FMAXNM(i.dest.reg().S4(), src.S4(), Q0.S4()); e.LDR(Q0, VConstData, e.GetVConstOffset(VPackD3DCOLORSat)); - e.FMIN(i.dest.reg().S4(), src.S4(), Q0.S4()); + e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q0.S4()); + // Extract bytes. // RGBA (XYZW) -> ARGB (WXYZ) // w = ((src1.uw & 0xFF) << 24) | ((src1.ux & 0xFF) << 16) | From 284024165b9415c480fcc4bc112a858ef234378a Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:49:14 +0900 Subject: [PATCH 168/179] [A64/Vector] Handle FLOAT16 pack sentinels - Emulate 360 saturation sentinels in FLOAT16_2 pack - Detect half-float infinities and replace with 0x7FFF/0xFFFF - Use reinterpret_cast loads/stores for NEON emulation helpers --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 71 +++++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 85bb5251f..87aa3bc03 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1390,14 +1390,30 @@ struct PACK : Sequence> { static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { alignas(16) float a[4]; alignas(16) uint16_t b[8]; - vst1q_u8(a, vld1q_u8(src1)); + + // Load NEON registers into a C array by casting to uint8_t* + vst1q_u8(reinterpret_cast(a), + vld1q_u8(reinterpret_cast(src1))); std::memset(b, 0, sizeof(b)); for (int i = 0; i < 2; i++) { - b[7 - i] = half_float::detail::float2half(a[i]); + float x = a[i]; + uint16_t h; + + // Xbox 360 saturation behavior + if (x >= 65504.0f) { + h = 0x7FFF; // Positive saturation sentinel + } else if (x <= -65504.0f) { + h = 0xFFFF; // Negative saturation sentinel + } else { + h = half_float::detail::float2half(x); + } + + b[7 - i] = h; } - return vld1q_u8(b); + // Store the uint16_t array into a uint8x16_t NEON register + return vld1q_u8(reinterpret_cast(b)); } static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->IsConstantZero()); @@ -1409,7 +1425,48 @@ struct PACK : Sequence> { if (i.src1.is_constant) { e.LoadConstantV(src1, i.src1.constant()); } + + // Perform IEEE conversion first e.FCVTN(i.dest.reg().toD().H4(), src1.S4()); + + // Xbox 360 rule: infinity values should be replaced with 0x7FFF/0xFFFF + // After FCVTN, +inf becomes 0x7C00, -inf becomes 0xFC00 + // We need to detect these and replace them + + // Create masks for infinity detection + QReg abs_mask = Q1; + QReg inf_pattern = Q2; + QReg is_inf = Q3; + QReg sign_mask = Q4; + QReg sentinel = Q5; + + // Get absolute value of halfwords (clear sign bit) + e.MOVI(abs_mask.B16(), 0xFF); // 0xFFFF + e.USHR(abs_mask.H8(), abs_mask.H8(), 1); // 0x7FFF + e.AND(is_inf.toD().B8(), i.dest.reg().toD().B8(), abs_mask.toD().B8()); + + // Check if abs value == 0x7C00 (infinity) + e.MOVI(inf_pattern.H8(), 0x7C, oaknut::LslSymbol{}, 8); // 0x7C00 + e.CMEQ(is_inf.toD().H4(), is_inf.toD().H4(), inf_pattern.toD().H4()); + + // Get sign bits + e.MOVI(sign_mask.H8(), 0x80, oaknut::LslSymbol{}, 8); // 0x8000 + e.AND(sign_mask.toD().B8(), i.dest.reg().toD().B8(), + sign_mask.toD().B8()); + + // Create sentinel value: sign | 0x7FFF + e.MOVI(sentinel.B16(), 0xFF); // 0xFFFF + e.USHR(sentinel.H8(), sentinel.H8(), 1); // 0x7FFF + e.ORR(sentinel.toD().B8(), sentinel.toD().B8(), sign_mask.toD().B8()); + + // Apply replacement where infinity was detected + // Use BSL to replace infinities with sentinels + // BSL: result = (mask & src1) | (~mask & src2) + // We want: where is_inf is true, use sentinel; else use original + e.BSL(is_inf.toD().B8(), sentinel.toD().B8(), i.dest.reg().toD().B8()); + e.MOV(i.dest.reg().toD().B8(), is_inf.toD().B8()); + + // Keep existing layout adjustments e.MOVI(Q0.B16(), 0); e.EXT(i.dest.reg().B16(), Q0.B16(), i.dest.reg().B16(), 4); e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); @@ -1427,7 +1484,10 @@ struct PACK : Sequence> { static uint8x16_t EmulateFLOAT16_4(void*, std::byte src1[16]) { alignas(16) float a[4]; alignas(16) uint16_t b[8]; - vst1q_u8(a, vld1q_u8(src1)); + + // Load NEON registers into a C array by casting to uint8_t* + vst1q_u8(reinterpret_cast(a), + vld1q_u8(reinterpret_cast(src1))); std::memset(b, 0, sizeof(b)); for (int i = 0; i < 4; i++) { @@ -1435,7 +1495,8 @@ struct PACK : Sequence> { half_float::detail::float2half(a[i]); } - return vld1q_u8(b); + // Store the uint16_t array into a uint8x16_t NEON register + return vld1q_u8(reinterpret_cast(b)); } static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->IsConstantZero()); From 3afa85c4c56c1bfa6d63dd9d952ac7e9437e35ca Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:50:27 +0900 Subject: [PATCH 169/179] [A64/Vector] Improve SHORT pack saturation - Add constant-zero fast path and preserve zero lanes - Copy src into dest when needed to keep register usage safe - Use FMAXNM/FMINNM and BIC to avoid NaN/zero edge cases --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 64 ++++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 87aa3bc03..3b0876654 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1524,45 +1524,93 @@ struct PACK : Sequence> { } static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->IsConstantZero()); + + // Check if input is constant zero - special case + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); + return; + } + QReg src = i.src1; if (i.src1.is_constant) { src = i.dest; e.LoadConstantV(src, i.src1.constant()); + } else { + // Copy src to dest if they're different registers + if (src.index() != i.dest.reg().index()) { + e.MOV(i.dest.reg().B16(), src.B16()); + } } + const XReg VConstData = X3; e.MOV(VConstData, e.GetVConstPtr()); - // Saturate + // Check if the value is zero (0x00000000) - if so, skip min/max + // Zero is a special case that doesn't use the mantissa trick + // Create a mask for zero elements + e.CMEQ(Q0.S4(), i.dest.reg().S4(), 0); + + // Save the zero mask + e.MOV(Q2.B16(), Q0.B16()); + + // Saturate non-zero values - use FMAXNM/FMINNM to handle NaN correctly e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); - e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); + e.FMAXNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Max)); - e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Use BIC to clear values that were originally zero + e.BIC(i.dest.reg().B16(), i.dest.reg().B16(), Q2.B16()); // Pack e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_2)); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q1.B16()); } static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->IsConstantZero()); + + // Check if input is constant zero - special case + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); + return; + } + QReg src = i.src1; if (i.src1.is_constant) { src = i.dest; e.LoadConstantV(src, i.src1.constant()); + } else { + // Copy src to dest if they're different registers + if (src.index() != i.dest.reg().index()) { + e.MOV(i.dest.reg().B16(), src.B16()); + } } + const XReg VConstData = X3; e.MOV(VConstData, e.GetVConstPtr()); - // Saturate + // Check if the value is zero (0x00000000) - if so, skip min/max + // Zero is a special case that doesn't use the mantissa trick + // Create a mask for zero elements + e.CMEQ(Q0.S4(), i.dest.reg().S4(), 0); + + // Save the zero mask + e.MOV(Q2.B16(), Q0.B16()); + + // Saturate non-zero values - use FMAXNM/FMINNM to handle NaN correctly e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); - e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); + e.FMAXNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Max)); - e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); + + // Use BIC to clear values that were originally zero + e.BIC(i.dest.reg().B16(), i.dest.reg().B16(), Q2.B16()); // Pack e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_4)); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q1.B16()); } static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { // https://www.opengl.org/registry/specs/ARB/vertex_type_2_10_10_10_rev.txt From 129462f32627f891a36fb39bbd24d604b8f9945b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:54:00 +0900 Subject: [PATCH 170/179] [A64/Vector] Gate pack ordering on Windows ARM64 - Swap src ordering for XT(N)/UQXTN/SQXTN/SQXTUN on _WIN32 - Add UMIN saturation before 32->16 unsigned packing - Keep non-Windows path matching existing macOS/Linux ordering --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 77 ++++++++++++++++++--- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 3b0876654..55925cadc 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1702,18 +1702,31 @@ struct PACK : Sequence> { if (i.src2.is_constant) { e.LoadConstantV(src2, i.src2.constant()); } + // Windows ARM64 requires src2->src1 ordering with EXT swap + // macOS works with src1->src2 without EXT +#ifdef _WIN32 e.UQXTN(i.dest.reg().toD().B8(), src2.H8()); e.UQXTN2(i.dest.reg().B16(), src1.H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.UQXTN(i.dest.reg().toD().B8(), src1.H8()); + e.UQXTN2(i.dest.reg().B16(), src2.H8()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } else { // unsigned -> unsigned + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 e.XTN(i.dest.reg().toD().B8(), i.src2.reg().H8()); e.XTN2(i.dest.reg().B16(), i.src1.reg().H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.XTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); + e.XTN2(i.dest.reg().B16(), i.src2.reg().H8()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } } else { if (IsPackOutSaturate(flags)) { @@ -1738,11 +1751,17 @@ struct PACK : Sequence> { e.LoadConstantV(src2, i.src2.constant()); } + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 e.SQXTUN(i.dest.reg().toD().B8(), src2.H8()); e.SQXTUN2(i.dest.reg().B16(), src1.H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.SQXTUN(i.dest.reg().toD().B8(), src1.H8()); + e.SQXTUN2(i.dest.reg().B16(), src2.H8()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } else { // signed -> unsigned assert_always(); @@ -1750,11 +1769,17 @@ struct PACK : Sequence> { } else { if (IsPackOutSaturate(flags)) { // signed -> signed + saturate + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 e.SQXTN(i.dest.reg().toD().B8(), i.src2.reg().H8()); e.SQXTN2(i.dest.reg().B16(), i.src1.reg().H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.SQXTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); + e.SQXTN2(i.dest.reg().B16(), i.src2.reg().H8()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } else { // signed -> signed assert_always(); @@ -1780,18 +1805,40 @@ struct PACK : Sequence> { e.LoadConstantV(src2, i.src2.constant()); } - e.UQXTN(i.dest.reg().toD().H4(), src2.S4()); - e.UQXTN2(i.dest.reg().H8(), src1.S4()); + // Create saturation limit: 0xFFFF in all lanes + e.MOV(W0, 0xFFFF); + e.DUP(Q2.S4(), W0); + // Saturate both sources + e.UMIN(Q0.S4(), src1.S4(), Q2.S4()); // Saturate src1 (v3) + e.UMIN(Q1.S4(), src2.S4(), Q2.S4()); // Saturate src2 (v4) + + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 + e.UQXTN(i.dest.reg().toD().H4(), + Q1.S4()); // Pack src2 to lower 64 bits + e.UQXTN2(i.dest.reg().H8(), Q0.S4()); // Pack src1 to upper 64 bits e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.UQXTN(i.dest.reg().toD().H4(), + Q0.S4()); // Pack src1 to lower 64 bits + e.UQXTN2(i.dest.reg().H8(), Q1.S4()); // Pack src2 to upper 64 bits + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } else { // unsigned -> unsigned + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 e.XTN(i.dest.reg().toD().H4(), i.src2.reg().S4()); e.XTN2(i.dest.reg().H8(), i.src1.reg().S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.XTN(i.dest.reg().toD().H4(), i.src1.reg().S4()); + e.XTN2(i.dest.reg().H8(), i.src2.reg().S4()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } } else { if (IsPackOutSaturate(flags)) { @@ -1806,11 +1853,17 @@ struct PACK : Sequence> { if (IsPackOutUnsigned(flags)) { if (IsPackOutSaturate(flags)) { // signed -> unsigned + saturate + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 e.SQXTUN(i.dest.reg().toD().H4(), i.src2.reg().S4()); e.SQXTUN2(i.dest.reg().H8(), i.src1.reg().S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.SQXTUN(i.dest.reg().toD().H4(), i.src1.reg().S4()); + e.SQXTUN2(i.dest.reg().H8(), i.src2.reg().S4()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } else { // signed -> unsigned assert_always(); @@ -1827,11 +1880,17 @@ struct PACK : Sequence> { if (i.src2.is_constant) { e.LoadConstantV(src2, i.src2.constant()); } + // Windows ARM64 requires src2->src1 ordering with EXT swap +#ifdef _WIN32 e.SQXTN(i.dest.reg().toD().H4(), src2.S4()); e.SQXTN2(i.dest.reg().H8(), src1.S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); +#else + e.SQXTN(i.dest.reg().toD().H4(), src1.S4()); + e.SQXTN2(i.dest.reg().H8(), src2.S4()); + e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); +#endif } else { // signed -> signed assert_always(); From 82061a5ffd701161302cebd8abced52f02dd7905 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 11:59:24 +0900 Subject: [PATCH 171/179] [A64/Vector] Normalize TBL list usage - Use List{} for NEON table shuffles in pack/unpack paths - Clarify D3DCOLOR unpack commentary while touching shuffles --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 55925cadc..34878ff15 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1643,14 +1643,14 @@ struct PACK : Sequence> { 0x03'02'01'00 + 0x04'04'04'04 * 3, 0x03'02'01'00 + 0x04'04'04'04 * 0, 0x03'02'01'00 + 0x04'04'04'04 * 1)); - e.TBL(Q0.B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.TBL(Q0.B16(), List{i.dest.reg().B16()}, Q1.B16()); e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); e.LoadConstantV(Q1, vec128i(0x03'02'01'00 + 0x04'04'04'04 * 1, 0x03'02'01'00 + 0x04'04'04'04 * 0, 0x03'02'01'00 + 0x04'04'04'04 * 3, 0x03'02'01'00 + 0x04'04'04'04 * 2)); - e.TBL(Q0.B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.TBL(Q0.B16(), List{i.dest.reg().B16()}, Q1.B16()); e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); } static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { @@ -1681,10 +1681,10 @@ struct PACK : Sequence> { // Place XZ where they're supposed to be. e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_PermuteXZ)); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16()}, Q1.B16()); + e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q1.B16()); // Place YW. e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_PermuteYW)); - e.TBL(Q0.B16(), oaknut::List{Q0.B16()}, Q1.B16()); + e.TBL(Q0.B16(), List{Q0.B16()}, Q1.B16()); // Merge XZ and YW. e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); } @@ -1956,12 +1956,14 @@ struct UNPACK : Sequence> { } else { src = i.src1; } - // src = ZZYYXXWW + // src = ZZYYXXWW (ARGB packed as bytes) // Unpack to 000000ZZ,000000YY,000000XX,000000WW e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackD3DCOLOR)); - e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); - // Add 1.0f to each. - e.FMOV(Q1.S4(), FImm8(0, 7, 0)); + e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); + // Convert to float by adding float bias + // XOR with 1.0f (0x3F800000) sets the exponent for values 0-127 + // For values 128-255, we need to handle differently + e.FMOV(Q1.S4(), FImm8(0, 7, 0)); // Load 1.0f (0x3F800000) e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); // To convert to 0 to 1, games multiply by 0x47008081 and add 0xC7008081. } @@ -2068,7 +2070,7 @@ struct UNPACK : Sequence> { } // Shuffle bytes. e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_2)); - e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); // If negative, make smaller than 3 - sign extend before adding. e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 16); @@ -2110,7 +2112,7 @@ struct UNPACK : Sequence> { } // Shuffle bytes. e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_4)); - e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); // If negative, make smaller than 3 - sign extend before adding. e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 16); @@ -2190,7 +2192,7 @@ struct UNPACK : Sequence> { // Extract pairs of nibbles to XZYW. XZ will have excess 4 upper bits, YW // will have excess 4 lower bits. e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackULONG_4202020_Permute)); - e.TBL(i.dest.reg().B16(), oaknut::List{src.B16()}, Q1.B16()); + e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); // Drop the excess nibble of YW. e.USHR(Q0.S4(), i.dest.reg().S4(), 4); @@ -2199,16 +2201,14 @@ struct UNPACK : Sequence> { 2 * 0x04'04'04'04 + 0x03'02'01'00, 1 * 0x04'04'04'04 + 0x03'02'01'00, 0 * 0x04'04'04'04 + 0x03'02'01'00)); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16(), Q0.B16()}, - Q1.B16()); + e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16(), Q0.B16()}, Q1.B16()); // Reorder as XYZW. e.LoadConstantV(Q1, vec128i(3 * 0x04'04'04'04 + 0x03'02'01'00, 1 * 0x04'04'04'04 + 0x03'02'01'00, 2 * 0x04'04'04'04 + 0x03'02'01'00, 0 * 0x04'04'04'04 + 0x03'02'01'00)); - e.TBL(i.dest.reg().B16(), oaknut::List{i.dest.reg().B16(), Q0.B16()}, - Q1.B16()); + e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16(), Q0.B16()}, Q1.B16()); // Drop the excess upper nibble in XZ and sign-extend XYZ. e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 12); e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 12); From a8b39dc20a5ae4fd948c1f5a376b07a00ec20726 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 12:00:17 +0900 Subject: [PATCH 172/179] [A64/Vector] Handle FLOAT16 unpack sentinels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Map 0x7FFF/0xFFFF halfword sentinels to ±131008.0f - Rewrite FLOAT16_2 emit path to preserve Xbox 360 ordering - Use reinterpret_cast loads/stores in unpack emulation helpers --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 108 ++++++++++++++++++-- 1 file changed, 97 insertions(+), 11 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 34878ff15..d3d1bdbbb 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -1970,34 +1970,116 @@ struct UNPACK : Sequence> { static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { alignas(16) uint16_t a[4]; alignas(16) float b[8]; - vst1q_u8(a, vld1q_u8(src1)); + + // Load NEON registers into a C array by casting to uint8_t* + vst1q_u8(reinterpret_cast(a), + vld1q_u8(reinterpret_cast(src1))); std::memset(b, 0, sizeof(b)); for (int i = 0; i < 2; i++) { - b[i] = half_float::detail::half2float(a[VEC128_W(6 + i)]); + uint16_t h = a[VEC128_W(6 + i)]; + + // Xbox 360 sentinel value handling + if (h == 0x7FFF) { + b[i] = 131008.0f; // Special positive sentinel (0x47FFE000) + } else if (h == 0xFFFF) { + b[i] = -131008.0f; // Special negative sentinel (0xC7FFE000) + } else { + b[i] = half_float::detail::half2float(h); + } } // Constants, or something b[2] = 0.f; b[3] = 1.f; - return vld1q_u8(b); + // Store the float array into a uint8x16_t NEON register + return vld1q_u8(reinterpret_cast(b)); } static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { // 1 bit sign, 5 bit exponent, 10 bit mantissa // D3D10 half float format if (e.IsFeatureEnabled(kA64EmitF16C)) { - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; + // Load source properly + const QReg src = i.src1.is_constant ? Q0 : i.src1; if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); + e.LoadConstantV(src, i.src1.constant()); } - // Move the upper 4 bytes to the lower 4 bytes, zero the rest - e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16(), 12); + // 1) Put src bytes [12..15] into bytes [0..3] of a temp + // EXT(..., #12) copies exactly those 4 bytes; the rest are zero + QReg halves = Q1; + e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); // Q0 = 0 + e.EXT(halves.B16(), src.B16(), Q0.B16(), + 12); // halves[0..3] = src[12..15] - e.FCVTL(i.dest.reg().S4(), i.dest.reg().toD().H4()); + // Keep copy for sentinel detection (before conversion) + QReg halves_copy = Q2; + e.MOV(halves_copy.B16(), halves.B16()); + + // 2) Convert low 4 halfwords -> 4 floats. FCVTL reads H0..H3 (low 64 + // bits) After the EXT, H0 = low halfword of element 3, H1 = high + // halfword, H2 = H3 = 0 + e.FCVTL(i.dest.reg().S4(), halves.toD().H4()); + + // Note: We do NOT swap the order - the natural order from EXT is correct + + // 3) Xbox 360 rule: 0x7FFF → +131008.0f, 0xFFFF → -131008.0f + + // Create sentinel patterns for halfword comparison + QReg h_7FFF = Q3; + QReg h_FFFF = Q4; + + // Create 0xFFFF - all bytes 0xFF + e.MOVI(h_FFFF.B16(), 0xFF); // All bytes 0xFF = 0xFFFF per halfword + // Create 0x7FFF by shifting 0xFFFF right by 1 + e.USHR(h_7FFF.H8(), h_FFFF.H8(), 1); // 0xFFFF >> 1 = 0x7FFF + + // Compare only the low 64 bits (H0-H3) with sentinels + // After EXT, only H0 and H1 have data, H2 and H3 are zero + QReg mask_7FFF_H = Q5; + QReg mask_FFFF_H = Q6; + // Initialize masks to zero first + e.EOR(mask_7FFF_H.B16(), mask_7FFF_H.B16(), mask_7FFF_H.B16()); + e.EOR(mask_FFFF_H.B16(), mask_FFFF_H.B16(), mask_FFFF_H.B16()); + // Compare only the low 64 bits + e.CMEQ(mask_7FFF_H.toD().H4(), halves_copy.toD().H4(), h_7FFF.toD().H4()); + e.CMEQ(mask_FFFF_H.toD().H4(), halves_copy.toD().H4(), h_FFFF.toD().H4()); + + // Widen halfword masks to word masks for blending with float32 values + // Use high-numbered registers to avoid any aliasing + QReg mask_7FFF_S = Q13; + QReg mask_FFFF_S = Q14; + e.SXTL(mask_7FFF_S.S4(), mask_7FFF_H.toD().H4()); + e.SXTL(mask_FFFF_S.S4(), mask_FFFF_H.toD().H4()); + + // Prepare replacement values: ±131008.0f (broadcast to all lanes) + QReg f_pos_131008 = Q9; + QReg f_neg_131008 = Q10; + // Load immediate into S[0] then duplicate + e.MOV(W0, 0x47FFE000); + e.MOV(f_pos_131008.Selem()[0], W0); + e.DUP(f_pos_131008.S4(), f_pos_131008.Selem()[0]); // +131008.0f + + e.MOV(W0, 0xC7FFE000); + e.MOV(f_neg_131008.Selem()[0], W0); + e.DUP(f_neg_131008.S4(), f_neg_131008.Selem()[0]); // -131008.0f + + // 5) Blend using BIT which has clearer semantics + // BIT Vd, Vn, Vm => Vd = (Vn & Vm) | (Vd & ~Vm) + // When mask==0: result = (replacement & 0) | (original & 0xFFFF) = + // original When mask==0xFFFF: result = (replacement & 0xFFFF) | (original + // & 0) = replacement This is what we want! + + // Apply sentinel replacements using BIT + // BIT Vd, Vn, Vm => Vd = (Vn & Vm) | (Vd & ~Vm) + // When mask is 0: keep original, when mask is 0xFFFFFFFF: use replacement + e.BIT(i.dest.reg().B16(), f_pos_131008.B16(), mask_7FFF_S.B16()); + e.BIT(i.dest.reg().B16(), f_neg_131008.B16(), mask_FFFF_S.B16()); + + // 6) Swap S0 and S1 to match Xbox 360 halfword read order + // The software reads halfword 7 first, then 6, but EXT gives us 6 then 7 e.REV64(i.dest.reg().S4(), i.dest.reg().S4()); // Write 1.0 to element 3 @@ -2017,13 +2099,17 @@ struct UNPACK : Sequence> { static uint8x16_t EmulateFLOAT16_4(void*, std::byte src1[16]) { alignas(16) uint16_t a[4]; alignas(16) float b[8]; - vst1q_u8(a, vld1q_u8(src1)); + + // Load NEON registers into a C array by casting to uint8_t* + vst1q_u8(reinterpret_cast(a), + vld1q_u8(reinterpret_cast(src1))); for (int i = 0; i < 4; i++) { b[i] = half_float::detail::half2float(a[VEC128_W(4 + i)]); } - return vld1q_u8(b); + // Store the float array into a uint8x16_t NEON register + return vld1q_u8(reinterpret_cast(b)); } static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { // src = [(dest.x | dest.y), (dest.z | dest.w), 0, 0] From ab942708ffe9f183ba4cfeb254d9cfd967299b8b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Sun, 18 Jan 2026 19:47:18 +0900 Subject: [PATCH 173/179] [A64] Handle constant F64<-F32 conversion - Load constant sources into a temp S register before FCVT. --- src/xenia/cpu/backend/a64/a64_sequences.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc index 3fbcfc73a..fd9d3bbda 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -344,8 +344,11 @@ struct CONVERT_F64_I64 struct CONVERT_F64_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.vcvtss2sd(i.dest, i.src1); - e.FCVT(i.dest.reg().toD(), i.src1.reg().toS()); + const SReg src = i.src1.is_constant ? S0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src.toQ(), i.src1.constant()); + } + e.FCVT(i.dest.reg().toD(), src.toS()); } }; EMITTER_OPCODE_TABLE(OPCODE_CONVERT, CONVERT_I32_F32, CONVERT_I32_F64, From a12dddcf27008544badfbaef348dd5e3b0bc6600 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Sun, 18 Jan 2026 20:21:13 +0900 Subject: [PATCH 174/179] [A64] Mask vector SHR immediates Keep USHR constant shifts within lane width to avoid Oaknut ImmRange asserts. --- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index d3d1bdbbb..f9a70f1bb 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -663,7 +663,7 @@ struct VECTOR_SHR_V128 } if (all_same) { // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0]); + e.USHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); @@ -687,7 +687,7 @@ struct VECTOR_SHR_V128 } if (all_same) { // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0]); + e.USHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0] & 0xF); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); @@ -711,7 +711,7 @@ struct VECTOR_SHR_V128 } if (all_same) { // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0]); + e.USHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0] & 0x1F); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); From 745581d4bd9cfb711a9bcc9dc78d80eaba1dc14c Mon Sep 17 00:00:00 2001 From: Will Martin Date: Mon, 19 Jan 2026 01:34:56 +0900 Subject: [PATCH 175/179] [A64] Handle large stack local offsets in local loads/stores - Emit register-based SP addressing when immediate offsets exceed the scaled 12-bit range. --- src/xenia/cpu/backend/a64/a64_seq_memory.cc | 61 ++++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_memory.cc b/src/xenia/cpu/backend/a64/a64_seq_memory.cc index d7d66a14d..3fb205a6a 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_memory.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_memory.cc @@ -286,52 +286,76 @@ EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, // OPCODE_LOAD_LOCAL // ============================================================================ // Note: all types are always aligned on the stack. +template +void EmitLocalAccess(A64Emitter& e, uint32_t offset, uint32_t scale, + const EmitFn& emit_fn) { + const bool imm_valid = (offset % scale) == 0 && (offset / scale) <= 0xFFF; + if (imm_valid) { + emit_fn(SP, offset); + return; + } + + auto addr = GetTempReg(e); + e.MOV(addr, offset); + e.ADD(addr, SP, addr); + emit_fn(addr, 0); +} + struct LOAD_LOCAL_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDRB(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 1, [&](auto base, uint32_t imm) { + e.LDRB(i.dest, base, imm); + }); // e.TraceLoadI8(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDRH(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 2, [&](auto base, uint32_t imm) { + e.LDRH(i.dest, base, imm); + }); // e.TraceLoadI16(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 4, + [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); // e.TraceLoadI32(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 8, + [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); // e.TraceLoadI64(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 4, + [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); // e.TraceLoadF32(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 8, + [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); // e.TraceLoadF64(DATA_LOCAL, i.src1.constant, i.dest); } }; struct LOAD_LOCAL_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 16, + [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); // e.TraceLoadV128(DATA_LOCAL, i.src1.constant, i.dest); } }; @@ -347,49 +371,58 @@ struct STORE_LOCAL_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI8(DATA_LOCAL, i.src1.constant, i.src2); - e.STRB(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 1, [&](auto base, uint32_t imm) { + e.STRB(i.src2, base, imm); + }); } }; struct STORE_LOCAL_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI16(DATA_LOCAL, i.src1.constant, i.src2); - e.STRH(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 2, [&](auto base, uint32_t imm) { + e.STRH(i.src2, base, imm); + }); } }; struct STORE_LOCAL_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI32(DATA_LOCAL, i.src1.constant, i.src2); - e.STR(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 4, + [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); } }; struct STORE_LOCAL_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreI64(DATA_LOCAL, i.src1.constant, i.src2); - e.STR(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 8, + [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); } }; struct STORE_LOCAL_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreF32(DATA_LOCAL, i.src1.constant, i.src2); - e.STR(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 4, + [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); } }; struct STORE_LOCAL_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreF64(DATA_LOCAL, i.src1.constant, i.src2); - e.STR(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 8, + [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); } }; struct STORE_LOCAL_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // e.TraceStoreV128(DATA_LOCAL, i.src1.constant, i.src2); - e.STR(i.src2, SP, i.src1.constant()); + EmitLocalAccess(e, i.src1.constant(), 16, + [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); } }; EMITTER_OPCODE_TABLE(OPCODE_STORE_LOCAL, STORE_LOCAL_I8, STORE_LOCAL_I16, From 8dbb4edaddb68dedc5a4bd2d6a4f4c1b57002e32 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 21:39:54 +0900 Subject: [PATCH 176/179] [A64] Update ARM64 backend from xenia-mac delta - sync sequences/emitter/code cache/tracers from macOS integration - add NEON helpers, denorm flush, SET_NJM, and half-float fixes --- src/xenia/cpu/backend/a64/a64_assembler.cc | 2 +- src/xenia/cpu/backend/a64/a64_backend.cc | 78 +-- src/xenia/cpu/backend/a64/a64_code_cache.cc | 117 +---- .../cpu/backend/a64/a64_code_cache_posix.cc | 16 - src/xenia/cpu/backend/a64/a64_emitter.cc | 474 +----------------- src/xenia/cpu/backend/a64/a64_emitter.h | 8 + src/xenia/cpu/backend/a64/a64_seq_control.cc | 26 +- src/xenia/cpu/backend/a64/a64_seq_memory.cc | 184 +++++++ src/xenia/cpu/backend/a64/a64_seq_vector.cc | 50 +- src/xenia/cpu/backend/a64/a64_sequences.cc | 165 ++---- src/xenia/cpu/backend/a64/a64_sequences.h | 19 +- src/xenia/cpu/backend/a64/a64_tracers.cc | 9 +- src/xenia/cpu/backend/a64/premake5.lua | 60 ++- 13 files changed, 406 insertions(+), 802 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc index 262de2849..7f24cab29 100644 --- a/src/xenia/cpu/backend/a64/a64_assembler.cc +++ b/src/xenia/cpu/backend/a64/a64_assembler.cc @@ -34,7 +34,7 @@ using xe::cpu::hir::HIRBuilder; A64Assembler::A64Assembler(A64Backend* backend) : Assembler(backend), a64_backend_(backend), capstone_handle_(0) { - if (cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != + if (cs_open(CS_ARCH_AARCH64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != CS_ERR_OK) { assert_always("Failed to initialize capstone"); } diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index 46b882023..b0718867b 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -70,15 +70,12 @@ class A64ThunkEmitter : public A64Emitter { }; A64Backend::A64Backend() : Backend(), code_cache_(nullptr) { - cs_err err = cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_); + cs_err err = + cs_open(CS_ARCH_AARCH64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_); if (err) { printf("Failed on cs_open() with error returned: %u\n", err); assert_always("Failed to initialize capstone"); } - // if (cs_open(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != - // CS_ERR_OK) { - // assert_always("Failed to initialize capstone"); - // } cs_option(capstone_handle_, CS_OPT_SYNTAX, CS_OPT_SYNTAX_INTEL); cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_ON); cs_option(capstone_handle_, CS_OPT_SKIPDATA, CS_OPT_OFF); @@ -160,7 +157,7 @@ std::unique_ptr A64Backend::CreateGuestFunction( return std::make_unique(module, address); } -uint64_t ReadCapstoneReg(HostThreadContext* context, arm64_reg reg) { +uint64_t ReadCapstoneReg(HostThreadContext* context, aarch64_reg reg) { switch (reg) { case ARM64_REG_X0: return context->x[0]; @@ -300,37 +297,37 @@ bool TestCapstonePstate(arm64_cc cond, uint32_t pstate) { const bool C = !!(pstate & 0x20000000); const bool V = !!(pstate & 0x10000000); switch (cond) { - case ARM64_CC_EQ: + case ARM64CC_EQ: return (Z == true); - case ARM64_CC_NE: + case ARM64CC_NE: return (Z == false); - case ARM64_CC_HS: + case ARM64CC_HS: return (C == true); - case ARM64_CC_LO: + case ARM64CC_LO: return (C == false); - case ARM64_CC_MI: + case ARM64CC_MI: return (N == true); - case ARM64_CC_PL: + case ARM64CC_PL: return (N == false); - case ARM64_CC_VS: + case ARM64CC_VS: return (V == true); - case ARM64_CC_VC: + case ARM64CC_VC: return (V == false); - case ARM64_CC_HI: + case ARM64CC_HI: return ((C == true) && (Z == false)); - case ARM64_CC_LS: + case ARM64CC_LS: return ((C == false) || (Z == true)); - case ARM64_CC_GE: + case ARM64CC_GE: return (N == V); - case ARM64_CC_LT: + case ARM64CC_LT: return (N != V); - case ARM64_CC_GT: + case ARM64CC_GT: return ((Z == false) && (N == V)); - case ARM64_CC_LE: + case ARM64CC_LE: return ((Z == true) || (N != V)); - case ARM64_CC_AL: + case ARM64CC_AL: return true; - case ARM64_CC_NV: + case ARM64CC_NV: return false; default: assert_unhandled_case(cond); @@ -348,14 +345,14 @@ uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, insn.detail = &all_detail; cs_disasm_iter(capstone_handle_, &machine_code_ptr, &remaining_machine_code_size, &host_address, &insn); - const auto& detail = all_detail.arm64; + const auto& detail = all_detail.aarch64; switch (insn.id) { case ARM64_INS_B: case ARM64_INS_BL: { assert_true(detail.operands[0].type == ARM64_OP_IMM); const int64_t pc_offset = static_cast(detail.operands[0].imm); const bool test_passed = - TestCapstonePstate(detail.cc, thread_info->host_context.cpsr); + TestCapstonePstate(detail.cc, thread_info->host_context.pstate); if (test_passed) { return current_pc + pc_offset; } else { @@ -451,39 +448,6 @@ bool A64Backend::ExceptionCallbackThunk(Exception* ex, void* data) { } bool A64Backend::ExceptionCallback(Exception* ex) { - if (ex->code() == Exception::Code::kAccessViolation) { - const uint64_t host_pc = ex->pc(); - const uint64_t fault_address = ex->fault_address(); - uint64_t guest_pc = 0; - uint32_t host_offset = 0; - auto function = code_cache_->LookupFunction(host_pc); - if (function && function->machine_code()) { - const uint64_t function_pc = - reinterpret_cast(function->machine_code()); - host_offset = static_cast(host_pc - function_pc); - if (const auto* entry = function->LookupMachineCodeOffset(host_offset)) { - guest_pc = entry->guest_address; - } - } -#if XE_ARCH_ARM64 - auto* thread_context = ex->thread_context(); - XELOGE( - "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " - "fault=0x{:016X} op={} x21=0x{:016X} x27=0x{:016X} x28=0x{:016X}", - host_pc, guest_pc, host_offset, fault_address, - static_cast(ex->access_violation_operation()), - thread_context ? thread_context->x[21] : 0, - thread_context ? thread_context->x[27] : 0, - thread_context ? thread_context->x[28] : 0); -#else - XELOGE( - "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " - "fault=0x{:016X} op={}", - host_pc, guest_pc, host_offset, fault_address, - static_cast(ex->access_violation_operation())); -#endif - return false; - } if (ex->code() != Exception::Code::kIllegalInstruction) { // We only care about illegal instructions. Other things will be handled by // other handlers (probably). If nothing else picks it up we'll be called diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index 1bc937cf0..b82ed7b71 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -9,7 +9,6 @@ #include "xenia/cpu/backend/a64/a64_code_cache.h" -#include #include #include @@ -20,7 +19,6 @@ #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" #include "xenia/base/clock.h" -#include "xenia/base/cvar.h" #include "xenia/base/literals.h" #include "xenia/base/logging.h" #include "xenia/base/math.h" @@ -28,11 +26,6 @@ #include "xenia/cpu/function.h" #include "xenia/cpu/module.h" -DEFINE_bool(a64_indirection_table_log, false, - "Log A64 indirection table mapping and updates.", "CPU"); -DEFINE_int32(a64_indirection_table_log_limit, 32, - "Maximum number of A64 indirection table log entries.", "CPU"); - namespace xe { namespace cpu { namespace backend { @@ -40,23 +33,6 @@ namespace a64 { using namespace xe::literals; -namespace { - -bool ShouldLogIndirectionTable() { - if (!cvars::a64_indirection_table_log) { - return false; - } - const int32_t limit = cvars::a64_indirection_table_log_limit; - if (limit <= 0) { - return false; - } - static std::atomic log_count{0}; - const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); - return count < limit; -} - -} // namespace - // Define static constants for linking const size_t A64CodeCache::kIndirectionTableSize; #if XE_A64_INDIRECTION_64BIT @@ -132,16 +108,17 @@ bool A64CodeCache::Initialize() { xe::memory::AllocationType::kReserve, xe::memory::PageAccess::kReadWrite)); if (!indirection_table_base_) { - XELOGW("Preferred indirection table base unavailable; falling back"); indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( nullptr, kIndirectionTableSize, xe::memory::AllocationType::kReserve, xe::memory::PageAccess::kReadWrite)); } if (!indirection_table_base_) { XELOGE("Unable to allocate code cache indirection table"); - XELOGE("Tried preferred range {:X}-{:X} with fallback to OS-chosen", - static_cast(kIndirectionTableBase), - kIndirectionTableBase + kIndirectionTableSize); + XELOGE( + "This is likely because the {:X}-{:X} range is in use by some other " + "system DLL", + static_cast(kIndirectionTableBase), + kIndirectionTableBase + kIndirectionTableSize); return false; } indirection_table_actual_base_ = @@ -153,18 +130,8 @@ bool A64CodeCache::Initialize() { #endif #endif - if (ShouldLogIndirectionTable()) { - XELOGI( - "A64 indirection table: guest_base=0x{:08X} table_base=0x{:016X} " - "size=0x{:X} entry_bytes={}", - static_cast(kIndirectionTableBase), - static_cast(indirection_table_actual_base_), - static_cast(kIndirectionTableSize), - static_cast(kIndirectionEntrySize)); - } - // Create mmap file. This allows us to share the code cache with the debugger. - file_name_ = fmt::format("xenia_code_cache"); + file_name_ = fmt::format("xenia_code_cache_{}", Clock::QueryHostTickCount()); mapping_ = xe::memory::CreateFileMappingHandle( file_name_, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, false); @@ -193,9 +160,6 @@ bool A64CodeCache::Initialize() { mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, 0)); if (!generated_code_execute_base_) { - XELOGW( - "Fixed address mapping for generated code failed, trying OS-chosen " - "address"); generated_code_execute_base_ = reinterpret_cast(xe::memory::MapFileView( mapping_, nullptr, kGeneratedCodeSize, @@ -231,9 +195,6 @@ bool A64CodeCache::Initialize() { mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); if (!generated_code_execute_base_) { - XELOGW( - "Fixed address mapping for execute code failed, trying OS-chosen " - "address"); generated_code_execute_base_ = reinterpret_cast( xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); @@ -243,9 +204,6 @@ bool A64CodeCache::Initialize() { mapping_, reinterpret_cast(kGeneratedCodeWriteBase), kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); if (!generated_code_write_base_) { - XELOGW( - "Fixed address mapping for write code failed, trying OS-chosen " - "address"); generated_code_write_base_ = reinterpret_cast( xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); @@ -309,51 +267,27 @@ void A64CodeCache::AddIndirection64(uint32_t guest_address, } if (guest_address < kIndirectionTableBase) { - XELOGE( - "A64CodeCache::AddIndirection64: guest_address 0x{:08X} below base " - "0x{:08X}", - guest_address, static_cast(kIndirectionTableBase)); return; } const uint64_t guest_delta = guest_address - kIndirectionTableBase; - if (guest_delta & 0x3) { - XELOGW( - "A64CodeCache::AddIndirection64: guest_address 0x{:08X} not 4-byte " - "aligned (delta=0x{:X})", - guest_address, guest_delta); - } // Calculate offset from the logical base (0x80000000), not from actual table // address. const uint64_t guest_offset = (guest_delta >> 2) * kIndirectionEntrySize; if (guest_offset + kIndirectionEntrySize > kIndirectionTableSize) { - XELOGE( - "A64CodeCache::AddIndirection64: guest_address 0x{:08X} offset 0x{:X} " - "exceeds table size 0x{:X}", - guest_address, guest_offset, - static_cast(kIndirectionTableSize)); return; } uint64_t* indirection_slot = reinterpret_cast(indirection_table_base_ + guest_offset); *indirection_slot = host_address; - - if (ShouldLogIndirectionTable()) { - XELOGI( - "A64 indirection add: guest=0x{:08X} delta=0x{:X} offset=0x{:X} " - "slot=0x{:016X} host=0x{:016X}", - guest_address, guest_delta, guest_offset, - reinterpret_cast(indirection_slot), host_address); - } } #endif void A64CodeCache::CommitExecutableRange(uint32_t guest_low, uint32_t guest_high) { if (!indirection_table_base_) { - XELOGE("CommitExecutableRange: indirection_table_base_ is null!"); return; } @@ -364,10 +298,6 @@ void A64CodeCache::CommitExecutableRange(uint32_t guest_low, // Calculate offsets from the guest address base, not the table base if (guest_low < kGuestAddressBase) { - XELOGE( - "CommitExecutableRange: guest_low 0x{:08X} is below guest base " - "0x{:08X}", - guest_low, kGuestAddressBase); return; } @@ -377,10 +307,6 @@ void A64CodeCache::CommitExecutableRange(uint32_t guest_low, // Sanity check bounds; the table should fully cover the XEX guest range now. if (start_offset + size > kIndirectionTableSize) { - XELOGE( - "CommitExecutableRange: range [0x{:08X}, 0x{:08X}) exceeds table (size " - "0x{:X})", - guest_low, guest_high, (unsigned)kIndirectionTableSize); return; } @@ -391,14 +317,6 @@ void A64CodeCache::CommitExecutableRange(uint32_t guest_low, for (uint32_t i = 0; i < entry_count; i++) { p[i] = indirection_default_value_; } - - if (ShouldLogIndirectionTable()) { - XELOGI( - "A64 indirection commit: guest=[0x{:08X},0x{:08X}) " - "offset=0x{:X} size=0x{:X} entries={} base=0x{:016X}", - guest_low, guest_high, start_offset, size, entry_count, - static_cast(indirection_table_actual_base_)); - } #else // Other platforms: use 32-bit entries uint32_t start_offset = (guest_low - kIndirectionTableBase); @@ -536,20 +454,10 @@ void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, // Calculate offset from the logical guest base (0x80000000) if (guest_address < kIndirectionTableBase) { - XELOGE( - "A64CodeCache::PlaceGuestCode: ERROR - guest_address 0x{:08X} is " - "below logical base 0x{:08X}!", - guest_address, static_cast(kIndirectionTableBase)); return; } uintptr_t guest_diff = guest_address - kIndirectionTableBase; - if (guest_diff & 0x3) { - XELOGW( - "A64CodeCache::PlaceGuestCode: guest_address 0x{:08X} not 4-byte " - "aligned (delta=0x{:X})", - guest_address, guest_diff); - } uintptr_t guest_offset = (guest_diff >> 2) * kIndirectionEntrySize; // 8-byte entries uintptr_t slot_address = @@ -560,23 +468,10 @@ void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, uintptr_t table_end = reinterpret_cast(indirection_table_base_) + kIndirectionTableSize; if (slot_address >= table_end) { - XELOGE( - "A64CodeCache::PlaceGuestCode: slot 0x{:016X} beyond table end " - "0x{:016X}", - slot_address, table_end); return; } *indirection_slot = reinterpret_cast(code_execute_address); - - if (ShouldLogIndirectionTable()) { - XELOGI( - "A64 indirection place: guest=0x{:08X} diff=0x{:X} offset=0x{:X} " - "slot=0x{:016X} host=0x{:016X}", - guest_address, guest_diff, guest_offset, slot_address, - static_cast( - reinterpret_cast(code_execute_address))); - } #else uint32_t* indirection_slot = reinterpret_cast( indirection_table_base_ + (guest_address - kIndirectionTableBase)); diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc index c2c8b5df7..0f6329c46 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc @@ -21,7 +21,6 @@ #include "xenia/base/assert.h" #include "xenia/base/clock.h" -#include "xenia/base/logging.h" #include "xenia/base/math.h" #include "xenia/base/memory.h" #include "xenia/cpu/function.h" @@ -138,21 +137,6 @@ void PosixA64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, // Store in the reserved slot unwind_table_[unwind_reservation.table_slot] = unwind_info; - // Validate address alignment before cache flushing - if ((uintptr_t)code_execute_address % 4 != 0) { - XELOGW( - "PosixA64CodeCache::PlaceCode: WARNING - code address 0x{:016X} is not " - "4-byte aligned", - (uintptr_t)code_execute_address); - } - - if (func_info.code_size.total % 4 != 0) { - XELOGW( - "PosixA64CodeCache::PlaceCode: WARNING - code size {} is not 4-byte " - "aligned", - func_info.code_size.total); - } - // Flush instruction cache #ifdef XE_PLATFORM_MAC // On macOS, use sys_icache_invalidate diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index a07de29f1..cae5778e6 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -9,18 +9,13 @@ #include "xenia/cpu/backend/a64/a64_emitter.h" -#include -#include #include -#include #include #include #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" -#include "xenia/base/atomic.h" -#include "xenia/base/byte_order.h" #include "xenia/base/debugging.h" #include "xenia/base/literals.h" #include "xenia/base/logging.h" @@ -36,7 +31,6 @@ #include "xenia/cpu/cpu_flags.h" #include "xenia/cpu/function.h" #include "xenia/cpu/function_debug_info.h" -#include "xenia/cpu/ppc/ppc_opcode_info.h" #include "xenia/cpu/processor.h" #include "xenia/cpu/symbol.h" #include "xenia/cpu/thread_state.h" @@ -49,15 +43,9 @@ DEFINE_bool(debugprint_trap_log, false, "Log debugprint traps to the active debugger", "CPU"); DEFINE_bool(ignore_undefined_externs, true, "Don't exit when an undefined extern is called.", "CPU"); -DEFINE_bool(log_undefined_extern_args, false, - "Log PPC args for undefined externs (once per function).", "CPU"); DEFINE_bool(emit_source_annotations, false, "Add extra movs and nops to make disassembly easier to read.", "CPU"); -DEFINE_bool(a64_resolve_function_log, false, - "Log A64 ResolveFunction failures with module ranges.", "CPU"); -DEFINE_int32(a64_resolve_function_log_limit, 8, - "Maximum ResolveFunction failure logs.", "CPU"); namespace xe { namespace cpu { @@ -71,19 +59,6 @@ using namespace oaknut::util; namespace { -bool ShouldLogResolveFailure() { - if (!cvars::a64_resolve_function_log) { - return false; - } - const int32_t limit = cvars::a64_resolve_function_log_limit; - if (limit <= 0) { - return false; - } - static std::atomic log_count{0}; - const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); - return count < limit; -} - void AdjustStackPointer(A64Emitter& emitter, size_t stack_size, bool add) { if (!stack_size) { return; @@ -311,7 +286,7 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // Mark block labels. auto label = block->label_head; while (label) { - l(label_lookup_[label->name]); + l(*lookup_label(label)); label = label->next; } @@ -324,8 +299,9 @@ bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { // No sequence found! // NOTE: If you encounter this after adding a new instruction, do a full // rebuild! + XELOGE("Unable to process HIR opcode {}", + hir::GetOpcodeName(instr->opcode)); assert_always(); - XELOGE("Unable to process HIR opcode {}", instr->opcode->name); break; } instr = new_tail; @@ -420,313 +396,6 @@ uint64_t TrapDebugPrint(void* raw_context, uint64_t address) { return 0; } -uint64_t TrapLogRegs(void* raw_context, uint64_t address) { - static volatile int32_t log_count = 0; - if (xe::atomic_inc(&log_count) > 8) { - return 0; - } - auto guest_context = reinterpret_cast(raw_context); - if (!guest_context) { - return 0; - } - auto thread_state = guest_context->thread_state; - XELOGI( - "TraceOnInstruction 0x{:08X}: r3=0x{:016X} r4=0x{:016X} r11=0x{:016X} " - "r30=0x{:016X} r31=0x{:016X} lr=0x{:016X} ctr=0x{:016X}", - static_cast(cvars::break_on_instruction), guest_context->r[3], - guest_context->r[4], guest_context->r[11], guest_context->r[30], - guest_context->r[31], guest_context->lr, guest_context->ctr); - if (thread_state) { - auto memory = thread_state->memory(); - if (memory) { - auto page_access_to_string = [](xe::memory::PageAccess access) { - switch (access) { - case xe::memory::PageAccess::kNoAccess: - return "no-access"; - case xe::memory::PageAccess::kReadOnly: - return "read-only"; - case xe::memory::PageAccess::kReadWrite: - return "read-write"; - case xe::memory::PageAccess::kExecuteReadOnly: - return "exec-read"; - case xe::memory::PageAccess::kExecuteReadWrite: - return "exec-read-write"; - } - return "unknown"; - }; - auto heap_type_to_string = [](HeapType type) { - switch (type) { - case HeapType::kGuestVirtual: - return "guest-virtual"; - case HeapType::kGuestXex: - return "guest-xex"; - case HeapType::kGuestPhysical: - return "guest-physical"; - case HeapType::kHostPhysical: - return "host-physical"; - } - return "unknown"; - }; - auto can_read_guest = [&](uint32_t addr) -> bool { - if (!addr) { - return false; - } - auto* heap = memory->LookupHeap(addr); - if (!heap) { - return false; - } - return heap->QueryRangeAccess(addr, addr) != - xe::memory::PageAccess::kNoAccess; - }; - auto log_guest_bytes = [&](uint32_t addr, const char* label) { - if (!can_read_guest(addr)) { - auto* heap = memory->LookupHeap(addr); - XELOGI( - "TraceOnInstruction {}: addr=0x{:08X} unreadable heap={} " - "access={}", - label, addr, - heap ? heap_type_to_string(heap->heap_type()) : "none", - heap ? page_access_to_string(heap->QueryRangeAccess(addr, addr)) - : "no-access"); - return; - } - const auto* heap = memory->LookupHeap(addr); - const uint8_t* host_ptr = nullptr; - if (heap && heap->heap_type() == HeapType::kGuestPhysical) { - uint32_t physical_address = memory->GetPhysicalAddress(addr); - host_ptr = memory->TranslatePhysical(physical_address); - } else { - host_ptr = memory->TranslateVirtual(addr); - } - if (!host_ptr) { - XELOGI("TraceOnInstruction {}: addr=0x{:08X} null", label, addr); - return; - } - uint8_t bytes[16] = {}; - std::memcpy(bytes, host_ptr, sizeof(bytes)); - char ascii[sizeof(bytes) + 1] = {}; - for (size_t i = 0; i < sizeof(bytes); ++i) { - uint8_t ch = bytes[i]; - ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; - } - XELOGI( - "TraceOnInstruction {}: addr=0x{:08X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", - label, addr, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], - bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], - bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); - }; - auto log_guest_string = [&](uint32_t addr, const char* label) { - if (!can_read_guest(addr)) { - return; - } - const uint8_t* ptr = memory->TranslateVirtual(addr); - if (!ptr) { - return; - } - char buffer[129] = {}; - size_t len = 0; - for (; len < sizeof(buffer) - 1; ++len) { - char ch = static_cast(ptr[len]); - if (!ch) { - break; - } - if (!std::isprint(static_cast(ch))) { - return; - } - buffer[len] = ch; - } - if (len > 0) { - XELOGI("TraceOnInstruction {}: {}", label, buffer); - } - }; - const uint32_t guest_address = static_cast(guest_context->r[4]); - const auto* heap = memory->LookupHeap(guest_address); - if (heap) { - const uint8_t* host_ptr = nullptr; - if (heap->heap_type() == HeapType::kGuestPhysical) { - uint32_t physical_address = memory->GetPhysicalAddress(guest_address); - host_ptr = memory->TranslatePhysical(physical_address); - } else { - host_ptr = memory->TranslateVirtual(guest_address); - } - if (host_ptr) { - uint8_t bytes[16] = {}; - std::memcpy(bytes, host_ptr, sizeof(bytes)); - char ascii[sizeof(bytes) + 1] = {}; - for (size_t i = 0; i < sizeof(bytes); ++i) { - uint8_t ch = bytes[i]; - ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; - } - XELOGI( - "TraceOnInstruction mem[r4]=0x{:08X}: {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", - guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], - bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], - bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); - } - } - - auto read_u32 = [&](uint32_t addr, uint32_t* out) -> bool { - if (!can_read_guest(addr)) { - return false; - } - const auto* heap = memory->LookupHeap(addr); - if (heap->heap_type() == HeapType::kGuestPhysical) { - uint32_t physical_address = memory->GetPhysicalAddress(addr); - auto ptr = - memory->TranslatePhysical(physical_address); - if (!ptr) { - return false; - } - *out = xe::load_and_swap(ptr); - return true; - } - auto ptr = memory->TranslateVirtual(addr); - if (!ptr) { - return false; - } - *out = xe::load_and_swap(ptr); - return true; - }; - - const uint32_t trace_pc = - static_cast(cvars::break_on_instruction); - auto log_trace_instr = [&](uint32_t pc, const char* label) { - if (!can_read_guest(pc)) { - auto* heap = memory->LookupHeap(pc); - XELOGI( - "TraceOnInstruction {}: pc=0x{:08X} unreadable heap={} " - "access={}", - label, pc, heap ? heap_type_to_string(heap->heap_type()) : "none", - heap ? page_access_to_string(heap->QueryRangeAccess(pc, pc)) - : "no-access"); - return; - } - uint32_t instr = 0; - if (!read_u32(pc, &instr)) { - XELOGI("TraceOnInstruction {}: pc=0x{:08X} unreadable", label, pc); - return; - } - xe::StringBuffer disasm; - if (cpu::ppc::DisasmPPC(pc, instr, &disasm)) { - XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X} {}", label, - pc, instr, disasm.to_string_view()); - } else { - XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X}", label, pc, - instr); - } - }; - if (trace_pc) { - log_trace_instr(trace_pc - 4, "target-4"); - log_trace_instr(trace_pc, "target"); - log_trace_instr(trace_pc + 4, "target+4"); - } - if (memory->LookupHeap(trace_pc)) { - for (int offset = -4; offset <= 4; ++offset) { - uint32_t pc = trace_pc + offset * 4; - if (!memory->LookupHeap(pc)) { - continue; - } - uint32_t instr = 0; - if (!read_u32(pc, &instr)) { - XELOGI("TraceOnInstruction window: pc=0x{:08X} unreadable", pc); - continue; - } - xe::StringBuffer disasm_window; - if (cpu::ppc::DisasmPPC(pc, instr, &disasm_window)) { - XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X} {}", - pc, instr, disasm_window.to_string_view()); - } else { - XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X}", pc, - instr); - } - } - } - - const uint32_t obj_address = static_cast(guest_context->r[3]); - if (!obj_address) { - XELOGI("TraceOnInstruction r3 fields: base=0x00000000"); - } else { - const auto* obj_heap = memory->LookupHeap(obj_address); - if (obj_heap) { - uint32_t value_4 = 0; - uint32_t value_8 = 0; - uint32_t value_c = 0; - uint32_t value_20 = 0; - uint32_t value_470 = 0; - bool have_any = false; - have_any |= read_u32(obj_address + 0x4, &value_4); - have_any |= read_u32(obj_address + 0x8, &value_8); - have_any |= read_u32(obj_address + 0xC, &value_c); - have_any |= read_u32(obj_address + 0x20, &value_20); - have_any |= read_u32(obj_address + 0x470, &value_470); - if (have_any) { - XELOGI( - "TraceOnInstruction r3 fields: base=0x{:08X} +0x4=0x{:08X} " - "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", - obj_address, value_4, value_8, value_c, value_20, value_470); - if (value_20) { - log_guest_bytes(value_20, "r3+0x20"); - } - if (value_470) { - log_guest_bytes(value_470, "r3+0x470"); - } - } else { - XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} unmapped", - obj_address); - } - } else { - XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} heap=null", - obj_address); - } - log_guest_string(obj_address, "r3 string"); - } - - const uint32_t r31_address = static_cast(guest_context->r[31]); - if (!r31_address) { - XELOGI("TraceOnInstruction r31 fields: base=0x00000000"); - } else { - const auto* r31_heap = memory->LookupHeap(r31_address); - if (r31_heap) { - uint32_t value_4 = 0; - uint32_t value_8 = 0; - uint32_t value_c = 0; - uint32_t value_20 = 0; - uint32_t value_470 = 0; - bool have_any = false; - have_any |= read_u32(r31_address + 0x4, &value_4); - have_any |= read_u32(r31_address + 0x8, &value_8); - have_any |= read_u32(r31_address + 0xC, &value_c); - have_any |= read_u32(r31_address + 0x20, &value_20); - have_any |= read_u32(r31_address + 0x470, &value_470); - if (have_any) { - XELOGI( - "TraceOnInstruction r31 fields: base=0x{:08X} +0x4=0x{:08X} " - "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", - r31_address, value_4, value_8, value_c, value_20, value_470); - if (value_20) { - log_guest_bytes(value_20, "r31+0x20"); - } - if (value_470) { - log_guest_bytes(value_470, "r31+0x470"); - } - } else { - XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} unmapped", - r31_address); - } - } else { - XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} heap=null", - r31_address); - } - } - } - } - return 0; -} - uint64_t TrapDebugBreak(void* raw_context, uint64_t address) { [[maybe_unused]] auto thread_state = *reinterpret_cast(raw_context); @@ -744,9 +413,6 @@ void A64Emitter::Trap(uint16_t trap_type) { // 0x0FE00014 is a 'debug print' where r3 = buffer r4 = length CallNative(TrapDebugPrint, 0); break; - case 27: - CallNative(TrapLogRegs, 0); - break; case 0: case 22: // Always trap? @@ -771,16 +437,16 @@ void A64Emitter::UnimplementedInstr(const hir::Instr* i) { // This is used by the A64ThunkEmitter's ResolveFunctionThunk. uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { - auto thread_state = *reinterpret_cast(raw_context); - auto guest_context = thread_state->context(); + auto guest_context = reinterpret_cast(raw_context); + assert_not_null(guest_context); + auto thread_state = guest_context->thread_state; + assert_not_null(thread_state); - uint32_t guest_address; + assert_not_zero(target_address); - // Check if this is a 64-bit host address that needs to be mapped back to - // guest address + uint32_t guest_address = 0; if (target_address > 0xFFFFFFFF) { - // Precise guard: if target_address is within the PPCContext, this is a bug. - auto ctx_ptr = reinterpret_cast(thread_state->context()); + auto ctx_ptr = reinterpret_cast(guest_context); if (target_address >= ctx_ptr && target_address < ctx_ptr + sizeof(ppc::PPCContext)) { XELOGE( @@ -793,7 +459,6 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { return 0; } - // Try to find a function that contains this host address auto code_cache = static_cast( thread_state->processor()->backend()->code_cache()); auto guest_function = code_cache->LookupFunction(target_address); @@ -801,30 +466,14 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { guest_address = guest_function->MapMachineCodeToGuestAddress(target_address); } else { - // This might be a guest memory address stored in 64-bit form - // Extract the lower 32 bits as the potential guest address - uint32_t potential_guest = static_cast(target_address); - guest_address = potential_guest; + guest_address = static_cast(target_address); } } else { - // Normal 32-bit guest address guest_address = static_cast(target_address); } - // Xbox 360 guest addresses can be in these ranges: - // 0x00000000-0x3FFFFFFF: v00000000 heap (virtual) - // 0x40000000-0x7EFFFFFF: v40000000 heap (virtual) - // 0x70000000-0x7F000000: Thread stacks - // 0x80000000-0x8FFFFFFF: v80000000 heap (XEX) - // 0x90000000-0x9FFFFFFF: v90000000 heap (XEX) - // 0xA0000000-0xBFFFFFFF: vA0000000 heap (physical) - // 0xC0000000-0xDFFFFFFF: vC0000000 heap (physical) - // 0xE0000000-0xFFCFFFFF: vE0000000 heap (physical) - - // Most executable code should be in XEX ranges (0x80000000-0x9FFFFFFF) - // but allow other ranges as they may contain valid code if (guest_address == 0) { - XELOGE("ResolveFunction: guest_address is 0! This should not happen"); + XELOGE("ResolveFunction: guest_address is 0"); return 0; } @@ -835,74 +484,10 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { guest_address); XELOGE("ResolveFunction: Original target_address was 0x{:016X}", target_address); - if (ShouldLogResolveFailure()) { - const uint32_t lr_guest = static_cast(guest_context->lr); - XELOGI( - "ResolveFunction: lr=0x{:016X} ctr=0x{:016X} thread_id={} " - "target_is_host={} guest_address=0x{:08X}", - guest_context->lr, guest_context->ctr, guest_context->thread_id, - target_address > 0xFFFFFFFF, guest_address); - auto log_modules_for_address = [&](uint32_t address, const char* label) { - bool found = false; - for (auto* module : thread_state->processor()->GetModules()) { - if (!module) { - continue; - } - if (module->ContainsAddress(address)) { - XELOGI("ResolveFunction: {} module '{}' contains 0x{:08X}", label, - module->name(), address); - found = true; - } - } - if (!found) { - XELOGI("ResolveFunction: {} no module contains 0x{:08X}", label, - address); - } - }; - log_modules_for_address(lr_guest, "lr"); - log_modules_for_address(guest_address, "guest"); - - auto lr_functions = - thread_state->processor()->FindFunctionsWithAddress(lr_guest); - if (lr_functions.empty()) { - XELOGI("ResolveFunction: no resolved function covers LR 0x{:08X}", - lr_guest); - } else { - const auto* fn = lr_functions.front(); - XELOGI("ResolveFunction: LR function {} [0x{:08X},0x{:08X}) name='{}'", - lr_functions.size(), fn->address(), fn->end_address(), - fn->name()); - } - - auto* memory = thread_state->memory(); - if (!memory) { - XELOGI("ResolveFunction: no Memory available for guest dump"); - } else if (!memory->LookupHeap(guest_address)) { - XELOGI( - "ResolveFunction: guest_address 0x{:08X} not in any heap for dump", - guest_address); - } else { - const uint8_t* data = - memory->TranslateVirtual(guest_address); - std::array bytes = {}; - std::memcpy(bytes.data(), data, bytes.size()); - XELOGI( - "ResolveFunction: guest[0x{:08X}] = {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X}", - guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], - bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], - bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]); - } - } - - // This can happen if the target address doesn't point to a valid function - // Return 0 to indicate failure - the calling code should handle this return 0; } auto a64_fn = static_cast(fn); - uint64_t addr = reinterpret_cast(a64_fn->machine_code()); if (!a64_fn->machine_code()) { XELOGE( "ResolveFunction: Function at guest address 0x{:08X} has no machine " @@ -911,7 +496,7 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { return 0; } - return addr; + return reinterpret_cast(a64_fn->machine_code()); } void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { @@ -952,9 +537,11 @@ void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { } #endif } else { - XELOGE("A64 indirection table missing; ResolveFunction fallback removed"); - BRK(0xF000); - B(epilog_label()); + // Old-style resolve. + // Not too important because indirection table is almost always available. + // TODO: Overwrite the call-site with a straight call. + CallNative(&ResolveFunction, function->address()); + MOV(X16, X0); } // Actually jump/call to X16. @@ -1015,9 +602,14 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, } #endif } else { - XELOGE("A64 indirection table missing; ResolveFunction fallback removed"); - BRK(0xF000); - B(epilog_label()); + // Old-style resolve. + // Not too important because indirection table is almost always available. + MOV(X0, GetContextReg()); + MOV(W1, reg.toW()); + + MOV(X16, reinterpret_cast(ResolveFunction)); + BLR(X16); + MOV(X16, X0); } // Actually jump/call to X16. @@ -1044,19 +636,6 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) { auto function = reinterpret_cast(function_ptr); - if (cvars::log_undefined_extern_args && - function->name() == "XeKeysConsolePrivateKeySign") { - static std::atomic logged{false}; - if (!logged.exchange(true)) { - auto* context = reinterpret_cast(raw_context); - XELOGI( - "Undefined extern {} args: r3={:016X} r4={:016X} r5={:016X} " - "r6={:016X} r7={:016X} r8={:016X} r9={:016X} r10={:016X}", - function->name(), context->r[3], context->r[4], context->r[5], - context->r[6], context->r[7], context->r[8], context->r[9], - context->r[10]); - } - } if (!cvars::ignore_undefined_externs) { xe::FatalError(fmt::format("undefined extern call to {:08X} {}", function->address(), function->name().c_str())); @@ -1308,6 +887,7 @@ static const vec128_t v_consts[] = { /* VQNaN */ vec128i(0x7FC00000u), /* VInt127 */ vec128i(0x7Fu), /* V2To32 */ vec128f(0x1.0p32f), + /* VSingleDenormalMask */ vec128i(0x7F800000u), }; // First location to try and place constants. diff --git a/src/xenia/cpu/backend/a64/a64_emitter.h b/src/xenia/cpu/backend/a64/a64_emitter.h index 9330b41dc..c2352c85e 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.h +++ b/src/xenia/cpu/backend/a64/a64_emitter.h @@ -115,6 +115,7 @@ enum VConst { VQNaN, VInt127, V2To32, + VSingleDenormalMask, }; enum A64EmitterFeatureFlags { @@ -174,6 +175,13 @@ class A64Emitter : public oaknut::VectorCodeGenerator { oaknut::Label* lookup_label(const char* label_name) { return &label_lookup_[label_name]; } + oaknut::Label* lookup_label(hir::Label* label) { + assert_not_null(label); + if (label->name) { + return &label_lookup_[label->name]; + } + return &label_lookup_[label->GetIdString()]; + } oaknut::Label& epilog_label() { return *epilog_label_; } diff --git a/src/xenia/cpu/backend/a64/a64_seq_control.cc b/src/xenia/cpu/backend/a64/a64_seq_control.cc index e68d2955b..8e2d8bb11 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_control.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_control.cc @@ -419,7 +419,7 @@ EMITTER_OPCODE_TABLE(OPCODE_SET_RETURN_ADDRESS, SET_RETURN_ADDRESS); // ============================================================================ struct BRANCH : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src1.value->name); + oaknut::Label* label = e.lookup_label(i.src1.value); assert_not_null(label); e.B(*label); } @@ -432,7 +432,7 @@ EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH); struct BRANCH_TRUE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBNZ(i.src1, *label); } @@ -440,7 +440,7 @@ struct BRANCH_TRUE_I8 struct BRANCH_TRUE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBNZ(i.src1, *label); } @@ -448,7 +448,7 @@ struct BRANCH_TRUE_I16 struct BRANCH_TRUE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBNZ(i.src1, *label); } @@ -456,7 +456,7 @@ struct BRANCH_TRUE_I32 struct BRANCH_TRUE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBNZ(i.src1, *label); } @@ -464,7 +464,7 @@ struct BRANCH_TRUE_I64 struct BRANCH_TRUE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.FCMP(i.src1, 0); e.B(Cond::NE, *label); @@ -473,7 +473,7 @@ struct BRANCH_TRUE_F32 struct BRANCH_TRUE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.FCMP(i.src1, 0); e.B(Cond::NE, *label); @@ -489,7 +489,7 @@ EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16, struct BRANCH_FALSE_I8 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBZ(i.src1, *label); } @@ -498,7 +498,7 @@ struct BRANCH_FALSE_I16 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBZ(i.src1, *label); } @@ -507,7 +507,7 @@ struct BRANCH_FALSE_I32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBZ(i.src1, *label); } @@ -516,7 +516,7 @@ struct BRANCH_FALSE_I64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.CBZ(i.src1, *label); } @@ -525,7 +525,7 @@ struct BRANCH_FALSE_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.FCMP(i.src1, 0); e.B(Cond::EQ, *label); @@ -535,7 +535,7 @@ struct BRANCH_FALSE_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value->name); + oaknut::Label* label = e.lookup_label(i.src2.value); assert_not_null(label); e.FCMP(i.src1, 0); e.B(Cond::EQ, *label); diff --git a/src/xenia/cpu/backend/a64/a64_seq_memory.cc b/src/xenia/cpu/backend/a64/a64_seq_memory.cc index 3fb205a6a..d9dbf4934 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_memory.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_memory.cc @@ -23,6 +23,11 @@ namespace a64 { volatile int anchor_memory = 0; +// vec128b stores bytes in reversed 32-bit chunks; use reversed args for 0..15. +static const vec128_t kStvlShuffle = + vec128b(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); +static const vec128_t kStvrSwapMask = vec128b(static_cast(0x83)); + template XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, WReg address_register = W3) { @@ -176,6 +181,132 @@ EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_EXCHANGE, ATOMIC_EXCHANGE_I8, ATOMIC_EXCHANGE_I16, ATOMIC_EXCHANGE_I32, ATOMIC_EXCHANGE_I64); +// ============================================================================ +// OPCODE_LVL/LVR/STVL/STVR +// ============================================================================ +struct LVL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W4); + e.AND(W0, address.toW(), 0xF); + e.SUB(X1, address, X0); + + e.LDR(Q2, X1); + + e.MOV(X2, e.GetVConstPtr()); + e.LDR(Q0, X2, e.GetVConstOffset(VByteSwapMask)); + e.DUP(Q1.B16(), W0); + e.ADD(Q0.B16(), Q0.B16(), Q1.B16()); + e.TBL(i.dest.reg().B16(), List{Q2.B16()}, Q0.B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LVL, LVL_V128); + +struct LVR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W4); + e.AND(W0, address.toW(), 0xF); + e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); + + oaknut::Label done; + e.CBZ(W0, done); + + e.SUB(X1, address, X0); + e.LDR(Q2, X1); + + e.MOV(X2, e.GetVConstPtr()); + e.LDR(Q0, X2, e.GetVConstOffset(VByteSwapMask)); + e.DUP(Q1.B16(), W0); + e.ADD(Q0.B16(), Q0.B16(), Q1.B16()); + + e.MOVI(Q1.B16(), 0x10); + e.CMHS(Q3.B16(), Q0.B16(), Q1.B16()); + e.SUB(Q0.B16(), Q0.B16(), Q1.B16()); + e.MOVI(Q1.B16(), 0x80); + e.BSL(Q3.B16(), Q0.B16(), Q1.B16()); + + e.TBL(i.dest.reg().B16(), List{Q2.B16()}, Q3.B16()); + e.l(done); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LVR, LVR_V128); + +struct STVL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W4); + e.AND(W0, address.toW(), 0xF); + e.SUB(X1, address, X0); + + e.LDR(Q2, X1); + + e.MOV(X2, reinterpret_cast(&kStvlShuffle)); + e.LDR(Q0, X2); + e.DUP(Q1.B16(), W0); + e.SUB(Q0.B16(), Q0.B16(), Q1.B16()); + + e.MOV(X2, e.GetVConstPtr()); + e.LDR(Q1, X2, e.GetVConstOffset(VSwapWordMask)); + e.EOR(Q0.B16(), Q0.B16(), Q1.B16()); + + const QReg shuffled = Q3; + if (i.src2.is_constant) { + e.LoadConstantV(shuffled, i.src2.constant()); + } else { + e.MOV(shuffled.B16(), i.src2.reg().B16()); + } + e.TBL(shuffled.B16(), List{shuffled.B16()}, Q0.B16()); + + e.MOVI(Q1.B16(), 0x80); + e.CMHS(Q1.B16(), Q0.B16(), Q1.B16()); + e.BSL(Q1.B16(), Q2.B16(), shuffled.B16()); + e.STR(Q1, X1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128); + +struct STVR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W4); + e.AND(W0, address.toW(), 0xF); + + oaknut::Label done; + e.CBZ(W0, done); + + e.SUB(X1, address, X0); + e.LDR(Q2, X1); + + e.MOV(X2, reinterpret_cast(&kStvlShuffle)); + e.LDR(Q0, X2); + e.DUP(Q1.B16(), W0); + e.SUB(Q0.B16(), Q0.B16(), Q1.B16()); + + e.MOV(X2, reinterpret_cast(&kStvrSwapMask)); + e.LDR(Q1, X2); + e.EOR(Q0.B16(), Q0.B16(), Q1.B16()); + + e.MOVI(Q1.B16(), 0x0F); + e.AND(Q1.B16(), Q0.B16(), Q1.B16()); + e.MOVI(Q3.B16(), 0x80); + e.AND(Q3.B16(), Q0.B16(), Q3.B16()); + e.ORR(Q1.B16(), Q1.B16(), Q3.B16()); + + const QReg shuffled = Q3; + if (i.src2.is_constant) { + e.LoadConstantV(shuffled, i.src2.constant()); + } else { + e.MOV(shuffled.B16(), i.src2.reg().B16()); + } + e.TBL(shuffled.B16(), List{shuffled.B16()}, Q1.B16()); + + e.MOVI(Q1.B16(), 0x80); + e.CMHS(Q1.B16(), Q0.B16(), Q1.B16()); + e.BSL(Q1.B16(), Q2.B16(), shuffled.B16()); + e.STR(Q1, X1); + + e.l(done); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STVR, STVR_V128); + // ============================================================================ // OPCODE_ATOMIC_COMPARE_EXCHANGE // ============================================================================ @@ -282,6 +413,59 @@ struct ATOMIC_COMPARE_EXCHANGE_I64 EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, ATOMIC_COMPARE_EXCHANGE_I32, ATOMIC_COMPARE_EXCHANGE_I64); +// ============================================================================ +// OPCODE_RESERVED_LOAD / OPCODE_RESERVED_STORE +// ============================================================================ +struct RESERVED_LOAD_INT32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W3); + e.LDAXR(i.dest, address); + } +}; +struct RESERVED_LOAD_INT64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W3); + e.LDAXR(i.dest, address); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RESERVED_LOAD, RESERVED_LOAD_INT32, + RESERVED_LOAD_INT64); + +struct RESERVED_STORE_INT32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W3); + const WReg value = i.src2.is_constant ? W4 : i.src2; + if (i.src2.is_constant) { + e.MOV(value, static_cast(i.src2.constant())); + } + e.STLXR(W0, value, address); + e.CMP(W0, 0); + e.CSET(i.dest, Cond::EQ); + } +}; + +struct RESERVED_STORE_INT64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const XReg address = ComputeMemoryAddress(e, i.src1, W3); + const XReg value = i.src2.is_constant ? X4 : i.src2; + if (i.src2.is_constant) { + e.MOV(value, i.src2.constant()); + } + e.STLXR(W0, value, address); + e.CMP(W0, 0); + e.CSET(i.dest, Cond::EQ); + } +}; + +EMITTER_OPCODE_TABLE(OPCODE_RESERVED_STORE, RESERVED_STORE_INT32, + RESERVED_STORE_INT64); + // ============================================================================ // OPCODE_LOAD_LOCAL // ============================================================================ diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index f9a70f1bb..95517a35b 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -8,6 +8,7 @@ */ #include "xenia/cpu/backend/a64/a64_sequences.h" +#include #include #include @@ -63,6 +64,26 @@ struct VECTOR_CONVERT_F2I }; EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_F2I, VECTOR_CONVERT_F2I); +// ============================================================================ +// OPCODE_VECTOR_DENORMFLUSH +// ============================================================================ +struct VECTOR_DENORMFLUSH + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Clear denormals to signed zero, preserving sign bits. + e.MOV(X2, e.GetVConstPtr()); + e.LDR(Q0, X2, e.GetVConstOffset(VSingleDenormalMask)); + e.AND(Q0.B16(), i.src1.reg().B16(), Q0.B16()); + e.CMEQ(Q0.S4(), Q0.S4(), 0); + e.BIC(Q1.B16(), i.src1.reg().B16(), Q0.B16()); + e.LDR(Q2, X2, e.GetVConstOffset(VSignMaskF32)); + e.AND(Q2.B16(), i.src1.reg().B16(), Q2.B16()); + e.ORR(i.dest.reg().B16(), Q1.B16(), Q2.B16()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_DENORMFLUSH, VECTOR_DENORMFLUSH); + // ============================================================================ // OPCODE_LOAD_VECTOR_SHL // ============================================================================ @@ -1985,7 +2006,7 @@ struct UNPACK : Sequence> { } else if (h == 0xFFFF) { b[i] = -131008.0f; // Special negative sentinel (0xC7FFE000) } else { - b[i] = half_float::detail::half2float(h); + b[i] = half_float::detail::half2float(h); } } @@ -2105,7 +2126,7 @@ struct UNPACK : Sequence> { vld1q_u8(reinterpret_cast(src1))); for (int i = 0; i < 4; i++) { - b[i] = half_float::detail::half2float(a[VEC128_W(4 + i)]); + b[i] = half_float::detail::half2float(a[VEC128_W(4 + i)]); } // Store the float array into a uint8x16_t NEON register @@ -2414,6 +2435,31 @@ struct UNPACK : Sequence> { }; EMITTER_OPCODE_TABLE(OPCODE_UNPACK, UNPACK); +namespace { +thread_local bool a64_njm_enabled = true; + +uint64_t SetNJMForwarder(void* raw_context, uint64_t value) { + (void)raw_context; + a64_njm_enabled = value != 0; + return 0; +} +} // namespace + +// ============================================================================ +// OPCODE_SET_NJM +// ============================================================================ +struct SET_NJM_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.CallNative(SetNJMForwarder, static_cast(i.src1.constant())); + return; + } + e.UXTB(W1, i.src1); + e.CallNativeSafe(reinterpret_cast(SetNJMForwarder)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SET_NJM, SET_NJM_I8); + } // namespace a64 } // namespace backend } // namespace cpu diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc index fd9d3bbda..ae5720280 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -25,8 +25,6 @@ #include "xenia/cpu/backend/a64/a64_sequences.h" #include -#include - #include "xenia/base/assert.h" #include "xenia/base/clock.h" #include "xenia/base/logging.h" @@ -53,7 +51,6 @@ using namespace xe::cpu::hir; using xe::cpu::hir::Instr; typedef bool (*SequenceSelectFn)(A64Emitter&, const Instr*); -// std::unordered_map sequence_table; Removed // ============================================================================ // OPCODE_COMMENT @@ -355,6 +352,22 @@ EMITTER_OPCODE_TABLE(OPCODE_CONVERT, CONVERT_I32_F32, CONVERT_I32_F64, CONVERT_I64_F64, CONVERT_F32_I32, CONVERT_F32_F64, CONVERT_F64_I64, CONVERT_F64_F32); +// ============================================================================ +// OPCODE_TO_SINGLE +// ============================================================================ +struct TOSINGLE_F64_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const DReg src = i.src1.is_constant ? D1 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src.toQ(), i.src1.constant()); + } + e.FCVT(S0, src); + e.FCVT(i.dest, S0); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TO_SINGLE, TOSINGLE_F64_F64); + // ============================================================================ // OPCODE_ROUND // ============================================================================ @@ -727,107 +740,6 @@ EMITTER_OPCODE_TABLE(OPCODE_SELECT, SELECT_I8, SELECT_I16, SELECT_I32, SELECT_I64, SELECT_F32, SELECT_F64, SELECT_V128_I8, SELECT_V128_V128); -// ============================================================================ -// OPCODE_IS_TRUE -// ============================================================================ -struct IS_TRUE_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::NE); - } -}; -struct IS_TRUE_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::NE); - } -}; -struct IS_TRUE_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::NE); - } -}; -struct IS_TRUE_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::NE); - } -}; -struct IS_TRUE_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::NE); - } -}; -struct IS_TRUE_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::NE); - } -}; -struct IS_TRUE_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UMAXV(Q0.toS(), i.src1.reg().S4()); - e.MOV(W0, Q0.Selem()[0]); - e.CMP(W0, 0); - e.CSET(i.dest, Cond::NE); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_IS_TRUE, IS_TRUE_I8, IS_TRUE_I16, IS_TRUE_I32, - IS_TRUE_I64, IS_TRUE_F32, IS_TRUE_F64, IS_TRUE_V128); - -// ============================================================================ -// OPCODE_IS_FALSE -// ============================================================================ -struct IS_FALSE_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::EQ); - } -}; -struct IS_FALSE_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::EQ); - } -}; -struct IS_FALSE_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::EQ); - } -}; -struct IS_FALSE_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::EQ); - } -}; -struct IS_FALSE_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::EQ); - } -}; -struct IS_FALSE_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1.reg(), 0); - e.CSET(i.dest, Cond::EQ); - } -}; -struct IS_FALSE_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UMAXV(Q0.toS(), i.src1.reg().S4()); - e.MOV(W0, Q0.Selem()[0]); - e.CMP(W0, 0); - e.CSET(i.dest, Cond::EQ); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_IS_FALSE, IS_FALSE_I8, IS_FALSE_I16, IS_FALSE_I32, - IS_FALSE_I64, IS_FALSE_F32, IS_FALSE_F64, IS_FALSE_V128); - // ============================================================================ // OPCODE_IS_NAN // ============================================================================ @@ -2116,15 +2028,17 @@ EMITTER_OPCODE_TABLE(OPCODE_LOG2, LOG2_F32, LOG2_F64, LOG2_V128); // ============================================================================ struct DOT_PRODUCT_3_V128 : Sequence> { + I> { static void Emit(A64Emitter& e, const EmitArgType& i) { // https://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, QReg src1, QReg src2) { - e.FMUL(dest.toQ().S4(), src1.S4(), src2.S4()); - e.MOV(dest.toQ().Selem()[3], WZR); - e.FADDP(dest.toQ().S4(), dest.toQ().S4(), dest.toQ().S4()); - e.FADDP(dest.toS(), dest.toD().S2()); + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMUL(dest.S4(), src1.S4(), src2.S4()); + e.MOV(dest.Selem()[3], WZR); + e.FADDP(dest.S4(), dest.S4(), dest.S4()); + e.FADDP(S0, dest.toD().S2()); + e.FMOV(W0, S0); + e.DUP(dest.S4(), W0); }); } }; @@ -2135,14 +2049,16 @@ EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_3, DOT_PRODUCT_3_V128); // ============================================================================ struct DOT_PRODUCT_4_V128 : Sequence> { + I> { static void Emit(A64Emitter& e, const EmitArgType& i) { // https://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, QReg src1, QReg src2) { - e.FMUL(dest.toQ().S4(), src1.S4(), src2.S4()); - e.FADDP(dest.toQ().S4(), dest.toQ().S4(), dest.toQ().S4()); - e.FADDP(dest.toS(), dest.toD().S2()); + e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { + e.FMUL(dest.S4(), src1.S4(), src2.S4()); + e.FADDP(dest.S4(), dest.S4(), dest.S4()); + e.FADDP(S0, dest.toD().S2()); + e.FMOV(W0, S0); + e.DUP(dest.S4(), W0); }); } }; @@ -2782,6 +2698,18 @@ struct SET_ROUNDING_MODE_I32 }; EMITTER_OPCODE_TABLE(OPCODE_SET_ROUNDING_MODE, SET_ROUNDING_MODE_I32); +static void MaybeYieldForwarder(void* ctx) { xe::threading::MaybeYield(); } +// ============================================================================ +// OPCODE_DELAY_EXECUTION +// ============================================================================ +struct DELAY_EXECUTION + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CallNativeSafe(reinterpret_cast(MaybeYieldForwarder)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DELAY_EXECUTION, DELAY_EXECUTION); + // Include anchors to other sequence sources so they get included in the build. extern volatile int anchor_control; static int anchor_control_dest = anchor_control; @@ -2795,15 +2723,14 @@ static int anchor_vector_dest = anchor_vector; bool SelectSequence(A64Emitter* e, const hir::Instr* i, const hir::Instr** new_tail) { const InstrKey key(i); - auto& table = GetSequenceTable(); // Use the singleton accessor - auto it = table.find(key); - if (it != table.end()) { + auto it = GetSequenceTable().find(key); + if (it != GetSequenceTable().end()) { if (it->second(*e, i)) { *new_tail = i->next; return true; } } - XELOGE("No sequence match for variant {}", i->opcode->name); + XELOGE("No sequence match for variant {}", hir::GetOpcodeName(i->opcode)); return false; } diff --git a/src/xenia/cpu/backend/a64/a64_sequences.h b/src/xenia/cpu/backend/a64/a64_sequences.h index 65d4adad3..eaf4fb0f1 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.h +++ b/src/xenia/cpu/backend/a64/a64_sequences.h @@ -10,11 +10,12 @@ #ifndef XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ #define XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ -#include -#include // For logging -#include #include "xenia/cpu/hir/instr.h" +#include + +#include "xenia/base/logging.h" + namespace xe { namespace cpu { namespace backend { @@ -24,33 +25,31 @@ class A64Emitter; typedef bool (*SequenceSelectFn)(A64Emitter&, const hir::Instr*); -// Singleton accessor for sequence_table +// Singleton accessor for sequence table. inline std::unordered_map& GetSequenceTable() { static std::unordered_map sequence_table; return sequence_table; } -// Registration Functions template bool RegisterSingle() { bool inserted = GetSequenceTable().emplace(T::head_key(), T::Select).second; if (!inserted) { - std::cerr << "Warning: Duplicate head_key detected for key " - << T::head_key() << std::endl; + XELOGW("A64 sequence registration duplicate key 0x{:08X}", T::head_key()); } return inserted; } template bool RegisterAll() { - return (RegisterSingle() && ...); // Fold expression (C++17) + bool ok = true; + ((ok &= RegisterSingle()), ...); + return ok; } -// Macro for Registration #define EMITTER_OPCODE_TABLE(name, ...) \ static const bool A64_INSTR_##name = RegisterAll<__VA_ARGS__>(); -// Function to Select Sequence bool SelectSequence(A64Emitter* e, const hir::Instr* i, const hir::Instr** new_tail); diff --git a/src/xenia/cpu/backend/a64/a64_tracers.cc b/src/xenia/cpu/backend/a64/a64_tracers.cc index 146f50982..49b9ed1cb 100644 --- a/src/xenia/cpu/backend/a64/a64_tracers.cc +++ b/src/xenia/cpu/backend/a64/a64_tracers.cc @@ -34,11 +34,12 @@ bool trace_enabled = true; #define IFLUSH() #define IPRINT(s) \ if (trace_enabled && THREAD_MATCH) \ - xe::logging::AppendLogLine(xe::LogLevel::Debug, 't', s) + xe::logging::AppendLogLine(xe::LogLevel::Debug, 't', s, xe::LogSrc::Cpu) #define DFLUSH() -#define DPRINT(...) \ - if (trace_enabled && THREAD_MATCH) \ - xe::logging::AppendLogLineFormat(xe::LogLevel::Debug, 't', __VA_ARGS__) +#define DPRINT(...) \ + if (trace_enabled && THREAD_MATCH) \ + xe::logging::AppendLogLineFormat(xe::LogSrc::Cpu, xe::LogLevel::Debug, 't', \ + __VA_ARGS__) uint32_t GetTracingMode() { uint32_t mode = 0; diff --git a/src/xenia/cpu/backend/a64/premake5.lua b/src/xenia/cpu/backend/a64/premake5.lua index 32b2d51a0..2ff243630 100644 --- a/src/xenia/cpu/backend/a64/premake5.lua +++ b/src/xenia/cpu/backend/a64/premake5.lua @@ -4,28 +4,44 @@ include(project_root.."/tools/build") group("src") project("xenia-cpu-backend-a64") uuid("495f3f3e-f5e8-489a-bd0f-289d0495bc08") + + -- Apply settings only for ARM64 filter("architecture:ARM64") kind("StaticLib") - filter("architecture:not ARM64") - kind("None") + language("C++") + cppdialect("C++20") + links({ + "fmt", + "xenia-base", + "xenia-cpu", + }) + sysincludedirs({ + project_root.."/third_party/oaknut/include", + }) + defines({ + }) + + -- Add oaknut as external include to suppress warnings + filter("toolset:clang or toolset:gcc") + externalincludedirs({ + project_root.."/third_party/oaknut/include", + }) + -- Also explicitly disable the warning for third-party code + buildoptions({ + "-Wno-shorten-64-to-32", + }) + filter("toolset:msc") + includedirs({ + project_root.."/third_party/oaknut/include", + }) + filter("architecture:ARM64") + + -- Include only ARM64-specific files + local_platform_files() + + -- For non-ARM64 architectures, create an empty static lib + filter("architecture:x86_64") + kind("None") + + -- Reset filter filter({}) - language("C++") - cppdialect("C++20") - links({ - "fmt", - "xenia-base", - "xenia-cpu", - }) - defines({ - }) - - disablewarnings({ - -- Silence errors in oaknut - "4146", -- unary minus operator applied to unsigned type, result still unsigned - "4267" -- 'initializing': conversion from 'size_t' to 'uint32_t', possible loss of data - }) - - includedirs({ - project_root.."/third_party/oaknut/include", - }) - local_platform_files() From 62e5238ad1c7380a2aeeb3778245e614cbc3e46b Mon Sep 17 00:00:00 2001 From: Will Martin Date: Wed, 21 Jan 2026 22:41:42 +0900 Subject: [PATCH 177/179] [A64] Restore mac-app debug tracing and diagnostics - Reintroduce TraceOnInstruction and ResolveFunction logging helpers - Add undefined extern arg logging and resolve failure throttling cvars - Restore A64 access violation logging in exception callback - Add indirection table mapping diagnostics and alignment warnings --- src/xenia/cpu/backend/a64/a64_backend.cc | 33 ++ src/xenia/cpu/backend/a64/a64_code_cache.cc | 79 +++- .../cpu/backend/a64/a64_code_cache_posix.cc | 16 + src/xenia/cpu/backend/a64/a64_emitter.cc | 416 +++++++++++++++++- 4 files changed, 541 insertions(+), 3 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc index b0718867b..1ef9840d7 100644 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -448,6 +448,39 @@ bool A64Backend::ExceptionCallbackThunk(Exception* ex, void* data) { } bool A64Backend::ExceptionCallback(Exception* ex) { + if (ex->code() == Exception::Code::kAccessViolation) { + const uint64_t host_pc = ex->pc(); + const uint64_t fault_address = ex->fault_address(); + uint64_t guest_pc = 0; + uint32_t host_offset = 0; + auto function = code_cache_->LookupFunction(host_pc); + if (function && function->machine_code()) { + const uint64_t function_pc = + reinterpret_cast(function->machine_code()); + host_offset = static_cast(host_pc - function_pc); + if (const auto* entry = function->LookupMachineCodeOffset(host_offset)) { + guest_pc = entry->guest_address; + } + } +#if XE_ARCH_ARM64 + auto* thread_context = ex->thread_context(); + XELOGE( + "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " + "fault=0x{:016X} op={} x21=0x{:016X} x27=0x{:016X} x28=0x{:016X}", + host_pc, guest_pc, host_offset, fault_address, + static_cast(ex->access_violation_operation()), + thread_context ? thread_context->x[21] : 0, + thread_context ? thread_context->x[27] : 0, + thread_context ? thread_context->x[28] : 0); +#else + XELOGE( + "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " + "fault=0x{:016X} op={}", + host_pc, guest_pc, host_offset, fault_address, + static_cast(ex->access_violation_operation())); +#endif + return false; + } if (ex->code() != Exception::Code::kIllegalInstruction) { // We only care about illegal instructions. Other things will be handled by // other handlers (probably). If nothing else picks it up we'll be called diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc index b82ed7b71..416dbd7d2 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -9,6 +9,7 @@ #include "xenia/cpu/backend/a64/a64_code_cache.h" +#include #include #include @@ -19,6 +20,7 @@ #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" #include "xenia/base/clock.h" +#include "xenia/base/cvar.h" #include "xenia/base/literals.h" #include "xenia/base/logging.h" #include "xenia/base/math.h" @@ -33,6 +35,28 @@ namespace a64 { using namespace xe::literals; +DEFINE_bool(a64_indirection_table_log, false, + "Log A64 indirection table mapping and updates.", "CPU"); +DEFINE_int32(a64_indirection_table_log_limit, 32, + "Maximum number of A64 indirection table log entries.", "CPU"); + +namespace { + +bool ShouldLogIndirectionTable() { + if (!cvars::a64_indirection_table_log) { + return false; + } + const int32_t limit = cvars::a64_indirection_table_log_limit; + if (limit <= 0) { + return false; + } + static std::atomic log_count{0}; + const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); + return count < limit; +} + +} // namespace + // Define static constants for linking const size_t A64CodeCache::kIndirectionTableSize; #if XE_A64_INDIRECTION_64BIT @@ -108,6 +132,7 @@ bool A64CodeCache::Initialize() { xe::memory::AllocationType::kReserve, xe::memory::PageAccess::kReadWrite)); if (!indirection_table_base_) { + XELOGW("Preferred indirection table base unavailable; falling back"); indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( nullptr, kIndirectionTableSize, xe::memory::AllocationType::kReserve, xe::memory::PageAccess::kReadWrite)); @@ -115,8 +140,7 @@ bool A64CodeCache::Initialize() { if (!indirection_table_base_) { XELOGE("Unable to allocate code cache indirection table"); XELOGE( - "This is likely because the {:X}-{:X} range is in use by some other " - "system DLL", + "Tried preferred range {:X}-{:X} with fallback to OS-chosen", static_cast(kIndirectionTableBase), kIndirectionTableBase + kIndirectionTableSize); return false; @@ -130,6 +154,16 @@ bool A64CodeCache::Initialize() { #endif #endif + if (ShouldLogIndirectionTable()) { + XELOGI( + "A64 indirection table: guest_base=0x{:08X} table_base=0x{:016X} " + "size=0x{:X} entry_bytes={}", + static_cast(kIndirectionTableBase), + static_cast(indirection_table_actual_base_), + static_cast(kIndirectionTableSize), + static_cast(kIndirectionEntrySize)); + } + // Create mmap file. This allows us to share the code cache with the debugger. file_name_ = fmt::format("xenia_code_cache_{}", Clock::QueryHostTickCount()); mapping_ = xe::memory::CreateFileMappingHandle( @@ -160,6 +194,9 @@ bool A64CodeCache::Initialize() { mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, 0)); if (!generated_code_execute_base_) { + XELOGW( + "Fixed address mapping for generated code failed, trying OS-chosen " + "address"); generated_code_execute_base_ = reinterpret_cast(xe::memory::MapFileView( mapping_, nullptr, kGeneratedCodeSize, @@ -195,6 +232,9 @@ bool A64CodeCache::Initialize() { mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); if (!generated_code_execute_base_) { + XELOGW( + "Fixed address mapping for generated code failed, trying OS-chosen " + "address"); generated_code_execute_base_ = reinterpret_cast( xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); @@ -204,6 +244,9 @@ bool A64CodeCache::Initialize() { mapping_, reinterpret_cast(kGeneratedCodeWriteBase), kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); if (!generated_code_write_base_) { + XELOGW( + "Fixed address mapping for generated code failed, trying OS-chosen " + "address"); generated_code_write_base_ = reinterpret_cast( xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); @@ -267,27 +310,51 @@ void A64CodeCache::AddIndirection64(uint32_t guest_address, } if (guest_address < kIndirectionTableBase) { + XELOGE( + "A64CodeCache::AddIndirection64: guest_address 0x{:08X} below base " + "0x{:08X}", + guest_address, static_cast(kIndirectionTableBase)); return; } const uint64_t guest_delta = guest_address - kIndirectionTableBase; + if (guest_delta & 0x3) { + XELOGW( + "A64CodeCache::AddIndirection64: guest_address 0x{:08X} not 4-byte " + "aligned (delta=0x{:X})", + guest_address, guest_delta); + } // Calculate offset from the logical base (0x80000000), not from actual table // address. const uint64_t guest_offset = (guest_delta >> 2) * kIndirectionEntrySize; if (guest_offset + kIndirectionEntrySize > kIndirectionTableSize) { + XELOGE( + "A64CodeCache::AddIndirection64: guest_address 0x{:08X} offset 0x{:X} " + "exceeds table size 0x{:X}", + guest_address, guest_offset, + static_cast(kIndirectionTableSize)); return; } uint64_t* indirection_slot = reinterpret_cast(indirection_table_base_ + guest_offset); *indirection_slot = host_address; + + if (ShouldLogIndirectionTable()) { + XELOGI( + "A64 indirection add: guest=0x{:08X} delta=0x{:X} offset=0x{:X} " + "slot=0x{:016X} host=0x{:016X}", + guest_address, guest_delta, guest_offset, + reinterpret_cast(indirection_slot), host_address); + } } #endif void A64CodeCache::CommitExecutableRange(uint32_t guest_low, uint32_t guest_high) { if (!indirection_table_base_) { + XELOGE("CommitExecutableRange: indirection_table_base_ is null!"); return; } @@ -298,6 +365,10 @@ void A64CodeCache::CommitExecutableRange(uint32_t guest_low, // Calculate offsets from the guest address base, not the table base if (guest_low < kGuestAddressBase) { + XELOGE( + "CommitExecutableRange: guest_low 0x{:08X} is below guest base " + "0x{:08X}", + guest_low, kGuestAddressBase); return; } @@ -307,6 +378,10 @@ void A64CodeCache::CommitExecutableRange(uint32_t guest_low, // Sanity check bounds; the table should fully cover the XEX guest range now. if (start_offset + size > kIndirectionTableSize) { + XELOGE( + "CommitExecutableRange: range [0x{:08X}, 0x{:08X}) exceeds table (size " + "0x{:X})", + guest_low, guest_high, static_cast(kIndirectionTableSize)); return; } diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc index 0f6329c46..639cd4f7b 100644 --- a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc +++ b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc @@ -21,6 +21,7 @@ #include "xenia/base/assert.h" #include "xenia/base/clock.h" +#include "xenia/base/logging.h" #include "xenia/base/math.h" #include "xenia/base/memory.h" #include "xenia/cpu/function.h" @@ -137,6 +138,21 @@ void PosixA64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, // Store in the reserved slot unwind_table_[unwind_reservation.table_slot] = unwind_info; + // Validate address alignment before cache flushing + if (reinterpret_cast(code_execute_address) % 4 != 0) { + XELOGW( + "PosixA64CodeCache::PlaceCode: WARNING - code address 0x{:016X} is not " + "4-byte aligned", + reinterpret_cast(code_execute_address)); + } + + if (func_info.code_size.total % 4 != 0) { + XELOGW( + "PosixA64CodeCache::PlaceCode: WARNING - code size {} is not 4-byte " + "aligned", + func_info.code_size.total); + } + // Flush instruction cache #ifdef XE_PLATFORM_MAC // On macOS, use sys_icache_invalidate diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc index cae5778e6..89fc1bc63 100644 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -9,19 +9,25 @@ #include "xenia/cpu/backend/a64/a64_emitter.h" +#include +#include #include +#include #include #include #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/assert.h" +#include "xenia/base/atomic.h" +#include "xenia/base/byte_order.h" #include "xenia/base/debugging.h" #include "xenia/base/literals.h" #include "xenia/base/logging.h" #include "xenia/base/math.h" #include "xenia/base/memory.h" #include "xenia/base/profiling.h" +#include "xenia/base/string_buffer.h" #include "xenia/base/vec128.h" #include "xenia/cpu/backend/a64/a64_backend.h" #include "xenia/cpu/backend/a64/a64_code_cache.h" @@ -31,6 +37,7 @@ #include "xenia/cpu/cpu_flags.h" #include "xenia/cpu/function.h" #include "xenia/cpu/function_debug_info.h" +#include "xenia/cpu/ppc/ppc_opcode_info.h" #include "xenia/cpu/processor.h" #include "xenia/cpu/symbol.h" #include "xenia/cpu/thread_state.h" @@ -43,9 +50,15 @@ DEFINE_bool(debugprint_trap_log, false, "Log debugprint traps to the active debugger", "CPU"); DEFINE_bool(ignore_undefined_externs, true, "Don't exit when an undefined extern is called.", "CPU"); +DEFINE_bool(log_undefined_extern_args, false, + "Log PPC args for undefined externs (once per function).", "CPU"); DEFINE_bool(emit_source_annotations, false, "Add extra movs and nops to make disassembly easier to read.", "CPU"); +DEFINE_bool(a64_resolve_function_log, false, + "Log A64 ResolveFunction failures with module ranges.", "CPU"); +DEFINE_int32(a64_resolve_function_log_limit, 8, + "Maximum ResolveFunction failure logs.", "CPU"); namespace xe { namespace cpu { @@ -59,6 +72,19 @@ using namespace oaknut::util; namespace { +bool ShouldLogResolveFailure() { + if (!cvars::a64_resolve_function_log) { + return false; + } + const int32_t limit = cvars::a64_resolve_function_log_limit; + if (limit <= 0) { + return false; + } + static std::atomic log_count{0}; + const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); + return count < limit; +} + void AdjustStackPointer(A64Emitter& emitter, size_t stack_size, bool add) { if (!stack_size) { return; @@ -396,6 +422,318 @@ uint64_t TrapDebugPrint(void* raw_context, uint64_t address) { return 0; } +uint64_t TrapLogRegs(void* raw_context, uint64_t address) { + static volatile int32_t log_count = 0; + if (xe::atomic_inc(&log_count) > 8) { + return 0; + } + auto guest_context = reinterpret_cast(raw_context); + if (!guest_context) { + return 0; + } + auto thread_state = guest_context->thread_state; + XELOGI( + "TraceOnInstruction 0x{:08X}: r3=0x{:016X} r4=0x{:016X} r11=0x{:016X} " + "r30=0x{:016X} r31=0x{:016X} lr=0x{:016X} ctr=0x{:016X}", + static_cast(cvars::break_on_instruction), + guest_context->r[3], guest_context->r[4], guest_context->r[11], + guest_context->r[30], guest_context->r[31], guest_context->lr, + guest_context->ctr); + if (thread_state) { + auto memory = thread_state->memory(); + if (memory) { + auto page_access_to_string = [](xe::memory::PageAccess access) { + switch (access) { + case xe::memory::PageAccess::kNoAccess: + return "no-access"; + case xe::memory::PageAccess::kReadOnly: + return "read-only"; + case xe::memory::PageAccess::kReadWrite: + return "read-write"; + case xe::memory::PageAccess::kExecuteReadOnly: + return "exec-read"; + case xe::memory::PageAccess::kExecuteReadWrite: + return "exec-read-write"; + } + return "unknown"; + }; + auto heap_type_to_string = [](HeapType type) { + switch (type) { + case HeapType::kGuestVirtual: + return "guest-virtual"; + case HeapType::kGuestXex: + return "guest-xex"; + case HeapType::kGuestPhysical: + return "guest-physical"; + case HeapType::kHostPhysical: + return "host-physical"; + } + return "unknown"; + }; + auto can_read_guest = [&](uint32_t addr) -> bool { + if (!addr) { + return false; + } + auto* heap = memory->LookupHeap(addr); + if (!heap) { + return false; + } + return heap->QueryRangeAccess(addr, addr) != + xe::memory::PageAccess::kNoAccess; + }; + auto log_guest_bytes = [&](uint32_t addr, const char* label) { + if (!can_read_guest(addr)) { + auto* heap = memory->LookupHeap(addr); + XELOGI( + "TraceOnInstruction {}: addr=0x{:08X} unreadable heap={} " + "access={}", + label, addr, heap ? heap_type_to_string(heap->heap_type()) + : "none", + heap ? page_access_to_string(heap->QueryRangeAccess(addr, addr)) + : "no-access"); + return; + } + const auto* heap = memory->LookupHeap(addr); + const uint8_t* host_ptr = nullptr; + if (heap && heap->heap_type() == HeapType::kGuestPhysical) { + uint32_t physical_address = memory->GetPhysicalAddress(addr); + host_ptr = memory->TranslatePhysical(physical_address); + } else { + host_ptr = memory->TranslateVirtual(addr); + } + if (!host_ptr) { + XELOGI("TraceOnInstruction {}: addr=0x{:08X} null", label, addr); + return; + } + uint8_t bytes[16] = {}; + std::memcpy(bytes, host_ptr, sizeof(bytes)); + char ascii[sizeof(bytes) + 1] = {}; + for (size_t i = 0; i < sizeof(bytes); ++i) { + uint8_t ch = bytes[i]; + ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; + } + XELOGI( + "TraceOnInstruction {}: addr=0x{:08X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", + label, addr, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], + bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); + }; + auto log_guest_string = [&](uint32_t addr, const char* label) { + if (!can_read_guest(addr)) { + return; + } + const uint8_t* ptr = memory->TranslateVirtual(addr); + if (!ptr) { + return; + } + char buffer[129] = {}; + size_t len = 0; + for (; len < sizeof(buffer) - 1; ++len) { + char ch = static_cast(ptr[len]); + if (!ch) { + break; + } + if (!std::isprint(static_cast(ch))) { + return; + } + buffer[len] = ch; + } + if (len > 0) { + XELOGI("TraceOnInstruction {}: {}", label, buffer); + } + }; + const uint32_t guest_address = + static_cast(guest_context->r[4]); + const auto* heap = memory->LookupHeap(guest_address); + if (heap) { + const uint8_t* host_ptr = nullptr; + if (heap->heap_type() == HeapType::kGuestPhysical) { + uint32_t physical_address = + memory->GetPhysicalAddress(guest_address); + host_ptr = memory->TranslatePhysical(physical_address); + } else { + host_ptr = memory->TranslateVirtual(guest_address); + } + if (host_ptr) { + uint8_t bytes[16] = {}; + std::memcpy(bytes, host_ptr, sizeof(bytes)); + char ascii[sizeof(bytes) + 1] = {}; + for (size_t i = 0; i < sizeof(bytes); ++i) { + uint8_t ch = bytes[i]; + ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; + } + XELOGI( + "TraceOnInstruction mem[r4]=0x{:08X}: {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", + guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], + bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); + } + } + + auto read_u32 = [&](uint32_t addr, uint32_t* out) -> bool { + if (!can_read_guest(addr)) { + return false; + } + const auto* heap = memory->LookupHeap(addr); + if (heap->heap_type() == HeapType::kGuestPhysical) { + uint32_t physical_address = memory->GetPhysicalAddress(addr); + auto ptr = + memory->TranslatePhysical(physical_address); + if (!ptr) { + return false; + } + *out = xe::load_and_swap(ptr); + return true; + } + auto ptr = memory->TranslateVirtual(addr); + if (!ptr) { + return false; + } + *out = xe::load_and_swap(ptr); + return true; + }; + const uint32_t trace_pc = + static_cast(cvars::break_on_instruction); + auto log_trace_instr = [&](uint32_t pc, const char* label) { + if (!can_read_guest(pc)) { + auto* heap = memory->LookupHeap(pc); + XELOGI( + "TraceOnInstruction {}: pc=0x{:08X} unreadable heap={} " + "access={}", + label, pc, heap ? heap_type_to_string(heap->heap_type()) : "none", + heap ? page_access_to_string(heap->QueryRangeAccess(pc, pc)) + : "no-access"); + return; + } + uint32_t instr = 0; + if (!read_u32(pc, &instr)) { + XELOGI("TraceOnInstruction {}: pc=0x{:08X} unreadable", label, pc); + return; + } + xe::StringBuffer disasm; + if (cpu::ppc::DisasmPPC(pc, instr, &disasm)) { + XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X} {}", label, + pc, instr, disasm.to_string_view()); + } else { + XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X}", label, + pc, instr); + } + }; + if (trace_pc) { + log_trace_instr(trace_pc - 4, "target-4"); + log_trace_instr(trace_pc, "target"); + log_trace_instr(trace_pc + 4, "target+4"); + } + if (memory->LookupHeap(trace_pc)) { + for (int offset = -4; offset <= 4; ++offset) { + uint32_t pc = trace_pc + offset * 4; + if (!memory->LookupHeap(pc)) { + continue; + } + uint32_t instr = 0; + if (!read_u32(pc, &instr)) { + XELOGI("TraceOnInstruction window: pc=0x{:08X} unreadable", pc); + continue; + } + xe::StringBuffer disasm_window; + if (cpu::ppc::DisasmPPC(pc, instr, &disasm_window)) { + XELOGI( + "TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X} {}", + pc, instr, disasm_window.to_string_view()); + } else { + XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X}", pc, + instr); + } + } + } + + const uint32_t obj_address = + static_cast(guest_context->r[3]); + if (!obj_address) { + XELOGI("TraceOnInstruction r3 fields: base=0x00000000"); + } else { + const auto* obj_heap = memory->LookupHeap(obj_address); + if (obj_heap) { + uint32_t value_4 = 0; + uint32_t value_8 = 0; + uint32_t value_c = 0; + uint32_t value_20 = 0; + uint32_t value_470 = 0; + bool have_any = false; + have_any |= read_u32(obj_address + 0x4, &value_4); + have_any |= read_u32(obj_address + 0x8, &value_8); + have_any |= read_u32(obj_address + 0xC, &value_c); + have_any |= read_u32(obj_address + 0x20, &value_20); + have_any |= read_u32(obj_address + 0x470, &value_470); + if (have_any) { + XELOGI( + "TraceOnInstruction r3 fields: base=0x{:08X} +0x4=0x{:08X} " + "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", + obj_address, value_4, value_8, value_c, value_20, value_470); + if (value_20) { + log_guest_bytes(value_20, "r3+0x20"); + } + if (value_470) { + log_guest_bytes(value_470, "r3+0x470"); + } + } else { + XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} unmapped", + obj_address); + } + } else { + XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} heap=null", + obj_address); + } + log_guest_string(obj_address, "r3 string"); + } + + const uint32_t r31_address = + static_cast(guest_context->r[31]); + if (!r31_address) { + XELOGI("TraceOnInstruction r31 fields: base=0x00000000"); + } else { + const auto* r31_heap = memory->LookupHeap(r31_address); + if (r31_heap) { + uint32_t value_4 = 0; + uint32_t value_8 = 0; + uint32_t value_c = 0; + uint32_t value_20 = 0; + uint32_t value_470 = 0; + bool have_any = false; + have_any |= read_u32(r31_address + 0x4, &value_4); + have_any |= read_u32(r31_address + 0x8, &value_8); + have_any |= read_u32(r31_address + 0xC, &value_c); + have_any |= read_u32(r31_address + 0x20, &value_20); + have_any |= read_u32(r31_address + 0x470, &value_470); + if (have_any) { + XELOGI( + "TraceOnInstruction r31 fields: base=0x{:08X} +0x4=0x{:08X} " + "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", + r31_address, value_4, value_8, value_c, value_20, value_470); + if (value_20) { + log_guest_bytes(value_20, "r31+0x20"); + } + if (value_470) { + log_guest_bytes(value_470, "r31+0x470"); + } + } else { + XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} unmapped", + r31_address); + } + } else { + XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} heap=null", + r31_address); + } + } + } + } + return 0; +} + uint64_t TrapDebugBreak(void* raw_context, uint64_t address) { [[maybe_unused]] auto thread_state = *reinterpret_cast(raw_context); @@ -413,6 +751,9 @@ void A64Emitter::Trap(uint16_t trap_type) { // 0x0FE00014 is a 'debug print' where r3 = buffer r4 = length CallNative(TrapDebugPrint, 0); break; + case 27: + CallNative(TrapLogRegs, 0); + break; case 0: case 22: // Always trap? @@ -473,7 +814,7 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { } if (guest_address == 0) { - XELOGE("ResolveFunction: guest_address is 0"); + XELOGE("ResolveFunction: guest_address is 0! This should not happen"); return 0; } @@ -484,6 +825,66 @@ uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { guest_address); XELOGE("ResolveFunction: Original target_address was 0x{:016X}", target_address); + if (ShouldLogResolveFailure()) { + const uint32_t lr_guest = static_cast(guest_context->lr); + XELOGI( + "ResolveFunction: lr=0x{:016X} ctr=0x{:016X} thread_id={} " + "target_is_host={} guest_address=0x{:08X}", + guest_context->lr, guest_context->ctr, guest_context->thread_id, + target_address > 0xFFFFFFFF, guest_address); + auto log_modules_for_address = [&](uint32_t address, const char* label) { + bool found = false; + for (auto* module : thread_state->processor()->GetModules()) { + if (!module) { + continue; + } + if (module->ContainsAddress(address)) { + XELOGI("ResolveFunction: {} module '{}' contains 0x{:08X}", label, + module->name(), address); + found = true; + } + } + if (!found) { + XELOGI("ResolveFunction: {} no module contains 0x{:08X}", label, + address); + } + }; + log_modules_for_address(lr_guest, "lr"); + log_modules_for_address(guest_address, "guest"); + + auto lr_functions = + thread_state->processor()->FindFunctionsWithAddress(lr_guest); + if (lr_functions.empty()) { + XELOGI("ResolveFunction: no resolved function covers LR 0x{:08X}", + lr_guest); + } else { + const auto* fn = lr_functions.front(); + XELOGI( + "ResolveFunction: LR function {} [0x{:08X},0x{:08X}) name='{}'", + lr_functions.size(), fn->address(), fn->end_address(), fn->name()); + } + + auto* memory = thread_state->memory(); + if (!memory) { + XELOGI("ResolveFunction: no Memory available for guest dump"); + } else if (!memory->LookupHeap(guest_address)) { + XELOGI( + "ResolveFunction: guest_address 0x{:08X} not in any heap for dump", + guest_address); + } else { + const uint8_t* data = + memory->TranslateVirtual(guest_address); + std::array bytes = {}; + std::memcpy(bytes.data(), data, bytes.size()); + XELOGI( + "ResolveFunction: guest[0x{:08X}] = {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " + "{:02X} {:02X} {:02X} {:02X}", + guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], + bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]); + } + } return 0; } @@ -636,6 +1037,19 @@ void A64Emitter::CallIndirect(const hir::Instr* instr, uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) { auto function = reinterpret_cast(function_ptr); + if (cvars::log_undefined_extern_args && + function->name() == "XeKeysConsolePrivateKeySign") { + static std::atomic logged{false}; + if (!logged.exchange(true)) { + auto* context = reinterpret_cast(raw_context); + XELOGI( + "Undefined extern {} args: r3={:016X} r4={:016X} r5={:016X} " + "r6={:016X} r7={:016X} r8={:016X} r9={:016X} r10={:016X}", + function->name(), context->r[3], context->r[4], context->r[5], + context->r[6], context->r[7], context->r[8], context->r[9], + context->r[10]); + } + } if (!cvars::ignore_undefined_externs) { xe::FatalError(fmt::format("undefined extern call to {:08X} {}", function->address(), function->name().c_str())); From 6a5acb46663c121dcb922ef202a6bd40012032c4 Mon Sep 17 00:00:00 2001 From: Will Martin Date: Mon, 26 Jan 2026 00:38:37 +0900 Subject: [PATCH 178/179] [CPU/A64] Preserve 32-bit wrap in guest address adds Use 32-bit registers for guest address addition to keep wrap-around semantics before adding the host membase. --- src/xenia/cpu/backend/a64/a64_seq_memory.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_seq_memory.cc b/src/xenia/cpu/backend/a64/a64_seq_memory.cc index d9dbf4934..41f29bd06 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_memory.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_memory.cc @@ -64,8 +64,9 @@ XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, // TODO(benvanik): find a way to avoid doing this. e.MOV(W0, guest.reg().toW()); } - e.MOV(X1, offset_const); - e.ADD(X0, X0, X1); + // Guest addresses are 32-bit and wrap on addition. + e.MOV(W1, offset_const); + e.ADD(W0, W0, W1); e.ADD(address_register.toX(), e.GetMembaseReg(), X0); return address_register.toX(); From d54d2666b28b4981cb5775e928ef4dc9604aac7a Mon Sep 17 00:00:00 2001 From: Will Martin Date: Mon, 26 Jan 2026 00:42:21 +0900 Subject: [PATCH 179/179] [CPU/A64] Handle constant operands in vector sequences Load constant vectors explicitly for A64 emission helpers and sequence ops, avoiding invalid reg assumptions. --- src/xenia/cpu/backend/a64/a64_op.h | 16 +- src/xenia/cpu/backend/a64/a64_seq_vector.cc | 93 ++++++---- src/xenia/cpu/backend/a64/a64_sequences.cc | 183 ++++++++++++++++---- 3 files changed, 221 insertions(+), 71 deletions(-) diff --git a/src/xenia/cpu/backend/a64/a64_op.h b/src/xenia/cpu/backend/a64/a64_op.h index d50bd0bb3..863d9865e 100644 --- a/src/xenia/cpu/backend/a64/a64_op.h +++ b/src/xenia/cpu/backend/a64/a64_op.h @@ -533,12 +533,14 @@ struct Sequence { template static void EmitCommutativeBinaryVOp(A64Emitter& e, const EmitArgType& i, const FN& fn) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); + if (i.src1.is_constant && i.src2.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + e.LoadConstantV(Q1, i.src2.constant()); + fn(e, i.dest, REG(0), REG(1)); + } else if (i.src1.is_constant) { e.LoadConstantV(Q0, i.src1.constant()); fn(e, i.dest, REG(0), i.src2); } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); e.LoadConstantV(Q0, i.src2.constant()); fn(e, i.dest, i.src1, REG(0)); } else { @@ -549,12 +551,14 @@ struct Sequence { template static void EmitAssociativeBinaryVOp(A64Emitter& e, const EmitArgType& i, const FN& fn) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); + if (i.src1.is_constant && i.src2.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + e.LoadConstantV(Q1, i.src2.constant()); + fn(e, i.dest, REG(0), REG(1)); + } else if (i.src1.is_constant) { e.LoadConstantV(Q0, i.src1.constant()); fn(e, i.dest, REG(0), i.src2); } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); e.LoadConstantV(Q0, i.src2.constant()); fn(e, i.dest, i.src1, REG(0)); } else { diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 95517a35b..685844cab 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -558,6 +558,10 @@ struct VECTOR_SHL_V128 } static void EmitInt8(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -569,19 +573,23 @@ struct VECTOR_SHL_V128 } if (all_same) { // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); + e.SHL(i.dest.reg().B16(), src1.B16(), shamt.u8[0] & 0x7); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitInt16(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -593,19 +601,23 @@ struct VECTOR_SHL_V128 } if (all_same) { // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u8[0] & 0xF); + e.SHL(i.dest.reg().H8(), src1.H8(), shamt.u8[0] & 0xF); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitInt32(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -617,14 +629,14 @@ struct VECTOR_SHL_V128 } if (all_same) { // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u8[0] & 0x1F); + e.SHL(i.dest.reg().S4(), src1.S4(), shamt.u8[0] & 0x1F); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -673,6 +685,10 @@ struct VECTOR_SHR_V128 } static void EmitInt8(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -684,19 +700,23 @@ struct VECTOR_SHR_V128 } if (all_same) { // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); + e.USHR(i.dest.reg().B16(), src1.B16(), shamt.u8[0] & 0x7); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitInt16(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -708,19 +728,23 @@ struct VECTOR_SHR_V128 } if (all_same) { // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0] & 0xF); + e.USHR(i.dest.reg().H8(), src1.H8(), shamt.u16[0] & 0xF); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitInt32(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -732,14 +756,14 @@ struct VECTOR_SHR_V128 } if (all_same) { // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0] & 0x1F); + e.USHR(i.dest.reg().S4(), src1.S4(), shamt.u32[0] & 0x1F); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -769,6 +793,10 @@ struct VECTOR_SHA_V128 } static void EmitInt8(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -780,19 +808,23 @@ struct VECTOR_SHA_V128 } if (all_same) { // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().B16(), i.src1.reg().B16(), shamt.u8[0] & 0x7); + e.SSHR(i.dest.reg().B16(), src1.B16(), shamt.u8[0] & 0x7); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitInt16(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -804,19 +836,23 @@ struct VECTOR_SHA_V128 } if (all_same) { // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().H8(), i.src1.reg().H8(), shamt.u16[0] & 0xF); + e.SSHR(i.dest.reg().H8(), src1.H8(), shamt.u16[0] & 0xF); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } static void EmitInt32(A64Emitter& e, const EmitArgType& i) { + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } if (i.src2.is_constant) { const auto& shamt = i.src2.constant(); bool all_same = true; @@ -828,14 +864,14 @@ struct VECTOR_SHA_V128 } if (all_same) { // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().S4(), i.src1.reg().S4(), shamt.u32[0] & 0x1F); + e.SSHR(i.dest.reg().S4(), src1.S4(), shamt.u32[0] & 0x1F); return; } e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); } else { e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -1030,11 +1066,14 @@ struct EXTRACT_I32 vec128b(15, 14, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), }; if (i.src2.is_constant) { - e.UMOV(i.dest, i.src1.reg().Selem()[VEC128_D(i.src2.constant())]); + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(src1, i.src1.constant()); + } + e.UMOV(i.dest, src1.Selem()[VEC128_D(i.src2.constant())]); } else { - QReg src1 = i.src1.reg(); + QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); if (i.src1.is_constant) { - src1 = Q1; e.LoadConstantV(src1, i.src1.constant()); } @@ -1379,9 +1418,8 @@ struct PACK : Sequence> { } static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { assert_true(i.src2.value->IsConstantZero()); - QReg src = i.src1; + QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); if (i.src1.is_constant) { - src = i.dest; e.LoadConstantV(src, i.src1.constant()); } @@ -1552,9 +1590,8 @@ struct PACK : Sequence> { return; } - QReg src = i.src1; + QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); if (i.src1.is_constant) { - src = i.dest; e.LoadConstantV(src, i.src1.constant()); } else { // Copy src to dest if they're different registers @@ -1597,9 +1634,8 @@ struct PACK : Sequence> { return; } - QReg src = i.src1; + QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); if (i.src1.is_constant) { - src = i.dest; e.LoadConstantV(src, i.src1.constant()); } else { // Copy src to dest if they're different registers @@ -1677,9 +1713,8 @@ struct PACK : Sequence> { static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { // XYZ are 20 bits, signed and saturated. // W is 4 bits, unsigned and saturated. - QReg src = i.src1; + QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); if (i.src1.is_constant) { - src = i.dest; e.LoadConstantV(src, i.src1.constant()); } const XReg VConstData = X3; diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc index ae5720280..ec0522658 100644 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -123,7 +123,11 @@ struct ASSIGN_F64 : Sequence> { }; struct ASSIGN_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest.reg().B16(), i.src1.reg().B16()); + if (i.src1.is_constant) { + e.LoadConstantV(i.dest.reg(), i.src1.constant()); + } else { + e.MOV(i.dest.reg().B16(), i.src1.reg().B16()); + } } }; EMITTER_OPCODE_TABLE(OPCODE_ASSIGN, ASSIGN_I8, ASSIGN_I16, ASSIGN_I32, @@ -291,10 +295,14 @@ struct CONVERT_I32_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // TODO(benvanik): saturation check? cvtt* (trunc?) + const SReg src = i.src1.is_constant ? S0 : i.src1.reg().toS(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } if (i.instr->flags == ROUND_TO_ZERO) { - e.FCVTZS(i.dest, i.src1.reg().toS()); + e.FCVTZS(i.dest, src); } else { - e.FCVTNS(i.dest, i.src1.reg().toS()); + e.FCVTNS(i.dest, src); } } }; @@ -303,20 +311,28 @@ struct CONVERT_I32_F64 static void Emit(A64Emitter& e, const EmitArgType& i) { // Intel returns 0x80000000 if the double value does not fit within an int32 // ARM64 and PPC saturates the value instead + const DReg src = i.src1.is_constant ? D0 : i.src1.reg().toD(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } if (i.instr->flags == ROUND_TO_ZERO) { - e.FCVTZS(i.dest, i.src1.reg().toD()); + e.FCVTZS(i.dest, src); } else { - e.FCVTNS(i.dest, i.src1.reg().toD()); + e.FCVTNS(i.dest, src); } } }; struct CONVERT_I64_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + const DReg src = i.src1.is_constant ? D0 : i.src1.reg().toD(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } if (i.instr->flags == ROUND_TO_ZERO) { - e.FCVTZS(i.dest, i.src1.reg().toD()); + e.FCVTZS(i.dest, src); } else { - e.FCVTNS(i.dest, i.src1.reg().toD()); + e.FCVTNS(i.dest, src); } } }; @@ -329,7 +345,11 @@ struct CONVERT_F32_I32 struct CONVERT_F32_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCVT(i.dest.reg().toS(), i.src1.reg().toD()); + const DReg src = i.src1.is_constant ? D0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(src.toQ(), i.src1.constant()); + } + e.FCVT(i.dest.reg().toS(), src.toD()); } }; struct CONVERT_F64_I64 @@ -373,54 +393,66 @@ EMITTER_OPCODE_TABLE(OPCODE_TO_SINGLE, TOSINGLE_F64_F64); // ============================================================================ struct ROUND_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + const SReg src = i.src1.is_constant ? S0 : i.src1.reg().toS(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } switch (i.instr->flags) { case ROUND_TO_ZERO: - e.FRINTZ(i.dest.reg().toS(), i.src1.reg().toS()); + e.FRINTZ(i.dest.reg().toS(), src); break; case ROUND_TO_NEAREST: - e.FRINTN(i.dest.reg().toS(), i.src1.reg().toS()); + e.FRINTN(i.dest.reg().toS(), src); break; case ROUND_TO_MINUS_INFINITY: - e.FRINTM(i.dest.reg().toS(), i.src1.reg().toS()); + e.FRINTM(i.dest.reg().toS(), src); break; case ROUND_TO_POSITIVE_INFINITY: - e.FRINTP(i.dest.reg().toS(), i.src1.reg().toS()); + e.FRINTP(i.dest.reg().toS(), src); break; } } }; struct ROUND_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + const DReg src = i.src1.is_constant ? D0 : i.src1; + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } switch (i.instr->flags) { case ROUND_TO_ZERO: - e.FRINTZ(i.dest, i.src1); + e.FRINTZ(i.dest, src); break; case ROUND_TO_NEAREST: - e.FRINTN(i.dest, i.src1); + e.FRINTN(i.dest, src); break; case ROUND_TO_MINUS_INFINITY: - e.FRINTM(i.dest, i.src1); + e.FRINTM(i.dest, src); break; case ROUND_TO_POSITIVE_INFINITY: - e.FRINTP(i.dest, i.src1); + e.FRINTP(i.dest, src); break; } } }; struct ROUND_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { + QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } switch (i.instr->flags) { case ROUND_TO_ZERO: - e.FRINTZ(i.dest.reg().S4(), i.src1.reg().S4()); + e.FRINTZ(i.dest.reg().S4(), src.S4()); break; case ROUND_TO_NEAREST: - e.FRINTN(i.dest.reg().S4(), i.src1.reg().S4()); + e.FRINTN(i.dest.reg().S4(), src.S4()); break; case ROUND_TO_MINUS_INFINITY: - e.FRINTM(i.dest.reg().S4(), i.src1.reg().S4()); + e.FRINTM(i.dest.reg().S4(), src.S4()); break; case ROUND_TO_POSITIVE_INFINITY: - e.FRINTP(i.dest.reg().S4(), i.src1.reg().S4()); + e.FRINTP(i.dest.reg().S4(), src.S4()); break; } } @@ -1814,7 +1846,11 @@ struct NEG_F64 : Sequence> { struct NEG_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { assert_true(!i.instr->flags); - e.FNEG(i.dest.reg().S4(), i.src1.reg().S4()); + QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } + e.FNEG(i.dest.reg().S4(), src.S4()); } }; EMITTER_OPCODE_TABLE(OPCODE_NEG, NEG_I8, NEG_I16, NEG_I32, NEG_I64, NEG_F32, @@ -1835,7 +1871,11 @@ struct ABS_F64 : Sequence> { }; struct ABS_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FABS(i.dest.reg().S4(), i.src1.reg().S4()); + QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } + e.FABS(i.dest.reg().S4(), src.S4()); } }; EMITTER_OPCODE_TABLE(OPCODE_ABS, ABS_F32, ABS_F64, ABS_V128); @@ -1845,17 +1885,34 @@ EMITTER_OPCODE_TABLE(OPCODE_ABS, ABS_F32, ABS_F64, ABS_V128); // ============================================================================ struct SQRT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FSQRT(i.dest, i.src1); + SReg src1 = S0; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + e.FSQRT(i.dest, src1); } }; struct SQRT_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FSQRT(i.dest, i.src1); + DReg src1 = D0; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + e.FSQRT(i.dest, src1); } }; struct SQRT_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FSQRT(i.dest.reg().S4(), i.src1.reg().S4()); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + e.FSQRT(i.dest.reg().S4(), Q0.S4()); + } else { + e.FSQRT(i.dest.reg().S4(), i.src1.reg().S4()); + } } }; EMITTER_OPCODE_TABLE(OPCODE_SQRT, SQRT_F32, SQRT_F64, SQRT_V128); @@ -1866,17 +1923,34 @@ EMITTER_OPCODE_TABLE(OPCODE_SQRT, SQRT_F32, SQRT_F64, SQRT_V128); // Altivec guarantees an error of < 1/4096 for vrsqrtefp struct RSQRT_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FRSQRTE(i.dest, i.src1); + SReg src1 = S0; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + e.FRSQRTE(i.dest, src1); } }; struct RSQRT_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FRSQRTE(i.dest, i.src1); + DReg src1 = D0; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + e.FRSQRTE(i.dest, src1); } }; struct RSQRT_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FRSQRTE(i.dest.reg().S4(), i.src1.reg().S4()); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + e.FRSQRTE(i.dest.reg().S4(), Q0.S4()); + } else { + e.FRSQRTE(i.dest.reg().S4(), i.src1.reg().S4()); + } } }; EMITTER_OPCODE_TABLE(OPCODE_RSQRT, RSQRT_F32, RSQRT_F64, RSQRT_V128); @@ -1887,17 +1961,34 @@ EMITTER_OPCODE_TABLE(OPCODE_RSQRT, RSQRT_F32, RSQRT_F64, RSQRT_V128); // Altivec guarantees an error of < 1/4096 for vrefp struct RECIP_F32 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FRECPE(i.dest, i.src1); + SReg src1 = S0; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + e.FRECPE(i.dest, src1); } }; struct RECIP_F64 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FRECPE(i.dest, i.src1); + DReg src1 = D0; + if (i.src1.is_constant) { + e.LoadConstantV(src1.toQ(), i.src1.constant()); + } else { + src1 = i.src1.reg(); + } + e.FRECPE(i.dest, src1); } }; struct RECIP_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FRECPE(i.dest.reg().S4(), i.src1.reg().S4()); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + e.FRECPE(i.dest.reg().S4(), Q0.S4()); + } else { + e.FRECPE(i.dest.reg().S4(), i.src1.reg().S4()); + } } }; EMITTER_OPCODE_TABLE(OPCODE_RECIP, RECIP_F32, RECIP_F64, RECIP_V128); @@ -1918,7 +2009,11 @@ struct POW2_F32 : Sequence> { } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); + } e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.FMOV(i.dest, S0); } @@ -1934,7 +2029,11 @@ struct POW2_F64 : Sequence> { } static void Emit(A64Emitter& e, const EmitArgType& i) { assert_always(); - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); + } e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.FMOV(i.dest, D0); } @@ -1949,7 +2048,11 @@ struct POW2_V128 : Sequence> { return vld1q_f32(values); } static void Emit(A64Emitter& e, const EmitArgType& i) { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); + if (i.src1.is_constant) { + e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); + } else { + e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); + } e.CallNativeSafe(reinterpret_cast(EmulatePow2)); e.MOV(i.dest.reg().B16(), Q0.B16()); } @@ -2295,7 +2398,11 @@ struct NOT_I64 : Sequence> { }; struct NOT_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { - e.NOT(i.dest.reg().B16(), i.src1.reg().B16()); + QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } + e.NOT(i.dest.reg().B16(), src.B16()); } }; EMITTER_OPCODE_TABLE(OPCODE_NOT, NOT_I8, NOT_I16, NOT_I32, NOT_I64, NOT_V128); @@ -2620,7 +2727,11 @@ struct BYTE_SWAP_V128 : Sequence> { static void Emit(A64Emitter& e, const EmitArgType& i) { // Reverse upper and lower 64-bit halfs - e.REV32(i.dest.reg().B16(), i.src1.reg().B16()); + QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); + if (i.src1.is_constant) { + e.LoadConstantV(Q0, i.src1.constant()); + } + e.REV32(i.dest.reg().B16(), src.B16()); } }; EMITTER_OPCODE_TABLE(OPCODE_BYTE_SWAP, BYTE_SWAP_I16, BYTE_SWAP_I32,