diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc deleted file mode 100644 index 7f24cab29..000000000 --- a/src/xenia/cpu/backend/a64/a64_assembler.cc +++ /dev/null @@ -1,152 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_assembler.h" - -#include - -#include "third_party/capstone/include/capstone/arm64.h" -#include "third_party/capstone/include/capstone/capstone.h" -#include "xenia/base/profiling.h" -#include "xenia/base/reset_scope.h" -#include "xenia/base/string.h" -#include "xenia/cpu/backend/a64/a64_backend.h" -#include "xenia/cpu/backend/a64/a64_code_cache.h" -#include "xenia/cpu/backend/a64/a64_emitter.h" -#include "xenia/cpu/backend/a64/a64_function.h" -#include "xenia/cpu/cpu_flags.h" -#include "xenia/cpu/hir/hir_builder.h" -#include "xenia/cpu/hir/label.h" -#include "xenia/cpu/processor.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -using xe::cpu::hir::HIRBuilder; - -A64Assembler::A64Assembler(A64Backend* backend) - : Assembler(backend), a64_backend_(backend), capstone_handle_(0) { - if (cs_open(CS_ARCH_AARCH64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_) != - CS_ERR_OK) { - assert_always("Failed to initialize capstone"); - } - // Remove Intel syntax option as it's not applicable to ARM64 - cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_OFF); -} - -A64Assembler::~A64Assembler() { - // Emitter must be freed before the allocator. - emitter_.reset(); - - if (capstone_handle_) { - cs_close(&capstone_handle_); - } -} - -bool A64Assembler::Initialize() { - if (!Assembler::Initialize()) { - return false; - } - - emitter_.reset(new A64Emitter(a64_backend_)); - - return true; -} - -void A64Assembler::Reset() { - string_buffer_.Reset(); - Assembler::Reset(); -} - -bool A64Assembler::Assemble(GuestFunction* function, HIRBuilder* builder, - uint32_t debug_info_flags, - std::unique_ptr debug_info) { - SCOPE_profile_cpu_f("cpu"); - - // Reset when we leave. - xe::make_reset_scope(this); - - // Lower HIR -> a64. - void* machine_code = nullptr; - size_t code_size = 0; - if (!emitter_->Emit(function, builder, debug_info_flags, debug_info.get(), - &machine_code, &code_size, &function->source_map())) { - return false; - } - - // Stash generated machine code. - if (debug_info_flags & DebugInfoFlags::kDebugInfoDisasmMachineCode) { - DumpMachineCode(machine_code, code_size, function->source_map(), - &string_buffer_); - debug_info->set_machine_code_disasm(xe_strdup(string_buffer_.buffer())); - string_buffer_.Reset(); - } - - function->set_debug_info(std::move(debug_info)); - static_cast(function)->Setup( - reinterpret_cast(machine_code), code_size); - // Install into indirection table. - const uint64_t host_address = reinterpret_cast(machine_code); -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, machine code might be allocated in high address space. - // Use the 64-bit version of AddIndirection to store the full address. - reinterpret_cast(backend_->code_cache()) - ->AddIndirection64(function->address(), host_address); -#else - assert_true((host_address >> 32) == 0); - reinterpret_cast(backend_->code_cache()) - ->AddIndirection(function->address(), - static_cast(host_address)); -#endif - - return true; -} - -void A64Assembler::DumpMachineCode( - void* machine_code, size_t code_size, - const std::vector& source_map, StringBuffer* str) { - if (source_map.empty()) { - return; - } - auto source_map_index = 0; - uint32_t next_code_offset = source_map[0].code_offset; - - const uint8_t* code_ptr = reinterpret_cast(machine_code); - size_t remaining_code_size = code_size; - uint64_t address = uint64_t(machine_code); - cs_insn insn = {}; - while (remaining_code_size && - cs_disasm_iter(capstone_handle_, &code_ptr, &remaining_code_size, - &address, &insn)) { - // Look up source offset. - auto code_offset = - uint32_t(code_ptr - reinterpret_cast(machine_code)); - if (code_offset >= next_code_offset && - source_map_index < static_cast(source_map.size())) { - auto& source_map_entry = source_map[source_map_index]; - str->AppendFormat("{:08X} ", source_map_entry.guest_address); - ++source_map_index; - next_code_offset = source_map_index < static_cast(source_map.size()) - ? source_map[source_map_index].code_offset - : UINT_MAX; - } else { - str->Append(" "); - } - - str->AppendFormat("{:08X} {:<6} {}\n", uint32_t(insn.address), - insn.mnemonic, insn.op_str); - } -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_assembler.h b/src/xenia/cpu/backend/a64/a64_assembler.h deleted file mode 100644 index 95e0a6f1e..000000000 --- a/src/xenia/cpu/backend/a64/a64_assembler.h +++ /dev/null @@ -1,59 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ -#define XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ - -#include -#include - -#include "xenia/base/string_buffer.h" -#include "xenia/cpu/backend/assembler.h" -#include "xenia/cpu/function.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -class A64Backend; -class A64Emitter; - -class A64Assembler : public Assembler { - public: - explicit A64Assembler(A64Backend* backend); - ~A64Assembler() override; - - bool Initialize() override; - - void Reset() override; - - bool Assemble(GuestFunction* function, hir::HIRBuilder* builder, - uint32_t debug_info_flags, - std::unique_ptr debug_info) override; - - private: - void DumpMachineCode(void* machine_code, size_t code_size, - const std::vector& source_map, - StringBuffer* str); - - private: - A64Backend* a64_backend_; - std::unique_ptr emitter_; - uintptr_t capstone_handle_; - - StringBuffer string_buffer_; -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc deleted file mode 100644 index 1ef9840d7..000000000 --- a/src/xenia/cpu/backend/a64/a64_backend.cc +++ /dev/null @@ -1,779 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_backend.h" - -#include - -#include "third_party/capstone/include/capstone/arm64.h" -#include "third_party/capstone/include/capstone/capstone.h" - -#include "xenia/base/exception_handler.h" -#include "xenia/base/logging.h" -#include "xenia/cpu/backend/a64/a64_assembler.h" -#include "xenia/cpu/backend/a64/a64_code_cache.h" -#include "xenia/cpu/backend/a64/a64_emitter.h" -#include "xenia/cpu/backend/a64/a64_function.h" -#include "xenia/cpu/backend/a64/a64_sequences.h" -#include "xenia/cpu/backend/a64/a64_stack_layout.h" -#include "xenia/cpu/breakpoint.h" -#include "xenia/cpu/processor.h" -#include "xenia/cpu/stack_walker.h" - -DEFINE_int32(a64_extension_mask, -1, - "Allow the detection and utilization of specific instruction set " - "features.\n" - " 0 = armv8.0\n" - " 1 = LSE\n" - " 2 = F16C\n" - " -1 = Detect and utilize all possible processor features\n", - "a64"); - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -using namespace oaknut::util; - -class A64ThunkEmitter : public A64Emitter { - public: - A64ThunkEmitter(A64Backend* backend); - ~A64ThunkEmitter() override; - HostToGuestThunk EmitHostToGuestThunk(); - GuestToHostThunk EmitGuestToHostThunk(); - ResolveFunctionThunk EmitResolveFunctionThunk(); - - private: - // The following four functions provide save/load functionality for registers. - // They assume at least StackLayout::THUNK_STACK_SIZE bytes have been - // allocated on the stack. - - // Caller saved: - // Dont assume these registers will survive a subroutine call - // x0, v0 is not saved for use as arg0/return - // x1-x15, x30 | v0-v7 and v16-v31 - void EmitSaveVolatileRegs(); - void EmitLoadVolatileRegs(); - - // Callee saved: - // Subroutines must preserve these registers if they intend to use them - // x19-x30 | d8-d15 - void EmitSaveNonvolatileRegs(); - void EmitLoadNonvolatileRegs(); -}; - -A64Backend::A64Backend() : Backend(), code_cache_(nullptr) { - cs_err err = - cs_open(CS_ARCH_AARCH64, CS_MODE_LITTLE_ENDIAN, &capstone_handle_); - if (err) { - printf("Failed on cs_open() with error returned: %u\n", err); - assert_always("Failed to initialize capstone"); - } - cs_option(capstone_handle_, CS_OPT_SYNTAX, CS_OPT_SYNTAX_INTEL); - cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_ON); - cs_option(capstone_handle_, CS_OPT_SKIPDATA, CS_OPT_OFF); -} - -A64Backend::~A64Backend() { - if (capstone_handle_) { - cs_close(&capstone_handle_); - } - - A64Emitter::FreeConstData(emitter_data_); - ExceptionHandler::Uninstall(&ExceptionCallbackThunk, this); -} - -bool A64Backend::Initialize(Processor* processor) { - if (!Backend::Initialize(processor)) { - return false; - } - - auto& gprs = machine_info_.register_sets[0]; - gprs.id = 0; - std::strcpy(gprs.name, "x"); - gprs.types = MachineInfo::RegisterSet::INT_TYPES; - gprs.count = A64Emitter::GPR_COUNT; - - auto& fprs = machine_info_.register_sets[1]; - fprs.id = 1; - std::strcpy(fprs.name, "v"); - fprs.types = MachineInfo::RegisterSet::FLOAT_TYPES | - MachineInfo::RegisterSet::VEC_TYPES; - fprs.count = A64Emitter::FPR_COUNT; - - code_cache_ = A64CodeCache::Create(); - Backend::code_cache_ = code_cache_.get(); - if (!code_cache_->Initialize()) { - return false; - } - - // Generate thunks used to transition between jitted code and host code. - A64ThunkEmitter thunk_emitter(this); - host_to_guest_thunk_ = thunk_emitter.EmitHostToGuestThunk(); - guest_to_host_thunk_ = thunk_emitter.EmitGuestToHostThunk(); - resolve_function_thunk_ = thunk_emitter.EmitResolveFunctionThunk(); - -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, we use 64-bit addresses and the indirection table now - // supports 64-bit entries, so we can store the actual thunk address directly. - static_cast(code_cache_.get()) - ->set_indirection_default_64(uint64_t(resolve_function_thunk_)); -#else - assert_zero(uint64_t(resolve_function_thunk_) & 0xFFFFFFFF00000000ull); - code_cache_->set_indirection_default( - uint32_t(uint64_t(resolve_function_thunk_))); -#endif - - // Allocate some special indirections. - code_cache_->CommitExecutableRange(0x9FFF0000, 0x9FFFFFFF); - - // Allocate emitter constant data. - emitter_data_ = A64Emitter::PlaceConstData(); - - // Setup exception callback - ExceptionHandler::Install(&ExceptionCallbackThunk, this); - - return true; -} - -void A64Backend::CommitExecutableRange(uint32_t guest_low, - uint32_t guest_high) { - code_cache_->CommitExecutableRange(guest_low, guest_high); -} - -std::unique_ptr A64Backend::CreateAssembler() { - return std::make_unique(this); -} - -std::unique_ptr A64Backend::CreateGuestFunction( - Module* module, uint32_t address) { - return std::make_unique(module, address); -} - -uint64_t ReadCapstoneReg(HostThreadContext* context, aarch64_reg reg) { - switch (reg) { - case ARM64_REG_X0: - return context->x[0]; - case ARM64_REG_X1: - return context->x[1]; - case ARM64_REG_X2: - return context->x[2]; - case ARM64_REG_X3: - return context->x[3]; - case ARM64_REG_X4: - return context->x[4]; - case ARM64_REG_X5: - return context->x[5]; - case ARM64_REG_X6: - return context->x[6]; - case ARM64_REG_X7: - return context->x[7]; - case ARM64_REG_X8: - return context->x[8]; - case ARM64_REG_X9: - return context->x[9]; - case ARM64_REG_X10: - return context->x[10]; - case ARM64_REG_X11: - return context->x[11]; - case ARM64_REG_X12: - return context->x[12]; - case ARM64_REG_X13: - return context->x[13]; - case ARM64_REG_X14: - return context->x[14]; - case ARM64_REG_X15: - return context->x[15]; - case ARM64_REG_X16: - return context->x[16]; - case ARM64_REG_X17: - return context->x[17]; - case ARM64_REG_X18: - return context->x[18]; - case ARM64_REG_X19: - return context->x[19]; - case ARM64_REG_X20: - return context->x[20]; - case ARM64_REG_X21: - return context->x[21]; - case ARM64_REG_X22: - return context->x[22]; - case ARM64_REG_X23: - return context->x[23]; - case ARM64_REG_X24: - return context->x[24]; - case ARM64_REG_X25: - return context->x[25]; - case ARM64_REG_X26: - return context->x[26]; - case ARM64_REG_X27: - return context->x[27]; - case ARM64_REG_X28: - return context->x[28]; - case ARM64_REG_X29: - return context->x[29]; - case ARM64_REG_X30: - return context->x[30]; - case ARM64_REG_W0: - return uint32_t(context->x[0]); - case ARM64_REG_W1: - return uint32_t(context->x[1]); - case ARM64_REG_W2: - return uint32_t(context->x[2]); - case ARM64_REG_W3: - return uint32_t(context->x[3]); - case ARM64_REG_W4: - return uint32_t(context->x[4]); - case ARM64_REG_W5: - return uint32_t(context->x[5]); - case ARM64_REG_W6: - return uint32_t(context->x[6]); - case ARM64_REG_W7: - return uint32_t(context->x[7]); - case ARM64_REG_W8: - return uint32_t(context->x[8]); - case ARM64_REG_W9: - return uint32_t(context->x[9]); - case ARM64_REG_W10: - return uint32_t(context->x[10]); - case ARM64_REG_W11: - return uint32_t(context->x[11]); - case ARM64_REG_W12: - return uint32_t(context->x[12]); - case ARM64_REG_W13: - return uint32_t(context->x[13]); - case ARM64_REG_W14: - return uint32_t(context->x[14]); - case ARM64_REG_W15: - return uint32_t(context->x[15]); - case ARM64_REG_W16: - return uint32_t(context->x[16]); - case ARM64_REG_W17: - return uint32_t(context->x[17]); - case ARM64_REG_W18: - return uint32_t(context->x[18]); - case ARM64_REG_W19: - return uint32_t(context->x[19]); - case ARM64_REG_W20: - return uint32_t(context->x[20]); - case ARM64_REG_W21: - return uint32_t(context->x[21]); - case ARM64_REG_W22: - return uint32_t(context->x[22]); - case ARM64_REG_W23: - return uint32_t(context->x[23]); - case ARM64_REG_W24: - return uint32_t(context->x[24]); - case ARM64_REG_W25: - return uint32_t(context->x[25]); - case ARM64_REG_W26: - return uint32_t(context->x[26]); - case ARM64_REG_W27: - return uint32_t(context->x[27]); - case ARM64_REG_W28: - return uint32_t(context->x[28]); - case ARM64_REG_W29: - return uint32_t(context->x[29]); - case ARM64_REG_W30: - return uint32_t(context->x[30]); - default: - assert_unhandled_case(reg); - return 0; - } -} - -bool TestCapstonePstate(arm64_cc cond, uint32_t pstate) { - // https://devblogs.microsoft.com/oldnewthing/20220815-00/?p=106975 - // Upper 4 bits of pstate are NZCV - const bool N = !!(pstate & 0x80000000); - const bool Z = !!(pstate & 0x40000000); - const bool C = !!(pstate & 0x20000000); - const bool V = !!(pstate & 0x10000000); - switch (cond) { - case ARM64CC_EQ: - return (Z == true); - case ARM64CC_NE: - return (Z == false); - case ARM64CC_HS: - return (C == true); - case ARM64CC_LO: - return (C == false); - case ARM64CC_MI: - return (N == true); - case ARM64CC_PL: - return (N == false); - case ARM64CC_VS: - return (V == true); - case ARM64CC_VC: - return (V == false); - case ARM64CC_HI: - return ((C == true) && (Z == false)); - case ARM64CC_LS: - return ((C == false) || (Z == true)); - case ARM64CC_GE: - return (N == V); - case ARM64CC_LT: - return (N != V); - case ARM64CC_GT: - return ((Z == false) && (N == V)); - case ARM64CC_LE: - return ((Z == true) || (N != V)); - case ARM64CC_AL: - return true; - case ARM64CC_NV: - return false; - default: - assert_unhandled_case(cond); - return false; - } -} - -uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, - uint64_t current_pc) { - auto machine_code_ptr = reinterpret_cast(current_pc); - size_t remaining_machine_code_size = 64; - uint64_t host_address = current_pc; - cs_insn insn = {}; - cs_detail all_detail = {}; - insn.detail = &all_detail; - cs_disasm_iter(capstone_handle_, &machine_code_ptr, - &remaining_machine_code_size, &host_address, &insn); - const auto& detail = all_detail.aarch64; - switch (insn.id) { - case ARM64_INS_B: - case ARM64_INS_BL: { - assert_true(detail.operands[0].type == ARM64_OP_IMM); - const int64_t pc_offset = static_cast(detail.operands[0].imm); - const bool test_passed = - TestCapstonePstate(detail.cc, thread_info->host_context.pstate); - if (test_passed) { - return current_pc + pc_offset; - } else { - return current_pc + insn.size; - } - } break; - case ARM64_INS_BR: - case ARM64_INS_BLR: { - assert_true(detail.operands[0].type == ARM64_OP_REG); - const uint64_t target_pc = - ReadCapstoneReg(&thread_info->host_context, detail.operands[0].reg); - return target_pc; - } break; - case ARM64_INS_RET: { - assert_true(detail.operands[0].type == ARM64_OP_REG); - const uint64_t target_pc = - ReadCapstoneReg(&thread_info->host_context, detail.operands[0].reg); - return target_pc; - } break; - case ARM64_INS_CBNZ: { - assert_true(detail.operands[0].type == ARM64_OP_REG); - assert_true(detail.operands[1].type == ARM64_OP_IMM); - const int64_t pc_offset = static_cast(detail.operands[1].imm); - const bool test_passed = (0 != ReadCapstoneReg(&thread_info->host_context, - detail.operands[0].reg)); - if (test_passed) { - return current_pc + pc_offset; - } else { - return current_pc + insn.size; - } - } break; - case ARM64_INS_CBZ: { - assert_true(detail.operands[0].type == ARM64_OP_REG); - assert_true(detail.operands[1].type == ARM64_OP_IMM); - const int64_t pc_offset = static_cast(detail.operands[1].imm); - const bool test_passed = (0 == ReadCapstoneReg(&thread_info->host_context, - detail.operands[0].reg)); - if (test_passed) { - return current_pc + pc_offset; - } else { - return current_pc + insn.size; - } - } break; - default: { - // Not a branching instruction - just move over it. - return current_pc + insn.size; - } break; - } -} - -void A64Backend::InstallBreakpoint(Breakpoint* breakpoint) { - breakpoint->ForEachHostAddress([breakpoint](uint64_t host_address) { - auto ptr = reinterpret_cast(host_address); - auto original_bytes = xe::load_and_swap(ptr); - assert_true(original_bytes != 0x0000'dead); - xe::store_and_swap(ptr, 0x0000'dead); - breakpoint->backend_data().emplace_back(host_address, original_bytes); - }); -} - -void A64Backend::InstallBreakpoint(Breakpoint* breakpoint, Function* fn) { - assert_true(breakpoint->address_type() == Breakpoint::AddressType::kGuest); - assert_true(fn->is_guest()); - auto guest_function = reinterpret_cast(fn); - auto host_address = - guest_function->MapGuestAddressToMachineCode(breakpoint->guest_address()); - if (!host_address) { - assert_always(); - return; - } - - // Assume we haven't already installed a breakpoint in this spot. - auto ptr = reinterpret_cast(host_address); - auto original_bytes = xe::load_and_swap(ptr); - assert_true(original_bytes != 0x0000'dead); - xe::store_and_swap(ptr, 0x0000'dead); - breakpoint->backend_data().emplace_back(host_address, original_bytes); -} - -void A64Backend::UninstallBreakpoint(Breakpoint* breakpoint) { - for (auto& pair : breakpoint->backend_data()) { - auto ptr = reinterpret_cast(pair.first); - auto instruction_bytes = xe::load_and_swap(ptr); - assert_true(instruction_bytes == 0x0000'dead); - xe::store_and_swap(ptr, static_cast(pair.second)); - } - breakpoint->backend_data().clear(); -} - -bool A64Backend::ExceptionCallbackThunk(Exception* ex, void* data) { - auto backend = reinterpret_cast(data); - return backend->ExceptionCallback(ex); -} - -bool A64Backend::ExceptionCallback(Exception* ex) { - if (ex->code() == Exception::Code::kAccessViolation) { - const uint64_t host_pc = ex->pc(); - const uint64_t fault_address = ex->fault_address(); - uint64_t guest_pc = 0; - uint32_t host_offset = 0; - auto function = code_cache_->LookupFunction(host_pc); - if (function && function->machine_code()) { - const uint64_t function_pc = - reinterpret_cast(function->machine_code()); - host_offset = static_cast(host_pc - function_pc); - if (const auto* entry = function->LookupMachineCodeOffset(host_offset)) { - guest_pc = entry->guest_address; - } - } -#if XE_ARCH_ARM64 - auto* thread_context = ex->thread_context(); - XELOGE( - "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " - "fault=0x{:016X} op={} x21=0x{:016X} x27=0x{:016X} x28=0x{:016X}", - host_pc, guest_pc, host_offset, fault_address, - static_cast(ex->access_violation_operation()), - thread_context ? thread_context->x[21] : 0, - thread_context ? thread_context->x[27] : 0, - thread_context ? thread_context->x[28] : 0); -#else - XELOGE( - "A64 AV: host_pc=0x{:016X} guest_pc=0x{:08X} host_off=0x{:X} " - "fault=0x{:016X} op={}", - host_pc, guest_pc, host_offset, fault_address, - static_cast(ex->access_violation_operation())); -#endif - return false; - } - if (ex->code() != Exception::Code::kIllegalInstruction) { - // We only care about illegal instructions. Other things will be handled by - // other handlers (probably). If nothing else picks it up we'll be called - // with OnUnhandledException to do real crash handling. - return false; - } - - // Verify an expected illegal instruction. - auto instruction_bytes = - xe::load_and_swap(reinterpret_cast(ex->pc())); - if (instruction_bytes != 0x0000'dead) { - // Not our `udf #0xdead` - not us. - return false; - } - - // Let the processor handle things. - return processor()->OnThreadBreakpointHit(ex); -} - -A64ThunkEmitter::A64ThunkEmitter(A64Backend* backend) : A64Emitter(backend) {} - -A64ThunkEmitter::~A64ThunkEmitter() {} - -HostToGuestThunk A64ThunkEmitter::EmitHostToGuestThunk() { - // X0 = target - // X1 = arg0 (context) - // X2 = arg1 (guest return address) - - struct _code_offsets { - size_t prolog; - size_t prolog_stack_alloc; - size_t body; - size_t epilog; - size_t tail; - } code_offsets = {}; - - const size_t stack_size = StackLayout::THUNK_STACK_SIZE; - - code_offsets.prolog = offset(); - - SUB(SP, SP, stack_size); - - code_offsets.prolog_stack_alloc = offset(); - code_offsets.body = offset(); - - EmitSaveNonvolatileRegs(); - - MOV(X16, X0); - MOV(GetContextReg(), X1); // context - MOV(X0, X2); // return address - BLR(X16); - - EmitLoadNonvolatileRegs(); - - code_offsets.epilog = offset(); - - ADD(SP, SP, stack_size); - - RET(); - - code_offsets.tail = offset(); - - assert_zero(code_offsets.prolog); - EmitFunctionInfo func_info = {}; - func_info.code_size.total = offset(); - func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; - func_info.code_size.body = code_offsets.epilog - code_offsets.body; - func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; - func_info.code_size.tail = offset() - code_offsets.tail; - func_info.prolog_stack_alloc_offset = - code_offsets.prolog_stack_alloc - code_offsets.prolog; - func_info.stack_size = stack_size; - - void* fn = Emplace(func_info); - return (HostToGuestThunk)fn; -} - -GuestToHostThunk A64ThunkEmitter::EmitGuestToHostThunk() { - // X0 = target function - // X1 = arg0 - // X2 = arg1 - // X3 = arg2 - - struct _code_offsets { - size_t prolog; - size_t prolog_stack_alloc; - size_t body; - size_t epilog; - size_t tail; - } code_offsets = {}; - - const size_t stack_size = StackLayout::THUNK_STACK_SIZE; - - code_offsets.prolog = offset(); - - SUB(SP, SP, stack_size); - - code_offsets.prolog_stack_alloc = offset(); - code_offsets.body = offset(); - - EmitSaveVolatileRegs(); - - MOV(X16, X0); // function - MOV(X0, GetContextReg()); // context - BLR(X16); - - EmitLoadVolatileRegs(); - - code_offsets.epilog = offset(); - - ADD(SP, SP, stack_size); - RET(); - - code_offsets.tail = offset(); - - assert_zero(code_offsets.prolog); - EmitFunctionInfo func_info = {}; - func_info.code_size.total = offset(); - func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; - func_info.code_size.body = code_offsets.epilog - code_offsets.body; - func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; - func_info.code_size.tail = offset() - code_offsets.tail; - func_info.prolog_stack_alloc_offset = - code_offsets.prolog_stack_alloc - code_offsets.prolog; - func_info.stack_size = stack_size; - - void* fn = Emplace(func_info); - return (GuestToHostThunk)fn; -} - -// A64Emitter handles actually resolving functions. -uint64_t ResolveFunction(void* raw_context, uint64_t target_address); - -ResolveFunctionThunk A64ThunkEmitter::EmitResolveFunctionThunk() { - // Entry: - // W17 = target PPC address - // X0 = context - - struct _code_offsets { - size_t prolog; - size_t prolog_stack_alloc; - size_t body; - size_t epilog; - size_t tail; - } code_offsets = {}; - - const size_t stack_size = StackLayout::THUNK_STACK_SIZE; - - code_offsets.prolog = offset(); - - // Preserve context register - STP(ZR, X0, SP, PRE_INDEXED, -16); - - SUB(SP, SP, stack_size); - - code_offsets.prolog_stack_alloc = offset(); - code_offsets.body = offset(); - - EmitSaveVolatileRegs(); - - // mov(rcx, rsi); // context - // mov(rdx, rbx); - // mov(rax, reinterpret_cast(&ResolveFunction)); - // call(rax) - MOV(X0, GetContextReg()); // context - MOV(W1, W17); - MOV(X16, reinterpret_cast(&ResolveFunction)); - BLR(X16); - MOV(X16, X0); - - EmitLoadVolatileRegs(); - - code_offsets.epilog = offset(); - - // add(rsp, stack_size); - // jmp(rax); - ADD(SP, SP, stack_size); - - // Reload context register - LDP(ZR, X0, SP, POST_INDEXED, 16); - oaknut::Label resolve_failed; - CBZ(X16, resolve_failed); - BR(X16); - l(resolve_failed); - RET(); - - code_offsets.tail = offset(); - - assert_zero(code_offsets.prolog); - EmitFunctionInfo func_info = {}; - func_info.code_size.total = offset(); - func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; - func_info.code_size.body = code_offsets.epilog - code_offsets.body; - func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; - func_info.code_size.tail = offset() - code_offsets.tail; - func_info.prolog_stack_alloc_offset = - code_offsets.prolog_stack_alloc - code_offsets.prolog; - func_info.stack_size = stack_size; - - void* fn = Emplace(func_info); - return (ResolveFunctionThunk)fn; -} - -void A64ThunkEmitter::EmitSaveVolatileRegs() { - // Save off volatile registers. - // Preserve arguments passed to and returned from a subroutine - // STR(X0, SP, offsetof(StackLayout::Thunk, r[0])); - STP(X1, X2, SP, offsetof(StackLayout::Thunk, r[0])); - STP(X3, X4, SP, offsetof(StackLayout::Thunk, r[2])); - STP(X5, X6, SP, offsetof(StackLayout::Thunk, r[4])); - STP(X7, X8, SP, offsetof(StackLayout::Thunk, r[6])); - STP(X9, X10, SP, offsetof(StackLayout::Thunk, r[8])); - STP(X11, X12, SP, offsetof(StackLayout::Thunk, r[10])); - STP(X13, X14, SP, offsetof(StackLayout::Thunk, r[12])); - STP(X15, X30, SP, offsetof(StackLayout::Thunk, r[14])); - - // Preserve arguments passed to and returned from a subroutine - // STR(Q0, SP, offsetof(StackLayout::Thunk, xmm[0])); - STP(Q1, Q2, SP, offsetof(StackLayout::Thunk, xmm[0])); - STP(Q3, Q4, SP, offsetof(StackLayout::Thunk, xmm[2])); - STP(Q5, Q6, SP, offsetof(StackLayout::Thunk, xmm[4])); - STP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); - STP(Q17, Q18, SP, offsetof(StackLayout::Thunk, xmm[8])); - STP(Q19, Q20, SP, offsetof(StackLayout::Thunk, xmm[10])); - STP(Q21, Q22, SP, offsetof(StackLayout::Thunk, xmm[12])); - STP(Q23, Q24, SP, offsetof(StackLayout::Thunk, xmm[14])); - STP(Q25, Q26, SP, offsetof(StackLayout::Thunk, xmm[16])); - STP(Q27, Q28, SP, offsetof(StackLayout::Thunk, xmm[18])); - STP(Q29, Q30, SP, offsetof(StackLayout::Thunk, xmm[20])); - STR(Q31, SP, offsetof(StackLayout::Thunk, xmm[21])); -} - -void A64ThunkEmitter::EmitLoadVolatileRegs() { - // Preserve arguments passed to and returned from a subroutine - // LDR(X0, SP, offsetof(StackLayout::Thunk, r[0])); - LDP(X1, X2, SP, offsetof(StackLayout::Thunk, r[0])); - LDP(X3, X4, SP, offsetof(StackLayout::Thunk, r[2])); - LDP(X5, X6, SP, offsetof(StackLayout::Thunk, r[4])); - LDP(X7, X8, SP, offsetof(StackLayout::Thunk, r[6])); - LDP(X9, X10, SP, offsetof(StackLayout::Thunk, r[8])); - LDP(X11, X12, SP, offsetof(StackLayout::Thunk, r[10])); - LDP(X13, X14, SP, offsetof(StackLayout::Thunk, r[12])); - LDP(X15, X30, SP, offsetof(StackLayout::Thunk, r[14])); - - // Preserve arguments passed to and returned from a subroutine - // LDR(Q0, SP, offsetof(StackLayout::Thunk, xmm[0])); - LDP(Q1, Q2, SP, offsetof(StackLayout::Thunk, xmm[0])); - LDP(Q3, Q4, SP, offsetof(StackLayout::Thunk, xmm[2])); - LDP(Q5, Q6, SP, offsetof(StackLayout::Thunk, xmm[4])); - LDP(Q7, Q16, SP, offsetof(StackLayout::Thunk, xmm[6])); - LDP(Q17, Q18, SP, offsetof(StackLayout::Thunk, xmm[8])); - LDP(Q19, Q20, SP, offsetof(StackLayout::Thunk, xmm[10])); - LDP(Q21, Q22, SP, offsetof(StackLayout::Thunk, xmm[12])); - LDP(Q23, Q24, SP, offsetof(StackLayout::Thunk, xmm[14])); - LDP(Q25, Q26, SP, offsetof(StackLayout::Thunk, xmm[16])); - LDP(Q27, Q28, SP, offsetof(StackLayout::Thunk, xmm[18])); - LDP(Q29, Q30, SP, offsetof(StackLayout::Thunk, xmm[20])); - LDR(Q31, SP, offsetof(StackLayout::Thunk, xmm[21])); -} - -void A64ThunkEmitter::EmitSaveNonvolatileRegs() { - STP(X19, X20, SP, offsetof(StackLayout::Thunk, r[0])); - STP(X21, X22, SP, offsetof(StackLayout::Thunk, r[2])); - STP(X23, X24, SP, offsetof(StackLayout::Thunk, r[4])); - STP(X25, X26, SP, offsetof(StackLayout::Thunk, r[6])); - STP(X27, X28, SP, offsetof(StackLayout::Thunk, r[8])); - STP(X29, X30, SP, offsetof(StackLayout::Thunk, r[10])); - - STR(X17, SP, offsetof(StackLayout::Thunk, r[12])); - - STP(D8, D9, SP, offsetof(StackLayout::Thunk, xmm[0])); - STP(D10, D11, SP, offsetof(StackLayout::Thunk, xmm[1])); - STP(D12, D13, SP, offsetof(StackLayout::Thunk, xmm[2])); - STP(D14, D15, SP, offsetof(StackLayout::Thunk, xmm[3])); -} - -void A64ThunkEmitter::EmitLoadNonvolatileRegs() { - LDP(X19, X20, SP, offsetof(StackLayout::Thunk, r[0])); - LDP(X21, X22, SP, offsetof(StackLayout::Thunk, r[2])); - LDP(X23, X24, SP, offsetof(StackLayout::Thunk, r[4])); - LDP(X25, X26, SP, offsetof(StackLayout::Thunk, r[6])); - LDP(X27, X28, SP, offsetof(StackLayout::Thunk, r[8])); - LDP(X29, X30, SP, offsetof(StackLayout::Thunk, r[10])); - - LDR(X17, SP, offsetof(StackLayout::Thunk, r[12])); - - LDP(D8, D9, SP, offsetof(StackLayout::Thunk, xmm[0])); - LDP(D10, D11, SP, offsetof(StackLayout::Thunk, xmm[1])); - LDP(D12, D13, SP, offsetof(StackLayout::Thunk, xmm[2])); - LDP(D14, D15, SP, offsetof(StackLayout::Thunk, xmm[3])); -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_backend.h b/src/xenia/cpu/backend/a64/a64_backend.h deleted file mode 100644 index 57557414c..000000000 --- a/src/xenia/cpu/backend/a64/a64_backend.h +++ /dev/null @@ -1,88 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ -#define XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ - -#include - -#include "xenia/base/cvar.h" -#include "xenia/cpu/backend/backend.h" - -DECLARE_int32(a64_extension_mask); - -namespace xe { -class Exception; -} // namespace xe -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -class A64CodeCache; - -typedef void* (*HostToGuestThunk)(void* target, void* arg0, void* arg1); -typedef void* (*GuestToHostThunk)(void* target, void* arg0, void* arg1); -typedef void (*ResolveFunctionThunk)(); - -class A64Backend : public Backend { - public: - static const uint32_t kForceReturnAddress = 0x9FFF0000u; - - explicit A64Backend(); - ~A64Backend() override; - - A64CodeCache* code_cache() const { return code_cache_.get(); } - uintptr_t emitter_data() const { return emitter_data_; } - - // Call a generated function, saving all stack parameters. - HostToGuestThunk host_to_guest_thunk() const { return host_to_guest_thunk_; } - // Function that guest code can call to transition into host code. - GuestToHostThunk guest_to_host_thunk() const { return guest_to_host_thunk_; } - // Function that thunks to the ResolveFunction in A64Emitter. - ResolveFunctionThunk resolve_function_thunk() const { - return resolve_function_thunk_; - } - - bool Initialize(Processor* processor) override; - - void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high) override; - - std::unique_ptr CreateAssembler() override; - - std::unique_ptr CreateGuestFunction(Module* module, - uint32_t address) override; - - uint64_t CalculateNextHostInstruction(ThreadDebugInfo* thread_info, - uint64_t current_pc) override; - - void InstallBreakpoint(Breakpoint* breakpoint) override; - void InstallBreakpoint(Breakpoint* breakpoint, Function* fn) override; - void UninstallBreakpoint(Breakpoint* breakpoint) override; - - private: - static bool ExceptionCallbackThunk(Exception* ex, void* data); - bool ExceptionCallback(Exception* ex); - - uintptr_t capstone_handle_ = 0; - - std::unique_ptr code_cache_; - uintptr_t emitter_data_ = 0; - - HostToGuestThunk host_to_guest_thunk_; - GuestToHostThunk guest_to_host_thunk_; - ResolveFunctionThunk resolve_function_thunk_; -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc deleted file mode 100644 index 3464578d0..000000000 --- a/src/xenia/cpu/backend/a64/a64_code_cache.cc +++ /dev/null @@ -1,643 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_code_cache.h" - -#include -#include -#include - -#if XE_PLATFORM_MAC -#include -#endif - -#include "third_party/fmt/include/fmt/format.h" -#include "xenia/base/assert.h" -#include "xenia/base/clock.h" -#include "xenia/base/cvar.h" -#include "xenia/base/literals.h" -#include "xenia/base/logging.h" -#include "xenia/base/math.h" -#include "xenia/base/memory.h" -#include "xenia/cpu/function.h" -#include "xenia/cpu/module.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -using namespace xe::literals; - -DEFINE_bool(a64_indirection_table_log, false, - "Log A64 indirection table mapping and updates.", "CPU"); -DEFINE_int32(a64_indirection_table_log_limit, 32, - "Maximum number of A64 indirection table log entries.", "CPU"); - -namespace { - -bool ShouldLogIndirectionTable() { - if (!cvars::a64_indirection_table_log) { - return false; - } - const int32_t limit = cvars::a64_indirection_table_log_limit; - if (limit <= 0) { - return false; - } - static std::atomic log_count{0}; - const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); - return count < limit; -} - -} // namespace - -// Define static constants for linking -const size_t A64CodeCache::kIndirectionTableSize; -#if XE_A64_INDIRECTION_64BIT -// On ARM64 platforms, this will be set dynamically during initialization -uintptr_t A64CodeCache::kIndirectionTableBase = 0x80000000; -#else -const uintptr_t A64CodeCache::kIndirectionTableBase; -#endif - -A64CodeCache::A64CodeCache() = default; - -A64CodeCache::~A64CodeCache() { - if (indirection_table_base_) { - xe::memory::DeallocFixed(indirection_table_base_, kIndirectionTableSize, - xe::memory::DeallocationType::kRelease); - } - - // Unmap all views and close mapping. - if (mapping_ != xe::memory::kFileMappingHandleInvalid) { -#if XE_PLATFORM_MAC && XE_ARCH_ARM64 - // On macOS ARM64, we used AllocFixed instead of MapFileView, so use - // DeallocFixed - if (generated_code_execute_base_) { - xe::memory::DeallocFixed(generated_code_execute_base_, kGeneratedCodeSize, - xe::memory::DeallocationType::kRelease); - } -#else - // Other platforms use MapFileView/UnmapFileView - if (generated_code_write_base_ && - generated_code_write_base_ != generated_code_execute_base_) { - xe::memory::UnmapFileView(mapping_, generated_code_write_base_, - kGeneratedCodeSize); - } - if (generated_code_execute_base_) { - xe::memory::UnmapFileView(mapping_, generated_code_execute_base_, - kGeneratedCodeSize); - } -#endif - xe::memory::CloseFileMappingHandle(mapping_, file_name_); - mapping_ = xe::memory::kFileMappingHandleInvalid; - } -} - -bool A64CodeCache::Initialize() { -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, allocate the indirection table wherever the OS allows, - // then update our base address to match. This gives us the same direct - // access pattern as x64 without needing complex offset calculations. - - indirection_table_base_ = reinterpret_cast( - xe::memory::AllocFixed(nullptr, kIndirectionTableSize, - xe::memory::AllocationType::kReserveCommit, - xe::memory::PageAccess::kReadWrite)); - - if (!indirection_table_base_) { - XELOGE("Unable to allocate indirection table at any address"); - return false; - } - - // Keep kIndirectionTableBase as 0x80000000 for offset calculations - // Store the actual allocated address separately - indirection_table_actual_base_ = - reinterpret_cast(indirection_table_base_); -#if XE_A64_INDIRECTION_64BIT - indirection_table_base_bias_ = - indirection_table_actual_base_ - - (static_cast(kIndirectionTableBase) * 2); -#endif -#else - // Other platforms: try to allocate at the preferred address first. - indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( - reinterpret_cast(kIndirectionTableBase), kIndirectionTableSize, - xe::memory::AllocationType::kReserve, - xe::memory::PageAccess::kReadWrite)); - if (!indirection_table_base_) { - XELOGW("Preferred indirection table base unavailable; falling back"); - indirection_table_base_ = reinterpret_cast(xe::memory::AllocFixed( - nullptr, kIndirectionTableSize, xe::memory::AllocationType::kReserve, - xe::memory::PageAccess::kReadWrite)); - } - if (!indirection_table_base_) { - XELOGE("Unable to allocate code cache indirection table"); - XELOGE("Tried preferred range {:X}-{:X} with fallback to OS-chosen", - static_cast(kIndirectionTableBase), - kIndirectionTableBase + kIndirectionTableSize); - return false; - } - indirection_table_actual_base_ = - reinterpret_cast(indirection_table_base_); -#if XE_A64_INDIRECTION_64BIT - indirection_table_base_bias_ = - indirection_table_actual_base_ - - (static_cast(kIndirectionTableBase) * 2); -#endif -#endif - - if (ShouldLogIndirectionTable()) { - XELOGI( - "A64 indirection table: guest_base=0x{:08X} table_base=0x{:016X} " - "size=0x{:X} entry_bytes={}", - static_cast(kIndirectionTableBase), - static_cast(indirection_table_actual_base_), - static_cast(kIndirectionTableSize), - static_cast(kIndirectionEntrySize)); - } - - // Create mmap file. This allows us to share the code cache with the debugger. - file_name_ = fmt::format("xenia_code_cache_{}", Clock::QueryHostTickCount()); - mapping_ = xe::memory::CreateFileMappingHandle( - file_name_, kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, - false); - if (mapping_ == xe::memory::kFileMappingHandleInvalid) { - XELOGE("Unable to create code cache mmap"); - return false; - } - - // Map generated code region into the file. Pages are committed as required. - if (xe::memory::IsWritableExecutableMemoryPreferred()) { -#if XE_PLATFORM_MAC && XE_ARCH_ARM64 - // On macOS ARM64, always use OS-chosen MAP_JIT memory. - generated_code_execute_base_ = reinterpret_cast( - xe::memory::AllocFixed(nullptr, kGeneratedCodeSize, - xe::memory::AllocationType::kReserveCommit, - xe::memory::PageAccess::kExecuteReadWrite)); - generated_code_write_base_ = generated_code_execute_base_; - if (!generated_code_execute_base_ || !generated_code_write_base_) { - XELOGE("Unable to allocate code cache generated code storage"); - return false; - } - // On macOS ARM64, verify the memory is properly allocated for MAP_JIT -#else - generated_code_execute_base_ = - reinterpret_cast(xe::memory::MapFileView( - mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), - kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadWrite, 0)); - if (!generated_code_execute_base_) { - XELOGW( - "Fixed address mapping for generated code failed, trying OS-chosen " - "address"); - generated_code_execute_base_ = - reinterpret_cast(xe::memory::MapFileView( - mapping_, nullptr, kGeneratedCodeSize, - xe::memory::PageAccess::kExecuteReadWrite, 0)); - } - generated_code_write_base_ = generated_code_execute_base_; - if (!generated_code_execute_base_ || !generated_code_write_base_) { - XELOGE("Unable to allocate code cache generated code storage"); - XELOGE( - "This is likely because the {:X}-{:X} range is in use by some other " - "system DLL", - uint64_t(kGeneratedCodeExecuteBase), - uint64_t(kGeneratedCodeExecuteBase + kGeneratedCodeSize)); - return false; - } -#endif - } else { -#if XE_PLATFORM_MAC && XE_ARCH_ARM64 - // On macOS ARM64, always use OS-chosen addresses for the views. - generated_code_execute_base_ = reinterpret_cast( - xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, - xe::memory::PageAccess::kExecuteReadOnly, 0)); - generated_code_write_base_ = reinterpret_cast( - xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, - xe::memory::PageAccess::kReadWrite, 0)); - if (!generated_code_execute_base_ || !generated_code_write_base_) { - XELOGE("Unable to allocate code cache generated code storage"); - return false; - } -#else - generated_code_execute_base_ = - reinterpret_cast(xe::memory::MapFileView( - mapping_, reinterpret_cast(kGeneratedCodeExecuteBase), - kGeneratedCodeSize, xe::memory::PageAccess::kExecuteReadOnly, 0)); - if (!generated_code_execute_base_) { - XELOGW( - "Fixed address mapping for generated code failed, trying OS-chosen " - "address"); - generated_code_execute_base_ = reinterpret_cast( - xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, - xe::memory::PageAccess::kExecuteReadOnly, 0)); - } - generated_code_write_base_ = - reinterpret_cast(xe::memory::MapFileView( - mapping_, reinterpret_cast(kGeneratedCodeWriteBase), - kGeneratedCodeSize, xe::memory::PageAccess::kReadWrite, 0)); - if (!generated_code_write_base_) { - XELOGW( - "Fixed address mapping for generated code failed, trying OS-chosen " - "address"); - generated_code_write_base_ = reinterpret_cast( - xe::memory::MapFileView(mapping_, nullptr, kGeneratedCodeSize, - xe::memory::PageAccess::kReadWrite, 0)); - } - if (!generated_code_execute_base_ || !generated_code_write_base_) { - XELOGE("Unable to allocate code cache generated code storage"); - XELOGE( - "This is likely because the {:X}-{:X} and {:X}-{:X} ranges are in " - "use by some other system DLL", - uint64_t(kGeneratedCodeExecuteBase), - uint64_t(kGeneratedCodeExecuteBase + kGeneratedCodeSize), - uint64_t(kGeneratedCodeWriteBase), - uint64_t(kGeneratedCodeWriteBase + kGeneratedCodeSize)); - return false; - } -#endif - } - - // Preallocate the function map to a large, reasonable size. - generated_code_map_.reserve(kMaximumFunctionCount); - - return true; -} - -void A64CodeCache::set_indirection_default(uint32_t default_value) { -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, we extend 32-bit values to 64-bit - indirection_default_value_ = default_value; -#else - indirection_default_value_ = default_value; -#endif -} - -#if XE_A64_INDIRECTION_64BIT -void A64CodeCache::set_indirection_default_64(uint64_t default_value) { - indirection_default_value_ = default_value; -} -#endif - -void A64CodeCache::AddIndirection(uint32_t guest_address, - uint32_t host_address) { -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, delegate to the 64-bit version - AddIndirection64(guest_address, host_address); -#else - if (!indirection_table_base_) { - return; - } - - uint32_t* indirection_slot = reinterpret_cast( - indirection_table_base_ + (guest_address - kIndirectionTableBase)); - *indirection_slot = host_address; -#endif -} - -#if XE_A64_INDIRECTION_64BIT -void A64CodeCache::AddIndirection64(uint32_t guest_address, - uint64_t host_address) { - if (!indirection_table_base_) { - return; - } - - if (guest_address < kIndirectionTableBase) { - XELOGE( - "A64CodeCache::AddIndirection64: guest_address 0x{:08X} below base " - "0x{:08X}", - guest_address, static_cast(kIndirectionTableBase)); - return; - } - - const uint64_t guest_delta = guest_address - kIndirectionTableBase; - if (guest_delta & 0x3) { - XELOGW( - "A64CodeCache::AddIndirection64: guest_address 0x{:08X} not 4-byte " - "aligned (delta=0x{:X})", - guest_address, guest_delta); - } - - // Calculate offset from the logical base (0x80000000), not from actual table - // address. - const uint64_t guest_offset = (guest_delta >> 2) * kIndirectionEntrySize; - if (guest_offset + kIndirectionEntrySize > kIndirectionTableSize) { - XELOGE( - "A64CodeCache::AddIndirection64: guest_address 0x{:08X} offset 0x{:X} " - "exceeds table size 0x{:X}", - guest_address, guest_offset, - static_cast(kIndirectionTableSize)); - return; - } - - uint64_t* indirection_slot = - reinterpret_cast(indirection_table_base_ + guest_offset); - *indirection_slot = host_address; - - if (ShouldLogIndirectionTable()) { - XELOGI( - "A64 indirection add: guest=0x{:08X} delta=0x{:X} offset=0x{:X} " - "slot=0x{:016X} host=0x{:016X}", - guest_address, guest_delta, guest_offset, - reinterpret_cast(indirection_slot), host_address); - } -} -#endif - -void A64CodeCache::CommitExecutableRange(uint32_t guest_low, - uint32_t guest_high) { - if (!indirection_table_base_) { - XELOGE("CommitExecutableRange: indirection_table_base_ is null!"); - return; - } - -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms: use offset-based addressing from guest base - // (0x80000000) - static const uintptr_t kGuestAddressBase = 0x80000000; - - // Calculate offsets from the guest address base, not the table base - if (guest_low < kGuestAddressBase) { - XELOGE( - "CommitExecutableRange: guest_low 0x{:08X} is below guest base " - "0x{:08X}", - guest_low, kGuestAddressBase); - return; - } - - uint32_t start_offset = - ((guest_low - kGuestAddressBase) >> 2) * kIndirectionEntrySize; - uint32_t size = ((guest_high - guest_low) >> 2) * kIndirectionEntrySize; - - // Sanity check bounds; the table should fully cover the XEX guest range now. - if (start_offset + size > kIndirectionTableSize) { - XELOGE( - "CommitExecutableRange: range [0x{:08X}, 0x{:08X}) exceeds table (size " - "0x{:X})", - guest_low, guest_high, static_cast(kIndirectionTableSize)); - return; - } - - // The memory should already be allocated, just fill with default value - void* target_memory = indirection_table_base_ + start_offset; - uint64_t* p = reinterpret_cast(target_memory); - uint32_t entry_count = size / kIndirectionEntrySize; - for (uint32_t i = 0; i < entry_count; i++) { - p[i] = indirection_default_value_; - } -#else - // Other platforms: use 32-bit entries - uint32_t start_offset = (guest_low - kIndirectionTableBase); - uint32_t size = (guest_high - guest_low); - - xe::memory::AllocFixed(indirection_table_base_ + start_offset, size, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kReadWrite); - - uint32_t* p = reinterpret_cast(indirection_table_base_); - for (uint32_t address = guest_low; address < guest_high; address += 4) { - p[(address - kIndirectionTableBase) / 4] = indirection_default_value_; - } -#endif -} - -void A64CodeCache::PlaceHostCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - void*& code_execute_address_out, - void*& code_write_address_out) { - // Same for now. We may use different pools or whatnot later on, like when - // we only want to place guest code in a serialized cache on disk. - PlaceGuestCode(guest_address, machine_code, func_info, nullptr, - code_execute_address_out, code_write_address_out); -} - -void A64CodeCache::PlaceGuestCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - GuestFunction* function_info, - void*& code_execute_address_out, - void*& code_write_address_out) { - // Hold a lock while we bump the pointers up. This is important as the - // unwind table requires entries AND code to be sorted in order. - [[maybe_unused]] size_t low_mark; - size_t high_mark; - uint8_t* code_execute_address; - UnwindReservation unwind_reservation; - { - auto global_lock = global_critical_region_.Acquire(); - - low_mark = generated_code_offset_; - - // Reserve code. - // Always move the code to land on 16b alignment. - code_execute_address = - generated_code_execute_base_ + generated_code_offset_; - code_execute_address_out = code_execute_address; - uint8_t* code_write_address = - generated_code_write_base_ + generated_code_offset_; - code_write_address_out = code_write_address; - generated_code_offset_ += xe::round_up(func_info.code_size.total, 16); - - auto tail_write_address = - generated_code_write_base_ + generated_code_offset_; - - // Reserve unwind info. - // We go on the high size of the unwind info as we don't know how big we - // need it, and a few extra bytes of padding isn't the worst thing. - unwind_reservation = RequestUnwindReservation(generated_code_write_base_ + - generated_code_offset_); - generated_code_offset_ += xe::round_up(unwind_reservation.data_size, 16); - - auto end_write_address = - generated_code_write_base_ + generated_code_offset_; - - high_mark = generated_code_offset_; - - // Store in map. It is maintained in sorted order of host PC dependent on - // us also being append-only. - generated_code_map_.emplace_back( - (uint64_t(code_execute_address - generated_code_execute_base_) << 32) | - generated_code_offset_, - function_info); - - // TODO(DrChat): The following code doesn't really need to be under the - // global lock except for PlaceCode (but it depends on the previous code - // already being ran) - - // If we are going above the high water mark of committed memory, commit - // some more. It's ok if multiple threads do this, as redundant commits - // aren't harmful. - size_t old_commit_mark, new_commit_mark; - do { - old_commit_mark = generated_code_commit_mark_; - if (high_mark <= old_commit_mark) break; - - new_commit_mark = old_commit_mark + 16_MiB; - if (generated_code_execute_base_ == generated_code_write_base_) { - xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kExecuteReadWrite); - } else { - xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kExecuteReadOnly); - xe::memory::AllocFixed(generated_code_write_base_, new_commit_mark, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kReadWrite); - } - } while (generated_code_commit_mark_.compare_exchange_weak( - old_commit_mark, new_commit_mark)); - - // Copy code and fill padding while in write mode on MAP_JIT. -#if XE_PLATFORM_MAC && defined(__aarch64__) - const bool jit_write = - (generated_code_execute_base_ == generated_code_write_base_); - if (jit_write) { - pthread_jit_write_protect_np(0); - } -#endif - CopyMachineCode(code_write_address, machine_code, - func_info.code_size.total); - if (end_write_address > tail_write_address) { - std::memset(tail_write_address, 0x00, - static_cast(end_write_address - tail_write_address)); - } -#if XE_PLATFORM_MAC && defined(__aarch64__) - if (jit_write) { - pthread_jit_write_protect_np(1); - } -#endif - - // Notify subclasses of placed code. - PlaceCode(guest_address, machine_code, func_info, code_execute_address, - unwind_reservation); - } - - // Now that everything is ready, fix up the indirection table. - // Note that we do support code that doesn't have an indirection fixup, so - // ignore those when we see them. - if (guest_address && indirection_table_base_) { -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, map guest addresses to table offsets using logical - // base kIndirectionTableBase remains 0x80000000 for calculation purposes - - // Calculate offset from the logical guest base (0x80000000) - if (guest_address < kIndirectionTableBase) { - return; - } - - uintptr_t guest_diff = guest_address - kIndirectionTableBase; - uintptr_t guest_offset = - (guest_diff >> 2) * kIndirectionEntrySize; // 8-byte entries - uintptr_t slot_address = - reinterpret_cast(indirection_table_base_) + guest_offset; - uint64_t* indirection_slot = reinterpret_cast(slot_address); - - // Check if the slot address is within bounds - uintptr_t table_end = reinterpret_cast(indirection_table_base_) + - kIndirectionTableSize; - if (slot_address >= table_end) { - return; - } - - *indirection_slot = reinterpret_cast(code_execute_address); -#else - uint32_t* indirection_slot = reinterpret_cast( - indirection_table_base_ + (guest_address - kIndirectionTableBase)); - *indirection_slot = - uint32_t(reinterpret_cast(code_execute_address)); -#endif - } -} - -uint32_t A64CodeCache::PlaceData(const void* data, size_t length) { - // Hold a lock while we bump the pointers up. - size_t high_mark; - uint8_t* data_address = nullptr; - { - auto global_lock = global_critical_region_.Acquire(); - - // Reserve code. - // Always move the code to land on 16b alignment. - data_address = generated_code_write_base_ + generated_code_offset_; - generated_code_offset_ += xe::round_up(length, 16); - - high_mark = generated_code_offset_; - } - - // If we are going above the high water mark of committed memory, commit some - // more. It's ok if multiple threads do this, as redundant commits aren't - // harmful. - size_t old_commit_mark, new_commit_mark; - do { - old_commit_mark = generated_code_commit_mark_; - if (high_mark <= old_commit_mark) break; - - new_commit_mark = old_commit_mark + 16_MiB; - if (generated_code_execute_base_ == generated_code_write_base_) { - xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kExecuteReadWrite); - } else { - xe::memory::AllocFixed(generated_code_execute_base_, new_commit_mark, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kExecuteReadOnly); - xe::memory::AllocFixed(generated_code_write_base_, new_commit_mark, - xe::memory::AllocationType::kCommit, - xe::memory::PageAccess::kReadWrite); - } - } while (generated_code_commit_mark_.compare_exchange_weak(old_commit_mark, - new_commit_mark)); - - // Copy data. -#if XE_PLATFORM_MAC && defined(__aarch64__) - if (generated_code_execute_base_ == generated_code_write_base_) { - pthread_jit_write_protect_np(0); - std::memcpy(data_address, data, length); - pthread_jit_write_protect_np(1); - } else { - std::memcpy(data_address, data, length); - } -#else - std::memcpy(data_address, data, length); -#endif - - return uint32_t(uintptr_t(data_address)); -} - -GuestFunction* A64CodeCache::LookupFunction(uint64_t host_pc) { - uint32_t key = uint32_t(host_pc - execute_base_address()); - void* fn_entry = std::bsearch( - &key, generated_code_map_.data(), generated_code_map_.size() + 1, - sizeof(std::pair), - [](const void* key_ptr, const void* element_ptr) { - auto key = *reinterpret_cast(key_ptr); - auto element = - reinterpret_cast*>( - element_ptr); - if (key < (element->first >> 32)) { - return -1; - } else if (key > uint32_t(element->first)) { - return 1; - } else { - return 0; - } - }); - if (fn_entry) { - return reinterpret_cast*>( - fn_entry) - ->second; - } else { - return nullptr; - } -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h deleted file mode 100644 index 1da3f27ba..000000000 --- a/src/xenia/cpu/backend/a64/a64_code_cache.h +++ /dev/null @@ -1,219 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ -#define XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ - -#include -#include -#include -#include -#include -#include -#include - -#include "xenia/base/memory.h" -#include "xenia/base/mutex.h" -#include "xenia/cpu/backend/code_cache.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -#if XE_ARCH_ARM64 -#define XE_A64_INDIRECTION_64BIT 1 -#else -#define XE_A64_INDIRECTION_64BIT 0 -#endif - -struct EmitFunctionInfo { - struct _code_size { - size_t prolog; - size_t body; - size_t epilog; - size_t tail; - size_t total; - } code_size; - size_t prolog_stack_alloc_offset; // offset of instruction after stack alloc - size_t stack_size; -}; - -class A64CodeCache : public CodeCache { - public: - ~A64CodeCache() override; - - static std::unique_ptr Create(); - - virtual bool Initialize(); - - const std::filesystem::path& file_name() const override { return file_name_; } - uintptr_t execute_base_address() const override { - return generated_code_execute_base_ - ? reinterpret_cast(generated_code_execute_base_) - : kGeneratedCodeExecuteBase; - } - size_t total_size() const override { return kGeneratedCodeSize; } - - // TODO(benvanik): ELF serialization/etc - // TODO(benvanik): keep track of code blocks - // TODO(benvanik): padding/guards/etc - - bool has_indirection_table() { return indirection_table_base_ != nullptr; } - void set_indirection_default(uint32_t default_value); -#if XE_A64_INDIRECTION_64BIT - void set_indirection_default_64(uint64_t default_value); -#endif - void AddIndirection(uint32_t guest_address, uint32_t host_address); -#if XE_A64_INDIRECTION_64BIT - void AddIndirection64(uint32_t guest_address, uint64_t host_address); -#endif - - void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high); - - void PlaceHostCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - void*& code_execute_address_out, - void*& code_write_address_out); - void PlaceGuestCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - GuestFunction* function_info, - void*& code_execute_address_out, - void*& code_write_address_out); - uint32_t PlaceData(const void* data, size_t length); - - GuestFunction* LookupFunction(uint64_t host_pc) override; - - // Access to indirection table base for emitter - uint8_t* indirection_table_base() const { return indirection_table_base_; } - - // Returns the actual base address used for indirection table - uintptr_t indirection_table_base_address() const { - return indirection_table_actual_base_; - } -#if XE_A64_INDIRECTION_64BIT - uintptr_t indirection_table_base_bias() const { - return indirection_table_base_bias_; - } -#endif - - public: - // All executable code falls within 0x80000000 to 0x9FFFFFFF, so we can - // only map enough for lookups within that range. - // Size of the indirection table in bytes. - // On ARM64 platforms we store 64-bit entries (8 bytes) per 4-byte guest slot - // for the 0x2000_0000-byte guest executable range (0x8000_0000..0xA000_0000), - // so we need 0x4000_0000 bytes to cover the full space. -#if XE_A64_INDIRECTION_64BIT - static const size_t kIndirectionTableSize = 0x40000000; // 1 GiB -#else - static const size_t kIndirectionTableSize = - 0x20000000 - 1; // 512 MiB - 1 (legacy) -#endif -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, the base address is determined dynamically at runtime - // based on where the OS allows us to allocate memory - static uintptr_t kIndirectionTableBase; -#else - static const uintptr_t kIndirectionTableBase = 0x80000000; -#endif - // The code range is 512MB, but we know the total code games will have is - // pretty small (dozens of mb at most) and our expansion is reasonablish - // so 256MB should be more than enough. - static const size_t kGeneratedCodeSize = 0x0FFFFFFF; - static const uintptr_t kGeneratedCodeExecuteBase = 0xA0000000; - // Used for writing when PageAccess::kExecuteReadWrite is not supported. - static const uintptr_t kGeneratedCodeWriteBase = - kGeneratedCodeExecuteBase + kGeneratedCodeSize + 1; - - // This is picked to be high enough to cover whatever we can reasonably - // expect. If we hit issues with this it probably means some corner case - // in analysis triggering. - static const size_t kMaximumFunctionCount = 100000; - - struct UnwindReservation { - size_t data_size = 0; - size_t table_slot = 0; - uint8_t* entry_address = 0; - }; - - A64CodeCache(); - - virtual UnwindReservation RequestUnwindReservation(uint8_t* entry_address) { - return UnwindReservation(); - } - virtual void PlaceCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - void* code_execute_address, - UnwindReservation unwind_reservation) {} - - // Platform-specific code copying with JIT protection handling - virtual void CopyMachineCode(void* dest, const void* src, size_t size) { - std::memcpy(dest, src, size); - } - - std::filesystem::path file_name_; - xe::memory::FileMappingHandle mapping_ = - xe::memory::kFileMappingHandleInvalid; - - // NOTE: the global critical region must be held when manipulating the offsets - // or counts of anything, to keep the tables consistent and ordered. - xe::global_critical_region global_critical_region_; - - // Value that the indirection table will be initialized with upon commit. -#if XE_A64_INDIRECTION_64BIT - uint64_t indirection_default_value_ = 0xFEEDF00D; -#else - uint32_t indirection_default_value_ = 0xFEEDF00D; -#endif - -#if XE_A64_INDIRECTION_64BIT - // On ARM64 platforms, we use 64-bit pointers in the indirection table to - // handle high addresses that can't fit in 32-bit values. - using indirection_entry_t = uint64_t; - static constexpr size_t kIndirectionEntrySize = 8; -#else - // Other platforms use 32-bit pointers - using indirection_entry_t = uint32_t; - static constexpr size_t kIndirectionEntrySize = 4; -#endif - - // Fixed at kIndirectionTableBase in host space, holding pointers into - // the generated code table that correspond to the PPC functions in guest - // space. - uint8_t* indirection_table_base_ = nullptr; - // Actual base address of the indirection table (may differ from - // kIndirectionTableBase on systems where fixed address allocation fails) - uintptr_t indirection_table_actual_base_ = 0; -#if XE_A64_INDIRECTION_64BIT - uintptr_t indirection_table_base_bias_ = 0; -#endif - // Fixed at kGeneratedCodeExecuteBase and holding all generated code, growing - // as needed. - uint8_t* generated_code_execute_base_ = nullptr; - // View of the memory that backs generated_code_execute_base_ when - // PageAccess::kExecuteReadWrite is not supported, for writing the generated - // code. Equals to generated_code_execute_base_ when it's supported. - uint8_t* generated_code_write_base_ = nullptr; - // Current offset to empty space in generated code. - size_t generated_code_offset_ = 0; - // Current high water mark of COMMITTED code. - std::atomic generated_code_commit_mark_ = {0}; - // Sorted map by host PC base offsets to source function info. - // This can be used to bsearch on host PC to find the guest function. - // The key is [start address | end address]. - std::vector> generated_code_map_; -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc deleted file mode 100644 index 639cd4f7b..000000000 --- a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc +++ /dev/null @@ -1,201 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2025 Ben Vanik. All rights reserved. - * Released under the BSD license - see LICENSE in the root for more details. - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_code_cache.h" - -#include -#include -#include -#include - -#ifdef XE_PLATFORM_MAC -#include -#include -#endif - -#include "xenia/base/assert.h" -#include "xenia/base/clock.h" -#include "xenia/base/logging.h" -#include "xenia/base/math.h" -#include "xenia/base/memory.h" -#include "xenia/cpu/function.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -// ARM64 unwind-op codes for POSIX (simplified) -typedef enum _UNWIND_OP_CODES_POSIX { - UWOP_POSIX_NOP = 0x00, - UWOP_POSIX_ALLOC_STACK = 0x01, - UWOP_POSIX_SAVE_FP_LR = 0x02, - UWOP_POSIX_SET_FP = 0x03, - UWOP_POSIX_END = 0xFF, -} UNWIND_CODE_OPS_POSIX; - -using UNWIND_CODE_POSIX = uint8_t; - -// Size of unwind info per function. -static const size_t kUnwindInfoSize = 16; - -class PosixA64CodeCache : public A64CodeCache { - public: - PosixA64CodeCache(); - ~PosixA64CodeCache() override; - - bool Initialize() override; - - void* LookupUnwindInfo(uint64_t host_pc) override; - - protected: - void CopyMachineCode(void* dest, const void* src, size_t size) override; - - private: - struct UnwindInfo { - uint64_t begin_address; - uint64_t end_address; - // Additional unwind information can be added here - }; - - UnwindReservation RequestUnwindReservation(uint8_t* entry_address) override; - void PlaceCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, void* code_execute_address, - UnwindReservation unwind_reservation) override; - - void InitializeUnwindEntry(uint8_t* unwind_entry_address, - size_t unwind_table_slot, - void* code_execute_address, - const EmitFunctionInfo& func_info); - - // Unwind table entries. - std::vector unwind_table_; - // Current number of entries in the table. - std::atomic unwind_table_count_ = {0}; -}; - -std::unique_ptr A64CodeCache::Create() { - return std::make_unique(); -} - -PosixA64CodeCache::PosixA64CodeCache() = default; - -PosixA64CodeCache::~PosixA64CodeCache() { - // Cleanup if necessary -} - -bool PosixA64CodeCache::Initialize() { - if (!A64CodeCache::Initialize()) { - return false; - } - - // Resize (not reserve) space for unwind table entries to ensure vector has - // actual elements - unwind_table_.resize(kMaximumFunctionCount); - - // Additional POSIX-specific initialization can be done here - - return true; -} - -void PosixA64CodeCache::CopyMachineCode(void* dest, const void* src, - size_t size) { - std::memcpy(dest, src, size); -} - -PosixA64CodeCache::UnwindReservation -PosixA64CodeCache::RequestUnwindReservation(uint8_t* entry_address) { - uint32_t current_count = unwind_table_count_.fetch_add(1); - assert_false(current_count >= kMaximumFunctionCount); - UnwindReservation unwind_reservation; - unwind_reservation.data_size = xe::round_up(kUnwindInfoSize, 16); - unwind_reservation.table_slot = current_count; - unwind_reservation.entry_address = entry_address; - return unwind_reservation; -} - -void PosixA64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - void* code_execute_address, - UnwindReservation unwind_reservation) { - // Add unwind info. - InitializeUnwindEntry( - reinterpret_cast(unwind_reservation.entry_address), - unwind_reservation.table_slot, code_execute_address, func_info); - - // Add entry to unwind table at the reserved slot only - UnwindInfo unwind_info; - unwind_info.begin_address = reinterpret_cast(code_execute_address); - unwind_info.end_address = - unwind_info.begin_address + func_info.code_size.total; - - // Store in the reserved slot - unwind_table_[unwind_reservation.table_slot] = unwind_info; - - // Validate address alignment before cache flushing - if (reinterpret_cast(code_execute_address) % 4 != 0) { - XELOGW( - "PosixA64CodeCache::PlaceCode: WARNING - code address 0x{:016X} is not " - "4-byte aligned", - reinterpret_cast(code_execute_address)); - } - - if (func_info.code_size.total % 4 != 0) { - XELOGW( - "PosixA64CodeCache::PlaceCode: WARNING - code size {} is not 4-byte " - "aligned", - func_info.code_size.total); - } - - // Flush instruction cache -#ifdef XE_PLATFORM_MAC - // On macOS, use sys_icache_invalidate - sys_icache_invalidate(code_execute_address, func_info.code_size.total); -#else - // On Linux and other POSIX systems, use GCC builtin - __builtin___clear_cache( - static_cast(code_execute_address), - static_cast(code_execute_address) + func_info.code_size.total); -#endif -} - -void PosixA64CodeCache::InitializeUnwindEntry( - uint8_t* unwind_entry_address, size_t unwind_table_slot, - void* code_execute_address, const EmitFunctionInfo& func_info) { - // Initialize unwind information for POSIX (simplified example) - // In practice, you would populate this with proper unwind info - // based on the function prologue and epilogue. - - // NOTE: Unwind info is already stored in PlaceCode, so we don't store it - // again here to avoid the double-storage bug that was causing memory - // corruption. -} - -void* PosixA64CodeCache::LookupUnwindInfo(uint64_t host_pc) { - // Binary search the unwind table for the given program counter - size_t left = 0; - size_t right = unwind_table_count_.load(); - while (left < right) { - size_t mid = left + (right - left) / 2; - const UnwindInfo& info = unwind_table_[mid]; - if (host_pc < info.begin_address) { - right = mid; - } else if (host_pc >= info.end_address) { - left = mid + 1; - } else { - return &unwind_table_[mid]; - } - } - return nullptr; -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_win.cc b/src/xenia/cpu/backend/a64/a64_code_cache_win.cc deleted file mode 100644 index 21a87e9f2..000000000 --- a/src/xenia/cpu/backend/a64/a64_code_cache_win.cc +++ /dev/null @@ -1,319 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_code_cache.h" - -#include -#include - -#include "xenia/base/assert.h" -#include "xenia/base/clock.h" -#include "xenia/base/logging.h" -#include "xenia/base/math.h" -#include "xenia/base/memory.h" -#include "xenia/base/platform_win.h" -#include "xenia/cpu/function.h" - -// Function pointer definitions -using FnRtlAddGrowableFunctionTable = decltype(&RtlAddGrowableFunctionTable); -using FnRtlGrowFunctionTable = decltype(&RtlGrowFunctionTable); -using FnRtlDeleteGrowableFunctionTable = - decltype(&RtlDeleteGrowableFunctionTable); - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -// ARM64 unwind-op codes -// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes -// https://www.corsix.org/content/windows-arm64-unwind-codes -typedef enum _UNWIND_OP_CODES { - UWOP_NOP = 0xE3, - UWOP_ALLOC_S = 0x00, // sub sp, sp, i*16 - UWOP_ALLOC_L = 0xE0'00'00'00, // sub sp, sp, i*16 - UWOP_SAVE_FPLR = 0x40, // stp fp, lr, [sp+i*8] - UWOP_SAVE_FPLRX = 0x80, // stp fp, lr, [sp-(i+1)*8]! - UWOP_SET_FP = 0xE1, // mov fp, sp - UWOP_END = 0xE4, -} UNWIND_CODE_OPS; - -using UNWIND_CODE = uint32_t; - -static_assert(sizeof(UNWIND_CODE) == sizeof(uint32_t)); - -// UNWIND_INFO defines the static part (first 32-bit) of the .xdata record -typedef struct _UNWIND_INFO { - uint32_t FunctionLength : 18; - uint32_t Version : 2; - uint32_t X : 1; - uint32_t E : 1; - uint32_t EpilogCount : 5; - uint32_t CodeWords : 5; - UNWIND_CODE UnwindCodes[2]; -} UNWIND_INFO, *PUNWIND_INFO; - -static_assert(offsetof(UNWIND_INFO, UnwindCodes[0]) == 4); -static_assert(offsetof(UNWIND_INFO, UnwindCodes[1]) == 8); - -// Size of unwind info per function. -static const uint32_t kUnwindInfoSize = sizeof(UNWIND_INFO); - -class Win32A64CodeCache : public A64CodeCache { - public: - Win32A64CodeCache(); - ~Win32A64CodeCache() override; - - bool Initialize() override; - - void* LookupUnwindInfo(uint64_t host_pc) override; - - private: - UnwindReservation RequestUnwindReservation(uint8_t* entry_address) override; - void PlaceCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, void* code_execute_address, - UnwindReservation unwind_reservation) override; - - void InitializeUnwindEntry(uint8_t* unwind_entry_address, - size_t unwind_table_slot, - void* code_execute_address, - const EmitFunctionInfo& func_info); - - // Growable function table system handle. - void* unwind_table_handle_ = nullptr; - // Actual unwind table entries. - std::vector unwind_table_; - // Current number of entries in the table. - std::atomic unwind_table_count_ = {0}; - // Does this version of Windows support growable funciton tables? - bool supports_growable_table_ = false; - - FnRtlAddGrowableFunctionTable add_growable_table_ = nullptr; - FnRtlDeleteGrowableFunctionTable delete_growable_table_ = nullptr; - FnRtlGrowFunctionTable grow_table_ = nullptr; -}; - -std::unique_ptr A64CodeCache::Create() { - return std::make_unique(); -} - -Win32A64CodeCache::Win32A64CodeCache() = default; - -Win32A64CodeCache::~Win32A64CodeCache() { - if (supports_growable_table_) { - if (unwind_table_handle_) { - delete_growable_table_(unwind_table_handle_); - } - } else { - if (generated_code_execute_base_) { - RtlDeleteFunctionTable(reinterpret_cast( - reinterpret_cast(generated_code_execute_base_) | 0x3)); - } - } -} - -bool Win32A64CodeCache::Initialize() { - if (!A64CodeCache::Initialize()) { - return false; - } - - // Compute total number of unwind entries we should allocate. - // We don't support reallocing right now, so this should be high. - unwind_table_.resize(kMaximumFunctionCount); - - // Check if this version of Windows supports growable function tables. - auto ntdll_handle = GetModuleHandleW(L"ntdll.dll"); - if (!ntdll_handle) { - add_growable_table_ = nullptr; - delete_growable_table_ = nullptr; - grow_table_ = nullptr; - } else { - add_growable_table_ = (FnRtlAddGrowableFunctionTable)GetProcAddress( - ntdll_handle, "RtlAddGrowableFunctionTable"); - delete_growable_table_ = (FnRtlDeleteGrowableFunctionTable)GetProcAddress( - ntdll_handle, "RtlDeleteGrowableFunctionTable"); - grow_table_ = (FnRtlGrowFunctionTable)GetProcAddress( - ntdll_handle, "RtlGrowFunctionTable"); - } - supports_growable_table_ = - add_growable_table_ && delete_growable_table_ && grow_table_; - - // Create table and register with the system. It's empty now, but we'll grow - // it as functions are added. - if (supports_growable_table_) { - if (add_growable_table_( - &unwind_table_handle_, unwind_table_.data(), unwind_table_count_, - DWORD(unwind_table_.size()), - reinterpret_cast(generated_code_execute_base_), - reinterpret_cast(generated_code_execute_base_ + - kGeneratedCodeSize))) { - XELOGE("Unable to create unwind function table"); - return false; - } - } else { - // Install a callback that the debugger will use to lookup unwind info on - // demand. - if (!RtlInstallFunctionTableCallback( - reinterpret_cast(generated_code_execute_base_) | 0x3, - reinterpret_cast(generated_code_execute_base_), - kGeneratedCodeSize, - [](DWORD64 control_pc, PVOID context) { - auto code_cache = reinterpret_cast(context); - return reinterpret_cast( - code_cache->LookupUnwindInfo(control_pc)); - }, - this, nullptr)) { - XELOGE("Unable to install function table callback"); - return false; - } - } - - return true; -} - -Win32A64CodeCache::UnwindReservation -Win32A64CodeCache::RequestUnwindReservation(uint8_t* entry_address) { - assert_false(unwind_table_count_ >= kMaximumFunctionCount); - UnwindReservation unwind_reservation; - unwind_reservation.data_size = xe::round_up(kUnwindInfoSize, 16); - unwind_reservation.table_slot = unwind_table_count_++; - unwind_reservation.entry_address = entry_address; - return unwind_reservation; -} - -void Win32A64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, - const EmitFunctionInfo& func_info, - void* code_execute_address, - UnwindReservation unwind_reservation) { - // Add unwind info. - InitializeUnwindEntry(unwind_reservation.entry_address, - unwind_reservation.table_slot, code_execute_address, - func_info); - - if (supports_growable_table_) { - // Notify that the unwind table has grown. - // We do this outside of the lock, but with the latest total count. - grow_table_(unwind_table_handle_, unwind_table_count_); - } - - // https://docs.microsoft.com/en-us/uwp/win32-and-com/win32-apis - FlushInstructionCache(GetCurrentProcess(), code_execute_address, - func_info.code_size.total); -} - -constexpr UNWIND_CODE UnwindOpWord(uint8_t code0 = UWOP_NOP, - uint8_t code1 = UWOP_NOP, - uint8_t code2 = UWOP_NOP, - uint8_t code3 = UWOP_NOP) { - return static_cast(code0) | (static_cast(code1) << 8) | - (static_cast(code2) << 16) | - (static_cast(code3) << 24); -} - -// 8-byte unwind code for "stp fp, lr, [sp, #-16]! -// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes -static uint8_t OpSaveFpLrX(int16_t pre_index_offset) { - assert_true(pre_index_offset <= -8); - assert_true(pre_index_offset >= -512); - // 16-byte aligned - constexpr int IndexShift = 3; - constexpr int IndexMask = (1 << IndexShift) - 1; - assert_true((pre_index_offset & IndexMask) == 0); - const uint32_t encoded_value = (-pre_index_offset >> IndexShift) - 1; - return UWOP_SAVE_FPLRX | encoded_value; -} - -// Ensure a 16-byte aligned stack -static constexpr size_t StackAlignShift = 4; // n / 16 -static constexpr size_t StackAlignMask = (1 << StackAlignShift) - 1; // n % 16 - -// 8-byte unwind code for up to +512-byte "sub sp, sp, #stack_space" -// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes -static uint8_t OpAllocS(int16_t stack_space) { - assert_true(stack_space >= 0); - assert_true(stack_space < 512); - assert_true((stack_space & StackAlignMask) == 0); - return UWOP_ALLOC_S | (stack_space >> StackAlignShift); -} - -// 4-byte unwind code for +256MiB "sub sp, sp, #stack_space" -// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#unwind-codes -uint32_t OpAllocL(int32_t stack_space) { - assert_true(stack_space >= 0); - assert_true(stack_space < (0xFFFFFF * 16)); - assert_true((stack_space & StackAlignMask) == 0); - return xe::byte_swap(UWOP_ALLOC_L | - ((stack_space >> StackAlignShift) & 0xFF'FF'FF)); -} - -void Win32A64CodeCache::InitializeUnwindEntry( - uint8_t* unwind_entry_address, size_t unwind_table_slot, - void* code_execute_address, const EmitFunctionInfo& func_info) { - auto unwind_info = reinterpret_cast(unwind_entry_address); - - *unwind_info = {}; - // ARM64 instructions are always multiples of 4 bytes - // Windows ignores the bottom 2 bits - unwind_info->FunctionLength = func_info.code_size.total / 4; - unwind_info->CodeWords = 2; - - // https://learn.microsoft.com/en-us/cpp/build/arm64-exception-handling?view=msvc-170#unwind-codes - // The array of unwind codes is a pool of sequences that describe exactly how - // to undo the effects of the prolog. They're stored in the same order the - // operations need to be undone. The unwind codes can be thought of as a small - // instruction set, encoded as a string of bytes. When execution is complete, - // the return address to the calling function is in the lr register. And, all - // non-volatile registers are restored to their values at the time the - // function was called. - - // Function frames are generally: - // STP(X29, X30, SP, PRE_INDEXED, -16); - // MOV(X29, XSP); - // SUB(XSP, XSP, stack_size); - // ... function body ... - // ADD(XSP, XSP, stack_size); - // MOV(XSP, X29); - // LDP(X29, X30, SP, POST_INDEXED, 16); - - // These opcodes must undo the epilog and put the return address within lr - unwind_info->UnwindCodes[0] = OpAllocL(func_info.stack_size); - unwind_info->UnwindCodes[1] = - UnwindOpWord(UWOP_SET_FP, OpSaveFpLrX(-16), UWOP_END); - - // Add entry. - RUNTIME_FUNCTION& fn_entry = unwind_table_[unwind_table_slot]; - fn_entry.BeginAddress = - DWORD(reinterpret_cast(code_execute_address) - - generated_code_execute_base_); - fn_entry.UnwindData = - DWORD(unwind_entry_address - generated_code_execute_base_); -} - -void* Win32A64CodeCache::LookupUnwindInfo(uint64_t host_pc) { - return std::bsearch( - &host_pc, unwind_table_.data(), unwind_table_count_, - sizeof(RUNTIME_FUNCTION), - [](const void* key_ptr, const void* element_ptr) { - auto key = *reinterpret_cast(key_ptr) - - kGeneratedCodeExecuteBase; - auto element = reinterpret_cast(element_ptr); - if (key < element->BeginAddress) { - return -1; - } else if (key > (element->BeginAddress + element->FunctionLength)) { - return 1; - } else { - return 0; - } - }); -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc deleted file mode 100644 index 58d25acbd..000000000 --- a/src/xenia/cpu/backend/a64/a64_emitter.cc +++ /dev/null @@ -1,1606 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_emitter.h" - -#include -#include -#include - -#include -#include -#include - -#include "third_party/fmt/include/fmt/format.h" -#include "xenia/base/assert.h" -#include "xenia/base/atomic.h" -#include "xenia/base/byte_order.h" -#include "xenia/base/debugging.h" -#include "xenia/base/literals.h" -#include "xenia/base/logging.h" -#include "xenia/base/math.h" -#include "xenia/base/memory.h" -#include "xenia/base/profiling.h" -#include "xenia/base/string_buffer.h" -#include "xenia/base/vec128.h" -#include "xenia/cpu/backend/a64/a64_backend.h" -#include "xenia/cpu/backend/a64/a64_code_cache.h" -#include "xenia/cpu/backend/a64/a64_function.h" -#include "xenia/cpu/backend/a64/a64_sequences.h" -#include "xenia/cpu/backend/a64/a64_stack_layout.h" -#include "xenia/cpu/cpu_flags.h" -#include "xenia/cpu/function.h" -#include "xenia/cpu/function_debug_info.h" -#include "xenia/cpu/ppc/ppc_opcode_info.h" -#include "xenia/cpu/processor.h" -#include "xenia/cpu/symbol.h" -#include "xenia/cpu/thread_state.h" - -#include "oaknut/feature_detection/cpu_feature.hpp" -#include "oaknut/feature_detection/feature_detection.hpp" -#include "oaknut/feature_detection/feature_detection_idregs.hpp" - -DEFINE_bool(debugprint_trap_log, false, - "Log debugprint traps to the active debugger", "CPU"); -DEFINE_bool(ignore_undefined_externs, true, - "Don't exit when an undefined extern is called.", "CPU"); -DEFINE_bool(log_undefined_extern_args, false, - "Log PPC args for undefined externs (once per function).", "CPU"); -DEFINE_bool(emit_source_annotations, false, - "Add extra movs and nops to make disassembly easier to read.", - "CPU"); -DEFINE_bool(a64_resolve_function_log, false, - "Log A64 ResolveFunction failures with module ranges.", "CPU"); -DEFINE_int32(a64_resolve_function_log_limit, 8, - "Maximum ResolveFunction failure logs.", "CPU"); - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -using xe::cpu::hir::HIRBuilder; -using xe::cpu::hir::Instr; -using namespace xe::literals; -using namespace oaknut::util; - -namespace { - -bool ShouldLogResolveFailure() { - if (!cvars::a64_resolve_function_log) { - return false; - } - const int32_t limit = cvars::a64_resolve_function_log_limit; - if (limit <= 0) { - return false; - } - static std::atomic log_count{0}; - const int32_t count = log_count.fetch_add(1, std::memory_order_relaxed); - return count < limit; -} - -void AdjustStackPointer(A64Emitter& emitter, size_t stack_size, bool add) { - if (!stack_size) { - return; - } - const uint64_t size_u64 = static_cast(stack_size); - const bool imm_valid = size_u64 <= 0xFFF || - ((size_u64 & 0xFFF) == 0 && (size_u64 >> 12) <= 0xFFF); - if (imm_valid) { - if (add) { - emitter.ADD(SP, SP, size_u64); - } else { - emitter.SUB(SP, SP, size_u64); - } - return; - } - emitter.MOV(X15, size_u64); - if (add) { - emitter.ADD(SP, SP, X15); - } else { - emitter.SUB(SP, SP, X15); - } -} - -} // namespace - -static const size_t kStashOffset = 32; -// static const size_t kStashOffsetHigh = 32 + 32; - -// Register indices that the HIR is allowed to use for operands -const uint8_t A64Emitter::gpr_reg_map_[A64Emitter::GPR_COUNT] = { - 19, 20, 21, 22, 23, 24, 25, 26, -}; - -const uint8_t A64Emitter::fpr_reg_map_[A64Emitter::FPR_COUNT] = { - 8, 9, 10, 11, 12, 13, 14, 15, -}; - -A64Emitter::A64Emitter(A64Backend* backend) - : VectorCodeGenerator(assembly_buffer), - processor_(backend->processor()), - backend_(backend), - code_cache_(backend->code_cache()) { - oaknut::CpuFeatures cpu_ = oaknut::detect_features(); - - // Combine with id register detection -#if OAKNUT_SUPPORTS_READING_ID_REGISTERS > 0 -#if OAKNUT_SUPPORTS_READING_ID_REGISTERS == 1 - const std::optional id_registers = - oaknut::read_id_registers(); -#elif OAKNUT_SUPPORTS_READING_ID_REGISTERS == 2 - const std::optional id_registers = - oaknut::read_id_registers(0); -#endif - if (id_registers.has_value()) { - cpu_ = cpu_ | oaknut::detect_features_via_id_registers(*id_registers); - } -#endif - -#define TEST_EMIT_FEATURE(emit, ext) \ - if ((cvars::a64_extension_mask & emit) == emit) { \ - feature_flags_ |= (cpu_.has(ext) ? emit : 0); \ - } - - TEST_EMIT_FEATURE(kA64EmitLSE, oaknut::CpuFeature::LSE); - TEST_EMIT_FEATURE(kA64EmitF16C, oaknut::CpuFeature::FP16Conv); - -#undef TEST_EMIT_FEATURE -} - -A64Emitter::~A64Emitter() = default; - -bool A64Emitter::Emit(GuestFunction* function, HIRBuilder* builder, - uint32_t debug_info_flags, FunctionDebugInfo* debug_info, - void** out_code_address, size_t* out_code_size, - std::vector* out_source_map) { - SCOPE_profile_cpu_f("cpu"); - - // Reset. - debug_info_ = debug_info; - debug_info_flags_ = debug_info_flags; - trace_data_ = &function->trace_data(); - source_map_arena_.Reset(); - - // Fill the generator with code. - EmitFunctionInfo func_info = {}; - if (!Emit(builder, func_info)) { - return false; - } - - // Copy the final code to the cache and relocate it. - *out_code_size = offset(); - *out_code_address = Emplace(func_info, function); - - // Stash source map. - source_map_arena_.CloneContents(out_source_map); - - return true; -} - -void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, - GuestFunction* function) { - // Copy the current oaknut instruction-buffer into the code-cache - void* new_execute_address; - void* new_write_address; - - assert_true(func_info.code_size.total == static_cast(offset())); - - if (function) { - code_cache_->PlaceGuestCode(function->address(), assembly_buffer.data(), - func_info, function, new_execute_address, - new_write_address); - } else { - code_cache_->PlaceHostCode(0, assembly_buffer.data(), func_info, - new_execute_address, new_write_address); - } - - // Reset the oaknut instruction-buffer - assembly_buffer.clear(); - label_lookup_.clear(); - - return new_execute_address; -} - -void A64Emitter::EmitBtiJc() { dw(0xD503241F); } - -bool A64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) { - oaknut::Label epilog_label; - epilog_label_ = &epilog_label; - - // Calculate stack size. We need to align things to their natural sizes. - // This could be much better (sort by type/etc). - auto locals = builder->locals(); - size_t stack_offset = StackLayout::GUEST_STACK_SIZE; - for (auto it = locals.begin(); it != locals.end(); ++it) { - auto slot = *it; - size_t type_size = GetTypeSize(slot->type); - - // Align to natural size. - stack_offset = xe::align(stack_offset, type_size); - slot->set_constant((uint32_t)stack_offset); - stack_offset += type_size; - } - - // Ensure 16b alignment. - stack_offset -= StackLayout::GUEST_STACK_SIZE; - stack_offset = xe::align(stack_offset, static_cast(16)); - - struct _code_offsets { - size_t prolog; - size_t prolog_stack_alloc; - size_t body; - size_t epilog; - size_t tail; - } code_offsets = {}; - - code_offsets.prolog = offset(); - - // Function prolog. - // Must be 16b aligned. - // Windows is very strict about the form of this and the epilog: - // https://docs.microsoft.com/en-us/cpp/build/prolog-and-epilog?view=vs-2017 - // IMPORTANT: any changes to the prolog must be kept in sync with - // A64CodeCache, which dynamically generates exception information. - // Adding or changing anything here must be matched! - const size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; - assert_true(stack_size % 16 == 0); - func_info.stack_size = stack_size; - stack_size_ = stack_size; - - EmitBtiJc(); - - STP(X29, X30, SP, PRE_INDEXED, -16); - MOV(X29, SP); - - AdjustStackPointer(*this, stack_size, false); - - code_offsets.prolog_stack_alloc = offset(); - code_offsets.body = offset(); - - STR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); - STR(X0, SP, StackLayout::GUEST_RET_ADDR); - STR(XZR, SP, StackLayout::GUEST_CALL_RET_ADDR); - - // Safe now to do some tracing. - if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctions) { - // We require 32-bit addresses. - assert_true(uint64_t(trace_data_->header()) < UINT_MAX); - auto trace_header = trace_data_->header(); - - // Call count. - MOV(W0, 1); - MOV(X5, reinterpret_cast( - low_address(&trace_header->function_call_count))); - LDADDAL(X0, X0, X5); - - // Get call history slot. - static_assert(FunctionTraceData::kFunctionCallerHistoryCount == 4, - "bitmask depends on count"); - LDR(X0, X5); - AND(W0, W0, 0b00000011); - - // Record call history value into slot (guest addr in W1). - MOV(X5, reinterpret_cast( - low_address(&trace_header->function_caller_history))); - STR(W1, X5, X0, oaknut::IndexExt::LSL, 2); - - // Calling thread. Load X0 with thread ID. - EmitGetCurrentThreadId(); - MOV(W5, 1); - LSL(W0, W5, W0); - - MOV(X5, reinterpret_cast( - low_address(&trace_header->function_thread_use))); - LDSET(W0, WZR, X5); - } - - // Load membase. - LDR(GetMembaseReg(), GetContextReg(), - offsetof(ppc::PPCContext, virtual_membase)); - - // Body. - auto block = builder->first_block(); - [[maybe_unused]] int block_count = 0; - while (block) { - // Mark block labels. - auto label = block->label_head; - while (label) { - l(*lookup_label(label)); - label = label->next; - } - - // Process instructions. - const Instr* instr = block->instr_head; - [[maybe_unused]] int instr_count = 0; - while (instr) { - const Instr* new_tail = instr; - if (!SelectSequence(this, instr, &new_tail)) { - // No sequence found! - // NOTE: If you encounter this after adding a new instruction, do a full - // rebuild! - XELOGE("Unable to process HIR opcode {}", - hir::GetOpcodeName(instr->opcode)); - assert_always(); - break; - } - instr = new_tail; - } - - block = block->next; - } - - // Function epilog. - l(epilog_label); - epilog_label_ = nullptr; - EmitTraceUserCallReturn(); - LDR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); - - code_offsets.epilog = offset(); - - AdjustStackPointer(*this, stack_size, true); - - MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 16); - - RET(); - - code_offsets.tail = offset(); - - if (cvars::emit_source_annotations) { - NOP(); - NOP(); - NOP(); - NOP(); - NOP(); - } - - assert_zero(code_offsets.prolog); - func_info.code_size.total = offset(); - func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; - func_info.code_size.body = code_offsets.epilog - code_offsets.body; - func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; - func_info.code_size.tail = offset() - code_offsets.tail; - func_info.prolog_stack_alloc_offset = - code_offsets.prolog_stack_alloc - code_offsets.prolog; - - return true; -} - -void A64Emitter::MarkSourceOffset(const Instr* i) { - auto entry = source_map_arena_.Alloc(); - entry->guest_address = static_cast(i->src1.offset); - entry->hir_offset = uint32_t(i->block->ordinal << 16) | i->ordinal; - entry->code_offset = static_cast(offset()); - - if (cvars::emit_source_annotations) { - NOP(); - NOP(); - MOV(X0, entry->guest_address); - NOP(); - NOP(); - } - - if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctionCoverage) { - const uint32_t instruction_index = - (entry->guest_address - trace_data_->start_address()) / 4; - MOV(X0, 1); - MOV(X1, reinterpret_cast( - low_address(trace_data_->instruction_execute_counts() + - instruction_index * 8))); - LDADDAL(X0, ZR, X1); - } -} - -void A64Emitter::EmitGetCurrentThreadId() { - // X27 must point to context. We could fetch from the stack if needed. - LDRH(W0, GetContextReg(), offsetof(ppc::PPCContext, thread_id)); -} - -void A64Emitter::EmitTraceUserCallReturn() {} - -void A64Emitter::DebugBreak() { BRK(0xF000); } - -uint64_t TrapDebugPrint(void* raw_context, uint64_t address) { - auto thread_state = *reinterpret_cast(raw_context); - uint32_t str_ptr = uint32_t(thread_state->context()->r[3]); - // uint16_t str_len = uint16_t(thread_state->context()->r[4]); - auto str = thread_state->memory()->TranslateVirtual(str_ptr); - // TODO(benvanik): truncate to length? - XELOGD("(DebugPrint) {}", str); - - if (cvars::debugprint_trap_log) { - debugging::DebugPrint("(DebugPrint) {}", str); - } - - return 0; -} - -uint64_t TrapLogRegs(void* raw_context, uint64_t address) { - static volatile int32_t log_count = 0; - if (xe::atomic_inc(&log_count) > 8) { - return 0; - } - auto guest_context = reinterpret_cast(raw_context); - if (!guest_context) { - return 0; - } - auto thread_state = guest_context->thread_state; - XELOGI( - "TraceOnInstruction 0x{:08X}: r3=0x{:016X} r4=0x{:016X} r11=0x{:016X} " - "r30=0x{:016X} r31=0x{:016X} lr=0x{:016X} ctr=0x{:016X}", - static_cast(cvars::break_on_instruction), guest_context->r[3], - guest_context->r[4], guest_context->r[11], guest_context->r[30], - guest_context->r[31], guest_context->lr, guest_context->ctr); - if (thread_state) { - auto memory = thread_state->memory(); - if (memory) { - auto page_access_to_string = [](xe::memory::PageAccess access) { - switch (access) { - case xe::memory::PageAccess::kNoAccess: - return "no-access"; - case xe::memory::PageAccess::kReadOnly: - return "read-only"; - case xe::memory::PageAccess::kReadWrite: - return "read-write"; - case xe::memory::PageAccess::kExecuteReadOnly: - return "exec-read"; - case xe::memory::PageAccess::kExecuteReadWrite: - return "exec-read-write"; - } - return "unknown"; - }; - auto heap_type_to_string = [](HeapType type) { - switch (type) { - case HeapType::kGuestVirtual: - return "guest-virtual"; - case HeapType::kGuestXex: - return "guest-xex"; - case HeapType::kGuestPhysical: - return "guest-physical"; - case HeapType::kHostPhysical: - return "host-physical"; - } - return "unknown"; - }; - auto can_read_guest = [&](uint32_t addr) -> bool { - if (!addr) { - return false; - } - auto* heap = memory->LookupHeap(addr); - if (!heap) { - return false; - } - return heap->QueryRangeAccess(addr, addr) != - xe::memory::PageAccess::kNoAccess; - }; - auto log_guest_bytes = [&](uint32_t addr, const char* label) { - if (!can_read_guest(addr)) { - auto* heap = memory->LookupHeap(addr); - XELOGI( - "TraceOnInstruction {}: addr=0x{:08X} unreadable heap={} " - "access={}", - label, addr, - heap ? heap_type_to_string(heap->heap_type()) : "none", - heap ? page_access_to_string(heap->QueryRangeAccess(addr, addr)) - : "no-access"); - return; - } - const auto* heap = memory->LookupHeap(addr); - const uint8_t* host_ptr = nullptr; - if (heap && heap->heap_type() == HeapType::kGuestPhysical) { - uint32_t physical_address = memory->GetPhysicalAddress(addr); - host_ptr = memory->TranslatePhysical(physical_address); - } else { - host_ptr = memory->TranslateVirtual(addr); - } - if (!host_ptr) { - XELOGI("TraceOnInstruction {}: addr=0x{:08X} null", label, addr); - return; - } - uint8_t bytes[16] = {}; - std::memcpy(bytes, host_ptr, sizeof(bytes)); - char ascii[sizeof(bytes) + 1] = {}; - for (size_t i = 0; i < sizeof(bytes); ++i) { - uint8_t ch = bytes[i]; - ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; - } - XELOGI( - "TraceOnInstruction {}: addr=0x{:08X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", - label, addr, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], - bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], - bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); - }; - auto log_guest_string = [&](uint32_t addr, const char* label) { - if (!can_read_guest(addr)) { - return; - } - const uint8_t* ptr = memory->TranslateVirtual(addr); - if (!ptr) { - return; - } - char buffer[129] = {}; - size_t len = 0; - for (; len < sizeof(buffer) - 1; ++len) { - char ch = static_cast(ptr[len]); - if (!ch) { - break; - } - if (!std::isprint(static_cast(ch))) { - return; - } - buffer[len] = ch; - } - if (len > 0) { - XELOGI("TraceOnInstruction {}: {}", label, buffer); - } - }; - const uint32_t guest_address = static_cast(guest_context->r[4]); - const auto* heap = memory->LookupHeap(guest_address); - if (heap) { - const uint8_t* host_ptr = nullptr; - if (heap->heap_type() == HeapType::kGuestPhysical) { - uint32_t physical_address = memory->GetPhysicalAddress(guest_address); - host_ptr = memory->TranslatePhysical(physical_address); - } else { - host_ptr = memory->TranslateVirtual(guest_address); - } - if (host_ptr) { - uint8_t bytes[16] = {}; - std::memcpy(bytes, host_ptr, sizeof(bytes)); - char ascii[sizeof(bytes) + 1] = {}; - for (size_t i = 0; i < sizeof(bytes); ++i) { - uint8_t ch = bytes[i]; - ascii[i] = (ch >= 0x20 && ch <= 0x7E) ? static_cast(ch) : '.'; - } - XELOGI( - "TraceOnInstruction mem[r4]=0x{:08X}: {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} ascii={}", - guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], - bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], - bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], ascii); - } - } - - auto read_u32 = [&](uint32_t addr, uint32_t* out) -> bool { - if (!can_read_guest(addr)) { - return false; - } - const auto* heap = memory->LookupHeap(addr); - if (heap->heap_type() == HeapType::kGuestPhysical) { - uint32_t physical_address = memory->GetPhysicalAddress(addr); - auto ptr = - memory->TranslatePhysical(physical_address); - if (!ptr) { - return false; - } - *out = xe::load_and_swap(ptr); - return true; - } - auto ptr = memory->TranslateVirtual(addr); - if (!ptr) { - return false; - } - *out = xe::load_and_swap(ptr); - return true; - }; - const uint32_t trace_pc = - static_cast(cvars::break_on_instruction); - auto log_trace_instr = [&](uint32_t pc, const char* label) { - if (!can_read_guest(pc)) { - auto* heap = memory->LookupHeap(pc); - XELOGI( - "TraceOnInstruction {}: pc=0x{:08X} unreadable heap={} " - "access={}", - label, pc, heap ? heap_type_to_string(heap->heap_type()) : "none", - heap ? page_access_to_string(heap->QueryRangeAccess(pc, pc)) - : "no-access"); - return; - } - uint32_t instr = 0; - if (!read_u32(pc, &instr)) { - XELOGI("TraceOnInstruction {}: pc=0x{:08X} unreadable", label, pc); - return; - } - xe::StringBuffer disasm; - if (cpu::ppc::DisasmPPC(pc, instr, &disasm)) { - XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X} {}", label, - pc, instr, disasm.to_string_view()); - } else { - XELOGI("TraceOnInstruction {}: pc=0x{:08X} instr=0x{:08X}", label, pc, - instr); - } - }; - if (trace_pc) { - log_trace_instr(trace_pc - 4, "target-4"); - log_trace_instr(trace_pc, "target"); - log_trace_instr(trace_pc + 4, "target+4"); - } - if (memory->LookupHeap(trace_pc)) { - for (int offset = -4; offset <= 4; ++offset) { - uint32_t pc = trace_pc + offset * 4; - if (!memory->LookupHeap(pc)) { - continue; - } - uint32_t instr = 0; - if (!read_u32(pc, &instr)) { - XELOGI("TraceOnInstruction window: pc=0x{:08X} unreadable", pc); - continue; - } - xe::StringBuffer disasm_window; - if (cpu::ppc::DisasmPPC(pc, instr, &disasm_window)) { - XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X} {}", - pc, instr, disasm_window.to_string_view()); - } else { - XELOGI("TraceOnInstruction window: pc=0x{:08X} instr=0x{:08X}", pc, - instr); - } - } - } - - const uint32_t obj_address = static_cast(guest_context->r[3]); - if (!obj_address) { - XELOGI("TraceOnInstruction r3 fields: base=0x00000000"); - } else { - const auto* obj_heap = memory->LookupHeap(obj_address); - if (obj_heap) { - uint32_t value_4 = 0; - uint32_t value_8 = 0; - uint32_t value_c = 0; - uint32_t value_20 = 0; - uint32_t value_470 = 0; - bool have_any = false; - have_any |= read_u32(obj_address + 0x4, &value_4); - have_any |= read_u32(obj_address + 0x8, &value_8); - have_any |= read_u32(obj_address + 0xC, &value_c); - have_any |= read_u32(obj_address + 0x20, &value_20); - have_any |= read_u32(obj_address + 0x470, &value_470); - if (have_any) { - XELOGI( - "TraceOnInstruction r3 fields: base=0x{:08X} +0x4=0x{:08X} " - "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", - obj_address, value_4, value_8, value_c, value_20, value_470); - if (value_20) { - log_guest_bytes(value_20, "r3+0x20"); - } - if (value_470) { - log_guest_bytes(value_470, "r3+0x470"); - } - } else { - XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} unmapped", - obj_address); - } - } else { - XELOGI("TraceOnInstruction r3 fields: base=0x{:08X} heap=null", - obj_address); - } - log_guest_string(obj_address, "r3 string"); - } - - const uint32_t r31_address = static_cast(guest_context->r[31]); - if (!r31_address) { - XELOGI("TraceOnInstruction r31 fields: base=0x00000000"); - } else { - const auto* r31_heap = memory->LookupHeap(r31_address); - if (r31_heap) { - uint32_t value_4 = 0; - uint32_t value_8 = 0; - uint32_t value_c = 0; - uint32_t value_20 = 0; - uint32_t value_470 = 0; - bool have_any = false; - have_any |= read_u32(r31_address + 0x4, &value_4); - have_any |= read_u32(r31_address + 0x8, &value_8); - have_any |= read_u32(r31_address + 0xC, &value_c); - have_any |= read_u32(r31_address + 0x20, &value_20); - have_any |= read_u32(r31_address + 0x470, &value_470); - if (have_any) { - XELOGI( - "TraceOnInstruction r31 fields: base=0x{:08X} +0x4=0x{:08X} " - "+0x8=0x{:08X} +0xC=0x{:08X} +0x20=0x{:08X} +0x470=0x{:08X}", - r31_address, value_4, value_8, value_c, value_20, value_470); - if (value_20) { - log_guest_bytes(value_20, "r31+0x20"); - } - if (value_470) { - log_guest_bytes(value_470, "r31+0x470"); - } - } else { - XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} unmapped", - r31_address); - } - } else { - XELOGI("TraceOnInstruction r31 fields: base=0x{:08X} heap=null", - r31_address); - } - } - } - } - return 0; -} - -uint64_t TrapDebugBreak(void* raw_context, uint64_t address) { - [[maybe_unused]] auto thread_state = - *reinterpret_cast(raw_context); - XELOGE("tw/td forced trap hit! This should be a crash!"); - if (cvars::break_on_debugbreak) { - xe::debugging::Break(); - } - return 0; -} - -void A64Emitter::Trap(uint16_t trap_type) { - switch (trap_type) { - case 20: - case 26: - // 0x0FE00014 is a 'debug print' where r3 = buffer r4 = length - CallNative(TrapDebugPrint, 0); - break; - case 27: - CallNative(TrapLogRegs, 0); - break; - case 0: - case 22: - // Always trap? - // TODO(benvanik): post software interrupt to debugger. - CallNative(TrapDebugBreak, 0); - break; - case 25: - // ? - break; - default: - XELOGW("Unknown trap type {}", trap_type); - BRK(0xF000); - break; - } -} - -void A64Emitter::UnimplementedInstr(const hir::Instr* i) { - // TODO(benvanik): notify debugger. - BRK(0xF000); - assert_always(); -} - -// This is used by the A64ThunkEmitter's ResolveFunctionThunk. -uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { - auto guest_context = reinterpret_cast(raw_context); - assert_not_null(guest_context); - auto thread_state = guest_context->thread_state; - assert_not_null(thread_state); - - assert_not_zero(target_address); - - uint32_t guest_address = 0; - if (target_address > 0xFFFFFFFF) { - auto ctx_ptr = reinterpret_cast(guest_context); - if (target_address >= ctx_ptr && - target_address < ctx_ptr + sizeof(ppc::PPCContext)) { - XELOGE( - "ResolveFunction: target_address 0x{:016X} is within PPCContext " - "[0x{:016X}, 0x{:016X})", - target_address, ctx_ptr, ctx_ptr + sizeof(ppc::PPCContext)); - XELOGE( - "ResolveFunction: The target register contains a context pointer " - "instead of a function address"); - return 0; - } - - auto code_cache = static_cast( - thread_state->processor()->backend()->code_cache()); - auto guest_function = code_cache->LookupFunction(target_address); - if (guest_function) { - guest_address = - guest_function->MapMachineCodeToGuestAddress(target_address); - } else { - guest_address = static_cast(target_address); - } - } else { - guest_address = static_cast(target_address); - } - - if (guest_address == 0) { - XELOGE("ResolveFunction: guest_address is 0! This should not happen"); - return 0; - } - - auto fn = thread_state->processor()->ResolveFunction(guest_address); - if (!fn) { - XELOGE( - "ResolveFunction: Failed to resolve function at guest address 0x{:08X}", - guest_address); - XELOGE("ResolveFunction: Original target_address was 0x{:016X}", - target_address); - if (ShouldLogResolveFailure()) { - const uint32_t lr_guest = static_cast(guest_context->lr); - XELOGI( - "ResolveFunction: lr=0x{:016X} ctr=0x{:016X} thread_id={} " - "target_is_host={} guest_address=0x{:08X}", - guest_context->lr, guest_context->ctr, guest_context->thread_id, - target_address > 0xFFFFFFFF, guest_address); - auto log_modules_for_address = [&](uint32_t address, const char* label) { - bool found = false; - for (auto* module : thread_state->processor()->GetModules()) { - if (!module) { - continue; - } - if (module->ContainsAddress(address)) { - XELOGI("ResolveFunction: {} module '{}' contains 0x{:08X}", label, - module->name(), address); - found = true; - } - } - if (!found) { - XELOGI("ResolveFunction: {} no module contains 0x{:08X}", label, - address); - } - }; - log_modules_for_address(lr_guest, "lr"); - log_modules_for_address(guest_address, "guest"); - - auto lr_functions = - thread_state->processor()->FindFunctionsWithAddress(lr_guest); - if (lr_functions.empty()) { - XELOGI("ResolveFunction: no resolved function covers LR 0x{:08X}", - lr_guest); - } else { - const auto* fn = lr_functions.front(); - XELOGI("ResolveFunction: LR function {} [0x{:08X},0x{:08X}) name='{}'", - lr_functions.size(), fn->address(), fn->end_address(), - fn->name()); - } - - auto* memory = thread_state->memory(); - if (!memory) { - XELOGI("ResolveFunction: no Memory available for guest dump"); - } else if (!memory->LookupHeap(guest_address)) { - XELOGI( - "ResolveFunction: guest_address 0x{:08X} not in any heap for dump", - guest_address); - } else { - const uint8_t* data = - memory->TranslateVirtual(guest_address); - std::array bytes = {}; - std::memcpy(bytes.data(), data, bytes.size()); - XELOGI( - "ResolveFunction: guest[0x{:08X}] = {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} {:02X} " - "{:02X} {:02X} {:02X} {:02X}", - guest_address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], - bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], - bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]); - } - } - return 0; - } - - auto a64_fn = static_cast(fn); - if (!a64_fn->machine_code()) { - XELOGE( - "ResolveFunction: Function at guest address 0x{:08X} has no machine " - "code", - guest_address); - return 0; - } - - return reinterpret_cast(a64_fn->machine_code()); -} - -void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { - assert_not_null(function); - auto fn = static_cast(function); - // Resolve address to the function to call and store in X16. - if (fn->machine_code()) { - // TODO(benvanik): is it worth it to do this? It removes the need for - // a ResolveFunction call, but makes the table less useful. -#if XE_A64_INDIRECTION_64BIT - MOV(X16, reinterpret_cast(fn->machine_code())); -#else - assert_zero(uint64_t(fn->machine_code()) & 0xFFFFFFFF00000000); - MOV(X16, uint32_t(uint64_t(fn->machine_code()))); -#endif - } else if (code_cache_->has_indirection_table()) { - // Load the pointer to the indirection table maintained in A64CodeCache. - // The target dword will either contain the address of the generated code - // or a thunk to ResolveAddress. - MOV(W17, function->address()); -#if XE_A64_INDIRECTION_64BIT - // ARM64 platforms with 64-bit indirection entries must compute an offset. - MOV(X16, code_cache_->indirection_table_base_bias()); - LSL(X15, X17, 1); - ADD(X16, X16, X15); - LDR(X16, X16); -#else - // Other platforms use 32-bit addresses mapped at guest address space. - if (code_cache_->indirection_table_base_address() == - A64CodeCache::kIndirectionTableBase) { - LDR(W16, X17); - } else { - MOV(W16, static_cast(A64CodeCache::kIndirectionTableBase)); - SUB(W17, W17, W16); - MOV(X16, code_cache_->indirection_table_base_address()); - ADD(X16, X16, W17, UXTW); - LDR(W16, X16); - } -#endif - } else { - // Old-style resolve. - // Not too important because indirection table is almost always available. - // TODO: Overwrite the call-site with a straight call. - CallNative(&ResolveFunction, function->address()); - MOV(X16, X0); - } - - // Actually jump/call to X16. - if (instr->flags & hir::CALL_TAIL) { - // Since we skip the prolog we need to mark the return here. - EmitTraceUserCallReturn(); - - // Pass the callers return address over. - LDR(X0, SP, StackLayout::GUEST_RET_ADDR); - - AdjustStackPointer(*this, stack_size(), true); - - MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 16); - - BR(X16); - } else { - // Return address is from the previous SET_RETURN_ADDRESS. - LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); - - BLR(X16); - } -} - -void A64Emitter::CallIndirect(const hir::Instr* instr, - const oaknut::XReg& reg) { - // Check if return. - if (instr->flags & hir::CALL_POSSIBLE_RETURN) { - LDR(W16, SP, StackLayout::GUEST_RET_ADDR); - CMP(reg.toW(), W16); - B(oaknut::Cond::EQ, epilog_label()); - } - - // Load the pointer to the indirection table maintained in A64CodeCache. - // The target dword will either contain the address of the generated code - // or a thunk to ResolveAddress. - if (code_cache_->has_indirection_table()) { - if (reg.toW().index() != W17.index()) { - MOV(W17, reg.toW()); - } -#if XE_A64_INDIRECTION_64BIT - // ARM64 platforms with 64-bit indirection entries must compute an offset. - MOV(X16, code_cache_->indirection_table_base_bias()); - LSL(X15, X17, 1); - ADD(X16, X16, X15); - LDR(X16, X16); -#else - // Other platforms use 32-bit addresses mapped at guest address space. - if (code_cache_->indirection_table_base_address() == - A64CodeCache::kIndirectionTableBase) { - LDR(W16, X17); - } else { - MOV(W16, static_cast(A64CodeCache::kIndirectionTableBase)); - SUB(W17, W17, W16); - MOV(X16, code_cache_->indirection_table_base_address()); - ADD(X16, X16, W17, UXTW); - LDR(W16, X16); - } -#endif - } else { - // Old-style resolve. - // Not too important because indirection table is almost always available. - MOV(X0, GetContextReg()); - MOV(W1, reg.toW()); - - MOV(X16, reinterpret_cast(ResolveFunction)); - BLR(X16); - MOV(X16, X0); - } - - // Actually jump/call to X16. - if (instr->flags & hir::CALL_TAIL) { - // Since we skip the prolog we need to mark the return here. - EmitTraceUserCallReturn(); - - // Pass the callers return address over. - LDR(X0, SP, StackLayout::GUEST_RET_ADDR); - - AdjustStackPointer(*this, stack_size(), true); - - MOV(SP, X29); - LDP(X29, X30, SP, POST_INDEXED, 16); - - BR(X16); - } else { - // Return address is from the previous SET_RETURN_ADDRESS. - LDR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); - - BLR(X16); - } -} - -uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) { - auto function = reinterpret_cast(function_ptr); - if (cvars::log_undefined_extern_args && - function->name() == "XeKeysConsolePrivateKeySign") { - static std::atomic logged{false}; - if (!logged.exchange(true)) { - auto* context = reinterpret_cast(raw_context); - XELOGI( - "Undefined extern {} args: r3={:016X} r4={:016X} r5={:016X} " - "r6={:016X} r7={:016X} r8={:016X} r9={:016X} r10={:016X}", - function->name(), context->r[3], context->r[4], context->r[5], - context->r[6], context->r[7], context->r[8], context->r[9], - context->r[10]); - } - } - if (!cvars::ignore_undefined_externs) { - xe::FatalError(fmt::format("undefined extern call to {:08X} {}", - function->address(), function->name().c_str())); - } else { - XELOGE("undefined extern call to {:08X} {}", function->address(), - function->name()); - } - return 0; -} -void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { - bool undefined = true; - if (function->behavior() == Function::Behavior::kBuiltin) { - auto builtin_function = static_cast(function); - if (builtin_function->handler()) { - undefined = false; - // x0 = target function - // x1 = arg0 - // x2 = arg1 - // x3 = arg2 - MOV(X0, reinterpret_cast(builtin_function->handler())); - MOV(X1, reinterpret_cast(builtin_function->arg0())); - MOV(X2, reinterpret_cast(builtin_function->arg1())); - - auto thunk = backend()->guest_to_host_thunk(); - MOV(X16, reinterpret_cast(thunk)); - BLR(X16); - - // x0 = host return - } - } else if (function->behavior() == Function::Behavior::kExtern) { - auto extern_function = static_cast(function); - if (extern_function->extern_handler()) { - undefined = false; - // x0 = target function - // x1 = arg0 - // x2 = arg1 - // x3 = arg2 - MOV(X0, reinterpret_cast(extern_function->extern_handler())); - LDR(X1, GetContextReg(), offsetof(ppc::PPCContext, kernel_state)); - - auto thunk = backend()->guest_to_host_thunk(); - MOV(X16, reinterpret_cast(thunk)); - BLR(X16); - - // x0 = host return - } - } - if (undefined) { - CallNative(UndefinedCallExtern, reinterpret_cast(function)); - } -} - -void A64Emitter::CallNative(void* fn) { CallNativeSafe(fn); } - -void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context)) { - CallNativeSafe(reinterpret_cast(fn)); -} - -void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0)) { - CallNativeSafe(reinterpret_cast(fn)); -} - -void A64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0), - uint64_t arg0) { - MOV(GetNativeParam(0), arg0); - CallNativeSafe(reinterpret_cast(fn)); -} - -void A64Emitter::CallNativeSafe(void* fn) { - // X0 = target function - // X1 = arg0 - // X2 = arg1 - // X3 = arg2 - auto thunk = backend()->guest_to_host_thunk(); - - MOV(X0, reinterpret_cast(fn)); - - MOV(X16, reinterpret_cast(thunk)); - BLR(X16); - - // X0 = host return -} - -void A64Emitter::SetReturnAddress(uint64_t value) { - MOV(X0, value); - STR(X0, SP, StackLayout::GUEST_CALL_RET_ADDR); -} - -oaknut::XReg A64Emitter::GetNativeParam(uint32_t param) { - if (param == 0) - return X1; - else if (param == 1) - return X2; - else if (param == 2) - return X3; - - assert_always(); - return X3; -} - -// Important: If you change these, you must update the thunks in a64_backend.cc! -oaknut::XReg A64Emitter::GetContextReg() { return X27; } -oaknut::XReg A64Emitter::GetMembaseReg() { return X28; } - -void A64Emitter::ReloadContext() { - LDR(GetContextReg(), SP, StackLayout::GUEST_CTX_HOME); -} - -void A64Emitter::ReloadMembase() { - LDR(GetMembaseReg(), GetContextReg(), - offsetof(ppc::PPCContext, virtual_membase)); -} - -bool A64Emitter::ConstantFitsIn32Reg(uint64_t v) { - if ((v & ~0x7FFFFFFF) == 0) { - // Fits under 31 bits, so just load using normal mov. - return true; - } else if ((v & ~0x7FFFFFFFUL) == ~0x7FFFFFFFUL) { - // Negative number that fits in 32bits. - return true; - } - return false; -} - -void A64Emitter::MovMem64(const oaknut::XRegSp& addr, intptr_t offset, - uint64_t v) { - if (v == 0) { - STR(XZR, addr, offset); - } else if (!(v >> 32)) { - // All high bits are zero, 32-bit MOV - MOV(W0, static_cast(v)); - STR(X0, addr, offset); - } else { - // 64bit number that needs double movs. - MOV(X0, v); - STR(X0, addr, offset); - } -} - -static const vec128_t v_consts[] = { - /* VZero */ vec128f(0.0f), - /* VOnePD */ vec128d(1.0), - /* VNegativeOne */ vec128f(-1.0f, -1.0f, -1.0f, -1.0f), - /* VFFFF */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu), - /* VMaskX16Y16 */ - vec128i(0x0000FFFFu, 0xFFFF0000u, 0x00000000u, 0x00000000u), - /* VFlipX16Y16 */ - vec128i(0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u), - /* VFixX16Y16 */ vec128f(-32768.0f, 0.0f, 0.0f, 0.0f), - /* VNormalizeX16Y16 */ - vec128f(1.0f / 32767.0f, 1.0f / (32767.0f * 65536.0f), 0.0f, 0.0f), - /* V0001 */ vec128f(0.0f, 0.0f, 0.0f, 1.0f), - /* V3301 */ vec128f(3.0f, 3.0f, 0.0f, 1.0f), - /* V3331 */ vec128f(3.0f, 3.0f, 3.0f, 1.0f), - /* V3333 */ vec128f(3.0f, 3.0f, 3.0f, 3.0f), - /* VSignMaskPS */ - vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u), - /* VSignMaskPD */ - vec128i(0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u), - /* VAbsMaskPS */ - vec128i(0x7FFFFFFFu, 0x7FFFFFFFu, 0x7FFFFFFFu, 0x7FFFFFFFu), - /* VAbsMaskPD */ - vec128i(0xFFFFFFFFu, 0x7FFFFFFFu, 0xFFFFFFFFu, 0x7FFFFFFFu), - /* VByteSwapMask */ - vec128i(0x00010203u, 0x04050607u, 0x08090A0Bu, 0x0C0D0E0Fu), - /* VByteOrderMask */ - vec128i(0x01000302u, 0x05040706u, 0x09080B0Au, 0x0D0C0F0Eu), - /* VPermuteControl15 */ vec128b(15), - /* VPermuteByteMask */ vec128b(0x1F), - /* VPackD3DCOLORSat */ vec128i(0x404000FFu), - /* VPackD3DCOLOR */ - // Note: x86 PSHUFB uses 0xFF to zero bytes, ARM TBL uses indices >= 16 - // Keep original 0xFF for consistency, handle in implementation - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0C000408u), - /* VUnpackD3DCOLOR */ - vec128i(0xFFFFFF0Eu, 0xFFFFFF0Du, 0xFFFFFF0Cu, 0xFFFFFF0Fu), - /* VPackFLOAT16_2 */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000302u), - /* VUnpackFLOAT16_2 */ - vec128i(0x0D0C0F0Eu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu), - /* VPackFLOAT16_4 */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000302u, 0x05040706u), - /* VUnpackFLOAT16_4 */ - vec128i(0x09080B0Au, 0x0D0C0F0Eu, 0xFFFFFFFFu, 0xFFFFFFFFu), - /* VPackSHORT_Min */ vec128i(0x403F8001u), - /* VPackSHORT_Max */ vec128i(0x40407FFFu), - /* VPackSHORT_2 */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000504u), - /* VPackSHORT_4 */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000504u, 0x09080D0Cu), - /* VUnpackSHORT_2 */ - vec128i(0xFFFF0F0Eu, 0xFFFF0D0Cu, 0xFFFFFFFFu, 0xFFFFFFFFu), - /* VUnpackSHORT_4 */ - vec128i(0xFFFF0B0Au, 0xFFFF0908u, 0xFFFF0F0Eu, 0xFFFF0D0Cu), - /* VUnpackSHORT_Overflow */ vec128i(0x403F8000u), - /* VPackUINT_2101010_MinUnpacked */ - vec128i(0x403FFE01u, 0x403FFE01u, 0x403FFE01u, 0x40400000u), - /* VPackUINT_2101010_MaxUnpacked */ - vec128i(0x404001FFu, 0x404001FFu, 0x404001FFu, 0x40400003u), - /* VPackUINT_2101010_MaskUnpacked */ - vec128i(0x3FFu, 0x3FFu, 0x3FFu, 0x3u), - /* VPackUINT_2101010_MaskPacked */ - vec128i(0x3FFu, 0x3FFu << 10, 0x3FFu << 20, 0x3u << 30), - /* VPackUINT_2101010_Shift */ vec128i(0, 10, 20, 30), - /* VUnpackUINT_2101010_Overflow */ vec128i(0x403FFE00u), - /* VPackULONG_4202020_MinUnpacked */ - vec128i(0x40380001u, 0x40380001u, 0x40380001u, 0x40400000u), - /* VPackULONG_4202020_MaxUnpacked */ - vec128i(0x4047FFFFu, 0x4047FFFFu, 0x4047FFFFu, 0x4040000Fu), - /* VPackULONG_4202020_MaskUnpacked */ - vec128i(0xFFFFFu, 0xFFFFFu, 0xFFFFFu, 0xFu), - /* VPackULONG_4202020_PermuteXZ */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x0A0908FFu, 0xFF020100u), - /* VPackULONG_4202020_PermuteYW */ - vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x0CFFFF06u, 0x0504FFFFu), - /* VUnpackULONG_4202020_Permute */ - vec128i(0xFF0E0D0Cu, 0xFF0B0A09u, 0xFF080F0Eu, 0xFFFFFF0Bu), - /* VUnpackULONG_4202020_Overflow */ vec128i(0x40380000u), - /* VOneOver255 */ vec128f(1.0f / 255.0f), - /* VMaskEvenPI16 */ - vec128i(0x0000FFFFu, 0x0000FFFFu, 0x0000FFFFu, 0x0000FFFFu), - /* VShiftMaskEvenPI16 */ - vec128i(0x0000000Fu, 0x0000000Fu, 0x0000000Fu, 0x0000000Fu), - /* VShiftMaskPS */ - vec128i(0x0000001Fu, 0x0000001Fu, 0x0000001Fu, 0x0000001Fu), - /* VShiftByteMask */ - vec128i(0x000000FFu, 0x000000FFu, 0x000000FFu, 0x000000FFu), - /* VSwapWordMask */ - vec128i(0x03030303u, 0x03030303u, 0x03030303u, 0x03030303u), - /* VUnsignedDwordMax */ - vec128i(0xFFFFFFFFu, 0x00000000u, 0xFFFFFFFFu, 0x00000000u), - /* V255 */ vec128f(255.0f), - /* VPI32 */ vec128i(32), - /* VSignMaskI8 */ - vec128i(0x80808080u, 0x80808080u, 0x80808080u, 0x80808080u), - /* VSignMaskI16 */ - vec128i(0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u), - /* VSignMaskI32 */ - vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u), - /* VSignMaskF32 */ - vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u), - /* VShortMinPS */ vec128f(SHRT_MIN), - /* VShortMaxPS */ vec128f(SHRT_MAX), - /* VIntMin */ vec128i(INT_MIN), - /* VIntMax */ vec128i(INT_MAX), - /* VIntMaxPD */ vec128d(INT_MAX), - /* VPosIntMinPS */ vec128f((float)0x80000000u), - /* VQNaN */ vec128i(0x7FC00000u), - /* VInt127 */ vec128i(0x7Fu), - /* V2To32 */ vec128f(0x1.0p32f), - /* VSingleDenormalMask */ vec128i(0x7F800000u), -}; - -// First location to try and place constants. -[[maybe_unused]] static const uintptr_t kConstDataLocation = 0x20000000; -static const uintptr_t kConstDataSize = sizeof(v_consts); - -// Increment the location by this amount for every allocation failure. -[[maybe_unused]] static const uintptr_t kConstDataIncrement = 0x00001000; - -// This function places constant data that is used by the emitter later on. -// Only called once and used by multiple instances of the emitter. -// -// TODO(DrChat): This should be placed in the code cache with the code, but -// doing so requires RIP-relative addressing, which is difficult to support -// given the current setup. -uintptr_t A64Emitter::PlaceConstData() { - void* mem = nullptr; -#if XE_PLATFORM_MAC && XE_ARCH_ARM64 - // macOS ARM64 PAGEZERO blocks low fixed mappings; use OS-chosen addresses. - mem = memory::AllocFixed( - nullptr, xe::round_up(kConstDataSize, memory::page_size()), - memory::AllocationType::kReserveCommit, memory::PageAccess::kReadWrite); -#else - uint8_t* ptr = reinterpret_cast(kConstDataLocation); - while (!mem) { - mem = memory::AllocFixed( - ptr, xe::round_up(kConstDataSize, memory::page_size()), - memory::AllocationType::kReserveCommit, memory::PageAccess::kReadWrite); - - ptr += kConstDataIncrement; - } -#endif - -#if XE_PLATFORM_MAC && XE_ARCH_ARM64 - // On macOS ARM64, memory is often allocated in high address space - if (reinterpret_cast(mem) & ~0x7FFFFFFF) { - XELOGD( - "Const data allocated at high address {:#x}, may cause compatibility " - "issues", - reinterpret_cast(mem)); - // Continue anyway since we'll handle it later - } -#else - // The pointer must not be greater than 31 bits. - assert_zero(reinterpret_cast(mem) & ~0x7FFFFFFF); -#endif - std::memcpy(mem, v_consts, sizeof(v_consts)); - memory::Protect(mem, kConstDataSize, memory::PageAccess::kReadOnly, nullptr); - - return reinterpret_cast(mem); -} - -void A64Emitter::FreeConstData(uintptr_t data) { - memory::DeallocFixed(reinterpret_cast(data), 0, - memory::DeallocationType::kRelease); -} - -uintptr_t A64Emitter::GetVConstPtr() const { return backend_->emitter_data(); } - -uintptr_t A64Emitter::GetVConstPtr(VConst id) const { - // Load through fixed constant table setup by PlaceConstData. - // It's important that the pointer is not signed, as it will be sign-extended. - return GetVConstPtr() + GetVConstOffset(id); -} - -// Attempts to convert an fp32 bit-value into an fp8-immediate value for FMOV -// returns false if the value cannot be represented -// C2.2.3 Modified immediate constants in A64 floating-point instructions -// abcdefgh -// V -// aBbbbbbc defgh000 00000000 00000000 -// B = NOT(b) -static bool f32_to_fimm8(uint32_t u32, oaknut::FImm8& fp8) { - const uint32_t sign = (u32 >> 31) & 1; - int32_t exp = ((u32 >> 23) & 0xff) - 127; - int64_t mantissa = u32 & 0x7fffff; - - // Too many mantissa bits - if (mantissa & 0x7ffff) { - return false; - } - // Too many exp bits - if (exp < -3 || exp > 4) { - return false; - } - - // mantissa = (16 + e:f:g:h) / 16. - mantissa >>= 19; - if ((mantissa & 0b1111) != mantissa) { - return false; - } - - // exp = (NOT(b):c:d) - 3 - exp = ((exp + 3) & 0b111) ^ 0b100; - - fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); - return true; -} - -// Attempts to convert an fp64 bit-value into an fp8-immediate value for FMOV -// returns false if the value cannot be represented -// C2.2.3 Modified immediate constants in A64 floating-point instructions -// abcdefgh -// V -// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 -// B = NOT(b) -static bool f64_to_fimm8(uint64_t u64, oaknut::FImm8& fp8) { - const uint32_t sign = (u64 >> 63) & 1; - int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; - int64_t mantissa = u64 & 0xfffffffffffffULL; - - // Too many mantissa bits - if (mantissa & 0xffffffffffffULL) { - return false; - } - // Too many exp bits - if (exp < -3 || exp > 4) { - return false; - } - - // mantissa = (16 + e:f:g:h) / 16. - mantissa >>= 48; - if ((mantissa & 0b1111) != mantissa) { - return false; - } - - // exp = (NOT(b):c:d) - 3 - exp = ((exp + 3) & 0b111) ^ 0b100; - - fp8 = oaknut::FImm8(sign, exp, uint8_t(mantissa)); - return true; -} - -// Implies possible StashV(0, ...)! -void A64Emitter::LoadConstantV(oaknut::QReg dest, const vec128_t& v) { - if (!v.low && !v.high) { - // 0000... - // MOVI is implemented as a register-rename while EOR(x, x, x) is not - // https://dougallj.github.io/applecpu/firestorm.html - MOVI(dest.B16(), 0); - } else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) { - // 1111... - MOVI(dest.B16(), 0xFF); - } else { - // Try to figure out some common splat-patterns to utilize MOVI rather than - // stashing to memory. - const bool all_same_u8 = - std::adjacent_find(std::cbegin(v.u8), std::cend(v.u8), - std::not_equal_to<>()) == std::cend(v.u8); - - if (all_same_u8) { - // 0xXX, 0xXX, 0xXX... - MOVI(dest.B16(), v.u8[0]); - return; - } - - const bool all_same_u16 = - std::adjacent_find(std::cbegin(v.u16), std::cend(v.u16), - std::not_equal_to<>()) == std::cend(v.u16); - - if (all_same_u16) { - if ((v.u16[0] & 0xFF00) == 0) { - // 0x00XX, 0x00XX, 0x00XX... - MOVI(dest.H8(), uint8_t(v.u16[0])); - return; - } else if ((v.u16[0] & 0x00FF) == 0) { - // 0xXX00, 0xXX00, 0xXX00... - MOVI(dest.H8(), uint8_t(v.u16[0] >> 8), oaknut::util::LSL, 8); - return; - } - } - - const bool all_same_u32 = - std::adjacent_find(std::cbegin(v.u32), std::cend(v.u32), - std::not_equal_to<>()) == std::cend(v.u32); - - if (all_same_u32) { - if ((v.u32[0] & 0x00FFFFFF) == 0) { - // This is used a lot for certain float-splats and should be checked - // first before the others - // 0xXX000000, 0xXX000000, 0xXX000000... - MOVI(dest.S4(), uint8_t(v.u32[0] >> 24), oaknut::util::LSL, 24); - return; - } else if ((v.u32[0] & 0xFFFFFF00) == 0) { - // 0x000000XX, 0x000000XX, 0x000000XX... - MOVI(dest.S4(), uint8_t(v.u32[0])); - return; - } else if ((v.u32[0] & 0xFFFF00FF) == 0) { - // 0x0000XX00, 0x0000XX00, 0x0000XX00... - MOVI(dest.S4(), uint8_t(v.u32[0] >> 8), oaknut::util::LSL, 8); - return; - } else if ((v.u32[0] & 0xFF00FFFF) == 0) { - // 0x00XX0000, 0x00XX0000, 0x00XX0000... - MOVI(dest.S4(), uint8_t(v.u32[0] >> 16), oaknut::util::LSL, 16); - return; - } - - // Try to utilize FMOV if possible - oaknut::FImm8 fp8(0); - if (f32_to_fimm8(v.u32[0], fp8)) { - FMOV(dest.S4(), fp8); - return; - } - } - - // TODO(benvanik): see what other common values are. - // TODO(benvanik): build constant table - 99% are reused. - MovMem64(SP, kStashOffset, v.low); - MovMem64(SP, kStashOffset + 8, v.high); - LDR(dest, SP, kStashOffset); - } -} - -void A64Emitter::LoadConstantV(oaknut::QReg dest, float v) { - union { - float f; - uint32_t i; - } x = {v}; - if (!x.i) { - // +0.0f (but not -0.0f because it may be used to flip the sign via xor). - MOVI(dest.B16(), 0); - } else if (x.i == ~uint32_t(0)) { - // 1111... - MOVI(dest.B16(), 0xFF); - } else { - // TODO(benvanik): see what other common values are. - // TODO(benvanik): build constant table - 99% are reused. - - // Try to utilize FMOV if possible - oaknut::FImm8 fp8(0); - if (f32_to_fimm8(x.i, fp8)) { - FMOV(dest.toS(), fp8); - return; - } - - MOV(W0, x.i); - FMOV(dest.toS(), W0); - } -} - -void A64Emitter::LoadConstantV(oaknut::QReg dest, double v) { - union { - double d; - uint64_t i; - } x = {v}; - if (!x.i) { - // +0.0 (but not -0.0 because it may be used to flip the sign via xor). - MOVI(dest.toD(), oaknut::RepImm(0)); - } else if (x.i == ~uint64_t(0)) { - // 1111... - MOVI(dest.toD(), oaknut::RepImm(0xFF)); - } else { - // TODO(benvanik): see what other common values are. - // TODO(benvanik): build constant table - 99% are reused. - - // Try to utilize FMOV if possible - oaknut::FImm8 fp8(0); - if (f64_to_fimm8(x.i, fp8)) { - FMOV(dest.toD(), fp8); - return; - } - - MOV(X0, x.i); - FMOV(dest.toD(), X0); - } -} - -uintptr_t A64Emitter::StashV(int index, const oaknut::QReg& r) { - // auto addr = ptr[rsp + kStashOffset + (index * 16)]; - // vmovups(addr, r); - const auto addr = kStashOffset + (index * 16); - STR(r, SP, addr); - return addr; -} - -uintptr_t A64Emitter::StashConstantV(int index, float v) { - union { - float f; - uint32_t i; - } x = {v}; - const auto addr = kStashOffset + (index * 16); - MovMem64(SP, addr, x.i); - MovMem64(SP, addr + 8, 0); - return addr; -} - -uintptr_t A64Emitter::StashConstantV(int index, double v) { - union { - double d; - uint64_t i; - } x = {v}; - const auto addr = kStashOffset + (index * 16); - MovMem64(SP, addr, x.i); - MovMem64(SP, addr + 8, 0); - return addr; -} - -uintptr_t A64Emitter::StashConstantV(int index, const vec128_t& v) { - const auto addr = kStashOffset + (index * 16); - MovMem64(SP, addr, v.low); - MovMem64(SP, addr + 8, v.high); - return addr; -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_emitter.h b/src/xenia/cpu/backend/a64/a64_emitter.h deleted file mode 100644 index c2352c85e..000000000 --- a/src/xenia/cpu/backend/a64/a64_emitter.h +++ /dev/null @@ -1,277 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ -#define XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ - -#include -#include - -#include "xenia/base/arena.h" -#include "xenia/cpu/function.h" -#include "xenia/cpu/function_trace_data.h" -#include "xenia/cpu/hir/hir_builder.h" -#include "xenia/cpu/hir/instr.h" -#include "xenia/cpu/hir/value.h" -#include "xenia/memory.h" - -#include "oaknut/code_block.hpp" -#include "oaknut/oaknut.hpp" - -namespace xe { -namespace cpu { -class Processor; -} // namespace cpu -} // namespace xe - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -class A64Backend; -class A64CodeCache; - -struct EmitFunctionInfo; - -enum RegisterFlags { - REG_DEST = (1 << 0), - REG_ABCD = (1 << 1), -}; - -enum VConst { - VZero = 0, - VOnePD, - VNegativeOne, - VFFFF, - VMaskX16Y16, - VFlipX16Y16, - VFixX16Y16, - VNormalizeX16Y16, - V0001, - V3301, - V3331, - V3333, - VSignMaskPS, - VSignMaskPD, - VAbsMaskPS, - VAbsMaskPD, - VByteSwapMask, - VByteOrderMask, - VPermuteControl15, - VPermuteByteMask, - VPackD3DCOLORSat, - VPackD3DCOLOR, - VUnpackD3DCOLOR, - VPackFLOAT16_2, - VUnpackFLOAT16_2, - VPackFLOAT16_4, - VUnpackFLOAT16_4, - VPackSHORT_Min, - VPackSHORT_Max, - VPackSHORT_2, - VPackSHORT_4, - VUnpackSHORT_2, - VUnpackSHORT_4, - VUnpackSHORT_Overflow, - VPackUINT_2101010_MinUnpacked, - VPackUINT_2101010_MaxUnpacked, - VPackUINT_2101010_MaskUnpacked, - VPackUINT_2101010_MaskPacked, - VPackUINT_2101010_Shift, - VUnpackUINT_2101010_Overflow, - VPackULONG_4202020_MinUnpacked, - VPackULONG_4202020_MaxUnpacked, - VPackULONG_4202020_MaskUnpacked, - VPackULONG_4202020_PermuteXZ, - VPackULONG_4202020_PermuteYW, - VUnpackULONG_4202020_Permute, - VUnpackULONG_4202020_Overflow, - VOneOver255, - VMaskEvenPI16, - VShiftMaskEvenPI16, - VShiftMaskPS, - VShiftByteMask, - VSwapWordMask, - VUnsignedDwordMax, - V255, - VPI32, - VSignMaskI8, - VSignMaskI16, - VSignMaskI32, - VSignMaskF32, - VShortMinPS, - VShortMaxPS, - VIntMin, - VIntMax, - VIntMaxPD, - VPosIntMinPS, - VQNaN, - VInt127, - V2To32, - VSingleDenormalMask, -}; - -enum A64EmitterFeatureFlags { - kA64EmitLSE = 1 << 0, - kA64EmitF16C = 1 << 1, -}; - -class A64Emitter : public oaknut::VectorCodeGenerator { - public: - A64Emitter(A64Backend* backend); - virtual ~A64Emitter(); - - Processor* processor() const { return processor_; } - A64Backend* backend() const { return backend_; } - - static uintptr_t PlaceConstData(); - static void FreeConstData(uintptr_t data); - - bool Emit(GuestFunction* function, hir::HIRBuilder* builder, - uint32_t debug_info_flags, FunctionDebugInfo* debug_info, - void** out_code_address, size_t* out_code_size, - std::vector* out_source_map); - - public: - // Reserved: XSP, X27, X28 - // Scratch: X1-X15, X30 | V0-v7 and V16-V31 - // V0-2 - // Available: X19-X26 - // V4-V15 (save to get V3) - static const size_t GPR_COUNT = 8; - static const size_t FPR_COUNT = 8; - - static void SetupReg(const hir::Value* v, oaknut::WReg& r) { - const auto idx = gpr_reg_map_[v->reg.index]; - r = oaknut::WReg(idx); - } - static void SetupReg(const hir::Value* v, oaknut::XReg& r) { - const auto idx = gpr_reg_map_[v->reg.index]; - r = oaknut::XReg(idx); - } - static void SetupReg(const hir::Value* v, oaknut::SReg& r) { - const auto idx = fpr_reg_map_[v->reg.index]; - r = oaknut::SReg(idx); - } - static void SetupReg(const hir::Value* v, oaknut::DReg& r) { - const auto idx = fpr_reg_map_[v->reg.index]; - r = oaknut::DReg(idx); - } - static void SetupReg(const hir::Value* v, oaknut::QReg& r) { - const auto idx = fpr_reg_map_[v->reg.index]; - r = oaknut::QReg(idx); - } - - void EmitBtiJc(); - - // Gets(and possibly create) an HIR label with the specified name - oaknut::Label* lookup_label(const char* label_name) { - return &label_lookup_[label_name]; - } - oaknut::Label* lookup_label(hir::Label* label) { - assert_not_null(label); - if (label->name) { - return &label_lookup_[label->name]; - } - return &label_lookup_[label->GetIdString()]; - } - - oaknut::Label& epilog_label() { return *epilog_label_; } - - void MarkSourceOffset(const hir::Instr* i); - - void DebugBreak(); - void Trap(uint16_t trap_type = 0); - void UnimplementedInstr(const hir::Instr* i); - - void Call(const hir::Instr* instr, GuestFunction* function); - void CallIndirect(const hir::Instr* instr, const oaknut::XReg& reg); - void CallExtern(const hir::Instr* instr, const Function* function); - void CallNative(void* fn); - void CallNative(uint64_t (*fn)(void* raw_context)); - void CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0)); - void CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0), - uint64_t arg0); - void CallNativeSafe(void* fn); - void SetReturnAddress(uint64_t value); - - static oaknut::XReg GetNativeParam(uint32_t param); - - static oaknut::XReg GetContextReg(); - static oaknut::XReg GetMembaseReg(); - void ReloadContext(); - void ReloadMembase(); - - // Moves a 64bit immediate into memory. - static bool ConstantFitsIn32Reg(uint64_t v); - void MovMem64(const oaknut::XRegSp& addr, intptr_t offset, uint64_t v); - - uintptr_t GetVConstPtr() const; - uintptr_t GetVConstPtr(VConst id) const; - static constexpr uintptr_t GetVConstOffset(VConst id) { - return sizeof(vec128_t) * id; - } - void LoadConstantV(oaknut::QReg dest, float v); - void LoadConstantV(oaknut::QReg dest, double v); - void LoadConstantV(oaknut::QReg dest, const vec128_t& v); - - // Returned addresses are relative to XSP - uintptr_t StashV(int index, const oaknut::QReg& r); - uintptr_t StashConstantV(int index, float v); - uintptr_t StashConstantV(int index, double v); - uintptr_t StashConstantV(int index, const vec128_t& v); - - bool IsFeatureEnabled(uint32_t feature_flag) const { - return (feature_flags_ & feature_flag) == feature_flag; - } - - FunctionDebugInfo* debug_info() const { return debug_info_; } - - size_t stack_size() const { return stack_size_; } - - protected: - void* Emplace(const EmitFunctionInfo& func_info, - GuestFunction* function = nullptr); - bool Emit(hir::HIRBuilder* builder, EmitFunctionInfo& func_info); - void EmitGetCurrentThreadId(); - void EmitTraceUserCallReturn(); - - protected: - Processor* processor_ = nullptr; - A64Backend* backend_ = nullptr; - A64CodeCache* code_cache_ = nullptr; - uint32_t feature_flags_ = 0; - - std::vector assembly_buffer; - - oaknut::Label* epilog_label_ = nullptr; - - // Convert from plain-text label-names into oaknut-labels - std::unordered_map label_lookup_; - - hir::Instr* current_instr_ = nullptr; - - FunctionDebugInfo* debug_info_ = nullptr; - uint32_t debug_info_flags_ = 0; - FunctionTraceData* trace_data_ = nullptr; - Arena source_map_arena_; - - size_t stack_size_ = 0; - - static const uint8_t gpr_reg_map_[GPR_COUNT]; - static const uint8_t fpr_reg_map_[FPR_COUNT]; -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ diff --git a/src/xenia/cpu/backend/a64/a64_function.cc b/src/xenia/cpu/backend/a64/a64_function.cc deleted file mode 100644 index 0738a1bc4..000000000 --- a/src/xenia/cpu/backend/a64/a64_function.cc +++ /dev/null @@ -1,75 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_function.h" - -#ifdef XE_PLATFORM_MAC -#include -#include -#include -#include -#endif - -#include "xenia/base/logging.h" -#include "xenia/cpu/backend/a64/a64_backend.h" -#include "xenia/cpu/processor.h" -#include "xenia/cpu/thread_state.h" - -#if XE_PLATFORM_MAC && defined(__aarch64__) -thread_local bool jit_thread_initialized = false; - -// Initialize JIT execution for the current thread -static void InitializeJITThread() { - if (!jit_thread_initialized) { - // Ensure this thread can execute JIT code by setting execute mode - pthread_jit_write_protect_np(1); // Enable execute, disable write - jit_thread_initialized = true; - } -} -#endif - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -A64Function::A64Function(Module* module, uint32_t address) - : GuestFunction(module, address) {} - -A64Function::~A64Function() { - // machine_code_ is freed by code cache. -} - -void A64Function::Setup(uint8_t* machine_code, size_t machine_code_length) { - machine_code_ = machine_code; - machine_code_length_ = machine_code_length; -} - -bool A64Function::CallImpl(ThreadState* thread_state, uint32_t return_address) { -#if XE_PLATFORM_MAC && defined(__aarch64__) - // Initialize JIT execution for this thread - // This ensures pthread_jit_write_protect_np is set correctly for execution - InitializeJITThread(); -#endif - - auto backend = - reinterpret_cast(thread_state->processor()->backend()); - auto thunk = backend->host_to_guest_thunk(); - - // Make the actual thunk call - thunk(machine_code_, thread_state->context(), - reinterpret_cast(uintptr_t(return_address))); - - return true; -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_function.h b/src/xenia/cpu/backend/a64/a64_function.h deleted file mode 100644 index d4c568567..000000000 --- a/src/xenia/cpu/backend/a64/a64_function.h +++ /dev/null @@ -1,44 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ -#define XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ - -#include "xenia/cpu/function.h" -#include "xenia/cpu/thread_state.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -class A64Function : public GuestFunction { - public: - A64Function(Module* module, uint32_t address); - ~A64Function() override; - - uint8_t* machine_code() const override { return machine_code_; } - size_t machine_code_length() const override { return machine_code_length_; } - - void Setup(uint8_t* machine_code, size_t machine_code_length); - - protected: - bool CallImpl(ThreadState* thread_state, uint32_t return_address) override; - - private: - uint8_t* machine_code_ = nullptr; - size_t machine_code_length_ = 0; -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ diff --git a/src/xenia/cpu/backend/a64/a64_op.h b/src/xenia/cpu/backend/a64/a64_op.h deleted file mode 100644 index 863d9865e..000000000 --- a/src/xenia/cpu/backend/a64/a64_op.h +++ /dev/null @@ -1,630 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Xenia Developers. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ -#ifndef XENIA_CPU_BACKEND_A64_A64_OP_H_ -#define XENIA_CPU_BACKEND_A64_A64_OP_H_ - -#include "xenia/cpu/backend/a64/a64_emitter.h" - -#include "xenia/cpu/hir/instr.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -// TODO(benvanik): direct usings. -using namespace xe::cpu; -using namespace xe::cpu::hir; -using namespace oaknut; -using namespace oaknut::util; - -// Selects the right byte/word/etc from a vector. We need to flip logical -// indices (0,1,2,3,4,5,6,7,...) = (3,2,1,0,7,6,5,4,...) -#define VEC128_B(n) ((n) ^ 0x3) -#define VEC128_W(n) ((n) ^ 0x1) -#define VEC128_D(n) (n) -#define VEC128_F(n) (n) - -enum KeyType { - KEY_TYPE_X = OPCODE_SIG_TYPE_X, - KEY_TYPE_L = OPCODE_SIG_TYPE_L, - KEY_TYPE_O = OPCODE_SIG_TYPE_O, - KEY_TYPE_S = OPCODE_SIG_TYPE_S, - KEY_TYPE_V_I8 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT8_TYPE), - KEY_TYPE_V_I16 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT16_TYPE), - KEY_TYPE_V_I32 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT32_TYPE), - KEY_TYPE_V_I64 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(INT64_TYPE), - KEY_TYPE_V_F32 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(FLOAT32_TYPE), - KEY_TYPE_V_F64 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(FLOAT64_TYPE), - KEY_TYPE_V_V128 = - static_cast(OPCODE_SIG_TYPE_V) + static_cast(VEC128_TYPE), -}; - -#pragma pack(push, 1) -union InstrKey { - uint32_t value; - struct { - uint32_t opcode : 8; - uint32_t dest : 5; - uint32_t src1 : 5; - uint32_t src2 : 5; - uint32_t src3 : 5; - uint32_t reserved : 4; - }; - - operator uint32_t() const { return value; } - - InstrKey() : value(0) { static_assert_size(*this, sizeof(value)); } - InstrKey(uint32_t v) : value(v) {} - InstrKey(const Instr* i) : value(0) { - opcode = i->opcode->num; - uint32_t sig = i->opcode->signature; - dest = GET_OPCODE_SIG_TYPE_DEST(sig) ? static_cast(OPCODE_SIG_TYPE_V) + - static_cast(i->dest->type) - : 0; - src1 = GET_OPCODE_SIG_TYPE_SRC1(sig); - if (src1 == OPCODE_SIG_TYPE_V) { - src1 += static_cast(i->src1.value->type); - } - src2 = GET_OPCODE_SIG_TYPE_SRC2(sig); - if (src2 == OPCODE_SIG_TYPE_V) { - src2 += static_cast(i->src2.value->type); - } - src3 = GET_OPCODE_SIG_TYPE_SRC3(sig); - if (src3 == OPCODE_SIG_TYPE_V) { - src3 += static_cast(i->src3.value->type); - } - } - - template - struct Construct { - static const uint32_t value = - (OPCODE) | (DEST << 8) | (SRC1 << 13) | (SRC2 << 18) | (SRC3 << 23); - }; -}; -#pragma pack(pop) -static_assert(sizeof(InstrKey) <= 4, "Key must be 4 bytes"); - -template -struct CombinedStruct; -template <> -struct CombinedStruct<> {}; -template -struct CombinedStruct : T, CombinedStruct {}; - -struct OpBase {}; - -template -struct Op : OpBase { - static const KeyType key_type = KEY_TYPE; -}; - -struct VoidOp : Op { - protected: - friend struct Op; - template - friend struct I; - void Load(const Instr::Op& op) {} -}; - -struct OffsetOp : Op { - uint64_t value; - - protected: - friend struct Op; - template - friend struct I; - void Load(const Instr::Op& op) { this->value = op.offset; } -}; - -struct SymbolOp : Op { - Function* value; - - protected: - friend struct Op; - template - friend struct I; - bool Load(const Instr::Op& op) { - this->value = op.symbol; - return true; - } -}; - -struct LabelOp : Op { - hir::Label* value; - - protected: - friend struct Op; - template - friend struct I; - void Load(const Instr::Op& op) { this->value = op.label; } -}; - -template -struct ValueOp : Op, KEY_TYPE> { - typedef REG_TYPE reg_type; - const Value* value; - bool is_constant; - virtual bool ConstantFitsIn32Reg() const { return true; } - const REG_TYPE& reg() const { - assert_true(!is_constant); - return reg_; - } - operator const REG_TYPE&() const { return reg(); } - bool IsEqual(const T& b) const { - if (is_constant && b.is_constant) { - return reinterpret_cast(this)->constant() == b.constant(); - } else if (!is_constant && !b.is_constant) { - return reg_.index() == b.reg_.index(); - } else { - return false; - } - } - bool IsEqual(const oaknut::Reg& b) const { - if (is_constant) { - return false; - } else if (!is_constant) { - return reg_.index() == b.index(); - } else { - return false; - } - } - bool operator==(const T& b) const { return IsEqual(b); } - bool operator!=(const T& b) const { return !IsEqual(b); } - bool operator==(const oaknut::Reg& b) const { return IsEqual(b); } - bool operator!=(const oaknut::Reg& b) const { return !IsEqual(b); } - void Load(const Instr::Op& op) { - value = op.value; - is_constant = value->IsConstant(); - if (!is_constant) { - A64Emitter::SetupReg(value, reg_); - } - } - - protected: - REG_TYPE reg_ = REG_TYPE(0); -}; - -struct I8Op : ValueOp { - typedef ValueOp BASE; - int8_t constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.i8; - } -}; -struct I16Op : ValueOp { - typedef ValueOp BASE; - int16_t constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.i16; - } -}; -struct I32Op : ValueOp { - typedef ValueOp BASE; - int32_t constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.i32; - } -}; -struct I64Op : ValueOp { - typedef ValueOp BASE; - int64_t constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.i64; - } - bool ConstantFitsIn32Reg() const override { - int64_t v = BASE::value->constant.i64; - if ((v & ~0x7FFFFFFF) == 0) { - // Fits under 31 bits, so just load using normal mov. - return true; - } else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) { - // Negative number that fits in 32bits. - return true; - } - return false; - } -}; -struct F32Op : ValueOp { - typedef ValueOp BASE; - float constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.f32; - } -}; -struct F64Op : ValueOp { - typedef ValueOp BASE; - double constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.f64; - } -}; -struct V128Op : ValueOp { - typedef ValueOp BASE; - const vec128_t& constant() const { - assert_true(BASE::is_constant); - return BASE::value->constant.v128; - } -}; - -template -struct DestField; -template -struct DestField { - DEST dest; - - protected: - bool LoadDest(const Instr* i) { - Instr::Op op; - op.value = i->dest; - dest.Load(op); - return true; - } -}; -template <> -struct DestField { - protected: - bool LoadDest(const Instr* i) { return true; } -}; - -template -struct I; -template -struct I : DestField { - typedef DestField BASE; - static const hir::Opcode opcode = OPCODE; - static const uint32_t key = - InstrKey::Construct::value; - static const KeyType dest_type = DEST::key_type; - const Instr* instr; - - protected: - template - friend struct Sequence; - bool Load(const Instr* i) { - if (InstrKey(i).value == key && BASE::LoadDest(i)) { - instr = i; - return true; - } - return false; - } -}; -template -struct I : DestField { - typedef DestField BASE; - static const hir::Opcode opcode = OPCODE; - static const uint32_t key = - InstrKey::Construct::value; - static const KeyType dest_type = DEST::key_type; - static const KeyType src1_type = SRC1::key_type; - const Instr* instr; - SRC1 src1 = {}; - - protected: - template - friend struct Sequence; - bool Load(const Instr* i) { - if (InstrKey(i).value == key && BASE::LoadDest(i)) { - instr = i; - src1.Load(i->src1); - return true; - } - return false; - } -}; -template -struct I : DestField { - typedef DestField BASE; - static const hir::Opcode opcode = OPCODE; - static const uint32_t key = - InstrKey::Construct::value; - static const KeyType dest_type = DEST::key_type; - static const KeyType src1_type = SRC1::key_type; - static const KeyType src2_type = SRC2::key_type; - const Instr* instr; - SRC1 src1; - SRC2 src2; - - protected: - template - friend struct Sequence; - bool Load(const Instr* i) { - if (InstrKey(i).value == key && BASE::LoadDest(i)) { - instr = i; - src1.Load(i->src1); - src2.Load(i->src2); - return true; - } - return false; - } -}; -template -struct I : DestField { - typedef DestField BASE; - static const hir::Opcode opcode = OPCODE; - static const uint32_t key = - InstrKey::Construct::value; - static const KeyType dest_type = DEST::key_type; - static const KeyType src1_type = SRC1::key_type; - static const KeyType src2_type = SRC2::key_type; - static const KeyType src3_type = SRC3::key_type; - const Instr* instr; - SRC1 src1; - SRC2 src2; - SRC3 src3; - - protected: - template - friend struct Sequence; - bool Load(const Instr* i) { - if (InstrKey(i).value == key && BASE::LoadDest(i)) { - instr = i; - src1.Load(i->src1); - src2.Load(i->src2); - src3.Load(i->src3); - return true; - } - return false; - } -}; - -template -static const T GetTempReg(A64Emitter& e); -template <> -[[maybe_unused]] const WReg GetTempReg(A64Emitter& e) { - return W0; -} -template <> -[[maybe_unused]] const XReg GetTempReg(A64Emitter& e) { - return X0; -} - -template -struct Sequence { - typedef T EmitArgType; - - static constexpr uint32_t head_key() { return T::key; } - - static bool Select(A64Emitter& e, const Instr* i) { - T args; - if (!args.Load(i)) { - return false; - } - SEQ::Emit(e, args); - return true; - } - - template - static void EmitUnaryOp(A64Emitter& e, const EmitArgType& i, - const REG_FN& reg_fn) { - if (i.src1.is_constant) { - e.MOV(i.dest, i.src1.constant()); - reg_fn(e, i.dest); - } else { - if (i.dest != i.src1) { - e.MOV(i.dest, i.src1); - } - reg_fn(e, i.dest); - } - } - - template - static void EmitCommutativeBinaryOp(A64Emitter& e, const EmitArgType& i, - const REG_REG_FN& reg_reg_fn, - const REG_CONST_FN& reg_const_fn) { - if (i.src1.is_constant) { - if (i.src2.is_constant) { - // Both constants. - if (i.src1.ConstantFitsIn32Reg()) { - e.MOV(i.dest, i.src2.constant()); - reg_const_fn(e, i.dest, static_cast(i.src1.constant())); - } else if (i.src2.ConstantFitsIn32Reg()) { - e.MOV(i.dest, i.src1.constant()); - reg_const_fn(e, i.dest, static_cast(i.src2.constant())); - } else { - e.MOV(i.dest, i.src1.constant()); - auto temp = GetTempReg(e); - e.MOV(temp, i.src2.constant()); - reg_reg_fn(e, i.dest, temp); - } - } else { - // src1 constant. - if (i.dest == i.src2) { - if (i.src1.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.dest, static_cast(i.src1.constant())); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src1.constant()); - reg_reg_fn(e, i.dest, temp); - } - } else { - e.MOV(i.dest, i.src1.constant()); - reg_reg_fn(e, i.dest, i.src2); - } - } - } else if (i.src2.is_constant) { - if (i.dest == i.src1) { - if (i.src2.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.dest, static_cast(i.src2.constant())); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2.constant()); - reg_reg_fn(e, i.dest, temp); - } - } else { - e.MOV(i.dest, i.src2.constant()); - reg_reg_fn(e, i.dest, i.src1); - } - } else { - if (i.dest == i.src1) { - reg_reg_fn(e, i.dest, i.src2); - } else if (i.dest == i.src2) { - reg_reg_fn(e, i.dest, i.src1); - } else { - e.MOV(i.dest, i.src1); - reg_reg_fn(e, i.dest, i.src2); - } - } - } - template - static void EmitAssociativeBinaryOp(A64Emitter& e, const EmitArgType& i, - const REG_REG_FN& reg_reg_fn, - const REG_CONST_FN& reg_const_fn) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - if (i.dest == i.src2) { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2); - e.MOV(i.dest, i.src1.constant()); - reg_reg_fn(e, i.dest, temp); - } else { - e.MOV(i.dest, i.src1.constant()); - reg_reg_fn(e, i.dest, i.src2); - } - } else if (i.src2.is_constant) { - if (i.dest == i.src1) { - if (i.src2.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.dest, static_cast(i.src2.constant())); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2.constant()); - reg_reg_fn(e, i.dest, temp); - } - } else { - e.MOV(i.dest, i.src1); - if (i.src2.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.dest, static_cast(i.src2.constant())); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2.constant()); - reg_reg_fn(e, i.dest, temp); - } - } - } else { - if (i.dest == i.src1) { - reg_reg_fn(e, i.dest, i.src2); - } else if (i.dest == i.src2) { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2); - e.MOV(i.dest, i.src1); - reg_reg_fn(e, i.dest, temp); - } else { - e.MOV(i.dest, i.src1); - reg_reg_fn(e, i.dest, i.src2); - } - } - } - - template - static void EmitCommutativeBinaryVOp(A64Emitter& e, const EmitArgType& i, - const FN& fn) { - if (i.src1.is_constant && i.src2.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - e.LoadConstantV(Q1, i.src2.constant()); - fn(e, i.dest, REG(0), REG(1)); - } else if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - fn(e, i.dest, REG(0), i.src2); - } else if (i.src2.is_constant) { - e.LoadConstantV(Q0, i.src2.constant()); - fn(e, i.dest, i.src1, REG(0)); - } else { - fn(e, i.dest, i.src1, i.src2); - } - } - - template - static void EmitAssociativeBinaryVOp(A64Emitter& e, const EmitArgType& i, - const FN& fn) { - if (i.src1.is_constant && i.src2.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - e.LoadConstantV(Q1, i.src2.constant()); - fn(e, i.dest, REG(0), REG(1)); - } else if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - fn(e, i.dest, REG(0), i.src2); - } else if (i.src2.is_constant) { - e.LoadConstantV(Q0, i.src2.constant()); - fn(e, i.dest, i.src1, REG(0)); - } else { - fn(e, i.dest, i.src1, i.src2); - } - } - - template - static void EmitCommutativeCompareOp(A64Emitter& e, const EmitArgType& i, - const REG_REG_FN& reg_reg_fn, - const REG_CONST_FN& reg_const_fn) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - if (i.src1.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.src2, static_cast(i.src1.constant())); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src1.constant()); - reg_reg_fn(e, i.src2, temp); - } - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - if (i.src2.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.src1, static_cast(i.src2.constant())); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2.constant()); - reg_reg_fn(e, i.src1, temp); - } - } else { - reg_reg_fn(e, i.src1, i.src2); - } - } - template - static void EmitAssociativeCompareOp(A64Emitter& e, const EmitArgType& i, - const REG_REG_FN& reg_reg_fn, - const REG_CONST_FN& reg_const_fn) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - if (i.src1.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.dest, i.src2, static_cast(i.src1.constant()), - true); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src1.constant()); - reg_reg_fn(e, i.dest, i.src2, temp, true); - } - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - if (i.src2.ConstantFitsIn32Reg()) { - reg_const_fn(e, i.dest, i.src1, static_cast(i.src2.constant()), - false); - } else { - auto temp = GetTempReg(e); - e.MOV(temp, i.src2.constant()); - reg_reg_fn(e, i.dest, i.src1, temp, false); - } - } else { - reg_reg_fn(e, i.dest, i.src1, i.src2, false); - } - } -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_OP_H_ diff --git a/src/xenia/cpu/backend/a64/a64_seq_control.cc b/src/xenia/cpu/backend/a64/a64_seq_control.cc deleted file mode 100644 index fd2ce41d3..000000000 --- a/src/xenia/cpu/backend/a64/a64_seq_control.cc +++ /dev/null @@ -1,551 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Xenia Developers. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_sequences.h" - -#include -#include - -#include "xenia/cpu/backend/a64/a64_op.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -volatile int anchor_control = 0; - -// ============================================================================ -// OPCODE_DEBUG_BREAK -// ============================================================================ -struct DEBUG_BREAK : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { e.DebugBreak(); } -}; -EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK, DEBUG_BREAK); - -// ============================================================================ -// OPCODE_DEBUG_BREAK_TRUE -// ============================================================================ -struct DEBUG_BREAK_TRUE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.DebugBreak(); - e.l(skip); - } -}; -struct DEBUG_BREAK_TRUE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.DebugBreak(); - e.l(skip); - } -}; -struct DEBUG_BREAK_TRUE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.DebugBreak(); - e.l(skip); - } -}; -struct DEBUG_BREAK_TRUE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.DebugBreak(); - e.l(skip); - } -}; -struct DEBUG_BREAK_TRUE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.DebugBreak(); - e.l(skip); - } -}; -struct DEBUG_BREAK_TRUE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.DebugBreak(); - e.l(skip); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK_TRUE, DEBUG_BREAK_TRUE_I8, - DEBUG_BREAK_TRUE_I16, DEBUG_BREAK_TRUE_I32, - DEBUG_BREAK_TRUE_I64, DEBUG_BREAK_TRUE_F32, - DEBUG_BREAK_TRUE_F64); - -// ============================================================================ -// OPCODE_TRAP -// ============================================================================ -struct TRAP : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.Trap(i.instr->flags); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_TRAP, TRAP); - -// ============================================================================ -// OPCODE_TRAP_TRUE -// ============================================================================ -struct TRAP_TRUE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Trap(i.instr->flags); - e.l(skip); - } -}; -struct TRAP_TRUE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Trap(i.instr->flags); - e.l(skip); - } -}; -struct TRAP_TRUE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Trap(i.instr->flags); - e.l(skip); - } -}; -struct TRAP_TRUE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Trap(i.instr->flags); - e.l(skip); - } -}; -struct TRAP_TRUE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.Trap(i.instr->flags); - e.l(skip); - } -}; -struct TRAP_TRUE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.Trap(i.instr->flags); - e.l(skip); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_TRAP_TRUE, TRAP_TRUE_I8, TRAP_TRUE_I16, - TRAP_TRUE_I32, TRAP_TRUE_I64, TRAP_TRUE_F32, - TRAP_TRUE_F64); - -// ============================================================================ -// OPCODE_CALL -// ============================================================================ -struct CALL : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src1.value->is_guest()); - e.Call(i.instr, static_cast(i.src1.value)); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CALL, CALL); - -// ============================================================================ -// OPCODE_CALL_TRUE -// ============================================================================ -struct CALL_TRUE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->is_guest()); - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Call(i.instr, static_cast(i.src2.value)); - e.l(skip); - } -}; -struct CALL_TRUE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->is_guest()); - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Call(i.instr, static_cast(i.src2.value)); - e.l(skip); - } -}; -struct CALL_TRUE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->is_guest()); - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Call(i.instr, static_cast(i.src2.value)); - e.l(skip); - } -}; -struct CALL_TRUE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->is_guest()); - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.Call(i.instr, static_cast(i.src2.value)); - e.l(skip); - } -}; -struct CALL_TRUE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->is_guest()); - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.Call(i.instr, static_cast(i.src2.value)); - e.l(skip); - } -}; -struct CALL_TRUE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->is_guest()); - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.Call(i.instr, static_cast(i.src2.value)); - e.l(skip); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CALL_TRUE, CALL_TRUE_I8, CALL_TRUE_I16, - CALL_TRUE_I32, CALL_TRUE_I64, CALL_TRUE_F32, - CALL_TRUE_F64); - -// ============================================================================ -// OPCODE_CALL_INDIRECT -// ============================================================================ -struct CALL_INDIRECT - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CallIndirect(i.instr, i.src1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT, CALL_INDIRECT); - -// ============================================================================ -// OPCODE_CALL_INDIRECT_TRUE -// ============================================================================ -struct CALL_INDIRECT_TRUE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.CallIndirect(i.instr, i.src2); - e.l(skip); - } -}; -struct CALL_INDIRECT_TRUE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.CallIndirect(i.instr, i.src2); - e.l(skip); - } -}; -struct CALL_INDIRECT_TRUE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.CallIndirect(i.instr, i.src2); - e.l(skip); - } -}; -struct CALL_INDIRECT_TRUE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.CBZ(i.src1, skip); - e.CallIndirect(i.instr, i.src2); - e.l(skip); - } -}; -struct CALL_INDIRECT_TRUE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.CallIndirect(i.instr, i.src2); - e.l(skip); - } -}; -struct CALL_INDIRECT_TRUE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label skip; - e.FCMP(i.src1, 0); - e.B(Cond::EQ, skip); - e.CallIndirect(i.instr, i.src2); - e.l(skip); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT_TRUE, CALL_INDIRECT_TRUE_I8, - CALL_INDIRECT_TRUE_I16, CALL_INDIRECT_TRUE_I32, - CALL_INDIRECT_TRUE_I64, CALL_INDIRECT_TRUE_F32, - CALL_INDIRECT_TRUE_F64); - -// ============================================================================ -// OPCODE_CALL_EXTERN -// ============================================================================ -struct CALL_EXTERN - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CallExtern(i.instr, i.src1.value); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CALL_EXTERN, CALL_EXTERN); - -// ============================================================================ -// OPCODE_RETURN -// ============================================================================ -struct RETURN : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // If this is the last instruction in the last block, just let us - // fall through. - if (i.instr->next || i.instr->block->next) { - e.B(e.epilog_label()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_RETURN, RETURN); - -// ============================================================================ -// OPCODE_RETURN_TRUE -// ============================================================================ -struct RETURN_TRUE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CBNZ(i.src1, e.epilog_label()); - } -}; -struct RETURN_TRUE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CBNZ(i.src1, e.epilog_label()); - } -}; -struct RETURN_TRUE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CBNZ(i.src1, e.epilog_label()); - } -}; -struct RETURN_TRUE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CBNZ(i.src1, e.epilog_label()); - } -}; -struct RETURN_TRUE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1, 0); - e.B(Cond::NE, e.epilog_label()); - } -}; -struct RETURN_TRUE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1, 0); - e.B(Cond::NE, e.epilog_label()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_RETURN_TRUE, RETURN_TRUE_I8, RETURN_TRUE_I16, - RETURN_TRUE_I32, RETURN_TRUE_I64, RETURN_TRUE_F32, - RETURN_TRUE_F64); - -// ============================================================================ -// OPCODE_SET_RETURN_ADDRESS -// ============================================================================ -struct SET_RETURN_ADDRESS - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SetReturnAddress(i.src1.constant()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SET_RETURN_ADDRESS, SET_RETURN_ADDRESS); - -// ============================================================================ -// OPCODE_BRANCH -// ============================================================================ -struct BRANCH : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src1.value); - assert_not_null(label); - e.B(*label); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH); - -// ============================================================================ -// OPCODE_BRANCH_TRUE -// ============================================================================ -struct BRANCH_TRUE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBNZ(i.src1, *label); - } -}; -struct BRANCH_TRUE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBNZ(i.src1, *label); - } -}; -struct BRANCH_TRUE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBNZ(i.src1, *label); - } -}; -struct BRANCH_TRUE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBNZ(i.src1, *label); - } -}; -struct BRANCH_TRUE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.FCMP(i.src1, 0); - e.B(Cond::NE, *label); - } -}; -struct BRANCH_TRUE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.FCMP(i.src1, 0); - e.B(Cond::NE, *label); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16, - BRANCH_TRUE_I32, BRANCH_TRUE_I64, BRANCH_TRUE_F32, - BRANCH_TRUE_F64); - -// ============================================================================ -// OPCODE_BRANCH_FALSE -// ============================================================================ -struct BRANCH_FALSE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBZ(i.src1, *label); - } -}; -struct BRANCH_FALSE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBZ(i.src1, *label); - } -}; -struct BRANCH_FALSE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBZ(i.src1, *label); - } -}; -struct BRANCH_FALSE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.CBZ(i.src1, *label); - } -}; -struct BRANCH_FALSE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.FCMP(i.src1, 0); - e.B(Cond::EQ, *label); - } -}; -struct BRANCH_FALSE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - oaknut::Label* label = e.lookup_label(i.src2.value); - assert_not_null(label); - e.FCMP(i.src1, 0); - e.B(Cond::EQ, *label); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_BRANCH_FALSE, BRANCH_FALSE_I8, BRANCH_FALSE_I16, - BRANCH_FALSE_I32, BRANCH_FALSE_I64, BRANCH_FALSE_F32, - BRANCH_FALSE_F64); - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_seq_memory.cc b/src/xenia/cpu/backend/a64/a64_seq_memory.cc deleted file mode 100644 index 41f29bd06..000000000 --- a/src/xenia/cpu/backend/a64/a64_seq_memory.cc +++ /dev/null @@ -1,1425 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Xenia Developers. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_sequences.h" - -#include -#include - -#include "xenia/base/memory.h" -#include "xenia/cpu/backend/a64/a64_op.h" -#include "xenia/cpu/backend/a64/a64_tracers.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -volatile int anchor_memory = 0; - -// vec128b stores bytes in reversed 32-bit chunks; use reversed args for 0..15. -static const vec128_t kStvlShuffle = - vec128b(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); -static const vec128_t kStvrSwapMask = vec128b(static_cast(0x83)); - -template -XReg ComputeMemoryAddressOffset(A64Emitter& e, const T& guest, const T& offset, - WReg address_register = W3) { - assert_true(offset.is_constant); - const int32_t offset_const = static_cast(offset.constant()); - - if (guest.is_constant) { - uint32_t address = static_cast(guest.constant()); - address += offset_const; - if (address < 0x80000000) { - e.MOV(address_register.toX(), address); - e.ADD(address_register.toX(), e.GetMembaseReg(), address_register.toX()); - return address_register.toX(); - } else { - if (address >= 0xE0000000 && - xe::memory::allocation_granularity() > 0x1000) { - e.MOV(W0, address + 0x1000); - } else { - e.MOV(W0, address); - } - e.ADD(address_register.toX(), e.GetMembaseReg(), X0); - return address_register.toX(); - } - } else { - if (xe::memory::allocation_granularity() > 0x1000) { - // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do - // it via memory mapping. - e.MOV(W0, 0xE0000000 - offset_const); - e.CMP(guest.reg().toW(), W0); - e.CSET(W0, Cond::HS); - e.ADD(W0, guest.reg().toW(), W0, LSL, 12); - } else { - // Clear the top 32 bits, as they are likely garbage. - // TODO(benvanik): find a way to avoid doing this. - e.MOV(W0, guest.reg().toW()); - } - // Guest addresses are 32-bit and wrap on addition. - e.MOV(W1, offset_const); - e.ADD(W0, W0, W1); - - e.ADD(address_register.toX(), e.GetMembaseReg(), X0); - return address_register.toX(); - } -} - -// Note: most *should* be aligned, but needs to be checked! -template -XReg ComputeMemoryAddress(A64Emitter& e, const T& guest, - WReg address_register = W3) { - if (guest.is_constant) { - // TODO(benvanik): figure out how to do this without a temp. - // Since the constant is often 0x8... if we tried to use that as a - // displacement it would be sign extended and mess things up. - const uint32_t address = static_cast(guest.constant()); - if (address < 0x80000000) { - e.MOV(W0, address); - e.ADD(address_register.toX(), e.GetMembaseReg(), X0); - return address_register.toX(); - } else { - if (address >= 0xE0000000 && - xe::memory::allocation_granularity() > 0x1000) { - e.MOV(W0, address + 0x1000u); - } else { - e.MOV(W0, address); - } - e.ADD(address_register.toX(), e.GetMembaseReg(), X0); - return address_register.toX(); - } - } else { - if (xe::memory::allocation_granularity() > 0x1000) { - // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do - // it via memory mapping. - e.MOV(W0, 0xE0000000); - e.CMP(guest.reg().toW(), W0); - e.CSET(W0, Cond::HS); - e.ADD(W0, guest.reg().toW(), W0, LSL, 12); - } else { - // Clear the top 32 bits, as they are likely garbage. - // TODO(benvanik): find a way to avoid doing this. - e.MOV(W0, guest.reg().toW()); - } - e.ADD(address_register.toX(), e.GetMembaseReg(), X0); - return address_register.toX(); - } -} - -// ============================================================================ -// OPCODE_ATOMIC_EXCHANGE -// ============================================================================ -// Note that the address we use here is a real, host address! -// This is weird, and should be fixed. -template -void EmitAtomicExchangeXX(A64Emitter& e, const ARGS& i, const FN& fn) { - if (i.dest == i.src1) { - e.MOV(X0, i.src1); - if (i.dest != i.src2) { - if (i.src2.is_constant) { - e.MOV(i.dest, i.src2.constant()); - } else { - e.MOV(i.dest, i.src2); - } - } - fn(e, i.dest, X0); - } else { - if (i.dest != i.src2) { - if (i.src2.is_constant) { - e.MOV(i.dest, i.src2.constant()); - } else { - e.MOV(i.dest, i.src2); - } - } - fn(e, i.dest, i.src1); - } -} -struct ATOMIC_EXCHANGE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX( - e, i, - [](A64Emitter& e, WReg dest, XReg src) { e.SWPALB(dest, dest, src); }); - } -}; -struct ATOMIC_EXCHANGE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX( - e, i, - [](A64Emitter& e, WReg dest, XReg src) { e.SWPALH(dest, dest, src); }); - } -}; -struct ATOMIC_EXCHANGE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX( - e, i, - [](A64Emitter& e, WReg dest, XReg src) { e.SWPAL(dest, dest, src); }); - } -}; -struct ATOMIC_EXCHANGE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAtomicExchangeXX( - e, i, - [](A64Emitter& e, XReg dest, XReg src) { e.SWPAL(dest, dest, src); }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_EXCHANGE, ATOMIC_EXCHANGE_I8, - ATOMIC_EXCHANGE_I16, ATOMIC_EXCHANGE_I32, - ATOMIC_EXCHANGE_I64); - -// ============================================================================ -// OPCODE_LVL/LVR/STVL/STVR -// ============================================================================ -struct LVL_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W4); - e.AND(W0, address.toW(), 0xF); - e.SUB(X1, address, X0); - - e.LDR(Q2, X1); - - e.MOV(X2, e.GetVConstPtr()); - e.LDR(Q0, X2, e.GetVConstOffset(VByteSwapMask)); - e.DUP(Q1.B16(), W0); - e.ADD(Q0.B16(), Q0.B16(), Q1.B16()); - e.TBL(i.dest.reg().B16(), List{Q2.B16()}, Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LVL, LVL_V128); - -struct LVR_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W4); - e.AND(W0, address.toW(), 0xF); - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); - - oaknut::Label done; - e.CBZ(W0, done); - - e.SUB(X1, address, X0); - e.LDR(Q2, X1); - - e.MOV(X2, e.GetVConstPtr()); - e.LDR(Q0, X2, e.GetVConstOffset(VByteSwapMask)); - e.DUP(Q1.B16(), W0); - e.ADD(Q0.B16(), Q0.B16(), Q1.B16()); - - e.MOVI(Q1.B16(), 0x10); - e.CMHS(Q3.B16(), Q0.B16(), Q1.B16()); - e.SUB(Q0.B16(), Q0.B16(), Q1.B16()); - e.MOVI(Q1.B16(), 0x80); - e.BSL(Q3.B16(), Q0.B16(), Q1.B16()); - - e.TBL(i.dest.reg().B16(), List{Q2.B16()}, Q3.B16()); - e.l(done); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LVR, LVR_V128); - -struct STVL_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W4); - e.AND(W0, address.toW(), 0xF); - e.SUB(X1, address, X0); - - e.LDR(Q2, X1); - - e.MOV(X2, reinterpret_cast(&kStvlShuffle)); - e.LDR(Q0, X2); - e.DUP(Q1.B16(), W0); - e.SUB(Q0.B16(), Q0.B16(), Q1.B16()); - - e.MOV(X2, e.GetVConstPtr()); - e.LDR(Q1, X2, e.GetVConstOffset(VSwapWordMask)); - e.EOR(Q0.B16(), Q0.B16(), Q1.B16()); - - const QReg shuffled = Q3; - if (i.src2.is_constant) { - e.LoadConstantV(shuffled, i.src2.constant()); - } else { - e.MOV(shuffled.B16(), i.src2.reg().B16()); - } - e.TBL(shuffled.B16(), List{shuffled.B16()}, Q0.B16()); - - e.MOVI(Q1.B16(), 0x80); - e.CMHS(Q1.B16(), Q0.B16(), Q1.B16()); - e.BSL(Q1.B16(), Q2.B16(), shuffled.B16()); - e.STR(Q1, X1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128); - -struct STVR_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W4); - e.AND(W0, address.toW(), 0xF); - - oaknut::Label done; - e.CBZ(W0, done); - - e.SUB(X1, address, X0); - e.LDR(Q2, X1); - - e.MOV(X2, reinterpret_cast(&kStvlShuffle)); - e.LDR(Q0, X2); - e.DUP(Q1.B16(), W0); - e.SUB(Q0.B16(), Q0.B16(), Q1.B16()); - - e.MOV(X2, reinterpret_cast(&kStvrSwapMask)); - e.LDR(Q1, X2); - e.EOR(Q0.B16(), Q0.B16(), Q1.B16()); - - e.MOVI(Q1.B16(), 0x0F); - e.AND(Q1.B16(), Q0.B16(), Q1.B16()); - e.MOVI(Q3.B16(), 0x80); - e.AND(Q3.B16(), Q0.B16(), Q3.B16()); - e.ORR(Q1.B16(), Q1.B16(), Q3.B16()); - - const QReg shuffled = Q3; - if (i.src2.is_constant) { - e.LoadConstantV(shuffled, i.src2.constant()); - } else { - e.MOV(shuffled.B16(), i.src2.reg().B16()); - } - e.TBL(shuffled.B16(), List{shuffled.B16()}, Q1.B16()); - - e.MOVI(Q1.B16(), 0x80); - e.CMHS(Q1.B16(), Q0.B16(), Q1.B16()); - e.BSL(Q1.B16(), Q2.B16(), shuffled.B16()); - e.STR(Q1, X1); - - e.l(done); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STVR, STVR_V128); - -// ============================================================================ -// OPCODE_ATOMIC_COMPARE_EXCHANGE -// ============================================================================ -struct ATOMIC_COMPARE_EXCHANGE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (xe::memory::allocation_granularity() > 0x1000) { - // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do - // it via memory mapping. - e.MOV(W3, 0xE0000000); - e.CMP(i.src1.reg().toW(), W3); - e.CSET(W1, Cond::HS); - e.ADD(W1, i.src1.reg().toW(), W1, LSL, 12); - } else { - e.MOV(W1, i.src1.reg().toW()); - } - e.ADD(X1, e.GetMembaseReg(), X1); - - const XReg address = X1; - const WReg expected = i.src2; - const WReg desired = i.src3; - const WReg status = W0; - - if (e.IsFeatureEnabled(kA64EmitLSE)) { - e.MOV(status, expected); - - // if([C] == A) [C] = B - // else A = [C] - e.CASAL(status, desired, address); - e.CMP(status, expected); - e.CSET(i.dest, Cond::EQ); - return; - } - - oaknut::Label success, fail, retry; - - e.l(retry); - e.LDAXR(W4, address); - e.CMP(W4, expected); - e.B(Cond::NE, fail); - - e.STLXR(status.toW(), desired, address); - e.CBNZ(status, retry); - e.B(success); - - e.l(fail); - e.CLREX(); - - e.l(success); - e.CSET(i.dest, Cond::EQ); - } -}; -struct ATOMIC_COMPARE_EXCHANGE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (xe::memory::allocation_granularity() > 0x1000) { - // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do - // it via memory mapping. - e.MOV(W3, 0xE0000000); - e.CMP(i.src1.reg(), X3); - e.CSET(W1, Cond::HS); - e.ADD(W1, i.src1.reg().toW(), W1, LSL, 12); - } else { - e.MOV(W1, i.src1.reg().toW()); - } - e.ADD(X1, e.GetMembaseReg(), X1); - - const XReg address = X1; - const XReg expected = i.src2; - const XReg desired = i.src3; - const XReg status = X0; - - if (e.IsFeatureEnabled(kA64EmitLSE)) { - e.MOV(status, expected); - - // if([C] == A) [C] = B - // else A = [C] - e.CASAL(status, desired, address); - e.CMP(status, expected); - e.CSET(i.dest, Cond::EQ); - return; - } - - oaknut::Label success, fail, retry; - - e.l(retry); - e.LDAXR(X4, address); - e.CMP(X4, expected); - e.B(Cond::NE, fail); - - e.STLXR(status.toW(), desired, address); - e.CBNZ(status, retry); - e.B(success); - - e.l(fail); - e.CLREX(); - - e.l(success); - e.CSET(i.dest, Cond::EQ); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, - ATOMIC_COMPARE_EXCHANGE_I32, ATOMIC_COMPARE_EXCHANGE_I64); - -// ============================================================================ -// OPCODE_RESERVED_LOAD / OPCODE_RESERVED_STORE -// ============================================================================ -struct RESERVED_LOAD_INT32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W3); - e.LDAXR(i.dest, address); - } -}; -struct RESERVED_LOAD_INT64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W3); - e.LDAXR(i.dest, address); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_RESERVED_LOAD, RESERVED_LOAD_INT32, - RESERVED_LOAD_INT64); - -struct RESERVED_STORE_INT32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W3); - const WReg value = i.src2.is_constant ? W4 : i.src2; - if (i.src2.is_constant) { - e.MOV(value, static_cast(i.src2.constant())); - } - e.STLXR(W0, value, address); - e.CMP(W0, 0); - e.CSET(i.dest, Cond::EQ); - } -}; - -struct RESERVED_STORE_INT64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const XReg address = ComputeMemoryAddress(e, i.src1, W3); - const XReg value = i.src2.is_constant ? X4 : i.src2; - if (i.src2.is_constant) { - e.MOV(value, i.src2.constant()); - } - e.STLXR(W0, value, address); - e.CMP(W0, 0); - e.CSET(i.dest, Cond::EQ); - } -}; - -EMITTER_OPCODE_TABLE(OPCODE_RESERVED_STORE, RESERVED_STORE_INT32, - RESERVED_STORE_INT64); - -// ============================================================================ -// OPCODE_LOAD_LOCAL -// ============================================================================ -// Note: all types are always aligned on the stack. -template -void EmitLocalAccess(A64Emitter& e, uint32_t offset, uint32_t scale, - const EmitFn& emit_fn) { - const bool imm_valid = (offset % scale) == 0 && (offset / scale) <= 0xFFF; - if (imm_valid) { - emit_fn(SP, offset); - return; - } - - auto addr = GetTempReg(e); - e.MOV(addr, offset); - e.ADD(addr, SP, addr); - emit_fn(addr, 0); -} - -struct LOAD_LOCAL_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 1, [&](auto base, uint32_t imm) { - e.LDRB(i.dest, base, imm); - }); - // e.TraceLoadI8(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -struct LOAD_LOCAL_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 2, [&](auto base, uint32_t imm) { - e.LDRH(i.dest, base, imm); - }); - // e.TraceLoadI16(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -struct LOAD_LOCAL_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 4, - [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); - // e.TraceLoadI32(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -struct LOAD_LOCAL_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 8, - [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); - // e.TraceLoadI64(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -struct LOAD_LOCAL_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 4, - [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); - // e.TraceLoadF32(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -struct LOAD_LOCAL_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 8, - [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); - // e.TraceLoadF64(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -struct LOAD_LOCAL_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitLocalAccess(e, i.src1.constant(), 16, - [&](auto base, uint32_t imm) { e.LDR(i.dest, base, imm); }); - // e.TraceLoadV128(DATA_LOCAL, i.src1.constant, i.dest); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_LOCAL, LOAD_LOCAL_I8, LOAD_LOCAL_I16, - LOAD_LOCAL_I32, LOAD_LOCAL_I64, LOAD_LOCAL_F32, - LOAD_LOCAL_F64, LOAD_LOCAL_V128); - -// ============================================================================ -// OPCODE_STORE_LOCAL -// ============================================================================ -// Note: all types are always aligned on the stack. -struct STORE_LOCAL_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreI8(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 1, [&](auto base, uint32_t imm) { - e.STRB(i.src2, base, imm); - }); - } -}; -struct STORE_LOCAL_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreI16(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 2, [&](auto base, uint32_t imm) { - e.STRH(i.src2, base, imm); - }); - } -}; -struct STORE_LOCAL_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreI32(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 4, - [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); - } -}; -struct STORE_LOCAL_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreI64(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 8, - [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); - } -}; -struct STORE_LOCAL_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreF32(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 4, - [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); - } -}; -struct STORE_LOCAL_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreF64(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 8, - [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); - } -}; -struct STORE_LOCAL_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // e.TraceStoreV128(DATA_LOCAL, i.src1.constant, i.src2); - EmitLocalAccess(e, i.src1.constant(), 16, - [&](auto base, uint32_t imm) { e.STR(i.src2, base, imm); }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STORE_LOCAL, STORE_LOCAL_I8, STORE_LOCAL_I16, - STORE_LOCAL_I32, STORE_LOCAL_I64, STORE_LOCAL_F32, - STORE_LOCAL_F64, STORE_LOCAL_V128); - -// ============================================================================ -// OPCODE_LOAD_CONTEXT -// ============================================================================ -struct LOAD_CONTEXT_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDRB(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.MOV(e.GetNativeParam(0), i.src1.value); - e.LDRB(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadI8)); - } - } -}; -struct LOAD_CONTEXT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDRH(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.LDRH(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadI16)); - } - } -}; -struct LOAD_CONTEXT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.LDR(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadI32)); - } - } -}; -struct LOAD_CONTEXT_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.LDR(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadI64)); - } - } -}; -struct LOAD_CONTEXT_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadF32)); - } - } -}; -struct LOAD_CONTEXT_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadF64)); - } - } -}; -struct LOAD_CONTEXT_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.LDR(i.dest, e.GetContextReg(), i.src1.value); - if (IsTracingData()) { - e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextLoadV128)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_CONTEXT, LOAD_CONTEXT_I8, LOAD_CONTEXT_I16, - LOAD_CONTEXT_I32, LOAD_CONTEXT_I64, LOAD_CONTEXT_F32, - LOAD_CONTEXT_F64, LOAD_CONTEXT_V128); - -// ============================================================================ -// OPCODE_STORE_CONTEXT -// ============================================================================ -// Note: all types are always aligned on the stack. -struct STORE_CONTEXT_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.constant()); - e.STRB(W0, e.GetContextReg(), i.src1.value); - } else { - e.STRB(i.src2.reg(), e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.LDRB(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreI8)); - } - } -}; -struct STORE_CONTEXT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.constant()); - e.STRH(W0, e.GetContextReg(), i.src1.value); - } else { - e.STRH(i.src2.reg(), e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.LDRH(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreI16)); - } - } -}; -struct STORE_CONTEXT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.constant()); - e.STR(W0, e.GetContextReg(), i.src1.value); - } else { - e.STR(i.src2.reg(), e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.LDR(e.GetNativeParam(1).toW(), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreI32)); - } - } -}; -struct STORE_CONTEXT_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.MOV(X0, i.src2.constant()); - e.STR(X0, e.GetContextReg(), i.src1.value); - } else { - e.STR(i.src2.reg(), e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.LDR(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreI64)); - } - } -}; -struct STORE_CONTEXT_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.value->constant.i32); - e.STR(W0, e.GetContextReg(), i.src1.value); - } else { - e.STR(i.src2, e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreF32)); - } - } -}; -struct STORE_CONTEXT_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.MOV(X0, i.src2.value->constant.i64); - e.STR(X0, e.GetContextReg(), i.src1.value); - } else { - e.STR(i.src2, e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreF64)); - } - } -}; -struct STORE_CONTEXT_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.LoadConstantV(Q0, i.src2.constant()); - e.STR(Q0, e.GetContextReg(), i.src1.value); - } else { - e.STR(i.src2, e.GetContextReg(), i.src1.value); - } - if (IsTracingData()) { - e.ADD(e.GetNativeParam(1), e.GetContextReg(), i.src1.value); - e.MOV(e.GetNativeParam(0), i.src1.value); - e.CallNative(reinterpret_cast(TraceContextStoreV128)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STORE_CONTEXT, STORE_CONTEXT_I8, STORE_CONTEXT_I16, - STORE_CONTEXT_I32, STORE_CONTEXT_I64, STORE_CONTEXT_F32, - STORE_CONTEXT_F64, STORE_CONTEXT_V128); - -// ============================================================================ -// OPCODE_LOAD_MMIO -// ============================================================================ -// Note: all types are always aligned in the context. -struct LOAD_MMIO_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // uint64_t (context, addr) - const auto mmio_range = reinterpret_cast(i.src1.value); - const auto read_address = uint32_t(i.src2.value); - e.MOV(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); - e.MOV(e.GetNativeParam(1).toW(), read_address); - e.CallNativeSafe(reinterpret_cast(mmio_range->read)); - e.REV(i.dest, W0); - if (IsTracingData()) { - e.MOV(e.GetNativeParam(0).toW(), i.dest); - e.MOV(X1, read_address); - e.CallNative(reinterpret_cast(TraceContextLoadI32)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_MMIO, LOAD_MMIO_I32); - -// ============================================================================ -// OPCODE_STORE_MMIO -// ============================================================================ -// Note: all types are always aligned on the stack. -struct STORE_MMIO_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // void (context, addr, value) - const auto mmio_range = reinterpret_cast(i.src1.value); - const auto write_address = uint32_t(i.src2.value); - e.MOV(e.GetNativeParam(0), uint64_t(mmio_range->callback_context)); - e.MOV(e.GetNativeParam(1).toW(), write_address); - if (i.src3.is_constant) { - e.MOV(e.GetNativeParam(2).toW(), xe::byte_swap(i.src3.constant())); - } else { - e.REV(e.GetNativeParam(2).toW(), i.src3); - } - e.CallNativeSafe(reinterpret_cast(mmio_range->write)); - if (IsTracingData()) { - if (i.src3.is_constant) { - e.MOV(e.GetNativeParam(0).toW(), i.src3.constant()); - } else { - e.MOV(e.GetNativeParam(0).toW(), i.src3); - } - e.MOV(X1, write_address); - e.CallNative(reinterpret_cast(TraceContextStoreI32)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STORE_MMIO, STORE_MMIO_I32); - -// ============================================================================ -// OPCODE_LOAD_OFFSET -// ============================================================================ -struct LOAD_OFFSET_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - e.LDRB(i.dest, addr_reg); - } -}; - -struct LOAD_OFFSET_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - e.LDRH(i.dest, addr_reg); - e.REV16(i.dest, i.dest); - } else { - e.LDRH(i.dest, addr_reg); - } - } -}; - -struct LOAD_OFFSET_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - e.LDR(i.dest, addr_reg); - e.REV(i.dest, i.dest); - } else { - e.LDR(i.dest, addr_reg); - } - } -}; - -struct LOAD_OFFSET_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - e.LDR(i.dest, addr_reg); - e.REV(i.dest, i.dest); - } else { - e.LDR(i.dest, addr_reg); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_OFFSET, LOAD_OFFSET_I8, LOAD_OFFSET_I16, - LOAD_OFFSET_I32, LOAD_OFFSET_I64); - -// ============================================================================ -// OPCODE_STORE_OFFSET -// ============================================================================ -struct STORE_OFFSET_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.src3.is_constant) { - e.MOV(W0, i.src3.constant()); - e.STRB(W0, addr_reg); - } else { - e.STRB(i.src3, addr_reg); - } - } -}; - -struct STORE_OFFSET_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src3.is_constant); - assert_always("not implemented"); - } else { - if (i.src3.is_constant) { - e.MOV(W0, i.src3.constant()); - e.STRH(W0, addr_reg); - } else { - e.STRH(i.src3, addr_reg); - } - } - } -}; - -struct STORE_OFFSET_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src3.is_constant); - assert_always("not implemented"); - } else { - if (i.src3.is_constant) { - e.MOV(W0, i.src3.constant()); - e.STR(W0, addr_reg); - } else { - e.STR(i.src3, addr_reg); - } - } - } -}; - -struct STORE_OFFSET_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddressOffset(e, i.src1, i.src2); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src3.is_constant); - assert_always("not implemented"); - } else { - if (i.src3.is_constant) { - e.MovMem64(addr_reg, 0, i.src3.constant()); - } else { - e.STR(i.src3, addr_reg); - } - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STORE_OFFSET, STORE_OFFSET_I8, STORE_OFFSET_I16, - STORE_OFFSET_I32, STORE_OFFSET_I64); - -// ============================================================================ -// OPCODE_LOAD -// ============================================================================ -struct LOAD_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDRB(i.dest, addr_reg); - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1).toW(), i.dest); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadI8)); - } - } -}; -struct LOAD_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - e.LDRH(i.dest, addr_reg); - e.REV16(i.dest, i.dest); - } else { - e.LDRH(i.dest, addr_reg); - } - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1).toW(), i.dest); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadI16)); - } - } -}; -struct LOAD_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - e.LDR(i.dest, addr_reg); - e.REV(i.dest, i.dest); - } else { - e.LDR(i.dest, addr_reg); - } - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1).toW(), i.dest); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadI32)); - } - } -}; -struct LOAD_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - e.LDR(i.dest, addr_reg); - e.REV64(i.dest, i.dest); - } else { - e.LDR(i.dest, addr_reg); - } - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1), i.dest); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadI64)); - } - } -}; -struct LOAD_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDR(i.dest, addr_reg); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_always("not implemented yet"); - } - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadF32)); - } - } -}; -struct LOAD_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDR(i.dest, addr_reg); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_always("not implemented yet"); - } - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadF64)); - } - } -}; -struct LOAD_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDR(i.dest, addr_reg); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - // Reverse upper and lower 64-bit halfs - e.REV64(i.dest.reg().B16(), i.dest.reg().B16()); - // Reverse the 64-bit halfs themselves - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); - } - if (IsTracingData()) { - e.MOV(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryLoadV128)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD, LOAD_I8, LOAD_I16, LOAD_I32, LOAD_I64, - LOAD_F32, LOAD_F64, LOAD_V128); - -// ============================================================================ -// OPCODE_STORE -// ============================================================================ -// Note: most *should* be aligned, but needs to be checked! -struct STORE_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.src2.is_constant) { - e.MOV(W0, i.src2.constant()); - e.STRB(W0, addr_reg); - } else { - e.STRB(i.src2.reg(), addr_reg); - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDRB(e.GetNativeParam(1).toW(), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreI8)); - } - } -}; -struct STORE_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src2.is_constant); - assert_always("not implemented"); - } else { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.constant()); - e.STRH(W0, addr_reg); - } else { - e.STRH(i.src2.reg(), addr_reg); - } - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDRH(e.GetNativeParam(1).toW(), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreI16)); - } - } -}; -struct STORE_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src2.is_constant); - assert_always("not implemented"); - } else { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.constant()); - e.STR(W0, addr_reg); - } else { - e.STR(i.src2.reg(), addr_reg); - } - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDR(e.GetNativeParam(1).toW(), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreI32)); - } - } -}; -struct STORE_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src2.is_constant); - assert_always("not implemented"); - } else { - if (i.src2.is_constant) { - e.MovMem64(addr_reg, 0, i.src2.constant()); - } else { - e.STR(i.src2.reg(), addr_reg); - } - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.LDR(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreI64)); - } - } -}; -struct STORE_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src2.is_constant); - assert_always("not yet implemented"); - } else { - if (i.src2.is_constant) { - e.MOV(W0, i.src2.value->constant.i32); - e.STR(W0, addr_reg); - } else { - e.STR(i.src2, addr_reg); - } - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.MOV(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreF32)); - } - } -}; -struct STORE_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src2.is_constant); - assert_always("not yet implemented"); - } else { - if (i.src2.is_constant) { - e.MOV(X0, i.src2.value->constant.i64); - e.STR(X0, addr_reg); - } else { - e.STR(i.src2, addr_reg); - } - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.MOV(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreF64)); - } - } -}; -struct STORE_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto addr_reg = ComputeMemoryAddress(e, i.src1); - if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { - assert_false(i.src2.is_constant); - // Reverse upper and lower 64-bit halfs - e.REV64(Q0.B16(), i.src2.reg().B16()); - // Reverse the 64-bit halfs themselves - e.EXT(Q0.B16(), Q0.B16(), Q0.B16(), 8); - e.STR(Q0, addr_reg); - } else { - if (i.src2.is_constant) { - e.LoadConstantV(Q0, i.src2.constant()); - e.STR(Q0, addr_reg); - } else { - e.STR(i.src2, addr_reg); - } - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.MOV(e.GetNativeParam(1), addr_reg); - e.MOV(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemoryStoreV128)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_STORE, STORE_I8, STORE_I16, STORE_I32, STORE_I64, - STORE_F32, STORE_F64, STORE_V128); - -// ============================================================================ -// OPCODE_CACHE_CONTROL -// ============================================================================ -struct CACHE_CONTROL - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - bool is_clflush = false, is_prefetch = false; - switch (CacheControlType(i.instr->flags)) { - case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH: - case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH_FOR_STORE: - is_prefetch = true; - break; - case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE: - case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE_AND_FLUSH: - is_clflush = true; - break; - default: - assert_unhandled_case(CacheControlType(i.instr->flags)); - return; - } - size_t cache_line_size = i.src2.value; - - XReg addr = X0; - uint32_t address_constant; - if (i.src1.is_constant) { - // TODO(benvanik): figure out how to do this without a temp. - // Since the constant is often 0x8... if we tried to use that as a - // displacement it would be sign extended and mess things up. - address_constant = static_cast(i.src1.constant()); - if (address_constant < 0x80000000) { - e.ADD(addr, e.GetMembaseReg(), address_constant); - } else { - if (address_constant >= 0xE0000000 && - xe::memory::allocation_granularity() > 0x1000) { - e.MOV(X1, address_constant + 0x1000); - } else { - e.MOV(X1, address_constant); - } - e.ADD(addr, e.GetMembaseReg(), X1); - } - } else { - if (xe::memory::allocation_granularity() > 0x1000) { - // Emulate the 4 KB physical address offset in 0xE0000000+ when can't do - // it via memory mapping. - e.MOV(X1, 0xE0000000); - e.CMP(i.src1.reg(), X1); - e.CSET(X1, Cond::HS); - e.ADD(X1, i.src1.reg(), X1, LSL, 12); - } else { - // Clear the top 32 bits, as they are likely garbage. - e.MOV(W1, i.src1.reg().toW()); - } - e.ADD(addr, e.GetMembaseReg(), X1); - } - - if (is_clflush) { - // TODO(wunkolo): These kind of cache-maintenance instructions cause an - // illegal-instruction on windows, but is trapped to proper EL1 code on - // Linux. Need a way to do cache-maintenance on Windows-Arm - // e.DC(DcOp::CIVAC, addr); - - // Full data sync - e.DSB(BarrierOp::ISH); - } - if (is_prefetch) { - e.PRFM(PrfOp::PLDL1KEEP, addr); - } - - if (cache_line_size >= 128) { - // Prefetch the other 64 bytes of the 128-byte cache line. - if (i.src1.is_constant && address_constant < 0x80000000) { - e.ADD(addr, e.GetMembaseReg(), address_constant ^ 64); - } else { - e.EOR(X1, X1, 64); - } - if (is_clflush) { - // TODO(wunkolo): These kind of cache-maintenance instructions cause an - // illegal-instruction on windows, but is trapped to proper EL1 code on - // Linux. Need a way to do cache-maintenance on Windows-Arm - // e.DC(DcOp::CIVAC, addr); - - // Full data sync - e.DSB(BarrierOp::ISH); - } - if (is_prefetch) { - e.PRFM(PrfOp::PLDL1KEEP, addr); - } - assert_true(cache_line_size == 128); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); - -// ============================================================================ -// OPCODE_MEMORY_BARRIER -// ============================================================================ -struct MEMORY_BARRIER - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.DMB(BarrierOp::SY); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MEMORY_BARRIER, MEMORY_BARRIER); - -// ============================================================================ -// OPCODE_MEMSET -// ============================================================================ -struct MEMSET_I64_I8_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.is_constant); - assert_true(i.src3.is_constant); - assert_true(i.src2.constant() == 0); - e.MOVI(Q0.B16(), 0); - auto addr_reg = ComputeMemoryAddress(e, i.src1); - switch (i.src3.constant()) { - case 32: - e.STP(Q0, Q0, addr_reg, 0 * 16); - break; - case 128: - e.STP(Q0, Q0, addr_reg, 0 * 16); - e.STP(Q0, Q0, addr_reg, 2 * 16); - e.STP(Q0, Q0, addr_reg, 4 * 16); - e.STP(Q0, Q0, addr_reg, 6 * 16); - break; - default: - assert_unhandled_case(i.src3.constant()); - break; - } - if (IsTracingData()) { - addr_reg = ComputeMemoryAddress(e, i.src1); - e.MOV(e.GetNativeParam(2), i.src3.constant()); - e.MOV(e.GetNativeParam(1), i.src2.constant()); - e.LDR(e.GetNativeParam(0), addr_reg); - e.CallNative(reinterpret_cast(TraceMemset)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MEMSET, MEMSET_I64_I8_I64); - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc deleted file mode 100644 index 685844cab..000000000 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ /dev/null @@ -1,2501 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2025 Xenia Developers. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ -#include "xenia/cpu/backend/a64/a64_sequences.h" - -#include -#include -#include - -#include "xenia/cpu/backend/a64/a64_op.h" - -// For OPCODE_PACK/OPCODE_UNPACK -#include "third_party/half/include/half.hpp" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -volatile int anchor_vector = 0; - -// ============================================================================ -// OPCODE_VECTOR_CONVERT_I2F -// ============================================================================ -struct VECTOR_CONVERT_I2F - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const QReg src = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - e.UCVTF(i.dest.reg().S4(), src.S4()); - } else { - e.SCVTF(i.dest.reg().S4(), src.S4()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_I2F, VECTOR_CONVERT_I2F); - -// ============================================================================ -// OPCODE_VECTOR_CONVERT_F2I -// ============================================================================ -struct VECTOR_CONVERT_F2I - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const QReg src = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - e.FCVTZU(i.dest.reg().S4(), src.S4()); - } else { - e.FCVTZS(i.dest.reg().S4(), src.S4()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_F2I, VECTOR_CONVERT_F2I); - -// ============================================================================ -// OPCODE_VECTOR_DENORMFLUSH -// ============================================================================ -struct VECTOR_DENORMFLUSH - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // Clear denormals to signed zero, preserving sign bits. - e.MOV(X2, e.GetVConstPtr()); - e.LDR(Q0, X2, e.GetVConstOffset(VSingleDenormalMask)); - e.AND(Q0.B16(), i.src1.reg().B16(), Q0.B16()); - e.CMEQ(Q0.S4(), Q0.S4(), 0); - e.BIC(Q1.B16(), i.src1.reg().B16(), Q0.B16()); - e.LDR(Q2, X2, e.GetVConstOffset(VSignMaskF32)); - e.AND(Q2.B16(), i.src1.reg().B16(), Q2.B16()); - e.ORR(i.dest.reg().B16(), Q1.B16(), Q2.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_DENORMFLUSH, VECTOR_DENORMFLUSH); - -// ============================================================================ -// OPCODE_LOAD_VECTOR_SHL -// ============================================================================ -static const vec128_t lvsl_table[16] = { - vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), - vec128b(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), - vec128b(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17), - vec128b(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18), - vec128b(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19), - vec128b(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20), - vec128b(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21), - vec128b(7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22), - vec128b(8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23), - vec128b(9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24), - vec128b(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25), - vec128b(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), - vec128b(12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27), - vec128b(13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28), - vec128b(14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29), - vec128b(15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30), -}; -struct LOAD_VECTOR_SHL_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - auto sh = i.src1.constant(); - assert_true(static_cast(sh) < xe::countof(lvsl_table)); - e.MOV(X0, reinterpret_cast(&lvsl_table[sh])); - e.LDR(i.dest, X0); - } else { - e.MOV(X0, reinterpret_cast(lvsl_table)); - e.AND(X1, i.src1.reg().toX(), 0xf); - e.LDR(i.dest, X0, X1, IndexExt::LSL, 4); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHL, LOAD_VECTOR_SHL_I8); - -// ============================================================================ -// OPCODE_LOAD_VECTOR_SHR -// ============================================================================ -static const vec128_t lvsr_table[16] = { - vec128b(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31), - vec128b(15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30), - vec128b(14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29), - vec128b(13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28), - vec128b(12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27), - vec128b(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), - vec128b(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25), - vec128b(9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24), - vec128b(8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23), - vec128b(7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22), - vec128b(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21), - vec128b(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20), - vec128b(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19), - vec128b(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18), - vec128b(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17), - vec128b(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), -}; -struct LOAD_VECTOR_SHR_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - auto sh = i.src1.constant(); - assert_true(static_cast(sh) < xe::countof(lvsr_table)); - e.MOV(X0, reinterpret_cast(&lvsr_table[sh])); - e.LDR(i.dest, X0); - } else { - e.MOV(X0, reinterpret_cast(lvsr_table)); - e.AND(X1, i.src1.reg().toX(), 0xf); - e.LDR(i.dest, X0, X1, IndexExt::LSL, 4); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHR, LOAD_VECTOR_SHR_I8); - -// ============================================================================ -// OPCODE_VECTOR_MAX -// ============================================================================ -struct VECTOR_MAX - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - uint32_t part_type = i.instr->flags >> 8; - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - switch (part_type) { - case INT8_TYPE: - e.UMAX(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.UMAX(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.UMAX(dest.S4(), src1.S4(), src2.S4()); - break; - default: - assert_unhandled_case(part_type); - break; - } - } else { - switch (part_type) { - case INT8_TYPE: - e.SMAX(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.SMAX(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.SMAX(dest.S4(), src1.S4(), src2.S4()); - break; - default: - assert_unhandled_case(part_type); - break; - } - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MAX, VECTOR_MAX); - -// ============================================================================ -// OPCODE_VECTOR_MIN -// ============================================================================ -struct VECTOR_MIN - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - uint32_t part_type = i.instr->flags >> 8; - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - switch (part_type) { - case INT8_TYPE: - e.UMIN(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.UMIN(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.UMIN(dest.S4(), src1.S4(), src2.S4()); - break; - default: - assert_unhandled_case(part_type); - break; - } - } else { - switch (part_type) { - case INT8_TYPE: - e.SMIN(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.SMIN(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.SMIN(dest.S4(), src1.S4(), src2.S4()); - break; - default: - assert_unhandled_case(part_type); - break; - } - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MIN, VECTOR_MIN); - -// ============================================================================ -// OPCODE_VECTOR_COMPARE_EQ -// ============================================================================ -struct VECTOR_COMPARE_EQ_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAssociativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - switch (i.instr->flags) { - case INT8_TYPE: - e.CMEQ(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.CMEQ(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.CMEQ(dest.S4(), src1.S4(), src2.S4()); - break; - case FLOAT32_TYPE: - e.FCMEQ(dest.S4(), src1.S4(), src2.S4()); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_EQ, VECTOR_COMPARE_EQ_V128); - -// ============================================================================ -// OPCODE_VECTOR_COMPARE_SGT -// ============================================================================ -struct VECTOR_COMPARE_SGT_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAssociativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - switch (i.instr->flags) { - case INT8_TYPE: - e.CMGT(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.CMGT(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.CMGT(dest.S4(), src1.S4(), src2.S4()); - break; - case FLOAT32_TYPE: - e.FCMGT(dest.S4(), src1.S4(), src2.S4()); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGT, VECTOR_COMPARE_SGT_V128); - -// ============================================================================ -// OPCODE_VECTOR_COMPARE_SGE -// ============================================================================ -struct VECTOR_COMPARE_SGE_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAssociativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - switch (i.instr->flags) { - case INT8_TYPE: - e.CMGE(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.CMGE(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.CMGE(dest.S4(), src1.S4(), src2.S4()); - break; - case FLOAT32_TYPE: - e.FCMGE(dest.S4(), src1.S4(), src2.S4()); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGE, VECTOR_COMPARE_SGE_V128); - -// ============================================================================ -// OPCODE_VECTOR_COMPARE_UGT -// ============================================================================ -struct VECTOR_COMPARE_UGT_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAssociativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - switch (i.instr->flags) { - case INT8_TYPE: - e.CMHI(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.CMHI(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.CMHI(dest.S4(), src1.S4(), src2.S4()); - break; - case FLOAT32_TYPE: - e.FABS(Q0.S4(), src1.S4()); - e.FABS(Q1.S4(), src2.S4()); - e.FCMGT(dest.S4(), Q0.S4(), Q1.S4()); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGT, VECTOR_COMPARE_UGT_V128); - -// ============================================================================ -// OPCODE_VECTOR_COMPARE_UGE -// ============================================================================ -struct VECTOR_COMPARE_UGE_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAssociativeBinaryVOp( - e, i, [&i](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - switch (i.instr->flags) { - case INT8_TYPE: - e.CMHS(dest.B16(), src1.B16(), src2.B16()); - break; - case INT16_TYPE: - e.CMHS(dest.H8(), src1.H8(), src2.H8()); - break; - case INT32_TYPE: - e.CMHS(dest.S4(), src1.S4(), src2.S4()); - break; - case FLOAT32_TYPE: - e.FABS(Q0.S4(), src1.S4()); - e.FABS(Q1.S4(), src2.S4()); - e.FCMGE(dest.S4(), Q0.S4(), Q1.S4()); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGE, VECTOR_COMPARE_UGE_V128); - -// ============================================================================ -// OPCODE_VECTOR_ADD -// ============================================================================ -struct VECTOR_ADD - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, const QReg& dest, QReg src1, QReg src2) { - const TypeName part_type = - static_cast(i.instr->flags & 0xFF); - const uint32_t arithmetic_flags = i.instr->flags >> 8; - bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED); - bool saturate = !!(arithmetic_flags & ARITHMETIC_SATURATE); - switch (part_type) { - case INT8_TYPE: - if (saturate) { - if (is_unsigned) { - e.UQADD(dest.B16(), src1.B16(), src2.B16()); - } else { - e.SQADD(dest.B16(), src1.B16(), src2.B16()); - } - } else { - e.ADD(dest.B16(), src1.B16(), src2.B16()); - } - break; - case INT16_TYPE: - if (saturate) { - if (is_unsigned) { - e.UQADD(dest.H8(), src1.H8(), src2.H8()); - } else { - e.SQADD(dest.H8(), src1.H8(), src2.H8()); - } - } else { - e.ADD(dest.H8(), src1.H8(), src2.H8()); - } - break; - case INT32_TYPE: - if (saturate) { - if (is_unsigned) { - e.UQADD(dest.S4(), src1.S4(), src2.S4()); - } else { - e.SQADD(dest.S4(), src1.S4(), src2.S4()); - } - } else { - e.ADD(dest.S4(), src1.S4(), src2.S4()); - } - break; - case FLOAT32_TYPE: - assert_false(is_unsigned); - assert_false(saturate); - e.FADD(dest.S4(), src1.S4(), src2.S4()); - break; - default: - assert_unhandled_case(part_type); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ADD, VECTOR_ADD); - -// ============================================================================ -// OPCODE_VECTOR_SUB -// ============================================================================ -struct VECTOR_SUB - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [&i](A64Emitter& e, const QReg& dest, QReg src1, QReg src2) { - const TypeName part_type = - static_cast(i.instr->flags & 0xFF); - const uint32_t arithmetic_flags = i.instr->flags >> 8; - bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED); - bool saturate = !!(arithmetic_flags & ARITHMETIC_SATURATE); - switch (part_type) { - case INT8_TYPE: - if (saturate) { - if (is_unsigned) { - e.UQSUB(dest.B16(), src1.B16(), src2.B16()); - } else { - e.SQSUB(dest.B16(), src1.B16(), src2.B16()); - } - } else { - e.SUB(dest.B16(), src1.B16(), src2.B16()); - } - break; - case INT16_TYPE: - if (saturate) { - if (is_unsigned) { - e.UQSUB(dest.H8(), src1.H8(), src2.H8()); - } else { - e.SQSUB(dest.H8(), src1.H8(), src2.H8()); - } - } else { - e.SUB(dest.H8(), src1.H8(), src2.H8()); - } - break; - case INT32_TYPE: - if (saturate) { - if (is_unsigned) { - e.UQSUB(dest.S4(), src1.S4(), src2.S4()); - } else { - e.SQSUB(dest.S4(), src1.S4(), src2.S4()); - } - } else { - e.SUB(dest.S4(), src1.S4(), src2.S4()); - } - break; - case FLOAT32_TYPE: - assert_false(is_unsigned); - assert_false(saturate); - e.FSUB(dest.S4(), src1.S4(), src2.S4()); - break; - default: - assert_unhandled_case(part_type); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB); - -// ============================================================================ -// OPCODE_VECTOR_SHL -// ============================================================================ -template ::value, int> = 0> -static uint8x16_t EmulateVectorShl(void*, std::byte src1[16], - std::byte src2[16]) { - alignas(16) T value[16 / sizeof(T)]; - alignas(16) T shamt[16 / sizeof(T)]; - - // Load NEON registers into a C array. - vst1q_u8(reinterpret_cast(value), - vld1q_u8(reinterpret_cast(src1))); - vst1q_u8(reinterpret_cast(shamt), - vld1q_u8(reinterpret_cast(src2))); - - for (size_t i = 0; i < (16 / sizeof(T)); ++i) { - value[i] = value[i] << (shamt[i] & ((sizeof(T) * 8) - 1)); - } - - // Store result and return it. - return vld1q_u8(reinterpret_cast(value)); -} -struct VECTOR_SHL_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - switch (i.instr->flags) { - case INT8_TYPE: - EmitInt8(e, i); - break; - case INT16_TYPE: - EmitInt16(e, i); - break; - case INT32_TYPE: - EmitInt32(e, i); - break; - default: - assert_always(); - break; - } - } - - static void EmitInt8(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 16 - n; ++n) { - if (shamt.u8[n] != shamt.u8[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().B16(), src1.B16(), shamt.u8[0] & 0x7); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - - static void EmitInt16(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 16 - n; ++n) { - if (shamt.u8[n] != shamt.u8[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().H8(), src1.H8(), shamt.u8[0] & 0xF); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - - static void EmitInt32(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 16 - n; ++n) { - if (shamt.u8[n] != shamt.u8[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use SHL - e.SHL(i.dest.reg().S4(), src1.S4(), shamt.u8[0] & 0x1F); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShl)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128); - -// ============================================================================ -// OPCODE_VECTOR_SHR -// ============================================================================ -template ::value, int> = 0> -static uint8x16_t EmulateVectorShr(void*, std::byte src1[16], - std::byte src2[16]) { - alignas(16) T value[16 / sizeof(T)]; - alignas(16) T shamt[16 / sizeof(T)]; - - // Load NEON registers into a C array by casting to uint8_t* - vst1q_u8(reinterpret_cast(value), - vld1q_u8(reinterpret_cast(src1))); - vst1q_u8(reinterpret_cast(shamt), - vld1q_u8(reinterpret_cast(src2))); - - for (size_t i = 0; i < (16 / sizeof(T)); ++i) { - value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1)); - } - - // Store result and return it by casting to uint8_t* - return vld1q_u8(reinterpret_cast(value)); -} -struct VECTOR_SHR_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - switch (i.instr->flags) { - case INT8_TYPE: - EmitInt8(e, i); - break; - case INT16_TYPE: - EmitInt16(e, i); - break; - case INT32_TYPE: - EmitInt32(e, i); - break; - default: - assert_always(); - break; - } - } - - static void EmitInt8(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 16 - n; ++n) { - if (shamt.u8[n] != shamt.u8[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().B16(), src1.B16(), shamt.u8[0] & 0x7); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - - static void EmitInt16(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 8 - n; ++n) { - if (shamt.u16[n] != shamt.u16[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().H8(), src1.H8(), shamt.u16[0] & 0xF); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - - static void EmitInt32(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 4 - n; ++n) { - if (shamt.u32[n] != shamt.u32[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use USHR - e.USHR(i.dest.reg().S4(), src1.S4(), shamt.u32[0] & 0x1F); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHR, VECTOR_SHR_V128); - -// ============================================================================ -// OPCODE_VECTOR_SHA -// ============================================================================ -struct VECTOR_SHA_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - switch (i.instr->flags) { - case INT8_TYPE: - EmitInt8(e, i); - break; - case INT16_TYPE: - EmitInt16(e, i); - break; - case INT32_TYPE: - EmitInt32(e, i); - break; - default: - assert_always(); - break; - } - } - - static void EmitInt8(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 16 - n; ++n) { - if (shamt.u8[n] != shamt.u8[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().B16(), src1.B16(), shamt.u8[0] & 0x7); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - - static void EmitInt16(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 8 - n; ++n) { - if (shamt.u16[n] != shamt.u16[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().H8(), src1.H8(), shamt.u16[0] & 0xF); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - - static void EmitInt32(A64Emitter& e, const EmitArgType& i) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - if (i.src2.is_constant) { - const auto& shamt = i.src2.constant(); - bool all_same = true; - for (size_t n = 0; n < 4 - n; ++n) { - if (shamt.u32[n] != shamt.u32[n + 1]) { - all_same = false; - break; - } - } - if (all_same) { - // Every count is the same, so we can use SSHR - e.SSHR(i.dest.reg().S4(), src1.S4(), shamt.u32[0] & 0x1F); - return; - } - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, src1)); - e.CallNativeSafe(reinterpret_cast(EmulateVectorShr)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128); - -// ============================================================================ -// OPCODE_VECTOR_ROTATE_LEFT -// ============================================================================ -template ::value, int> = 0> -static uint8x16_t EmulateVectorRotateLeft(void*, std::byte src1[16], - std::byte src2[16]) { - alignas(16) T value[16 / sizeof(T)]; - alignas(16) T shamt[16 / sizeof(T)]; - - // Load NEON registers into a C array by casting to uint8_t* - vst1q_u8(reinterpret_cast(value), - vld1q_u8(reinterpret_cast(src1))); - vst1q_u8(reinterpret_cast(shamt), - vld1q_u8(reinterpret_cast(src2))); - - for (size_t i = 0; i < (16 / sizeof(T)); ++i) { - value[i] = xe::rotate_left(value[i], shamt[i] & ((sizeof(T) * 8) - 1)); - } - - // Store result and return it by casting to uint8_t* - return vld1q_u8(reinterpret_cast(value)); -} -struct VECTOR_ROTATE_LEFT_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.ADD(e.GetNativeParam(1), SP, e.StashConstantV(1, i.src2.constant())); - } else { - e.ADD(e.GetNativeParam(1), SP, e.StashV(1, i.src2)); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - switch (i.instr->flags) { - case INT8_TYPE: - e.CallNativeSafe( - reinterpret_cast(EmulateVectorRotateLeft)); - break; - case INT16_TYPE: - e.CallNativeSafe( - reinterpret_cast(EmulateVectorRotateLeft)); - break; - case INT32_TYPE: - e.CallNativeSafe( - reinterpret_cast(EmulateVectorRotateLeft)); - break; - default: - assert_always(); - break; - } - e.MOV(i.dest.reg().B16(), Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128); - -// ============================================================================ -// OPCODE_VECTOR_AVERAGE -// ============================================================================ -struct VECTOR_AVERAGE - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, - [&i](A64Emitter& e, const QReg& dest, const QReg& src1, - const QReg& src2) { - const TypeName part_type = - static_cast(i.instr->flags & 0xFF); - const uint32_t arithmetic_flags = i.instr->flags >> 8; - bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED); - switch (part_type) { - case INT8_TYPE: - if (is_unsigned) { - e.URHADD(dest.B16(), src1.B16(), src2.B16()); - } else { - e.SRHADD(dest.B16(), src1.B16(), src2.B16()); - assert_always(); - } - break; - case INT16_TYPE: - if (is_unsigned) { - e.URHADD(dest.H8(), src1.H8(), src2.H8()); - } else { - e.SRHADD(dest.H8(), src1.H8(), src2.H8()); - } - break; - case INT32_TYPE: - if (is_unsigned) { - e.URHADD(dest.S4(), src1.S4(), src2.S4()); - } else { - e.SRHADD(dest.S4(), src1.S4(), src2.S4()); - } - break; - default: - assert_unhandled_case(part_type); - break; - } - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_VECTOR_AVERAGE, VECTOR_AVERAGE); - -// ============================================================================ -// OPCODE_INSERT -// ============================================================================ -struct INSERT_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.is_constant); - e.MOV(i.dest.reg().Belem()[i.src2.constant() ^ 0x3], i.src3.reg()); - } -}; -struct INSERT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.is_constant); - e.MOV(i.dest.reg().Helem()[i.src2.constant() ^ 0x1], i.src3.reg()); - } -}; -struct INSERT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.is_constant); - e.MOV(i.dest.reg().Selem()[i.src2.constant()], i.src3.reg()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_INSERT, INSERT_I8, INSERT_I16, INSERT_I32); - -// ============================================================================ -// OPCODE_EXTRACT -// ============================================================================ -// TODO(benvanik): sequence extract/splat: -// v0.i32 = extract v0.v128, 0 -// v0.v128 = splat v0.i32 -// This can be a single broadcast. -struct EXTRACT_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.UMOV(i.dest, i.src1.reg().Belem()[VEC128_B(i.src2.constant())]); - } else { - // Fixup index - e.EOR(W0, i.src2, 0b11); - e.AND(W0, W0, 0x1F); - e.DUP(Q0.B16(), W0); - // Byte-table lookup - e.TBL(Q0.B16(), List{i.src1.reg().B16()}, Q0.B16()); - // Get lowest element - e.UMOV(i.dest, Q0.Belem()[0]); - } - } -}; -struct EXTRACT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src2.is_constant) { - e.UMOV(i.dest, i.src1.reg().Helem()[VEC128_W(i.src2.constant())]); - } else { - // Fixup index - e.EOR(W0, i.src2, 0b01); - e.LSL(W0, W0, 1); - - // Replicate index as byte - e.MOV(W1, 0x01'01); - e.MUL(W0, W0, W1); - - // Byte indices - e.ADD(W0, W0, 0x01'00); - e.UXTH(W0, W0); - - // Replicate byte indices - e.DUP(Q0.H8(), W0); - // Byte-table lookup - e.TBL(Q0.B16(), List{i.src1.reg().B16()}, Q0.B16()); - // Get lowest element - e.UMOV(i.dest, Q0.Helem()[0]); - } - } -}; -struct EXTRACT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - static const vec128_t extract_table_32[4] = { - vec128b(3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - vec128b(7, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - vec128b(11, 10, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - vec128b(15, 14, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - }; - if (i.src2.is_constant) { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - e.UMOV(i.dest, src1.Selem()[VEC128_D(i.src2.constant())]); - } else { - QReg src1 = i.src1.is_constant ? Q1 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - - e.AND(X0, i.src2.reg().toX(), 0b11); - e.LSL(X0, X0, 4); - - e.MOV(X1, reinterpret_cast(extract_table_32)); - e.LDR(Q0, X1, X0); - - // Byte-table lookup - e.TBL(Q0.B16(), List{src1.B16()}, Q0.B16()); - // Get lowest element - e.UMOV(i.dest, Q0.Selem()[0]); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); - -// ============================================================================ -// OPCODE_SPLAT -// ============================================================================ -// Copy a value into all elements of a vector -struct SPLAT_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - // int8_t is always <= 0xFF, so this is always true - // Keep the MOVI path for constants - e.MOVI(i.dest.reg().B16(), i.src1.constant()); - } else { - e.DUP(i.dest.reg().B16(), i.src1); - } - } -}; -struct SPLAT_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - if (i.src1.constant() <= 0xFF) { - e.MOVI(i.dest.reg().H8(), i.src1.constant()); - return; - } - e.MOV(W0, i.src1.constant()); - e.DUP(i.dest.reg().H8(), W0); - } else { - e.DUP(i.dest.reg().H8(), i.src1); - } - } -}; -struct SPLAT_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - if (i.src1.constant() <= 0xFF) { - e.MOVI(i.dest.reg().S4(), i.src1.constant()); - return; - } - e.MOV(W0, i.src1.constant()); - e.DUP(i.dest.reg().S4(), W0); - } else { - e.DUP(i.dest.reg().S4(), i.src1); - } - } -}; -struct SPLAT_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - if (i.src1.value->constant.i32 <= 0xFF) { - e.MOVI(i.dest.reg().S4(), i.src1.value->constant.i32); - return; - } - e.MOV(W0, i.src1.value->constant.i32); - e.DUP(i.dest.reg().S4(), W0); - } else { - e.DUP(i.dest.reg().S4(), i.src1.reg().toQ().Selem()[0]); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SPLAT, SPLAT_I8, SPLAT_I16, SPLAT_I32, SPLAT_F32); - -// ============================================================================ -// OPCODE_PERMUTE -// ============================================================================ -struct PERMUTE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(i.instr->flags == INT32_TYPE); - // Permute words between src2 and src3. - if (i.src1.is_constant) { - // Each byte is a word-index - const uint32_t control = i.src1.constant(); - const QReg indices = Q0; - - // Word to byte index - e.MOV(W0, control * 4); - e.MOV(indices.Selem()[0], W0); - - // Widen int8 to int16 - e.ZIP1(indices.B16(), indices.B16(), indices.B16()); - // Widen int16 to int32 - e.ZIP1(indices.B16(), indices.B16(), indices.B16()); - - // Convert to byte-indices - e.MOV(W0, 0x03'02'01'00); - e.DUP(Q1.S4(), W0); - e.ADD(indices.S4(), indices.S4(), Q1.S4()); - - // Table-registers must be sequential indices - const QReg table0 = Q2; - if (i.src2.is_constant) { - e.LoadConstantV(table0, i.src2.constant()); - } else { - e.MOV(table0.B16(), i.src2.reg().B16()); - } - - const QReg table1 = Q3; - if (i.src3.is_constant) { - e.LoadConstantV(table1, i.src3.constant()); - } else { - e.MOV(table1.B16(), i.src3.reg().B16()); - } - - e.TBL(i.dest.reg().B16(), List{table0.B16(), table1.B16()}, - indices.B16()); - } else { - // Permute by non-constant. - assert_always(); - } - } -}; -struct PERMUTE_V128 - : Sequence> { - static void EmitByInt8(A64Emitter& e, const EmitArgType& i) { - // Permute bytes between src2 and src3. - // src1 is an array of indices corresponding to positions within src2 and - // src3. - if (i.src3.value->IsConstantZero()) { - if (i.src2.value->IsConstantZero()) { - // src2 & src3 are zero, so result will always be zero. - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); - return; - } - } - - const QReg indices = Q0; - if (i.src1.is_constant) { - e.LoadConstantV(indices, i.src1.constant()); - } else { - e.MOV(indices.B16(), i.src1.reg().B16()); - } - - // Indices must be endian-swapped - e.MOVI(Q1.B16(), 0b11); - e.EOR(indices.B16(), indices.B16(), Q1.B16()); - - // Modulo 32 the indices - e.MOVI(Q1.B16(), 0b0001'1111); - e.AND(indices.B16(), indices.B16(), Q1.B16()); - - // Table-registers must be sequential indices - const QReg table_lo = Q2; - if (i.src2.is_constant) { - e.LoadConstantV(table_lo, i.src2.constant()); - } else { - e.MOV(table_lo.B16(), i.src2.reg().B16()); - } - - const QReg table_hi = Q3; - if (i.src3.is_constant) { - e.LoadConstantV(table_hi, i.src3.constant()); - } else { - e.MOV(table_hi.B16(), i.src3.reg().B16()); - } - - e.TBL(i.dest.reg().B16(), List{table_lo.B16(), table_hi.B16()}, - indices.B16()); - } - - static void EmitByInt16(A64Emitter& e, const EmitArgType& i) { - // Permute 16-bit halfwords between src2 and src3. - // src1 is an array of 16-bit indices corresponding to positions within src2 - // and src3. - if (i.src3.value->IsConstantZero()) { - if (i.src2.value->IsConstantZero()) { - // src2 & src3 are zero, so result will always be zero. - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); - return; - } - } - - // This implementation is adapted from the x64 backend - assert_true(i.src1.is_constant); - - // Process the constant indices to create byte shuffle masks and blend - // control - vec128_t perm = (i.src1.constant() & vec128s(0xF)) ^ vec128s(0x1); - vec128_t perm_bytes = vec128b(0); - vec128_t blend_mask = vec128b(0); - - for (int idx = 0; idx < 8; idx++) { - bool from_src3 = perm.i16[idx] > 7; - - // Create byte indices for this 16-bit element - uint8_t base_byte_idx = uint8_t(perm.u16[idx] & 7) * 2; - perm_bytes.u8[idx * 2] = base_byte_idx; - perm_bytes.u8[idx * 2 + 1] = base_byte_idx + 1; - - // Create blend mask (0xFF means take from src3, 0x00 means take from - // src2) - blend_mask.u8[idx * 2] = from_src3 ? 0xFF : 0x00; - blend_mask.u8[idx * 2 + 1] = from_src3 ? 0xFF : 0x00; - } - - // Load the byte shuffle mask - e.LoadConstantV(Q0, perm_bytes); - - // Load the blend mask - e.LoadConstantV(Q1, blend_mask); - - // Load src2 and shuffle it - if (i.src2.is_constant) { - e.LoadConstantV(Q2, i.src2.constant()); - } else { - e.MOV(Q2.B16(), i.src2.reg().B16()); - } - e.TBL(Q2.B16(), List{Q2.B16()}, Q0.B16()); - - // Load src3 and shuffle it - if (i.src3.is_constant) { - e.LoadConstantV(Q3, i.src3.constant()); - } else { - e.MOV(Q3.B16(), i.src3.reg().B16()); - } - e.TBL(Q3.B16(), List{Q3.B16()}, Q0.B16()); - - // Blend the results: dest = (Q3 & Q1) | (Q2 & ~Q1) - e.AND(Q3.B16(), Q3.B16(), Q1.B16()); // Q3 & mask - e.BIC(Q2.B16(), Q2.B16(), Q1.B16()); // Q2 & ~mask - e.ORR(i.dest.reg().B16(), Q3.B16(), Q2.B16()); // combine - } - - static void EmitByInt32(A64Emitter& e, const EmitArgType& i) { - assert_always(); - } - - static void Emit(A64Emitter& e, const EmitArgType& i) { - switch (i.instr->flags) { - case INT8_TYPE: - EmitByInt8(e, i); - break; - case INT16_TYPE: - EmitByInt16(e, i); - break; - case INT32_TYPE: - EmitByInt32(e, i); - break; - default: - assert_unhandled_case(i.instr->flags); - return; - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_PERMUTE, PERMUTE_I32, PERMUTE_V128); - -// ============================================================================ -// OPCODE_SWIZZLE -// ============================================================================ -struct SWIZZLE - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - auto element_type = i.instr->flags; - if (element_type == INT8_TYPE) { - assert_always(); - } else if (element_type == INT16_TYPE) { - assert_always(); - } else if (element_type == INT32_TYPE || element_type == FLOAT32_TYPE) { - // Four 2-bit word-indices packed into one 8-bit value - const uint8_t swizzle_mask = static_cast(i.src2.value); - - // Convert to byte-indices - const vec128_t indice_vec = - vec128i(((swizzle_mask >> 0) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, - ((swizzle_mask >> 2) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, - ((swizzle_mask >> 4) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00, - ((swizzle_mask >> 6) & 0b11) * 0x04'04'04'04 + 0x03'02'01'00); - - const QReg indices = Q1; - e.LoadConstantV(indices, indice_vec); - - QReg table0 = Q0; - if (i.src1.is_constant) { - e.LoadConstantV(table0, i.src1.constant()); - } else { - table0 = i.src1; - } - - e.TBL(i.dest.reg().B16(), List{table0.B16()}, indices.B16()); - } else if (element_type == INT64_TYPE || element_type == FLOAT64_TYPE) { - assert_always(); - } else { - assert_always(); - } - }; -}; -EMITTER_OPCODE_TABLE(OPCODE_SWIZZLE, SWIZZLE); - -// ============================================================================ -// OPCODE_PACK -// ============================================================================ -struct PACK : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - switch (i.instr->flags & PACK_TYPE_MODE) { - case PACK_TYPE_D3DCOLOR: - EmitD3DCOLOR(e, i); - break; - case PACK_TYPE_FLOAT16_2: - EmitFLOAT16_2(e, i); - break; - case PACK_TYPE_FLOAT16_4: - EmitFLOAT16_4(e, i); - break; - case PACK_TYPE_SHORT_2: - EmitSHORT_2(e, i); - break; - case PACK_TYPE_SHORT_4: - EmitSHORT_4(e, i); - break; - case PACK_TYPE_UINT_2101010: - EmitUINT_2101010(e, i); - break; - case PACK_TYPE_ULONG_4202020: - EmitULONG_4202020(e, i); - break; - case PACK_TYPE_8_IN_16: - Emit8_IN_16(e, i, i.instr->flags); - break; - case PACK_TYPE_16_IN_32: - Emit16_IN_32(e, i, i.instr->flags); - break; - default: - assert_unhandled_case(i.instr->flags); - break; - } - } - static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->IsConstantZero()); - QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } - - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - // Saturate to [3,3....] so that only values between 3...[00] and 3...[FF] - // are valid - max before min to pack NaN as zero (5454082B is heavily - // affected by the order - packs 0xFFFFFFFF in matrix code to get a 0 - // constant). - // Use FMAXNM/FMINNM (numeric max/min) instead of FMAX/FMIN to handle NaN - // correctly. FMAXNM returns the non-NaN value when one operand is NaN, - // matching x86 behavior and Windows ARM64 expectations. - e.LDR(Q0, VConstData, e.GetVConstOffset(V3333)); - e.FMAXNM(i.dest.reg().S4(), src.S4(), Q0.S4()); - - e.LDR(Q0, VConstData, e.GetVConstOffset(VPackD3DCOLORSat)); - e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q0.S4()); - - // Extract bytes. - // RGBA (XYZW) -> ARGB (WXYZ) - // w = ((src1.uw & 0xFF) << 24) | ((src1.ux & 0xFF) << 16) | - // ((src1.uy & 0xFF) << 8) | (src1.uz & 0xFF) - e.LDR(Q0, VConstData, e.GetVConstOffset(VPackD3DCOLOR)); - e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q0.B16()); - } - static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { - alignas(16) float a[4]; - alignas(16) uint16_t b[8]; - - // Load NEON registers into a C array by casting to uint8_t* - vst1q_u8(reinterpret_cast(a), - vld1q_u8(reinterpret_cast(src1))); - std::memset(b, 0, sizeof(b)); - - for (int i = 0; i < 2; i++) { - float x = a[i]; - uint16_t h; - - // Xbox 360 saturation behavior - if (x >= 65504.0f) { - h = 0x7FFF; // Positive saturation sentinel - } else if (x <= -65504.0f) { - h = 0xFFFF; // Negative saturation sentinel - } else { - h = half_float::detail::float2half(x); - } - - b[7 - i] = h; - } - - // Store the uint16_t array into a uint8x16_t NEON register - return vld1q_u8(reinterpret_cast(b)); - } - static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->IsConstantZero()); - // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx - // dest = [(src1.x | src1.y), 0, 0, 0] - - if (e.IsFeatureEnabled(kA64EmitF16C)) { - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - - // Perform IEEE conversion first - e.FCVTN(i.dest.reg().toD().H4(), src1.S4()); - - // Xbox 360 rule: infinity values should be replaced with 0x7FFF/0xFFFF - // After FCVTN, +inf becomes 0x7C00, -inf becomes 0xFC00 - // We need to detect these and replace them - - // Create masks for infinity detection - QReg abs_mask = Q1; - QReg inf_pattern = Q2; - QReg is_inf = Q3; - QReg sign_mask = Q4; - QReg sentinel = Q5; - - // Get absolute value of halfwords (clear sign bit) - e.MOVI(abs_mask.B16(), 0xFF); // 0xFFFF - e.USHR(abs_mask.H8(), abs_mask.H8(), 1); // 0x7FFF - e.AND(is_inf.toD().B8(), i.dest.reg().toD().B8(), abs_mask.toD().B8()); - - // Check if abs value == 0x7C00 (infinity) - e.MOVI(inf_pattern.H8(), 0x7C, oaknut::LslSymbol{}, 8); // 0x7C00 - e.CMEQ(is_inf.toD().H4(), is_inf.toD().H4(), inf_pattern.toD().H4()); - - // Get sign bits - e.MOVI(sign_mask.H8(), 0x80, oaknut::LslSymbol{}, 8); // 0x8000 - e.AND(sign_mask.toD().B8(), i.dest.reg().toD().B8(), - sign_mask.toD().B8()); - - // Create sentinel value: sign | 0x7FFF - e.MOVI(sentinel.B16(), 0xFF); // 0xFFFF - e.USHR(sentinel.H8(), sentinel.H8(), 1); // 0x7FFF - e.ORR(sentinel.toD().B8(), sentinel.toD().B8(), sign_mask.toD().B8()); - - // Apply replacement where infinity was detected - // Use BSL to replace infinities with sentinels - // BSL: result = (mask & src1) | (~mask & src2) - // We want: where is_inf is true, use sentinel; else use original - e.BSL(is_inf.toD().B8(), sentinel.toD().B8(), i.dest.reg().toD().B8()); - e.MOV(i.dest.reg().toD().B8(), is_inf.toD().B8()); - - // Keep existing layout adjustments - e.MOVI(Q0.B16(), 0); - e.EXT(i.dest.reg().B16(), Q0.B16(), i.dest.reg().B16(), 4); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - return; - } - - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - } - e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_2)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static uint8x16_t EmulateFLOAT16_4(void*, std::byte src1[16]) { - alignas(16) float a[4]; - alignas(16) uint16_t b[8]; - - // Load NEON registers into a C array by casting to uint8_t* - vst1q_u8(reinterpret_cast(a), - vld1q_u8(reinterpret_cast(src1))); - std::memset(b, 0, sizeof(b)); - - for (int i = 0; i < 4; i++) { - b[7 - (i ^ 2)] = - half_float::detail::float2half(a[i]); - } - - // Store the uint16_t array into a uint8x16_t NEON register - return vld1q_u8(reinterpret_cast(b)); - } - static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->IsConstantZero()); - // http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx - // dest = [(src1.z | src1.w), (src1.x | src1.y), 0, 0] - - if (e.IsFeatureEnabled(kA64EmitF16C)) { - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - e.FCVTN(i.dest.reg().toD().H4(), src1.S4()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - return; - } - - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - } - e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_4)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->IsConstantZero()); - - // Check if input is constant zero - special case - if (i.src1.is_constant && i.src1.value->IsConstantZero()) { - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); - return; - } - - QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } else { - // Copy src to dest if they're different registers - if (src.index() != i.dest.reg().index()) { - e.MOV(i.dest.reg().B16(), src.B16()); - } - } - - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - // Check if the value is zero (0x00000000) - if so, skip min/max - // Zero is a special case that doesn't use the mantissa trick - // Create a mask for zero elements - e.CMEQ(Q0.S4(), i.dest.reg().S4(), 0); - - // Save the zero mask - e.MOV(Q2.B16(), Q0.B16()); - - // Saturate non-zero values - use FMAXNM/FMINNM to handle NaN correctly - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); - e.FMAXNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Max)); - e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Use BIC to clear values that were originally zero - e.BIC(i.dest.reg().B16(), i.dest.reg().B16(), Q2.B16()); - - // Pack - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_2)); - e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q1.B16()); - } - static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { - assert_true(i.src2.value->IsConstantZero()); - - // Check if input is constant zero - special case - if (i.src1.is_constant && i.src1.value->IsConstantZero()) { - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16()); - return; - } - - QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } else { - // Copy src to dest if they're different registers - if (src.index() != i.dest.reg().index()) { - e.MOV(i.dest.reg().B16(), src.B16()); - } - } - - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - // Check if the value is zero (0x00000000) - if so, skip min/max - // Zero is a special case that doesn't use the mantissa trick - // Create a mask for zero elements - e.CMEQ(Q0.S4(), i.dest.reg().S4(), 0); - - // Save the zero mask - e.MOV(Q2.B16(), Q0.B16()); - - // Saturate non-zero values - use FMAXNM/FMINNM to handle NaN correctly - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Min)); - e.FMAXNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_Max)); - e.FMINNM(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Use BIC to clear values that were originally zero - e.BIC(i.dest.reg().B16(), i.dest.reg().B16(), Q2.B16()); - - // Pack - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackSHORT_4)); - e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q1.B16()); - } - static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { - // https://www.opengl.org/registry/specs/ARB/vertex_type_2_10_10_10_rev.txt - // XYZ are 10 bits, signed and saturated. - // W is 2 bits, unsigned and saturated. - const QReg src = i.dest; - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - // Saturate. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MinUnpacked)); - e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MaxUnpacked)); - e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Remove the unneeded bits of the floats. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MaskUnpacked)); - e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); - - // Shift the components up. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_Shift)); - e.USHL(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Combine the components. - e.LoadConstantV(Q1, vec128i(0x03'02'01'00 + 0x04'04'04'04 * 2, - 0x03'02'01'00 + 0x04'04'04'04 * 3, - 0x03'02'01'00 + 0x04'04'04'04 * 0, - 0x03'02'01'00 + 0x04'04'04'04 * 1)); - e.TBL(Q0.B16(), List{i.dest.reg().B16()}, Q1.B16()); - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); - - e.LoadConstantV(Q1, vec128i(0x03'02'01'00 + 0x04'04'04'04 * 1, - 0x03'02'01'00 + 0x04'04'04'04 * 0, - 0x03'02'01'00 + 0x04'04'04'04 * 3, - 0x03'02'01'00 + 0x04'04'04'04 * 2)); - e.TBL(Q0.B16(), List{i.dest.reg().B16()}, Q1.B16()); - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); - } - static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { - // XYZ are 20 bits, signed and saturated. - // W is 4 bits, unsigned and saturated. - QReg src = i.src1.is_constant ? i.dest.reg() : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - // Saturate. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MinUnpacked)); - e.FMAX(i.dest.reg().S4(), src.S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MaxUnpacked)); - e.FMIN(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Remove the unneeded bits of the floats (so excess nibbles will also be - // cleared). - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_MaskUnpacked)); - e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); - - // Store Y and W shifted left by 4 so vpshufb can be used with them. - e.SHL(Q0.S4(), i.dest.reg().S4(), 4); - - // Place XZ where they're supposed to be. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_PermuteXZ)); - e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16()}, Q1.B16()); - // Place YW. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackULONG_4202020_PermuteYW)); - e.TBL(Q0.B16(), List{Q0.B16()}, Q1.B16()); - // Merge XZ and YW. - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q0.B16()); - } - static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { - if (IsPackInUnsigned(flags)) { - if (IsPackOutUnsigned(flags)) { - if (IsPackOutSaturate(flags)) { - // unsigned -> unsigned + saturate - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - - const QReg src2 = i.src2.is_constant ? Q1 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2, i.src2.constant()); - } - // Windows ARM64 requires src2->src1 ordering with EXT swap - // macOS works with src1->src2 without EXT -#ifdef _WIN32 - e.UQXTN(i.dest.reg().toD().B8(), src2.H8()); - e.UQXTN2(i.dest.reg().B16(), src1.H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.UQXTN(i.dest.reg().toD().B8(), src1.H8()); - e.UQXTN2(i.dest.reg().B16(), src2.H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } else { - // unsigned -> unsigned - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.XTN(i.dest.reg().toD().B8(), i.src2.reg().H8()); - e.XTN2(i.dest.reg().B16(), i.src1.reg().H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.XTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); - e.XTN2(i.dest.reg().B16(), i.src2.reg().H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } - } else { - if (IsPackOutSaturate(flags)) { - // unsigned -> signed + saturate - assert_always(); - } else { - // unsigned -> signed - assert_always(); - } - } - } else { - if (IsPackOutUnsigned(flags)) { - if (IsPackOutSaturate(flags)) { - // signed -> unsigned + saturate - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - - const QReg src2 = i.src2.is_constant ? Q1 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2, i.src2.constant()); - } - - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.SQXTUN(i.dest.reg().toD().B8(), src2.H8()); - e.SQXTUN2(i.dest.reg().B16(), src1.H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.SQXTUN(i.dest.reg().toD().B8(), src1.H8()); - e.SQXTUN2(i.dest.reg().B16(), src2.H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } else { - // signed -> unsigned - assert_always(); - } - } else { - if (IsPackOutSaturate(flags)) { - // signed -> signed + saturate - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.SQXTN(i.dest.reg().toD().B8(), i.src2.reg().H8()); - e.SQXTN2(i.dest.reg().B16(), i.src1.reg().H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.SQXTN(i.dest.reg().toD().B8(), i.src1.reg().H8()); - e.SQXTN2(i.dest.reg().B16(), i.src2.reg().H8()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } else { - // signed -> signed - assert_always(); - } - } - } - } - // Pack 2 32-bit vectors into a 16-bit vector. - static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, - uint32_t flags) { - // TODO(benvanik): handle src2 (or src1) being constant zero - if (IsPackInUnsigned(flags)) { - if (IsPackOutUnsigned(flags)) { - if (IsPackOutSaturate(flags)) { - // unsigned -> unsigned + saturate - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - - const QReg src2 = i.src2.is_constant ? Q1 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2, i.src2.constant()); - } - - // Create saturation limit: 0xFFFF in all lanes - e.MOV(W0, 0xFFFF); - e.DUP(Q2.S4(), W0); - - // Saturate both sources - e.UMIN(Q0.S4(), src1.S4(), Q2.S4()); // Saturate src1 (v3) - e.UMIN(Q1.S4(), src2.S4(), Q2.S4()); // Saturate src2 (v4) - - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.UQXTN(i.dest.reg().toD().H4(), - Q1.S4()); // Pack src2 to lower 64 bits - e.UQXTN2(i.dest.reg().H8(), Q0.S4()); // Pack src1 to upper 64 bits - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.UQXTN(i.dest.reg().toD().H4(), - Q0.S4()); // Pack src1 to lower 64 bits - e.UQXTN2(i.dest.reg().H8(), Q1.S4()); // Pack src2 to upper 64 bits - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } else { - // unsigned -> unsigned - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.XTN(i.dest.reg().toD().H4(), i.src2.reg().S4()); - e.XTN2(i.dest.reg().H8(), i.src1.reg().S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.XTN(i.dest.reg().toD().H4(), i.src1.reg().S4()); - e.XTN2(i.dest.reg().H8(), i.src2.reg().S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } - } else { - if (IsPackOutSaturate(flags)) { - // unsigned -> signed + saturate - assert_always(); - } else { - // unsigned -> signed - assert_always(); - } - } - } else { - if (IsPackOutUnsigned(flags)) { - if (IsPackOutSaturate(flags)) { - // signed -> unsigned + saturate - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.SQXTUN(i.dest.reg().toD().H4(), i.src2.reg().S4()); - e.SQXTUN2(i.dest.reg().H8(), i.src1.reg().S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.SQXTUN(i.dest.reg().toD().H4(), i.src1.reg().S4()); - e.SQXTUN2(i.dest.reg().H8(), i.src2.reg().S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } else { - // signed -> unsigned - assert_always(); - } - } else { - if (IsPackOutSaturate(flags)) { - // signed -> signed + saturate - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - - const QReg src2 = i.src2.is_constant ? Q1 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2, i.src2.constant()); - } - // Windows ARM64 requires src2->src1 ordering with EXT swap -#ifdef _WIN32 - e.SQXTN(i.dest.reg().toD().H4(), src2.S4()); - e.SQXTN2(i.dest.reg().H8(), src1.S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.dest.reg().B16(), 8); -#else - e.SQXTN(i.dest.reg().toD().H4(), src1.S4()); - e.SQXTN2(i.dest.reg().H8(), src2.S4()); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); -#endif - } else { - // signed -> signed - assert_always(); - } - } - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_PACK, PACK); - -// ============================================================================ -// OPCODE_UNPACK -// ============================================================================ -struct UNPACK : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - switch (i.instr->flags & PACK_TYPE_MODE) { - case PACK_TYPE_D3DCOLOR: - EmitD3DCOLOR(e, i); - break; - case PACK_TYPE_FLOAT16_2: - EmitFLOAT16_2(e, i); - break; - case PACK_TYPE_FLOAT16_4: - EmitFLOAT16_4(e, i); - break; - case PACK_TYPE_SHORT_2: - EmitSHORT_2(e, i); - break; - case PACK_TYPE_SHORT_4: - EmitSHORT_4(e, i); - break; - case PACK_TYPE_UINT_2101010: - EmitUINT_2101010(e, i); - break; - case PACK_TYPE_ULONG_4202020: - EmitULONG_4202020(e, i); - break; - case PACK_TYPE_8_IN_16: - Emit8_IN_16(e, i, i.instr->flags); - break; - case PACK_TYPE_16_IN_32: - Emit16_IN_32(e, i, i.instr->flags); - break; - default: - assert_unhandled_case(i.instr->flags); - break; - } - } - static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { - // ARGB (WXYZ) -> RGBA (XYZW) - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - QReg src(0); - - if (i.src1.is_constant) { - if (i.src1.value->IsConstantZero()) { - e.FMOV(i.dest.reg().S4(), FImm8(0, 7, 0)); - return; - } - src = i.dest; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - // src = ZZYYXXWW (ARGB packed as bytes) - // Unpack to 000000ZZ,000000YY,000000XX,000000WW - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackD3DCOLOR)); - e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); - // Convert to float by adding float bias - // XOR with 1.0f (0x3F800000) sets the exponent for values 0-127 - // For values 128-255, we need to handle differently - e.FMOV(Q1.S4(), FImm8(0, 7, 0)); // Load 1.0f (0x3F800000) - e.EOR(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); - // To convert to 0 to 1, games multiply by 0x47008081 and add 0xC7008081. - } - static uint8x16_t EmulateFLOAT16_2(void*, std::byte src1[16]) { - alignas(16) uint16_t a[4]; - alignas(16) float b[8]; - - // Load NEON registers into a C array by casting to uint8_t* - vst1q_u8(reinterpret_cast(a), - vld1q_u8(reinterpret_cast(src1))); - std::memset(b, 0, sizeof(b)); - - for (int i = 0; i < 2; i++) { - uint16_t h = a[VEC128_W(6 + i)]; - - // Xbox 360 sentinel value handling - if (h == 0x7FFF) { - b[i] = 131008.0f; // Special positive sentinel (0x47FFE000) - } else if (h == 0xFFFF) { - b[i] = -131008.0f; // Special negative sentinel (0xC7FFE000) - } else { - b[i] = half_float::detail::half2float(h); - } - } - - // Constants, or something - b[2] = 0.f; - b[3] = 1.f; - - // Store the float array into a uint8x16_t NEON register - return vld1q_u8(reinterpret_cast(b)); - } - static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { - // 1 bit sign, 5 bit exponent, 10 bit mantissa - // D3D10 half float format - - if (e.IsFeatureEnabled(kA64EmitF16C)) { - // Load source properly - const QReg src = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src, i.src1.constant()); - } - - // 1) Put src bytes [12..15] into bytes [0..3] of a temp - // EXT(..., #12) copies exactly those 4 bytes; the rest are zero - QReg halves = Q1; - e.EOR(Q0.B16(), Q0.B16(), Q0.B16()); // Q0 = 0 - e.EXT(halves.B16(), src.B16(), Q0.B16(), - 12); // halves[0..3] = src[12..15] - - // Keep copy for sentinel detection (before conversion) - QReg halves_copy = Q2; - e.MOV(halves_copy.B16(), halves.B16()); - - // 2) Convert low 4 halfwords -> 4 floats. FCVTL reads H0..H3 (low 64 - // bits) After the EXT, H0 = low halfword of element 3, H1 = high - // halfword, H2 = H3 = 0 - e.FCVTL(i.dest.reg().S4(), halves.toD().H4()); - - // Note: We do NOT swap the order - the natural order from EXT is correct - - // 3) Xbox 360 rule: 0x7FFF → +131008.0f, 0xFFFF → -131008.0f - - // Create sentinel patterns for halfword comparison - QReg h_7FFF = Q3; - QReg h_FFFF = Q4; - - // Create 0xFFFF - all bytes 0xFF - e.MOVI(h_FFFF.B16(), 0xFF); // All bytes 0xFF = 0xFFFF per halfword - // Create 0x7FFF by shifting 0xFFFF right by 1 - e.USHR(h_7FFF.H8(), h_FFFF.H8(), 1); // 0xFFFF >> 1 = 0x7FFF - - // Compare only the low 64 bits (H0-H3) with sentinels - // After EXT, only H0 and H1 have data, H2 and H3 are zero - QReg mask_7FFF_H = Q5; - QReg mask_FFFF_H = Q6; - // Initialize masks to zero first - e.EOR(mask_7FFF_H.B16(), mask_7FFF_H.B16(), mask_7FFF_H.B16()); - e.EOR(mask_FFFF_H.B16(), mask_FFFF_H.B16(), mask_FFFF_H.B16()); - // Compare only the low 64 bits - e.CMEQ(mask_7FFF_H.toD().H4(), halves_copy.toD().H4(), h_7FFF.toD().H4()); - e.CMEQ(mask_FFFF_H.toD().H4(), halves_copy.toD().H4(), h_FFFF.toD().H4()); - - // Widen halfword masks to word masks for blending with float32 values - // Use high-numbered registers to avoid any aliasing - QReg mask_7FFF_S = Q13; - QReg mask_FFFF_S = Q14; - e.SXTL(mask_7FFF_S.S4(), mask_7FFF_H.toD().H4()); - e.SXTL(mask_FFFF_S.S4(), mask_FFFF_H.toD().H4()); - - // Prepare replacement values: ±131008.0f (broadcast to all lanes) - QReg f_pos_131008 = Q9; - QReg f_neg_131008 = Q10; - // Load immediate into S[0] then duplicate - e.MOV(W0, 0x47FFE000); - e.MOV(f_pos_131008.Selem()[0], W0); - e.DUP(f_pos_131008.S4(), f_pos_131008.Selem()[0]); // +131008.0f - - e.MOV(W0, 0xC7FFE000); - e.MOV(f_neg_131008.Selem()[0], W0); - e.DUP(f_neg_131008.S4(), f_neg_131008.Selem()[0]); // -131008.0f - - // 5) Blend using BIT which has clearer semantics - // BIT Vd, Vn, Vm => Vd = (Vn & Vm) | (Vd & ~Vm) - // When mask==0: result = (replacement & 0) | (original & 0xFFFF) = - // original When mask==0xFFFF: result = (replacement & 0xFFFF) | (original - // & 0) = replacement This is what we want! - - // Apply sentinel replacements using BIT - // BIT Vd, Vn, Vm => Vd = (Vn & Vm) | (Vd & ~Vm) - // When mask is 0: keep original, when mask is 0xFFFFFFFF: use replacement - e.BIT(i.dest.reg().B16(), f_pos_131008.B16(), mask_7FFF_S.B16()); - e.BIT(i.dest.reg().B16(), f_neg_131008.B16(), mask_FFFF_S.B16()); - - // 6) Swap S0 and S1 to match Xbox 360 halfword read order - // The software reads halfword 7 first, then 6, but EXT gives us 6 then 7 - e.REV64(i.dest.reg().S4(), i.dest.reg().S4()); - - // Write 1.0 to element 3 - e.FMOV(S0, oaknut::FImm8(0, 7, 0)); - e.MOV(i.dest.reg().Selem()[3], Q0.Selem()[0]); - return; - } - - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - } - e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_2)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static uint8x16_t EmulateFLOAT16_4(void*, std::byte src1[16]) { - alignas(16) uint16_t a[4]; - alignas(16) float b[8]; - - // Load NEON registers into a C array by casting to uint8_t* - vst1q_u8(reinterpret_cast(a), - vld1q_u8(reinterpret_cast(src1))); - - for (int i = 0; i < 4; i++) { - b[i] = half_float::detail::half2float(a[VEC128_W(4 + i)]); - } - - // Store the float array into a uint8x16_t NEON register - return vld1q_u8(reinterpret_cast(b)); - } - static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { - // src = [(dest.x | dest.y), (dest.z | dest.w), 0, 0] - if (e.IsFeatureEnabled(kA64EmitF16C)) { - const QReg src1 = i.src1.is_constant ? Q0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } - e.EXT(i.dest.reg().B16(), i.dest.reg().B16(), i.src1.reg().B16(), 8); - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.FCVTL(i.dest.reg().S4(), i.dest.reg().toD().H4()); - return; - } - - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - } - e.CallNativeSafe(reinterpret_cast(EmulateFLOAT16_4)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { - // (VD.x) = 3.0 + (VB.x>>16)*2^-22 - // (VD.y) = 3.0 + (VB.x)*2^-22 - // (VD.z) = 0.0 - // (VD.w) = 1.0 (games splat W after unpacking to get vectors of 1.0f) - // src is (xx,xx,xx,VALUE) - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - QReg src(0); - if (i.src1.is_constant) { - if (i.src1.value->IsConstantZero()) { - src = i.dest; - e.LDR(i.dest, VConstData, e.GetVConstOffset(V3301)); - return; - } - // TODO(benvanik): check other common constants/perform shuffle/or here. - src = i.src1; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - // Shuffle bytes. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_2)); - e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); - - // If negative, make smaller than 3 - sign extend before adding. - e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 16); - e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 16); - - // Add 3,3,0,1. - e.LDR(Q1, VConstData, e.GetVConstOffset(V3301)); - e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Return quiet NaNs in case of negative overflow. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_Overflow)); - e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); - e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { - // (VD.x) = 3.0 + (VB.x>>16)*2^-22 - // (VD.y) = 3.0 + (VB.x)*2^-22 - // (VD.z) = 3.0 + (VB.y>>16)*2^-22 - // (VD.w) = 3.0 + (VB.y)*2^-22 - // src is (xx,xx,VALUE,VALUE) - - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - QReg src(0); - if (i.src1.is_constant) { - if (i.src1.value->IsConstantZero()) { - e.LDR(i.dest, VConstData, e.GetVConstOffset(V3333)); - return; - } - // TODO(benvanik): check other common constants/perform shuffle/or here. - src = i.dest; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - // Shuffle bytes. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_4)); - e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); - - // If negative, make smaller than 3 - sign extend before adding. - e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 16); - e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 16); - - // Add 3,3,3,3. - e.LDR(Q1, VConstData, e.GetVConstOffset(V3333)); - e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - - // Return quiet NaNs in case of negative overflow. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackSHORT_Overflow)); - e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); - e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - QReg src(0); - if (i.src1.is_constant) { - if (i.src1.value->IsConstantZero()) { - e.LDR(i.dest, VConstData, e.GetVConstOffset(V3331)); - return; - } - src = i.dest; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - - // Splat W. - e.DUP(i.dest.reg().S4(), src.Selem()[3]); - // Keep only the needed components. - // Red in 0-9 now, green in 10-19, blue in 20-29, alpha in 30-31. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_MaskPacked)); - e.AND(i.dest.reg().B16(), i.dest.reg().B16(), Q1.B16()); - - // Shift the components down. - e.LDR(Q1, VConstData, e.GetVConstOffset(VPackUINT_2101010_Shift)); - e.NEG(Q1.S4(), Q1.S4()); - e.USHL(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - // If XYZ are negative, make smaller than 3 - sign extend XYZ before adding. - // W is unsigned. - e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 22); - e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 22); - // Add 3,3,3,1. - e.LDR(Q1, VConstData, e.GetVConstOffset(V3331)); - e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - // Return quiet NaNs in case of negative overflow. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackUINT_2101010_Overflow)); - e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); - e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); - e.MOV(i.dest.reg().B16(), Q0.B16()); - // To convert XYZ to -1 to 1, games multiply by 0x46004020 & sub 0x46C06030. - // For W to 0 to 1, they multiply by and subtract 0x4A2AAAAB.} - } - static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { - const XReg VConstData = X3; - e.MOV(VConstData, e.GetVConstPtr()); - - QReg src(0); - if (i.src1.is_constant) { - if (i.src1.value->IsConstantZero()) { - e.LDR(i.dest, VConstData, e.GetVConstOffset(V3331)); - return; - } - src = i.dest; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - // Extract pairs of nibbles to XZYW. XZ will have excess 4 upper bits, YW - // will have excess 4 lower bits. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackULONG_4202020_Permute)); - e.TBL(i.dest.reg().B16(), List{src.B16()}, Q1.B16()); - - // Drop the excess nibble of YW. - e.USHR(Q0.S4(), i.dest.reg().S4(), 4); - // Merge XZ and YW now both starting at offset 0. - e.LoadConstantV(Q1, vec128i(3 * 0x04'04'04'04 + 0x03'02'01'00, - 2 * 0x04'04'04'04 + 0x03'02'01'00, - 1 * 0x04'04'04'04 + 0x03'02'01'00, - 0 * 0x04'04'04'04 + 0x03'02'01'00)); - e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16(), Q0.B16()}, Q1.B16()); - - // Reorder as XYZW. - e.LoadConstantV(Q1, vec128i(3 * 0x04'04'04'04 + 0x03'02'01'00, - 1 * 0x04'04'04'04 + 0x03'02'01'00, - 2 * 0x04'04'04'04 + 0x03'02'01'00, - 0 * 0x04'04'04'04 + 0x03'02'01'00)); - e.TBL(i.dest.reg().B16(), List{i.dest.reg().B16(), Q0.B16()}, Q1.B16()); - // Drop the excess upper nibble in XZ and sign-extend XYZ. - e.SHL(i.dest.reg().S4(), i.dest.reg().S4(), 12); - e.SSHR(i.dest.reg().S4(), i.dest.reg().S4(), 12); - // Add 3,3,3,1. - e.LDR(Q1, VConstData, e.GetVConstOffset(V3331)); - e.ADD(i.dest.reg().S4(), i.dest.reg().S4(), Q1.S4()); - // Return quiet NaNs in case of negative overflow. - e.LDR(Q1, VConstData, e.GetVConstOffset(VUnpackULONG_4202020_Overflow)); - e.CMEQ(Q0.S4(), i.dest.reg().S4(), Q1.S4()); - - e.LDR(Q1, VConstData, e.GetVConstOffset(VQNaN)); - e.BSL(Q0.B16(), Q1.B16(), i.dest.reg().B16()); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { - assert_false(IsPackOutSaturate(flags)); - QReg src(0); - if (i.src1.is_constant) { - src = i.dest; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - if (IsPackToLo(flags)) { - // Unpack to LO. - if (IsPackInUnsigned(flags)) { - if (IsPackOutUnsigned(flags)) { - // unsigned -> unsigned - assert_always(); - } else { - // unsigned -> signed - assert_always(); - } - } else { - if (IsPackOutUnsigned(flags)) { - // signed -> unsigned - assert_always(); - } else { - // signed -> signed - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.SXTL2(i.dest.reg().H8(), i.dest.reg().B16()); - } - } - } else { - // Unpack to HI. - if (IsPackInUnsigned(flags)) { - if (IsPackOutUnsigned(flags)) { - // unsigned -> unsigned - assert_always(); - } else { - // unsigned -> signed - assert_always(); - } - } else { - if (IsPackOutUnsigned(flags)) { - // signed -> unsigned - assert_always(); - } else { - // signed -> signed - e.REV32(i.dest.reg().H8(), i.dest.reg().H8()); - e.SXTL(i.dest.reg().H8(), i.dest.reg().toD().B8()); - } - } - } - } - static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, - uint32_t flags) { - assert_false(IsPackOutSaturate(flags)); - QReg src(0); - if (i.src1.is_constant) { - src = i.dest; - e.LoadConstantV(src, i.src1.constant()); - } else { - src = i.src1; - } - if (IsPackToLo(flags)) { - // Unpack to LO. - if (IsPackInUnsigned(flags)) { - if (IsPackOutUnsigned(flags)) { - // unsigned -> unsigned - assert_always(); - } else { - // unsigned -> signed - assert_always(); - } - } else { - if (IsPackOutUnsigned(flags)) { - // signed -> unsigned - assert_always(); - } else { - // signed -> signed - e.SXTL2(i.dest.reg().S4(), src.H8()); - } - } - } else { - // Unpack to HI. - if (IsPackInUnsigned(flags)) { - if (IsPackOutUnsigned(flags)) { - // unsigned -> unsigned - assert_always(); - } else { - // unsigned -> signed - assert_always(); - } - } else { - if (IsPackOutUnsigned(flags)) { - // signed -> unsigned - assert_always(); - } else { - // signed -> signed - e.SXTL(i.dest.reg().S4(), src.toD().H4()); - } - } - } - e.REV64(i.dest.reg().S4(), i.dest.reg().S4()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_UNPACK, UNPACK); - -namespace { -thread_local bool a64_njm_enabled = true; - -uint64_t SetNJMForwarder(void* raw_context, uint64_t value) { - (void)raw_context; - a64_njm_enabled = value != 0; - return 0; -} -} // namespace - -// ============================================================================ -// OPCODE_SET_NJM -// ============================================================================ -struct SET_NJM_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.CallNative(SetNJMForwarder, static_cast(i.src1.constant())); - return; - } - e.UXTB(W1, i.src1); - e.CallNativeSafe(reinterpret_cast(SetNJMForwarder)); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SET_NJM, SET_NJM_I8); - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc deleted file mode 100644 index ec0522658..000000000 --- a/src/xenia/cpu/backend/a64/a64_sequences.cc +++ /dev/null @@ -1,2851 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -// A note about vectors: -// Xenia represents vectors as xyzw pairs, with indices 0123. -// XMM registers are xyzw pairs with indices 3210, making them more like wzyx. -// This makes things somewhat confusing. It'd be nice to just shuffle the -// registers around on load/store, however certain operations require that -// data be in the right offset. -// Basically, this identity must hold: -// shuffle(vec, b00011011) -> {x,y,z,w} => {x,y,z,w} -// All indices and operations must respect that. -// -// Memory (big endian): -// [00 01 02 03] [04 05 06 07] [08 09 0A 0B] [0C 0D 0E 0F] (x, y, z, w) -// load into xmm register: -// [0F 0E 0D 0C] [0B 0A 09 08] [07 06 05 04] [03 02 01 00] (w, z, y, x) - -#include "xenia/cpu/backend/a64/a64_sequences.h" - -#include -#include "xenia/base/assert.h" -#include "xenia/base/clock.h" -#include "xenia/base/logging.h" -#include "xenia/base/string.h" -#include "xenia/base/threading.h" -#include "xenia/cpu/backend/a64/a64_emitter.h" -#include "xenia/cpu/backend/a64/a64_op.h" -#include "xenia/cpu/backend/a64/a64_tracers.h" -#include "xenia/cpu/backend/a64/a64_util.h" -#include "xenia/cpu/hir/hir_builder.h" -#include "xenia/cpu/processor.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -using namespace oaknut; - -// TODO(benvanik): direct usings. -using namespace xe::cpu; -using namespace xe::cpu::hir; - -using xe::cpu::hir::Instr; - -typedef bool (*SequenceSelectFn)(A64Emitter&, const Instr*); - -// ============================================================================ -// OPCODE_COMMENT -// ============================================================================ -struct COMMENT : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (IsTracingInstr()) { - auto str = reinterpret_cast(i.src1.value); - // TODO(benvanik): pass through. - // TODO(benvanik): don't just leak this memory. - auto str_copy = xe_strdup(str); - e.MOV(e.GetNativeParam(0), reinterpret_cast(str_copy)); - e.CallNative(reinterpret_cast(TraceString)); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_COMMENT, COMMENT); - -// ============================================================================ -// OPCODE_NOP -// ============================================================================ -struct NOP : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { e.NOP(); } -}; -EMITTER_OPCODE_TABLE(OPCODE_NOP, NOP); - -// ============================================================================ -// OPCODE_SOURCE_OFFSET -// ============================================================================ -struct SOURCE_OFFSET - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MarkSourceOffset(i.instr); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SOURCE_OFFSET, SOURCE_OFFSET); - -// ============================================================================ -// OPCODE_ASSIGN -// ============================================================================ -struct ASSIGN_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest, i.src1); - } -}; -struct ASSIGN_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTH(i.dest, i.src1); - } -}; -struct ASSIGN_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest, i.src1); - } -}; -struct ASSIGN_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest, i.src1); - } -}; -struct ASSIGN_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMOV(i.dest, i.src1); - } -}; -struct ASSIGN_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMOV(i.dest, i.src1); - } -}; -struct ASSIGN_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.LoadConstantV(i.dest.reg(), i.src1.constant()); - } else { - e.MOV(i.dest.reg().B16(), i.src1.reg().B16()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ASSIGN, ASSIGN_I8, ASSIGN_I16, ASSIGN_I32, - ASSIGN_I64, ASSIGN_F32, ASSIGN_F64, ASSIGN_V128); - -// ============================================================================ -// OPCODE_CAST -// ============================================================================ -struct CAST_I32_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMOV(i.dest, i.src1); - } -}; -struct CAST_I64_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMOV(i.dest, i.src1); - } -}; -struct CAST_F32_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMOV(i.dest, i.src1); - } -}; -struct CAST_F64_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FMOV(i.dest, i.src1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CAST, CAST_I32_F32, CAST_I64_F64, CAST_F32_I32, - CAST_F64_I64); - -// ============================================================================ -// OPCODE_ZERO_EXTEND -// ============================================================================ -struct ZERO_EXTEND_I16_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest, i.src1); - } -}; -struct ZERO_EXTEND_I32_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest, i.src1); - } -}; -struct ZERO_EXTEND_I64_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest.reg().toW(), i.src1); - } -}; -struct ZERO_EXTEND_I32_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTH(i.dest, i.src1); - } -}; -struct ZERO_EXTEND_I64_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTH(i.dest.reg().toW(), i.src1); - } -}; -struct ZERO_EXTEND_I64_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest.reg().toW(), i.src1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ZERO_EXTEND, ZERO_EXTEND_I16_I8, ZERO_EXTEND_I32_I8, - ZERO_EXTEND_I64_I8, ZERO_EXTEND_I32_I16, - ZERO_EXTEND_I64_I16, ZERO_EXTEND_I64_I32); - -// ============================================================================ -// OPCODE_SIGN_EXTEND -// ============================================================================ -struct SIGN_EXTEND_I16_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTB(i.dest, i.src1); - } -}; -struct SIGN_EXTEND_I32_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTB(i.dest, i.src1); - } -}; -struct SIGN_EXTEND_I64_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTB(i.dest, i.src1); - } -}; -struct SIGN_EXTEND_I32_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTH(i.dest, i.src1); - } -}; -struct SIGN_EXTEND_I64_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTH(i.dest, i.src1); - } -}; -struct SIGN_EXTEND_I64_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SXTW(i.dest, i.src1.reg()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SIGN_EXTEND, SIGN_EXTEND_I16_I8, SIGN_EXTEND_I32_I8, - SIGN_EXTEND_I64_I8, SIGN_EXTEND_I32_I16, - SIGN_EXTEND_I64_I16, SIGN_EXTEND_I64_I32); - -// ============================================================================ -// OPCODE_TRUNCATE -// ============================================================================ -struct TRUNCATE_I8_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest, i.src1); - } -}; -struct TRUNCATE_I8_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest, i.src1); - } -}; -struct TRUNCATE_I8_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTB(i.dest, i.src1.reg().toW()); - } -}; -struct TRUNCATE_I16_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTH(i.dest, i.src1); - } -}; -struct TRUNCATE_I16_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.UXTH(i.dest, i.src1.reg().toW()); - } -}; -struct TRUNCATE_I32_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.MOV(i.dest, i.src1.reg().toW()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_TRUNCATE, TRUNCATE_I8_I16, TRUNCATE_I8_I32, - TRUNCATE_I8_I64, TRUNCATE_I16_I32, TRUNCATE_I16_I64, - TRUNCATE_I32_I64); - -// ============================================================================ -// OPCODE_CONVERT -// ============================================================================ -struct CONVERT_I32_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): saturation check? cvtt* (trunc?) - const SReg src = i.src1.is_constant ? S0 : i.src1.reg().toS(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - if (i.instr->flags == ROUND_TO_ZERO) { - e.FCVTZS(i.dest, src); - } else { - e.FCVTNS(i.dest, src); - } - } -}; -struct CONVERT_I32_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // Intel returns 0x80000000 if the double value does not fit within an int32 - // ARM64 and PPC saturates the value instead - const DReg src = i.src1.is_constant ? D0 : i.src1.reg().toD(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - if (i.instr->flags == ROUND_TO_ZERO) { - e.FCVTZS(i.dest, src); - } else { - e.FCVTNS(i.dest, src); - } - } -}; -struct CONVERT_I64_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const DReg src = i.src1.is_constant ? D0 : i.src1.reg().toD(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - if (i.instr->flags == ROUND_TO_ZERO) { - e.FCVTZS(i.dest, src); - } else { - e.FCVTNS(i.dest, src); - } - } -}; -struct CONVERT_F32_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SCVTF(i.dest.reg().toS(), i.src1); - } -}; -struct CONVERT_F32_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const DReg src = i.src1.is_constant ? D0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src.toQ(), i.src1.constant()); - } - e.FCVT(i.dest.reg().toS(), src.toD()); - } -}; -struct CONVERT_F64_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.SCVTF(i.dest.reg().toD(), i.src1); - } -}; -struct CONVERT_F64_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const SReg src = i.src1.is_constant ? S0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src.toQ(), i.src1.constant()); - } - e.FCVT(i.dest.reg().toD(), src.toS()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CONVERT, CONVERT_I32_F32, CONVERT_I32_F64, - CONVERT_I64_F64, CONVERT_F32_I32, CONVERT_F32_F64, - CONVERT_F64_I64, CONVERT_F64_F32); - -// ============================================================================ -// OPCODE_TO_SINGLE -// ============================================================================ -struct TOSINGLE_F64_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const DReg src = i.src1.is_constant ? D1 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(src.toQ(), i.src1.constant()); - } - e.FCVT(S0, src); - e.FCVT(i.dest, S0); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_TO_SINGLE, TOSINGLE_F64_F64); - -// ============================================================================ -// OPCODE_ROUND -// ============================================================================ -struct ROUND_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const SReg src = i.src1.is_constant ? S0 : i.src1.reg().toS(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - switch (i.instr->flags) { - case ROUND_TO_ZERO: - e.FRINTZ(i.dest.reg().toS(), src); - break; - case ROUND_TO_NEAREST: - e.FRINTN(i.dest.reg().toS(), src); - break; - case ROUND_TO_MINUS_INFINITY: - e.FRINTM(i.dest.reg().toS(), src); - break; - case ROUND_TO_POSITIVE_INFINITY: - e.FRINTP(i.dest.reg().toS(), src); - break; - } - } -}; -struct ROUND_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const DReg src = i.src1.is_constant ? D0 : i.src1; - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - switch (i.instr->flags) { - case ROUND_TO_ZERO: - e.FRINTZ(i.dest, src); - break; - case ROUND_TO_NEAREST: - e.FRINTN(i.dest, src); - break; - case ROUND_TO_MINUS_INFINITY: - e.FRINTM(i.dest, src); - break; - case ROUND_TO_POSITIVE_INFINITY: - e.FRINTP(i.dest, src); - break; - } - } -}; -struct ROUND_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - switch (i.instr->flags) { - case ROUND_TO_ZERO: - e.FRINTZ(i.dest.reg().S4(), src.S4()); - break; - case ROUND_TO_NEAREST: - e.FRINTN(i.dest.reg().S4(), src.S4()); - break; - case ROUND_TO_MINUS_INFINITY: - e.FRINTM(i.dest.reg().S4(), src.S4()); - break; - case ROUND_TO_POSITIVE_INFINITY: - e.FRINTP(i.dest.reg().S4(), src.S4()); - break; - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ROUND, ROUND_F32, ROUND_F64, ROUND_V128); - -// ============================================================================ -// OPCODE_LOAD_CLOCK -// ============================================================================ -struct LOAD_CLOCK : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // When scaling is disabled and the raw clock source is selected, the code - // in the Clock class is actually just forwarding tick counts after one - // simple multiply and division. In that case we rather bake the scaling in - // here to cut extra function calls with CPU cache misses and stack frame - // overhead. - if (cvars::clock_no_scaling && cvars::clock_source_raw) { - auto ratio = Clock::guest_tick_ratio(); - // The 360 CPU is an in-order CPU, ARM64 usually isn't. Since it's - // resolution however is much higher than the 360's mftb instruction this - // can safely be ignored. - - // Read clock cycle count - e.MRS(i.dest, SystemReg::CNTVCT_EL0); - // Apply tick frequency scaling. - e.MOV(X0, ratio.first); - e.MUL(i.dest, i.dest, X0); - e.MOV(X0, ratio.second); - e.UDIV(i.dest, i.dest, X0); - } else { - e.CallNative(LoadClock); - e.MOV(i.dest, X0); - } - } - static uint64_t LoadClock(void* raw_context) { - return Clock::QueryGuestTickCount(); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOAD_CLOCK, LOAD_CLOCK); - -// ============================================================================ -// OPCODE_CONTEXT_BARRIER -// ============================================================================ -struct CONTEXT_BARRIER - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) {} -}; -EMITTER_OPCODE_TABLE(OPCODE_CONTEXT_BARRIER, CONTEXT_BARRIER); - -// ============================================================================ -// OPCODE_MAX -// ============================================================================ -struct MAX_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FMAX(dest, src1, src2); - }); - } -}; -struct MAX_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FMAX(dest, src1, src2); - }); - } -}; -struct MAX_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMAX(dest.S4(), src1.S4(), src2.S4()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MAX, MAX_F32, MAX_F64, MAX_V128); - -// ============================================================================ -// OPCODE_MIN -// ============================================================================ -struct MIN_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.CMP(dest_src, src); - e.CSEL(dest_src, dest_src, src, Cond::LO); - }, - [](A64Emitter& e, WReg dest_src, int32_t constant) { - e.MOV(W0, constant); - e.CMP(dest_src, W0); - e.CSEL(dest_src, dest_src, W0, Cond::LO); - }); - } -}; -struct MIN_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.CMP(dest_src, src); - e.CSEL(dest_src, dest_src, src, Cond::LO); - }, - [](A64Emitter& e, WReg dest_src, int32_t constant) { - e.MOV(W0, constant); - e.CMP(dest_src, W0); - e.CSEL(dest_src, dest_src, W0, Cond::LO); - }); - } -}; -struct MIN_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.CMP(dest_src, src); - e.CSEL(dest_src, dest_src, src, Cond::LO); - }, - [](A64Emitter& e, WReg dest_src, int32_t constant) { - e.MOV(W0, constant); - e.CMP(dest_src, W0); - e.CSEL(dest_src, dest_src, W0, Cond::LO); - }); - } -}; -struct MIN_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, XReg dest_src, XReg src) { - e.CMP(dest_src, src); - e.CSEL(dest_src, dest_src, src, Cond::LO); - }, - [](A64Emitter& e, XReg dest_src, int64_t constant) { - e.MOV(X0, constant); - e.CMP(dest_src, X0); - e.CSEL(dest_src, dest_src, X0, Cond::LO); - }); - } -}; -struct MIN_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FMIN(dest, src1, src2); - }); - } -}; -struct MIN_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FMIN(dest, src1, src2); - }); - } -}; -struct MIN_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMIN(dest.S4(), src1.S4(), src2.S4()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MIN, MIN_I8, MIN_I16, MIN_I32, MIN_I64, MIN_F32, - MIN_F64, MIN_V128); - -// ============================================================================ -// OPCODE_SELECT -// ============================================================================ -// dest = src1 ? src2 : src3 -// TODO(benvanik): match compare + select sequences, as often it's something -// like SELECT(VECTOR_COMPARE_SGE(a, b), a, b) -struct SELECT_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - WReg src2(0); - if (i.src2.is_constant) { - src2 = W0; - e.MOV(src2, i.src2.constant()); - } else { - src2 = i.src2; - } - e.CMP(i.src1.reg().toX(), 0); - e.CSEL(i.dest, src2, i.src3, Cond::NE); - } -}; -struct SELECT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - WReg src2(0); - if (i.src2.is_constant) { - src2 = W0; - e.MOV(src2, i.src2.constant()); - } else { - src2 = i.src2; - } - e.CMP(i.src1.reg().toX(), 0); - e.CSEL(i.dest, src2, i.src3, Cond::NE); - } -}; -struct SELECT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - WReg src2(0); - if (i.src2.is_constant) { - src2 = W0; - e.MOV(src2, i.src2.constant()); - } else { - src2 = i.src2; - } - e.CMP(i.src1.reg().toX(), 0); - e.CSEL(i.dest, src2, i.src3, Cond::NE); - } -}; -struct SELECT_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - XReg src2(0); - if (i.src2.is_constant) { - src2 = X0; - e.MOV(src2, i.src2.constant()); - } else { - src2 = i.src2; - } - e.CMP(i.src1.reg().toX(), 0); - e.CSEL(i.dest, src2, i.src3, Cond::NE); - } -}; -struct SELECT_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // dest = src1 != 0 ? src2 : src3 - - SReg src2 = i.src2.is_constant ? S2 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2.toQ(), i.src2.constant()); - } - - SReg src3 = i.src3.is_constant ? S3 : i.src3; - if (i.src3.is_constant) { - e.LoadConstantV(src3.toQ(), i.src3.constant()); - } - - e.CMP(i.src1.reg().toX(), 0); - e.FCSEL(i.dest, src2, src3, Cond::NE); - } -}; -struct SELECT_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // dest = src1 != 0 ? src2 : src3 - - const DReg src2 = i.src2.is_constant ? D2 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2.toQ(), i.src2.constant()); - } - - const DReg src3 = i.src3.is_constant ? D3 : i.src3; - if (i.src3.is_constant) { - e.LoadConstantV(src3.toQ(), i.src3.constant()); - } - - e.CMP(i.src1.reg().toX(), 0); - e.FCSEL(i.dest, src2, src3, Cond::NE); - } -}; -struct SELECT_V128_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // dest = src1 != 0 ? src2 : src3 - - const QReg src2 = i.src2.is_constant ? Q2 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2, i.src2.constant()); - } - - const QReg src3 = i.src3.is_constant ? Q3 : i.src3; - if (i.src3.is_constant) { - e.LoadConstantV(src3, i.src3.constant()); - } - - e.CMP(i.src1.reg().toX(), 0); - e.CSETM(W0, Cond::NE); - e.DUP(i.dest.reg().S4(), W0); - e.BSL(i.dest.reg().B16(), src2.B16(), src3.B16()); - } -}; -struct SELECT_V128_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const QReg src1 = Q0; - if (i.src1.is_constant) { - e.LoadConstantV(src1, i.src1.constant()); - } else { - e.MOV(src1.B16(), i.src1.reg().B16()); - } - - const QReg src2 = i.src2.is_constant ? Q2 : i.src2; - if (i.src2.is_constant) { - e.LoadConstantV(src2, i.src2.constant()); - } - - const QReg src3 = i.src3.is_constant ? Q3 : i.src3; - if (i.src3.is_constant) { - e.LoadConstantV(src3, i.src3.constant()); - } - - // src1 ? src2 : src3; - e.BSL(src1.B16(), src3.B16(), src2.B16()); - e.MOV(i.dest.reg().B16(), src1.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SELECT, SELECT_I8, SELECT_I16, SELECT_I32, - SELECT_I64, SELECT_F32, SELECT_F64, SELECT_V128_I8, - SELECT_V128_V128); - -// ============================================================================ -// OPCODE_IS_NAN -// ============================================================================ -struct IS_NAN_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1, i.src1); - e.CSET(i.dest, Cond::VS); - } -}; - -struct IS_NAN_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1, i.src1); - e.CSET(i.dest, Cond::VS); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_IS_NAN, IS_NAN_F32, IS_NAN_F64); - -// ============================================================================ -// OPCODE_COMPARE_EQ -// ============================================================================ -struct COMPARE_EQ_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, WReg src1, int32_t constant) { - e.MOV(W1, constant); - e.CMP(src1, W1); - }); - e.CSET(i.dest, Cond::EQ); - } -}; -struct COMPARE_EQ_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, WReg src1, int32_t constant) { - e.MOV(W1, constant); - e.CMP(src1, W1); - }); - e.CSET(i.dest, Cond::EQ); - } -}; -struct COMPARE_EQ_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, WReg src1, int32_t constant) { - e.MOV(W1, constant); - e.CMP(src1, W1); - }); - e.CSET(i.dest, Cond::EQ); - } -}; -struct COMPARE_EQ_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, XReg src1, XReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, XReg src1, int32_t constant) { - e.MOV(X1, constant); - e.CMP(src1, X1); - }); - e.CSET(i.dest, Cond::EQ); - } -}; -struct COMPARE_EQ_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, I8Op dest, const SReg& src1, const SReg& src2) { - e.FCMP(src1, src2); - }); - e.CSET(i.dest, Cond::EQ); - } -}; -struct COMPARE_EQ_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, I8Op dest, const DReg& src1, const DReg& src2) { - e.FCMP(src1, src2); - }); - e.CSET(i.dest, Cond::EQ); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_COMPARE_EQ, COMPARE_EQ_I8, COMPARE_EQ_I16, - COMPARE_EQ_I32, COMPARE_EQ_I64, COMPARE_EQ_F32, - COMPARE_EQ_F64); - -// ============================================================================ -// OPCODE_COMPARE_NE -// ============================================================================ -struct COMPARE_NE_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, WReg src1, int32_t constant) { - e.MOV(W1, constant); - e.CMP(src1, W1); - }); - e.CSET(i.dest, Cond::NE); - } -}; -struct COMPARE_NE_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, WReg src1, int32_t constant) { - e.MOV(W1, constant); - e.CMP(src1, W1); - }); - e.CSET(i.dest, Cond::NE); - } -}; -struct COMPARE_NE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, WReg src1, WReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, WReg src1, int32_t constant) { - e.MOV(W1, constant); - e.CMP(src1, W1); - }); - e.CSET(i.dest, Cond::NE); - } -}; -struct COMPARE_NE_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeCompareOp( - e, i, [](A64Emitter& e, XReg src1, XReg src2) { e.CMP(src1, src2); }, - [](A64Emitter& e, XReg src1, int32_t constant) { - e.MOV(X1, constant); - e.CMP(src1, X1); - }); - e.CSET(i.dest, Cond::NE); - } -}; -struct COMPARE_NE_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1, i.src2); - e.CSET(i.dest, Cond::NE); - } -}; -struct COMPARE_NE_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FCMP(i.src1, i.src2); - e.CSET(i.dest, Cond::NE); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_COMPARE_NE, COMPARE_NE_I8, COMPARE_NE_I16, - COMPARE_NE_I32, COMPARE_NE_I64, COMPARE_NE_F32, - COMPARE_NE_F64); - -// ============================================================================ -// OPCODE_COMPARE_* -// ============================================================================ -#define EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, type, \ - reg_type) \ - struct COMPARE_##op##_##type \ - : Sequence> { \ - static void Emit(A64Emitter& e, const EmitArgType& i) { \ - EmitAssociativeCompareOp( \ - e, i, \ - [](A64Emitter& e, WReg dest, const reg_type& src1, \ - const reg_type& src2, bool inverse) { \ - e.CMP(src1, src2); \ - if (!inverse) { \ - e.CSET(dest, cond); \ - } else { \ - e.CSET(dest, inverse_cond); \ - } \ - }, \ - [](A64Emitter& e, WReg dest, const reg_type& src1, int32_t constant, \ - bool inverse) { \ - e.MOV(reg_type(1), constant); \ - e.CMP(src1, reg_type(1)); \ - if (!inverse) { \ - e.CSET(dest, cond); \ - } else { \ - e.CSET(dest, inverse_cond); \ - } \ - }); \ - } \ - }; -#define EMITTER_ASSOCIATIVE_COMPARE_XX(op, cond, inverse_cond) \ - EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I8Op, WReg); \ - EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I16Op, WReg); \ - EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I32Op, WReg); \ - EMITTER_ASSOCIATIVE_COMPARE_INT(op, cond, inverse_cond, I64Op, XReg); \ - EMITTER_OPCODE_TABLE(OPCODE_COMPARE_##op, COMPARE_##op##_I8Op, \ - COMPARE_##op##_I16Op, COMPARE_##op##_I32Op, \ - COMPARE_##op##_I64Op); -EMITTER_ASSOCIATIVE_COMPARE_XX(SLT, Cond::LT, Cond::GT); // setl, setg -EMITTER_ASSOCIATIVE_COMPARE_XX(SLE, Cond::LE, Cond::GE); // setle, setge -EMITTER_ASSOCIATIVE_COMPARE_XX(SGT, Cond::GT, Cond::LT); // setg, setl -EMITTER_ASSOCIATIVE_COMPARE_XX(SGE, Cond::GE, Cond::LE); // setge, setle -EMITTER_ASSOCIATIVE_COMPARE_XX(ULT, Cond::LO, Cond::HI); // setb, seta -EMITTER_ASSOCIATIVE_COMPARE_XX(ULE, Cond::LS, Cond::HS); // setbe, setae -EMITTER_ASSOCIATIVE_COMPARE_XX(UGE, Cond::HS, Cond::LS); // setae, setbe -EMITTER_ASSOCIATIVE_COMPARE_XX(UGT, Cond::HI, Cond::LO); // seta, setb - -// https://web.archive.org/web/20171129015931/https://x86.renejeschke.de/html/file_module_x86_id_288.html -// Original link: https://x86.renejeschke.de/html/file_module_x86_id_288.html -#define EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(op, cond) \ - struct COMPARE_##op##_F32 \ - : Sequence> { \ - static void Emit(A64Emitter& e, const EmitArgType& i) { \ - e.FCMP(i.src1, i.src2); \ - e.CSET(i.dest, cond); \ - } \ - }; \ - struct COMPARE_##op##_F64 \ - : Sequence> { \ - static void Emit(A64Emitter& e, const EmitArgType& i) { \ - if (i.src1.is_constant) { \ - e.LoadConstantV(Q0, i.src1.constant()); \ - e.FCMP(D0, i.src2); \ - } else if (i.src2.is_constant) { \ - e.LoadConstantV(Q0, i.src2.constant()); \ - e.FCMP(i.src1, D0); \ - } else { \ - e.FCMP(i.src1, i.src2); \ - } \ - e.CSET(i.dest, cond); \ - } \ - }; \ - EMITTER_OPCODE_TABLE(OPCODE_COMPARE_##op##_FLT, COMPARE_##op##_F32, \ - COMPARE_##op##_F64); -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SLT, Cond::LT); // setb -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SLE, Cond::LE); // setbe -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SGT, Cond::GT); // seta -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(SGE, Cond::GE); // setae -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(ULT, Cond::LO); // setb -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(ULE, Cond::LS); // setbe -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(UGT, Cond::HI); // seta -EMITTER_ASSOCIATIVE_COMPARE_FLT_XX(UGE, Cond::HS); // setae - -// ============================================================================ -// OPCODE_DID_SATURATE -// ============================================================================ -struct DID_SATURATE - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // Bit 27 in the FPSR is the QC bit - e.MRS(X0, SystemReg::FPSR); - e.UBFX(i.dest, W0, 27, 1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_DID_SATURATE, DID_SATURATE); - -// ============================================================================ -// OPCODE_ADD -// ============================================================================ -// TODO(benvanik): put dest/src1|2 together. -template -void EmitAddXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.ADD(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int32_t constant) { - e.MOV(REG(1), constant); - e.ADD(dest_src, dest_src, REG(1)); - }); -} -struct ADD_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddXX(e, i); - } -}; -struct ADD_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddXX(e, i); - } -}; -struct ADD_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddXX(e, i); - } -}; -struct ADD_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddXX(e, i); - } -}; -struct ADD_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FADD(dest, src1, src2); - }); - } -}; -struct ADD_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FADD(dest, src1, src2); - }); - } -}; -struct ADD_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FADD(dest.S4(), src1.S4(), src2.S4()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ADD, ADD_I8, ADD_I16, ADD_I32, ADD_I64, ADD_F32, - ADD_F64, ADD_V128); - -// ============================================================================ -// OPCODE_ADD_CARRY -// ============================================================================ -// TODO(benvanik): put dest/src1|2 together. -template -void EmitAddCarryXX(A64Emitter& e, const ARGS& i) { - // TODO(benvanik): faster setting? we could probably do some fun math tricks - // here to get the carry flag set. - if (i.src3.is_constant) { - e.MOV(W0, WZR); - if (i.src3.constant()) { - // Set carry - // This is implicitly "SUBS 0 - 0" - e.CMP(W0, 0); - } else { - // Clear carry - e.CMN(W0, 0); - } - } else { - // If src3 is non-zero, set the carry flag - e.CMP(i.src3.reg().toW(), 0); - e.CSET(X0, Cond::NE); - - e.MRS(X1, SystemReg::NZCV); - // Assign carry bit - e.BFI(X1, X0, 29, 1); - e.MSR(SystemReg::NZCV, X1); - } - SEQ::EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, const REG& dest_src, const REG& src) { - e.ADC(dest_src, dest_src, src); - }, - [](A64Emitter& e, const REG& dest_src, int32_t constant) { - e.MOV(REG(1), constant); - e.ADC(dest_src, dest_src, REG(1)); - }); -} -struct ADD_CARRY_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddCarryXX(e, i); - } -}; -struct ADD_CARRY_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddCarryXX(e, i); - } -}; -struct ADD_CARRY_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddCarryXX(e, i); - } -}; -struct ADD_CARRY_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAddCarryXX(e, i); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ADD_CARRY, ADD_CARRY_I8, ADD_CARRY_I16, - ADD_CARRY_I32, ADD_CARRY_I64); - -// ============================================================================ -// OPCODE_SUB -// ============================================================================ -// TODO(benvanik): put dest/src1|2 together. -template -void EmitSubXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.SUB(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int32_t constant) { - e.MOV(REG(1), constant); - e.SUB(dest_src, dest_src, REG(1)); - }); -} -struct SUB_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSubXX(e, i); - } -}; -struct SUB_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSubXX(e, i); - } -}; -struct SUB_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSubXX(e, i); - } -}; -struct SUB_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitSubXX(e, i); - } -}; -struct SUB_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitAssociativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FSUB(dest, src1, src2); - }); - } -}; -struct SUB_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitAssociativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FSUB(dest, src1, src2); - }); - } -}; -struct SUB_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitAssociativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FSUB(dest.S4(), src1.S4(), src2.S4()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SUB, SUB_I8, SUB_I16, SUB_I32, SUB_I64, SUB_F32, - SUB_F64, SUB_V128); - -// ============================================================================ -// OPCODE_MUL -// ============================================================================ -// Sign doesn't matter here, as we don't use the high bits. -// We exploit mulx here to avoid creating too much register pressure. -struct MUL_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - } -}; -struct MUL_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - } -}; -struct MUL_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - } -}; -struct MUL_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(X0, i.src1.constant()); - e.MUL(i.dest, X0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(X0, i.src2.constant()); - e.MUL(i.dest, i.src1, X0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - } -}; -struct MUL_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FMUL(dest, src1, src2); - }); - } -}; -struct MUL_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FMUL(dest, src1, src2); - }); - } -}; -struct MUL_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMUL(dest.S4(), src1.S4(), src2.S4()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MUL, MUL_I8, MUL_I16, MUL_I32, MUL_I64, MUL_F32, - MUL_F64, MUL_V128); - -// ============================================================================ -// OPCODE_MUL_HI -// ============================================================================ -struct MUL_HI_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - e.UBFX(i.dest, i.dest, 8, 8); - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - e.SBFX(i.dest, i.dest, 8, 8); - } - } -}; -struct MUL_HI_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - e.UBFX(i.dest, i.dest, 16, 16); - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.MUL(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.MUL(i.dest, i.src1, W0); - } else { - e.MUL(i.dest, i.src1, i.src2); - } - e.SBFX(i.dest, i.dest, 16, 16); - } - } -}; -struct MUL_HI_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.UMULL(X0, W0, i.src2); - e.UBFX(X0, X0, 32, 32); - e.MOV(i.dest, X0.toW()); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.UMULL(X0, W0, i.src1); - e.UBFX(X0, X0, 32, 32); - e.MOV(i.dest, X0.toW()); - } else { - e.UMULL(X0, i.src1, i.src2); - e.UBFX(X0, X0, 32, 32); - e.MOV(i.dest, X0.toW()); - } - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.SMULL(X0, W0, i.src2); - e.SBFX(X0, X0, 32, 32); - e.MOV(i.dest, X0.toW()); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.SMULL(X0, W0, i.src1); - e.SBFX(X0, X0, 32, 32); - e.MOV(i.dest, X0.toW()); - } else { - e.SMULL(X0, i.src1, i.src2); - e.SBFX(X0, X0, 32, 32); - e.MOV(i.dest, X0.toW()); - } - } - } -}; -struct MUL_HI_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(X0, i.src1.constant()); - e.UMULH(i.dest, X0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(X0, i.src2.constant()); - e.UMULH(i.dest, i.src1, X0); - } else { - e.UMULH(i.dest, i.src1, i.src2); - } - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(X0, i.src1.constant()); - e.SMULH(i.dest, X0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(X0, i.src2.constant()); - e.SMULH(i.dest, i.src1, X0); - } else { - e.SMULH(i.dest, i.src1, i.src2); - } - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MUL_HI, MUL_HI_I8, MUL_HI_I16, MUL_HI_I32, - MUL_HI_I64); - -// ============================================================================ -// OPCODE_DIV -// ============================================================================ -struct DIV_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.UDIV(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.UDIV(i.dest, i.src1, W0); - } else { - e.UDIV(i.dest, i.src1, i.src2); - } - e.UXTB(i.dest, i.dest); - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.SDIV(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.SDIV(i.dest, i.src1, W0); - } else { - e.SDIV(i.dest, i.src1, i.src2); - } - e.SXTB(i.dest, i.dest); - } - } -}; -struct DIV_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.UDIV(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.UDIV(i.dest, i.src1, W0); - } else { - e.UDIV(i.dest, i.src1, i.src2); - } - e.UXTH(i.dest, i.dest); - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.SDIV(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.SDIV(i.dest, i.src1, W0); - } else { - e.SDIV(i.dest, i.src1, i.src2); - } - e.SXTH(i.dest, i.dest); - } - } -}; -struct DIV_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.UDIV(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.UDIV(i.dest, i.src1, W0); - } else { - e.UDIV(i.dest, i.src1, i.src2); - } - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(W0, i.src1.constant()); - e.SDIV(i.dest, W0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(W0, i.src2.constant()); - e.SDIV(i.dest, i.src1, W0); - } else { - e.SDIV(i.dest, i.src1, i.src2); - } - } - } -}; -struct DIV_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.instr->flags & ARITHMETIC_UNSIGNED) { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(X0, i.src1.constant()); - e.UDIV(i.dest, X0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(X0, i.src2.constant()); - e.UDIV(i.dest, i.src1, X0); - } else { - e.UDIV(i.dest, i.src1, i.src2); - } - } else { - if (i.src1.is_constant) { - assert_true(!i.src2.is_constant); - e.MOV(X0, i.src1.constant()); - e.SDIV(i.dest, X0, i.src2); - } else if (i.src2.is_constant) { - assert_true(!i.src1.is_constant); - e.MOV(X0, i.src2.constant()); - e.SDIV(i.dest, i.src1, X0); - } else { - e.SDIV(i.dest, i.src1, i.src2); - } - } - } -}; -struct DIV_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitAssociativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FDIV(dest, src1, src2); - }); - } -}; -struct DIV_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitAssociativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FDIV(dest, src1, src2); - }); - } -}; -struct DIV_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - EmitAssociativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FDIV(dest.S4(), src1.S4(), src2.S4()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_DIV, DIV_I8, DIV_I16, DIV_I32, DIV_I64, DIV_F32, - DIV_F64, DIV_V128); - -// ============================================================================ -// OPCODE_MUL_ADD -// ============================================================================ -// d = 1 * 2 + 3 -// $0 = $1x$0 + $2 -struct MUL_ADD_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - SReg src3 = S3; - if (i.src3.is_constant) { - e.LoadConstantV(src3.toQ(), i.src3.constant()); - } else { - src3 = i.src3.reg(); - } - - SReg src2 = S2; - if (i.src2.is_constant) { - e.LoadConstantV(src2.toQ(), i.src2.constant()); - } else { - src2 = i.src2.reg(); - } - - SReg src1 = S1; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - - e.FMADD(i.dest, src1, src2, src3); - } -}; -struct MUL_ADD_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - DReg src3 = D3; - if (i.src3.is_constant) { - e.LoadConstantV(src3.toQ(), i.src3.constant()); - } else { - src3 = i.src3.reg(); - } - - DReg src2 = D2; - if (i.src2.is_constant) { - e.LoadConstantV(src2.toQ(), i.src2.constant()); - } else { - src2 = i.src2.reg(); - } - - DReg src1 = D1; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - - e.FMADD(i.dest, src1, src2, src3); - } -}; -struct MUL_ADD_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - const QReg dest = i.dest.reg(); - - // Always use temporary registers to avoid conflicts - QReg temp1 = Q0; - QReg temp2 = Q1; - QReg temp3 = Q2; - - if (i.src1.is_constant) { - e.LoadConstantV(temp1, i.src1.constant()); - } else { - e.MOV(temp1.B16(), i.src1.reg().B16()); - } - - if (i.src2.is_constant) { - e.LoadConstantV(temp2, i.src2.constant()); - } else { - e.MOV(temp2.B16(), i.src2.reg().B16()); - } - - if (i.src3.is_constant) { - e.LoadConstantV(temp3, i.src3.constant()); - } else { - e.MOV(temp3.B16(), i.src3.reg().B16()); - } - - // First multiply: dest = temp1 * temp2 - e.FMUL(dest.S4(), temp1.S4(), temp2.S4()); - - // Then add: dest = dest + temp3 - e.FADD(dest.S4(), dest.S4(), temp3.S4()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MUL_ADD, MUL_ADD_F32, MUL_ADD_F64, MUL_ADD_V128); - -// ============================================================================ -// OPCODE_MUL_SUB -// ============================================================================ -// d = 1 * 2 - 3 -// $0 = $2x$0 - $3 -// TODO(benvanik): use other forms (132/213/etc) to avoid register shuffling. -// dest could be src2 or src3 - need to ensure it's not before overwriting dest -// perhaps use other 132/213/etc -// Forms: -// - 132 -> $1 = $1 * $3 - $2 -// - 213 -> $1 = $2 * $1 - $3 -// - 231 -> $1 = $2 * $3 - $1 -struct MUL_SUB_F32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - SReg src3(1); - if (i.src3.is_constant) { - src3 = S1; - e.LoadConstantV(src3.toQ(), i.src3.constant()); - } else { - // If i.dest == i.src3, back up i.src3 so we don't overwrite it. - src3 = i.src3.reg(); - if (i.dest.reg().index() == i.src3.reg().index()) { - e.FMOV(S1, i.src3); - src3 = S1; - } - } - - // Multiply operation is commutative. - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, SReg dest, SReg src1, SReg src2) { - e.FMUL(dest, src1, src2); // $0 = $1 * $2 - }); - - e.FSUB(i.dest, i.dest, src3); // $0 = $1 - $2 - } -}; -struct MUL_SUB_F64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - DReg src3(1); - if (i.src3.is_constant) { - src3 = D1; - e.LoadConstantV(src3.toQ(), i.src3.constant()); - } else { - // If i.dest == i.src3, back up i.src3 so we don't overwrite it. - src3 = i.src3.reg(); - if (i.dest.reg().index() == i.src3.reg().index()) { - e.FMOV(D1, i.src3); - src3 = D1; - } - } - - // Multiply operation is commutative. - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, DReg dest, DReg src1, DReg src2) { - e.FMUL(dest, src1, src2); // $0 = $1 * $2 - }); - - e.FSUB(i.dest, i.dest, src3); // $0 = $1 + $2 - } -}; -struct MUL_SUB_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - QReg src3(1); - if (i.src3.is_constant) { - src3 = Q1; - e.LoadConstantV(src3, i.src3.constant()); - } else { - // If i.dest == i.src3, back up i.src3 so we don't overwrite it. - src3 = i.src3; - if (i.dest == i.src3) { - e.MOV(Q1.B16(), i.src3.reg().B16()); - src3 = Q1; - } - } - - // Multiply operation is commutative. - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMUL(dest.S4(), src1.S4(), src2.S4()); // $0 = $1 * $2 - }); - - e.FSUB(i.dest.reg().S4(), i.dest.reg().S4(), src3.S4()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_MUL_SUB, MUL_SUB_F32, MUL_SUB_F64, MUL_SUB_V128); - -// ============================================================================ -// OPCODE_NEG -// ============================================================================ -// TODO(benvanik): put dest/src1 together. -template -void EmitNegXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitUnaryOp( - e, i, [](A64Emitter& e, REG dest_src) { e.NEG(dest_src, dest_src); }); -} -struct NEG_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNegXX(e, i); - } -}; -struct NEG_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNegXX(e, i); - } -}; -struct NEG_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNegXX(e, i); - } -}; -struct NEG_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNegXX(e, i); - } -}; -struct NEG_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FNEG(i.dest, i.src1); - } -}; -struct NEG_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FNEG(i.dest, i.src1); - } -}; -struct NEG_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_true(!i.instr->flags); - QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - e.FNEG(i.dest.reg().S4(), src.S4()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_NEG, NEG_I8, NEG_I16, NEG_I32, NEG_I64, NEG_F32, - NEG_F64, NEG_V128); - -// ============================================================================ -// OPCODE_ABS -// ============================================================================ -struct ABS_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FABS(i.dest, i.src1); - } -}; -struct ABS_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.FABS(i.dest, i.src1); - } -}; -struct ABS_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - e.FABS(i.dest.reg().S4(), src.S4()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ABS, ABS_F32, ABS_F64, ABS_V128); - -// ============================================================================ -// OPCODE_SQRT -// ============================================================================ -struct SQRT_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - SReg src1 = S0; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - e.FSQRT(i.dest, src1); - } -}; -struct SQRT_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - DReg src1 = D0; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - e.FSQRT(i.dest, src1); - } -}; -struct SQRT_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - e.FSQRT(i.dest.reg().S4(), Q0.S4()); - } else { - e.FSQRT(i.dest.reg().S4(), i.src1.reg().S4()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SQRT, SQRT_F32, SQRT_F64, SQRT_V128); - -// ============================================================================ -// OPCODE_RSQRT -// ============================================================================ -// Altivec guarantees an error of < 1/4096 for vrsqrtefp -struct RSQRT_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - SReg src1 = S0; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - e.FRSQRTE(i.dest, src1); - } -}; -struct RSQRT_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - DReg src1 = D0; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - e.FRSQRTE(i.dest, src1); - } -}; -struct RSQRT_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - e.FRSQRTE(i.dest.reg().S4(), Q0.S4()); - } else { - e.FRSQRTE(i.dest.reg().S4(), i.src1.reg().S4()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_RSQRT, RSQRT_F32, RSQRT_F64, RSQRT_V128); - -// ============================================================================ -// OPCODE_RECIP -// ============================================================================ -// Altivec guarantees an error of < 1/4096 for vrefp -struct RECIP_F32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - SReg src1 = S0; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - e.FRECPE(i.dest, src1); - } -}; -struct RECIP_F64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - DReg src1 = D0; - if (i.src1.is_constant) { - e.LoadConstantV(src1.toQ(), i.src1.constant()); - } else { - src1 = i.src1.reg(); - } - e.FRECPE(i.dest, src1); - } -}; -struct RECIP_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - e.FRECPE(i.dest.reg().S4(), Q0.S4()); - } else { - e.FRECPE(i.dest.reg().S4(), i.src1.reg().S4()); - } - } -}; -EMITTER_OPCODE_TABLE(OPCODE_RECIP, RECIP_F32, RECIP_F64, RECIP_V128); - -// ============================================================================ -// OPCODE_POW2 -// ============================================================================ -// TODO(benvanik): use approx here: -// https://jrfonseca.blogspot.com/2008/09/fast-sse2-pow-tables-or-polynomials.html -struct POW2_F32 : Sequence> { - static float32x4_t EmulatePow2(void*, std::byte src[16]) { - float src_value; - vst1q_lane_f32(&src_value, vld1q_u8(reinterpret_cast(src)), - 0); - const float result = std::exp2(src_value); - return vld1q_lane_f32(&result, - vld1q_u8(reinterpret_cast(src)), 0); - } - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_always(); - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); - } - e.CallNativeSafe(reinterpret_cast(EmulatePow2)); - e.FMOV(i.dest, S0); - } -}; -struct POW2_F64 : Sequence> { - static float64x2_t EmulatePow2(void*, std::byte src[16]) { - double src_value; - vst1q_lane_f64(&src_value, vld1q_u8(reinterpret_cast(src)), - 0); - const double result = std::exp2(src_value); - return vld1q_lane_f64(&result, - vld1q_u8(reinterpret_cast(src)), 0); - } - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_always(); - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); - } - e.CallNativeSafe(reinterpret_cast(EmulatePow2)); - e.FMOV(i.dest, D0); - } -}; -struct POW2_V128 : Sequence> { - static float32x4_t EmulatePow2(void*, std::byte src[16]) { - alignas(16) float values[4]; - vst1q_f32(values, vld1q_u8(reinterpret_cast(src))); - for (size_t i = 0; i < 4; ++i) { - values[i] = std::exp2(values[i]); - } - return vld1q_f32(values); - } - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); - } - e.CallNativeSafe(reinterpret_cast(EmulatePow2)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_POW2, POW2_F32, POW2_F64, POW2_V128); - -// ============================================================================ -// OPCODE_LOG2 -// ============================================================================ -// TODO(benvanik): use approx here: -// https://jrfonseca.blogspot.com/2008/09/fast-sse2-pow-tables-or-polynomials.html -// TODO(benvanik): this emulated fn destroys all xmm registers! don't do it! -struct LOG2_F32 : Sequence> { - static float32x4_t EmulateLog2(void*, std::byte src[16]) { - float src_value; - vst1q_lane_f32(&src_value, vld1q_u8(reinterpret_cast(src)), - 0); - float result = std::log2(src_value); - return vld1q_lane_f32(&result, - vld1q_u8(reinterpret_cast(src)), 0); - } - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_always(); - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); - } - e.CallNativeSafe(reinterpret_cast(EmulateLog2)); - e.FMOV(i.dest, S0); - } -}; -struct LOG2_F64 : Sequence> { - static float64x2_t EmulateLog2(void*, std::byte src[16]) { - double src_value; - vst1q_lane_f64(&src_value, vld1q_u8(reinterpret_cast(src)), - 0); - double result = std::log2(src_value); - return vld1q_lane_f64(&result, - vld1q_u8(reinterpret_cast(src)), 0); - } - static void Emit(A64Emitter& e, const EmitArgType& i) { - assert_always(); - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); - } - e.CallNativeSafe(reinterpret_cast(EmulateLog2)); - e.FMOV(i.dest, D0); - } -}; -struct LOG2_V128 : Sequence> { - static float32x4_t EmulateLog2(void*, std::byte src[16]) { - alignas(16) float values[4]; - vst1q_f32(values, vld1q_u8(reinterpret_cast(src))); - for (size_t i = 0; i < 4; ++i) { - values[i] = std::log2(values[i]); - } - return vld1q_f32(values); - } - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.ADD(e.GetNativeParam(0), SP, e.StashConstantV(0, i.src1.constant())); - } else { - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1.reg().toQ())); - } - e.CallNativeSafe(reinterpret_cast(EmulateLog2)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_LOG2, LOG2_F32, LOG2_F64, LOG2_V128); - -// ============================================================================ -// OPCODE_DOT_PRODUCT_3 -// ============================================================================ -struct DOT_PRODUCT_3_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // https://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMUL(dest.S4(), src1.S4(), src2.S4()); - e.MOV(dest.Selem()[3], WZR); - e.FADDP(dest.S4(), dest.S4(), dest.S4()); - e.FADDP(S0, dest.toD().S2()); - e.FMOV(W0, S0); - e.DUP(dest.S4(), W0); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_3, DOT_PRODUCT_3_V128); - -// ============================================================================ -// OPCODE_DOT_PRODUCT_4 -// ============================================================================ -struct DOT_PRODUCT_4_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // https://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.FMUL(dest.S4(), src1.S4(), src2.S4()); - e.FADDP(dest.S4(), dest.S4(), dest.S4()); - e.FADDP(S0, dest.toD().S2()); - e.FMOV(W0, S0); - e.DUP(dest.S4(), W0); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_4, DOT_PRODUCT_4_V128); - -// ============================================================================ -// OPCODE_AND -// ============================================================================ -// TODO(benvanik): put dest/src1|2 together. -template -void EmitAndXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.AND(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int32_t constant) { - e.MOV(REG(1), constant); - e.AND(dest_src, dest_src, REG(1)); - }); -} -struct AND_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndXX(e, i); - } -}; -struct AND_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndXX(e, i); - } -}; -struct AND_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndXX(e, i); - } -}; -struct AND_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndXX(e, i); - } -}; -struct AND_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.AND(dest.B16(), src1.B16(), src2.B16()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_AND, AND_I8, AND_I16, AND_I32, AND_I64, AND_V128); - -// ============================================================================ -// OPCODE_AND_NOT -// ============================================================================ -template -void EmitAndNotXX(A64Emitter& e, const ARGS& i) { - if (i.src1.is_constant) { - // src1 constant. - auto temp = GetTempReg(e); - e.MOV(temp, i.src1.constant()); - e.BIC(i.dest, temp, i.src2); - } else if (i.src2.is_constant) { - // src2 constant. - if (i.dest.reg().index() == i.src1.reg().index()) { - auto temp = GetTempReg(e); - e.MOV(temp, ~i.src2.constant()); - e.AND(i.dest, i.dest, temp); - } else { - e.MOV(i.dest, i.src1); - auto temp = GetTempReg(e); - e.MOV(temp, ~i.src2.constant()); - e.AND(i.dest, i.dest, temp); - } - } else { - // neither are constant - e.BIC(i.dest, i.src1, i.src2); - } -} -struct AND_NOT_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndNotXX(e, i); - } -}; -struct AND_NOT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndNotXX(e, i); - } -}; -struct AND_NOT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndNotXX(e, i); - } -}; -struct AND_NOT_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitAndNotXX(e, i); - } -}; -struct AND_NOT_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.BIC(dest.B16(), src1.B16(), src2.B16()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_AND_NOT, AND_NOT_I8, AND_NOT_I16, AND_NOT_I32, - AND_NOT_I64, AND_NOT_V128); - -// ============================================================================ -// OPCODE_OR -// ============================================================================ -// TODO(benvanik): put dest/src1|2 together. -template -void EmitOrXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.ORR(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int32_t constant) { - e.MOV(REG(1), constant); - e.ORR(dest_src, dest_src, REG(1)); - }); -} -struct OR_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitOrXX(e, i); - } -}; -struct OR_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitOrXX(e, i); - } -}; -struct OR_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitOrXX(e, i); - } -}; -struct OR_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitOrXX(e, i); - } -}; -struct OR_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.ORR(dest.B16(), src1.B16(), src2.B16()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_OR, OR_I8, OR_I16, OR_I32, OR_I64, OR_V128); - -// ============================================================================ -// OPCODE_XOR -// ============================================================================ -// TODO(benvanik): put dest/src1|2 together. -template -void EmitXorXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitCommutativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, REG src) { - e.EOR(dest_src, dest_src, src); - }, - [](A64Emitter& e, REG dest_src, int32_t constant) { - e.MOV(REG(1), constant); - e.EOR(dest_src, dest_src, REG(1)); - }); -} -struct XOR_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitXorXX(e, i); - } -}; -struct XOR_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitXorXX(e, i); - } -}; -struct XOR_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitXorXX(e, i); - } -}; -struct XOR_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitXorXX(e, i); - } -}; -struct XOR_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitCommutativeBinaryVOp( - e, i, [](A64Emitter& e, QReg dest, QReg src1, QReg src2) { - e.EOR(dest.B16(), src1.B16(), src2.B16()); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_XOR, XOR_I8, XOR_I16, XOR_I32, XOR_I64, XOR_V128); - -// ============================================================================ -// OPCODE_NOT -// ============================================================================ -// TODO(benvanik): put dest/src1 together. -template -void EmitNotXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitUnaryOp( - e, i, [](A64Emitter& e, REG dest_src) { e.MVN(dest_src, dest_src); }); -} -struct NOT_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNotXX(e, i); - } -}; -struct NOT_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNotXX(e, i); - } -}; -struct NOT_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNotXX(e, i); - } -}; -struct NOT_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitNotXX(e, i); - } -}; -struct NOT_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - e.NOT(i.dest.reg().B16(), src.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_NOT, NOT_I8, NOT_I16, NOT_I32, NOT_I64, NOT_V128); - -// ============================================================================ -// OPCODE_SHL -// ============================================================================ -// TODO(benvanik): optimize common shifts. -template -void EmitShlXX(A64Emitter& e, const ARGS& i) { - SEQ::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, REG dest_src, WReg src) { - e.LSL(dest_src, dest_src, REG(src.index())); - }, - [](A64Emitter& e, REG dest_src, int8_t constant) { - e.LSL(dest_src, dest_src, constant); - }); -} -struct SHL_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShlXX(e, i); - } -}; -struct SHL_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShlXX(e, i); - } -}; -struct SHL_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShlXX(e, i); - } -}; -struct SHL_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitShlXX(e, i); - } -}; -struct SHL_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): native version (with shift magic). - if (i.src2.is_constant) { - e.MOV(e.GetNativeParam(1), i.src2.constant()); - } else { - e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - e.CallNativeSafe(reinterpret_cast(EmulateShlV128)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static float32x4_t EmulateShlV128(void*, std::byte src1[16], uint8_t src2) { - // Almost all instances are shamt = 1, but non-constant. - // shamt is [0,7] - uint8_t shamt = src2 & 0x7; - - // Load `src1` as a byte vector (uint8x16_t) then work on byte shifting - uint8x16_t byte_vec = vld1q_u8(reinterpret_cast(src1)); - alignas(16) vec128_t value; - vst1q_u8(reinterpret_cast(&value), byte_vec); - for (int i = 0; i < 15; ++i) { - value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] << shamt) | - (value.u8[(i + 1) ^ 0x3] >> (8 - shamt)); - } - value.u8[15 ^ 0x3] = value.u8[15 ^ 0x3] << shamt; - return vreinterpretq_f32_u8( - vld1q_u8(reinterpret_cast(&value))); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SHL, SHL_I8, SHL_I16, SHL_I32, SHL_I64, SHL_V128); - -// ============================================================================ -// OPCODE_SHR -// ============================================================================ -struct SHR_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.LSR(dest_src, dest_src, src); - }, - [](A64Emitter& e, WReg dest_src, int8_t constant) { - e.LSR(dest_src, dest_src, constant); - }); - } -}; -struct SHR_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.LSR(dest_src, dest_src, src); - }, - [](A64Emitter& e, WReg dest_src, int8_t constant) { - e.LSR(dest_src, dest_src, constant); - }); - } -}; -struct SHR_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.LSR(dest_src, dest_src, src); - }, - [](A64Emitter& e, WReg dest_src, int8_t constant) { - e.LSR(dest_src, dest_src, constant); - }); - } -}; -struct SHR_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, XReg dest_src, WReg src) { - e.LSR(dest_src, dest_src, src.toX()); - }, - [](A64Emitter& e, XReg dest_src, int8_t constant) { - e.LSR(dest_src, dest_src, constant); - }); - } -}; -struct SHR_V128 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // TODO(benvanik): native version (with shift magic). - if (i.src2.is_constant) { - e.MOV(e.GetNativeParam(1), i.src2.constant()); - } else { - e.MOV(e.GetNativeParam(1), i.src2.reg().toX()); - } - e.ADD(e.GetNativeParam(0), SP, e.StashV(0, i.src1)); - e.CallNativeSafe(reinterpret_cast(EmulateShrV128)); - e.MOV(i.dest.reg().B16(), Q0.B16()); - } - static float32x4_t EmulateShrV128(void*, std::byte src1[16], uint8_t src2) { - // Almost all instances are shamt = 1, but non-constant. - // shamt is [0,7] - uint8_t shamt = src2 & 0x7; - // Load `src1` as a byte vector (uint8x16_t) and store it into `value` - uint8x16_t byte_vec = vld1q_u8(reinterpret_cast(src1)); - alignas(16) vec128_t value; - vst1q_u8(reinterpret_cast(&value), byte_vec); - - for (int i = 15; i > 0; --i) { - value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] >> shamt) | - (value.u8[(i - 1) ^ 0x3] << (8 - shamt)); - } - value.u8[0 ^ 0x3] = value.u8[0 ^ 0x3] >> shamt; - // Convert to float32x4_t by reinterpreting the processed `value` - return vreinterpretq_f32_u8( - vld1q_u8(reinterpret_cast(&value))); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SHR, SHR_I8, SHR_I16, SHR_I32, SHR_I64, SHR_V128); - -// ============================================================================ -// OPCODE_SHA -// ============================================================================ -struct SHA_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.SXTB(dest_src, dest_src); - e.ASR(dest_src, dest_src, src); - }, - [](A64Emitter& e, WReg dest_src, int8_t constant) { - e.SXTB(dest_src, dest_src); - e.ASR(dest_src, dest_src, constant); - }); - } -}; -struct SHA_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.SXTH(dest_src, dest_src); - e.ASR(dest_src, dest_src, src); - }, - [](A64Emitter& e, WReg dest_src, int8_t constant) { - e.ASR(dest_src, dest_src, constant); - }); - } -}; -struct SHA_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, WReg dest_src, WReg src) { - e.ASR(dest_src, dest_src, src); - }, - [](A64Emitter& e, WReg dest_src, int8_t constant) { - e.ASR(dest_src, dest_src, constant); - }); - } -}; -struct SHA_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - Sequence::EmitAssociativeBinaryOp( - e, i, - [](A64Emitter& e, XReg dest_src, WReg src) { - e.ASR(dest_src, dest_src, src.toX()); - }, - [](A64Emitter& e, XReg dest_src, int8_t constant) { - e.ASR(dest_src, dest_src, constant); - }); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SHA, SHA_I8, SHA_I16, SHA_I32, SHA_I64); - -// ============================================================================ -// OPCODE_ROTATE_LEFT -// ============================================================================ -// TODO(benvanik): put dest/src1 together, src2 in cl. -template -void EmitRotateLeftXX(A64Emitter& e, const ARGS& i) { - // ; rotate r1 left by r2, producing r0 - // ; (destroys r2) - // ; r1 = ABCDEFGH - // lslv r0, r1, r2 ; r0 = EFGH0000 - // mvn r2, r2 ; r2 = leftover bits - // lsrv r2, r1, r2 ; r2 = 0000ABCD - // orr r0, r0, r2 ; r0 = EFGHABCD - if (i.src1.is_constant) { - e.MOV(REG(0), i.src1.constant()); - } else { - e.MOV(REG(0), i.src1.reg()); - } - - if (i.src2.is_constant) { - e.MOV(REG(1), i.src2.constant()); - } else { - e.MOV(W1, i.src2.reg().toW()); - } - - e.LSLV(i.dest, REG(0), REG(1)); - e.MVN(REG(1), REG(1)); - e.LSRV(REG(1), REG(0), REG(1)); - e.ORR(i.dest, i.dest, REG(1)); -} -struct ROTATE_LEFT_I8 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitRotateLeftXX(e, i); - } -}; -struct ROTATE_LEFT_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitRotateLeftXX(e, i); - } -}; -struct ROTATE_LEFT_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.MOV(W0, i.src1.constant()); - } else { - e.MOV(W0, i.src1.reg()); - } - - if (i.src2.is_constant) { - e.MOV(W1, i.src2.constant()); - } else { - e.SXTB(W1, i.src2.reg()); - } - e.NEG(W1, W1); - - e.ROR(i.dest, W0, W1); - } -}; -struct ROTATE_LEFT_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - if (i.src1.is_constant) { - e.MOV(X0, i.src1.constant()); - } else { - e.MOV(X0, i.src1.reg()); - } - - if (i.src2.is_constant) { - e.MOV(X1, i.src2.constant()); - } else { - e.SXTB(X1, i.src2.reg().toW()); - } - e.NEG(X1, X1); - - e.ROR(i.dest, X0, X1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_ROTATE_LEFT, ROTATE_LEFT_I8, ROTATE_LEFT_I16, - ROTATE_LEFT_I32, ROTATE_LEFT_I64); - -// ============================================================================ -// OPCODE_BYTE_SWAP -// ============================================================================ -// TODO(benvanik): put dest/src1 together. -struct BYTE_SWAP_I16 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitUnaryOp(e, i, [](A64Emitter& e, WReg dest_src) { - e.REV16(dest_src, dest_src); - }); - } -}; -struct BYTE_SWAP_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitUnaryOp( - e, i, [](A64Emitter& e, WReg dest_src) { e.REV(dest_src, dest_src); }); - } -}; -struct BYTE_SWAP_I64 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - EmitUnaryOp( - e, i, [](A64Emitter& e, XReg dest_src) { e.REV(dest_src, dest_src); }); - } -}; -struct BYTE_SWAP_V128 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // Reverse upper and lower 64-bit halfs - QReg src = i.src1.is_constant ? Q0 : i.src1.reg(); - if (i.src1.is_constant) { - e.LoadConstantV(Q0, i.src1.constant()); - } - e.REV32(i.dest.reg().B16(), src.B16()); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_BYTE_SWAP, BYTE_SWAP_I16, BYTE_SWAP_I32, - BYTE_SWAP_I64, BYTE_SWAP_V128); - -// ============================================================================ -// OPCODE_CNTLZ -// Count leading zeroes -// ============================================================================ -struct CNTLZ_I8 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // No 8bit lzcnt, so do 32 and sub 24. - e.UXTB(i.dest, i.src1); - e.CLZ(i.dest, i.dest); - e.SUB(i.dest.reg(), i.dest.reg(), 24); - } -}; -struct CNTLZ_I16 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // No 16bit lzcnt, so do 32 and sub 16. - e.UXTH(i.dest, i.src1); - e.CLZ(i.dest, i.dest); - e.SUB(i.dest.reg(), i.dest.reg(), 16); - } -}; -struct CNTLZ_I32 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CLZ(i.dest, i.src1); - } -}; -struct CNTLZ_I64 : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CLZ(i.dest.reg().toX(), i.src1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_CNTLZ, CNTLZ_I8, CNTLZ_I16, CNTLZ_I32, CNTLZ_I64); - -// ============================================================================ -// OPCODE_SET_ROUNDING_MODE -// ============================================================================ -// Input: FPSCR (PPC format) -// Convert from PPC rounding mode to ARM -// PPC | ARM | -// 00 | 00 | nearest -// 01 | 11 | toward zero -// 10 | 01 | toward +infinity -// 11 | 10 | toward -infinity -static const uint8_t fpcr_table[] = { - 0b0'00, // |--|nearest - 0b0'11, // |--|toward zero - 0b0'01, // |--|toward +infinity - 0b0'10, // |--|toward -infinity - 0b1'00, // |FZ|nearest - 0b1'11, // |FZ|toward zero - 0b1'01, // |FZ|toward +infinity - 0b1'10, // |FZ|toward -infinity -}; -struct SET_ROUNDING_MODE_I32 - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - // Low 3 bits are |Non-IEEE:1|RoundingMode:2| - // Non-IEEE bit is flush-to-zero - e.AND(W1, i.src1, 0b111); - - // Use the low 3 bits as an index into a LUT - e.MOV(X0, reinterpret_cast(fpcr_table)); - e.LDRB(W0, X0, X1); - - // Replace FPCR bits with new value - e.MRS(X1, SystemReg::FPCR); - e.BFI(X1, X0, 23, 3); - e.MSR(SystemReg::FPCR, X1); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_SET_ROUNDING_MODE, SET_ROUNDING_MODE_I32); - -static void MaybeYieldForwarder(void* ctx) { xe::threading::MaybeYield(); } -// ============================================================================ -// OPCODE_DELAY_EXECUTION -// ============================================================================ -struct DELAY_EXECUTION - : Sequence> { - static void Emit(A64Emitter& e, const EmitArgType& i) { - e.CallNativeSafe(reinterpret_cast(MaybeYieldForwarder)); - } -}; -EMITTER_OPCODE_TABLE(OPCODE_DELAY_EXECUTION, DELAY_EXECUTION); - -// Include anchors to other sequence sources so they get included in the build. -extern volatile int anchor_control; -static int anchor_control_dest = anchor_control; - -extern volatile int anchor_memory; -static int anchor_memory_dest = anchor_memory; - -extern volatile int anchor_vector; -static int anchor_vector_dest = anchor_vector; - -bool SelectSequence(A64Emitter* e, const hir::Instr* i, - const hir::Instr** new_tail) { - const InstrKey key(i); - auto it = GetSequenceTable().find(key); - if (it != GetSequenceTable().end()) { - if (it->second(*e, i)) { - *new_tail = i->next; - return true; - } - } - XELOGE("No sequence match for variant {}", hir::GetOpcodeName(i->opcode)); - return false; -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_sequences.h b/src/xenia/cpu/backend/a64/a64_sequences.h deleted file mode 100644 index eaf4fb0f1..000000000 --- a/src/xenia/cpu/backend/a64/a64_sequences.h +++ /dev/null @@ -1,61 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ -#define XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ - -#include "xenia/cpu/hir/instr.h" - -#include - -#include "xenia/base/logging.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -class A64Emitter; - -typedef bool (*SequenceSelectFn)(A64Emitter&, const hir::Instr*); - -// Singleton accessor for sequence table. -inline std::unordered_map& GetSequenceTable() { - static std::unordered_map sequence_table; - return sequence_table; -} - -template -bool RegisterSingle() { - bool inserted = GetSequenceTable().emplace(T::head_key(), T::Select).second; - if (!inserted) { - XELOGW("A64 sequence registration duplicate key 0x{:08X}", T::head_key()); - } - return inserted; -} - -template -bool RegisterAll() { - bool ok = true; - ((ok &= RegisterSingle()), ...); - return ok; -} - -#define EMITTER_OPCODE_TABLE(name, ...) \ - static const bool A64_INSTR_##name = RegisterAll<__VA_ARGS__>(); - -bool SelectSequence(A64Emitter* e, const hir::Instr* i, - const hir::Instr** new_tail); - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ diff --git a/src/xenia/cpu/backend/a64/a64_stack_layout.h b/src/xenia/cpu/backend/a64/a64_stack_layout.h deleted file mode 100644 index ee8cbcfac..000000000 --- a/src/xenia/cpu/backend/a64/a64_stack_layout.h +++ /dev/null @@ -1,129 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ -#define XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ - -#include "xenia/base/vec128.h" -#include "xenia/cpu/backend/a64/a64_backend.h" -#include "xenia/cpu/backend/a64/a64_emitter.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -class StackLayout { - public: - /** - * Stack Layout - * ---------------------------- - * NOTE: stack must always be 16b aligned. - * - * Thunk stack: - * Non-Volatile Volatile - * +------------------+------------------+ - * | arg temp, 3 * 8 | arg temp, 3 * 8 | sp + 0x000 - * | | | - * | | | - * +------------------+------------------+ - * | rbx | (unused) | sp + 0x018 - * +------------------+------------------+ - * | rbp | X1 | sp + 0x020 - * +------------------+------------------+ - * | rcx (Win32) | X2 | sp + 0x028 - * +------------------+------------------+ - * | rsi (Win32) | X3 | sp + 0x030 - * +------------------+------------------+ - * | rdi (Win32) | X4 | sp + 0x038 - * +------------------+------------------+ - * | r12 | X5 | sp + 0x040 - * +------------------+------------------+ - * | r13 | X6 | sp + 0x048 - * +------------------+------------------+ - * | r14 | X7 | sp + 0x050 - * +------------------+------------------+ - * | r15 | X8 | sp + 0x058 - * +------------------+------------------+ - * | xmm6 (Win32) | X9 | sp + 0x060 - * | | | - * +------------------+------------------+ - * | xmm7 (Win32) | X10 | sp + 0x070 - * | | | - * +------------------+------------------+ - * | xmm8 (Win32) | X11 | sp + 0x080 - * | | | - * +------------------+------------------+ - * | xmm9 (Win32) | X12 | sp + 0x090 - * | | | - * +------------------+------------------+ - * | xmm10 (Win32) | X13 | sp + 0x0A0 - * | | | - * +------------------+------------------+ - * | xmm11 (Win32) | X14 | sp + 0x0B0 - * | | | - * +------------------+------------------+ - * | xmm12 (Win32) | X15 | sp + 0x0C0 - * | | | - * +------------------+------------------+ - * | xmm13 (Win32) | X16 | sp + 0x0D0 - * | | | - * +------------------+------------------+ - * | xmm14 (Win32) | X17 | sp + 0x0E0 - * | | | - * +------------------+------------------+ - * | xmm15 (Win32) | X18 | sp + 0x0F0 - * | | | - * +------------------+------------------+ - */ - XEPACKEDSTRUCT(Thunk, { - uint64_t arg_temp[3]; - uint64_t r[17]; - vec128_t xmm[22]; - }); - static_assert(sizeof(Thunk) % 16 == 0, - "sizeof(Thunk) must be a multiple of 16!"); - static const size_t THUNK_STACK_SIZE = sizeof(Thunk); - - /** - * - * - * Guest stack: - * +------------------+ - * | arg temp, 3 * 8 | sp + 0 - * | | - * | | - * +------------------+ - * | scratch, 48b | sp + 32(kStashOffset) - * | | - * +------------------+ - * | X0 / context | sp + 80 - * +------------------+ - * | guest ret addr | sp + 88 - * +------------------+ - * | call ret addr | sp + 96 - * +------------------+ - * ... locals ... - * +------------------+ - * | (return address) | - * +------------------+ - * - */ - static const size_t GUEST_STACK_SIZE = 96 + 16; - static const size_t GUEST_CTX_HOME = 80; - static const size_t GUEST_RET_ADDR = 88; - static const size_t GUEST_CALL_RET_ADDR = 96; -}; - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ diff --git a/src/xenia/cpu/backend/a64/a64_tracers.cc b/src/xenia/cpu/backend/a64/a64_tracers.cc deleted file mode 100644 index 49b9ed1cb..000000000 --- a/src/xenia/cpu/backend/a64/a64_tracers.cc +++ /dev/null @@ -1,226 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/cpu/backend/a64/a64_tracers.h" - -#include - -#include "xenia/base/logging.h" -#include "xenia/base/vec128.h" -#include "xenia/cpu/backend/a64/a64_emitter.h" -#include "xenia/cpu/processor.h" -#include "xenia/cpu/thread_state.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { - -#define ITRACE 0 -#define DTRACE 0 - -#define TARGET_THREAD 0 - -bool trace_enabled = true; - -#define THREAD_MATCH \ - (!TARGET_THREAD || thread_state->thread_id() == TARGET_THREAD) -#define IFLUSH() -#define IPRINT(s) \ - if (trace_enabled && THREAD_MATCH) \ - xe::logging::AppendLogLine(xe::LogLevel::Debug, 't', s, xe::LogSrc::Cpu) -#define DFLUSH() -#define DPRINT(...) \ - if (trace_enabled && THREAD_MATCH) \ - xe::logging::AppendLogLineFormat(xe::LogSrc::Cpu, xe::LogLevel::Debug, 't', \ - __VA_ARGS__) - -uint32_t GetTracingMode() { - uint32_t mode = 0; -#if ITRACE - mode |= TRACING_INSTR; -#endif // ITRACE -#if DTRACE - mode |= TRACING_DATA; -#endif // DTRACE - return mode; -} - -void TraceString(void* raw_context, const char* str) { - auto thread_state = *reinterpret_cast(raw_context); - IPRINT(str); - IFLUSH(); -} - -void TraceContextLoadI8(void* raw_context, uint64_t offset, uint8_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = ctx i8 +{}\n", (int8_t)value, value, offset); -} -void TraceContextLoadI16(void* raw_context, uint64_t offset, uint16_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = ctx i16 +{}\n", (int16_t)value, value, offset); -} -void TraceContextLoadI32(void* raw_context, uint64_t offset, uint32_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = ctx i32 +{}\n", (int32_t)value, value, offset); -} -void TraceContextLoadI64(void* raw_context, uint64_t offset, uint64_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = ctx i64 +{}\n", (int64_t)value, value, offset); -} -void TraceContextLoadF32(void* raw_context, uint64_t offset, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = ctx f32 +{}\n", xe::m128_f32<0>(value), - xe::m128_i32<0>(value), offset); -} -void TraceContextLoadF64(void* raw_context, uint64_t offset, - const double* value) { - auto thread_state = *reinterpret_cast(raw_context); - // auto v = _mm_loadu_pd(value); - auto v = vld1q_f64(value); - DPRINT("{} ({:X}) = ctx f64 +{}\n", xe::m128_f64<0>(v), xe::m128_i64<0>(v), - offset); -} -void TraceContextLoadV128(void* raw_context, uint64_t offset, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("[{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}] = ctx v128 +{}\n", - xe::m128_f32<0>(value), xe::m128_f32<1>(value), xe::m128_f32<2>(value), - xe::m128_f32<3>(value), xe::m128_i32<0>(value), xe::m128_i32<1>(value), - xe::m128_i32<2>(value), xe::m128_i32<3>(value), offset); -} - -void TraceContextStoreI8(void* raw_context, uint64_t offset, uint8_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("ctx i8 +{} = {} ({:X})\n", offset, (int8_t)value, value); -} -void TraceContextStoreI16(void* raw_context, uint64_t offset, uint16_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("ctx i16 +{} = {} ({:X})\n", offset, (int16_t)value, value); -} -void TraceContextStoreI32(void* raw_context, uint64_t offset, uint32_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("ctx i32 +{} = {} ({:X})\n", offset, (int32_t)value, value); -} -void TraceContextStoreI64(void* raw_context, uint64_t offset, uint64_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("ctx i64 +{} = {} ({:X})\n", offset, (int64_t)value, value); -} -void TraceContextStoreF32(void* raw_context, uint64_t offset, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("ctx f32 +{} = {} ({:X})\n", offset, xe::m128_f32<0>(value), - xe::m128_i32<0>(value)); -} -void TraceContextStoreF64(void* raw_context, uint64_t offset, - const double* value) { - auto thread_state = *reinterpret_cast(raw_context); - // auto v = _mm_loadu_pd(value); - auto v = vld1q_f64(value); - DPRINT("ctx f64 +{} = {} ({:X})\n", offset, xe::m128_f64<0>(v), - xe::m128_i64<0>(v)); -} -void TraceContextStoreV128(void* raw_context, uint64_t offset, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("ctx v128 +{} = [{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}]\n", - offset, xe::m128_f32<0>(value), xe::m128_f32<1>(value), - xe::m128_f32<2>(value), xe::m128_f32<3>(value), xe::m128_i32<0>(value), - xe::m128_i32<1>(value), xe::m128_i32<2>(value), - xe::m128_i32<3>(value)); -} - -void TraceMemoryLoadI8(void* raw_context, uint32_t address, uint8_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = load.i8 {:08X}\n", (int8_t)value, value, address); -} -void TraceMemoryLoadI16(void* raw_context, uint32_t address, uint16_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = load.i16 {:08X}\n", (int16_t)value, value, address); -} -void TraceMemoryLoadI32(void* raw_context, uint32_t address, uint32_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = load.i32 {:08X}\n", (int32_t)value, value, address); -} -void TraceMemoryLoadI64(void* raw_context, uint32_t address, uint64_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = load.i64 {:08X}\n", (int64_t)value, value, address); -} -void TraceMemoryLoadF32(void* raw_context, uint32_t address, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = load.f32 {:08X}\n", xe::m128_f32<0>(value), - xe::m128_i32<0>(value), address); -} -void TraceMemoryLoadF64(void* raw_context, uint32_t address, - float64x2_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("{} ({:X}) = load.f64 {:08X}\n", xe::m128_f64<0>(value), - xe::m128_i64<0>(value), address); -} -void TraceMemoryLoadV128(void* raw_context, uint32_t address, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT( - "[{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}] = load.v128 {:08X}\n", - xe::m128_f32<0>(value), xe::m128_f32<1>(value), xe::m128_f32<2>(value), - xe::m128_f32<3>(value), xe::m128_i32<0>(value), xe::m128_i32<1>(value), - xe::m128_i32<2>(value), xe::m128_i32<3>(value), address); -} - -void TraceMemoryStoreI8(void* raw_context, uint32_t address, uint8_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("store.i8 {:08X} = {} ({:X})\n", address, (int8_t)value, value); -} -void TraceMemoryStoreI16(void* raw_context, uint32_t address, uint16_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("store.i16 {:08X} = {} ({:X})\n", address, (int16_t)value, value); -} -void TraceMemoryStoreI32(void* raw_context, uint32_t address, uint32_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("store.i32 {:08X} = {} ({:X})\n", address, (int32_t)value, value); -} -void TraceMemoryStoreI64(void* raw_context, uint32_t address, uint64_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("store.i64 {:08X} = {} ({:X})\n", address, (int64_t)value, value); -} -void TraceMemoryStoreF32(void* raw_context, uint32_t address, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("store.f32 {:08X} = {} ({:X})\n", address, xe::m128_f32<0>(value), - xe::m128_i32<0>(value)); -} -void TraceMemoryStoreF64(void* raw_context, uint32_t address, - float64x2_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("store.f64 {:08X} = {} ({:X})\n", address, xe::m128_f64<0>(value), - xe::m128_i64<0>(value)); -} -void TraceMemoryStoreV128(void* raw_context, uint32_t address, - float32x4_t value) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT( - "store.v128 {:08X} = [{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}]\n", - address, xe::m128_f32<0>(value), xe::m128_f32<1>(value), - xe::m128_f32<2>(value), xe::m128_f32<3>(value), xe::m128_i32<0>(value), - xe::m128_i32<1>(value), xe::m128_i32<2>(value), xe::m128_i32<3>(value)); -} - -void TraceMemset(void* raw_context, uint32_t address, uint8_t value, - uint32_t length) { - auto thread_state = *reinterpret_cast(raw_context); - DPRINT("memset {:08X}-{:08X} ({}) = {:02X}", address, address + length, - length, value); -} - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_tracers.h b/src/xenia/cpu/backend/a64/a64_tracers.h deleted file mode 100644 index d004aef6c..000000000 --- a/src/xenia/cpu/backend/a64/a64_tracers.h +++ /dev/null @@ -1,82 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ -#define XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ - -#include -#include - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 { -class A64Emitter; - -enum TracingMode { - TRACING_INSTR = (1 << 1), - TRACING_DATA = (1 << 2), -}; - -uint32_t GetTracingMode(); -inline bool IsTracingInstr() { return (GetTracingMode() & TRACING_INSTR) != 0; } -inline bool IsTracingData() { return (GetTracingMode() & TRACING_DATA) != 0; } - -void TraceString(void* raw_context, const char* str); - -void TraceContextLoadI8(void* raw_context, uint64_t offset, uint8_t value); -void TraceContextLoadI16(void* raw_context, uint64_t offset, uint16_t value); -void TraceContextLoadI32(void* raw_context, uint64_t offset, uint32_t value); -void TraceContextLoadI64(void* raw_context, uint64_t offset, uint64_t value); -void TraceContextLoadF32(void* raw_context, uint64_t offset, float32x4_t value); -void TraceContextLoadF64(void* raw_context, uint64_t offset, - const double* value); -void TraceContextLoadV128(void* raw_context, uint64_t offset, - float32x4_t value); - -void TraceContextStoreI8(void* raw_context, uint64_t offset, uint8_t value); -void TraceContextStoreI16(void* raw_context, uint64_t offset, uint16_t value); -void TraceContextStoreI32(void* raw_context, uint64_t offset, uint32_t value); -void TraceContextStoreI64(void* raw_context, uint64_t offset, uint64_t value); -void TraceContextStoreF32(void* raw_context, uint64_t offset, - float32x4_t value); -void TraceContextStoreF64(void* raw_context, uint64_t offset, - const double* value); -void TraceContextStoreV128(void* raw_context, uint64_t offset, - float32x4_t value); - -void TraceMemoryLoadI8(void* raw_context, uint32_t address, uint8_t value); -void TraceMemoryLoadI16(void* raw_context, uint32_t address, uint16_t value); -void TraceMemoryLoadI32(void* raw_context, uint32_t address, uint32_t value); -void TraceMemoryLoadI64(void* raw_context, uint32_t address, uint64_t value); -void TraceMemoryLoadF32(void* raw_context, uint32_t address, float32x4_t value); -void TraceMemoryLoadF64(void* raw_context, uint32_t address, float64x2_t value); -void TraceMemoryLoadV128(void* raw_context, uint32_t address, - float32x4_t value); - -void TraceMemoryStoreI8(void* raw_context, uint32_t address, uint8_t value); -void TraceMemoryStoreI16(void* raw_context, uint32_t address, uint16_t value); -void TraceMemoryStoreI32(void* raw_context, uint32_t address, uint32_t value); -void TraceMemoryStoreI64(void* raw_context, uint32_t address, uint64_t value); -void TraceMemoryStoreF32(void* raw_context, uint32_t address, - float32x4_t value); -void TraceMemoryStoreF64(void* raw_context, uint32_t address, - float64x2_t value); -void TraceMemoryStoreV128(void* raw_context, uint32_t address, - float32x4_t value); - -void TraceMemset(void* raw_context, uint32_t address, uint8_t value, - uint32_t length); - -} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ diff --git a/src/xenia/cpu/backend/a64/a64_util.h b/src/xenia/cpu/backend/a64/a64_util.h deleted file mode 100644 index e3a34ac00..000000000 --- a/src/xenia/cpu/backend/a64/a64_util.h +++ /dev/null @@ -1,25 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2024 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_CPU_BACKEND_A64_A64_UTIL_H_ -#define XENIA_CPU_BACKEND_A64_A64_UTIL_H_ - -#include "xenia/base/vec128.h" -#include "xenia/cpu/backend/a64/a64_backend.h" -#include "xenia/cpu/backend/a64/a64_emitter.h" - -namespace xe { -namespace cpu { -namespace backend { -namespace a64 {} // namespace a64 -} // namespace backend -} // namespace cpu -} // namespace xe - -#endif // XENIA_CPU_BACKEND_A64_A64_UTIL_H_