From b7c8244dafea9dd97876fecb693606bef93e4cb9 Mon Sep 17 00:00:00 2001 From: Mphatso Raymond Mataka <45720369+HorseTrain@users.noreply.github.com> Date: Tue, 21 Jan 2025 23:42:00 -0800 Subject: [PATCH] fixed global move propagation. --- src/assembly/x86/x86_assembler.cpp | 152 +- src/assembly/x86/x86_pipeline.cpp | 7 + src/assembly/x86/x86_pre_allocator.cpp | 304 +- .../aarch64/aarch64_externals_interpreter.cpp | 10 + .../aarch64/aarch64_externals_jit.cpp | 59 + src/emulator/aarch64/aarch64_impl.cpp | 2947 +++++++++++------ src/emulator/aarch64/aarch64_impl.h | 56 +- src/emulator/atomic.h | 1 + src/emulator/guest_process.cpp | 16 +- src/emulator/guest_process.h | 1 + src/emulator/ssa_emit_context.h | 2 + src/export.cpp | 8 +- src/ir/ir.h | 46 +- src/ir/ir_operation_block.cpp | 3 + src/ir/ssa.cpp | 212 +- src/jit/jit_context.cpp | 7 +- src/jit/jit_context.h | 2 +- 17 files changed, 2789 insertions(+), 1044 deletions(-) diff --git a/src/assembly/x86/x86_assembler.cpp b/src/assembly/x86/x86_assembler.cpp index a46f891..cfd1052 100755 --- a/src/assembly/x86/x86_assembler.cpp +++ b/src/assembly/x86/x86_assembler.cpp @@ -99,6 +99,17 @@ static void assert_valid_binary_float_operation(ir_operation* operation) assert(ir_operand::is_vector(&operation->sources[0])); } +static void assert_valid_binary_float_comparison_operation(ir_operation* operation) +{ + assert_operand_count(operation, 1, 3); + + assert_same_registers(operation->destinations[0], operation->sources[0]); + assert_same_size({operation->destinations[0], operation->sources[0], operation->sources[1]}); + assert(ir_operand::is_vector(&operation->sources[0])); + + assert_is_constant(operation->sources[2]); +} + void assemble_x86_64_code(void** result_code, uint64_t* result_code_size, ir_operation_block* source_ir) { arena_allocator* allocator = source_ir->allocator; @@ -156,7 +167,7 @@ void assemble_x86_64_code(void** result_code, uint64_t* result_code_size, ir_ope }; break; - case ir_compare_and_swap: + case x86_cmpxchg: { assert_operand_count(&working_operation, 1, 3); @@ -368,11 +379,8 @@ void assemble_x86_64_code(void** result_code, uint64_t* result_code_size, ir_ope assert_is_register(destinations[0]); assert_is_register(sources[0]); - - //assert_all_registers(&working_operation); - - assert(destinations[0].meta_data == sources[0].meta_data); - assert(destinations[0].meta_data == sources[1].meta_data); + + assert_same_size({destinations[0], sources[0], sources[1]}); if (ir_operand::is_constant(&sources[1])) { @@ -814,13 +822,26 @@ void assemble_x86_64_code(void** result_code, uint64_t* result_code_size, ir_ope ir_operand d = working_operation.destinations[0]; ir_operand s0 = working_operation.sources[0]; ir_operand s1 = working_operation.sources[1]; - - switch (d.meta_data) - { - case int32: c.lea(create_operand(d), c.ptr[create_operand(s0) + create_operand(s1)]); break; - case int64: c.lea(create_operand(d), c.ptr[create_operand(s0) + create_operand(s1)]); break; - default: throw_error(); + if (ir_operand::is_constant(&s1)) + { + switch (d.meta_data) + { + case int32: c.lea(create_operand(d), c.ptr[create_operand(s0) + s1.value]); break; + case int64: c.lea(create_operand(d), c.ptr[create_operand(s0) + s1.value]); break; + + default: throw_error(); + } + } + else + { + switch (d.meta_data) + { + case int32: c.lea(create_operand(d), c.ptr[create_operand(s0) + create_operand(s1)]); break; + case int64: c.lea(create_operand(d), c.ptr[create_operand(s0) + create_operand(s1)]); break; + + default: throw_error(); + } } }; break; @@ -852,6 +873,19 @@ void assemble_x86_64_code(void** result_code, uint64_t* result_code_size, ir_ope }; break; + case x86_cmpss: assert_valid_binary_float_comparison_operation(&working_operation); c.cmpss(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1]), working_operation.sources[2].value); break; + case x86_cmpsd: assert_valid_binary_float_comparison_operation(&working_operation); c.cmpsd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1]), working_operation.sources[2].value); break; + case x86_cmpps: assert_valid_binary_float_comparison_operation(&working_operation); c.cmpps(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1]), working_operation.sources[2].value); break; + case x86_cmppd: assert_valid_binary_float_comparison_operation(&working_operation); c.cmppd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1]), working_operation.sources[2].value); break; + + case x86_haddpd: assert_valid_binary_float_operation(&working_operation); c.haddpd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + case x86_haddps: assert_valid_binary_float_operation(&working_operation); c.haddps(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + + case x86_paddb: assert_valid_binary_float_operation(&working_operation); c.paddb(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + case x86_paddw: assert_valid_binary_float_operation(&working_operation); c.paddw(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + case x86_paddd: assert_valid_binary_float_operation(&working_operation); c.paddd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + case x86_paddq: assert_valid_binary_float_operation(&working_operation); c.paddq(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + case x86_addpd: assert_valid_binary_float_operation(&working_operation); c.addpd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; case x86_addps: assert_valid_binary_float_operation(&working_operation); c.addps(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; case x86_addsd: assert_valid_binary_float_operation(&working_operation); c.addsd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; @@ -878,9 +912,103 @@ void assemble_x86_64_code(void** result_code, uint64_t* result_code_size, ir_ope case x86_pand: assert_valid_binary_float_operation(&working_operation); c.pand(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; case x86_pandn: assert_valid_binary_float_operation(&working_operation); c.pandn(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[1])); break; + case x86_cvtsd2ss: c.cvtsd2ss(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[0])); break; + case x86_cvtss2sd: c.cvtss2sd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[0])); break; case x86_sqrtss: c.sqrtss(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[0])); break; case x86_sqrtsd: c.sqrtsd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[0])); break; + case x86_sqrtps: c.sqrtps(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[0])); break; + case x86_sqrtpd: c.sqrtpd(create_operand(working_operation.destinations[0]), create_operand(working_operation.sources[0])); break; + + case x86_popcnt: + { + ir_operand destination = working_operation.destinations[0]; + ir_operand source = working_operation.sources[0]; + + assert_same_size({destination, source}); + + switch (destination.meta_data) + { + case int16: c.popcnt(create_operand(destination), create_operand(source)); break; + case int32: c.popcnt(create_operand(destination), create_operand(source)); break; + case int64: c.popcnt(create_operand(destination), create_operand(source)); break; + default: throw_error(); + } + }; break; + + case x86_lzcnt: + { + ir_operand destination = working_operation.destinations[0]; + ir_operand source = working_operation.sources[0]; + + assert_same_size({destination, source}); + + switch (destination.meta_data) + { + case int32: c.lzcnt(create_operand(destination), create_operand(source)); break; + case int64: c.lzcnt(create_operand(destination), create_operand(source)); break; + default: throw_error(); + } + } break; + + case x86_add_flags: + case x86_sub_flags: + { + ir_operand* destinations = working_operation.destinations.data; + ir_operand* sources = working_operation.sources.data; + + assert_is_register(destinations[0]); + assert_same_registers(destinations[0], sources[0]); + + assert(ir_operand::is_register(&destinations[0])); + assert(ir_operand::is_register(&sources[0])); + + Xbyak::Operand dn = create_operand(destinations[0]); + + assert_operand_count(&working_operation, 5, 2); + + if (ir_operand::is_constant(&sources[1])) + { + uint64_t imm = sources[1].value; + + if (imm >= INT32_MAX) + { + throw_error(); + } + + switch (instruction) + { + case x86_add_flags: c.add(dn, imm); break; + case x86_sub_flags: c.sub(dn, imm); break; + default: throw_error(); + } + } + else + { + assert(destinations[0].value == sources[0].value); + assert(destinations[0].meta_data == sources[0].meta_data); + assert(destinations[0].meta_data == sources[1].meta_data); + + Xbyak::Operand m = create_operand(sources[1]); + + switch (instruction) + { + case x86_add_flags: c.add(dn, m); break; + case x86_sub_flags: c.sub(dn, m); break; + default: throw_error(); + } + } + + c.sets(create_operand(destinations[1])); + c.sete(create_operand(destinations[2])); + c.setc(create_operand(destinations[3])); + c.seto(create_operand(destinations[4])); + + for (int i = 1; i < 5; ++i) + { + c.and_(create_operand(destinations[i]), 1); + } + }; break; case ir_floating_point_compare_equal: case ir_floating_point_compare_less: diff --git a/src/assembly/x86/x86_pipeline.cpp b/src/assembly/x86/x86_pipeline.cpp index a7edacd..dd8b4f6 100755 --- a/src/assembly/x86/x86_pipeline.cpp +++ b/src/assembly/x86/x86_pipeline.cpp @@ -7,6 +7,8 @@ #include "ir/basic_register_allocator.h" #include "ir/undefined_behavior_check.h" +#include + void assemble_x86_64_pipeline(void** result_code, uint64_t* result_code_size, ir_operation_block* source_ir, bool optimize, abi working_abi, compiler_flags flags) { arena_allocator* allocator = source_ir->allocator; @@ -33,6 +35,11 @@ void assemble_x86_64_pipeline(void** result_code, uint64_t* result_code_size, ir if (flags & optimize_ssa) { ssa_construct_and_optimize(source_ir, flags); + + if (flags & mathmatical_fold) + { + ir_operation_block::log(source_ir); + } } x86_pre_allocator_context::run_pass(&pre_allocation_data, pre_allocated_code, source_ir, working_abi.cpu,working_abi.os); diff --git a/src/assembly/x86/x86_pre_allocator.cpp b/src/assembly/x86/x86_pre_allocator.cpp index 8dd7c20..55b0a33 100755 --- a/src/assembly/x86/x86_pre_allocator.cpp +++ b/src/assembly/x86/x86_pre_allocator.cpp @@ -189,6 +189,9 @@ static bool instruction_is_commutative(uint64_t instruction) case x86_mulpd: case x86_mulss: case x86_mulsd: + case x86_add_flags: + case ir_compare_equal: + case ir_compare_not_equal: { return true; }; @@ -205,7 +208,7 @@ static void swap(ir_operand* l, ir_operand* r) *r = tmp; } -static void emit_d_n_f_d_n_m(x86_pre_allocator_context* result, ir_instructions instruction, ir_operand destination, ir_operand source_0, ir_operand source_1) +static void emit_d_n_f_d_n_m(x86_pre_allocator_context* result, ir_instructions instruction, ir_operand destination, ir_operand source_0, ir_operand source_1, ir_operation* source_operation) { assert_same_size({ destination, source_0, source_1 }); @@ -214,48 +217,113 @@ static void emit_d_n_f_d_n_m(x86_pre_allocator_context* result, ir_instructions swap(&source_0, &source_1); } + if (instruction_is_commutative(instruction) && ir_operand::are_equal(destination, source_1) && !ir_operand::are_equal(destination, source_0)) + { + swap(&source_0, &source_1); + } + + intrusive_linked_list_element* core_operation = nullptr; + if ((ir_operand::is_constant(&source_1) && source_1.value < INT32_MAX) && !ir_operand::is_constant(&source_0) && instruction != ir_multiply) { + if (instruction == ir_add && ir_operand::get_raw_size(&destination) > int16) + { + ir_operation_block::emitds(result->ir, x86_lea, destination, source_0, source_1); + + return; + } + if (!ir_operand::are_equal(destination, source_0)) { emit_move(result,destination, source_0); } - ir_operation_block::emitds(result->ir, instruction, destination, destination, source_1); - - return; - } - - ir_operand working_destination = destination; - ir_operand working_source_0 = register_or_constant(result, &source_0); - ir_operand working_source_1 = register_or_constant(result, &source_1); - - if (instruction_is_commutative(instruction) && ir_operand::are_equal(working_destination, working_source_1) && !ir_operand::are_equal(working_destination, working_source_0)) - { - swap(&source_0, &source_1); - } - - if (ir_operand::are_equal(working_destination, working_source_0)) - { - ir_operation_block::emitds(result->ir, instruction, working_destination, working_destination, working_source_1); - } - else if (instruction == ir_add && ir_operand::get_raw_size(&working_destination) > int16) - { - ir_operation_block::emitds(result->ir, x86_lea, working_destination, working_source_0, working_source_1); - } - else if (!ir_operand::are_equal(working_destination, working_source_1)) - { - emit_move(result,destination, source_0); - ir_operation_block::emitds(result->ir, instruction, working_destination, working_destination, working_source_1); + core_operation = ir_operation_block::emitds(result->ir, instruction, destination, destination, source_1); } else { - ir_operand working_scrap = create_scrap_operand(result, destination.meta_data); + ir_operand working_destination = destination; + ir_operand working_source_0 = register_or_constant(result, &source_0); + ir_operand working_source_1 = register_or_constant(result, &source_1); - emit_move(result, working_scrap, working_source_0); - ir_operation_block::emitds(result->ir, instruction, working_scrap, working_scrap, working_source_1); - - emit_move(result, working_destination, working_scrap); + if (ir_operand::are_equal(working_destination, working_source_0)) + { + core_operation = ir_operation_block::emitds(result->ir, instruction, working_destination, working_destination, working_source_1); + } + else if (instruction == ir_add && ir_operand::get_raw_size(&working_destination) > int16) + { + ir_operation_block::emitds(result->ir, x86_lea, working_destination, working_source_0, working_source_1); + } + else if (!ir_operand::are_equal(working_destination, working_source_1)) + { + emit_move(result,destination, source_0); + core_operation = ir_operation_block::emitds(result->ir, instruction, working_destination, working_destination, working_source_1); + } + else + { + ir_operand working_scrap = create_scrap_operand(result, destination.meta_data); + + emit_move(result, working_scrap, working_source_0); + core_operation = ir_operation_block::emitds(result->ir, instruction, working_scrap, working_scrap, working_source_1); + + emit_move(result, working_destination, working_scrap); + } + } + + switch (instruction) + { + case x86_add_flags: + case x86_sub_flags: + { + if (core_operation == nullptr) + { + throw_error(); + } + + ir_operand new_destinations[5]; + ir_operand* new_sources = core_operation->data.sources.data; + + new_destinations[0] = core_operation->data.destinations[0]; + + for (int i = 1; i < 5; ++i) + { + new_destinations[i] = source_operation->destinations[i]; + } + + ir_operation_block::emit_with(result->ir, instruction, new_destinations, 5, new_sources, 2, core_operation); + + core_operation->data.instruction = ir_no_operation; + core_operation->data.sources.count = 0; + core_operation->data.destinations.count = 0; + }; break; + + case x86_cmpss: + case x86_cmpsd: + case x86_cmpps: + case x86_cmppd: + { + if (core_operation == nullptr) + { + throw_error(); + } + + ir_operand sources_with_control[3]; + + for (int i = 0; i < 2; ++i) + { + sources_with_control[i] = core_operation->data.sources[i]; + } + + sources_with_control[2] = source_operation->sources[2]; + + assert_is_constant(sources_with_control[2]); + + ir_operation_block::emit_with(result->ir, instruction, core_operation->data.destinations.data, 1, sources_with_control, 3, core_operation); + + core_operation->data.instruction = ir_no_operation; + core_operation->data.sources.count = 0; + core_operation->data.destinations.count = 0; + }; break; } } @@ -327,10 +395,24 @@ static void emit_compare(x86_pre_allocator_context* result, uint64_t instruction { assert_same_size({ destination, source_0, source_1 }); + if (ir_operand::is_constant(&source_0) && !ir_operand::is_constant(&source_1) && instruction_is_commutative(instruction)) + { + swap(&source_0, &source_1); + } + ir_operand working_destination = destination; ir_operand working_source_0 = register_or_constant(result, &source_0); - ir_operand working_source_1 = register_or_constant(result, &source_1); + ir_operand working_source_1; + + if (ir_operand::is_constant(&source_1) && source_1.value < INT32_MAX) + { + working_source_1 = source_1; + } + else + { + working_source_1 = register_or_constant(result, &source_1); + } ir_operation_block::emitds(result->ir, instruction, working_destination, working_source_0, working_source_1); } @@ -679,7 +761,7 @@ static void emit_compare_and_swap(x86_pre_allocator_context* result, ir_operatio ir_operand rax = RAX(expecting.meta_data); emit_move(result, rax, expecting); - ir_operation_block::emitds(result->ir, ir_compare_and_swap, destination, address, rax, to_swap); + ir_operation_block::emitds(result->ir, x86_cmpxchg, destination, address, rax, to_swap); ir_operation_block::emits(result->ir, ir_instructions::ir_register_allocator_p_unlock, RAX(int64)); } @@ -843,6 +925,91 @@ static void emit_shuf(x86_pre_allocator_context* result, ir_operation* operation emit_move(result, destination, working); } +static void emit_store(x86_pre_allocator_context* result, ir_operation* operation) +{ + ir_operand source_temp[3]; + + ir_operation_block* ir = result->ir; + + for (int i = 0; i < operation->sources.count; ++i) + { + ir_operand working = operation->sources[i]; + + if (ir_operand::is_constant(&working) && working.value > INT32_MAX) + { + working = register_or_constant(result, working); + } + + source_temp[i] = working; + } + + if (operation->sources.count == 3) + { + if (ir_operand::is_constant(&operation->sources[0]) && !ir_operand::is_constant(&operation->sources[1])) + { + swap(&source_temp[0], &source_temp[1]); + } + } + + if (ir_operand::is_constant(&source_temp[0])) + { + source_temp[0] = register_or_constant(result, source_temp[0]); + } + + int last = operation->sources.count - 1; + + if (ir_operand::is_constant(&source_temp[last])) + { + source_temp[last] = register_or_constant(result, source_temp[last]); + } + + ir_operation_block::emit_with( + ir, + operation->instruction, + operation->destinations.data, operation->destinations.count, + source_temp, operation->sources.count + ); +} + +static void emit_load(x86_pre_allocator_context* result, ir_operation* operation) +{ + ir_operation_block* ir = result->ir; + + ir_operand source_temp[2]; + + for (int i = 0; i < operation->sources.count; ++i) + { + ir_operand working = operation->sources[i]; + + if (ir_operand::is_constant(&working) && working.value > INT32_MAX) + { + working = register_or_constant(result, working); + } + + source_temp[i] = working; + } + + if (operation->sources.count == 2) + { + if (ir_operand::is_constant(&operation->sources[0]) && !ir_operand::is_constant(&operation->sources[1])) + { + swap(&source_temp[0], &source_temp[1]); + } + } + + if (ir_operand::is_constant(&source_temp[0])) + { + source_temp[0] = register_or_constant(result, source_temp[0]); + } + + ir_operation_block::emit_with( + ir, + operation->instruction, + operation->destinations.data, operation->destinations.count, + source_temp, operation->sources.count + ); +} + static void emit_pre_allocation_instruction(x86_pre_allocator_context* pre_allocator_context, ir_operation* operation, os_information os) { ir_instructions working_instruction = (ir_instructions)operation->instruction; @@ -876,18 +1043,46 @@ static void emit_pre_allocation_instruction(x86_pre_allocator_context* pre_alloc case x86_subps: case x86_subsd: case x86_subss: + + case x86_paddb: + case x86_paddw: + case x86_paddd: + case x86_paddq: case x86_xorps: case x86_pand: case x86_orps: case x86_pandn: + + case x86_haddpd: + case x86_haddps: { assert_operand_count(operation, 1, 2); assert_is_register(operation->destinations[0]); - emit_d_n_f_d_n_m(pre_allocator_context, operation->instruction, operation->destinations[0], operation->sources[0], operation->sources[1]); + emit_d_n_f_d_n_m(pre_allocator_context, operation->instruction, operation->destinations[0], operation->sources[0], operation->sources[1], nullptr); }; break; + case x86_cmpss: + case x86_cmpsd: + case x86_cmpps: + case x86_cmppd: + { + assert_operand_count(operation, 1, 3); + assert_is_register(operation->destinations[0]); + + emit_d_n_f_d_n_m(pre_allocator_context, operation->instruction, operation->destinations[0], operation->sources[0], operation->sources[1], operation); + }; break; + + case x86_add_flags: + case x86_sub_flags: + { + assert_operand_count(operation, 5, 2); + assert_is_register(operation->destinations[0]); + + emit_d_n_f_d_n_m(pre_allocator_context, operation->instruction, operation->destinations[0], operation->sources[0], operation->sources[1], operation); + }; break; + case ir_floating_point_add: case ir_floating_point_subtract: case ir_floating_point_multiply: @@ -1036,11 +1231,26 @@ static void emit_pre_allocation_instruction(x86_pre_allocator_context* pre_alloc }; break; case ir_load: + { + emit_load(pre_allocator_context, operation); + }; break; + + case ir_store: + { + emit_store(pre_allocator_context, operation); + }; break; + case ir_jump_if: case ir_vector_zero: case ir_vector_one: case x86_roundss: case x86_roundsd: + case x86_cvtsd2ss: + case x86_cvtss2sd: + case x86_sqrtps: + case x86_sqrtpd: + case x86_sqrtss: + case x86_sqrtsd: { emit_as_is(pre_allocator_context, operation); }; break; @@ -1053,7 +1263,9 @@ static void emit_pre_allocation_instruction(x86_pre_allocator_context* pre_alloc case ir_assert_false: case ir_assert_true: - case ir_store: //TODO this can be optimized + + case x86_lzcnt: + case x86_popcnt: { emit_with_possible_remaps(pre_allocator_context, operation); }; break; @@ -1097,7 +1309,7 @@ static void emit_pre_allocation_instruction(x86_pre_allocator_context* pre_alloc emit_external_call(pre_allocator_context, operation, os); }; break; - case ir_compare_and_swap: + case x86_cmpxchg: { emit_compare_and_swap(pre_allocator_context, operation); }; break; @@ -1119,6 +1331,24 @@ static void emit_pre_allocation_instruction(x86_pre_allocator_context* pre_alloc throw_error(); } + switch (operation->instruction) + { + case ir_compare_equal: + case ir_compare_greater_equal_signed: + case ir_compare_greater_equal_unsigned: + case ir_compare_greater_signed: + case ir_compare_greater_unsigned: + case ir_compare_less_equal_signed: + case ir_compare_less_equal_unsigned: + case ir_compare_less_signed: + case ir_compare_less_unsigned: + case ir_compare_not_equal: + return; + + default: + break; + } + for (int i = 0; i < operation->destinations.count; ++i) { ir_operand destination = operation->destinations[i]; diff --git a/src/emulator/aarch64/aarch64_externals_interpreter.cpp b/src/emulator/aarch64/aarch64_externals_interpreter.cpp index 7846c21..c32ae1d 100644 --- a/src/emulator/aarch64/aarch64_externals_interpreter.cpp +++ b/src/emulator/aarch64/aarch64_externals_interpreter.cpp @@ -205,4 +205,14 @@ uint64_t use_fast_float_interpreter(interpreter_data* ctx) uint64_t use_x86_sse41_interpreter(interpreter_data* ctx) { return true; +} + +uint64_t use_x86_interpreter(interpreter_data* ctx) +{ + return false; +} + +uint64_t use_x86_lzcnt_interpreter(interpreter_data* ctx) +{ + return false; } \ No newline at end of file diff --git a/src/emulator/aarch64/aarch64_externals_jit.cpp b/src/emulator/aarch64/aarch64_externals_jit.cpp index 287663b..b4d33a6 100644 --- a/src/emulator/aarch64/aarch64_externals_jit.cpp +++ b/src/emulator/aarch64/aarch64_externals_jit.cpp @@ -235,6 +235,60 @@ uint64_t use_x86_sse41_jit(ssa_emit_context* ctx) return 1; } +uint64_t use_x86_jit(ssa_emit_context* ctx) +{ + return 1; +} + +uint64_t use_x86_lzcnt_jit(ssa_emit_context* ctx) +{ + return 1; +} + +static ir_operand x86_add_subtract_set_flags_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m, bool is_add) +{ + aarch64_emit_context* actx = (aarch64_emit_context*)ctx->context_data; + aarch64_context_offsets offsets = actx->process->guest_context_offset_data; + + ir_operand result = ssa_emit_context::create_local(ctx, O); + + ir_operand destinations[] = + { + result, + aarch64_emit_context::get_context_reg_raw(actx, offsets.n_offset), + aarch64_emit_context::get_context_reg_raw(actx, offsets.z_offset), + aarch64_emit_context::get_context_reg_raw(actx, offsets.c_offset), + aarch64_emit_context::get_context_reg_raw(actx, offsets.v_offset), + }; + + ir_operand sources[] = + { + n, + m + }; + + ir_operation_block::emit_with(ctx->ir, is_add ? x86_add_flags : x86_sub_flags, destinations, 5, sources, 2); + + if (!is_add) + { + ir_operand c = aarch64_emit_context::get_context_reg_raw(actx, offsets.c_offset); + + ir_operation_block::emitds(ctx->ir, ir_bitwise_exclusive_or, c, c, ir_operand::create_con(1, c.meta_data)); + } + + return result; +} + +ir_operand x86_add_set_flags_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m) +{ + return x86_add_subtract_set_flags_jit(ctx, O, n, m, true); +} + +ir_operand x86_subtract_set_flags_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m) +{ + return x86_add_subtract_set_flags_jit(ctx, O, n, m, false); +} + ir_operand intrinsic_unary_jit(ssa_emit_context* ctx,uint64_t R, uint64_t instruction, ir_operand source) { return ssa_emit_context::emit_ssa(ctx, (ir_instructions)instruction, source, R); @@ -253,4 +307,9 @@ ir_operand intrinsic_binary_imm_jit(ssa_emit_context* ctx,uint64_t R, uint64_t i ir_operand intrinsic_ternary_imm_jit(ssa_emit_context* ctx,uint64_t R, uint64_t instruction, ir_operand source_0, ir_operand source_1, uint64_t source_2) { return ssa_emit_context::emit_ssa(ctx, (ir_instructions)instruction, source_0, source_1, ir_operand::create_con(source_2)); +} + +ir_operand intrinsic_ternary_jit(ssa_emit_context* ctx,uint64_t R, uint64_t instruction, ir_operand source_0, ir_operand source_1, ir_operand source_2) +{ + return ssa_emit_context::emit_ssa(ctx, (ir_instructions)instruction, source_0, source_1, source_2); } \ No newline at end of file diff --git a/src/emulator/aarch64/aarch64_impl.cpp b/src/emulator/aarch64/aarch64_impl.cpp index de2bfed..44eca1f 100644 --- a/src/emulator/aarch64/aarch64_impl.cpp +++ b/src/emulator/aarch64/aarch64_impl.cpp @@ -116,6 +116,18 @@ R intrinsic_ternary_imm_interpreter(interpreter_data* ctx, uint64_t instruction, } +template +O x86_add_set_flags_interpreter(interpreter_data* ctx, O n, O m) +{ + +} + +template +O x86_subtract_set_flags_interpreter(interpreter_data* ctx, O n, O m) +{ + +} + static void call_add_subtract_imm12_interpreter(interpreter_data* ctx, uint32_t instruction) { int sf = (instruction >> 31) & 1; @@ -1430,6 +1442,120 @@ static void emit_faddp_scalar_jit(ssa_emit_context* ctx, uint32_t instruction) faddp_scalar_jit(ctx, sz, Rn, Rd); } +static void call_fcmeq_vector_zero_interpreter(interpreter_data* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmeq_vector_zero_interpreter(ctx, Q, sz, Rn, Rd); +} + +static void emit_fcmeq_vector_zero_jit(ssa_emit_context* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmeq_vector_zero_jit(ctx, Q, sz, Rn, Rd); +} + +static void call_fcmgt_vector_zero_interpreter(interpreter_data* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmgt_vector_zero_interpreter(ctx, Q, sz, Rn, Rd); +} + +static void emit_fcmgt_vector_zero_jit(ssa_emit_context* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmgt_vector_zero_jit(ctx, Q, sz, Rn, Rd); +} + +static void call_fcmge_vector_zero_interpreter(interpreter_data* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmge_vector_zero_interpreter(ctx, Q, sz, Rn, Rd); +} + +static void emit_fcmge_vector_zero_jit(ssa_emit_context* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmge_vector_zero_jit(ctx, Q, sz, Rn, Rd); +} + +static void call_fcmeq_vector_register_interpreter(interpreter_data* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rm = (instruction >> 16) & 31; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmeq_vector_register_interpreter(ctx, Q, sz, Rm, Rn, Rd); +} + +static void emit_fcmeq_vector_register_jit(ssa_emit_context* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rm = (instruction >> 16) & 31; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmeq_vector_register_jit(ctx, Q, sz, Rm, Rn, Rd); +} + +static void call_fcmgt_vector_register_interpreter(interpreter_data* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rm = (instruction >> 16) & 31; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmgt_vector_register_interpreter(ctx, Q, sz, Rm, Rn, Rd); +} + +static void emit_fcmgt_vector_register_jit(ssa_emit_context* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rm = (instruction >> 16) & 31; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmgt_vector_register_jit(ctx, Q, sz, Rm, Rn, Rd); +} + +static void call_fcmge_vector_register_interpreter(interpreter_data* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rm = (instruction >> 16) & 31; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmge_vector_register_interpreter(ctx, Q, sz, Rm, Rn, Rd); +} + +static void emit_fcmge_vector_register_jit(ssa_emit_context* ctx, uint32_t instruction) +{ + int Q = (instruction >> 30) & 1; + int sz = (instruction >> 22) & 1; + int Rm = (instruction >> 16) & 31; + int Rn = (instruction >> 5) & 31; + int Rd = (instruction >> 0) & 31; + fcmge_vector_register_jit(ctx, Q, sz, Rm, Rn, Rd); +} + static void call_fadd_scalar_interpreter(interpreter_data* ctx, uint32_t instruction) { int ftype = (instruction >> 22) & 3; @@ -2616,120 +2742,6 @@ static void emit_st1_single_structure_post_index_jit(ssa_emit_context* ctx, uint st1_single_structure_post_index_jit(ctx, Q, Rm, opcode, S, size, Rn, Rt); } -static void call_fcmeq_vector_zero_interpreter(interpreter_data* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmeq_vector_zero_interpreter(ctx, Q, sz, Rn, Rd); -} - -static void emit_fcmeq_vector_zero_jit(ssa_emit_context* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmeq_vector_zero_jit(ctx, Q, sz, Rn, Rd); -} - -static void call_fcmgt_vector_zero_interpreter(interpreter_data* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmgt_vector_zero_interpreter(ctx, Q, sz, Rn, Rd); -} - -static void emit_fcmgt_vector_zero_jit(ssa_emit_context* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmgt_vector_zero_jit(ctx, Q, sz, Rn, Rd); -} - -static void call_fcmge_vector_zero_interpreter(interpreter_data* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmge_vector_zero_interpreter(ctx, Q, sz, Rn, Rd); -} - -static void emit_fcmge_vector_zero_jit(ssa_emit_context* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmge_vector_zero_jit(ctx, Q, sz, Rn, Rd); -} - -static void call_fcmeq_vector_register_interpreter(interpreter_data* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rm = (instruction >> 16) & 31; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmeq_vector_register_interpreter(ctx, Q, sz, Rm, Rn, Rd); -} - -static void emit_fcmeq_vector_register_jit(ssa_emit_context* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rm = (instruction >> 16) & 31; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmeq_vector_register_jit(ctx, Q, sz, Rm, Rn, Rd); -} - -static void call_fcmgt_vector_register_interpreter(interpreter_data* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rm = (instruction >> 16) & 31; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmgt_vector_register_interpreter(ctx, Q, sz, Rm, Rn, Rd); -} - -static void emit_fcmgt_vector_register_jit(ssa_emit_context* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rm = (instruction >> 16) & 31; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmgt_vector_register_jit(ctx, Q, sz, Rm, Rn, Rd); -} - -static void call_fcmge_vector_register_interpreter(interpreter_data* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rm = (instruction >> 16) & 31; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmge_vector_register_interpreter(ctx, Q, sz, Rm, Rn, Rd); -} - -static void emit_fcmge_vector_register_jit(ssa_emit_context* ctx, uint32_t instruction) -{ - int Q = (instruction >> 30) & 1; - int sz = (instruction >> 22) & 1; - int Rm = (instruction >> 16) & 31; - int Rn = (instruction >> 5) & 31; - int Rd = (instruction >> 0) & 31; - fcmge_vector_register_jit(ctx, Q, sz, Rm, Rn, Rd); -} - static void call_floating_point_conditional_select_interpreter(interpreter_data* ctx, uint32_t instruction) { int ftype = (instruction >> 22) & 3; @@ -2855,6 +2867,12 @@ void init_aarch64_decoder(guest_process* process) append_table(process, "010111111-------0-01-0----------", (void*)emit_fmul_accumulate_scalar_jit, (void*)call_fmul_accumulate_scalar_interpreter,nullptr, "fmul_accumulate_scalar"); append_table(process, "0-0011111-------0-01-0----------", (void*)emit_fmul_accumulate_element_jit, (void*)call_fmul_accumulate_element_interpreter,nullptr, "fmul_accumulate_element"); append_table(process, "011111100-110000110110----------", (void*)emit_faddp_scalar_jit, (void*)call_faddp_scalar_interpreter,nullptr, "faddp_scalar"); + append_table(process, "0-0011101-100000110110----------", (void*)emit_fcmeq_vector_zero_jit, (void*)call_fcmeq_vector_zero_interpreter,nullptr, "fcmeq_vector_zero"); + append_table(process, "0-0011101-100000110010----------", (void*)emit_fcmgt_vector_zero_jit, (void*)call_fcmgt_vector_zero_interpreter,nullptr, "fcmgt_vector_zero"); + append_table(process, "0-1011101-100000110010----------", (void*)emit_fcmge_vector_zero_jit, (void*)call_fcmge_vector_zero_interpreter,nullptr, "fcmge_vector_zero"); + append_table(process, "0-0011100-1-----111001----------", (void*)emit_fcmeq_vector_register_jit, (void*)call_fcmeq_vector_register_interpreter,nullptr, "fcmeq_vector_register"); + append_table(process, "0-1011101-1-----111001----------", (void*)emit_fcmgt_vector_register_jit, (void*)call_fcmgt_vector_register_interpreter,nullptr, "fcmgt_vector_register"); + append_table(process, "0-1011100-1-----111001----------", (void*)emit_fcmge_vector_register_jit, (void*)call_fcmge_vector_register_interpreter,nullptr, "fcmge_vector_register"); append_table(process, "00011110--1-----001010----------", (void*)emit_fadd_scalar_jit, (void*)call_fadd_scalar_interpreter,nullptr, "fadd_scalar"); append_table(process, "00011110--1-----001110----------", (void*)emit_fsub_scalar_jit, (void*)call_fsub_scalar_interpreter,nullptr, "fsub_scalar"); append_table(process, "00011110--1-----000010----------", (void*)emit_fmul_scalar_jit, (void*)call_fmul_scalar_interpreter,nullptr, "fmul_scalar"); @@ -2915,12 +2933,6 @@ void init_aarch64_decoder(guest_process* process) append_table(process, "0-001100100-----1000------------", (void*)emit_st2_multiple_structures_post_index_jit, (void*)call_st2_multiple_structures_post_index_interpreter,nullptr, "st2_multiple_structures_post_index"); append_table(process, "0-00110100000000--0-------------", (void*)emit_st1_single_structure_no_offset_jit, (void*)call_st1_single_structure_no_offset_interpreter,nullptr, "st1_single_structure_no_offset"); append_table(process, "0-001101100-------0-------------", (void*)emit_st1_single_structure_post_index_jit, (void*)call_st1_single_structure_post_index_interpreter,nullptr, "st1_single_structure_post_index"); - append_table(process, "0-0011101-100000110110----------", (void*)emit_fcmeq_vector_zero_jit, (void*)call_fcmeq_vector_zero_interpreter,nullptr, "fcmeq_vector_zero"); - append_table(process, "0-0011101-100000110010----------", (void*)emit_fcmgt_vector_zero_jit, (void*)call_fcmgt_vector_zero_interpreter,nullptr, "fcmgt_vector_zero"); - append_table(process, "0-1011101-100000110010----------", (void*)emit_fcmge_vector_zero_jit, (void*)call_fcmge_vector_zero_interpreter,nullptr, "fcmge_vector_zero"); - append_table(process, "0-0011100-1-----111001----------", (void*)emit_fcmeq_vector_register_jit, (void*)call_fcmeq_vector_register_interpreter,nullptr, "fcmeq_vector_register"); - append_table(process, "0-1011101-1-----111001----------", (void*)emit_fcmgt_vector_register_jit, (void*)call_fcmgt_vector_register_interpreter,nullptr, "fcmgt_vector_register"); - append_table(process, "0-1011100-1-----111001----------", (void*)emit_fcmge_vector_register_jit, (void*)call_fcmge_vector_register_interpreter,nullptr, "fcmge_vector_register"); append_table(process, "00011110--1---------11----------", (void*)emit_floating_point_conditional_select_jit, (void*)call_floating_point_conditional_select_interpreter,nullptr, "floating_point_conditional_select"); append_table(process, "00011110--1-----001000-----0-000", (void*)emit_fcmp_jit, (void*)call_fcmp_interpreter,nullptr, "fcmp"); append_table(process, "00011110--1---------01-----0----", (void*)emit_fccmp_jit, (void*)call_fccmp_interpreter,nullptr, "fccmp"); @@ -3119,6 +3131,18 @@ template O add_subtract_impl_interpreter(interpreter_data* ctx, O n, O m, uint64_t set_flags, uint64_t is_add) { O d; + if ((((uint64_t)set_flags && (uint64_t)use_x86_interpreter(ctx)))) + { + if ((is_add)) + { + d = x86_add_set_flags_interpreter(ctx,n,m); + } + else + { + d = x86_subtract_set_flags_interpreter(ctx,n,m); + } + return d; + } if ((is_add)) { d = ((O)n + (O)m); @@ -3233,6 +3257,13 @@ void branch_long_universal_interpreter(interpreter_data* ctx, uint64_t Rn, uint6 _branch_long_interpreter(ctx,branch_location); } +uint64_t select_interpreter(interpreter_data* ctx, uint64_t condition, uint64_t yes, uint64_t no) +{ + if ((condition)) + return yes; + return no; +} + uint64_t create_mask_interpreter(interpreter_data* ctx, uint64_t bits) { if ((((uint64_t)bits >= (uint64_t)64ULL))) @@ -3254,6 +3285,32 @@ uint64_t shift_left_check_interpreter(interpreter_data* ctx, uint64_t to_shift, return result; } +uint64_t get_x86_rounding_mode_interpreter(interpreter_data* ctx, uint64_t rounding) +{ + uint64_t rounding_control; + if ((((uint64_t)rounding == (uint64_t)FPRounding_TIEEVEN))) + { + rounding_control = 0ULL; + } + else if ((((uint64_t)rounding == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)rounding == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)rounding == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else + { + undefined_interpreter(ctx); + } + return rounding_control; +} + uint64_t shift_right_check_interpreter(interpreter_data* ctx, uint64_t to_shift, uint64_t shift, uint64_t size, uint64_t is_unsigned) { uint64_t result = 0ULL; @@ -4109,45 +4166,6 @@ void memory_1_interpreter(interpreter_data* ctx, uint64_t wback, uint64_t Q, uin } -void fcm_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz) -{ - uint128_t n = V_interpreter(ctx,Rn); - uint128_t m; - uint128_t result = 0; - if ((((uint64_t)Rm == (uint64_t)-1ULL))) - { - m = 0; - } - else - { - m = V_interpreter(ctx,Rm); - } - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); - uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) - { - uint64_t element1 = uint128_t::extract(n, e, esize); - uint64_t element2 = uint128_t::extract(m, e, esize); - uint64_t element_result; - if ((((uint64_t)mode == (uint64_t)0ULL))) - { - element_result = ((uint64_t)0ULL - (uint64_t)FPCompareEQ_interpreter(ctx,element1,element2,fpcr_state,esize)); - } - else if ((((uint64_t)mode == (uint64_t)1ULL))) - { - element_result = ((uint64_t)0ULL - (uint64_t)FPCompareGT_interpreter(ctx,element1,element2,fpcr_state,esize)); - } - else if ((((uint64_t)mode == (uint64_t)2ULL))) - { - element_result = ((uint64_t)0ULL - (uint64_t)FPCompareGE_interpreter(ctx,element1,element2,fpcr_state,esize)); - } - uint128_t::insert(&result, e, esize, element_result); - } - V_interpreter(ctx,Rd,result); -} - uint64_t bits_r_interpreter(interpreter_data* ctx, uint64_t operand, uint64_t top, uint64_t bottom) { top = ((uint64_t)top + (uint64_t)1ULL); @@ -4693,6 +4711,10 @@ uint64_t FixedToFP_interpreter(interpreter_data* ctx, uint64_t source, uint64_t { working_result = convert_to_float((uint32_t)source, 1); } + if ((((uint64_t)fracbits == (uint64_t)0ULL))) + { + return working_result; + } return (undefined_value()); } if (from == 64ULL) @@ -4711,6 +4733,10 @@ uint64_t FixedToFP_interpreter(interpreter_data* ctx, uint64_t source, uint64_t { working_result = convert_to_float((uint64_t)source, 1); } + if ((((uint64_t)fracbits == (uint64_t)0ULL))) + { + return working_result; + } return (undefined_value()); } @@ -4733,6 +4759,10 @@ uint64_t FixedToFP_interpreter(interpreter_data* ctx, uint64_t source, uint64_t { working_result = convert_to_float((uint32_t)source, 1); } + if ((((uint64_t)fracbits == (uint64_t)0ULL))) + { + return working_result; + } return (undefined_value()); } if (from == 64ULL) @@ -4751,6 +4781,10 @@ uint64_t FixedToFP_interpreter(interpreter_data* ctx, uint64_t source, uint64_t { working_result = convert_to_float((uint64_t)source, 1); } + if ((((uint64_t)fracbits == (uint64_t)0ULL))) + { + return working_result; + } return (undefined_value()); } @@ -4762,7 +4796,7 @@ uint64_t FixedToFP_interpreter(interpreter_data* ctx, uint64_t source, uint64_t uint64_t FPToFixed_interpreter(interpreter_data* ctx, uint64_t source, uint64_t fracbits, uint64_t is_unsigned, uint64_t round, uint64_t to, uint64_t from) { - if ((((uint64_t)((uint64_t)((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)((uint64_t)to != (uint64_t)16ULL)) && (uint64_t)((uint64_t)from != (uint64_t)16ULL)) && (uint64_t)((uint64_t)round != (uint64_t)FPRounding_ODD)))) + if ((((uint64_t)((uint64_t)((uint64_t)((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)((uint64_t)to != (uint64_t)16ULL)) && (uint64_t)((uint64_t)from != (uint64_t)16ULL)) && (uint64_t)((uint64_t)round != (uint64_t)FPRounding_ODD)) && (uint64_t)use_x86_sse41_interpreter(ctx)))) { if (from == 32ULL) { @@ -4792,50 +4826,43 @@ uint64_t FPToFixed_interpreter(interpreter_data* ctx, uint64_t source, uint64_t uint32_t power = create_fixed_from_fbits_interpreter(ctx,fracbits,from); working = (undefined_value()); } - if ((use_x86_sse41_interpreter(ctx))) + uint64_t rounding_control; + if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) { - uint64_t rounding_control; - if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) + rounding_control = 0ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) + { + rounding_control = 2ULL; + if (((undefined_value()))) { - rounding_control = 0ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) - { - rounding_control = 1ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) - { - rounding_control = 2ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) - { - rounding_control = 3ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) - { - rounding_control = 2ULL; - if (((undefined_value()))) - { - working = ((undefined_value())); - } - } - else - { - undefined_interpreter(ctx); - } - if ((((uint64_t)from == (uint64_t)32ULL))) - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); - } - else - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + working = ((undefined_value())); } } else { undefined_interpreter(ctx); } + if ((((uint64_t)from == (uint64_t)32ULL))) + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); + } + else + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + } uint32_t result = 0ULL; if (((undefined_value()))) { @@ -4893,50 +4920,43 @@ uint64_t FPToFixed_interpreter(interpreter_data* ctx, uint64_t source, uint64_t uint32_t power = create_fixed_from_fbits_interpreter(ctx,fracbits,from); working = (undefined_value()); } - if ((use_x86_sse41_interpreter(ctx))) + uint64_t rounding_control; + if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) { - uint64_t rounding_control; - if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) + rounding_control = 0ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) + { + rounding_control = 2ULL; + if (((undefined_value()))) { - rounding_control = 0ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) - { - rounding_control = 1ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) - { - rounding_control = 2ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) - { - rounding_control = 3ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) - { - rounding_control = 2ULL; - if (((undefined_value()))) - { - working = ((undefined_value())); - } - } - else - { - undefined_interpreter(ctx); - } - if ((((uint64_t)from == (uint64_t)32ULL))) - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); - } - else - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + working = ((undefined_value())); } } else { undefined_interpreter(ctx); } + if ((((uint64_t)from == (uint64_t)32ULL))) + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); + } + else + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + } uint64_t result = 0ULL; if (((undefined_value()))) { @@ -4998,50 +5018,43 @@ uint64_t FPToFixed_interpreter(interpreter_data* ctx, uint64_t source, uint64_t uint64_t power = create_fixed_from_fbits_interpreter(ctx,fracbits,from); working = (undefined_value()); } - if ((use_x86_sse41_interpreter(ctx))) + uint64_t rounding_control; + if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) { - uint64_t rounding_control; - if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) + rounding_control = 0ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) + { + rounding_control = 2ULL; + if (((undefined_value()))) { - rounding_control = 0ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) - { - rounding_control = 1ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) - { - rounding_control = 2ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) - { - rounding_control = 3ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) - { - rounding_control = 2ULL; - if (((undefined_value()))) - { - working = ((undefined_value())); - } - } - else - { - undefined_interpreter(ctx); - } - if ((((uint64_t)from == (uint64_t)32ULL))) - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); - } - else - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + working = ((undefined_value())); } } else { undefined_interpreter(ctx); } + if ((((uint64_t)from == (uint64_t)32ULL))) + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); + } + else + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + } uint32_t result = 0ULL; if (((undefined_value()))) { @@ -5099,50 +5112,43 @@ uint64_t FPToFixed_interpreter(interpreter_data* ctx, uint64_t source, uint64_t uint64_t power = create_fixed_from_fbits_interpreter(ctx,fracbits,from); working = (undefined_value()); } - if ((use_x86_sse41_interpreter(ctx))) + uint64_t rounding_control; + if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) { - uint64_t rounding_control; - if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) + rounding_control = 0ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) + { + rounding_control = 2ULL; + if (((undefined_value()))) { - rounding_control = 0ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) - { - rounding_control = 1ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) - { - rounding_control = 2ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) - { - rounding_control = 3ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) - { - rounding_control = 2ULL; - if (((undefined_value()))) - { - working = ((undefined_value())); - } - } - else - { - undefined_interpreter(ctx); - } - if ((((uint64_t)from == (uint64_t)32ULL))) - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); - } - else - { - working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + working = ((undefined_value())); } } else { undefined_interpreter(ctx); } + if ((((uint64_t)from == (uint64_t)32ULL))) + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundss,(uint128_t)working,rounding_control); + } + else + { + working = intrinsic_binary_imm_interpreter(ctx,(uint64_t)x86_roundsd,(uint128_t)working,rounding_control); + } uint64_t result = 0ULL; if (((undefined_value()))) { @@ -5183,8 +5189,22 @@ uint64_t FPToFixed_interpreter(interpreter_data* ctx, uint64_t source, uint64_t uint64_t FPConvert_interpreter(interpreter_data* ctx, uint64_t source, uint64_t to, uint64_t from) { + if ((((uint64_t)to == (uint64_t)from))) + { + undefined_interpreter(ctx); + } if ((((uint64_t)((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)((uint64_t)to != (uint64_t)16ULL)) && (uint64_t)((uint64_t)from != (uint64_t)16ULL)))) { + uint64_t result; + if ((((uint64_t)((uint64_t)to == (uint64_t)32ULL) && (uint64_t)((uint64_t)from == (uint64_t)64ULL)))) + { + result = intrinsic_unary_interpreter(ctx,(uint64_t)x86_cvtsd2ss,(uint128_t)source); + } + else + { + result = intrinsic_unary_interpreter(ctx,(uint64_t)x86_cvtss2sd,(uint128_t)source); + } + return result; } return call_interpreter(ctx,source,(uint64_t)0ULL,(uint64_t)0ULL,(uint64_t)to,(uint64_t)from,(uint64_t)0ULL,(uint64_t)FPConvert_I); } @@ -5280,13 +5300,23 @@ void float_binary_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_ void frint_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd, uint64_t rounding) { - uint128_t operand1 = V_interpreter(ctx,Rn); - uint128_t result = 0; + uint128_t operand = V_interpreter(ctx,Rn); uint64_t esize = get_flt_size_interpreter(ctx,ftype); - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - uint64_t working = FPRoundInt_interpreter(ctx,(uint64_t)operand1,fpcr_state,rounding,esize); - uint128_t::insert(&result, 0ULL, esize, working); - V_interpreter(ctx,Rd,result); + if ((((uint64_t)((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)) && (uint64_t)((uint64_t)rounding != (uint64_t)FPRounding_TIEAWAY)))) + { + uint64_t rounding_control = get_x86_rounding_mode_interpreter(ctx,rounding); + uint128_t result = intrinsic_binary_imm_interpreter(ctx,select_interpreter(ctx,((uint64_t)esize == (uint64_t)64ULL),(uint64_t)x86_roundsd,(uint64_t)x86_roundss),operand,rounding_control); + result = clear_vector_scalar_interpreter(ctx,result,esize); + V_interpreter(ctx,Rd,result); + } + else + { + uint128_t result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + uint64_t working = FPRoundInt_interpreter(ctx,(uint64_t)operand,fpcr_state,rounding,esize); + uint128_t::insert(&result, 0ULL, esize, working); + V_interpreter(ctx,Rd,result); + } } void intrinsic_float_binary_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t Q, uint64_t sz, uint64_t float_instruction, uint64_t double_instruction) @@ -5328,6 +5358,7 @@ void intrinsic_float_binary_scalar_interpreter(interpreter_data* ctx, uint64_t R { result = intrinsic_binary_interpreter(ctx,half_instruction,operand1,operand2); uint128_t::insert(&result, 1ULL, 16ULL, (uint64_t)0ULL); + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); } else { @@ -5404,6 +5435,264 @@ uint128_t sse_coppy_gp_across_lanes_interpreter(interpreter_data* ctx, uint64_t } } +void floating_point_multiply_scalar_element_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index) +{ + uint128_t operand1 = V_interpreter(ctx,Rn); + uint128_t operand2 = V_interpreter(ctx,Rm); + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + if ((use_x86_sse_interpreter(ctx))) + { + operand2 = sse_copy_to_xmm_from_xmm_element_interpreter(ctx,operand2,esize,index); + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)32ULL))) + multiply_instruction = (uint64_t)x86_mulss; + else + multiply_instruction = (uint64_t)x86_mulsd; + uint128_t result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,result); + } + else + { + uint128_t result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + uint64_t product = FPMul_interpreter(ctx,(uint64_t)operand1,uint128_t::extract(operand2, index, esize),fpcr_state,esize); + uint128_t::insert(&result, 0ULL, esize, product); + V_interpreter(ctx,Rd,result); + } +} + +void floating_point_multiply_vector_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index) +{ + uint128_t operand1 = V_interpreter(ctx,Rn); + uint128_t operand2 = V_interpreter(ctx,Rm); + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); + if ((use_x86_sse_interpreter(ctx))) + { + operand2 = sse_copy_to_xmm_from_xmm_element_interpreter(ctx,operand2,esize,index); + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)32ULL))) + multiply_instruction = (uint64_t)x86_mulps; + else + multiply_instruction = (uint64_t)x86_mulpd; + uint128_t result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,result); + } + else + { + uint128_t result = 0; + uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); + uint64_t element2 = uint128_t::extract(operand2, index, esize); + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(operand1, e, esize); + uint128_t::insert(&result, e, esize, FPMul_interpreter(ctx,element1,element2,fpcr_state,esize)); + } + V_interpreter(ctx,Rd,result); + } +} + +void floating_point_multiply_accumulate_scalar_element_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index) +{ + uint128_t operand1 = V_interpreter(ctx,Rn); + uint128_t operand2 = V_interpreter(ctx,Rm); + uint128_t operand3 = V_interpreter(ctx,Rd); + uint128_t result; + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t add_instruction; + uint64_t subtract_instruction; + uint64_t multiply_instruction; + operand2 = uint128_t::extract(operand2, index, esize); + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addsd; + subtract_instruction = (uint64_t)x86_subsd; + multiply_instruction = (uint64_t)x86_mulsd; + } + else + { + add_instruction = (uint64_t)x86_addss; + subtract_instruction = (uint64_t)x86_subss; + multiply_instruction = (uint64_t)x86_mulss; + } + result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); + if ((neg)) + { + result = intrinsic_binary_interpreter(ctx,subtract_instruction,operand3,result); + } + else + { + result = intrinsic_binary_interpreter(ctx,add_instruction,operand3,result); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + } + else + { + result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + uint64_t element1 = operand1; + if ((neg)) + { + element1 = FPNeg_interpreter(ctx,element1,fpcr_state,esize); + } + uint64_t product_accumalant = FPMulAdd_interpreter(ctx,(uint64_t)operand3,element1,uint128_t::extract(operand2, index, esize),fpcr_state,esize); + uint128_t::insert(&result, 0ULL, esize, product_accumalant); + } + V_interpreter(ctx,Rd,result); +} + +void floating_point_multiply_accumulate_vector_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index) +{ + uint128_t operand1 = V_interpreter(ctx,Rn); + uint128_t operand2 = V_interpreter(ctx,Rm); + uint128_t operand3 = V_interpreter(ctx,Rd); + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); + uint128_t result; + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t add_instruction; + uint64_t subtract_instruction; + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addpd; + subtract_instruction = (uint64_t)x86_subpd; + multiply_instruction = (uint64_t)x86_mulpd; + } + else + { + add_instruction = (uint64_t)x86_addps; + subtract_instruction = (uint64_t)x86_subps; + multiply_instruction = (uint64_t)x86_mulps; + } + operand2 = sse_copy_to_xmm_from_xmm_element_interpreter(ctx,operand2,esize,index); + result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); + if ((neg)) + { + result = intrinsic_binary_interpreter(ctx,subtract_instruction,operand3,result); + } + else + { + result = intrinsic_binary_interpreter(ctx,add_instruction,operand3,result); + } + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + } + else + { + result = 0; + uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); + uint64_t element2 = uint128_t::extract(operand2, index, esize); + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(operand1, e, esize); + uint64_t element3 = uint128_t::extract(operand3, e, esize); + if ((neg)) + { + element1 = FPNeg_interpreter(ctx,element1,fpcr_state,esize); + } + uint128_t::insert(&result, e, esize, FPMulAdd_interpreter(ctx,element3,element1,element2,fpcr_state,esize)); + } + } + V_interpreter(ctx,Rd,result); +} + +void fcm_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz) +{ + uint128_t n = V_interpreter(ctx,Rn); + uint128_t m; + if ((((uint64_t)Rm == (uint64_t)-1ULL))) + { + m = 0; + } + else + { + m = V_interpreter(ctx,Rm); + } + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); + if ((use_x86_sse_interpreter(ctx))) + { + uint128_t result; + uint64_t operation = select_interpreter(ctx,((uint64_t)esize == (uint64_t)64ULL),(uint64_t)x86_cmppd,(uint64_t)x86_cmpps); + uint64_t control; + if ((((uint64_t)mode == (uint64_t)0ULL))) + { + control = 0ULL; + } + else if ((((uint64_t)mode == (uint64_t)1ULL))) + { + control = 6ULL; + } + else if ((((uint64_t)mode == (uint64_t)2ULL))) + { + control = 5ULL; + } + result = intrinsic_ternary_imm_interpreter(ctx,operation,n,m,control); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,result); + } + else + { + uint128_t result = 0; + uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(n, e, esize); + uint64_t element2 = uint128_t::extract(m, e, esize); + uint64_t element_result; + if ((((uint64_t)mode == (uint64_t)0ULL))) + { + element_result = ((uint64_t)0ULL - (uint64_t)FPCompareEQ_interpreter(ctx,element1,element2,fpcr_state,esize)); + } + else if ((((uint64_t)mode == (uint64_t)1ULL))) + { + element_result = ((uint64_t)0ULL - (uint64_t)FPCompareGT_interpreter(ctx,element1,element2,fpcr_state,esize)); + } + else if ((((uint64_t)mode == (uint64_t)2ULL))) + { + element_result = ((uint64_t)0ULL - (uint64_t)FPCompareGE_interpreter(ctx,element1,element2,fpcr_state,esize)); + } + uint128_t::insert(&result, e, esize, element_result); + } + V_interpreter(ctx,Rd,result); + } +} + +uint128_t clear_vector_scalar_interpreter(interpreter_data* ctx, uint128_t working, uint64_t fltsize) +{ + if ((((uint64_t)fltsize == (uint64_t)32ULL))) + { + uint128_t::insert(&working, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&working, 1ULL, 64ULL, (uint64_t)0ULL); + return working; +} + void add_subtract_imm12_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t sh, uint64_t imm12, uint64_t Rn, uint64_t Rd) { if (sf == 0ULL) @@ -5725,28 +6014,42 @@ void divide_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t Rm, uint64_ } +uint64_t create_rbit_mask_interpreter(interpreter_data* ctx, uint64_t index) +{ + index = ((uint64_t)1ULL << (uint64_t)index); + uint64_t mask = ((uint64_t)(((uint64_t)(((uint64_t)1ULL << (uint64_t)index)) - (uint64_t)1ULL)) << (uint64_t)index); + mask = replicate_c_interpreter(ctx,mask,((uint64_t)index * (uint64_t)2ULL),((uint64_t)((uint64_t)64ULL / (uint64_t)index) / (uint64_t)2ULL)); + return mask; +} + void rbit_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t Rn, uint64_t Rd) { uint64_t datasize = ((uint64_t)32ULL << (uint64_t)sf); if (sf == 0ULL) { uint32_t operand = X_interpreter(ctx,Rn); - uint32_t result = 0ULL; - for (uint64_t i = 0; i < (datasize); i++) + uint32_t result = operand; + uint64_t max = ((uint64_t)5ULL + (uint64_t)sf); + for (uint64_t i = 0; i < (max); i++) { - uint32_t working = ((uint32_t)(((uint32_t)operand >> (uint32_t)i)) & (uint32_t)1ULL); - result = ((uint32_t)result | (uint32_t)(((uint32_t)working << (uint32_t)(((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL))))); + uint64_t n_mask = create_rbit_mask_interpreter(ctx,i); + uint64_t i_mask = ~n_mask; + uint64_t shift = ((uint64_t)1ULL << (uint64_t)i); + result = ((uint32_t)(((uint32_t)(((uint32_t)result & (uint32_t)n_mask)) >> (uint32_t)shift)) | (uint32_t)(((uint32_t)(((uint32_t)result & (uint32_t)i_mask)) << (uint32_t)shift))); } X_interpreter(ctx,Rd,(uint64_t)result); } if (sf == 1ULL) { uint64_t operand = X_interpreter(ctx,Rn); - uint64_t result = 0ULL; - for (uint64_t i = 0; i < (datasize); i++) + uint64_t result = operand; + uint64_t max = ((uint64_t)5ULL + (uint64_t)sf); + for (uint64_t i = 0; i < (max); i++) { - uint64_t working = ((uint64_t)(((uint64_t)operand >> (uint64_t)i)) & (uint64_t)1ULL); - result = ((uint64_t)result | (uint64_t)(((uint64_t)working << (uint64_t)(((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL))))); + uint64_t n_mask = create_rbit_mask_interpreter(ctx,i); + uint64_t i_mask = ~n_mask; + uint64_t shift = ((uint64_t)1ULL << (uint64_t)i); + result = ((uint64_t)(((uint64_t)(((uint64_t)result & (uint64_t)n_mask)) >> (uint64_t)shift)) | (uint64_t)(((uint64_t)(((uint64_t)result & (uint64_t)i_mask)) << (uint64_t)shift))); } X_interpreter(ctx,Rd,(uint64_t)result); } @@ -5832,23 +6135,42 @@ void count_leading_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t s, u sig_bit = ((uint32_t)(((uint32_t)operand >> (uint32_t)(((uint64_t)datasize - (uint64_t)1ULL)))) & (uint32_t)1ULL); datasize = ((uint64_t)datasize - (uint64_t)1ULL); } - for (uint64_t i = 0; i < (datasize); i++) + if ((use_x86_lzcnt_interpreter(ctx))) { - uint32_t working = ((uint32_t)(((uint32_t)operand >> (uint32_t)(((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)))) & (uint32_t)1ULL); if ((s)) { - if ((((uint32_t)working != (uint32_t)sig_bit))) + uint32_t nhigh = ((uint32_t)operand >> (uint32_t)1ULL); + uint32_t mask = -1ULL; + mask = ((uint32_t)mask >> (uint32_t)1ULL); + uint32_t nlow = ((uint32_t)operand & (uint32_t)mask); + result = intrinsic_unary_interpreter(ctx,(uint64_t)x86_lzcnt,(((uint32_t)nhigh ^ (uint32_t)nlow))); + result = ((uint32_t)result - (uint32_t)1ULL); + } + else + { + result = intrinsic_unary_interpreter(ctx,(uint64_t)x86_lzcnt,operand); + } + } + else + { + for (uint64_t i = 0; i < (datasize); i++) + { + uint32_t working = ((uint32_t)(((uint32_t)operand >> (uint32_t)(((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)))) & (uint32_t)1ULL); + if ((s)) + { + if ((((uint32_t)working != (uint32_t)sig_bit))) + { + done = 1ULL; + } + } + else if ((working)) { done = 1ULL; } - } - else if ((working)) - { - done = 1ULL; - } - if ((!done)) - { - result = ((uint32_t)result + (uint32_t)1ULL); + if ((!done)) + { + result = ((uint32_t)result + (uint32_t)1ULL); + } } } X_interpreter(ctx,Rd,(uint64_t)result); @@ -5864,23 +6186,42 @@ void count_leading_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t s, u sig_bit = ((uint64_t)(((uint64_t)operand >> (uint64_t)(((uint64_t)datasize - (uint64_t)1ULL)))) & (uint64_t)1ULL); datasize = ((uint64_t)datasize - (uint64_t)1ULL); } - for (uint64_t i = 0; i < (datasize); i++) + if ((use_x86_lzcnt_interpreter(ctx))) { - uint64_t working = ((uint64_t)(((uint64_t)operand >> (uint64_t)(((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)))) & (uint64_t)1ULL); if ((s)) { - if ((((uint64_t)working != (uint64_t)sig_bit))) + uint64_t nhigh = ((uint64_t)operand >> (uint64_t)1ULL); + uint64_t mask = -1ULL; + mask = ((uint64_t)mask >> (uint64_t)1ULL); + uint64_t nlow = ((uint64_t)operand & (uint64_t)mask); + result = intrinsic_unary_interpreter(ctx,(uint64_t)x86_lzcnt,(((uint64_t)nhigh ^ (uint64_t)nlow))); + result = ((uint64_t)result - (uint64_t)1ULL); + } + else + { + result = intrinsic_unary_interpreter(ctx,(uint64_t)x86_lzcnt,operand); + } + } + else + { + for (uint64_t i = 0; i < (datasize); i++) + { + uint64_t working = ((uint64_t)(((uint64_t)operand >> (uint64_t)(((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)))) & (uint64_t)1ULL); + if ((s)) + { + if ((((uint64_t)working != (uint64_t)sig_bit))) + { + done = 1ULL; + } + } + else if ((working)) { done = 1ULL; } - } - else if ((working)) - { - done = 1ULL; - } - if ((!done)) - { - result = ((uint64_t)result + (uint64_t)1ULL); + if ((!done)) + { + result = ((uint64_t)result + (uint64_t)1ULL); + } } } X_interpreter(ctx,Rd,(uint64_t)result); @@ -7631,6 +7972,11 @@ void load_store_exclusive_ordered_interpreter(interpreter_data* ctx, uint64_t si } } +uint64_t exclusive_address_mask_interpreter(interpreter_data* ctx) +{ + return ~(((uint64_t)(((uint64_t)4ULL << (uint64_t)4ULL)) - (uint64_t)1ULL)); +} + void load_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, uint64_t size, uint64_t Rn, uint64_t Rt) { uint64_t datasize = ((uint64_t)8ULL << (uint64_t)size); @@ -7640,7 +7986,7 @@ void load_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, ui uint8_t value = mem_interpreter(ctx,address); if ((is_exclusive)) { - _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)~63ULL)); + _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)exclusive_address_mask_interpreter(ctx))); _sys_interpreter(ctx,(uint64_t)exclusive_value,(uint64_t)value); } X_interpreter(ctx,Rt,(uint64_t)value); @@ -7650,7 +7996,7 @@ void load_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, ui uint16_t value = mem_interpreter(ctx,address); if ((is_exclusive)) { - _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)~63ULL)); + _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)exclusive_address_mask_interpreter(ctx))); _sys_interpreter(ctx,(uint64_t)exclusive_value,(uint64_t)value); } X_interpreter(ctx,Rt,(uint64_t)value); @@ -7660,7 +8006,7 @@ void load_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, ui uint32_t value = mem_interpreter(ctx,address); if ((is_exclusive)) { - _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)~63ULL)); + _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)exclusive_address_mask_interpreter(ctx))); _sys_interpreter(ctx,(uint64_t)exclusive_value,(uint64_t)value); } X_interpreter(ctx,Rt,(uint64_t)value); @@ -7670,7 +8016,7 @@ void load_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, ui uint64_t value = mem_interpreter(ctx,address); if ((is_exclusive)) { - _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)~63ULL)); + _sys_interpreter(ctx,(uint64_t)exclusive_address,((uint64_t)address & (uint64_t)exclusive_address_mask_interpreter(ctx))); _sys_interpreter(ctx,(uint64_t)exclusive_value,(uint64_t)value); } X_interpreter(ctx,Rt,(uint64_t)value); @@ -7686,14 +8032,14 @@ void store_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, u { if ((is_exclusive)) { - uint64_t mask = ~63ULL; + uint64_t mask = exclusive_address_mask_interpreter(ctx); uint64_t _exclusive_address = _sys_interpreter(ctx,(uint64_t)exclusive_address); if ((((uint64_t)_exclusive_address == (uint64_t)(((uint64_t)address & (uint64_t)mask))))) { uint8_t to_swap = X_interpreter(ctx,Rt); uint8_t expecting = _sys_interpreter(ctx,(uint64_t)exclusive_value); uint8_t cas_success = compare_and_swap_interpreter(ctx,address,(uint64_t)expecting,(uint64_t)to_swap,datasize); - X_interpreter(ctx,Rs,(uint64_t)((uint8_t)cas_success ^ (uint8_t)1ULL)); + X_interpreter(ctx,Rs,(uint64_t)((uint8_t)(((uint8_t)cas_success ^ (uint8_t)1ULL)) & (uint8_t)1ULL)); } else { @@ -7709,14 +8055,14 @@ void store_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, u { if ((is_exclusive)) { - uint64_t mask = ~63ULL; + uint64_t mask = exclusive_address_mask_interpreter(ctx); uint64_t _exclusive_address = _sys_interpreter(ctx,(uint64_t)exclusive_address); if ((((uint64_t)_exclusive_address == (uint64_t)(((uint64_t)address & (uint64_t)mask))))) { uint16_t to_swap = X_interpreter(ctx,Rt); uint16_t expecting = _sys_interpreter(ctx,(uint64_t)exclusive_value); uint16_t cas_success = compare_and_swap_interpreter(ctx,address,(uint64_t)expecting,(uint64_t)to_swap,datasize); - X_interpreter(ctx,Rs,(uint64_t)((uint16_t)cas_success ^ (uint16_t)1ULL)); + X_interpreter(ctx,Rs,(uint64_t)((uint16_t)(((uint16_t)cas_success ^ (uint16_t)1ULL)) & (uint16_t)1ULL)); } else { @@ -7732,14 +8078,14 @@ void store_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, u { if ((is_exclusive)) { - uint64_t mask = ~63ULL; + uint64_t mask = exclusive_address_mask_interpreter(ctx); uint64_t _exclusive_address = _sys_interpreter(ctx,(uint64_t)exclusive_address); if ((((uint64_t)_exclusive_address == (uint64_t)(((uint64_t)address & (uint64_t)mask))))) { uint32_t to_swap = X_interpreter(ctx,Rt); uint32_t expecting = _sys_interpreter(ctx,(uint64_t)exclusive_value); uint32_t cas_success = compare_and_swap_interpreter(ctx,address,(uint64_t)expecting,(uint64_t)to_swap,datasize); - X_interpreter(ctx,Rs,(uint64_t)((uint32_t)cas_success ^ (uint32_t)1ULL)); + X_interpreter(ctx,Rs,(uint64_t)((uint32_t)(((uint32_t)cas_success ^ (uint32_t)1ULL)) & (uint32_t)1ULL)); } else { @@ -7755,14 +8101,14 @@ void store_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, u { if ((is_exclusive)) { - uint64_t mask = ~63ULL; + uint64_t mask = exclusive_address_mask_interpreter(ctx); uint64_t _exclusive_address = _sys_interpreter(ctx,(uint64_t)exclusive_address); if ((((uint64_t)_exclusive_address == (uint64_t)(((uint64_t)address & (uint64_t)mask))))) { uint64_t to_swap = X_interpreter(ctx,Rt); uint64_t expecting = _sys_interpreter(ctx,(uint64_t)exclusive_value); uint64_t cas_success = compare_and_swap_interpreter(ctx,address,(uint64_t)expecting,(uint64_t)to_swap,datasize); - X_interpreter(ctx,Rs,(uint64_t)((uint64_t)cas_success ^ (uint64_t)1ULL)); + X_interpreter(ctx,Rs,(uint64_t)((uint64_t)(((uint64_t)cas_success ^ (uint64_t)1ULL)) & (uint64_t)1ULL)); } else { @@ -7967,15 +8313,41 @@ void frsqrte_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); uint128_t operand = V_interpreter(ctx,Rn); - uint128_t result = 0; - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_interpreter(ctx))) { - uint64_t element = uint128_t::extract(operand, e, esize); - element = FPRSqrtEstimate_interpreter(ctx,element,fpcr_state,esize); - uint128_t::insert(&result, e, esize, element); + uint128_t one = sse_coppy_gp_across_lanes_interpreter(ctx,float_imm_interpreter(ctx,(uint64_t)1ULL,esize),esize); + uint64_t sqrt_instruction; + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtpd; + divide_instruction = (uint64_t)x86_divpd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtps; + divide_instruction = (uint64_t)x86_divps; + } + uint128_t sqrt_result = intrinsic_unary_interpreter(ctx,sqrt_instruction,operand); + uint128_t reciprocal_result = intrinsic_binary_interpreter(ctx,divide_instruction,one,sqrt_result); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + uint128_t::insert(&reciprocal_result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,reciprocal_result); + } + else + { + uint128_t result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element = uint128_t::extract(operand, e, esize); + element = FPRSqrtEstimate_interpreter(ctx,element,fpcr_state,esize); + uint128_t::insert(&result, e, esize, element); + } + V_interpreter(ctx,Rd,result); } - V_interpreter(ctx,Rd,result); } void frsqrts_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -7985,15 +8357,48 @@ void frsqrts_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); uint128_t operand1 = V_interpreter(ctx,Rn); uint128_t operand2 = V_interpreter(ctx,Rm); - uint128_t result = 0; - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_interpreter(ctx))) { - uint64_t element1 = uint128_t::extract(operand1, e, esize); - uint64_t element2 = uint128_t::extract(operand2, e, esize); - uint128_t::insert(&result, e, esize, FPRSqrtStepFused_interpreter(ctx,element1,element2,fpcr_state,esize)); + uint128_t two = sse_coppy_gp_across_lanes_interpreter(ctx,float_imm_interpreter(ctx,(uint64_t)2ULL,esize),esize); + uint128_t three = sse_coppy_gp_across_lanes_interpreter(ctx,float_imm_interpreter(ctx,(uint64_t)3ULL,esize),esize); + uint128_t negator = sse_coppy_gp_across_lanes_interpreter(ctx,(uint64_t)((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL))),esize); + operand1 = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand1,negator); + uint64_t add_instruction; + uint64_t multiply_instruction; + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addpd; + multiply_instruction = (uint64_t)x86_mulpd; + divide_instruction = (uint64_t)x86_divpd; + } + else + { + add_instruction = (uint64_t)x86_addps; + multiply_instruction = (uint64_t)x86_mulps; + divide_instruction = (uint64_t)x86_divps; + } + uint128_t mul_result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); + uint128_t add_result = intrinsic_binary_interpreter(ctx,add_instruction,three,mul_result); + uint128_t div_result = intrinsic_binary_interpreter(ctx,divide_instruction,add_result,two); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + uint128_t::insert(&div_result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,div_result); + } + else + { + uint128_t result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(operand1, e, esize); + uint64_t element2 = uint128_t::extract(operand2, e, esize); + uint128_t::insert(&result, e, esize, FPRSqrtStepFused_interpreter(ctx,element1,element2,fpcr_state,esize)); + } + V_interpreter(ctx,Rd,result); } - V_interpreter(ctx,Rd,result); } void frecps_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -8003,15 +8408,42 @@ void frecps_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, u uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); uint128_t operand1 = V_interpreter(ctx,Rn); uint128_t operand2 = V_interpreter(ctx,Rm); - uint128_t result = 0; - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_interpreter(ctx))) { - uint64_t element1 = uint128_t::extract(operand1, e, esize); - uint64_t element2 = uint128_t::extract(operand2, e, esize); - uint128_t::insert(&result, e, esize, FPRecipStepFused_interpreter(ctx,element1,element2,fpcr_state,esize)); + uint128_t negator = sse_coppy_gp_across_lanes_interpreter(ctx,(uint64_t)((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL))),esize); + uint128_t two = sse_coppy_gp_across_lanes_interpreter(ctx,float_imm_interpreter(ctx,(uint64_t)2ULL,esize),esize); + operand1 = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand1,negator); + uint64_t add_instruction; + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addpd; + multiply_instruction = (uint64_t)x86_mulpd; + } + else + { + add_instruction = (uint64_t)x86_addps; + multiply_instruction = (uint64_t)x86_mulps; + } + uint128_t result = intrinsic_binary_interpreter(ctx,add_instruction,two,(uint128_t)intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2)); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,result); + } + else + { + uint128_t result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(operand1, e, esize); + uint64_t element2 = uint128_t::extract(operand2, e, esize); + uint128_t::insert(&result, e, esize, FPRecipStepFused_interpreter(ctx,element1,element2,fpcr_state,esize)); + } + V_interpreter(ctx,Rd,result); } - V_interpreter(ctx,Rd,result); } void fmul_scalar_by_element_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -8032,14 +8464,7 @@ void fmul_scalar_by_element_interpreter(interpreter_data* ctx, uint64_t sz, uint { undefined_interpreter(ctx); } - uint128_t operand1 = V_interpreter(ctx,Rn); - uint128_t operand2 = V_interpreter(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); - uint128_t result = 0; - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - uint64_t product = FPMul_interpreter(ctx,(uint64_t)operand1,uint128_t::extract(operand2, index, esize),fpcr_state,esize); - uint128_t::insert(&result, 0ULL, esize, product); - V_interpreter(ctx,Rd,result); + floating_point_multiply_scalar_element_interpreter(ctx,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),sz,index); } void fmul_vector_by_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -8060,20 +8485,7 @@ void fmul_vector_by_element_interpreter(interpreter_data* ctx, uint64_t Q, uint6 { undefined_interpreter(ctx); } - uint128_t operand1 = V_interpreter(ctx,Rn); - uint128_t operand2 = V_interpreter(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); - uint128_t result = 0; - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); - uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); - uint64_t element2 = uint128_t::extract(operand2, index, esize); - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) - { - uint64_t element1 = uint128_t::extract(operand1, e, esize); - uint128_t::insert(&result, e, esize, FPMul_interpreter(ctx,element1,element2,fpcr_state,esize)); - } - V_interpreter(ctx,Rd,result); + floating_point_multiply_vector_element_interpreter(ctx,Q,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),sz,index); } void fmul_accumulate_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -8094,57 +8506,7 @@ void fmul_accumulate_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint { undefined_interpreter(ctx); } - uint128_t operand1 = V_interpreter(ctx,Rn); - uint128_t operand2 = V_interpreter(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); - uint128_t operand3 = V_interpreter(ctx,Rd); - uint128_t result; - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) - { - uint64_t add_instruction; - uint64_t subtract_instruction; - uint64_t multiply_instruction; - operand2 = uint128_t::extract(operand2, index, esize); - if ((((uint64_t)esize == (uint64_t)64ULL))) - { - add_instruction = (uint64_t)x86_addsd; - subtract_instruction = (uint64_t)x86_subsd; - multiply_instruction = (uint64_t)x86_mulsd; - } - else - { - add_instruction = (uint64_t)x86_addss; - subtract_instruction = (uint64_t)x86_subss; - multiply_instruction = (uint64_t)x86_mulss; - } - result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); - if ((neg)) - { - result = intrinsic_binary_interpreter(ctx,subtract_instruction,operand3,result); - } - else - { - result = intrinsic_binary_interpreter(ctx,add_instruction,operand3,result); - } - uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); - if ((((uint64_t)esize == (uint64_t)32ULL))) - { - uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); - } - } - else - { - result = 0; - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - uint64_t element1 = operand1; - if ((neg)) - { - element1 = FPNeg_interpreter(ctx,element1,fpcr_state,esize); - } - uint64_t product_accumalant = FPMulAdd_interpreter(ctx,(uint64_t)operand3,element1,uint128_t::extract(operand2, index, esize),fpcr_state,esize); - uint128_t::insert(&result, 0ULL, esize, product_accumalant); - } - V_interpreter(ctx,Rd,result); + floating_point_multiply_accumulate_scalar_element_interpreter(ctx,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),neg,sz,index); } void fmul_accumulate_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -8168,59 +8530,7 @@ void fmul_accumulate_element_interpreter(interpreter_data* ctx, uint64_t Q, uint uint128_t operand1 = V_interpreter(ctx,Rn); uint128_t operand2 = V_interpreter(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); uint128_t operand3 = V_interpreter(ctx,Rd); - uint128_t result; - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); - if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) - { - uint64_t add_instruction; - uint64_t subtract_instruction; - uint64_t multiply_instruction; - if ((((uint64_t)esize == (uint64_t)64ULL))) - { - add_instruction = (uint64_t)x86_addpd; - subtract_instruction = (uint64_t)x86_subpd; - multiply_instruction = (uint64_t)x86_mulpd; - } - else - { - add_instruction = (uint64_t)x86_addps; - subtract_instruction = (uint64_t)x86_subps; - multiply_instruction = (uint64_t)x86_mulps; - } - operand2 = sse_copy_to_xmm_from_xmm_element_interpreter(ctx,operand2,esize,index); - result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); - if ((neg)) - { - result = intrinsic_binary_interpreter(ctx,subtract_instruction,operand3,result); - } - else - { - result = intrinsic_binary_interpreter(ctx,add_instruction,operand3,result); - } - if ((((uint64_t)datasize == (uint64_t)64ULL))) - { - uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); - } - } - else - { - result = 0; - uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); - uint64_t element2 = uint128_t::extract(operand2, index, esize); - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) - { - uint64_t element1 = uint128_t::extract(operand1, e, esize); - uint64_t element3 = uint128_t::extract(operand3, e, esize); - if ((neg)) - { - element1 = FPNeg_interpreter(ctx,element1,fpcr_state,esize); - } - uint128_t::insert(&result, e, esize, FPMulAdd_interpreter(ctx,element3,element1,element2,fpcr_state,esize)); - } - } - V_interpreter(ctx,Rd,result); + floating_point_multiply_accumulate_vector_element_interpreter(ctx,Q,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),neg,sz,index); } void faddp_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t Rn, uint64_t Rd) @@ -8228,12 +8538,55 @@ void faddp_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t Rn, u uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); uint64_t datasize = ((uint64_t)esize * (uint64_t)2ULL); uint128_t operand = V_interpreter(ctx,Rn); - uint128_t result = 0; - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - uint64_t bottom = uint128_t::extract(operand, 0ULL, esize); - uint64_t top = uint128_t::extract(operand, 1ULL, esize); - uint128_t::insert(&result, 0ULL, esize, FPAdd_interpreter(ctx,bottom,top,fpcr_state,esize)); - V_interpreter(ctx,Rd,result); + if ((use_x86_sse_interpreter(ctx))) + { + uint128_t result = intrinsic_binary_interpreter(ctx,select_interpreter(ctx,((uint64_t)esize == (uint64_t)64ULL),(uint64_t)x86_haddpd,(uint64_t)x86_haddps),operand,operand); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,result); + } + else + { + uint128_t result = 0; + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + uint64_t bottom = uint128_t::extract(operand, 0ULL, esize); + uint64_t top = uint128_t::extract(operand, 1ULL, esize); + uint128_t::insert(&result, 0ULL, esize, FPAdd_interpreter(ctx,bottom,top,fpcr_state,esize)); + V_interpreter(ctx,Rd,result); + } +} + +void fcmeq_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_interpreter(ctx,Rd,Rn,-1ULL,0ULL,Q,sz); +} + +void fcmgt_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_interpreter(ctx,Rd,Rn,-1ULL,1ULL,Q,sz); +} + +void fcmge_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_interpreter(ctx,Rd,Rn,-1ULL,2ULL,Q,sz); +} + +void fcmeq_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_interpreter(ctx,Rd,Rn,Rm,0ULL,Q,sz); +} + +void fcmgt_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_interpreter(ctx,Rd,Rn,Rm,1ULL,Q,sz); +} + +void fcmge_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_interpreter(ctx,Rd,Rn,Rm,2ULL,Q,sz); } void fadd_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -8308,36 +8661,171 @@ void fminnm_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t R void fnmul_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)) && (uint64_t)((uint64_t)ftype != (uint64_t)3ULL)))) + { + uint128_t operand1 = V_interpreter(ctx,Rn); + uint128_t operand2 = V_interpreter(ctx,Rm); + uint64_t esize = get_flt_size_interpreter(ctx,ftype); + uint128_t negator = (uint64_t)(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))); + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + multiply_instruction = (uint64_t)x86_mulss; + } + else if ((((uint64_t)esize == (uint64_t)64ULL))) + { + multiply_instruction = (uint64_t)x86_mulsd; + } + else + { + undefined_interpreter(ctx); + } + operand2 = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand2,negator); + uint128_t result = intrinsic_binary_interpreter(ctx,multiply_instruction,operand1,operand2); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,result); + return; + } float_binary_scalar_interpreter(ctx,Rd,Rn,Rm,ftype,(uint64_t)FPNMul_interpreter); } void fabs_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t esize = get_flt_size_interpreter(ctx,ftype); + uint128_t operand = V_interpreter(ctx,Rn); + uint128_t mask = (uint64_t)(((uint64_t)(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))) - (uint64_t)1ULL)); + uint128_t result = intrinsic_binary_interpreter(ctx,(uint64_t)x86_pand,operand,mask); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,result); + return; + } float_unary_scalar_interpreter(ctx,Rd,Rn,ftype,(uint64_t)FPAbs_interpreter); } void fneg_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t esize = get_flt_size_interpreter(ctx,ftype); + uint128_t operand = V_interpreter(ctx,Rn); + uint128_t mask = (uint64_t)(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))); + uint128_t result = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand,mask); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,result); + return; + } float_unary_scalar_interpreter(ctx,Rd,Rn,ftype,(uint64_t)FPNeg_interpreter); } void fneg_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint128_t operand = V_interpreter(ctx,Rn); + uint128_t mask = sse_coppy_gp_across_lanes_interpreter(ctx,(uint64_t)(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))),esize); + uint128_t result = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand,mask); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,result); + return; + } float_unary_vector_interpreter(ctx,Rd,Rn,Q,sz,(uint64_t)FPNeg_interpreter); } void fsqrt_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)) && (uint64_t)((uint64_t)ftype != (uint64_t)3ULL)))) + { + uint64_t esize = get_flt_size_interpreter(ctx,ftype); + uint128_t operand = V_interpreter(ctx,Rn); + uint64_t sqrt_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtsd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtss; + } + uint128_t result = intrinsic_unary_interpreter(ctx,sqrt_instruction,operand); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,result); + return; + } float_unary_scalar_interpreter(ctx,Rd,Rn,ftype,(uint64_t)FPSqrt_interpreter); } void fsqrt_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint128_t operand = V_interpreter(ctx,Rn); + uint64_t sqrt_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtpd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtps; + } + uint128_t result = intrinsic_unary_interpreter(ctx,sqrt_instruction,operand); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,result); + return; + } float_unary_vector_interpreter(ctx,Rd,Rn,Q,sz,(uint64_t)FPSqrt_interpreter); } void frecpe_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) + { + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint128_t operand = V_interpreter(ctx,Rn); + uint128_t one = sse_coppy_gp_across_lanes_interpreter(ctx,float_imm_interpreter(ctx,(uint64_t)1ULL,esize),esize); + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + divide_instruction = (uint64_t)x86_divpd; + } + else + { + divide_instruction = (uint64_t)x86_divps; + } + uint128_t result = intrinsic_binary_interpreter(ctx,divide_instruction,one,operand); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,result); + return; + } float_unary_vector_interpreter(ctx,Rd,Rn,Q,sz,(uint64_t)FPRecipEstimate_interpreter); } @@ -8345,13 +8833,40 @@ void frsqrte_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t Rn, { uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); uint64_t operand = V_interpreter(ctx,Rn); - uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); - uint64_t result = FPRSqrtEstimate_interpreter(ctx,operand,fpcr_state,esize); - if ((((uint64_t)esize == (uint64_t)32ULL))) + if ((((uint64_t)use_fast_float_interpreter(ctx) && (uint64_t)use_x86_sse_interpreter(ctx)))) { - result = ((uint64_t)result & (uint64_t)4294967295ULL); + uint128_t one = float_imm_interpreter(ctx,(uint64_t)1ULL,esize); + uint64_t sqrt_instruction; + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtsd; + divide_instruction = (uint64_t)x86_divsd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtss; + divide_instruction = (uint64_t)x86_divss; + } + uint128_t sqrt_result = intrinsic_unary_interpreter(ctx,sqrt_instruction,(uint128_t)operand); + uint128_t reciprocal_result = intrinsic_binary_interpreter(ctx,divide_instruction,one,sqrt_result); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + uint128_t::insert(&reciprocal_result, 1ULL, 32ULL, (uint64_t)0ULL); + } + uint128_t::insert(&reciprocal_result, 1ULL, 64ULL, (uint64_t)0ULL); + V_interpreter(ctx,Rd,reciprocal_result); + } + else + { + uint64_t fpcr_state = _sys_interpreter(ctx,(uint64_t)fpcr); + uint64_t result = FPRSqrtEstimate_interpreter(ctx,operand,fpcr_state,esize); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + result = ((uint64_t)result & (uint64_t)4294967295ULL); + } + V_interpreter(ctx,Rd,(uint128_t)result); } - V_interpreter(ctx,Rd,(uint128_t)result); } void fmov_scalar_immediate_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t imm8, uint64_t Rd) @@ -8363,6 +8878,18 @@ void fmov_scalar_immediate_interpreter(interpreter_data* ctx, uint64_t ftype, ui uint64_t _compare_and_swap_interpreter(interpreter_data* ctx, uint64_t physical_address, uint64_t expecting, uint64_t to_swap, uint64_t size) { + if ((use_x86_interpreter(ctx))) + { + if ((((uint64_t)size != (uint64_t)64ULL))) + { + uint64_t mask = ((uint64_t)(((uint64_t)1ULL << (uint64_t)size)) - (uint64_t)1ULL); + uint64_t masking_value = *(uint64_t*)physical_address; + masking_value = ((uint64_t)masking_value & (uint64_t)~mask); + expecting = ((uint64_t)(((uint64_t)expecting & (uint64_t)mask)) | (uint64_t)masking_value); + to_swap = ((uint64_t)(((uint64_t)to_swap & (uint64_t)mask)) | (uint64_t)masking_value); + } + return intrinsic_ternary_interpreter(ctx,(uint64_t)x86_cmpxchg,physical_address,expecting,to_swap); + } return call_interpreter(ctx,physical_address,expecting,to_swap,(uint64_t)size,(uint64_t)0ULL,(uint64_t)0ULL,(uint64_t)compare_and_swap_interpreter_cpp); } @@ -9220,12 +9747,35 @@ void add_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t size, ui uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); uint128_t operand1 = V_interpreter(ctx,Rn); uint128_t operand2 = V_interpreter(ctx,Rm); - uint128_t result = 0; - for (uint64_t e = 0; e < (elements); e++) + uint128_t result; + if ((use_x86_sse2_interpreter(ctx))) { - uint64_t element1 = uint128_t::extract(operand1, e, esize); - uint64_t element2 = uint128_t::extract(operand2, e, esize); - uint128_t::insert(&result, e, esize, (((uint64_t)element1 + (uint64_t)element2))); + uint64_t add_instruction; + if ((((uint64_t)esize == (uint64_t)8ULL))) + add_instruction = (uint64_t)x86_paddb; + else if ((((uint64_t)esize == (uint64_t)16ULL))) + add_instruction = (uint64_t)x86_paddw; + else if ((((uint64_t)esize == (uint64_t)32ULL))) + add_instruction = (uint64_t)x86_paddd; + else if ((((uint64_t)esize == (uint64_t)64ULL))) + add_instruction = (uint64_t)x86_paddq; + else + undefined_interpreter(ctx); + result = intrinsic_binary_interpreter(ctx,add_instruction,operand1,operand2); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + uint128_t::insert(&result, 1ULL, 64ULL, (uint64_t)0ULL); + } + } + else + { + result = 0; + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(operand1, e, esize); + uint64_t element2 = uint128_t::extract(operand2, e, esize); + uint128_t::insert(&result, e, esize, (((uint64_t)element1 + (uint64_t)element2))); + } } V_interpreter(ctx,Rd,result); } @@ -9280,12 +9830,19 @@ void cnt_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t size, uint64_t { uint8_t working = uint128_t::extract(source, e, esize); uint8_t count = 0ULL; - for (uint64_t b = 0; b < (esize); b++) + if ((use_x86_interpreter(ctx))) { - uint8_t bit = ((uint8_t)(((uint8_t)working >> (uint8_t)b)) & (uint8_t)1ULL); - if ((bit)) + count = intrinsic_unary_interpreter(ctx,(uint64_t)x86_popcnt,(uint16_t)working); + } + else + { + for (uint64_t b = 0; b < (esize); b++) { - count = (((uint8_t)count + (uint8_t)1ULL)); + uint8_t bit = ((uint8_t)(((uint8_t)working >> (uint8_t)b)) & (uint8_t)1ULL); + if ((bit)) + { + count = (((uint8_t)count + (uint8_t)1ULL)); + } } } uint128_t::insert(&result, e, esize, count); @@ -9329,15 +9886,29 @@ void bsl_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rm, uint uint128_t operand1 = V_interpreter(ctx,Rm); uint128_t operand2 = V_interpreter(ctx,Rd); uint128_t operand3 = V_interpreter(ctx,Rn); - uint128_t result = 0; - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_interpreter(ctx))) { - uint64_t element1 = uint128_t::extract(operand1, e, esize); - uint64_t element2 = uint128_t::extract(operand2, e, esize); - uint64_t element3 = uint128_t::extract(operand3, e, esize); - uint128_t::insert(&result, e, esize, (((uint64_t)element1 ^ (uint64_t)(((uint64_t)(((uint64_t)element1 ^ (uint64_t)element3)) & (uint64_t)element2))))); + uint128_t xor_result = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand1,operand3); + uint128_t and_result = intrinsic_binary_interpreter(ctx,(uint64_t)x86_pand,xor_result,operand2); + uint128_t final_result = intrinsic_binary_interpreter(ctx,(uint64_t)x86_xorps,operand1,and_result); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + uint128_t::insert(&final_result, 1ULL, 64ULL, (uint64_t)0ULL); + } + V_interpreter(ctx,Rd,final_result); + } + else + { + uint128_t result = 0; + for (uint64_t e = 0; e < (elements); e++) + { + uint64_t element1 = uint128_t::extract(operand1, e, esize); + uint64_t element2 = uint128_t::extract(operand2, e, esize); + uint64_t element3 = uint128_t::extract(operand3, e, esize); + uint128_t::insert(&result, e, esize, (((uint64_t)element1 ^ (uint64_t)(((uint64_t)(((uint64_t)element1 ^ (uint64_t)element3)) & (uint64_t)element2))))); + } + V_interpreter(ctx,Rd,result); } - V_interpreter(ctx,Rd,result); } void and_bic_vector_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t invert, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -9505,68 +10076,22 @@ void st1_single_structure_post_index_interpreter(interpreter_data* ctx, uint64_t memory_1_interpreter(ctx,1ULL,Q,1ULL,0ULL,Rm,0ULL,((uint64_t)opcode << (uint64_t)1ULL),S,size,Rn,Rt,0ULL); } -void fcmeq_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_interpreter(ctx,Rd,Rn,-1ULL,0ULL,Q,sz); -} - -void fcmgt_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_interpreter(ctx,Rd,Rn,-1ULL,1ULL,Q,sz); -} - -void fcmge_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_interpreter(ctx,Rd,Rn,-1ULL,2ULL,Q,sz); -} - -void fcmeq_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_interpreter(ctx,Rd,Rn,Rm,0ULL,Q,sz); -} - -void fcmgt_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_interpreter(ctx,Rd,Rn,Rm,1ULL,Q,sz); -} - -void fcmge_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_interpreter(ctx,Rd,Rn,Rm,2ULL,Q,sz); -} - void floating_point_conditional_select_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t cond, uint64_t Rn, uint64_t Rd) { uint64_t fltsize = get_flt_size_interpreter(ctx,ftype); uint128_t operand1 = V_interpreter(ctx,Rn); uint128_t operand2 = V_interpreter(ctx,Rm); - if (fltsize == 32ULL) + uint128_t result = 0; + if ((condition_holds_interpreter(ctx,cond))) { - uint32_t result = 0ULL; - if ((condition_holds_interpreter(ctx,cond))) - { - result = operand1; - } - else - { - result = operand2; - } - V_interpreter(ctx,Rd,(uint128_t)result); + result = operand1; } - if (fltsize == 64ULL) + else { - uint64_t result = 0ULL; - if ((condition_holds_interpreter(ctx,cond))) - { - result = operand1; - } - else - { - result = operand2; - } - V_interpreter(ctx,Rd,(uint128_t)result); + result = operand2; } - + result = clear_vector_scalar_interpreter(ctx,result,fltsize); + V_interpreter(ctx,Rd,result); } void fcmp_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t opc) @@ -9803,6 +10328,18 @@ uint64_t decode_add_subtract_imm_12_jit(ssa_emit_context* ctx, uint64_t source, ir_operand add_subtract_impl_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m, uint64_t set_flags, uint64_t is_add) { ir_operand d; + if ((((uint64_t)set_flags && (uint64_t)use_x86_jit(ctx)))) + { + if ((is_add)) + { + d = x86_add_set_flags_jit(ctx,O,n,m); + } + else + { + d = x86_subtract_set_flags_jit(ctx,O,n,m); + } + return d; + } if ((is_add)) { d = ssa_emit_context::emit_ssa(ctx, ir_add, n, m); @@ -9916,6 +10453,13 @@ void branch_long_universal_jit(ssa_emit_context* ctx, uint64_t Rn, uint64_t link _branch_long_jit(ctx,branch_location); } +uint64_t select_jit(ssa_emit_context* ctx, uint64_t condition, uint64_t yes, uint64_t no) +{ + if ((condition)) + return yes; + return no; +} + ir_operand create_mask_jit(ssa_emit_context* ctx, uint64_t bits) { if ((((uint64_t)bits >= (uint64_t)64ULL))) @@ -9949,6 +10493,32 @@ ir_operand shift_left_check_jit(ssa_emit_context* ctx, ir_operand to_shift, ir_o return result; } +uint64_t get_x86_rounding_mode_jit(ssa_emit_context* ctx, uint64_t rounding) +{ + uint64_t rounding_control; + if ((((uint64_t)rounding == (uint64_t)FPRounding_TIEEVEN))) + { + rounding_control = 0ULL; + } + else if ((((uint64_t)rounding == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)rounding == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)rounding == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else + { + undefined_jit(ctx); + } + return rounding_control; +} + ir_operand shift_right_check_jit(ssa_emit_context* ctx, ir_operand to_shift, ir_operand shift, uint64_t size, uint64_t is_unsigned) { ir_operand result = ssa_emit_context::emit_ssa(ctx, ir_move, ir_operand::create_con(0ULL, int64)); @@ -10394,45 +10964,6 @@ void memory_1_jit(ssa_emit_context* ctx, uint64_t wback, uint64_t Q, uint64_t L, } } -void fcm_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz) -{ - ir_operand n = V_jit(ctx,Rn); - ir_operand m; - ir_operand result = ssa_emit_context::vector_zero(ctx); - if ((((uint64_t)Rm == (uint64_t)-1ULL))) - { - m = ssa_emit_context::vector_zero(ctx); - } - else - { - m = V_jit(ctx,Rm); - } - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); - uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) - { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,n, e, esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,m, e, esize); - ir_operand element_result; - if ((((uint64_t)mode == (uint64_t)0ULL))) - { - element_result = ssa_emit_context::emit_ssa(ctx, ir_subtract, ir_operand::create_con(0ULL, int64), FPCompareEQ_jit(ctx,element1,element2,fpcr_state,esize)); - } - else if ((((uint64_t)mode == (uint64_t)1ULL))) - { - element_result = ssa_emit_context::emit_ssa(ctx, ir_subtract, ir_operand::create_con(0ULL, int64), FPCompareGT_jit(ctx,element1,element2,fpcr_state,esize)); - } - else if ((((uint64_t)mode == (uint64_t)2ULL))) - { - element_result = ssa_emit_context::emit_ssa(ctx, ir_subtract, ir_operand::create_con(0ULL, int64), FPCompareGE_jit(ctx,element1,element2,fpcr_state,esize)); - } - ssa_emit_context::vector_insert(ctx,result, e, esize, element_result); - } - V_jit(ctx,Rd,result); -} - ir_operand bits_r_jit(ssa_emit_context* ctx, ir_operand operand, uint64_t top, uint64_t bottom) { top = ((uint64_t)top + (uint64_t)1ULL); @@ -10960,6 +11491,10 @@ ir_operand FixedToFP_jit(ssa_emit_context* ctx, ir_operand source, uint64_t frac { working_result = ssa_emit_context::convert_to_float(ctx,copy_new_raw_size(ctx, source, I),F,I, 1); } + if ((((uint64_t)fracbits == (uint64_t)0ULL))) + { + return copy_new_raw_size(ctx, working_result, int64); + } return copy_new_raw_size(ctx, ssa_emit_context::emit_ssa(ctx, ir_floating_point_divide, working_result, power), int64); } } @@ -10969,7 +11504,7 @@ ir_operand FixedToFP_jit(ssa_emit_context* ctx, ir_operand source, uint64_t frac ir_operand FPToFixed_jit(ssa_emit_context* ctx, ir_operand source, uint64_t fracbits, uint64_t is_unsigned, uint64_t round, uint64_t to, uint64_t from) { - if ((((uint64_t)((uint64_t)((uint64_t)use_fast_float_jit(ctx) && (uint64_t)((uint64_t)to != (uint64_t)16ULL)) && (uint64_t)((uint64_t)from != (uint64_t)16ULL)) && (uint64_t)((uint64_t)round != (uint64_t)FPRounding_ODD)))) + if ((((uint64_t)((uint64_t)((uint64_t)((uint64_t)use_fast_float_jit(ctx) && (uint64_t)((uint64_t)to != (uint64_t)16ULL)) && (uint64_t)((uint64_t)from != (uint64_t)16ULL)) && (uint64_t)((uint64_t)round != (uint64_t)FPRounding_ODD)) && (uint64_t)use_x86_sse41_jit(ctx)))) { uint64_t F = from == 32ULL ? int32 : from == 64ULL ? int64 : 0; { @@ -10999,64 +11534,57 @@ ir_operand FPToFixed_jit(ssa_emit_context* ctx, ir_operand source, uint64_t frac ir_operand power = copy_new_raw_size(ctx, create_fixed_from_fbits_jit(ctx,F,fracbits,from), F); working = ssa_emit_context::emit_ssa(ctx, ir_floating_point_multiply, power, working); } - if ((use_x86_sse41_jit(ctx))) + uint64_t rounding_control; + if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) { - uint64_t rounding_control; - if ((((uint64_t)round == (uint64_t)FPRounding_TIEEVEN))) + rounding_control = 0ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) + { + rounding_control = 1ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) + { + rounding_control = 2ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) + { + rounding_control = 3ULL; + } + else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) + { + rounding_control = 2ULL; { - rounding_control = 0ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_NEGINF))) - { - rounding_control = 1ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_POSINF))) - { - rounding_control = 2ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_ZERO))) - { - rounding_control = 3ULL; - } - else if ((((uint64_t)round == (uint64_t)FPRounding_TIEAWAY))) - { - rounding_control = 2ULL; + ir_operand end = ir_operation_block::create_label(ctx->ir); + ir_operand yes = ir_operation_block::create_label(ctx->ir); + + ir_operand condition = ssa_emit_context::emit_ssa(ctx, ir_floating_point_compare_less_equal, working, ir_operand::create_con(0ULL, F)); + + ir_operation_block::jump_if(ctx->ir,yes, condition); + /* TODO if statements without a no should not have this*/ + + ir_operation_block::jump(ctx->ir,end); + ir_operation_block::mark_label(ctx->ir, yes); + { - ir_operand end = ir_operation_block::create_label(ctx->ir); - ir_operand yes = ir_operation_block::create_label(ctx->ir); - - ir_operand condition = ssa_emit_context::emit_ssa(ctx, ir_floating_point_compare_less_equal, working, ir_operand::create_con(0ULL, F)); - - ir_operation_block::jump_if(ctx->ir,yes, condition); - /* TODO if statements without a no should not have this*/ - - ir_operation_block::jump(ctx->ir,end); - ir_operation_block::mark_label(ctx->ir, yes); - - { - ssa_emit_context::move(ctx,working,ssa_emit_context::emit_ssa(ctx, ir_floating_point_subtract, working, copy_new_raw_size(ctx, float_imm_jit(ctx,ir_operand::create_con(1ULL, int64),from), F))); - } - - ir_operation_block::mark_label(ctx->ir, end); + ssa_emit_context::move(ctx,working,ssa_emit_context::emit_ssa(ctx, ir_floating_point_subtract, working, copy_new_raw_size(ctx, float_imm_jit(ctx,ir_operand::create_con(1ULL, int64),from), F))); } - } - else - { - undefined_jit(ctx); - } - if ((((uint64_t)from == (uint64_t)32ULL))) - { - working = copy_new_raw_size(ctx, intrinsic_binary_imm_jit(ctx,int128,(uint64_t)x86_roundss,copy_new_raw_size(ctx, working, int128),rounding_control), F); - } - else - { - working = copy_new_raw_size(ctx, intrinsic_binary_imm_jit(ctx,int128,(uint64_t)x86_roundsd,copy_new_raw_size(ctx, working, int128),rounding_control), F); + + ir_operation_block::mark_label(ctx->ir, end); } } else { undefined_jit(ctx); } + if ((((uint64_t)from == (uint64_t)32ULL))) + { + working = copy_new_raw_size(ctx, intrinsic_binary_imm_jit(ctx,int128,(uint64_t)x86_roundss,copy_new_raw_size(ctx, working, int128),rounding_control), F); + } + else + { + working = copy_new_raw_size(ctx, intrinsic_binary_imm_jit(ctx,int128,(uint64_t)x86_roundsd,copy_new_raw_size(ctx, working, int128),rounding_control), F); + } ir_operand result = ssa_emit_context::emit_ssa(ctx, ir_move, ir_operand::create_con(0ULL, I)); { ir_operand end = ir_operation_block::create_label(ctx->ir); @@ -11133,8 +11661,22 @@ ir_operand FPToFixed_jit(ssa_emit_context* ctx, ir_operand source, uint64_t frac ir_operand FPConvert_jit(ssa_emit_context* ctx, ir_operand source, uint64_t to, uint64_t from) { + if ((((uint64_t)to == (uint64_t)from))) + { + undefined_jit(ctx); + } if ((((uint64_t)((uint64_t)use_fast_float_jit(ctx) && (uint64_t)((uint64_t)to != (uint64_t)16ULL)) && (uint64_t)((uint64_t)from != (uint64_t)16ULL)))) { + ir_operand result; + if ((((uint64_t)((uint64_t)to == (uint64_t)32ULL) && (uint64_t)((uint64_t)from == (uint64_t)64ULL)))) + { + result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int128,(uint64_t)x86_cvtsd2ss,copy_new_raw_size(ctx, source, int128)), int64); + } + else + { + result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int128,(uint64_t)x86_cvtss2sd,copy_new_raw_size(ctx, source, int128)), int64); + } + return result; } return call_jit(ctx,source,ir_operand::create_con(0ULL, int64),ir_operand::create_con(0ULL, int64),ir_operand::create_con(to, int64),ir_operand::create_con(from, int64),ir_operand::create_con(0ULL, int64),(uint64_t)FPConvert_I); } @@ -11222,13 +11764,23 @@ void float_binary_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, ui void frint_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd, uint64_t rounding) { - ir_operand operand1 = V_jit(ctx,Rn); - ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand operand = V_jit(ctx,Rn); uint64_t esize = get_flt_size_jit(ctx,ftype); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - ir_operand working = FPRoundInt_jit(ctx,copy_new_raw_size(ctx, operand1, int64),fpcr_state,rounding,esize); - ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, working); - V_jit(ctx,Rd,result); + if ((((uint64_t)((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)) && (uint64_t)((uint64_t)rounding != (uint64_t)FPRounding_TIEAWAY)))) + { + uint64_t rounding_control = get_x86_rounding_mode_jit(ctx,rounding); + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_imm_jit(ctx,int128,select_jit(ctx,((uint64_t)esize == (uint64_t)64ULL),(uint64_t)x86_roundsd,(uint64_t)x86_roundss),operand,rounding_control), int128); + result = clear_vector_scalar_jit(ctx,result,esize); + V_jit(ctx,Rd,result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + ir_operand working = FPRoundInt_jit(ctx,copy_new_raw_size(ctx, operand, int64),fpcr_state,rounding,esize); + ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, working); + V_jit(ctx,Rd,result); + } } void intrinsic_float_binary_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t Q, uint64_t sz, uint64_t float_instruction, uint64_t double_instruction) @@ -11270,6 +11822,7 @@ void intrinsic_float_binary_scalar_jit(ssa_emit_context* ctx, uint64_t Rd, uint6 { result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,half_instruction,operand1,operand2), int128); ssa_emit_context::vector_insert(ctx,result, 1ULL, 16ULL, ir_operand::create_con(0ULL, int64)); + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); } else { @@ -11346,6 +11899,264 @@ ir_operand sse_coppy_gp_across_lanes_jit(ssa_emit_context* ctx, ir_operand sourc } } +void floating_point_multiply_scalar_element_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index) +{ + ir_operand operand1 = V_jit(ctx,Rn); + ir_operand operand2 = V_jit(ctx,Rm); + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + if ((use_x86_sse_jit(ctx))) + { + operand2 = sse_copy_to_xmm_from_xmm_element_jit(ctx,operand2,esize,index); + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)32ULL))) + multiply_instruction = (uint64_t)x86_mulss; + else + multiply_instruction = (uint64_t)x86_mulsd; + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + ir_operand product = FPMul_jit(ctx,copy_new_raw_size(ctx, operand1, int64),ssa_emit_context::vector_extract(ctx,operand2, index, esize),fpcr_state,esize); + ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, product); + V_jit(ctx,Rd,result); + } +} + +void floating_point_multiply_vector_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index) +{ + ir_operand operand1 = V_jit(ctx,Rn); + ir_operand operand2 = V_jit(ctx,Rm); + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); + if ((use_x86_sse_jit(ctx))) + { + operand2 = sse_copy_to_xmm_from_xmm_element_jit(ctx,operand2,esize,index); + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)32ULL))) + multiply_instruction = (uint64_t)x86_mulps; + else + multiply_instruction = (uint64_t)x86_mulpd; + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, index, esize); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); + ssa_emit_context::vector_insert(ctx,result, e, esize, FPMul_jit(ctx,element1,element2,fpcr_state,esize)); + } + V_jit(ctx,Rd,result); + } +} + +void floating_point_multiply_accumulate_scalar_element_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index) +{ + ir_operand operand1 = V_jit(ctx,Rn); + ir_operand operand2 = V_jit(ctx,Rm); + ir_operand operand3 = V_jit(ctx,Rd); + ir_operand result; + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t add_instruction; + uint64_t subtract_instruction; + uint64_t multiply_instruction; + operand2 = copy_new_raw_size(ctx, ssa_emit_context::vector_extract(ctx,operand2, index, esize), int128); + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addsd; + subtract_instruction = (uint64_t)x86_subsd; + multiply_instruction = (uint64_t)x86_mulsd; + } + else + { + add_instruction = (uint64_t)x86_addss; + subtract_instruction = (uint64_t)x86_subss; + multiply_instruction = (uint64_t)x86_mulss; + } + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); + if ((neg)) + { + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,subtract_instruction,operand3,result), int128); + } + else + { + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,operand3,result), int128); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + } + else + { + result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + ir_operand element1 = copy_new_raw_size(ctx, operand1, int64); + if ((neg)) + { + element1 = FPNeg_jit(ctx,element1,fpcr_state,esize); + } + ir_operand product_accumalant = FPMulAdd_jit(ctx,copy_new_raw_size(ctx, operand3, int64),element1,ssa_emit_context::vector_extract(ctx,operand2, index, esize),fpcr_state,esize); + ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, product_accumalant); + } + V_jit(ctx,Rd,result); +} + +void floating_point_multiply_accumulate_vector_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index) +{ + ir_operand operand1 = V_jit(ctx,Rn); + ir_operand operand2 = V_jit(ctx,Rm); + ir_operand operand3 = V_jit(ctx,Rd); + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); + ir_operand result; + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t add_instruction; + uint64_t subtract_instruction; + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addpd; + subtract_instruction = (uint64_t)x86_subpd; + multiply_instruction = (uint64_t)x86_mulpd; + } + else + { + add_instruction = (uint64_t)x86_addps; + subtract_instruction = (uint64_t)x86_subps; + multiply_instruction = (uint64_t)x86_mulps; + } + operand2 = sse_copy_to_xmm_from_xmm_element_jit(ctx,operand2,esize,index); + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); + if ((neg)) + { + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,subtract_instruction,operand3,result), int128); + } + else + { + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,operand3,result), int128); + } + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + } + else + { + result = ssa_emit_context::vector_zero(ctx); + uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, index, esize); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); + ir_operand element3 = ssa_emit_context::vector_extract(ctx,operand3, e, esize); + if ((neg)) + { + element1 = FPNeg_jit(ctx,element1,fpcr_state,esize); + } + ssa_emit_context::vector_insert(ctx,result, e, esize, FPMulAdd_jit(ctx,element3,element1,element2,fpcr_state,esize)); + } + } + V_jit(ctx,Rd,result); +} + +void fcm_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz) +{ + ir_operand n = V_jit(ctx,Rn); + ir_operand m; + if ((((uint64_t)Rm == (uint64_t)-1ULL))) + { + m = ssa_emit_context::vector_zero(ctx); + } + else + { + m = V_jit(ctx,Rm); + } + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); + if ((use_x86_sse_jit(ctx))) + { + ir_operand result; + uint64_t operation = select_jit(ctx,((uint64_t)esize == (uint64_t)64ULL),(uint64_t)x86_cmppd,(uint64_t)x86_cmpps); + uint64_t control; + if ((((uint64_t)mode == (uint64_t)0ULL))) + { + control = 0ULL; + } + else if ((((uint64_t)mode == (uint64_t)1ULL))) + { + control = 6ULL; + } + else if ((((uint64_t)mode == (uint64_t)2ULL))) + { + control = 5ULL; + } + result = copy_new_raw_size(ctx, intrinsic_ternary_imm_jit(ctx,int128,operation,n,m,control), int128); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,n, e, esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,m, e, esize); + ir_operand element_result; + if ((((uint64_t)mode == (uint64_t)0ULL))) + { + element_result = ssa_emit_context::emit_ssa(ctx, ir_subtract, ir_operand::create_con(0ULL, int64), FPCompareEQ_jit(ctx,element1,element2,fpcr_state,esize)); + } + else if ((((uint64_t)mode == (uint64_t)1ULL))) + { + element_result = ssa_emit_context::emit_ssa(ctx, ir_subtract, ir_operand::create_con(0ULL, int64), FPCompareGT_jit(ctx,element1,element2,fpcr_state,esize)); + } + else if ((((uint64_t)mode == (uint64_t)2ULL))) + { + element_result = ssa_emit_context::emit_ssa(ctx, ir_subtract, ir_operand::create_con(0ULL, int64), FPCompareGE_jit(ctx,element1,element2,fpcr_state,esize)); + } + ssa_emit_context::vector_insert(ctx,result, e, esize, element_result); + } + V_jit(ctx,Rd,result); + } +} + +ir_operand clear_vector_scalar_jit(ssa_emit_context* ctx, ir_operand working, uint64_t fltsize) +{ + if ((((uint64_t)fltsize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,working, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,working, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + return working; +} + void add_subtract_imm12_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t sh, uint64_t imm12, uint64_t Rn, uint64_t Rd) { uint64_t O = sf == 0ULL ? int32 : sf == 1ULL ? int64 : 0; @@ -11565,17 +12376,28 @@ void divide_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t Rm, uint64_t o1, ui } } +uint64_t create_rbit_mask_jit(ssa_emit_context* ctx, uint64_t index) +{ + index = ((uint64_t)1ULL << (uint64_t)index); + uint64_t mask = ((uint64_t)(((uint64_t)(((uint64_t)1ULL << (uint64_t)index)) - (uint64_t)1ULL)) << (uint64_t)index); + mask = replicate_c_jit(ctx,mask,((uint64_t)index * (uint64_t)2ULL),((uint64_t)((uint64_t)64ULL / (uint64_t)index) / (uint64_t)2ULL)); + return mask; +} + void rbit_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t Rn, uint64_t Rd) { uint64_t datasize = ((uint64_t)32ULL << (uint64_t)sf); uint64_t O = sf == 0ULL ? int32 : sf == 1ULL ? int64 : 0; { ir_operand operand = copy_new_raw_size(ctx, X_jit(ctx,Rn), O); - ir_operand result = ir_operand::create_con(0ULL, O); - for (uint64_t i = 0; i < (datasize); i++) + ir_operand result = operand; + uint64_t max = ((uint64_t)5ULL + (uint64_t)sf); + for (uint64_t i = 0; i < (max); i++) { - ir_operand working = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, operand, ir_operand::create_con(i, O)), ir_operand::create_con(1ULL, O)); - result = ssa_emit_context::emit_ssa(ctx, ir_bitwise_or, result, ssa_emit_context::emit_ssa(ctx, ir_shift_left, working, ir_operand::create_con((((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)), O))); + uint64_t n_mask = create_rbit_mask_jit(ctx,i); + uint64_t i_mask = ~n_mask; + uint64_t shift = ((uint64_t)1ULL << (uint64_t)i); + result = ssa_emit_context::emit_ssa(ctx, ir_bitwise_or, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, result, ir_operand::create_con(n_mask, O)), ir_operand::create_con(shift, O)), ssa_emit_context::emit_ssa(ctx, ir_shift_left, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, result, ir_operand::create_con(i_mask, O)), ir_operand::create_con(shift, O))); } X_jit(ctx,Rd,copy_new_raw_size(ctx, result, int64)); } @@ -11631,16 +12453,53 @@ void count_leading_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t s, uint64_t sig_bit = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, operand, ir_operand::create_con((((uint64_t)datasize - (uint64_t)1ULL)), O)), ir_operand::create_con(1ULL, O)); datasize = ((uint64_t)datasize - (uint64_t)1ULL); } - for (uint64_t i = 0; i < (datasize); i++) + if ((use_x86_lzcnt_jit(ctx))) { - ir_operand working = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, operand, ir_operand::create_con((((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)), O)), ir_operand::create_con(1ULL, O)); if ((s)) { + ir_operand nhigh = ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, operand, ir_operand::create_con(1ULL, O)); + ir_operand mask = ir_operand::create_con(-1ULL, O); + mask = ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, mask, ir_operand::create_con(1ULL, O)); + ir_operand nlow = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, operand, mask); + result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,O,(uint64_t)x86_lzcnt,ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, nhigh, nlow)), O); + result = ssa_emit_context::emit_ssa(ctx, ir_subtract, result, ir_operand::create_con(1ULL, O)); + } + else + { + result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,O,(uint64_t)x86_lzcnt,operand), O); + } + } + else + { + for (uint64_t i = 0; i < (datasize); i++) + { + ir_operand working = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, operand, ir_operand::create_con((((uint64_t)((uint64_t)datasize - (uint64_t)i) - (uint64_t)1ULL)), O)), ir_operand::create_con(1ULL, O)); + if ((s)) { + { + ir_operand end = ir_operation_block::create_label(ctx->ir); + ir_operand yes = ir_operation_block::create_label(ctx->ir); + + ir_operand condition = ssa_emit_context::emit_ssa(ctx, ir_compare_not_equal, working, sig_bit); + + ir_operation_block::jump_if(ctx->ir,yes, condition); + /* TODO if statements without a no should not have this*/ + + ir_operation_block::jump(ctx->ir,end); + ir_operation_block::mark_label(ctx->ir, yes); + + { + ssa_emit_context::move(ctx,done,ir_operand::create_con(1ULL, O)); + } + + ir_operation_block::mark_label(ctx->ir, end); + } + } + else { ir_operand end = ir_operation_block::create_label(ctx->ir); ir_operand yes = ir_operation_block::create_label(ctx->ir); - ir_operand condition = ssa_emit_context::emit_ssa(ctx, ir_compare_not_equal, working, sig_bit); + ir_operand condition = working; ir_operation_block::jump_if(ctx->ir,yes, condition); /* TODO if statements without a no should not have this*/ @@ -11654,42 +12513,24 @@ void count_leading_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t s, uint64_t ir_operation_block::mark_label(ctx->ir, end); } - } - else { - ir_operand end = ir_operation_block::create_label(ctx->ir); - ir_operand yes = ir_operation_block::create_label(ctx->ir); - - ir_operand condition = working; - - ir_operation_block::jump_if(ctx->ir,yes, condition); - /* TODO if statements without a no should not have this*/ - - ir_operation_block::jump(ctx->ir,end); - ir_operation_block::mark_label(ctx->ir, yes); - { - ssa_emit_context::move(ctx,done,ir_operand::create_con(1ULL, O)); + ir_operand end = ir_operation_block::create_label(ctx->ir); + ir_operand yes = ir_operation_block::create_label(ctx->ir); + + ir_operand condition = ssa_emit_context::emit_ssa(ctx, ir_logical_not, done); + + ir_operation_block::jump_if(ctx->ir,yes, condition); + /* TODO if statements without a no should not have this*/ + + ir_operation_block::jump(ctx->ir,end); + ir_operation_block::mark_label(ctx->ir, yes); + + { + ssa_emit_context::move(ctx,result,ssa_emit_context::emit_ssa(ctx, ir_add, result, ir_operand::create_con(1ULL, O))); + } + + ir_operation_block::mark_label(ctx->ir, end); } - - ir_operation_block::mark_label(ctx->ir, end); - } - { - ir_operand end = ir_operation_block::create_label(ctx->ir); - ir_operand yes = ir_operation_block::create_label(ctx->ir); - - ir_operand condition = ssa_emit_context::emit_ssa(ctx, ir_logical_not, done); - - ir_operation_block::jump_if(ctx->ir,yes, condition); - /* TODO if statements without a no should not have this*/ - - ir_operation_block::jump(ctx->ir,end); - ir_operation_block::mark_label(ctx->ir, yes); - - { - ssa_emit_context::move(ctx,result,ssa_emit_context::emit_ssa(ctx, ir_add, result, ir_operand::create_con(1ULL, O))); - } - - ir_operation_block::mark_label(ctx->ir, end); } } X_jit(ctx,Rd,copy_new_raw_size(ctx, result, int64)); @@ -12560,6 +13401,11 @@ void load_store_exclusive_ordered_jit(ssa_emit_context* ctx, uint64_t size, uint } } +ir_operand exclusive_address_mask_jit(ssa_emit_context* ctx) +{ + return ir_operand::create_con(~(((uint64_t)(((uint64_t)4ULL << (uint64_t)4ULL)) - (uint64_t)1ULL)), int64); +} + void load_exclusive_jit(ssa_emit_context* ctx, uint64_t is_exclusive, uint64_t size, uint64_t Rn, uint64_t Rt) { uint64_t datasize = ((uint64_t)8ULL << (uint64_t)size); @@ -12569,7 +13415,7 @@ void load_exclusive_jit(ssa_emit_context* ctx, uint64_t is_exclusive, uint64_t s ir_operand value = copy_new_raw_size(ctx, mem_jit(ctx,S,address), S); if ((is_exclusive)) { - _sys_jit(ctx,(uint64_t)exclusive_address,ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, address, ir_operand::create_con(~63ULL, int64))); + _sys_jit(ctx,(uint64_t)exclusive_address,ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, address, exclusive_address_mask_jit(ctx))); _sys_jit(ctx,(uint64_t)exclusive_value,copy_new_raw_size(ctx, value, int64)); } X_jit(ctx,Rt,copy_new_raw_size(ctx, value, int64)); @@ -12584,7 +13430,7 @@ void store_exclusive_jit(ssa_emit_context* ctx, uint64_t is_exclusive, uint64_t { if ((is_exclusive)) { - ir_operand mask = ir_operand::create_con(~63ULL, int64); + ir_operand mask = exclusive_address_mask_jit(ctx); ir_operand _exclusive_address = _sys_jit(ctx,(uint64_t)exclusive_address); { ir_operand end = ir_operation_block::create_label(ctx->ir); @@ -12604,7 +13450,7 @@ void store_exclusive_jit(ssa_emit_context* ctx, uint64_t is_exclusive, uint64_t ir_operand to_swap = copy_new_raw_size(ctx, X_jit(ctx,Rt), S); ir_operand expecting = copy_new_raw_size(ctx, _sys_jit(ctx,(uint64_t)exclusive_value), S); ir_operand cas_success = copy_new_raw_size(ctx, compare_and_swap_jit(ctx,address,copy_new_raw_size(ctx, expecting, int64),copy_new_raw_size(ctx, to_swap, int64),datasize), S); - X_jit(ctx,Rs,copy_new_raw_size(ctx, ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, cas_success, ir_operand::create_con(1ULL, S)), int64)); + X_jit(ctx,Rs,copy_new_raw_size(ctx, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, cas_success, ir_operand::create_con(1ULL, S)), ir_operand::create_con(1ULL, S)), int64)); } ir_operation_block::mark_label(ctx->ir, end); @@ -12807,15 +13653,41 @@ void frsqrte_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); ir_operand operand = V_jit(ctx,Rn); - ir_operand result = ssa_emit_context::vector_zero(ctx); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_jit(ctx))) { - ir_operand element = ssa_emit_context::vector_extract(ctx,operand, e, esize); - element = FPRSqrtEstimate_jit(ctx,element,fpcr_state,esize); - ssa_emit_context::vector_insert(ctx,result, e, esize, element); + ir_operand one = sse_coppy_gp_across_lanes_jit(ctx,float_imm_jit(ctx,ir_operand::create_con(1ULL, int64),esize),esize); + uint64_t sqrt_instruction; + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtpd; + divide_instruction = (uint64_t)x86_divpd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtps; + divide_instruction = (uint64_t)x86_divps; + } + ir_operand sqrt_result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int128,sqrt_instruction,operand), int128); + ir_operand reciprocal_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,divide_instruction,one,sqrt_result), int128); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + ssa_emit_context::vector_insert(ctx,reciprocal_result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,reciprocal_result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element = ssa_emit_context::vector_extract(ctx,operand, e, esize); + element = FPRSqrtEstimate_jit(ctx,element,fpcr_state,esize); + ssa_emit_context::vector_insert(ctx,result, e, esize, element); + } + V_jit(ctx,Rd,result); } - V_jit(ctx,Rd,result); } void frsqrts_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -12825,15 +13697,48 @@ void frsqrts_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); ir_operand operand1 = V_jit(ctx,Rn); ir_operand operand2 = V_jit(ctx,Rm); - ir_operand result = ssa_emit_context::vector_zero(ctx); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_jit(ctx))) { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); - ssa_emit_context::vector_insert(ctx,result, e, esize, FPRSqrtStepFused_jit(ctx,element1,element2,fpcr_state,esize)); + ir_operand two = sse_coppy_gp_across_lanes_jit(ctx,float_imm_jit(ctx,ir_operand::create_con(2ULL, int64),esize),esize); + ir_operand three = sse_coppy_gp_across_lanes_jit(ctx,float_imm_jit(ctx,ir_operand::create_con(3ULL, int64),esize),esize); + ir_operand negator = sse_coppy_gp_across_lanes_jit(ctx,ir_operand::create_con(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL))), int64),esize); + operand1 = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand1,negator), int128); + uint64_t add_instruction; + uint64_t multiply_instruction; + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addpd; + multiply_instruction = (uint64_t)x86_mulpd; + divide_instruction = (uint64_t)x86_divpd; + } + else + { + add_instruction = (uint64_t)x86_addps; + multiply_instruction = (uint64_t)x86_mulps; + divide_instruction = (uint64_t)x86_divps; + } + ir_operand mul_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); + ir_operand add_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,three,mul_result), int128); + ir_operand div_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,divide_instruction,add_result,two), int128); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + ssa_emit_context::vector_insert(ctx,div_result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,div_result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); + ssa_emit_context::vector_insert(ctx,result, e, esize, FPRSqrtStepFused_jit(ctx,element1,element2,fpcr_state,esize)); + } + V_jit(ctx,Rd,result); } - V_jit(ctx,Rd,result); } void frecps_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -12843,15 +13748,42 @@ void frecps_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); ir_operand operand1 = V_jit(ctx,Rn); ir_operand operand2 = V_jit(ctx,Rm); - ir_operand result = ssa_emit_context::vector_zero(ctx); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_jit(ctx))) { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); - ssa_emit_context::vector_insert(ctx,result, e, esize, FPRecipStepFused_jit(ctx,element1,element2,fpcr_state,esize)); + ir_operand negator = sse_coppy_gp_across_lanes_jit(ctx,ir_operand::create_con(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL))), int64),esize); + ir_operand two = sse_coppy_gp_across_lanes_jit(ctx,float_imm_jit(ctx,ir_operand::create_con(2ULL, int64),esize),esize); + operand1 = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand1,negator), int128); + uint64_t add_instruction; + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + add_instruction = (uint64_t)x86_addpd; + multiply_instruction = (uint64_t)x86_mulpd; + } + else + { + add_instruction = (uint64_t)x86_addps; + multiply_instruction = (uint64_t)x86_mulps; + } + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,two,copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128)), int128); + if ((((uint64_t)datasize == (uint64_t)64ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); + ssa_emit_context::vector_insert(ctx,result, e, esize, FPRecipStepFused_jit(ctx,element1,element2,fpcr_state,esize)); + } + V_jit(ctx,Rd,result); } - V_jit(ctx,Rd,result); } void fmul_scalar_by_element_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -12872,14 +13804,7 @@ void fmul_scalar_by_element_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t L, { undefined_jit(ctx); } - ir_operand operand1 = V_jit(ctx,Rn); - ir_operand operand2 = V_jit(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); - ir_operand result = ssa_emit_context::vector_zero(ctx); - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - ir_operand product = FPMul_jit(ctx,copy_new_raw_size(ctx, operand1, int64),ssa_emit_context::vector_extract(ctx,operand2, index, esize),fpcr_state,esize); - ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, product); - V_jit(ctx,Rd,result); + floating_point_multiply_scalar_element_jit(ctx,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),sz,index); } void fmul_vector_by_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -12900,20 +13825,7 @@ void fmul_vector_by_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, { undefined_jit(ctx); } - ir_operand operand1 = V_jit(ctx,Rn); - ir_operand operand2 = V_jit(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); - ir_operand result = ssa_emit_context::vector_zero(ctx); - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); - uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, index, esize); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) - { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); - ssa_emit_context::vector_insert(ctx,result, e, esize, FPMul_jit(ctx,element1,element2,fpcr_state,esize)); - } - V_jit(ctx,Rd,result); + floating_point_multiply_vector_element_jit(ctx,Q,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),sz,index); } void fmul_accumulate_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -12934,57 +13846,7 @@ void fmul_accumulate_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t L, { undefined_jit(ctx); } - ir_operand operand1 = V_jit(ctx,Rn); - ir_operand operand2 = V_jit(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); - ir_operand operand3 = V_jit(ctx,Rd); - ir_operand result; - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) - { - uint64_t add_instruction; - uint64_t subtract_instruction; - uint64_t multiply_instruction; - operand2 = copy_new_raw_size(ctx, ssa_emit_context::vector_extract(ctx,operand2, index, esize), int128); - if ((((uint64_t)esize == (uint64_t)64ULL))) - { - add_instruction = (uint64_t)x86_addsd; - subtract_instruction = (uint64_t)x86_subsd; - multiply_instruction = (uint64_t)x86_mulsd; - } - else - { - add_instruction = (uint64_t)x86_addss; - subtract_instruction = (uint64_t)x86_subss; - multiply_instruction = (uint64_t)x86_mulss; - } - result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); - if ((neg)) - { - result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,subtract_instruction,operand3,result), int128); - } - else - { - result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,operand3,result), int128); - } - ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); - if ((((uint64_t)esize == (uint64_t)32ULL))) - { - ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); - } - } - else - { - result = ssa_emit_context::vector_zero(ctx); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - ir_operand element1 = copy_new_raw_size(ctx, operand1, int64); - if ((neg)) - { - element1 = FPNeg_jit(ctx,element1,fpcr_state,esize); - } - ir_operand product_accumalant = FPMulAdd_jit(ctx,copy_new_raw_size(ctx, operand3, int64),element1,ssa_emit_context::vector_extract(ctx,operand2, index, esize),fpcr_state,esize); - ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, product_accumalant); - } - V_jit(ctx,Rd,result); + floating_point_multiply_accumulate_scalar_element_jit(ctx,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),neg,sz,index); } void fmul_accumulate_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd) @@ -13008,59 +13870,7 @@ void fmul_accumulate_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, ir_operand operand1 = V_jit(ctx,Rn); ir_operand operand2 = V_jit(ctx,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL)))); ir_operand operand3 = V_jit(ctx,Rd); - ir_operand result; - uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); - uint64_t datasize = ((uint64_t)64ULL << (uint64_t)Q); - if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) - { - uint64_t add_instruction; - uint64_t subtract_instruction; - uint64_t multiply_instruction; - if ((((uint64_t)esize == (uint64_t)64ULL))) - { - add_instruction = (uint64_t)x86_addpd; - subtract_instruction = (uint64_t)x86_subpd; - multiply_instruction = (uint64_t)x86_mulpd; - } - else - { - add_instruction = (uint64_t)x86_addps; - subtract_instruction = (uint64_t)x86_subps; - multiply_instruction = (uint64_t)x86_mulps; - } - operand2 = sse_copy_to_xmm_from_xmm_element_jit(ctx,operand2,esize,index); - result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); - if ((neg)) - { - result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,subtract_instruction,operand3,result), int128); - } - else - { - result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,operand3,result), int128); - } - if ((((uint64_t)datasize == (uint64_t)64ULL))) - { - ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); - } - } - else - { - result = ssa_emit_context::vector_zero(ctx); - uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, index, esize); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - for (uint64_t e = 0; e < (elements); e++) - { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); - ir_operand element3 = ssa_emit_context::vector_extract(ctx,operand3, e, esize); - if ((neg)) - { - element1 = FPNeg_jit(ctx,element1,fpcr_state,esize); - } - ssa_emit_context::vector_insert(ctx,result, e, esize, FPMulAdd_jit(ctx,element3,element1,element2,fpcr_state,esize)); - } - } - V_jit(ctx,Rd,result); + floating_point_multiply_accumulate_vector_element_jit(ctx,Q,Rd,Rn,((uint64_t)Rm | (uint64_t)(((uint64_t)rm_hi << (uint64_t)4ULL))),neg,sz,index); } void faddp_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t Rn, uint64_t Rd) @@ -13068,12 +13878,55 @@ void faddp_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t Rn, uint64_t uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); uint64_t datasize = ((uint64_t)esize * (uint64_t)2ULL); ir_operand operand = V_jit(ctx,Rn); - ir_operand result = ssa_emit_context::vector_zero(ctx); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - ir_operand bottom = ssa_emit_context::vector_extract(ctx,operand, 0ULL, esize); - ir_operand top = ssa_emit_context::vector_extract(ctx,operand, 1ULL, esize); - ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, FPAdd_jit(ctx,bottom,top,fpcr_state,esize)); - V_jit(ctx,Rd,result); + if ((use_x86_sse_jit(ctx))) + { + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,select_jit(ctx,((uint64_t)esize == (uint64_t)64ULL),(uint64_t)x86_haddpd,(uint64_t)x86_haddps),operand,operand), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + ir_operand bottom = ssa_emit_context::vector_extract(ctx,operand, 0ULL, esize); + ir_operand top = ssa_emit_context::vector_extract(ctx,operand, 1ULL, esize); + ssa_emit_context::vector_insert(ctx,result, 0ULL, esize, FPAdd_jit(ctx,bottom,top,fpcr_state,esize)); + V_jit(ctx,Rd,result); + } +} + +void fcmeq_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_jit(ctx,Rd,Rn,-1ULL,0ULL,Q,sz); +} + +void fcmgt_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_jit(ctx,Rd,Rn,-1ULL,1ULL,Q,sz); +} + +void fcmge_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_jit(ctx,Rd,Rn,-1ULL,2ULL,Q,sz); +} + +void fcmeq_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_jit(ctx,Rd,Rn,Rm,0ULL,Q,sz); +} + +void fcmgt_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_jit(ctx,Rd,Rn,Rm,1ULL,Q,sz); +} + +void fcmge_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) +{ + fcm_vector_jit(ctx,Rd,Rn,Rm,2ULL,Q,sz); } void fadd_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -13148,36 +14001,171 @@ void fminnm_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint6 void fnmul_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)) && (uint64_t)((uint64_t)ftype != (uint64_t)3ULL)))) + { + ir_operand operand1 = V_jit(ctx,Rn); + ir_operand operand2 = V_jit(ctx,Rm); + uint64_t esize = get_flt_size_jit(ctx,ftype); + ir_operand negator = copy_new_raw_size(ctx, ir_operand::create_con((((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))), int64), int128); + uint64_t multiply_instruction; + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + multiply_instruction = (uint64_t)x86_mulss; + } + else if ((((uint64_t)esize == (uint64_t)64ULL))) + { + multiply_instruction = (uint64_t)x86_mulsd; + } + else + { + undefined_jit(ctx); + } + operand2 = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand2,negator), int128); + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,multiply_instruction,operand1,operand2), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,result); + return; + } float_binary_scalar_jit(ctx,Rd,Rn,Rm,ftype,(uint64_t)FPNMul_jit); } void fabs_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t esize = get_flt_size_jit(ctx,ftype); + ir_operand operand = V_jit(ctx,Rn); + ir_operand mask = copy_new_raw_size(ctx, ir_operand::create_con((((uint64_t)(((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))) - (uint64_t)1ULL)), int64), int128); + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_pand,operand,mask), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,result); + return; + } float_unary_scalar_jit(ctx,Rd,Rn,ftype,(uint64_t)FPAbs_jit); } void fneg_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t esize = get_flt_size_jit(ctx,ftype); + ir_operand operand = V_jit(ctx,Rn); + ir_operand mask = copy_new_raw_size(ctx, ir_operand::create_con((((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))), int64), int128); + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand,mask), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,result); + return; + } float_unary_scalar_jit(ctx,Rd,Rn,ftype,(uint64_t)FPNeg_jit); } void fneg_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + ir_operand operand = V_jit(ctx,Rn); + ir_operand mask = sse_coppy_gp_across_lanes_jit(ctx,ir_operand::create_con((((uint64_t)1ULL << (uint64_t)(((uint64_t)esize - (uint64_t)1ULL)))), int64),esize); + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand,mask), int128); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,result); + return; + } float_unary_vector_jit(ctx,Rd,Rn,Q,sz,(uint64_t)FPNeg_jit); } void fsqrt_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)) && (uint64_t)((uint64_t)ftype != (uint64_t)3ULL)))) + { + uint64_t esize = get_flt_size_jit(ctx,ftype); + ir_operand operand = V_jit(ctx,Rn); + uint64_t sqrt_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtsd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtss; + } + ir_operand result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int128,sqrt_instruction,operand), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,result); + return; + } float_unary_scalar_jit(ctx,Rd,Rn,ftype,(uint64_t)FPSqrt_jit); } void fsqrt_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + ir_operand operand = V_jit(ctx,Rn); + uint64_t sqrt_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtpd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtps; + } + ir_operand result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int128,sqrt_instruction,operand), int128); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,result); + return; + } float_unary_vector_jit(ctx,Rd,Rn,Q,sz,(uint64_t)FPSqrt_jit); } void frecpe_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) { + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) + { + uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); + ir_operand operand = V_jit(ctx,Rn); + ir_operand one = sse_coppy_gp_across_lanes_jit(ctx,float_imm_jit(ctx,ir_operand::create_con(1ULL, int64),esize),esize); + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + divide_instruction = (uint64_t)x86_divpd; + } + else + { + divide_instruction = (uint64_t)x86_divps; + } + ir_operand result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,divide_instruction,one,operand), int128); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,result); + return; + } float_unary_vector_jit(ctx,Rd,Rn,Q,sz,(uint64_t)FPRecipEstimate_jit); } @@ -13185,13 +14173,40 @@ void frsqrte_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t Rn, uint64_ { uint64_t esize = ((uint64_t)32ULL << (uint64_t)sz); ir_operand operand = copy_new_raw_size(ctx, V_jit(ctx,Rn), int64); - ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); - ir_operand result = FPRSqrtEstimate_jit(ctx,operand,fpcr_state,esize); - if ((((uint64_t)esize == (uint64_t)32ULL))) + if ((((uint64_t)use_fast_float_jit(ctx) && (uint64_t)use_x86_sse_jit(ctx)))) { - result = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, result, ir_operand::create_con(4294967295ULL, int64)); + ir_operand one = copy_new_raw_size(ctx, float_imm_jit(ctx,ir_operand::create_con(1ULL, int64),esize), int128); + uint64_t sqrt_instruction; + uint64_t divide_instruction; + if ((((uint64_t)esize == (uint64_t)64ULL))) + { + sqrt_instruction = (uint64_t)x86_sqrtsd; + divide_instruction = (uint64_t)x86_divsd; + } + else + { + sqrt_instruction = (uint64_t)x86_sqrtss; + divide_instruction = (uint64_t)x86_divss; + } + ir_operand sqrt_result = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int128,sqrt_instruction,copy_new_raw_size(ctx, operand, int128)), int128); + ir_operand reciprocal_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,divide_instruction,one,sqrt_result), int128); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + ssa_emit_context::vector_insert(ctx,reciprocal_result, 1ULL, 32ULL, ir_operand::create_con(0ULL, int64)); + } + ssa_emit_context::vector_insert(ctx,reciprocal_result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + V_jit(ctx,Rd,reciprocal_result); + } + else + { + ir_operand fpcr_state = _sys_jit(ctx,(uint64_t)fpcr); + ir_operand result = FPRSqrtEstimate_jit(ctx,operand,fpcr_state,esize); + if ((((uint64_t)esize == (uint64_t)32ULL))) + { + result = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, result, ir_operand::create_con(4294967295ULL, int64)); + } + V_jit(ctx,Rd,copy_new_raw_size(ctx, result, int128)); } - V_jit(ctx,Rd,copy_new_raw_size(ctx, result, int128)); } void fmov_scalar_immediate_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t imm8, uint64_t Rd) @@ -13203,6 +14218,18 @@ void fmov_scalar_immediate_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t i ir_operand _compare_and_swap_jit(ssa_emit_context* ctx, ir_operand physical_address, ir_operand expecting, ir_operand to_swap, uint64_t size) { + if ((use_x86_jit(ctx))) + { + if ((((uint64_t)size != (uint64_t)64ULL))) + { + ir_operand mask = ir_operand::create_con(((uint64_t)(((uint64_t)1ULL << (uint64_t)size)) - (uint64_t)1ULL), int64); + ir_operand masking_value = ssa_emit_context::emit_ssa(ctx, ir_load, physical_address, int64); + masking_value = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, masking_value, ssa_emit_context::emit_ssa(ctx, ir_bitwise_not, mask)); + expecting = ssa_emit_context::emit_ssa(ctx, ir_bitwise_or, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, expecting, mask), masking_value); + to_swap = ssa_emit_context::emit_ssa(ctx, ir_bitwise_or, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, to_swap, mask), masking_value); + } + return copy_new_raw_size(ctx, intrinsic_ternary_jit(ctx,int64,(uint64_t)x86_cmpxchg,physical_address,expecting,to_swap), int64); + } return call_jit(ctx,physical_address,expecting,to_swap,ir_operand::create_con(size, int64),ir_operand::create_con(0ULL, int64),ir_operand::create_con(0ULL, int64),(uint64_t)compare_and_swap_interpreter_cpp); } @@ -13804,12 +14831,35 @@ void add_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t size, uint64_t R uint64_t elements = ((uint64_t)datasize / (uint64_t)esize); ir_operand operand1 = V_jit(ctx,Rn); ir_operand operand2 = V_jit(ctx,Rm); - ir_operand result = ssa_emit_context::vector_zero(ctx); - for (uint64_t e = 0; e < (elements); e++) + ir_operand result; + if ((use_x86_sse2_jit(ctx))) { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); - ssa_emit_context::vector_insert(ctx,result, e, esize, ssa_emit_context::emit_ssa(ctx, ir_add, element1, element2)); + uint64_t add_instruction; + if ((((uint64_t)esize == (uint64_t)8ULL))) + add_instruction = (uint64_t)x86_paddb; + else if ((((uint64_t)esize == (uint64_t)16ULL))) + add_instruction = (uint64_t)x86_paddw; + else if ((((uint64_t)esize == (uint64_t)32ULL))) + add_instruction = (uint64_t)x86_paddd; + else if ((((uint64_t)esize == (uint64_t)64ULL))) + add_instruction = (uint64_t)x86_paddq; + else + undefined_jit(ctx); + result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,add_instruction,operand1,operand2), int128); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + ssa_emit_context::vector_insert(ctx,result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + } + else + { + result = ssa_emit_context::vector_zero(ctx); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); + ssa_emit_context::vector_insert(ctx,result, e, esize, ssa_emit_context::emit_ssa(ctx, ir_add, element1, element2)); + } } V_jit(ctx,Rd,result); } @@ -13851,26 +14901,33 @@ void cnt_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t size, uint64_t Rn, uint { ir_operand working = copy_new_raw_size(ctx, ssa_emit_context::vector_extract(ctx,source, e, esize), int8); ir_operand count = ssa_emit_context::emit_ssa(ctx, ir_move, ir_operand::create_con(0ULL, int8)); - for (uint64_t b = 0; b < (esize); b++) + if ((use_x86_jit(ctx))) { - ir_operand bit = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, working, ir_operand::create_con(b, int8)), ir_operand::create_con(1ULL, int8)); + count = copy_new_raw_size(ctx, intrinsic_unary_jit(ctx,int16,(uint64_t)x86_popcnt,copy_new_raw_size(ctx, working, int16)), int8); + } + else + { + for (uint64_t b = 0; b < (esize); b++) { - ir_operand end = ir_operation_block::create_label(ctx->ir); - ir_operand yes = ir_operation_block::create_label(ctx->ir); - - ir_operand condition = bit; - - ir_operation_block::jump_if(ctx->ir,yes, condition); - /* TODO if statements without a no should not have this*/ - - ir_operation_block::jump(ctx->ir,end); - ir_operation_block::mark_label(ctx->ir, yes); - + ir_operand bit = ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_shift_right_unsigned, working, ir_operand::create_con(b, int8)), ir_operand::create_con(1ULL, int8)); { - ssa_emit_context::move(ctx,count,ssa_emit_context::emit_ssa(ctx, ir_add, count, ir_operand::create_con(1ULL, int8))); + ir_operand end = ir_operation_block::create_label(ctx->ir); + ir_operand yes = ir_operation_block::create_label(ctx->ir); + + ir_operand condition = bit; + + ir_operation_block::jump_if(ctx->ir,yes, condition); + /* TODO if statements without a no should not have this*/ + + ir_operation_block::jump(ctx->ir,end); + ir_operation_block::mark_label(ctx->ir, yes); + + { + ssa_emit_context::move(ctx,count,ssa_emit_context::emit_ssa(ctx, ir_add, count, ir_operand::create_con(1ULL, int8))); + } + + ir_operation_block::mark_label(ctx->ir, end); } - - ir_operation_block::mark_label(ctx->ir, end); } } ssa_emit_context::vector_insert(ctx,result, e, esize, count); @@ -13914,15 +14971,29 @@ void bsl_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rm, uint64_t Rn, ir_operand operand1 = V_jit(ctx,Rm); ir_operand operand2 = V_jit(ctx,Rd); ir_operand operand3 = V_jit(ctx,Rn); - ir_operand result = ssa_emit_context::vector_zero(ctx); - for (uint64_t e = 0; e < (elements); e++) + if ((use_x86_sse_jit(ctx))) { - ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); - ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); - ir_operand element3 = ssa_emit_context::vector_extract(ctx,operand3, e, esize); - ssa_emit_context::vector_insert(ctx,result, e, esize, ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, element1, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, element1, element3), element2))); + ir_operand xor_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand1,operand3), int128); + ir_operand and_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_pand,xor_result,operand2), int128); + ir_operand final_result = copy_new_raw_size(ctx, intrinsic_binary_jit(ctx,int128,(uint64_t)x86_xorps,operand1,and_result), int128); + if ((((uint64_t)Q == (uint64_t)0ULL))) + { + ssa_emit_context::vector_insert(ctx,final_result, 1ULL, 64ULL, ir_operand::create_con(0ULL, int64)); + } + V_jit(ctx,Rd,final_result); + } + else + { + ir_operand result = ssa_emit_context::vector_zero(ctx); + for (uint64_t e = 0; e < (elements); e++) + { + ir_operand element1 = ssa_emit_context::vector_extract(ctx,operand1, e, esize); + ir_operand element2 = ssa_emit_context::vector_extract(ctx,operand2, e, esize); + ir_operand element3 = ssa_emit_context::vector_extract(ctx,operand3, e, esize); + ssa_emit_context::vector_insert(ctx,result, e, esize, ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, element1, ssa_emit_context::emit_ssa(ctx, ir_bitwise_and, ssa_emit_context::emit_ssa(ctx, ir_bitwise_exclusive_or, element1, element3), element2))); + } + V_jit(ctx,Rd,result); } - V_jit(ctx,Rd,result); } void and_bic_vector_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t invert, uint64_t Rm, uint64_t Rn, uint64_t Rd) @@ -14090,66 +15161,34 @@ void st1_single_structure_post_index_jit(ssa_emit_context* ctx, uint64_t Q, uint memory_1_jit(ctx,1ULL,Q,1ULL,0ULL,Rm,0ULL,((uint64_t)opcode << (uint64_t)1ULL),S,size,Rn,Rt,0ULL); } -void fcmeq_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_jit(ctx,Rd,Rn,-1ULL,0ULL,Q,sz); -} - -void fcmgt_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_jit(ctx,Rd,Rn,-1ULL,1ULL,Q,sz); -} - -void fcmge_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_jit(ctx,Rd,Rn,-1ULL,2ULL,Q,sz); -} - -void fcmeq_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_jit(ctx,Rd,Rn,Rm,0ULL,Q,sz); -} - -void fcmgt_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_jit(ctx,Rd,Rn,Rm,1ULL,Q,sz); -} - -void fcmge_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd) -{ - fcm_vector_jit(ctx,Rd,Rn,Rm,2ULL,Q,sz); -} - void floating_point_conditional_select_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t cond, uint64_t Rn, uint64_t Rd) { uint64_t fltsize = get_flt_size_jit(ctx,ftype); ir_operand operand1 = V_jit(ctx,Rn); ir_operand operand2 = V_jit(ctx,Rm); - uint64_t O = fltsize == 32ULL ? int32 : fltsize == 64ULL ? int64 : 0; + ir_operand result = ssa_emit_context::emit_ssa(ctx, ir_move, ssa_emit_context::vector_zero(ctx)); { - ir_operand result = ssa_emit_context::emit_ssa(ctx, ir_move, ir_operand::create_con(0ULL, O)); + ir_operand end = ir_operation_block::create_label(ctx->ir); + ir_operand yes = ir_operation_block::create_label(ctx->ir); + + ir_operand condition = condition_holds_jit(ctx,cond); + + ir_operation_block::jump_if(ctx->ir,yes, condition); { - ir_operand end = ir_operation_block::create_label(ctx->ir); - ir_operand yes = ir_operation_block::create_label(ctx->ir); - - ir_operand condition = condition_holds_jit(ctx,cond); - - ir_operation_block::jump_if(ctx->ir,yes, condition); - { - ssa_emit_context::move(ctx,result,copy_new_raw_size(ctx, operand2, O)); - } - - ir_operation_block::jump(ctx->ir,end); - ir_operation_block::mark_label(ctx->ir, yes); - - { - ssa_emit_context::move(ctx,result,copy_new_raw_size(ctx, operand1, O)); - } - - ir_operation_block::mark_label(ctx->ir, end); + ssa_emit_context::move(ctx,result,operand2); } - V_jit(ctx,Rd,copy_new_raw_size(ctx, result, int128)); + + ir_operation_block::jump(ctx->ir,end); + ir_operation_block::mark_label(ctx->ir, yes); + + { + ssa_emit_context::move(ctx,result,operand1); + } + + ir_operation_block::mark_label(ctx->ir, end); } + result = clear_vector_scalar_jit(ctx,result,fltsize); + V_jit(ctx,Rd,result); } void fcmp_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t opc) diff --git a/src/emulator/aarch64/aarch64_impl.h b/src/emulator/aarch64/aarch64_impl.h index ab2c6bc..cbcfda6 100644 --- a/src/emulator/aarch64/aarch64_impl.h +++ b/src/emulator/aarch64/aarch64_impl.h @@ -115,8 +115,10 @@ template O add_subtract_carry_impl_interpreter(interpreter_data* ctx, O n, O m, uint64_t set_flags, uint64_t is_add, O carry); uint8_t condition_holds_interpreter(interpreter_data* ctx, uint64_t cond); void branch_long_universal_interpreter(interpreter_data* ctx, uint64_t Rn, uint64_t link); +uint64_t select_interpreter(interpreter_data* ctx, uint64_t condition, uint64_t yes, uint64_t no); uint64_t create_mask_interpreter(interpreter_data* ctx, uint64_t bits); uint64_t shift_left_check_interpreter(interpreter_data* ctx, uint64_t to_shift, uint64_t shift, uint64_t size); +uint64_t get_x86_rounding_mode_interpreter(interpreter_data* ctx, uint64_t rounding); uint64_t shift_right_check_interpreter(interpreter_data* ctx, uint64_t to_shift, uint64_t shift, uint64_t size, uint64_t is_unsigned); uint64_t reverse_interpreter(interpreter_data* ctx, uint128_t word, uint64_t M, uint64_t N); void convert_to_int_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t ftype, uint64_t Rd, uint64_t Rn, uint64_t round, uint64_t is_unsigned, uint64_t to_vector); @@ -134,7 +136,6 @@ void convert_to_float_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t f uint128_t replicate_vector_interpreter(interpreter_data* ctx, uint128_t source, uint64_t v_size, uint64_t count); void st_interpreter(interpreter_data* ctx, uint64_t wback, uint64_t Q, uint64_t L, uint64_t opcode, uint64_t size, uint64_t Rm, uint64_t Rn, uint64_t Rt); void memory_1_interpreter(interpreter_data* ctx, uint64_t wback, uint64_t Q, uint64_t L, uint64_t R, uint64_t Rm, uint64_t o2, uint64_t opcode, uint64_t S, uint64_t size, uint64_t Rn, uint64_t Rt, uint64_t is_load); -void fcm_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz); uint64_t bits_r_interpreter(interpreter_data* ctx, uint64_t operand, uint64_t top, uint64_t bottom); uint64_t infinity_interpreter(interpreter_data* ctx, uint64_t sign, uint64_t N); uint64_t float_is_nan_interpreter(interpreter_data* ctx, uint64_t operand, uint64_t N); @@ -176,6 +177,12 @@ void intrinsic_float_binary_scalar_interpreter(interpreter_data* ctx, uint64_t R void x86_sse_logic_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t Q, uint64_t invert, uint64_t primary_instruction); uint128_t sse_copy_to_xmm_from_xmm_element_interpreter(interpreter_data* ctx, uint128_t source, uint64_t size, uint64_t index); uint128_t sse_coppy_gp_across_lanes_interpreter(interpreter_data* ctx, uint64_t source, uint64_t size); +void floating_point_multiply_scalar_element_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index); +void floating_point_multiply_vector_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index); +void floating_point_multiply_accumulate_scalar_element_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index); +void floating_point_multiply_accumulate_vector_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index); +void fcm_vector_interpreter(interpreter_data* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz); +uint128_t clear_vector_scalar_interpreter(interpreter_data* ctx, uint128_t working, uint64_t fltsize); void add_subtract_imm12_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t sh, uint64_t imm12, uint64_t Rn, uint64_t Rd); void add_subtract_shifted_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t shift, uint64_t Rm, uint64_t imm6, uint64_t Rn, uint64_t Rd); void add_subtract_extended_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t Rm, uint64_t option, uint64_t imm3, uint64_t Rn, uint64_t Rd); @@ -185,6 +192,7 @@ void multiply_with_32_interpreter(interpreter_data* ctx, uint64_t U, uint64_t Rm void multiply_hi_interpreter(interpreter_data* ctx, uint64_t U, uint64_t Rm, uint64_t o0, uint64_t Rn, uint64_t Rd); void multiply_additive_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t Rm, uint64_t o0, uint64_t Ra, uint64_t Rn, uint64_t Rd); void divide_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t Rm, uint64_t o1, uint64_t Rn, uint64_t Rd); +uint64_t create_rbit_mask_interpreter(interpreter_data* ctx, uint64_t index); void rbit_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t Rn, uint64_t Rd); void rev16_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t Rn, uint64_t Rd); void reverse_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t opc, uint64_t Rn, uint64_t Rd); @@ -220,6 +228,7 @@ void load_store_register_imm_unsigned_interpreter(interpreter_data* ctx, uint64_ void load_store_register_imm_unscaled_interpreter(interpreter_data* ctx, uint64_t size, uint64_t VR, uint64_t opc, uint64_t imm9, uint64_t wb, uint64_t Rn, uint64_t Rt); void load_store_register_offset_interpreter(interpreter_data* ctx, uint64_t size, uint64_t VR, uint64_t opc, uint64_t Rm, uint64_t option, uint64_t S, uint64_t Rn, uint64_t Rt); void load_store_exclusive_ordered_interpreter(interpreter_data* ctx, uint64_t size, uint64_t ordered, uint64_t L, uint64_t Rs, uint64_t o0, uint64_t Rn, uint64_t Rt); +uint64_t exclusive_address_mask_interpreter(interpreter_data* ctx); void load_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, uint64_t size, uint64_t Rn, uint64_t Rt); void store_exclusive_interpreter(interpreter_data* ctx, uint64_t is_exclusive, uint64_t size, uint64_t Rn, uint64_t Rt, uint64_t Rs); void conversion_between_floating_point_and_fixed_point_interpreter(interpreter_data* ctx, uint64_t sf, uint64_t S, uint64_t ftype, uint64_t rmode, uint64_t opcode, uint64_t scale, uint64_t Rn, uint64_t Rd); @@ -245,6 +254,12 @@ void fmul_vector_by_element_interpreter(interpreter_data* ctx, uint64_t Q, uint6 void fmul_accumulate_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd); void fmul_accumulate_element_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd); void faddp_scalar_interpreter(interpreter_data* ctx, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmeq_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmgt_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmge_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmeq_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); +void fcmgt_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); +void fcmge_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); void fadd_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd); void fsub_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd); void fmul_scalar_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd); @@ -315,12 +330,6 @@ void st2_multiple_structures_no_offset_interpreter(interpreter_data* ctx, uint64 void st2_multiple_structures_post_index_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rm, uint64_t size, uint64_t Rn, uint64_t Rt); void st1_single_structure_no_offset_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t opcode, uint64_t S, uint64_t size, uint64_t Rn, uint64_t Rt); void st1_single_structure_post_index_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t Rm, uint64_t opcode, uint64_t S, uint64_t size, uint64_t Rn, uint64_t Rt); -void fcmeq_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); -void fcmgt_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); -void fcmge_vector_zero_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); -void fcmeq_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); -void fcmgt_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); -void fcmge_vector_register_interpreter(interpreter_data* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); void floating_point_conditional_select_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t cond, uint64_t Rn, uint64_t Rd); void fcmp_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t opc); void fccmp_interpreter(interpreter_data* ctx, uint64_t ftype, uint64_t Rm, uint64_t cond, uint64_t Rn, uint64_t nzcv); @@ -340,6 +349,12 @@ uint64_t use_fast_float_interpreter(interpreter_data* ctx);//THIS FUNCTION IS US uint64_t use_x86_sse_interpreter(interpreter_data* ctx);//THIS FUNCTION IS USER DEFINED uint64_t use_x86_sse2_interpreter(interpreter_data* ctx);//THIS FUNCTION IS USER DEFINED uint64_t use_x86_sse41_interpreter(interpreter_data* ctx);//THIS FUNCTION IS USER DEFINED +uint64_t use_x86_interpreter(interpreter_data* ctx);//THIS FUNCTION IS USER DEFINED +uint64_t use_x86_lzcnt_interpreter(interpreter_data* ctx);//THIS FUNCTION IS USER DEFINED +template +O x86_add_set_flags_interpreter(interpreter_data* ctx, O n, O m);//THIS FUNCTION IS USER DEFINED +template +O x86_subtract_set_flags_interpreter(interpreter_data* ctx, O n, O m);//THIS FUNCTION IS USER DEFINED uint64_t _get_pc_interpreter(interpreter_data* ctx);//THIS FUNCTION IS USER DEFINED uint64_t translate_address_interpreter(interpreter_data* ctx, uint64_t address);//THIS FUNCTION IS USER DEFINED void call_supervisor_interpreter(interpreter_data* ctx, uint64_t svc);//THIS FUNCTION IS USER DEFINED @@ -376,8 +391,10 @@ ir_operand add_subtract_impl_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand add_subtract_carry_impl_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m, uint64_t set_flags, uint64_t is_add, ir_operand carry); ir_operand condition_holds_jit(ssa_emit_context* ctx, uint64_t cond); void branch_long_universal_jit(ssa_emit_context* ctx, uint64_t Rn, uint64_t link); +uint64_t select_jit(ssa_emit_context* ctx, uint64_t condition, uint64_t yes, uint64_t no); ir_operand create_mask_jit(ssa_emit_context* ctx, uint64_t bits); ir_operand shift_left_check_jit(ssa_emit_context* ctx, ir_operand to_shift, ir_operand shift, uint64_t size); +uint64_t get_x86_rounding_mode_jit(ssa_emit_context* ctx, uint64_t rounding); ir_operand shift_right_check_jit(ssa_emit_context* ctx, ir_operand to_shift, ir_operand shift, uint64_t size, uint64_t is_unsigned); ir_operand reverse_jit(ssa_emit_context* ctx, ir_operand word, uint64_t M, uint64_t N); void convert_to_int_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t ftype, uint64_t Rd, uint64_t Rn, uint64_t round, uint64_t is_unsigned, uint64_t to_vector); @@ -395,7 +412,6 @@ void convert_to_float_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t ftype, ui ir_operand replicate_vector_jit(ssa_emit_context* ctx, ir_operand source, uint64_t v_size, uint64_t count); void st_jit(ssa_emit_context* ctx, uint64_t wback, uint64_t Q, uint64_t L, uint64_t opcode, uint64_t size, uint64_t Rm, uint64_t Rn, uint64_t Rt); void memory_1_jit(ssa_emit_context* ctx, uint64_t wback, uint64_t Q, uint64_t L, uint64_t R, uint64_t Rm, uint64_t o2, uint64_t opcode, uint64_t S, uint64_t size, uint64_t Rn, uint64_t Rt, uint64_t is_load); -void fcm_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz); ir_operand bits_r_jit(ssa_emit_context* ctx, ir_operand operand, uint64_t top, uint64_t bottom); ir_operand infinity_jit(ssa_emit_context* ctx, uint64_t sign, uint64_t N); ir_operand float_is_nan_jit(ssa_emit_context* ctx, ir_operand operand, uint64_t N); @@ -436,6 +452,12 @@ void intrinsic_float_binary_scalar_jit(ssa_emit_context* ctx, uint64_t Rd, uint6 void x86_sse_logic_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t Q, uint64_t invert, uint64_t primary_instruction); ir_operand sse_copy_to_xmm_from_xmm_element_jit(ssa_emit_context* ctx, ir_operand source, uint64_t size, uint64_t index); ir_operand sse_coppy_gp_across_lanes_jit(ssa_emit_context* ctx, ir_operand source, uint64_t size); +void floating_point_multiply_scalar_element_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index); +void floating_point_multiply_vector_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t sz, uint64_t index); +void floating_point_multiply_accumulate_scalar_element_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index); +void floating_point_multiply_accumulate_vector_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t neg, uint64_t sz, uint64_t index); +void fcm_vector_jit(ssa_emit_context* ctx, uint64_t Rd, uint64_t Rn, uint64_t Rm, uint64_t mode, uint64_t Q, uint64_t sz); +ir_operand clear_vector_scalar_jit(ssa_emit_context* ctx, ir_operand working, uint64_t fltsize); void add_subtract_imm12_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t sh, uint64_t imm12, uint64_t Rn, uint64_t Rd); void add_subtract_shifted_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t shift, uint64_t Rm, uint64_t imm6, uint64_t Rn, uint64_t Rd); void add_subtract_extended_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t op, uint64_t S, uint64_t Rm, uint64_t option, uint64_t imm3, uint64_t Rn, uint64_t Rd); @@ -445,6 +467,7 @@ void multiply_with_32_jit(ssa_emit_context* ctx, uint64_t U, uint64_t Rm, uint64 void multiply_hi_jit(ssa_emit_context* ctx, uint64_t U, uint64_t Rm, uint64_t o0, uint64_t Rn, uint64_t Rd); void multiply_additive_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t Rm, uint64_t o0, uint64_t Ra, uint64_t Rn, uint64_t Rd); void divide_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t Rm, uint64_t o1, uint64_t Rn, uint64_t Rd); +uint64_t create_rbit_mask_jit(ssa_emit_context* ctx, uint64_t index); void rbit_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t Rn, uint64_t Rd); void rev16_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t Rn, uint64_t Rd); void reverse_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t opc, uint64_t Rn, uint64_t Rd); @@ -480,6 +503,7 @@ void load_store_register_imm_unsigned_jit(ssa_emit_context* ctx, uint64_t size, void load_store_register_imm_unscaled_jit(ssa_emit_context* ctx, uint64_t size, uint64_t VR, uint64_t opc, uint64_t imm9, uint64_t wb, uint64_t Rn, uint64_t Rt); void load_store_register_offset_jit(ssa_emit_context* ctx, uint64_t size, uint64_t VR, uint64_t opc, uint64_t Rm, uint64_t option, uint64_t S, uint64_t Rn, uint64_t Rt); void load_store_exclusive_ordered_jit(ssa_emit_context* ctx, uint64_t size, uint64_t ordered, uint64_t L, uint64_t Rs, uint64_t o0, uint64_t Rn, uint64_t Rt); +ir_operand exclusive_address_mask_jit(ssa_emit_context* ctx); void load_exclusive_jit(ssa_emit_context* ctx, uint64_t is_exclusive, uint64_t size, uint64_t Rn, uint64_t Rt); void store_exclusive_jit(ssa_emit_context* ctx, uint64_t is_exclusive, uint64_t size, uint64_t Rn, uint64_t Rt, uint64_t Rs); void conversion_between_floating_point_and_fixed_point_jit(ssa_emit_context* ctx, uint64_t sf, uint64_t S, uint64_t ftype, uint64_t rmode, uint64_t opcode, uint64_t scale, uint64_t Rn, uint64_t Rd); @@ -505,6 +529,12 @@ void fmul_vector_by_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, void fmul_accumulate_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd); void fmul_accumulate_element_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t L, uint64_t M, uint64_t Rm, uint64_t neg, uint64_t H, uint64_t Rn, uint64_t Rd); void faddp_scalar_jit(ssa_emit_context* ctx, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmeq_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmgt_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmge_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); +void fcmeq_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); +void fcmgt_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); +void fcmge_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); void fadd_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd); void fsub_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd); void fmul_scalar_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t Rd); @@ -573,12 +603,6 @@ void st2_multiple_structures_no_offset_jit(ssa_emit_context* ctx, uint64_t Q, ui void st2_multiple_structures_post_index_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rm, uint64_t size, uint64_t Rn, uint64_t Rt); void st1_single_structure_no_offset_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t opcode, uint64_t S, uint64_t size, uint64_t Rn, uint64_t Rt); void st1_single_structure_post_index_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t Rm, uint64_t opcode, uint64_t S, uint64_t size, uint64_t Rn, uint64_t Rt); -void fcmeq_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); -void fcmgt_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); -void fcmge_vector_zero_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rn, uint64_t Rd); -void fcmeq_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); -void fcmgt_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); -void fcmge_vector_register_jit(ssa_emit_context* ctx, uint64_t Q, uint64_t sz, uint64_t Rm, uint64_t Rn, uint64_t Rd); void floating_point_conditional_select_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t cond, uint64_t Rn, uint64_t Rd); void fcmp_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t Rn, uint64_t opc); void fccmp_jit(ssa_emit_context* ctx, uint64_t ftype, uint64_t Rm, uint64_t cond, uint64_t Rn, uint64_t nzcv); @@ -598,6 +622,10 @@ uint64_t use_fast_float_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFIN uint64_t use_x86_sse_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFINED uint64_t use_x86_sse2_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFINED uint64_t use_x86_sse41_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFINED +uint64_t use_x86_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFINED +uint64_t use_x86_lzcnt_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFINED +ir_operand x86_add_set_flags_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m);//THIS FUNCTION IS USER DEFINED +ir_operand x86_subtract_set_flags_jit(ssa_emit_context* ctx,uint64_t O, ir_operand n, ir_operand m);//THIS FUNCTION IS USER DEFINED uint64_t _get_pc_jit(ssa_emit_context* ctx);//THIS FUNCTION IS USER DEFINED ir_operand translate_address_jit(ssa_emit_context* ctx, ir_operand address);//THIS FUNCTION IS USER DEFINED void call_supervisor_jit(ssa_emit_context* ctx, uint64_t svc);//THIS FUNCTION IS USER DEFINED diff --git a/src/emulator/atomic.h b/src/emulator/atomic.h index ebcc61d..35fbbbf 100644 --- a/src/emulator/atomic.h +++ b/src/emulator/atomic.h @@ -22,6 +22,7 @@ static T compare_and_swap_impl(uint64_t address_src, T expecting, T to_swap) return false; } +//TODO: account for unalgined pointer static uint64_t compare_and_swap_interpreter_cpp(uint64_t physical_address, uint64_t expecting, uint64_t to_swap, uint64_t size) { global_lock.lock(); diff --git a/src/emulator/guest_process.cpp b/src/emulator/guest_process.cpp index 3461c75..414d35d 100644 --- a/src/emulator/guest_process.cpp +++ b/src/emulator/guest_process.cpp @@ -23,6 +23,7 @@ void guest_process::create(guest_process* result, guest_memory guest_memory_cont result->guest_functions.use_flt = true; result->guest_functions.retranslator_is_running = false; + result->log_native = nullptr; init_aarch64_decoder(result); } @@ -40,7 +41,7 @@ uint64_t guest_process::jit_function(guest_process* process, uint64_t guest_func void* arguments[] = { arm_context }; - if (function_to_execute.times_executed == 100 && !process->debug_mode) + if (function_to_execute.times_executed == 1000 && !process->debug_mode) { switch (function_to_execute.optimizations) { @@ -106,6 +107,10 @@ guest_function guest_process::translate_function(translate_request_data* data, g ssa_emit_context::reset_local(&ssa_emit); + ssa_emit.memory_base = ssa_emit_context::create_global(&ssa_emit, int64); + + ir_operation_block::emitds(raw_ir, ir_move, ssa_emit.memory_base, ir_operand::create_con((uint64_t)process->guest_memory_context.base)); + while (true) { std::unordered_set to_compile_que = aarch64_emit.basic_block_translate_que; @@ -204,7 +209,14 @@ guest_function guest_process::translate_function(translate_request_data* data, g backend_compiler_flags = compiler_flags::optimize_ssa | compiler_flags::mathmatical_fold; } - void* code = jit_context::compile_code(process->host_jit_context, raw_ir,(compiler_flags)backend_compiler_flags); + uint64_t code_size; + + void* code = jit_context::compile_code(process->host_jit_context, raw_ir,(compiler_flags)backend_compiler_flags, &code_size); + + if (((guest_process*)data->process)->log_native != nullptr && flags == guest_compiler_optimization_flags::level_three) + { + ((void(*)(void*, int))((guest_process*)data->process)->log_native)(code, code_size); + } guest_function result; diff --git a/src/emulator/guest_process.h b/src/emulator/guest_process.h index efce36e..886fd61 100644 --- a/src/emulator/guest_process.h +++ b/src/emulator/guest_process.h @@ -25,6 +25,7 @@ struct guest_process void* counter_function; void* undefined_instruction; bool debug_mode; + void* log_native; cpu_type process_type; cpu_size process_size; diff --git a/src/emulator/ssa_emit_context.h b/src/emulator/ssa_emit_context.h index 99f9a9a..f0cb167 100644 --- a/src/emulator/ssa_emit_context.h +++ b/src/emulator/ssa_emit_context.h @@ -9,6 +9,8 @@ struct ssa_emit_context uint64_t local_top; uint64_t global_bottom; + ir_operand memory_base; + void* context_data; static void create(ssa_emit_context* ctx, ir_operation_block* ir); diff --git a/src/export.cpp b/src/export.cpp index 0c153c9..168e51b 100644 --- a/src/export.cpp +++ b/src/export.cpp @@ -22,7 +22,7 @@ static void base_plus_va_jit(void* memory, ssa_emit_context* ir, ir_operand dest { ir_operation_block* ctx = ir->ir; - ir_operation_block::emitds(ctx, ir_add, destination, source, ir_operand::create_con((uint64_t)memory)); + ir_operation_block::emitds(ctx, ir_add, destination, source, ir->memory_base); } extern "C" @@ -31,6 +31,8 @@ extern "C" { external_context* result = new external_context; + result->process.log_native = nullptr; + jit_context::create(&result->memory, 5ULL * 1024 * 1024 * 1024, get_abi()); guest_process::create(&result->process, {memory, base_plus_va, base_plus_va_jit}, &result->memory, *context_offsets); @@ -53,8 +55,10 @@ extern "C" return guest_process::interperate_function(&context->process, virtual_address, guest_context, is_running); } - EXPORT uint64_t jit_until_long_jump(external_context* context, uint64_t virtual_address, void* guest_context, bool* is_running) + EXPORT uint64_t jit_until_long_jump(external_context* context, uint64_t virtual_address, void* guest_context, bool* is_running, void* log_native) { + context->process.log_native = log_native; + while (*is_running) { virtual_address = guest_process::jit_function(&context->process, virtual_address, guest_context); diff --git a/src/ir/ir.h b/src/ir/ir.h index 65e886c..000f28f 100755 --- a/src/ir/ir.h +++ b/src/ir/ir.h @@ -85,7 +85,6 @@ enum ir_instructions : uint64_t //Abi ir_close_and_return, - ir_compare_and_swap, ir_get_argument, ir_external_call, ir_table_jump, @@ -144,6 +143,18 @@ enum ir_instructions : uint64_t x86_subsd, x86_subss, + x86_paddb, + x86_paddw, + x86_paddd, + x86_paddq, + + x86_cmpss, + x86_cmpsd, + x86_cmpps, + x86_cmppd, + x86_haddps, + x86_haddpd, + x86_xorps, x86_pand, x86_orps, @@ -158,6 +169,16 @@ enum ir_instructions : uint64_t x86_sqrtsd, x86_sqrtps, x86_sqrtpd, + x86_popcnt, + x86_cvtsd2ss, + x86_cvtss2sd, + + x86_add_flags, + x86_sub_flags, + + x86_cmpxchg, + + x86_lzcnt, //Emulator Helpers ir_guest_store_context, @@ -239,7 +260,6 @@ static std::string instruction_names[] = { //Abi "ir_close_and_return", - "ir_compare_and_swap", "ir_get_argument", "ir_external_call", "ir_table_jump", @@ -298,6 +318,18 @@ static std::string instruction_names[] = { "x86_subsd", "x86_subss", + "x86_paddb", + "x86_paddw", + "x86_paddd", + "x86_paddq", + + "x86_cmpss", + "x86_cmpsd", + "x86_cmpps", + "x86_cmppd", + "x86_haddps", + "x86_haddpd", + "x86_xorps", "x86_pand", "x86_orps", @@ -312,6 +344,16 @@ static std::string instruction_names[] = { "x86_sqrtsd", "x86_sqrtps", "x86_sqrtpd", + "x86_popcnt", + "x86_cvtsd2ss", + "x86_cvtss2sd", + + "x86_add_flags", + "x86_sub_flags", + + "x86_cmpxchg", + + "x86_lzcnt", //Emulator Helpers "ir_guest_store_context", diff --git a/src/ir/ir_operation_block.cpp b/src/ir/ir_operation_block.cpp index c1630e9..284d389 100755 --- a/src/ir/ir_operation_block.cpp +++ b/src/ir/ir_operation_block.cpp @@ -435,6 +435,9 @@ std::string ir_operation_block::get_block_log(ir_operation_block* ir) { ir_operation working_operation = i->data; + if (working_operation.instruction == ir_no_operation) + continue; + std::string name = instruction_names[working_operation.instruction]; result += name + " "; diff --git a/src/ir/ssa.cpp b/src/ir/ssa.cpp index a8ccd0c..af83a0f 100644 --- a/src/ir/ssa.cpp +++ b/src/ir/ssa.cpp @@ -490,6 +490,13 @@ static uint64_t get_mask(int size) throw_error(); } +static void empty_operation(ir_operation* operation) +{ + operation->sources.count = 0; + operation->destinations.count = 0; + operation->instruction = ir_no_operation; +} + static bool perform_move_propagation(ssa_cf_node* node) { bool done = true; @@ -537,9 +544,7 @@ static bool perform_move_propagation(ssa_cf_node* node) break; } - i->data.instruction = ir_no_operation; - i->data.destinations.count = 0; - i->data.sources.count = 0; + empty_operation(&i->data); done = false; } @@ -547,6 +552,104 @@ static bool perform_move_propagation(ssa_cf_node* node) return done; } +static bool perform_load_store_addition_propagation(ssa_cf_node* node, std::unordered_map>* usage_count_map) +{ + bool done = true; + + ssa_context* ctx = node->context; + + for (auto i = node->raw_node->entry_instruction; i != node->raw_node->final_instruction->next; i = i->next) + { + auto working_instruction = i->data; + + int ins = working_instruction.instruction; + + if (ins != ir_load && ins != ir_store) + continue; + + if ((working_instruction.sources.count == 2 && ins == ir_load) || (working_instruction.sources.count == 3 && ins == ir_store)) + continue; + + ir_operand source = i->data.sources[0]; + + if (ir_operand::is_constant(&source)) + continue; + + if (is_global(ctx, source)) + continue; + + if ((*usage_count_map)[source.value][node] > 2) + { + continue; + } + + ir_operation* to_nop = nullptr; + + for (auto s = i->prev; s != nullptr; s = s->prev) + { + ir_operation* check_instruction = &s->data; + + if (check_instruction->destinations.count != 1) + continue; + + ir_operand destination_check = check_instruction->destinations[0]; + + if (ir_operand::are_equal(destination_check, source)) + { + to_nop = check_instruction; + + break; + } + + if (s == node->raw_node->entry_instruction) + break; + + if (s->data.instruction == ir_mark_label) + { + throw_error(); + } + } + + if (to_nop == nullptr) + { + throw_error(); + } + + if (to_nop->instruction != ir_add) + continue; + + if (ins == ir_load) + { + ir_operation_block::emit_with(ctx->ir, ir_load, working_instruction.destinations.data, 1, to_nop->sources.data, 2, i); + } + else if (ins == ir_store) + { + ir_operand new_sources[3]; + + new_sources[0] = to_nop->sources[0]; + new_sources[1] = to_nop->sources[1]; + new_sources[2] = working_instruction.sources[1]; + + ir_operation_block::emit_with(ctx->ir, ir_store, nullptr, 0, new_sources, 3, i); + } + + empty_operation(&i->data); + empty_operation(to_nop); + + done = false; + } + + return done; +} + +static void swap(ir_operand* l, ir_operand* r) +{ + ir_operand tmp = *l; + + *l = *r; + *r = tmp; +} + static bool perform_global_move_propagation(ssa_cf_node* node, std::unordered_map>* usage_count_map) { bool done = true; @@ -575,17 +678,15 @@ static bool perform_global_move_propagation(ssa_cf_node* node, std::unordered_ma ir_operand* to_replace = nullptr; - int usage_count = 0; - - if ((*usage_count_map)[source.value][node] > 1) - { - continue; - } - for (auto s = i->prev; s != nullptr; s = s->prev) { ir_operation* check_instruction = &s->data; + if (check_if_local_used(s->data.sources.data, s->data.sources.count, source.value)) + { + break; + } + if (check_instruction->destinations.count != 1) continue; @@ -612,8 +713,23 @@ static bool perform_global_move_propagation(ssa_cf_node* node, std::unordered_ma continue; } - if (usage_count != 0) - break; + for (auto s = i->next; s != nullptr; s = s->next) + { + if (check_if_local_used(s->data.sources.data, s->data.sources.count, source.value)) + { + to_replace = nullptr; + + break; + } + + if (s == node->raw_node->final_instruction) + break; + } + + if (to_replace == nullptr) + { + continue; + } to_replace->value = destination.value; @@ -627,6 +743,40 @@ static bool perform_global_move_propagation(ssa_cf_node* node, std::unordered_ma return done; } +static bool perform_dead_code_elimination(ssa_cf_node* node, std::unordered_map>* usage_count_map) +{ + bool done = true; + + ssa_context* ctx = node->context; + + for (auto i = node->raw_node->entry_instruction; i != node->raw_node->final_instruction->next; i = i->next) + { + auto working_instruction = i->data; + + if (working_instruction.instruction == ir_external_call) + continue; + + if (working_instruction.destinations.count != 1) + continue; + + ir_operand destination = working_instruction.destinations[0]; + + if (is_global(ctx,destination)) + continue; + + if ((*usage_count_map)[destination.value][node] >= 1) + continue; + + i->data.instruction = ir_no_operation; + i->data.destinations.count = 0; + i->data.sources.count = 0; + + done = false; + } + + return done; +} + static bool perform_math_propagation(ssa_cf_node* node) { bool done = true; @@ -655,6 +805,28 @@ static bool perform_math_propagation(ssa_cf_node* node) }; break; + case ir_load: + { + if (working_instruction->sources.count == 2) + { + if (ir_operand::is_constant(&sources[0]) && !ir_operand::is_constant(&sources[1])) + { + swap(&sources[0], &sources[1]); + + done = false; + } + else if (ir_operand::is_constant(&sources[0]) && ir_operand::is_constant(&sources[1])) + { + ir_operand new_operand = ir_operand::create_con(sources[0].value + sources[1].value, sources[0].meta_data); + + sources[0] = new_operand; + working_instruction->sources.count = 1; + + done = false; + } + } + }; break; + case ir_add: { if (check_constant(sources[0], 0)) @@ -914,13 +1086,6 @@ void ssa_construct_and_optimize(ir_operation_block* source, compiler_flags flags remap_ssa_sources(&ctx, &ssa_node_store[i]); } - std::unordered_map> total_register_usage_count; - - for (int i = 0; i < count; ++i) - { - append_usage_data_global(nullptr, &total_register_usage_count, &ssa_node_store[i]); - } - //ir_operation_block::log(source); if (flags & compiler_flags::mathmatical_fold) @@ -929,6 +1094,13 @@ void ssa_construct_and_optimize(ir_operation_block* source, compiler_flags flags { bool done = true; + std::unordered_map> total_register_usage_count; + + for (int i = 0; i < count; ++i) + { + append_usage_data_global(nullptr, &total_register_usage_count, &ssa_node_store[i]); + } + for (int i = 0; i < count; ++i) { ssa_cf_node* working_node = &ssa_node_store[i]; @@ -936,6 +1108,8 @@ void ssa_construct_and_optimize(ir_operation_block* source, compiler_flags flags done &= perform_move_propagation(working_node); done &= perform_math_propagation(working_node); done &= perform_global_move_propagation(working_node, &total_register_usage_count); + done &= perform_load_store_addition_propagation(working_node, &total_register_usage_count); + done &= perform_dead_code_elimination(working_node, &total_register_usage_count); } if (done) diff --git a/src/jit/jit_context.cpp b/src/jit/jit_context.cpp index 061f4c1..3786733 100644 --- a/src/jit/jit_context.cpp +++ b/src/jit/jit_context.cpp @@ -67,7 +67,7 @@ void* jit_context::append_to_jit_cache(jit_context* context, void* source_functi return growing_jit_cache::append_code(&context->jit_cache, source_function, function_size); } -void* jit_context::compile_code(jit_context* context, ir_operation_block* ir_operation_block_context, compiler_flags flags) +void* jit_context::compile_code(jit_context* context, ir_operation_block* ir_operation_block_context, compiler_flags flags, uint64_t* code_size_result) { void* code_buffer; uint64_t code_size; @@ -84,5 +84,10 @@ void* jit_context::compile_code(jit_context* context, ir_operation_block* ir_ope default: throw_error(); } + if (code_size_result != nullptr) + { + *code_size_result = code_size; + } + return jit_context::append_to_jit_cache(context, code_buffer, code_size); } \ No newline at end of file diff --git a/src/jit/jit_context.h b/src/jit/jit_context.h index ea1ec45..7175ef4 100644 --- a/src/jit/jit_context.h +++ b/src/jit/jit_context.h @@ -16,7 +16,7 @@ struct jit_context static void create_from_host(jit_context* result, uint64_t allocation_size); static uint64_t call_jitted_function(jit_context* context, void* function, uint64_t* arguments); static void* append_to_jit_cache(jit_context* context, void* source_function, uint64_t function_size); - static void* compile_code(jit_context* context, ir_operation_block* ir_operation_block_context, compiler_flags flags); + static void* compile_code(jit_context* context, ir_operation_block* ir_operation_block_context, compiler_flags flags, uint64_t* code_size_result = nullptr); }; #endif \ No newline at end of file