[A64/Sequences] Fix NEON byte loads

- Cast std::byte buffers to uint8_t* in pow2/log2 emulators

- Update vector shift/rotate helpers to use byte-wise loads
This commit is contained in:
Will Martin
2026-01-21 11:11:54 +09:00
parent 9fe9f2ab2a
commit a83bd019a4
2 changed files with 50 additions and 27 deletions
+19 -13
View File
@@ -499,15 +499,17 @@ static uint8x16_t EmulateVectorShl(void*, std::byte src1[16],
alignas(16) T shamt[16 / sizeof(T)];
// Load NEON registers into a C array.
vst1q_u8(reinterpret_cast<T*>(value), vld1q_u8(src1));
vst1q_u8(reinterpret_cast<T*>(shamt), vld1q_u8(src2));
vst1q_u8(reinterpret_cast<uint8_t*>(value),
vld1q_u8(reinterpret_cast<const uint8_t*>(src1)));
vst1q_u8(reinterpret_cast<uint8_t*>(shamt),
vld1q_u8(reinterpret_cast<const uint8_t*>(src2)));
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
value[i] = value[i] << (shamt[i] & ((sizeof(T) * 8) - 1));
}
// Store result and return it.
return vld1q_u8(value);
return vld1q_u8(reinterpret_cast<const uint8_t*>(value));
}
struct VECTOR_SHL_V128
: Sequence<VECTOR_SHL_V128, I<OPCODE_VECTOR_SHL, V128Op, V128Op, V128Op>> {
@@ -611,16 +613,18 @@ static uint8x16_t EmulateVectorShr(void*, std::byte src1[16],
alignas(16) T value[16 / sizeof(T)];
alignas(16) T shamt[16 / sizeof(T)];
// Load NEON registers into a C array.
vst1q_u8(reinterpret_cast<T*>(value), vld1q_u8(src1));
vst1q_u8(reinterpret_cast<T*>(shamt), vld1q_u8(src2));
// Load NEON registers into a C array by casting to uint8_t*
vst1q_u8(reinterpret_cast<uint8_t*>(value),
vld1q_u8(reinterpret_cast<const uint8_t*>(src1)));
vst1q_u8(reinterpret_cast<uint8_t*>(shamt),
vld1q_u8(reinterpret_cast<const uint8_t*>(src2)));
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1));
}
// Store result and return it.
return vld1q_u8(value);
// Store result and return it by casting to uint8_t*
return vld1q_u8(reinterpret_cast<const uint8_t*>(value));
}
struct VECTOR_SHR_V128
: Sequence<VECTOR_SHR_V128, I<OPCODE_VECTOR_SHR, V128Op, V128Op, V128Op>> {
@@ -820,16 +824,18 @@ static uint8x16_t EmulateVectorRotateLeft(void*, std::byte src1[16],
alignas(16) T value[16 / sizeof(T)];
alignas(16) T shamt[16 / sizeof(T)];
// Load NEON registers into a C array.
vst1q_u8(reinterpret_cast<T*>(value), vld1q_u8(src1));
vst1q_u8(reinterpret_cast<T*>(shamt), vld1q_u8(src2));
// Load NEON registers into a C array by casting to uint8_t*
vst1q_u8(reinterpret_cast<uint8_t*>(value),
vld1q_u8(reinterpret_cast<const uint8_t*>(src1)));
vst1q_u8(reinterpret_cast<uint8_t*>(shamt),
vld1q_u8(reinterpret_cast<const uint8_t*>(src2)));
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
value[i] = xe::rotate_left<T>(value[i], shamt[i] & ((sizeof(T) * 8) - 1));
}
// Store result and return it.
return vld1q_u8(value);
// Store result and return it by casting to uint8_t*
return vld1q_u8(reinterpret_cast<const uint8_t*>(value));
}
struct VECTOR_ROTATE_LEFT_V128
: Sequence<VECTOR_ROTATE_LEFT_V128,
+31 -14
View File
@@ -1995,9 +1995,11 @@ EMITTER_OPCODE_TABLE(OPCODE_RECIP, RECIP_F32, RECIP_F64, RECIP_V128);
struct POW2_F32 : Sequence<POW2_F32, I<OPCODE_POW2, F32Op, F32Op>> {
static float32x4_t EmulatePow2(void*, std::byte src[16]) {
float src_value;
vst1q_lane_f32(&src_value, vld1q_u8(src), 0);
vst1q_lane_f32(&src_value, vld1q_u8(reinterpret_cast<const uint8_t*>(src)),
0);
const float result = std::exp2(src_value);
return vld1q_lane_f32(&result, vld1q_u8(src), 0);
return vld1q_lane_f32(&result,
vld1q_u8(reinterpret_cast<const uint8_t*>(src)), 0);
}
static void Emit(A64Emitter& e, const EmitArgType& i) {
assert_always();
@@ -2009,9 +2011,11 @@ struct POW2_F32 : Sequence<POW2_F32, I<OPCODE_POW2, F32Op, F32Op>> {
struct POW2_F64 : Sequence<POW2_F64, I<OPCODE_POW2, F64Op, F64Op>> {
static float64x2_t EmulatePow2(void*, std::byte src[16]) {
double src_value;
vst1q_lane_f64(&src_value, vld1q_u8(src), 0);
vst1q_lane_f64(&src_value, vld1q_u8(reinterpret_cast<const uint8_t*>(src)),
0);
const double result = std::exp2(src_value);
return vld1q_lane_f64(&result, vld1q_u8(src), 0);
return vld1q_lane_f64(&result,
vld1q_u8(reinterpret_cast<const uint8_t*>(src)), 0);
}
static void Emit(A64Emitter& e, const EmitArgType& i) {
assert_always();
@@ -2023,7 +2027,7 @@ struct POW2_F64 : Sequence<POW2_F64, I<OPCODE_POW2, F64Op, F64Op>> {
struct POW2_V128 : Sequence<POW2_V128, I<OPCODE_POW2, V128Op, V128Op>> {
static float32x4_t EmulatePow2(void*, std::byte src[16]) {
alignas(16) float values[4];
vst1q_f32(values, vld1q_u8(src));
vst1q_f32(values, vld1q_u8(reinterpret_cast<const uint8_t*>(src)));
for (size_t i = 0; i < 4; ++i) {
values[i] = std::exp2(values[i]);
}
@@ -2046,9 +2050,11 @@ EMITTER_OPCODE_TABLE(OPCODE_POW2, POW2_F32, POW2_F64, POW2_V128);
struct LOG2_F32 : Sequence<LOG2_F32, I<OPCODE_LOG2, F32Op, F32Op>> {
static float32x4_t EmulateLog2(void*, std::byte src[16]) {
float src_value;
vst1q_lane_f32(&src_value, vld1q_u8(src), 0);
vst1q_lane_f32(&src_value, vld1q_u8(reinterpret_cast<const uint8_t*>(src)),
0);
float result = std::log2(src_value);
return vld1q_lane_f32(&result, vld1q_u8(src), 0);
return vld1q_lane_f32(&result,
vld1q_u8(reinterpret_cast<const uint8_t*>(src)), 0);
}
static void Emit(A64Emitter& e, const EmitArgType& i) {
assert_always();
@@ -2064,9 +2070,11 @@ struct LOG2_F32 : Sequence<LOG2_F32, I<OPCODE_LOG2, F32Op, F32Op>> {
struct LOG2_F64 : Sequence<LOG2_F64, I<OPCODE_LOG2, F64Op, F64Op>> {
static float64x2_t EmulateLog2(void*, std::byte src[16]) {
double src_value;
vst1q_lane_f64(&src_value, vld1q_u8(src), 0);
vst1q_lane_f64(&src_value, vld1q_u8(reinterpret_cast<const uint8_t*>(src)),
0);
double result = std::log2(src_value);
return vld1q_lane_f64(&result, vld1q_u8(src), 0);
return vld1q_lane_f64(&result,
vld1q_u8(reinterpret_cast<const uint8_t*>(src)), 0);
}
static void Emit(A64Emitter& e, const EmitArgType& i) {
assert_always();
@@ -2082,7 +2090,7 @@ struct LOG2_F64 : Sequence<LOG2_F64, I<OPCODE_LOG2, F64Op, F64Op>> {
struct LOG2_V128 : Sequence<LOG2_V128, I<OPCODE_LOG2, V128Op, V128Op>> {
static float32x4_t EmulateLog2(void*, std::byte src[16]) {
alignas(16) float values[4];
vst1q_f32(values, vld1q_u8(src));
vst1q_f32(values, vld1q_u8(reinterpret_cast<const uint8_t*>(src)));
for (size_t i = 0; i < 4; ++i) {
values[i] = std::log2(values[i]);
}
@@ -2424,14 +2432,18 @@ struct SHL_V128 : Sequence<SHL_V128, I<OPCODE_SHL, V128Op, V128Op, I8Op>> {
// Almost all instances are shamt = 1, but non-constant.
// shamt is [0,7]
uint8_t shamt = src2 & 0x7;
// Load `src1` as a byte vector (uint8x16_t) then work on byte shifting
uint8x16_t byte_vec = vld1q_u8(reinterpret_cast<const uint8_t*>(src1));
alignas(16) vec128_t value;
vst1q_f32(reinterpret_cast<float32x4_t*>(&value), vld1q_u8(src1));
vst1q_u8(reinterpret_cast<uint8_t*>(&value), byte_vec);
for (int i = 0; i < 15; ++i) {
value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] << shamt) |
(value.u8[(i + 1) ^ 0x3] >> (8 - shamt));
}
value.u8[15 ^ 0x3] = value.u8[15 ^ 0x3] << shamt;
return vld1q_f32(reinterpret_cast<float32x4_t*>(&value));
return vreinterpretq_f32_u8(
vld1q_u8(reinterpret_cast<const uint8_t*>(&value)));
}
};
EMITTER_OPCODE_TABLE(OPCODE_SHL, SHL_I8, SHL_I16, SHL_I32, SHL_I64, SHL_V128);
@@ -2503,14 +2515,19 @@ struct SHR_V128 : Sequence<SHR_V128, I<OPCODE_SHR, V128Op, V128Op, I8Op>> {
// Almost all instances are shamt = 1, but non-constant.
// shamt is [0,7]
uint8_t shamt = src2 & 0x7;
// Load `src1` as a byte vector (uint8x16_t) and store it into `value`
uint8x16_t byte_vec = vld1q_u8(reinterpret_cast<const uint8_t*>(src1));
alignas(16) vec128_t value;
vst1q_f32(reinterpret_cast<float32x4_t*>(&value), vld1q_u8(src1));
vst1q_u8(reinterpret_cast<uint8_t*>(&value), byte_vec);
for (int i = 15; i > 0; --i) {
value.u8[i ^ 0x3] = (value.u8[i ^ 0x3] >> shamt) |
(value.u8[(i - 1) ^ 0x3] << (8 - shamt));
}
value.u8[0 ^ 0x3] = value.u8[0 ^ 0x3] >> shamt;
return vld1q_f32(reinterpret_cast<float32x4_t*>(&value));
// Convert to float32x4_t by reinterpreting the processed `value`
return vreinterpretq_f32_u8(
vld1q_u8(reinterpret_cast<const uint8_t*>(&value)));
}
};
EMITTER_OPCODE_TABLE(OPCODE_SHR, SHR_I8, SHR_I16, SHR_I32, SHR_I64, SHR_V128);