From 2df75aa6044310ba4d3aab158940029637ab1636 Mon Sep 17 00:00:00 2001 From: Zulux91 Date: Wed, 12 Aug 2026 05:53:44 -0500 Subject: [PATCH 1/3] RSX: vectorize the primitive-restart index upload on ARM64 The primitive-restart variant of upload_untouched had no SIMD path on ARM64: the asmjit builder is x86-only, and clang cannot auto-vectorize the scalar loop (-Rpass-analysis: "value that could not be identified as reduction is used outside the loop") because the min/max updates are conditional on the restart compare -- while the non-restart loop next to it does auto-vectorize. Net effect: 16 scalar instructions per index on a path some titles saturate. Measured on a Snapdragon 8 Elite (Odin 3), Virtua Tennis 4 routes its entire indexed-draw traffic through this loop: 2.81 billion indices in a 9-minute match session, median 159k indices per frame. Port the x86 lane algebra to NEON, 8x u16 / 4x u32 per iteration: the restart-equal mask ORs the lane to all-ones for the min accumulator and the store (all-ones is index_limit, exactly what the scalar loop writes) and BICs it to zero for the max accumulator, so restart lanes can never win either reduction; UMINV/UMAXV reduce once at the end and the tail stays scalar. Baseline v8.0 AdvSIMD only. Supporting results, all on the Odin 3 with the system driver: - Correctness: 216-case differential (scalar vs NEON vs the dispatched path; every tail residue mod 8 and mod 4; restart index absent, present, 0, index_limit, all-restart, and index_limit present while not the restart value; u16 and u32) ran on device at RSX init in all four A/B runs: 0 mismatches. An independent 65,000-case host-side model of the same lane algebra also matched the scalar loop, and a blind review of the diff could not construct a diverging input. - Performance A/B (cntvct_el0 around the dispatch, null-region calibration subtracted, per-window medians over 120-flip windows with >10k restart indices/flip, runs interleaved scalar/NEON/NEON/scalar): scalar: 0.02504 and 0.02464 ticks/index (1.30 ns/index) NEON: 0.00346 and 0.00381 ticks/index (0.19 ns/index) ~6.8x faster per index; scalar-scalar repeatability 1.6%. Worst single-frame cost in this loop fell from 3.64 ms to 0.99 ms. FPS stayed 60/60 in all runs on this device; the win is RSX-thread occupancy and worst-frame cost, and would be frame time where the RSX thread is the bottleneck. Titles that never enable primitive restart are unaffected: they route through the untouched path, which clang already vectorizes. --- rpcs3/Emu/RSX/Common/BufferUtils.cpp | 100 +++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/rpcs3/Emu/RSX/Common/BufferUtils.cpp b/rpcs3/Emu/RSX/Common/BufferUtils.cpp index 0d7455de9..d0dc33178 100644 --- a/rpcs3/Emu/RSX/Common/BufferUtils.cpp +++ b/rpcs3/Emu/RSX/Common/BufferUtils.cpp @@ -253,6 +253,7 @@ DECLARE(copy_data_swap_u32) = copy_data_swap_u32_naive; DECLARE(copy_data_swap_u32_cmp) = copy_data_swap_u32_naive; #endif + namespace { template @@ -392,6 +393,102 @@ namespace return (u64{max_index} << 32) | u64{min_index}; } +#if defined(ARCH_ARM64) + // Eight u16 (four u32) per iteration instead of one. + // + // The SIMD build of this loop is assembled by asmjit under ARCH_X64 only, so ARM64 + // fell through to the scalar loop above -- and unlike the non-restart loop, clang + // cannot auto-vectorize this one ("value that could not be identified as reduction + // is used outside the loop"): the min/max updates are themselves conditional on the + // restart compare. Measured on a Snapdragon 8 Elite, Virtua Tennis 4 routes its + // entire indexed-draw traffic through here -- a median of ~159k indices per frame + // in a match -- at 16 scalar instructions per index. + // + // Same lane algebra as the x86 asmjit builder below: the restart-equal mask ORs + // the lane to all-ones for the min accumulator and the store (all-ones is + // index_limit, exactly what the scalar loop writes), and BICs it to zero for the + // max accumulator, so restart lanes can never win either reduction. Horizontal + // UMINV/UMAXV once at the end. Tail stays scalar. + // + // CONTRACT: src and dst must not overlap. The vector body reads a full lane + // group before writing it back, so a partial overlap diverges from the scalar + // loop's element-wise order. Both callers (VK/GL vertex upload) hand a guest + // memory span as src and a freshly mapped ring-buffer span as dst. + static inline u64 upload_untouched_neon(const be_t* src, u16* dst, u32 count, u16 restart_index) + { + u32 i = 0; + u16 min_index = index_limit(); + u16 max_index = 0; + + if (count >= 8) + { + const uint16x8_t vrestart = vdupq_n_u16(restart_index); + uint16x8_t vmin = vdupq_n_u16(0xffff); + uint16x8_t vmax = vdupq_n_u16(0); + + for (; i + 8 <= count; i += 8) + { + const uint16x8_t v = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16( + vld1q_u16(reinterpret_cast(src) + i)))); + const uint16x8_t eq = vceqq_u16(v, vrestart); + const uint16x8_t v_or_ones = vorrq_u16(v, eq); + + vmin = vminq_u16(vmin, v_or_ones); + vmax = vmaxq_u16(vmax, vbicq_u16(v, eq)); + vst1q_u16(dst + i, v_or_ones); + } + + min_index = vminvq_u16(vmin); + max_index = vmaxvq_u16(vmax); + } + + for (; i < count; ++i) + { + const u16 index = src[i].value(); + dst[i] = index == restart_index ? index_limit() : min_max(min_index, max_index, index); + } + + return (u64{max_index} << 32) | u64{min_index}; + } + + static inline u64 upload_untouched_neon(const be_t* src, u32* dst, u32 count, u32 restart_index) + { + u32 i = 0; + u32 min_index = index_limit(); + u32 max_index = 0; + + if (count >= 4) + { + const uint32x4_t vrestart = vdupq_n_u32(restart_index); + uint32x4_t vmin = vdupq_n_u32(0xffffffffu); + uint32x4_t vmax = vdupq_n_u32(0); + + for (; i + 4 <= count; i += 4) + { + const uint32x4_t v = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32( + vld1q_u32(reinterpret_cast(src) + i)))); + const uint32x4_t eq = vceqq_u32(v, vrestart); + const uint32x4_t v_or_ones = vorrq_u32(v, eq); + + vmin = vminq_u32(vmin, v_or_ones); + vmax = vmaxq_u32(vmax, vbicq_u32(v, eq)); + vst1q_u32(dst + i, v_or_ones); + } + + min_index = vminvq_u32(vmin); + max_index = vmaxvq_u32(vmax); + } + + for (; i < count; ++i) + { + const u32 index = src[i].value(); + dst[i] = index == restart_index ? index_limit() : min_max(min_index, max_index, index); + } + + return (u64{max_index} << 32) | u64{min_index}; + } +#endif + #ifdef ARCH_X64 template static void build_upload_untouched(asmjit::simd_builder& c, native_args& args) @@ -470,6 +567,8 @@ namespace r = upload_xi16(src.data(), dst.data(), count, restart_index); else r = upload_xi32(src.data(), dst.data(), count, restart_index); +#elif defined(ARCH_ARM64) + r = upload_untouched_neon(src.data(), dst.data(), count, restart_index); #else r = upload_untouched_naive(src.data(), dst.data(), count, restart_index); #endif @@ -884,3 +983,4 @@ std::tuple write_index_array_data_to_buffer(std::span fmt::throw_exception("Unreachable"); } } + From ad15f81d6ea3d74f6b2ce0003d3384644b5cdc8f Mon Sep 17 00:00:00 2001 From: Zulux91 Date: Wed, 12 Aug 2026 07:24:20 -0500 Subject: [PATCH 2/3] RSX: clarify comments on the ARM64 index-upload paths Explain why the non-restart loop stays scalar (clang already auto-vectorizes it) and spell out which allocations each caller passes, so the no-overlap contract of upload_untouched_neon is checkable. --- rpcs3/Emu/RSX/Common/BufferUtils.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rpcs3/Emu/RSX/Common/BufferUtils.cpp b/rpcs3/Emu/RSX/Common/BufferUtils.cpp index d0dc33178..1113db753 100644 --- a/rpcs3/Emu/RSX/Common/BufferUtils.cpp +++ b/rpcs3/Emu/RSX/Common/BufferUtils.cpp @@ -253,7 +253,6 @@ DECLARE(copy_data_swap_u32) = copy_data_swap_u32_naive; DECLARE(copy_data_swap_u32_cmp) = copy_data_swap_u32_naive; #endif - namespace { template @@ -366,6 +365,9 @@ namespace else r = upload_xi32(src.data(), dst.data(), count); #else + // No hand-written ARM64 path here on purpose: this loop has no conditional + // min/max, so clang auto-vectorizes it (unlike the restart variant below, + // which needs the explicit NEON port). r = upload_untouched_naive(src.data(), dst.data(), count); #endif @@ -412,8 +414,10 @@ namespace // // CONTRACT: src and dst must not overlap. The vector body reads a full lane // group before writing it back, so a partial overlap diverges from the scalar - // loop's element-wise order. Both callers (VK/GL vertex upload) hand a guest - // memory span as src and a freshly mapped ring-buffer span as dst. + // loop's element-wise order. Both callers (VK/GL vertex upload) pass disjoint + // allocations: src is guest memory (or the immediate-mode push buffer), dst a + // freshly mapped ring-buffer span (or, when the driver quirk forces restart + // emulation, a fresh heap staging block). static inline u64 upload_untouched_neon(const be_t* src, u16* dst, u32 count, u16 restart_index) { u32 i = 0; @@ -983,4 +987,3 @@ std::tuple write_index_array_data_to_buffer(std::span fmt::throw_exception("Unreachable"); } } - From fcdd7cd6af76cc167cdb6b93dac5d1f798d628d7 Mon Sep 17 00:00:00 2001 From: Zulux91 Date: Wed, 12 Aug 2026 07:49:32 -0500 Subject: [PATCH 3/3] RSX: static_assert the all-ones identity the NEON restart lanes rely on The vector body stores vorrq(v, eq) into restart lanes, which is all-ones regardless of what index_limit() returns -- it only matches the scalar tail's index_limit store because index_limit is all bits set. Assert that beside the splats so a change to index_limit fails the ARM64 build instead of silently diverging the vector body from its own tail. No codegen change (emitted assembly is identical). --- rpcs3/Emu/RSX/Common/BufferUtils.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rpcs3/Emu/RSX/Common/BufferUtils.cpp b/rpcs3/Emu/RSX/Common/BufferUtils.cpp index 1113db753..e72447d00 100644 --- a/rpcs3/Emu/RSX/Common/BufferUtils.cpp +++ b/rpcs3/Emu/RSX/Common/BufferUtils.cpp @@ -426,6 +426,10 @@ namespace if (count >= 8) { + // ORR with the compare mask writes all-ones into restart lanes; that only + // matches the tail's index_limit store while index_limit is all bits set. + static_assert(index_limit() == 0xffff); + const uint16x8_t vrestart = vdupq_n_u16(restart_index); uint16x8_t vmin = vdupq_n_u16(0xffff); uint16x8_t vmax = vdupq_n_u16(0); @@ -463,6 +467,9 @@ namespace if (count >= 4) { + // Same all-ones requirement as the u16 overload above. + static_assert(index_limit() == 0xffffffffu); + const uint32x4_t vrestart = vdupq_n_u32(restart_index); uint32x4_t vmin = vdupq_n_u32(0xffffffffu); uint32x4_t vmax = vdupq_n_u32(0);