GV-6: fuse FindMinMax into vertex kick emission

GSVertexTraceFMM::FindMinMax re-walks the draw's index list at flush
(strip vertices up to 3x redundant) with a non-pipelined FDIV per vertex
pair — 6.6% of the GS thread on the MQ65 UYA profile. Accumulate the
min/max at index-emission time instead, where the vertex is
register/L1-hot, and consume the accumulator in GSVertexTrace::Update.

- Per-buffer FmmAcc (position/texture/color pairs) in GSVertexBuff,
  maintained by VertexKickDirect for triangle strips/lists. A watermark
  dedups already-folded vertices (past strip warmup only the register-
  resident new vertex accumulates); rewind/compaction sites clamp it so
  rewritten positions re-accumulate. Fan emissions poison the draw's
  fused state — the fan head doesn't fit the watermark model and
  FlushPrim can rebuild fan indices (caught by GS_VERTEX_CROSSCHECK on
  the UYA dump corpus).
- FmmFinish reproduces the legacy tail bit-exactly or declines: STQ
  requires one constant, normal, nonzero Q (min(s/q) == min(s)/q by
  monotone IEEE division; negative Q swaps; FLT_MAX sentinels folded at
  quotient level) and no inf/NaN S/T — legacy masks NaN quotients per
  lane and reports vt.nan, which a min/max summary can't reproduce.
  Declined draws run the legacy FindMinMax unchanged.
- aarch64-only: NaN detection relies on FMIN/FMAX propagation (sticky in
  the raw accumulator); SSE min/max can drop a NaN again. x86 keeps the
  legacy walk everywhere.
- TME/FST/IIP are stable across one draw's emissions (TestDrawChanged
  flushes or buffer-switches on any draw-affecting PRIM change), so
  kick-time PRIM flags select the accumulation policy.

Gates: gs_vertex_tests +4 property sweeps (600k draws vs a transcription
of the legacy walk: benign must fuse and match bit-exactly, special
Q/ST must decline or match); GS_VERTEX_CROSSCHECK replay of all 10 local
dumps clean; sw+vk frame hashes bit-identical to pre-campaign baselines;
recompiler_tests 1359/1359.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Brian Degenhardt
2026-07-19 09:49:24 -07:00
co-authored by Claude
parent f346103ef7
commit c16b88cb76
5 changed files with 678 additions and 1 deletions
+74
View File
@@ -317,6 +317,9 @@ void GSState::ResetDrawBufferIdx()
{
memcpy(m_vertex_buffers[entry_ptr].xy, m_vertex_buffers[i].xy, sizeof(m_vertex_buffers[i].xy));
memcpy(m_vertex_buffers[entry_ptr].kick_ring, m_vertex_buffers[i].kick_ring, sizeof(m_vertex_buffers[i].kick_ring));
m_vertex_buffers[entry_ptr].fmm_acc = m_vertex_buffers[i].fmm_acc;
m_vertex_buffers[entry_ptr].fmm_watermark = m_vertex_buffers[i].fmm_watermark;
m_vertex_buffers[entry_ptr].fmm_valid = m_vertex_buffers[i].fmm_valid;
m_vertex_buffers[entry_ptr].xyhead = m_vertex_buffers[i].xyhead;
m_vertex_buffers[entry_ptr].xy_tail = m_vertex_buffers[i].xy_tail;
}
@@ -340,6 +343,8 @@ void GSState::ResetDrawBufferIdx()
memset(&m_env_buffers[i], 0, sizeof(GSDrawBufferEnv));
m_vertex_buffers[i].head = m_vertex_buffers[i].tail = m_vertex_buffers[i].next = 0;
m_vertex_buffers[i].xy_tail = 0;
m_vertex_buffers[i].fmm_watermark = 0;
m_vertex_buffers[i].fmm_valid = false;
}
}
@@ -500,6 +505,9 @@ void GSState::PushBuffer()
else
m_vertex->xy_tail = 0;
m_vertex->fmm_watermark = 0;
m_vertex->fmm_valid = false;
m_current_buffer_idx = m_used_buffers_idx;
temp_draw_rect = GSVector4i::zero();
m_dirty_gs_regs = 0;
@@ -1627,6 +1635,12 @@ __forceinline void GSState::ApplyPRIM(u32 prim)
m_vertex->next = 0;
m_vertex->head = m_vertex->tail = m_vertex->next; // remove unused vertices from the end of the vertex buffer
#ifdef ARCH_ARM64
// Tail rewind: subsequent kicks rewrite positions from next upward, which must
// re-accumulate into the fused-FMM state if referenced by an emitted prim.
m_vertex->fmm_watermark = std::min(m_vertex->fmm_watermark, m_vertex->next);
#endif
}
void GSState::GIFRegHandlerPRIM(const GIFReg* RESTRICT r)
@@ -6019,6 +6033,11 @@ __forceinline void GSState::VertexKickDirect(u32 skip, u32 xraw, u32 yraw, const
if constexpr (prim == GS_INVALID)
{
c.tail = c.head;
#ifdef ARCH_ARM64
// Tail rewind: positions at/above the new tail may be rewritten and must
// re-accumulate into the fused-FMM state if referenced again.
c.vb->fmm_watermark = std::min(c.vb->fmm_watermark, c.head);
#endif
return;
}
@@ -6212,6 +6231,11 @@ __forceinline void GSState::VertexKickDirect(u32 skip, u32 xraw, u32 yraw, const
c.vbuff[next + 1] = c.vbuff[head + 1];
head = next;
c.tail = next + 2;
#ifdef ARCH_ARM64
// Line class doesn't accumulate fused-FMM state today, but keep the
// watermark invariant uniform: moved vertices must re-accumulate.
c.vb->fmm_watermark = std::min(c.vb->fmm_watermark, next);
#endif
}
buff[0] = static_cast<u16>(head + 0);
buff[1] = static_cast<u16>(head + 1);
@@ -6235,6 +6259,10 @@ __forceinline void GSState::VertexKickDirect(u32 skip, u32 xraw, u32 yraw, const
c.vbuff[next + 2] = c.vbuff[head + 2];
head = next;
c.tail = next + 3;
#ifdef ARCH_ARM64
// Vertices moved below the fused-FMM watermark must re-accumulate.
c.vb->fmm_watermark = std::min(c.vb->fmm_watermark, next);
#endif
}
buff[0] = static_cast<u16>(head + 0);
buff[1] = static_cast<u16>(head + 1);
@@ -6267,6 +6295,52 @@ __forceinline void GSState::VertexKickDirect(u32 skip, u32 xraw, u32 yraw, const
ASSUME(0);
}
#ifdef ARCH_ARM64
// Fused vertex-trace bounds (see GSVertexKick.h): fold this prim's
// newly-referenced vertices into the per-buffer FindMinMax accumulator so
// FlushPrim doesn't re-walk the index list. The current vertex (v0/v1) is
// register-resident and, past strip warmup, the only one below the
// watermark; older references only occur right after a seam or compaction
// and load from the (L1-hot) buffer — which also picks up any bytes the
// texel-rounding pass mutated in carried-over vertices. TME/FST/IIP are
// stable across a draw's emissions: any draw-affecting PRIM change flushes
// or buffer-switches first (TestDrawChanged).
if constexpr (prim == GS_TRIANGLEFAN)
{
// Fans are triangle-class but reference {head, tail-2, tail-1} — the fan
// head doesn't fit the watermark model, and FlushPrim may rebuild fan
// indices entirely. Any fan emission sends the draw to the legacy walk.
c.vb->fmm_valid = false;
}
else if constexpr (primclass == GS_TRIANGLE_CLASS)
{
const u32 last = c.tail - 1; // last emitted index == the prim's provoking vertex
const bool tme = PRIM->TME != 0;
const bool fst = PRIM->FST != 0;
const bool iip = PRIM->IIP != 0;
GSVertexBuff* RESTRICT vb = c.vb;
if (c.itail == n)
{
GSVertexKernels::FmmAccReset(vb->fmm_acc, tme, fst);
vb->fmm_valid = true;
vb->fmm_watermark = last - 2;
}
if (vb->fmm_valid)
{
for (u32 j = std::max(vb->fmm_watermark, last - 2); j < last; j++)
{
const GSVector4i jm0(c.vbuff[j].m[0]);
const GSVector4i jm1(c.vbuff[j].m[1]);
GSVertexKernels::FmmAccumVertex(vb->fmm_acc, jm0, jm1, tme, fst, iip);
}
GSVertexKernels::FmmAccumVertex(vb->fmm_acc, v0, v1, tme, fst, true);
vb->fmm_watermark = last + 1;
}
}
#endif
// Update rectangle for the current draw (accumulated in the cursor, folded
// into temp_draw_rect with one scissor clamp at every seam). Needs exclusive
// endpoints.
+13
View File
@@ -19,6 +19,10 @@ class GSDumpBase;
class GSState : public GSAlignedClass<32>
{
// GSVertexTrace::Update consumes the per-buffer fused FindMinMax accumulator
// (m_vertex->fmm_*) directly.
friend class GSVertexTrace;
public:
GSState();
virtual ~GSState();
@@ -148,6 +152,15 @@ protected:
// Scalar mirror of xy[] for the outcode cull fast path: written wherever
// xy[] is written, outcodes re-derived on scissor change (RefreshKickMirror).
GSVertexKernels::CullMirrorEntry kick_ring[4];
// Fused vertex-trace bounds (aarch64 only): FindMinMax min/max accumulated
// at index emission over this buffer's referenced vertices. fmm_watermark is
// the first vertex position not yet folded in (clamped on rewinds/compaction
// so re-referenced positions re-accumulate); fmm_valid means the accumulator
// covers every emitted index of the pending draw. Reset lazily at the first
// emission of a draw (itail == n).
GSVertexKernels::FmmAcc fmm_acc;
u32 fmm_watermark;
bool fmm_valid;
};
GSVertexBuff m_vertex_buffers[MAX_DRAW_BUFFERS];
+200
View File
@@ -6,6 +6,7 @@
#include "GS/GSRegs.h"
#include "GS/GSVector.h"
#include <bit>
#include <cfloat>
// Pure kernels backing the fused GIF packed vertex handlers and the per-prim
@@ -336,4 +337,203 @@ namespace GSVertexKernels
return 0;
}
// ------------------------------------------------------------------------
// Fused vertex-trace bounds: accumulate GSVertexTraceFMM::FindMinMax's
// min/max at index-emission time over each newly-referenced vertex (the data
// is register/L1-hot in the kick), so the flush doesn't re-walk the index
// list (strip vertices up to 3x redundant) with a non-pipelined FDIV per
// vertex pair. The raw material is accumulated env-blind; the finish step
// reproduces the legacy tail bit-exactly or declines (caller then runs the
// legacy FindMinMax).
//
// Exactness notes (pinned by gs_vertex_tests + GS_VERTEX_CROSSCHECK):
// - min/max are idempotent and assoc/comm, so accumulating the referenced
// vertex SET (dedup'd by the caller's watermark) equals the legacy walk
// over the index list — provided no NaN is involved (see below).
// - STQ (!FST) folds min(s_i/q) into min(s_i)/q, exact by weak monotonicity
// of IEEE division in the numerator when q is one constant, normal,
// nonzero value (negative q swaps min/max; the minimum is attained, so
// weak monotonicity gives equality). The FLT_MAX sentinels the legacy
// min/max chains start from are folded at the quotient level, matching
// legacy when a quotient overflows to +/-inf. Inf/NaN S or T inputs (and
// any non-normal or non-constant q) make the fold decline: legacy masks
// NaN quotients per lane and reports them in vt.nan, which a min/max
// summary can't reproduce.
// - NaN DETECTION relies on AArch64 FMIN/FMAX propagating NaN into the raw
// accumulator (sticky). SSE MINPS keeps the second operand, which can
// drop a NaN again — so the fused path is aarch64-only; x86 keeps the
// legacy FindMinMax everywhere.
// ------------------------------------------------------------------------
struct FmmAcc
{
GSVector4i pmin, pmax; // u32 min/max of {x, y, z, fog-word} (the legacy p vectors)
GSVector4i tmin, tmax; // FST: u16 min/max of raw m[1] (elements 4/5 = U/V).
// !FST: float min/max of raw m[0] {S, T, rgba-bits, Q}.
GSVector4i cmin, cmax; // u8 min/max of m[0] (bytes 8-11 = RGBA); flat shading
// accumulates provoking vertices only.
};
// {x, y, z, fog-word} exactly as the legacy kernel builds its p vectors.
__forceinline_odr GSVector4i FmmPos(const GSVector4i& m1)
{
return m1.upl16().blend32<0xc>(m1.ywyw());
}
__forceinline_odr void FmmAccReset(FmmAcc& a, bool tme, bool fst)
{
a.pmin = GSVector4i::xffffffff();
a.pmax = GSVector4i::zero();
if (tme && !fst)
{
a.tmin = GSVector4i::cast(GSVector4(FLT_MAX));
a.tmax = GSVector4i::cast(GSVector4(-FLT_MAX));
}
else
{
a.tmin = GSVector4i::xffffffff();
a.tmax = GSVector4i::zero();
}
a.cmin = GSVector4i::xffffffff();
a.cmax = GSVector4i::zero();
}
// accumulate_color = iip || provoking, evaluated by the caller (flat shading
// only takes the provoking vertex's color; the provoking vertex is always the
// last-emitted index of the prim).
__forceinline_odr void FmmAccumVertex(FmmAcc& a, const GSVector4i& m0, const GSVector4i& m1,
bool tme, bool fst, bool accumulate_color)
{
const GSVector4i p = FmmPos(m1);
a.pmin = a.pmin.min_u32(p);
a.pmax = a.pmax.max_u32(p);
if (tme)
{
if (fst)
{
a.tmin = a.tmin.min_u16(m1);
a.tmax = a.tmax.max_u16(m1);
}
else
{
a.tmin = GSVector4i::cast(GSVector4::cast(a.tmin).min(GSVector4::cast(m0)));
a.tmax = GSVector4i::cast(GSVector4::cast(a.tmax).max(GSVector4::cast(m0)));
}
}
if (accumulate_color)
{
a.cmin = a.cmin.min_u8(m0);
a.cmax = a.cmax.max_u8(m0);
}
}
struct FmmResult
{
GSVector4 min_p, max_p, min_t, max_t;
GSVector4i min_c, max_c;
u32 nan_value; // only meaningful when write_nan
bool write_nan; // legacy leaves vt.nan untouched for TME && FST draws
};
// Reproduce the legacy FindMinMax tail from the accumulators. Returns false
// when that can't be done bit-exactly — the caller must run the legacy
// FindMinMax instead. tw/th are the draw context's TEX0.TW/TH.
__forceinline_odr bool FmmFinish(const FmmAcc& a, bool tme, bool fst, bool color,
const GIFRegXYOFFSET& ofs, u32 tw, u32 th, FmmResult& out)
{
out.write_nan = !(tme && fst);
out.nan_value = 0;
if (tme && !fst)
{
// One constant, normal, nonzero Q across the draw; no inf/NaN S or T
// (checked on the accumulator extremes: +/-inf and NaN always reach
// them — max surfaces +inf, min surfaces -inf, NaN is sticky).
const u32 qmin_bits = static_cast<u32>(a.tmin.U32[3]);
const u32 qmax_bits = static_cast<u32>(a.tmax.U32[3]);
if (qmin_bits != qmax_bits)
return false;
const u32 qexp = (qmin_bits >> 23) & 0xFF;
if (qexp == 0 || qexp == 0xFF)
return false;
const u32 st_lanes[4] = {static_cast<u32>(a.tmin.U32[0]), static_cast<u32>(a.tmin.U32[1]),
static_cast<u32>(a.tmax.U32[0]), static_cast<u32>(a.tmax.U32[1])};
for (const u32 bits : st_lanes)
{
if (((bits >> 23) & 0xFF) == 0xFF)
return false;
}
}
const GSVector4 o(ofs);
const GSVector4 s(1.0f / 16, 1.0f / 16, 2.0f, 1.0f);
out.min_p = (GSVector4(a.pmin) - o) * s;
out.max_p = (GSVector4(a.pmax) - o) * s;
// Fix signed int conversion of the Z lane, as the legacy tail does.
out.min_p = out.min_p.insert32<0, 2>(GSVector4::load(static_cast<float>(static_cast<u32>(a.pmin.extract32<2>()))));
out.max_p = out.max_p.insert32<0, 2>(GSVector4::load(static_cast<float>(static_cast<u32>(a.pmax.extract32<2>()))));
if (tme)
{
if (fst)
{
// Legacy converts each vertex's {U, V} u16s to float and min/maxes
// against FLT_MAX sentinels; u16 -> float is monotone and exact and
// the sentinels never survive, so min-in-u16-then-convert matches.
const GSVector4i uvmin(a.tmin.U16[4], a.tmin.U16[5], a.tmin.U16[4], a.tmin.U16[5]);
const GSVector4i uvmax(a.tmax.U16[4], a.tmax.U16[5], a.tmax.U16[4], a.tmax.U16[5]);
const GSVector4 sc = GSVector4(1.0f / 16, 1.0f).xxyy();
out.min_t = GSVector4(uvmin) * sc;
out.max_t = GSVector4(uvmax) * sc;
}
else
{
const u32 q_bits = static_cast<u32>(a.tmin.U32[3]);
const GSVector4 fmin_v = GSVector4::cast(a.tmin);
const GSVector4 fmax_v = GSVector4::cast(a.tmax);
const GSVector4 qv(std::bit_cast<float>(q_bits));
const GSVector4 lo = fmin_v / qv;
const GSVector4 hi = fmax_v / qv;
const bool qneg = (q_bits >> 31) != 0;
const GSVector4 amin = qneg ? hi : lo;
const GSVector4 amax = qneg ? lo : hi;
// Rebuild the legacy lane shape {S/q, T/q, q, q} and fold the
// FLT_MAX sentinels the legacy min/max chains start from (a
// quotient that overflows to +/-inf is clamped by them).
const GSVector4 tl = amin.xyww(qv).min(GSVector4(FLT_MAX));
const GSVector4 tu = amax.xyww(qv).max(GSVector4(-FLT_MAX));
const GSVector4 sc = GSVector4(1 << static_cast<int>(tw), 1 << static_cast<int>(th), 1, 1);
out.min_t = tl * sc;
out.max_t = tu * sc;
}
}
else
{
out.min_t = GSVector4::zero();
out.max_t = GSVector4::zero();
}
if (color)
{
out.min_c = a.cmin.zzzz().u8to32();
out.max_c = a.cmax.zzzz().u8to32();
}
else
{
out.min_c = GSVector4i::zero();
out.max_c = GSVector4i::zero();
}
return true;
}
} // namespace GSVertexKernels
+46 -1
View File
@@ -25,7 +25,52 @@ void GSVertexTrace::Update(const void* vertex, const u16* index, int v_count, in
const u32 fst = m_state->PRIM->FST;
const u32 color = !(m_state->PRIM->TME && m_state->m_context->TEX0.TFX == TFX_DECAL && m_state->m_context->TEX0.TCC);
m_fmm[color][fst][tme][iip][primclass](*this, vertex, index, i_count);
bool fused = false;
#ifdef ARCH_ARM64
// Fused vertex-trace bounds (see GSVertexKick.h): triangle-class draws
// accumulate their min/max at index-emission time; consume the accumulator
// instead of re-walking the index list when the finish step can reproduce
// the legacy tail bit-exactly (it declines on STQ hazards).
if (primclass == GS_TRIANGLE_CLASS && m_state->m_vertex->fmm_valid)
{
GSVertexKernels::FmmResult r;
if (GSVertexKernels::FmmFinish(m_state->m_vertex->fmm_acc, tme != 0, fst != 0, color != 0,
m_state->m_context->XYOFFSET, m_state->m_context->TEX0.TW, m_state->m_context->TEX0.TH, r))
{
m_min.p = r.min_p;
m_max.p = r.max_p;
m_min.t = r.min_t;
m_max.t = r.max_t;
m_min.c = r.min_c;
m_max.c = r.max_c;
if (r.write_nan)
nan.value = r.nan_value;
fused = true;
}
}
#endif
if (!fused)
{
m_fmm[color][fst][tme][iip][primclass](*this, vertex, index, i_count);
}
#if defined(ARCH_ARM64) && defined(GS_VERTEX_CROSSCHECK)
else
{
const GSVector4 xp_min = m_min.p, xp_max = m_max.p, xt_min = m_min.t, xt_max = m_max.t;
const GSVector4i xc_min = m_min.c, xc_max = m_max.c;
const u32 x_nan = nan.value;
m_fmm[color][fst][tme][iip][primclass](*this, vertex, index, i_count);
pxAssertRel(GSVector4i::cast(xp_min).eq(GSVector4i::cast(m_min.p)) &&
GSVector4i::cast(xp_max).eq(GSVector4i::cast(m_max.p)) &&
GSVector4i::cast(xt_min).eq(GSVector4i::cast(m_min.t)) &&
GSVector4i::cast(xt_max).eq(GSVector4i::cast(m_max.t)) &&
xc_min.eq(m_min.c) && xc_max.eq(m_max.c) && x_nan == nan.value,
"GS_VERTEX_CROSSCHECK: fused FindMinMax divergence");
}
#endif
// Potential float overflow detected. Better uses the slower division instead
// Note: If Q is too big, 1/Q will end up as 0. 1e30 is a random number
+345
View File
@@ -458,3 +458,348 @@ TEST(GsVertexCull, PointSweep)
{
RunCullSweep<1, GS_POINT_CLASS>(0x67763304, 500000);
}
#ifdef ARCH_ARM64
// ---------------------------------------------------------------------------
// Fused vertex-trace bounds (FmmAcc/FmmAccumVertex/FmmFinish vs the legacy
// FindMinMax walk). The oracle here is a faithful transcription of
// GSVertexTraceFMM::FindMinMax<GS_TRIANGLE_CLASS> (GSVertexTraceFMM.cpp) over
// {m0, m1} vector pairs — the property under test is that the fused
// reformulation reproduces the legacy walk bit-exactly whenever FmmFinish
// accepts, and that it accepts on the benign (real-game) configurations.
// End-to-end integration is separately pinned by GS_VERTEX_CROSSCHECK replay.
// aarch64-only: the fused path's NaN detection relies on FMIN/FMAX NaN
// propagation, which SSE min/max does not provide.
// ---------------------------------------------------------------------------
namespace
{
struct FmmRefOut
{
GSVector4 min_p, max_p, min_t, max_t;
GSVector4i min_c, max_c;
u32 nan_value;
bool wrote_nan;
};
// Transcription of FindMinMax<GS_TRIANGLE_CLASS, iip, tme, fst, color> plus
// its GetFMM argument fixups (real_fst = tme ? fst : false; real_iip = iip).
void RefFindMinMaxTriangle(const GSVector4i* m0s, const GSVector4i* m1s, const u16* index, int count,
bool iip, bool tme, bool fst_in, bool color, const GIFRegXYOFFSET& ofs, u32 tw, u32 th, FmmRefOut& out)
{
const bool fst = tme ? fst_in : false;
const GSVector4 s_minmax = GSVector4(FLT_MAX, -FLT_MAX, 0.f, 0.f);
GSVector4 tmin = s_minmax.xxxx();
GSVector4 tmax = s_minmax.yyyy();
GSVector4i tnan = GSVector4i::zero();
GSVector4i cmin = GSVector4i::xffffffff();
GSVector4i cmax = GSVector4i::zero();
GSVector4i pmin = GSVector4i::xffffffff();
GSVector4i pmax = GSVector4i::zero();
const auto processVertices = [&](int i0, int i1, bool finalVertex)
{
if (color)
{
const GSVector4i c0 = GSVector4i::load(static_cast<int>(static_cast<u32>(m0s[i0].U32[2])));
const GSVector4i c1 = GSVector4i::load(static_cast<int>(static_cast<u32>(m0s[i1].U32[2])));
if (iip || finalVertex)
{
cmin = cmin.min_u8(c0.min_u8(c1));
cmax = cmax.max_u8(c0.max_u8(c1));
}
// (n == 2 branch: line class only, not transcribed)
}
if (tme)
{
if (!fst)
{
GSVector4 stq0 = GSVector4::cast(m0s[i0]);
GSVector4 stq1 = GSVector4::cast(m0s[i1]);
const GSVector4 q = stq0.wwww(stq1);
const GSVector4 st = stq0.xyxy(stq1) / q;
stq0 = st.xyww(stq0);
stq1 = st.zwww(stq1);
const GSVector4i nan0 = GSVector4i::cast(stq0 != stq0);
const GSVector4i nan1 = GSVector4i::cast(stq1 != stq1);
tmin = tmin.blend32(tmin.min(stq0), GSVector4::cast(~nan0));
tmin = tmin.blend32(tmin.min(stq1), GSVector4::cast(~nan1));
tmax = tmax.blend32(tmax.max(stq0), GSVector4::cast(~nan0));
tmax = tmax.blend32(tmax.max(stq1), GSVector4::cast(~nan1));
tnan |= nan0 | nan1;
}
else
{
const GSVector4 st0 = GSVector4(m1s[i0].uph16()).xyxy();
const GSVector4 st1 = GSVector4(m1s[i1].uph16()).xyxy();
tmin = tmin.min(st0.min(st1));
tmax = tmax.max(st0.max(st1));
}
}
const GSVector4i xyzf0 = m1s[i0];
const GSVector4i xyzf1 = m1s[i1];
const GSVector4i xy0 = xyzf0.upl16();
const GSVector4i zf0 = xyzf0.ywyw();
const GSVector4i xy1 = xyzf1.upl16();
const GSVector4i zf1 = xyzf1.ywyw();
const GSVector4i p0 = xy0.blend32<0xc>(zf0);
const GSVector4i p1 = xy1.blend32<0xc>(zf1);
pmin = pmin.min_u32(p0.min_u32(p1));
pmax = pmax.max_u32(p0.max_u32(p1));
};
int i = 0;
if (iip)
{
for (; i < (count - 1); i += 2)
processVertices(index[i + 0], index[i + 1], true);
if (count & 1)
processVertices(index[i], index[i], true);
}
else
{
for (; i < (count - 3); i += 6)
{
processVertices(index[i + 0], index[i + 3], false);
processVertices(index[i + 1], index[i + 4], false);
processVertices(index[i + 2], index[i + 5], true);
}
if (count & 1)
{
processVertices(index[i + 0], index[i + 1], false);
processVertices(index[i + 2], index[i + 2], true);
}
}
const GSVector4 o(ofs);
const GSVector4 s(1.0f / 16, 1.0f / 16, 2.0f, 1.0f);
out.min_p = (GSVector4(pmin) - o) * s;
out.max_p = (GSVector4(pmax) - o) * s;
out.min_p = out.min_p.insert32<0, 2>(GSVector4::load(static_cast<float>(static_cast<u32>(pmin.extract32<2>()))));
out.max_p = out.max_p.insert32<0, 2>(GSVector4::load(static_cast<float>(static_cast<u32>(pmax.extract32<2>()))));
out.wrote_nan = true;
out.nan_value = 0;
if (tme)
{
GSVector4 sc;
if (fst)
{
sc = GSVector4(1.0f / 16, 1.0f).xxyy();
out.wrote_nan = false;
}
else
{
sc = GSVector4(1 << static_cast<int>(tw), 1 << static_cast<int>(th), 1, 1);
out.nan_value = static_cast<u32>(tnan.mask()) & ~4u;
}
out.min_t = tmin * sc;
out.max_t = tmax * sc;
}
else
{
out.min_t = GSVector4::zero();
out.max_t = GSVector4::zero();
}
if (color)
{
out.min_c = cmin.u8to32();
out.max_c = cmax.u8to32();
}
else
{
out.min_c = GSVector4i::zero();
out.max_c = GSVector4i::zero();
}
}
enum class FmmQMode
{
ConstantNormal, // one random normal q for the whole draw (either sign)
ConstantSpecial, // one q from {+-0, denormal, inf, nan} — fused must decline
Varying, // random normal q per vertex — fused must decline
};
u32 RandomNormalFloatBits(std::mt19937& rng)
{
const u32 exp = 1 + (rng() % 254);
return (rng() & 0x807FFFFFu) | (exp << 23);
}
u32 RandomSpecialFloatBits(std::mt19937& rng)
{
switch (rng() % 5)
{
case 0: return 0x00000000u; // +0
case 1: return 0x80000000u; // -0
case 2: return (rng() & 0x807FFFFFu); // denormal (or +-0)
case 3: return 0x7F800000u | (rng() & 0x80000000u); // +-inf
default: return 0x7FC00000u | (rng() & 0x8007FFFFu); // NaN
}
}
// One randomized draw: build vertices, emit a subset of prims (mirroring the
// GSState watermark/provoking accumulation), and compare FmmFinish against
// the legacy-walk oracle whenever it accepts.
// Returns true if the fused path accepted the draw.
bool RunFmmCase(std::mt19937& rng, bool iip, bool tme, bool fst, bool color, FmmQMode qmode, bool special_st,
bool strip_shaped)
{
const int prim_count = 1 + (rng() % 8);
const int vertex_count = strip_shaped ? (prim_count + 2) : (prim_count * 3);
GSVector4i m0s[26];
GSVector4i m1s[26];
const u32 const_q = (qmode == FmmQMode::ConstantSpecial) ? RandomSpecialFloatBits(rng) : RandomNormalFloatBits(rng);
for (int i = 0; i < vertex_count; i++)
{
const u32 s_bits = special_st && (rng() % 4 == 0) ? RandomSpecialFloatBits(rng) : RandomNormalFloatBits(rng);
const u32 t_bits = special_st && (rng() % 4 == 0) ? RandomSpecialFloatBits(rng) : RandomNormalFloatBits(rng);
const u32 q_bits = (qmode == FmmQMode::Varying) ? RandomNormalFloatBits(rng) : const_q;
m0s[i] = GSVector4i(static_cast<int>(s_bits), static_cast<int>(t_bits), static_cast<int>(rng()),
static_cast<int>(q_bits));
m1s[i] = GSVector4i(static_cast<int>((rng() & 0xFFFF) | (rng() << 16)), static_cast<int>(rng()),
static_cast<int>(rng()), static_cast<int>(rng()));
}
// Emit prims, skipping some (culled — excluded from both sides).
u16 index[24 * 3];
int icount = 0;
u32 watermark = 0;
GSVertexKernels::FmmAcc acc;
GSVertexKernels::FmmAccReset(acc, tme, fst);
for (int p = 0; p < prim_count; p++)
{
const u16 i0 = static_cast<u16>(strip_shaped ? p + 0 : p * 3 + 0);
const u16 i1 = static_cast<u16>(strip_shaped ? p + 1 : p * 3 + 1);
const u16 i2 = static_cast<u16>(strip_shaped ? p + 2 : p * 3 + 2);
// Culled prims are excluded from both sides; an accepted prim after a
// culled run still accumulates the culled prims' shared vertices (they
// sit at/above the watermark), mirroring VertexKickDirect.
if (rng() % 3 == 0)
continue;
index[icount++] = i0;
index[icount++] = i1;
index[icount++] = i2;
// Mirror VertexKickDirect: buffer-accumulate referenced vertices at or
// above the watermark (color only under iip), then the provoking vertex
// with color always.
for (u32 j = std::max<u32>(watermark, i2 >= 2 ? i2 - 2 : 0); j < i2; j++)
GSVertexKernels::FmmAccumVertex(acc, m0s[j], m1s[j], tme, fst, iip);
GSVertexKernels::FmmAccumVertex(acc, m0s[i2], m1s[i2], tme, fst, true);
watermark = static_cast<u32>(i2) + 1;
}
if (icount == 0)
return false;
GIFRegXYOFFSET ofs;
ofs.U64 = 0;
ofs.OFX = rng() & 0xFFFF;
ofs.OFY = rng() & 0xFFFF;
const u32 tw = rng() % 11;
const u32 th = rng() % 11;
GSVertexKernels::FmmResult r;
if (!GSVertexKernels::FmmFinish(acc, tme, fst, color, ofs, tw, th, r))
return false;
FmmRefOut ref;
RefFindMinMaxTriangle(m0s, m1s, index, icount, iip, tme, fst, color, ofs, tw, th, ref);
EXPECT_TRUE(GSVector4i::cast(r.min_p).eq(GSVector4i::cast(ref.min_p))) << "min_p divergence";
EXPECT_TRUE(GSVector4i::cast(r.max_p).eq(GSVector4i::cast(ref.max_p))) << "max_p divergence";
EXPECT_TRUE(GSVector4i::cast(r.min_t).eq(GSVector4i::cast(ref.min_t))) << "min_t divergence";
EXPECT_TRUE(GSVector4i::cast(r.max_t).eq(GSVector4i::cast(ref.max_t))) << "max_t divergence";
EXPECT_TRUE(r.min_c.eq(ref.min_c)) << "min_c divergence";
EXPECT_TRUE(r.max_c.eq(ref.max_c)) << "max_c divergence";
EXPECT_EQ(r.write_nan, ref.wrote_nan) << "write_nan policy divergence";
if (r.write_nan)
EXPECT_EQ(r.nan_value, ref.nan_value) << "nan divergence";
return true;
}
void RunFmmSweep(u32 seed, int iters, FmmQMode qmode, bool special_st, bool expect_accept)
{
std::mt19937 rng(seed);
int accepted = 0;
int attempted = 0;
for (int i = 0; i < iters; i++)
{
const bool iip = rng() & 1;
const bool tme = rng() & 1;
const bool fst = rng() & 1;
const bool color = (rng() % 4) != 0;
const bool strip = rng() & 1;
attempted++;
if (RunFmmCase(rng, iip, tme, fst, color, qmode, special_st, strip))
accepted++;
if (::testing::Test::HasFailure())
{
ADD_FAILURE() << "seed " << seed << " iter " << i;
return;
}
}
if (expect_accept)
{
// The benign configurations must ride the fused path (this is the
// perf guarantee, not just correctness).
EXPECT_GT(accepted, attempted / 2) << "fused acceptance collapsed";
}
}
} // namespace
TEST(GsVertexFmm, BenignSweep)
{
// Constant normal Q, finite S/T: every draw must fuse and match bit-exactly.
RunFmmSweep(0x67763320, 150000, FmmQMode::ConstantNormal, false, true);
}
TEST(GsVertexFmm, SpecialStSweep)
{
// Inf/NaN S/T mixed in: fused may decline (legacy masks per lane); when it
// accepts, it must match.
RunFmmSweep(0x67763321, 150000, FmmQMode::ConstantNormal, true, false);
}
TEST(GsVertexFmm, SpecialQSweep)
{
// Zero/denormal/inf/NaN Q: fused must decline the STQ path (and still match
// on FST/no-TME draws, where Q doesn't matter).
RunFmmSweep(0x67763322, 150000, FmmQMode::ConstantSpecial, false, false);
}
TEST(GsVertexFmm, VaryingQSweep)
{
// Per-vertex Q: fused must decline the STQ path.
RunFmmSweep(0x67763323, 150000, FmmQMode::Varying, false, false);
}
#endif // ARCH_ARM64