From a9d8b6d3c5d59fb3891fbe86cf239a243766943d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 2 Oct 2026 16:39:56 -0600 Subject: [PATCH] GEMath: GERowSum4 takes its left operands as one vector They're truncated to float24 with one AND, and their exponents, fractions and signs worked out for all terms at once, each term then broadcasting its lane (new CrossSIMD Vec4S32::SplatLane). The term count becomes a template parameter so the lanes are constants. About 75-83 M transforms/s on an M1, from 68; results unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- Common/Math/CrossSIMD.h | 12 ++++++++ Common/Math/SIMDHeaders.h | 9 ++++++ GPU/Software/GEMath.cpp | 9 ++++-- GPU/Software/GEMath.h | 56 ++++++++++++++++++++++------------ GPU/Software/Lighting.cpp | 6 ++-- GPU/Software/TransformUnit.cpp | 12 ++++---- unittest/TestGEMath.cpp | 13 ++++++-- 7 files changed, 83 insertions(+), 34 deletions(-) diff --git a/Common/Math/CrossSIMD.h b/Common/Math/CrossSIMD.h index 5c5d572111..e80ae845e2 100644 --- a/Common/Math/CrossSIMD.h +++ b/Common/Math/CrossSIMD.h @@ -170,6 +170,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ imm == 0 ? v : _mm_srai_epi32(v, imm) }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ _mm_shuffle_epi32(v, _MM_SHUFFLE(lane, lane, lane, lane)) }; } Vec4S32 Max(Vec4S32 other) const { const __m128i gt = _mm_cmpgt_epi32(v, other.v); return Vec4S32{ _mm_or_si128(_mm_and_si128(gt, v), _mm_andnot_si128(gt, other.v)) }; @@ -631,6 +634,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ vshrq_n_s32(v, imm) }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ vdupq_laneq_s32(v, lane) }; } Vec4S32 Max(Vec4S32 other) const { return Vec4S32{ vmaxq_s32(v, other.v) }; } void operator +=(Vec4S32 other) { v = vaddq_s32(v, other.v); } @@ -1174,6 +1180,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ __lsx_vsrai_w(v, imm) }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ __lsx_vreplvei_w(v, lane) }; } Vec4S32 Max(Vec4S32 other) const { return Vec4S32{ __lsx_vmax_w(v, other.v) }; } void operator +=(Vec4S32 other) { v = __lsx_vadd_w(v, other.v); } @@ -1670,6 +1679,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ { v[0] >> imm, v[1] >> imm, v[2] >> imm, v[3] >> imm } }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ { v[lane], v[lane], v[lane], v[lane] } }; } Vec4S32 Max(Vec4S32 other) const { Vec4S32 tmp; for (int i = 0; i < 4; i++) { diff --git a/Common/Math/SIMDHeaders.h b/Common/Math/SIMDHeaders.h index 0619efc0cd..8f710d6a3d 100644 --- a/Common/Math/SIMDHeaders.h +++ b/Common/Math/SIMDHeaders.h @@ -133,6 +133,15 @@ static inline float32x4_t vdupq_laneq_f32(float32x4_t vec, int lane) { } } +static inline int32x4_t vdupq_laneq_s32(int32x4_t vec, int lane) { + switch (lane & 3) { + case 0: return vdupq_lane_s32(vget_low_s32(vec), 0); + case 1: return vdupq_lane_s32(vget_low_s32(vec), 1); + case 2: return vdupq_lane_s32(vget_high_s32(vec), 0); + default: return vdupq_lane_s32(vget_high_s32(vec), 1); + } +} + #define vfmaq_laneq_f32 vmlaq_laneq_f32 static inline uint32x4_t vcgezq_f32(float32x4_t v) { diff --git a/GPU/Software/GEMath.cpp b/GPU/Software/GEMath.cpp index 3ece934b20..ab1ef9983d 100644 --- a/GPU/Software/GEMath.cpp +++ b/GPU/Software/GEMath.cpp @@ -114,9 +114,12 @@ float GEAddFloat24(float a, float b) { } // A dot product as the GE's dot product unit sums it (like a matrix row without translation). -Vec4F32 GERowSum4Fallback(const float a[4], const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result) { - alignas(16) float bv[4][4], rv[4]; +Vec4F32 GERowSum4Fallback(Vec4F32 a, const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result) { + alignas(16) float av[4], bv[4][4], rv[4]; alignas(16) int lv[4]; + a.Store(av); + for (int k = 0; k < 4; ++k) + av[k] = TruncateToFloat24(av[k]); for (int k = 0; k < count; ++k) Vec4F32(b[k]).Store(bv[k]); result.Store(rv); @@ -126,7 +129,7 @@ Vec4F32 GERowSum4Fallback(const float a[4], const Vec4F32 b[4], int count, Vec4S continue; GERowTerm terms[4]; for (int k = 0; k < count; ++k) - terms[k] = GEProduct(a[k], bv[k][i]); + terms[k] = GEProduct(av[k], bv[k][i]); rv[i] = GERowSum(terms, count); } return Vec4F32::Load(rv); diff --git a/GPU/Software/GEMath.h b/GPU/Software/GEMath.h index 70a2c8354d..843708e74c 100644 --- a/GPU/Software/GEMath.h +++ b/GPU/Software/GEMath.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "Common/CommonTypes.h" #include "Common/Math/CrossSIMD.h" @@ -170,38 +171,55 @@ inline float GEScreenZ(float clipZ, float clipW, float zScale, float zCenter) { } // GERowSum4's lanes that it can't do itself: a result that isn't a normal float. -Vec4F32 GERowSum4Fallback(const float a[4], const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result); +Vec4F32 GERowSum4Fallback(Vec4F32 a, const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result); // Four row sums at once: lane i is GERowSum of the products a[k] * b[k][i], k < count, so four rows -// sharing their left operands, like the four rows of a vertex transform. Every operand must be a float24 -// (its low 8 mantissa bits zero), as matrix entries and truncated coordinates are. That makes each exact -// product a 32-bit integer: (1.a' * 1.b') >> 15 = 32768 + a' + b' + (a' b' >> 15), a' and b' being the 15 -// fraction bits, and a' b' fits a 16-bit multiply. The terms are aligned to the largest lsb by scaling -// with a power of two and truncating, which drops low bits toward zero as GERowSum's shifts do. -inline Vec4F32 GERowSum4(const float a[4], const Vec4F32 b[4], int count) { +// sharing their left operands (the lanes of a), like the four rows of a vertex transform. a is truncated +// to float24s here; every b must be one already (its low 8 mantissa bits zero), as matrix entries are. +// That makes each exact product a 32-bit integer: (1.a' * 1.b') >> 15 = 32768 + a' + b' + (a' b' >> 15), +// a' and b' being the 15 fraction bits, and a' b' fits a 16-bit multiply. The terms are aligned to the +// largest lsb by scaling with a power of two and truncating, which drops low bits toward zero as +// GERowSum's shifts do. +template +inline Vec4F32 GERowSum4(Vec4F32 a, const Vec4F32 b[4]) { + static_assert(count >= 1 && count <= 4, "GERowSum4 sums up to four terms"); const Vec4S32 expMask = Vec4S32::Splat(0x7F800000); const Vec4S32 fracMask = Vec4S32::Splat(0x7FFF); // Below any real lsb, without overflowing the differences below. - const int noTerm = -100000; + const Vec4S32 noTerm = Vec4S32::Splat(-100000); + + // The left operands' parts, for all terms at once; each term takes its lane. + const Vec4S32 aBits = Vec4S32FromBits(a) & Vec4S32::Splat((int)0xFFFFFF00); + const Vec4S32 ea = (aBits & expMask).Shr<23>(); + const Vec4S32 af = aBits.Shr<8>() & fracMask; + const Vec4S32 aNone = ea.CompareEq(Vec4S32::Zero()); + const Vec4S32 aLsb = ea - Vec4S32::Splat(254 + 15); + Vec4S32 m[4], lsb[4]; - Vec4S32 lsbMax = Vec4S32::Splat(noTerm); - for (int k = 0; k < count; ++k) { - uint32_t abits; - memcpy(&abits, &a[k], sizeof(abits)); - const int ea = (abits >> 23) & 0xFF; - const int af = (abits >> 8) & 0x7FFF; + Vec4S32 lsbMax = noTerm; + auto term = [&](auto lane) { + constexpr int k = decltype(lane)::value; const Vec4S32 bb = Vec4S32FromBits(b[k]); const Vec4S32 eb = (bb & expMask).Shr<23>(); const Vec4S32 bf = bb.Shr<8>() & fracMask; - Vec4S32 prod = Vec4S32::Splat(32768 + af) + bf + bf.Mul16(Vec4S32::Splat(af)).Shr<15>(); - const Vec4S32 sign = (bb ^ Vec4S32::Splat((int)abits)).Shr<31>(); + const Vec4S32 afk = af.template SplatLane(); + Vec4S32 prod = afk + Vec4S32::Splat(32768) + bf + bf.Mul16(afk).template Shr<15>(); + const Vec4S32 sign = (bb ^ aBits.template SplatLane()).template Shr<31>(); prod = (prod ^ sign) - sign; // Zero and denormals give no term. - const Vec4S32 none = ea == 0 ? Vec4S32::Splat(-1) : eb.CompareEq(Vec4S32::Zero()); + const Vec4S32 none = aNone.template SplatLane() | eb.CompareEq(Vec4S32::Zero()); m[k] = prod.AndNot(none); - lsb[k] = (eb + Vec4S32::Splat(ea - 254 - 15)).AndNot(none) | (Vec4S32::Splat(noTerm) & none); + lsb[k] = (eb + aLsb.template SplatLane()).AndNot(none) | (noTerm & none); lsbMax = lsbMax.Max(lsb[k]); - } + }; + term(std::integral_constant{}); + if constexpr (count > 1) + term(std::integral_constant{}); + if constexpr (count > 2) + term(std::integral_constant{}); + if constexpr (count > 3) + term(std::integral_constant{}); + Vec4S32 sum = Vec4S32::Zero(); for (int k = 0; k < count; ++k) { // 2^(lsb - lsbMax), or zero once that's below the float range (a shift of 127 or more drops it anyway). diff --git a/GPU/Software/Lighting.cpp b/GPU/Software/Lighting.cpp index 759d536f68..320ab30755 100644 --- a/GPU/Software/Lighting.cpp +++ b/GPU/Software/Lighting.cpp @@ -204,14 +204,14 @@ static inline float GENormalDot(const Vec3f &v, const Vec3f &n, float nRsqrt) { static Vec3f GELightVector(const Vec3f &lpos, const Vec3f &modelpos) { const float *m = gstate.worldMatrix; // The three components at once; matrix row k is m[3k..3k+2], negated. - const float a[4] = { 1.0f, TruncateToFloat24(modelpos.x), TruncateToFloat24(modelpos.y), TruncateToFloat24(modelpos.z) }; + alignas(16) const float a[4] = { modelpos.x, modelpos.y, modelpos.z, 1.0f }; alignas(16) const float translated[4] = { GEAddFloat24(lpos[0], -TruncateToFloat24(m[9])), GEAddFloat24(lpos[1], -TruncateToFloat24(m[10])), GEAddFloat24(lpos[2], -TruncateToFloat24(m[11])), 0.0f }; // Negated by the sign bit, as -m is (a subtraction from zero wouldn't flip a NaN's). const Vec4S32 sign = Vec4S32::Splat((int)0x80000000); auto negate = [&](const float *p) { return Vec4F32FromBits(Vec4S32FromBits(Vec4F32::Load(p)) ^ sign); }; - const Vec4F32 b[4] = { Vec4F32::Load(translated), negate(m), negate(m + 3), negate(m + 6) }; + const Vec4F32 b[4] = { negate(m), negate(m + 3), negate(m + 6), Vec4F32::Load(translated) }; alignas(16) float out[4]; - GERowSum4(a, b, 4).Store(out); + GERowSum4<4>(Vec4F32::Load(a), b).Store(out); return Vec3f(out[0], out[1], out[2]); } diff --git a/GPU/Software/TransformUnit.cpp b/GPU/Software/TransformUnit.cpp index d8bb5c7d80..8663d07590 100644 --- a/GPU/Software/TransformUnit.cpp +++ b/GPU/Software/TransformUnit.cpp @@ -154,31 +154,31 @@ WorldCoords TransformUnit::ModelToWorldNormal(const ModelCoords &coords) { // Each component summed like a matrix row (gpu/probe exp61), the three at once. Matrix row k is m[3k..3k+2]; // the fourth lane reads the next row's first entry and goes unused. const float *m = gstate.worldMatrix; - const float a[3] = { TruncateToFloat24(coords.x), TruncateToFloat24(coords.y), TruncateToFloat24(coords.z) }; + alignas(16) const float a[4] = { coords.x, coords.y, coords.z, 0.0f }; const Vec4F32 b[3] = { Vec4F32::Load(m), Vec4F32::Load(m + 3), Vec4F32::Load(m + 6) }; alignas(16) float out[4]; - GERowSum4(a, b, 3).Store(out); + GERowSum4<3>(Vec4F32::Load(a), b).Store(out); return WorldCoords(out[0], out[1], out[2]); } // The clip space position from the combined matrix (gpu/probe exp32, exp34, exp42), each component summed // like a matrix row, the four at once. The position is a float24; the translation is a term of its own. static inline Vec4F32 GEClipPosition(const Vec3f &v, const float m[16]) { - const float a[4] = { TruncateToFloat24(v.x), TruncateToFloat24(v.y), TruncateToFloat24(v.z), 1.0f }; + alignas(16) const float a[4] = { v.x, v.y, v.z, 1.0f }; const Vec4F32 b[4] = { Vec4F32::Load(m), Vec4F32::Load(m + 4), Vec4F32::Load(m + 8), Vec4F32::Load(m + 12) }; - return GERowSum4(a, b, 4); + return GERowSum4<4>(Vec4F32::Load(a), b); } // The texture coordinates from the 4x3 texture matrix, summed like clip space rows (gpu/probe exp64). static inline Vec3Packedf GETexGen(const Vec3f &v, const float m[12]) { - const float a[4] = { TruncateToFloat24(v.x), TruncateToFloat24(v.y), TruncateToFloat24(v.z), 1.0f }; + alignas(16) const float a[4] = { v.x, v.y, v.z, 1.0f }; // Row k is m[3k..3k+2], padded so the last load stays inside. alignas(16) float padded[13]; memcpy(padded, m, 12 * sizeof(float)); padded[12] = 0.0f; const Vec4F32 b[4] = { Vec4F32::Load(padded), Vec4F32::Load(padded + 3), Vec4F32::Load(padded + 6), Vec4F32::Load(padded + 9) }; alignas(16) float out[4]; - GERowSum4(a, b, 4).Store(out); + GERowSum4<4>(Vec4F32::Load(a), b).Store(out); return Vec3Packedf(out[0], out[1], out[2]); } diff --git a/unittest/TestGEMath.cpp b/unittest/TestGEMath.cpp index 119b949b2a..b85c573fc3 100644 --- a/unittest/TestGEMath.cpp +++ b/unittest/TestGEMath.cpp @@ -136,7 +136,7 @@ static bool TestGERowSum4() { const int count = 1 + (i & 3); const int spread = (i % 7 == 0) ? 120 : ((i % 3 == 0) ? 20 : 3); const int center = (i % 11 == 0) ? (int)(next() % 254) + 1 : 127; - float a[4]; + alignas(16) float a[4] = {}; alignas(16) float b[4][4]; Vec4F32 bv[4]; for (int k = 0; k < count; ++k) { @@ -146,7 +146,14 @@ static bool TestGERowSum4() { bv[k] = Vec4F32::Load(b[k]); } alignas(16) float got[4]; - GERowSum4(a, bv, count).Store(got); + Vec4F32 av = Vec4F32::Load(a); + switch (count) { + case 1: av = GERowSum4<1>(av, bv); break; + case 2: av = GERowSum4<2>(av, bv); break; + case 3: av = GERowSum4<3>(av, bv); break; + default: av = GERowSum4<4>(av, bv); break; + } + av.Store(got); for (int l = 0; l < 4; ++l) { GERowTerm terms[4]; for (int k = 0; k < count; ++k) @@ -175,7 +182,7 @@ static bool TestGERowSum4() { const double simd = CallsPerSecond([&] { Vec4F32 acc = Vec4F32::Zero(); for (int i = 0; i < count; ++i) - acc = acc + GERowSum4(&pos[i * 4], rows, 4); + acc = acc + GERowSum4<4>(Vec4F32::Load(&pos[i * 4]), rows); alignas(16) float out[4]; acc.Store(out); sink = out[0];