diff --git a/Common/Math/CrossSIMD.h b/Common/Math/CrossSIMD.h index 5c5d572111..e80ae845e2 100644 --- a/Common/Math/CrossSIMD.h +++ b/Common/Math/CrossSIMD.h @@ -170,6 +170,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ imm == 0 ? v : _mm_srai_epi32(v, imm) }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ _mm_shuffle_epi32(v, _MM_SHUFFLE(lane, lane, lane, lane)) }; } Vec4S32 Max(Vec4S32 other) const { const __m128i gt = _mm_cmpgt_epi32(v, other.v); return Vec4S32{ _mm_or_si128(_mm_and_si128(gt, v), _mm_andnot_si128(gt, other.v)) }; @@ -631,6 +634,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ vshrq_n_s32(v, imm) }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ vdupq_laneq_s32(v, lane) }; } Vec4S32 Max(Vec4S32 other) const { return Vec4S32{ vmaxq_s32(v, other.v) }; } void operator +=(Vec4S32 other) { v = vaddq_s32(v, other.v); } @@ -1174,6 +1180,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ __lsx_vsrai_w(v, imm) }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ __lsx_vreplvei_w(v, lane) }; } Vec4S32 Max(Vec4S32 other) const { return Vec4S32{ __lsx_vmax_w(v, other.v) }; } void operator +=(Vec4S32 other) { v = __lsx_vadd_w(v, other.v); } @@ -1670,6 +1679,9 @@ struct Vec4S32 { // Arithmetic: shifts in copies of the sign bit. template Vec4S32 Shr() const { return Vec4S32{ { v[0] >> imm, v[1] >> imm, v[2] >> imm, v[3] >> imm } }; } + // Every lane set to this one's lane. + template + Vec4S32 SplatLane() const { return Vec4S32{ { v[lane], v[lane], v[lane], v[lane] } }; } Vec4S32 Max(Vec4S32 other) const { Vec4S32 tmp; for (int i = 0; i < 4; i++) { diff --git a/Common/Math/SIMDHeaders.h b/Common/Math/SIMDHeaders.h index 0619efc0cd..8f710d6a3d 100644 --- a/Common/Math/SIMDHeaders.h +++ b/Common/Math/SIMDHeaders.h @@ -133,6 +133,15 @@ static inline float32x4_t vdupq_laneq_f32(float32x4_t vec, int lane) { } } +static inline int32x4_t vdupq_laneq_s32(int32x4_t vec, int lane) { + switch (lane & 3) { + case 0: return vdupq_lane_s32(vget_low_s32(vec), 0); + case 1: return vdupq_lane_s32(vget_low_s32(vec), 1); + case 2: return vdupq_lane_s32(vget_high_s32(vec), 0); + default: return vdupq_lane_s32(vget_high_s32(vec), 1); + } +} + #define vfmaq_laneq_f32 vmlaq_laneq_f32 static inline uint32x4_t vcgezq_f32(float32x4_t v) { diff --git a/GPU/Software/GEMath.cpp b/GPU/Software/GEMath.cpp index 3ece934b20..ab1ef9983d 100644 --- a/GPU/Software/GEMath.cpp +++ b/GPU/Software/GEMath.cpp @@ -114,9 +114,12 @@ float GEAddFloat24(float a, float b) { } // A dot product as the GE's dot product unit sums it (like a matrix row without translation). -Vec4F32 GERowSum4Fallback(const float a[4], const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result) { - alignas(16) float bv[4][4], rv[4]; +Vec4F32 GERowSum4Fallback(Vec4F32 a, const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result) { + alignas(16) float av[4], bv[4][4], rv[4]; alignas(16) int lv[4]; + a.Store(av); + for (int k = 0; k < 4; ++k) + av[k] = TruncateToFloat24(av[k]); for (int k = 0; k < count; ++k) Vec4F32(b[k]).Store(bv[k]); result.Store(rv); @@ -126,7 +129,7 @@ Vec4F32 GERowSum4Fallback(const float a[4], const Vec4F32 b[4], int count, Vec4S continue; GERowTerm terms[4]; for (int k = 0; k < count; ++k) - terms[k] = GEProduct(a[k], bv[k][i]); + terms[k] = GEProduct(av[k], bv[k][i]); rv[i] = GERowSum(terms, count); } return Vec4F32::Load(rv); diff --git a/GPU/Software/GEMath.h b/GPU/Software/GEMath.h index 70a2c8354d..843708e74c 100644 --- a/GPU/Software/GEMath.h +++ b/GPU/Software/GEMath.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "Common/CommonTypes.h" #include "Common/Math/CrossSIMD.h" @@ -170,38 +171,55 @@ inline float GEScreenZ(float clipZ, float clipW, float zScale, float zCenter) { } // GERowSum4's lanes that it can't do itself: a result that isn't a normal float. -Vec4F32 GERowSum4Fallback(const float a[4], const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result); +Vec4F32 GERowSum4Fallback(Vec4F32 a, const Vec4F32 b[4], int count, Vec4S32 lanes, Vec4F32 result); // Four row sums at once: lane i is GERowSum of the products a[k] * b[k][i], k < count, so four rows -// sharing their left operands, like the four rows of a vertex transform. Every operand must be a float24 -// (its low 8 mantissa bits zero), as matrix entries and truncated coordinates are. That makes each exact -// product a 32-bit integer: (1.a' * 1.b') >> 15 = 32768 + a' + b' + (a' b' >> 15), a' and b' being the 15 -// fraction bits, and a' b' fits a 16-bit multiply. The terms are aligned to the largest lsb by scaling -// with a power of two and truncating, which drops low bits toward zero as GERowSum's shifts do. -inline Vec4F32 GERowSum4(const float a[4], const Vec4F32 b[4], int count) { +// sharing their left operands (the lanes of a), like the four rows of a vertex transform. a is truncated +// to float24s here; every b must be one already (its low 8 mantissa bits zero), as matrix entries are. +// That makes each exact product a 32-bit integer: (1.a' * 1.b') >> 15 = 32768 + a' + b' + (a' b' >> 15), +// a' and b' being the 15 fraction bits, and a' b' fits a 16-bit multiply. The terms are aligned to the +// largest lsb by scaling with a power of two and truncating, which drops low bits toward zero as +// GERowSum's shifts do. +template +inline Vec4F32 GERowSum4(Vec4F32 a, const Vec4F32 b[4]) { + static_assert(count >= 1 && count <= 4, "GERowSum4 sums up to four terms"); const Vec4S32 expMask = Vec4S32::Splat(0x7F800000); const Vec4S32 fracMask = Vec4S32::Splat(0x7FFF); // Below any real lsb, without overflowing the differences below. - const int noTerm = -100000; + const Vec4S32 noTerm = Vec4S32::Splat(-100000); + + // The left operands' parts, for all terms at once; each term takes its lane. + const Vec4S32 aBits = Vec4S32FromBits(a) & Vec4S32::Splat((int)0xFFFFFF00); + const Vec4S32 ea = (aBits & expMask).Shr<23>(); + const Vec4S32 af = aBits.Shr<8>() & fracMask; + const Vec4S32 aNone = ea.CompareEq(Vec4S32::Zero()); + const Vec4S32 aLsb = ea - Vec4S32::Splat(254 + 15); + Vec4S32 m[4], lsb[4]; - Vec4S32 lsbMax = Vec4S32::Splat(noTerm); - for (int k = 0; k < count; ++k) { - uint32_t abits; - memcpy(&abits, &a[k], sizeof(abits)); - const int ea = (abits >> 23) & 0xFF; - const int af = (abits >> 8) & 0x7FFF; + Vec4S32 lsbMax = noTerm; + auto term = [&](auto lane) { + constexpr int k = decltype(lane)::value; const Vec4S32 bb = Vec4S32FromBits(b[k]); const Vec4S32 eb = (bb & expMask).Shr<23>(); const Vec4S32 bf = bb.Shr<8>() & fracMask; - Vec4S32 prod = Vec4S32::Splat(32768 + af) + bf + bf.Mul16(Vec4S32::Splat(af)).Shr<15>(); - const Vec4S32 sign = (bb ^ Vec4S32::Splat((int)abits)).Shr<31>(); + const Vec4S32 afk = af.template SplatLane(); + Vec4S32 prod = afk + Vec4S32::Splat(32768) + bf + bf.Mul16(afk).template Shr<15>(); + const Vec4S32 sign = (bb ^ aBits.template SplatLane()).template Shr<31>(); prod = (prod ^ sign) - sign; // Zero and denormals give no term. - const Vec4S32 none = ea == 0 ? Vec4S32::Splat(-1) : eb.CompareEq(Vec4S32::Zero()); + const Vec4S32 none = aNone.template SplatLane() | eb.CompareEq(Vec4S32::Zero()); m[k] = prod.AndNot(none); - lsb[k] = (eb + Vec4S32::Splat(ea - 254 - 15)).AndNot(none) | (Vec4S32::Splat(noTerm) & none); + lsb[k] = (eb + aLsb.template SplatLane()).AndNot(none) | (noTerm & none); lsbMax = lsbMax.Max(lsb[k]); - } + }; + term(std::integral_constant{}); + if constexpr (count > 1) + term(std::integral_constant{}); + if constexpr (count > 2) + term(std::integral_constant{}); + if constexpr (count > 3) + term(std::integral_constant{}); + Vec4S32 sum = Vec4S32::Zero(); for (int k = 0; k < count; ++k) { // 2^(lsb - lsbMax), or zero once that's below the float range (a shift of 127 or more drops it anyway). diff --git a/GPU/Software/Lighting.cpp b/GPU/Software/Lighting.cpp index 759d536f68..320ab30755 100644 --- a/GPU/Software/Lighting.cpp +++ b/GPU/Software/Lighting.cpp @@ -204,14 +204,14 @@ static inline float GENormalDot(const Vec3f &v, const Vec3f &n, float nRsqrt) { static Vec3f GELightVector(const Vec3f &lpos, const Vec3f &modelpos) { const float *m = gstate.worldMatrix; // The three components at once; matrix row k is m[3k..3k+2], negated. - const float a[4] = { 1.0f, TruncateToFloat24(modelpos.x), TruncateToFloat24(modelpos.y), TruncateToFloat24(modelpos.z) }; + alignas(16) const float a[4] = { modelpos.x, modelpos.y, modelpos.z, 1.0f }; alignas(16) const float translated[4] = { GEAddFloat24(lpos[0], -TruncateToFloat24(m[9])), GEAddFloat24(lpos[1], -TruncateToFloat24(m[10])), GEAddFloat24(lpos[2], -TruncateToFloat24(m[11])), 0.0f }; // Negated by the sign bit, as -m is (a subtraction from zero wouldn't flip a NaN's). const Vec4S32 sign = Vec4S32::Splat((int)0x80000000); auto negate = [&](const float *p) { return Vec4F32FromBits(Vec4S32FromBits(Vec4F32::Load(p)) ^ sign); }; - const Vec4F32 b[4] = { Vec4F32::Load(translated), negate(m), negate(m + 3), negate(m + 6) }; + const Vec4F32 b[4] = { negate(m), negate(m + 3), negate(m + 6), Vec4F32::Load(translated) }; alignas(16) float out[4]; - GERowSum4(a, b, 4).Store(out); + GERowSum4<4>(Vec4F32::Load(a), b).Store(out); return Vec3f(out[0], out[1], out[2]); } diff --git a/GPU/Software/TransformUnit.cpp b/GPU/Software/TransformUnit.cpp index d8bb5c7d80..8663d07590 100644 --- a/GPU/Software/TransformUnit.cpp +++ b/GPU/Software/TransformUnit.cpp @@ -154,31 +154,31 @@ WorldCoords TransformUnit::ModelToWorldNormal(const ModelCoords &coords) { // Each component summed like a matrix row (gpu/probe exp61), the three at once. Matrix row k is m[3k..3k+2]; // the fourth lane reads the next row's first entry and goes unused. const float *m = gstate.worldMatrix; - const float a[3] = { TruncateToFloat24(coords.x), TruncateToFloat24(coords.y), TruncateToFloat24(coords.z) }; + alignas(16) const float a[4] = { coords.x, coords.y, coords.z, 0.0f }; const Vec4F32 b[3] = { Vec4F32::Load(m), Vec4F32::Load(m + 3), Vec4F32::Load(m + 6) }; alignas(16) float out[4]; - GERowSum4(a, b, 3).Store(out); + GERowSum4<3>(Vec4F32::Load(a), b).Store(out); return WorldCoords(out[0], out[1], out[2]); } // The clip space position from the combined matrix (gpu/probe exp32, exp34, exp42), each component summed // like a matrix row, the four at once. The position is a float24; the translation is a term of its own. static inline Vec4F32 GEClipPosition(const Vec3f &v, const float m[16]) { - const float a[4] = { TruncateToFloat24(v.x), TruncateToFloat24(v.y), TruncateToFloat24(v.z), 1.0f }; + alignas(16) const float a[4] = { v.x, v.y, v.z, 1.0f }; const Vec4F32 b[4] = { Vec4F32::Load(m), Vec4F32::Load(m + 4), Vec4F32::Load(m + 8), Vec4F32::Load(m + 12) }; - return GERowSum4(a, b, 4); + return GERowSum4<4>(Vec4F32::Load(a), b); } // The texture coordinates from the 4x3 texture matrix, summed like clip space rows (gpu/probe exp64). static inline Vec3Packedf GETexGen(const Vec3f &v, const float m[12]) { - const float a[4] = { TruncateToFloat24(v.x), TruncateToFloat24(v.y), TruncateToFloat24(v.z), 1.0f }; + alignas(16) const float a[4] = { v.x, v.y, v.z, 1.0f }; // Row k is m[3k..3k+2], padded so the last load stays inside. alignas(16) float padded[13]; memcpy(padded, m, 12 * sizeof(float)); padded[12] = 0.0f; const Vec4F32 b[4] = { Vec4F32::Load(padded), Vec4F32::Load(padded + 3), Vec4F32::Load(padded + 6), Vec4F32::Load(padded + 9) }; alignas(16) float out[4]; - GERowSum4(a, b, 4).Store(out); + GERowSum4<4>(Vec4F32::Load(a), b).Store(out); return Vec3Packedf(out[0], out[1], out[2]); } diff --git a/unittest/TestGEMath.cpp b/unittest/TestGEMath.cpp index 119b949b2a..b85c573fc3 100644 --- a/unittest/TestGEMath.cpp +++ b/unittest/TestGEMath.cpp @@ -136,7 +136,7 @@ static bool TestGERowSum4() { const int count = 1 + (i & 3); const int spread = (i % 7 == 0) ? 120 : ((i % 3 == 0) ? 20 : 3); const int center = (i % 11 == 0) ? (int)(next() % 254) + 1 : 127; - float a[4]; + alignas(16) float a[4] = {}; alignas(16) float b[4][4]; Vec4F32 bv[4]; for (int k = 0; k < count; ++k) { @@ -146,7 +146,14 @@ static bool TestGERowSum4() { bv[k] = Vec4F32::Load(b[k]); } alignas(16) float got[4]; - GERowSum4(a, bv, count).Store(got); + Vec4F32 av = Vec4F32::Load(a); + switch (count) { + case 1: av = GERowSum4<1>(av, bv); break; + case 2: av = GERowSum4<2>(av, bv); break; + case 3: av = GERowSum4<3>(av, bv); break; + default: av = GERowSum4<4>(av, bv); break; + } + av.Store(got); for (int l = 0; l < 4; ++l) { GERowTerm terms[4]; for (int k = 0; k < count; ++k) @@ -175,7 +182,7 @@ static bool TestGERowSum4() { const double simd = CallsPerSecond([&] { Vec4F32 acc = Vec4F32::Zero(); for (int i = 0; i < count; ++i) - acc = acc + GERowSum4(&pos[i * 4], rows, 4); + acc = acc + GERowSum4<4>(Vec4F32::Load(&pos[i * 4]), rows); alignas(16) float out[4]; acc.Store(out); sink = out[0];