diff --git a/GPU/Common/SplineCommon.cpp b/GPU/Common/SplineCommon.cpp index dc59dca432..aee2f6a5dc 100644 --- a/GPU/Common/SplineCommon.cpp +++ b/GPU/Common/SplineCommon.cpp @@ -685,38 +685,6 @@ struct GEBezierFixed { } }; -static NO_INLINE Vec4F32 GEAdd4Small(Vec4F32 a, Vec4F32 b, Vec4S32 small, Vec4F32 r) { - alignas(16) float av[4], bv[4], rv[4]; - alignas(16) int sv[4]; - a.Store(av); - b.Store(bv); - r.Store(rv); - small.Store(sv); - for (int i = 0; i < 4; ++i) { - if (sv[i]) - rv[i] = GEAdd(av[i], bv[i]); - } - return Vec4F32::Load(rv); -} - -// Four of GEAdd at once: both terms truncated to the precision of the larger one, then added exactly. -static __forceinline Vec4F32 GEAdd4(Vec4F32 a, Vec4F32 b) { - const Vec4S32 expMask = Vec4S32::Splat(0x7F800000); - const Vec4S32 absMask = Vec4S32::Splat(0x7FFFFFFF); - const Vec4S32 ba = Vec4S32FromBits(a); - const Vec4S32 bb = Vec4S32FromBits(b); - const Vec4F32 za = Select((ba & expMask).CompareEq(Vec4S32::Zero()), Vec4F32::Zero(), a); - const Vec4F32 zb = Select((bb & expMask).CompareEq(Vec4S32::Zero()), Vec4F32::Zero(), b); - const Vec4S32 e = Vec4S32FromBits(Vec4F32FromBits(ba & absMask).Max(Vec4F32FromBits(bb & absMask))) & expMask; - const Vec4F32 down = Vec4F32FromBits(Vec4S32::Splat((int)(269u << 23)) - e); - const Vec4F32 up = Vec4F32FromBits(e - Vec4S32::Splat(15 << 23)); - const Vec4F32 r = Vec4F32FromS32(Vec4S32FromF32(za * down) + Vec4S32FromF32(zb * down)) * up; - const Vec4S32 small = e.CompareLt(Vec4S32::Splat(16 << 23)); - if (AnyCompareBitsSet(small)) - return GEAdd4Small(a, b, small, r); - return r; -} - // Four de Casteljau evaluations (GEBezierEval), with the last two points for the tangents. static inline Vec4F32 GEBezierEval4(const Vec4F32 p[4], Vec4S32 k, Vec4S32 k0, Vec4S32 k256, Vec4F32 *ab = nullptr, Vec4F32 *bc = nullptr) { const Vec4F32 a = GELerp4(p[0], p[1], k, k0, k256); diff --git a/GPU/Software/GEMath.cpp b/GPU/Software/GEMath.cpp index 54322390fb..c5d9ce49bd 100644 --- a/GPU/Software/GEMath.cpp +++ b/GPU/Software/GEMath.cpp @@ -122,6 +122,34 @@ float GERsqrt(float d) { return ScaleByPow2((float)q, -16 - (e >> 1), 0); } +Vec4F32 GEAdd4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result) { + alignas(16) float av[4], bv[4], rv[4]; + alignas(16) int lv[4]; + a.Store(av); + b.Store(bv); + result.Store(rv); + lanes.Store(lv); + for (int i = 0; i < 4; ++i) { + if (lv[i]) + rv[i] = GEAdd(av[i], bv[i]); + } + return Vec4F32::Load(rv); +} + +Vec4F32 GEMulFloat24x4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result) { + alignas(16) float av[4], bv[4], rv[4]; + alignas(16) int lv[4]; + a.Store(av); + b.Store(bv); + result.Store(rv); + lanes.Store(lv); + for (int i = 0; i < 4; ++i) { + if (lv[i]) + rv[i] = ProductToFloat24((double)TruncateToFloat24(av[i]) * TruncateToFloat24(bv[i])); + } + return Vec4F32::Load(rv); +} + float GEAddFloat24(float a, float b) { return TruncateToFloat24(GEAdd(a, b)); } diff --git a/GPU/Software/GEMath.h b/GPU/Software/GEMath.h index 843708e74c..be5b24d7eb 100644 --- a/GPU/Software/GEMath.h +++ b/GPU/Software/GEMath.h @@ -240,6 +240,72 @@ inline Vec4F32 GERowSum4(Vec4F32 a, const Vec4F32 b[4]) { } float GEAddFloat24(float a, float b); + +// GEAdd4's lanes that it can't do itself: the larger exponent field from 1 to 15 (values under 2^-111) or +// 255 (inf and NaN). +Vec4F32 GEAdd4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result); + +// Four of GEAdd at once: each term goes to 16-bit fixed point at the larger exponent by scaling with a power +// of two and truncating toward zero, which drops what GEAdd's masks clear, then the sum is exact. +inline Vec4F32 GEAdd4(Vec4F32 a, Vec4F32 b) { + const Vec4S32 expMask = Vec4S32::Splat(0x7F800000); + const Vec4S32 ba = Vec4S32FromBits(a); + const Vec4S32 bb = Vec4S32FromBits(b); + const Vec4S32 ea = ba & expMask; + const Vec4S32 eb = bb & expMask; + // Denormals count as zero. + const Vec4F32 za = Vec4F32FromBits(ba.AndNot(ea.CompareEq(Vec4S32::Zero()))); + const Vec4F32 zb = Vec4F32FromBits(bb.AndNot(eb.CompareEq(Vec4S32::Zero()))); + // The larger exponent field, as integers: a float maximum's NaN handling differs between platforms. + const Vec4S32 e = ea.Max(eb); + // 2^(142 - e) and 2^(e - 142), e being the biased exponent. + const Vec4F32 down = Vec4F32FromBits(Vec4S32::Splat((int)(269u << 23)) - e); + const Vec4F32 up = Vec4F32FromBits(e - Vec4S32::Splat(15 << 23)); + // Both zero (or denormal) makes zero. + const Vec4S32 none = e.CompareEq(Vec4S32::Zero()); + const Vec4F32 sum = Vec4F32FromS32(Vec4S32FromF32(za * down) + Vec4S32FromF32(zb * down)) * up; + const Vec4F32 result = Vec4F32FromBits(Vec4S32FromBits(sum).AndNot(none)); + const Vec4S32 outside = e.CompareLt(Vec4S32::Splat(16 << 23)).AndNot(none) | e.CompareEq(expMask); + if (AnyCompareBitsSet(outside)) + return GEAdd4Fallback(a, b, outside, result); + return result; +} + +// GEMulFloat24x4's lanes that it can't do itself: a denormal, inf or NaN operand, or a result outside the +// normal range. +Vec4F32 GEMulFloat24x4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result); + +// Per lane, the product of a and b truncated to float24, as ProductToFloat24((double)a * b) with both +// truncated to float24 first, except that a zero operand gives +0. With 15 fraction bits each, the +// significand product truncated to 17 bits is 32768 + a' + b' + (a' b' >> 15), as in GERowSum4. +inline Vec4F32 GEMulFloat24x4(Vec4F32 a, Vec4F32 b) { + const Vec4S32 expMask = Vec4S32::Splat(0x7F800000); + const Vec4S32 fracMask = Vec4S32::Splat(0x7FFF); + const Vec4S32 aBits = Vec4S32FromBits(a) & Vec4S32::Splat((int)0xFFFFFF00); + const Vec4S32 bBits = Vec4S32FromBits(b) & Vec4S32::Splat((int)0xFFFFFF00); + const Vec4S32 ea = (aBits & expMask).Shr<23>(); + const Vec4S32 eb = (bBits & expMask).Shr<23>(); + const Vec4S32 af = aBits.Shr<8>() & fracMask; + const Vec4S32 bf = bBits.Shr<8>() & fracMask; + const Vec4S32 prod = af + Vec4S32::Splat(32768) + bf + bf.Mul16(af).Shr<15>(); + // prod * 2^(ea + eb - 254 - 15), by adding to the exponent of the exact float(prod), then truncated. + const Vec4S32 lsb = ea + eb - Vec4S32::Splat(254 + 15); + const Vec4S32 fbits = Vec4S32FromBits(Vec4F32FromS32(prod)); + const Vec4S32 resultExp = (fbits & expMask).Shr<23>() + lsb; + const Vec4S32 sign = (aBits ^ bBits) & Vec4S32::Splat((int)0x80000000); + const Vec4S32 zero = (aBits & Vec4S32::Splat(0x7FFFFFFF)).CompareEq(Vec4S32::Zero()) | (bBits & Vec4S32::Splat(0x7FFFFFFF)).CompareEq(Vec4S32::Zero()); + const Vec4F32 result = Vec4F32FromBits((((fbits + lsb.Shl<23>()) & Vec4S32::Splat((int)0xFFFFFF00)) | sign).AndNot(zero)); + const Vec4S32 special = ea.CompareEq(Vec4S32::Zero()) | eb.CompareEq(Vec4S32::Zero()) | ea.CompareEq(Vec4S32::Splat(255)) | eb.CompareEq(Vec4S32::Splat(255)); + const Vec4S32 outside = (special | resultExp.CompareLt(Vec4S32::Splat(1)) | resultExp.CompareGt(Vec4S32::Splat(254))).AndNot(zero); + if (AnyCompareBitsSet(outside)) + return GEMulFloat24x4Fallback(a, b, outside, result); + return result; +} + +// Four of GEAddFloat24 at once. +inline Vec4F32 GEAddFloat24x4(Vec4F32 a, Vec4F32 b) { + return Vec4F32FromBits(Vec4S32FromBits(GEAdd4(a, b)) & Vec4S32::Splat((int)0xFFFFFF00)); +} float GEDot(const Vec3f &a, const Vec3f &b); float GENormalize(Vec3f &v); diff --git a/GPU/Software/TransformUnit.cpp b/GPU/Software/TransformUnit.cpp index 8663d07590..8c566bc6c2 100644 --- a/GPU/Software/TransformUnit.cpp +++ b/GPU/Software/TransformUnit.cpp @@ -607,49 +607,57 @@ static float GEMorphComponent(const VertexDecoder &dec, const u8 *in, int off, i static void ApplyGESkinning(u8 *decoded, const VertexDecoder &dec, const u8 *raw, int count) { const DecVtxFormat &fmt = dec.GetDecVtxFmt(); const int nweights = dec.nweights; + // The x, y and z components go in lanes 0-2. Each one is a chain of GEAdds in a fixed order: per bone, the + // translation, then the three products. GEAdd drops a GEProduct's 17th bit, so float24 products (one-term + // row sums) give the same sums. + auto product = [](float a, Vec4F32 b) { + return GERowSum4<1>(Vec4F32::Splat(a), &b); + }; for (int v = 0; v < count; ++v) { const u8 *in = raw + v * dec.VertexSize(); u8 *out = decoded + v * fmt.stride; auto component = [&](int off, int c, auto read) { return dec.morphcount > 1 ? GEMorphComponent(dec, in, off, c, read) : read(in + off, c); }; - float bones[8][12]; + // Each bone's rows (matrix entries 3j..3j+2, the translation j = 3), times its weight. Lane 3 reads past + // the row (and past the matrices, for the last translation). + Vec4F32 bones[8][4]; + bool used[8]; for (int b = 0; b < nweights; ++b) { const float w = TruncateToFloat24(component(dec.weightoff, b, [&](const u8 *p, int i) { return ReadRawWeight(p, dec.weighttype, i); })); - for (int k = 0; k < 12; ++k) { - const float m = gstate.boneMatrix[b * 12 + k]; - bones[b][k] = m == 0.0f || w == 0.0f ? 0.0f : ProductToFloat24((double)TruncateToFloat24(m) * w); + used[b] = w != 0.0f; + if (!used[b]) + continue; + const float *m = gstate.boneMatrix + b * 12; + for (int j = 0; j < 4; ++j) { + bones[b][j] = GEMulFloat24x4(Vec4F32::Splat(w), Vec4F32::Load(m + 3 * j).WithLane3Zero()); } } - auto skin = [&](const float src[3], bool translate, float dst[3]) { - for (int c = 0; c < 3; ++c) { - float acc = 0.0f; - auto add = [&](float term) { - if (term != 0.0f) - acc = acc == 0.0f ? TruncateToFloat24(term) : TruncateToFloat24(GEAdd(acc, term)); - }; - for (int b = 0; b < nweights; ++b) { - if (translate) - add(bones[b][9 + c]); - for (int j = 0; j < 3; ++j) - add(GEProduct(TruncateToFloat24(src[j]), bones[b][j * 3 + c]).Value()); - } - dst[c] = acc; + auto skin = [&](const float src[3], bool translate, u8 *dst) { + Vec4F32 acc = Vec4F32::Zero(); + for (int b = 0; b < nweights; ++b) { + if (!used[b]) + continue; + if (translate) + acc = GEAddFloat24x4(acc, bones[b][3]); + for (int j = 0; j < 3; ++j) + acc = GEAddFloat24x4(acc, product(src[j], bones[b][j])); } + alignas(16) float result[4]; + acc.Store(result); + memcpy(dst, result, 3 * sizeof(float)); }; if (dec.pos) { - float pos[3], skinned[3]; + float pos[3]; for (int i = 0; i < 3; ++i) pos[i] = component(dec.posoff, i, [&](const u8 *p, int c) { return ReadRawComponent(p, dec.pos, c); }); - skin(pos, true, skinned); - memcpy(out + fmt.posoff, skinned, sizeof(skinned)); + skin(pos, true, out + fmt.posoff); } if (dec.nrm && fmt.nrmfmt == DEC_FLOAT_3) { - float nrm[3], skinned[3]; + float nrm[3]; for (int i = 0; i < 3; ++i) nrm[i] = component(dec.nrmoff, i, [&](const u8 *p, int c) { return ReadRawComponent(p, dec.nrm, c); }); - skin(nrm, false, skinned); - memcpy(out + fmt.nrmoff, skinned, sizeof(skinned)); + skin(nrm, false, out + fmt.nrmoff); } } } diff --git a/unittest/TestGEMath.cpp b/unittest/TestGEMath.cpp index b85c573fc3..fd4052e4f9 100644 --- a/unittest/TestGEMath.cpp +++ b/unittest/TestGEMath.cpp @@ -107,6 +107,89 @@ static bool TestGEDot() { return true; } +// GEAdd4 against GEAdd, bit for bit: any bits (not just float24s), close and distant exponents, signs, +// zeros, denormals, inf and NaN. +static bool TestGEAdd4() { + uint32_t state = 777; + auto next = [&]() { + state = state * 1664525u + 1013904223u; + return state; + }; + auto randomBits = [&](int center, int spread) { + const uint32_t r = next(); + uint32_t e; + switch (r & 15) { + case 0: e = 0; break; + case 1: e = 255; break; + default: e = (uint32_t)std::clamp(center + (int)(next() % (2 * spread + 1)) - spread, 1, 254); break; + } + const uint32_t bits = (e << 23) | (next() & 0x007FFFFF) | ((r & 16) ? 0x80000000 : 0); + float f; + memcpy(&f, &bits, sizeof(f)); + return f; + }; + for (int i = 0; i < 200000; ++i) { + const int spread = (i % 5 == 0) ? 120 : 20; + const int center = (i % 7 == 0) ? (int)(next() % 254) + 1 : 127; + alignas(16) float a[4], b[4], got[4]; + for (int l = 0; l < 4; ++l) { + a[l] = randomBits(center, spread); + b[l] = randomBits(center, spread); + } + GEAdd4(Vec4F32::Load(a), Vec4F32::Load(b)).Store(got); + for (int l = 0; l < 4; ++l) { + const float want = GEAdd(a[l], b[l]); + if (memcmp(&got[l], &want, sizeof(float)) != 0) { + printf("GEAdd4: %a + %a = %a, want %a\n", a[l], b[l], got[l], want); + return false; + } + } + } + return true; +} + +// GEMulFloat24x4 against ProductToFloat24, bit for bit, on the same kind of bits as TestGEAdd4. +static bool TestGEMulFloat24x4() { + uint32_t state = 4321; + auto next = [&]() { + state = state * 1664525u + 1013904223u; + return state; + }; + auto randomBits = [&](int center, int spread) { + const uint32_t r = next(); + uint32_t e; + switch (r & 15) { + case 0: e = 0; break; + case 1: e = 255; break; + default: e = (uint32_t)std::clamp(center + (int)(next() % (2 * spread + 1)) - spread, 1, 254); break; + } + const uint32_t mant = (r & 32) ? 0 : (next() & 0x007FFFFF); + const uint32_t bits = (e << 23) | mant | ((r & 16) ? 0x80000000 : 0); + float f; + memcpy(&f, &bits, sizeof(f)); + return f; + }; + for (int i = 0; i < 200000; ++i) { + const int spread = (i % 5 == 0) ? 120 : 20; + const int center = (i % 7 == 0) ? (int)(next() % 254) + 1 : 127; + alignas(16) float a[4], b[4], got[4]; + for (int l = 0; l < 4; ++l) { + a[l] = randomBits(center, spread); + b[l] = randomBits(center, spread); + } + GEMulFloat24x4(Vec4F32::Load(a), Vec4F32::Load(b)).Store(got); + for (int l = 0; l < 4; ++l) { + const float ta = TruncateToFloat24(a[l]), tb = TruncateToFloat24(b[l]); + const float want = ta == 0.0f || tb == 0.0f ? 0.0f : ProductToFloat24((double)ta * tb); + if (memcmp(&got[l], &want, sizeof(float)) != 0) { + printf("GEMulFloat24x4: %a * %a = %a, want %a\n", a[l], b[l], got[l], want); + return false; + } + } + } + return true; +} + // GERowSum4 against GERowSum, bit for bit, on random float24s: mixed signs, zeros and denormals, exponents // close together (so the terms' alignment matters) and far apart, including ones past the vector path. static bool TestGERowSum4() { @@ -318,6 +401,8 @@ bool TestGEMath() { ok = TestGEAdd() && ok; ok = TestGEDot() && ok; ok = TestGERowSum4() && ok; + ok = TestGEAdd4() && ok; + ok = TestGEMulFloat24x4() && ok; ok = TestGESetupRecip() && ok; ok = TestGELog16() && ok; ok = TestTexCoordPrecision() && ok;