From ce47944216dbb0a8ae2fa0cbc009a6fec48e4470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 10 Oct 2026 10:25:14 +0200 Subject: [PATCH] SoftGPU: Gather a span's UV plane values in registers GEUVSpan stored the four 64-bit plane values, and the reciprocal table's entries, to arrays one by one and loaded them as vectors, a store forwarding stall each time, which took most of its time on x86. CrossSIMD gets LoadS64Low (the low halves of four 64-bit values) and LoadPairs (four pairs of 32-bit values split into two vectors) to do it without memory. Wipeout runs about 2% faster. Co-Authored-By: Claude Opus 5.5 (1M context) --- Common/Math/CrossSIMD.h | 45 ++++++++++++++++++++++++++++++++++++++ GPU/Software/GEMath.h | 20 +++++++---------- unittest/TestCrossSIMD.cpp | 27 +++++++++++++++++++++++ 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/Common/Math/CrossSIMD.h b/Common/Math/CrossSIMD.h index 4536edc7df..18a7526d47 100644 --- a/Common/Math/CrossSIMD.h +++ b/Common/Math/CrossSIMD.h @@ -125,6 +125,20 @@ struct Vec4S32 { static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ _mm_load_si128((const __m128i *)src) }; } // Four 16-bit values, zero extended. static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ _mm_unpacklo_epi16(_mm_loadl_epi64((const __m128i *)src), _mm_setzero_si128()) }; } + // The low 32 bits of four 64-bit values. + static Vec4S32 LoadS64Low(const int64_t *src) { + const __m128 a = _mm_castsi128_ps(_mm_loadu_si128((const __m128i *)src)); + const __m128 b = _mm_castsi128_ps(_mm_loadu_si128((const __m128i *)(src + 2))); + return Vec4S32{ _mm_castps_si128(_mm_shuffle_ps(a, b, _MM_SHUFFLE(2, 0, 2, 0))) }; + } + // Four pairs of 32-bit values: lo gets each pair's first, hi its second. + static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) { + const __m128 p01 = _mm_castsi128_ps(_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)p0), _mm_loadl_epi64((const __m128i *)p1))); + const __m128 p23 = _mm_castsi128_ps(_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)p2), _mm_loadl_epi64((const __m128i *)p3))); + lo.v = _mm_castps_si128(_mm_shuffle_ps(p01, p23, _MM_SHUFFLE(2, 0, 2, 0))); + hi.v = _mm_castps_si128(_mm_shuffle_ps(p01, p23, _MM_SHUFFLE(3, 1, 3, 1))); + } + void Store(int *dst) { _mm_storeu_si128((__m128i *)dst, v); } void Store2(int *dst) { _mm_storel_epi64((__m128i *)dst, v); } void StoreAligned(int *dst) { _mm_store_si128((__m128i *)dst, v);} @@ -616,6 +630,17 @@ struct Vec4S32 { static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ vld1q_s32(src) }; } // Four 16-bit values, zero extended. static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ vreinterpretq_s32_u32(vmovl_u16(vld1_u16(src))) }; } + // The low 32 bits of four 64-bit values. + static Vec4S32 LoadS64Low(const int64_t *src) { + return Vec4S32{ vcombine_s32(vmovn_s64(vld1q_s64(src)), vmovn_s64(vld1q_s64(src + 2))) }; + } + // Four pairs of 32-bit values: lo gets each pair's first, hi its second. + static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) { + const int32x4x2_t uz = vuzpq_s32(vcombine_s32(vld1_s32(p0), vld1_s32(p1)), vcombine_s32(vld1_s32(p2), vld1_s32(p3))); + lo.v = uz.val[0]; + hi.v = uz.val[1]; + } + void Store(int *dst) { vst1q_s32(dst, v); } void Store2(int *dst) { vst1_s32(dst, vget_low_s32(v)); } void StoreAligned(int *dst) { vst1q_s32(dst, v); } @@ -1171,6 +1196,18 @@ struct Vec4S32 { static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ __lsx_vld(src, 0) }; } // Four 16-bit values, zero extended. static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ __lsx_vilvl_h(__lsx_vldi(0), __lsx_vldrepl_d(src, 0)) }; } + // The low 32 bits of four 64-bit values. + static Vec4S32 LoadS64Low(const int64_t *src) { + return Vec4S32{ __lsx_vpickev_w(__lsx_vld(src + 2, 0), __lsx_vld(src, 0)) }; + } + // Four pairs of 32-bit values: lo gets each pair's first, hi its second. + static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) { + const __m128i p01 = __lsx_vilvl_d(__lsx_vldrepl_d(p1, 0), __lsx_vldrepl_d(p0, 0)); + const __m128i p23 = __lsx_vilvl_d(__lsx_vldrepl_d(p3, 0), __lsx_vldrepl_d(p2, 0)); + lo.v = __lsx_vpickev_w(p23, p01); + hi.v = __lsx_vpickod_w(p23, p01); + } + void Store(int *dst) { __lsx_vst(v, dst, 0); } void Store2(int *dst) { __lsx_vstelm_d(v, dst, 0, 0); } void StoreAligned(int *dst) { __lsx_vst(v, dst, 0); } @@ -1645,6 +1682,14 @@ struct Vec4S32 { static Vec4S32 LoadAligned(const int *src) { return Load(src); } // Four 16-bit values, zero extended. static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ { src[0], src[1], src[2], src[3] } }; } + // The low 32 bits of four 64-bit values. + static Vec4S32 LoadS64Low(const int64_t *src) { return Vec4S32{ { (int32_t)src[0], (int32_t)src[1], (int32_t)src[2], (int32_t)src[3] } }; } + // Four pairs of 32-bit values: lo gets each pair's first, hi its second. + static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) { + lo = Vec4S32{ { p0[0], p1[0], p2[0], p3[0] } }; + hi = Vec4S32{ { p0[1], p1[1], p2[1], p3[1] } }; + } + void Store(int *dst) { memcpy(dst, v, sizeof(v)); } void Store2(int *dst) { memcpy(dst, v, sizeof(v[0]) * 2); } void StoreAligned(int *dst) { memcpy(dst, v, sizeof(v)); } diff --git a/GPU/Software/GEMath.h b/GPU/Software/GEMath.h index 9185548bce..a2a470a885 100644 --- a/GPU/Software/GEMath.h +++ b/GPU/Software/GEMath.h @@ -495,15 +495,14 @@ inline bool GEUVSpanCore(Vec4S32 qv, Vec4S32 sv, Vec4S32 tv, int expQ, int expS, const Vec4S32 qe = (qb & expMask).Shr<23>() - Vec4S32::Splat(127); if constexpr (check) bad = bad | qe.CompareGt(Vec4S32::Splat(125)); - alignas(16) int32_t qbits[4], segB[4], segM[4]; + alignas(16) int32_t qbits[4]; Vec4S32(qb).Store(qbits); - for (int i = 0; i < 4; ++i) { - const GERecipSegment &seg = geRecipSegments[(qbits[i] >> 16) & 0x7F]; - segB[i] = seg.b; - segM[i] = seg.m; - } + // Gathered in registers: stored to an array one by one and loaded as a vector, they'd stall. + Vec4S32 segB, segM; + auto seg = [&](int i) { return &geRecipSegments[(qbits[i] >> 16) & 0x7F].b; }; + Vec4S32::LoadPairs(seg(0), seg(1), seg(2), seg(3), segB, segM); const Vec4S32 qi = qb.Shr<8>() & Vec4S32::Splat(0xFF); - const Vec4S32 recip = (Vec4S32::Load(segB).Shl<6>() + Vec4S32::Splat(63) + Vec4S32::Load(segM).Mul(qi)).Shr<7>(); + const Vec4S32 recip = (segB.Shl<6>() + Vec4S32::Splat(63) + segM.Mul(qi)).Shr<7>(); const Vec4S32 rb = Vec4S32FromBits(Vec4F32FromS32(recip)) - (Vec4S32::Splat(16) + qe).Shl<23>(); // GEUVProduct(x * r): the 16-bit significands' product, cut to 24 bits. @@ -545,15 +544,12 @@ inline bool GEUVSpanCore(Vec4S32 qv, Vec4S32 sv, Vec4S32 tv, int expQ, int expS, // From 64-bit plane values, checked. inline bool GEUVSpan(const int64_t qs[4], const int64_t ss[4], const int64_t ts[4], int expQ, int expS, int expT, float *s, float *t, float *q) { - alignas(16) int32_t vals[3][4]; for (int i = 0; i < 4; ++i) { if (qs[i] <= -(1 << 24) || qs[i] >= (1 << 24) || ss[i] <= -(1 << 24) || ss[i] >= (1 << 24) || ts[i] <= -(1 << 24) || ts[i] >= (1 << 24)) return false; - vals[0][i] = (int32_t)qs[i]; - vals[1][i] = (int32_t)ss[i]; - vals[2][i] = (int32_t)ts[i]; } - return GEUVSpanCore(Vec4S32::Load(vals[0]), Vec4S32::Load(vals[1]), Vec4S32::Load(vals[2]), expQ, expS, expT, s, t, q); + // Picked in registers: storing them one by one and loading them as a vector stalls. + return GEUVSpanCore(Vec4S32::LoadS64Low(qs), Vec4S32::LoadS64Low(ss), Vec4S32::LoadS64Low(ts), expQ, expS, expT, s, t, q); } // Whether GEUVSpanCore covers any values below 2^24 with these plane exponents: the float24s, the diff --git a/unittest/TestCrossSIMD.cpp b/unittest/TestCrossSIMD.cpp index d4a47b21d2..56dd66e382 100644 --- a/unittest/TestCrossSIMD.cpp +++ b/unittest/TestCrossSIMD.cpp @@ -437,6 +437,33 @@ static bool TestVec4F32NaNInf() { static bool TestVec4F32Loads() { float result[4]; + // Load3One: three floats, lane 3 1.0f. + const float xyz[3] = { -1.5f, 2.0f, 1e20f }; + Vec4F32::Load3One(xyz).Store(result); + EXPECT_EQ_FLOAT(result[0], -1.5f); + EXPECT_EQ_FLOAT(result[1], 2.0f); + EXPECT_EQ_FLOAT(result[2], 1e20f); + EXPECT_EQ_FLOAT(result[3], 1.0f); + + // LoadS64Low: the low halves. + int iresult[4]; + const int64_t wide[4] = { 0x123456789LL, -2, (int64_t)0x7FFFFFFF << 1, -0x100000000LL + 7 }; + Vec4S32::LoadS64Low(wide).Store(iresult); + for (int i = 0; i < 4; i++) + EXPECT_EQ_INT(iresult[i], (int32_t)(uint32_t)(uint64_t)wide[i]); + + // LoadPairs: firsts and seconds. + const int32_t pairs[8] = { 1, -1, 2, -2, 3, -3, 4, -4 }; + Vec4S32 firsts, seconds; + Vec4S32::LoadPairs(pairs + 6, pairs, pairs + 4, pairs + 2, firsts, seconds); + const int32_t expectedFirsts[4] = { 4, 1, 3, 2 }; + firsts.Store(iresult); + for (int i = 0; i < 4; i++) + EXPECT_EQ_INT(iresult[i], expectedFirsts[i]); + seconds.Store(iresult); + for (int i = 0; i < 4; i++) + EXPECT_EQ_INT(iresult[i], -expectedFirsts[i]); + // LoadF24x3_One: three 24-bit values shifted up into floats, lane 3 forced to 1.0f. const uint32_t f24_values[4] = { 0x3F8000 >> 0, 0x400000, 0x3F0000, 0x123456 }; Vec4F32::LoadF24x3_One(f24_values).Store(result);