mirror of
https://github.com/hrydgard/ppsspp.git
synced 2026-10-14 15:00:30 +02:00
SoftGPU: Gather a span's UV plane values in registers
GEUVSpan stored the four 64-bit plane values, and the reciprocal table's entries, to arrays one by one and loaded them as vectors, a store forwarding stall each time, which took most of its time on x86. CrossSIMD gets LoadS64Low (the low halves of four 64-bit values) and LoadPairs (four pairs of 32-bit values split into two vectors) to do it without memory. Wipeout runs about 2% faster. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Henrik RydgårdandClaude Opus 5.5 committed
1 parent
716aa36ca8
commit
ce47944216
3 files changed
+80
-12
No files matched your search
@@ -125,6 +125,20 @@ struct Vec4S32 {
|
||||
static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ _mm_load_si128((const __m128i *)src) }; }
|
||||
// Four 16-bit values, zero extended.
|
||||
static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ _mm_unpacklo_epi16(_mm_loadl_epi64((const __m128i *)src), _mm_setzero_si128()) }; }
|
||||
// The low 32 bits of four 64-bit values.
|
||||
static Vec4S32 LoadS64Low(const int64_t *src) {
|
||||
const __m128 a = _mm_castsi128_ps(_mm_loadu_si128((const __m128i *)src));
|
||||
const __m128 b = _mm_castsi128_ps(_mm_loadu_si128((const __m128i *)(src + 2)));
|
||||
return Vec4S32{ _mm_castps_si128(_mm_shuffle_ps(a, b, _MM_SHUFFLE(2, 0, 2, 0))) };
|
||||
}
|
||||
// Four pairs of 32-bit values: lo gets each pair's first, hi its second.
|
||||
static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) {
|
||||
const __m128 p01 = _mm_castsi128_ps(_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)p0), _mm_loadl_epi64((const __m128i *)p1)));
|
||||
const __m128 p23 = _mm_castsi128_ps(_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)p2), _mm_loadl_epi64((const __m128i *)p3)));
|
||||
lo.v = _mm_castps_si128(_mm_shuffle_ps(p01, p23, _MM_SHUFFLE(2, 0, 2, 0)));
|
||||
hi.v = _mm_castps_si128(_mm_shuffle_ps(p01, p23, _MM_SHUFFLE(3, 1, 3, 1)));
|
||||
}
|
||||
|
||||
void Store(int *dst) { _mm_storeu_si128((__m128i *)dst, v); }
|
||||
void Store2(int *dst) { _mm_storel_epi64((__m128i *)dst, v); }
|
||||
void StoreAligned(int *dst) { _mm_store_si128((__m128i *)dst, v);}
|
||||
@@ -616,6 +630,17 @@ struct Vec4S32 {
|
||||
static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ vld1q_s32(src) }; }
|
||||
// Four 16-bit values, zero extended.
|
||||
static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ vreinterpretq_s32_u32(vmovl_u16(vld1_u16(src))) }; }
|
||||
// The low 32 bits of four 64-bit values.
|
||||
static Vec4S32 LoadS64Low(const int64_t *src) {
|
||||
return Vec4S32{ vcombine_s32(vmovn_s64(vld1q_s64(src)), vmovn_s64(vld1q_s64(src + 2))) };
|
||||
}
|
||||
// Four pairs of 32-bit values: lo gets each pair's first, hi its second.
|
||||
static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) {
|
||||
const int32x4x2_t uz = vuzpq_s32(vcombine_s32(vld1_s32(p0), vld1_s32(p1)), vcombine_s32(vld1_s32(p2), vld1_s32(p3)));
|
||||
lo.v = uz.val[0];
|
||||
hi.v = uz.val[1];
|
||||
}
|
||||
|
||||
void Store(int *dst) { vst1q_s32(dst, v); }
|
||||
void Store2(int *dst) { vst1_s32(dst, vget_low_s32(v)); }
|
||||
void StoreAligned(int *dst) { vst1q_s32(dst, v); }
|
||||
@@ -1171,6 +1196,18 @@ struct Vec4S32 {
|
||||
static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ __lsx_vld(src, 0) }; }
|
||||
// Four 16-bit values, zero extended.
|
||||
static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ __lsx_vilvl_h(__lsx_vldi(0), __lsx_vldrepl_d(src, 0)) }; }
|
||||
// The low 32 bits of four 64-bit values.
|
||||
static Vec4S32 LoadS64Low(const int64_t *src) {
|
||||
return Vec4S32{ __lsx_vpickev_w(__lsx_vld(src + 2, 0), __lsx_vld(src, 0)) };
|
||||
}
|
||||
// Four pairs of 32-bit values: lo gets each pair's first, hi its second.
|
||||
static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) {
|
||||
const __m128i p01 = __lsx_vilvl_d(__lsx_vldrepl_d(p1, 0), __lsx_vldrepl_d(p0, 0));
|
||||
const __m128i p23 = __lsx_vilvl_d(__lsx_vldrepl_d(p3, 0), __lsx_vldrepl_d(p2, 0));
|
||||
lo.v = __lsx_vpickev_w(p23, p01);
|
||||
hi.v = __lsx_vpickod_w(p23, p01);
|
||||
}
|
||||
|
||||
void Store(int *dst) { __lsx_vst(v, dst, 0); }
|
||||
void Store2(int *dst) { __lsx_vstelm_d(v, dst, 0, 0); }
|
||||
void StoreAligned(int *dst) { __lsx_vst(v, dst, 0); }
|
||||
@@ -1645,6 +1682,14 @@ struct Vec4S32 {
|
||||
static Vec4S32 LoadAligned(const int *src) { return Load(src); }
|
||||
// Four 16-bit values, zero extended.
|
||||
static Vec4S32 LoadU16(const uint16_t *src) { return Vec4S32{ { src[0], src[1], src[2], src[3] } }; }
|
||||
// The low 32 bits of four 64-bit values.
|
||||
static Vec4S32 LoadS64Low(const int64_t *src) { return Vec4S32{ { (int32_t)src[0], (int32_t)src[1], (int32_t)src[2], (int32_t)src[3] } }; }
|
||||
// Four pairs of 32-bit values: lo gets each pair's first, hi its second.
|
||||
static void LoadPairs(const int32_t *p0, const int32_t *p1, const int32_t *p2, const int32_t *p3, Vec4S32 &lo, Vec4S32 &hi) {
|
||||
lo = Vec4S32{ { p0[0], p1[0], p2[0], p3[0] } };
|
||||
hi = Vec4S32{ { p0[1], p1[1], p2[1], p3[1] } };
|
||||
}
|
||||
|
||||
void Store(int *dst) { memcpy(dst, v, sizeof(v)); }
|
||||
void Store2(int *dst) { memcpy(dst, v, sizeof(v[0]) * 2); }
|
||||
void StoreAligned(int *dst) { memcpy(dst, v, sizeof(v)); }
|
||||
|
||||
+8
-12
@@ -495,15 +495,14 @@ inline bool GEUVSpanCore(Vec4S32 qv, Vec4S32 sv, Vec4S32 tv, int expQ, int expS,
|
||||
const Vec4S32 qe = (qb & expMask).Shr<23>() - Vec4S32::Splat(127);
|
||||
if constexpr (check)
|
||||
bad = bad | qe.CompareGt(Vec4S32::Splat(125));
|
||||
alignas(16) int32_t qbits[4], segB[4], segM[4];
|
||||
alignas(16) int32_t qbits[4];
|
||||
Vec4S32(qb).Store(qbits);
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const GERecipSegment &seg = geRecipSegments[(qbits[i] >> 16) & 0x7F];
|
||||
segB[i] = seg.b;
|
||||
segM[i] = seg.m;
|
||||
}
|
||||
// Gathered in registers: stored to an array one by one and loaded as a vector, they'd stall.
|
||||
Vec4S32 segB, segM;
|
||||
auto seg = [&](int i) { return &geRecipSegments[(qbits[i] >> 16) & 0x7F].b; };
|
||||
Vec4S32::LoadPairs(seg(0), seg(1), seg(2), seg(3), segB, segM);
|
||||
const Vec4S32 qi = qb.Shr<8>() & Vec4S32::Splat(0xFF);
|
||||
const Vec4S32 recip = (Vec4S32::Load(segB).Shl<6>() + Vec4S32::Splat(63) + Vec4S32::Load(segM).Mul(qi)).Shr<7>();
|
||||
const Vec4S32 recip = (segB.Shl<6>() + Vec4S32::Splat(63) + segM.Mul(qi)).Shr<7>();
|
||||
const Vec4S32 rb = Vec4S32FromBits(Vec4F32FromS32(recip)) - (Vec4S32::Splat(16) + qe).Shl<23>();
|
||||
|
||||
// GEUVProduct(x * r): the 16-bit significands' product, cut to 24 bits.
|
||||
@@ -545,15 +544,12 @@ inline bool GEUVSpanCore(Vec4S32 qv, Vec4S32 sv, Vec4S32 tv, int expQ, int expS,
|
||||
|
||||
// From 64-bit plane values, checked.
|
||||
inline bool GEUVSpan(const int64_t qs[4], const int64_t ss[4], const int64_t ts[4], int expQ, int expS, int expT, float *s, float *t, float *q) {
|
||||
alignas(16) int32_t vals[3][4];
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (qs[i] <= -(1 << 24) || qs[i] >= (1 << 24) || ss[i] <= -(1 << 24) || ss[i] >= (1 << 24) || ts[i] <= -(1 << 24) || ts[i] >= (1 << 24))
|
||||
return false;
|
||||
vals[0][i] = (int32_t)qs[i];
|
||||
vals[1][i] = (int32_t)ss[i];
|
||||
vals[2][i] = (int32_t)ts[i];
|
||||
}
|
||||
return GEUVSpanCore<true>(Vec4S32::Load(vals[0]), Vec4S32::Load(vals[1]), Vec4S32::Load(vals[2]), expQ, expS, expT, s, t, q);
|
||||
// Picked in registers: storing them one by one and loading them as a vector stalls.
|
||||
return GEUVSpanCore<true>(Vec4S32::LoadS64Low(qs), Vec4S32::LoadS64Low(ss), Vec4S32::LoadS64Low(ts), expQ, expS, expT, s, t, q);
|
||||
}
|
||||
|
||||
// Whether GEUVSpanCore<false> covers any values below 2^24 with these plane exponents: the float24s, the
|
||||
|
||||
@@ -437,6 +437,33 @@ static bool TestVec4F32NaNInf() {
|
||||
static bool TestVec4F32Loads() {
|
||||
float result[4];
|
||||
|
||||
// Load3One: three floats, lane 3 1.0f.
|
||||
const float xyz[3] = { -1.5f, 2.0f, 1e20f };
|
||||
Vec4F32::Load3One(xyz).Store(result);
|
||||
EXPECT_EQ_FLOAT(result[0], -1.5f);
|
||||
EXPECT_EQ_FLOAT(result[1], 2.0f);
|
||||
EXPECT_EQ_FLOAT(result[2], 1e20f);
|
||||
EXPECT_EQ_FLOAT(result[3], 1.0f);
|
||||
|
||||
// LoadS64Low: the low halves.
|
||||
int iresult[4];
|
||||
const int64_t wide[4] = { 0x123456789LL, -2, (int64_t)0x7FFFFFFF << 1, -0x100000000LL + 7 };
|
||||
Vec4S32::LoadS64Low(wide).Store(iresult);
|
||||
for (int i = 0; i < 4; i++)
|
||||
EXPECT_EQ_INT(iresult[i], (int32_t)(uint32_t)(uint64_t)wide[i]);
|
||||
|
||||
// LoadPairs: firsts and seconds.
|
||||
const int32_t pairs[8] = { 1, -1, 2, -2, 3, -3, 4, -4 };
|
||||
Vec4S32 firsts, seconds;
|
||||
Vec4S32::LoadPairs(pairs + 6, pairs, pairs + 4, pairs + 2, firsts, seconds);
|
||||
const int32_t expectedFirsts[4] = { 4, 1, 3, 2 };
|
||||
firsts.Store(iresult);
|
||||
for (int i = 0; i < 4; i++)
|
||||
EXPECT_EQ_INT(iresult[i], expectedFirsts[i]);
|
||||
seconds.Store(iresult);
|
||||
for (int i = 0; i < 4; i++)
|
||||
EXPECT_EQ_INT(iresult[i], -expectedFirsts[i]);
|
||||
|
||||
// LoadF24x3_One: three 24-bit values shifted up into floats, lane 3 forced to 1.0f.
|
||||
const uint32_t f24_values[4] = { 0x3F8000 >> 0, 0x400000, 0x3F0000, 0x123456 };
|
||||
Vec4F32::LoadF24x3_One(f24_values).Store(result);
|
||||
|
||||
Reference in new issue
Block a user