SoftGPU: Skin the x, y and z components together in vector lanes

Skinning did each component's weighted bone entries and its chain of
GEAdds separately in scalar code. The components now go in lanes, with
GEMulFloat24x4 for the weighted entries, one-term row sums for the
products and GEAdd4 for the sums, in the same order. Checked against the
scalar version on every vertex of 143 skinned frame dumps.

GEMulFloat24x4 matches ProductToFloat24, so inf and NaN bone entries
behave as before. GEAdd4 handles two zero operands without its fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

GPU/Common/SplineCommon.cpp

@@ -685,38 +685,6 @@ struct GEBezierFixed {
 	}
 };
 
-static NO_INLINE Vec4F32 GEAdd4Small(Vec4F32 a, Vec4F32 b, Vec4S32 small, Vec4F32 r) {
-	alignas(16) float av[4], bv[4], rv[4];
-	alignas(16) int sv[4];
-	a.Store(av);
-	b.Store(bv);
-	r.Store(rv);
-	small.Store(sv);
-	for (int i = 0; i < 4; ++i) {
-		if (sv[i])
-			rv[i] = GEAdd(av[i], bv[i]);
-	}
-	return Vec4F32::Load(rv);
-}
-
-// Four of GEAdd at once: both terms truncated to the precision of the larger one, then added exactly.
-static __forceinline Vec4F32 GEAdd4(Vec4F32 a, Vec4F32 b) {
-	const Vec4S32 expMask = Vec4S32::Splat(0x7F800000);
-	const Vec4S32 absMask = Vec4S32::Splat(0x7FFFFFFF);
-	const Vec4S32 ba = Vec4S32FromBits(a);
-	const Vec4S32 bb = Vec4S32FromBits(b);
-	const Vec4F32 za = Select((ba & expMask).CompareEq(Vec4S32::Zero()), Vec4F32::Zero(), a);
-	const Vec4F32 zb = Select((bb & expMask).CompareEq(Vec4S32::Zero()), Vec4F32::Zero(), b);
-	const Vec4S32 e = Vec4S32FromBits(Vec4F32FromBits(ba & absMask).Max(Vec4F32FromBits(bb & absMask))) & expMask;
-	const Vec4F32 down = Vec4F32FromBits(Vec4S32::Splat((int)(269u << 23)) - e);
-	const Vec4F32 up = Vec4F32FromBits(e - Vec4S32::Splat(15 << 23));
-	const Vec4F32 r = Vec4F32FromS32(Vec4S32FromF32(za * down) + Vec4S32FromF32(zb * down)) * up;
-	const Vec4S32 small = e.CompareLt(Vec4S32::Splat(16 << 23));
-	if (AnyCompareBitsSet(small))
-		return GEAdd4Small(a, b, small, r);
-	return r;
-}
-
 // Four de Casteljau evaluations (GEBezierEval), with the last two points for the tangents.
 static inline Vec4F32 GEBezierEval4(const Vec4F32 p[4], Vec4S32 k, Vec4S32 k0, Vec4S32 k256, Vec4F32 *ab = nullptr, Vec4F32 *bc = nullptr) {
 	const Vec4F32 a = GELerp4(p[0], p[1], k, k0, k256);

GPU/Software/GEMath.cpp

@@ -122,6 +122,34 @@ float GERsqrt(float d) {
 	return ScaleByPow2((float)q, -16 - (e >> 1), 0);
 }
 
+Vec4F32 GEAdd4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result) {
+	alignas(16) float av[4], bv[4], rv[4];
+	alignas(16) int lv[4];
+	a.Store(av);
+	b.Store(bv);
+	result.Store(rv);
+	lanes.Store(lv);
+	for (int i = 0; i < 4; ++i) {
+		if (lv[i])
+			rv[i] = GEAdd(av[i], bv[i]);
+	}
+	return Vec4F32::Load(rv);
+}
+
+Vec4F32 GEMulFloat24x4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result) {
+	alignas(16) float av[4], bv[4], rv[4];
+	alignas(16) int lv[4];
+	a.Store(av);
+	b.Store(bv);
+	result.Store(rv);
+	lanes.Store(lv);
+	for (int i = 0; i < 4; ++i) {
+		if (lv[i])
+			rv[i] = ProductToFloat24((double)TruncateToFloat24(av[i]) * TruncateToFloat24(bv[i]));
+	}
+	return Vec4F32::Load(rv);
+}
+
 float GEAddFloat24(float a, float b) {
 	return TruncateToFloat24(GEAdd(a, b));
 }

GPU/Software/GEMath.h

@@ -240,6 +240,72 @@ inline Vec4F32 GERowSum4(Vec4F32 a, const Vec4F32 b[4]) {
 }
 
 float GEAddFloat24(float a, float b);
+
+// GEAdd4's lanes that it can't do itself: the larger exponent field from 1 to 15 (values under 2^-111) or
+// 255 (inf and NaN).
+Vec4F32 GEAdd4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result);
+
+// Four of GEAdd at once: each term goes to 16-bit fixed point at the larger exponent by scaling with a power
+// of two and truncating toward zero, which drops what GEAdd's masks clear, then the sum is exact.
+inline Vec4F32 GEAdd4(Vec4F32 a, Vec4F32 b) {
+	const Vec4S32 expMask = Vec4S32::Splat(0x7F800000);
+	const Vec4S32 ba = Vec4S32FromBits(a);
+	const Vec4S32 bb = Vec4S32FromBits(b);
+	const Vec4S32 ea = ba & expMask;
+	const Vec4S32 eb = bb & expMask;
+	// Denormals count as zero.
+	const Vec4F32 za = Vec4F32FromBits(ba.AndNot(ea.CompareEq(Vec4S32::Zero())));
+	const Vec4F32 zb = Vec4F32FromBits(bb.AndNot(eb.CompareEq(Vec4S32::Zero())));
+	// The larger exponent field, as integers: a float maximum's NaN handling differs between platforms.
+	const Vec4S32 e = ea.Max(eb);
+	// 2^(142 - e) and 2^(e - 142), e being the biased exponent.
+	const Vec4F32 down = Vec4F32FromBits(Vec4S32::Splat((int)(269u << 23)) - e);
+	const Vec4F32 up = Vec4F32FromBits(e - Vec4S32::Splat(15 << 23));
+	// Both zero (or denormal) makes zero.
+	const Vec4S32 none = e.CompareEq(Vec4S32::Zero());
+	const Vec4F32 sum = Vec4F32FromS32(Vec4S32FromF32(za * down) + Vec4S32FromF32(zb * down)) * up;
+	const Vec4F32 result = Vec4F32FromBits(Vec4S32FromBits(sum).AndNot(none));
+	const Vec4S32 outside = e.CompareLt(Vec4S32::Splat(16 << 23)).AndNot(none) | e.CompareEq(expMask);
+	if (AnyCompareBitsSet(outside))
+		return GEAdd4Fallback(a, b, outside, result);
+	return result;
+}
+
+// GEMulFloat24x4's lanes that it can't do itself: a denormal, inf or NaN operand, or a result outside the
+// normal range.
+Vec4F32 GEMulFloat24x4Fallback(Vec4F32 a, Vec4F32 b, Vec4S32 lanes, Vec4F32 result);
+
+// Per lane, the product of a and b truncated to float24, as ProductToFloat24((double)a * b) with both
+// truncated to float24 first, except that a zero operand gives +0. With 15 fraction bits each, the
+// significand product truncated to 17 bits is 32768 + a' + b' + (a' b' >> 15), as in GERowSum4.
+inline Vec4F32 GEMulFloat24x4(Vec4F32 a, Vec4F32 b) {
+	const Vec4S32 expMask = Vec4S32::Splat(0x7F800000);
+	const Vec4S32 fracMask = Vec4S32::Splat(0x7FFF);
+	const Vec4S32 aBits = Vec4S32FromBits(a) & Vec4S32::Splat((int)0xFFFFFF00);
+	const Vec4S32 bBits = Vec4S32FromBits(b) & Vec4S32::Splat((int)0xFFFFFF00);
+	const Vec4S32 ea = (aBits & expMask).Shr<23>();
+	const Vec4S32 eb = (bBits & expMask).Shr<23>();
+	const Vec4S32 af = aBits.Shr<8>() & fracMask;
+	const Vec4S32 bf = bBits.Shr<8>() & fracMask;
+	const Vec4S32 prod = af + Vec4S32::Splat(32768) + bf + bf.Mul16(af).Shr<15>();
+	// prod * 2^(ea + eb - 254 - 15), by adding to the exponent of the exact float(prod), then truncated.
+	const Vec4S32 lsb = ea + eb - Vec4S32::Splat(254 + 15);
+	const Vec4S32 fbits = Vec4S32FromBits(Vec4F32FromS32(prod));
+	const Vec4S32 resultExp = (fbits & expMask).Shr<23>() + lsb;
+	const Vec4S32 sign = (aBits ^ bBits) & Vec4S32::Splat((int)0x80000000);
+	const Vec4S32 zero = (aBits & Vec4S32::Splat(0x7FFFFFFF)).CompareEq(Vec4S32::Zero()) | (bBits & Vec4S32::Splat(0x7FFFFFFF)).CompareEq(Vec4S32::Zero());
+	const Vec4F32 result = Vec4F32FromBits((((fbits + lsb.Shl<23>()) & Vec4S32::Splat((int)0xFFFFFF00)) | sign).AndNot(zero));
+	const Vec4S32 special = ea.CompareEq(Vec4S32::Zero()) | eb.CompareEq(Vec4S32::Zero()) | ea.CompareEq(Vec4S32::Splat(255)) | eb.CompareEq(Vec4S32::Splat(255));
+	const Vec4S32 outside = (special | resultExp.CompareLt(Vec4S32::Splat(1)) | resultExp.CompareGt(Vec4S32::Splat(254))).AndNot(zero);
+	if (AnyCompareBitsSet(outside))
+		return GEMulFloat24x4Fallback(a, b, outside, result);
+	return result;
+}
+
+// Four of GEAddFloat24 at once.
+inline Vec4F32 GEAddFloat24x4(Vec4F32 a, Vec4F32 b) {
+	return Vec4F32FromBits(Vec4S32FromBits(GEAdd4(a, b)) & Vec4S32::Splat((int)0xFFFFFF00));
+}
 float GEDot(const Vec3f &a, const Vec3f &b);
 float GENormalize(Vec3f &v);
 

GPU/Software/TransformUnit.cpp

@@ -607,49 +607,57 @@ static float GEMorphComponent(const VertexDecoder &dec, const u8 *in, int off, i
 static void ApplyGESkinning(u8 *decoded, const VertexDecoder &dec, const u8 *raw, int count) {
 	const DecVtxFormat &fmt = dec.GetDecVtxFmt();
 	const int nweights = dec.nweights;
+	// The x, y and z components go in lanes 0-2. Each one is a chain of GEAdds in a fixed order: per bone, the
+	// translation, then the three products. GEAdd drops a GEProduct's 17th bit, so float24 products (one-term
+	// row sums) give the same sums.
+	auto product = [](float a, Vec4F32 b) {
+		return GERowSum4<1>(Vec4F32::Splat(a), &b);
+	};
 	for (int v = 0; v < count; ++v) {
 		const u8 *in = raw + v * dec.VertexSize();
 		u8 *out = decoded + v * fmt.stride;
 		auto component = [&](int off, int c, auto read) {
 			return dec.morphcount > 1 ? GEMorphComponent(dec, in, off, c, read) : read(in + off, c);
 		};
-		float bones[8][12];
+		// Each bone's rows (matrix entries 3j..3j+2, the translation j = 3), times its weight. Lane 3 reads past
+		// the row (and past the matrices, for the last translation).
+		Vec4F32 bones[8][4];
+		bool used[8];
 		for (int b = 0; b < nweights; ++b) {
 			const float w = TruncateToFloat24(component(dec.weightoff, b, [&](const u8 *p, int i) { return ReadRawWeight(p, dec.weighttype, i); }));
-			for (int k = 0; k < 12; ++k) {
-				const float m = gstate.boneMatrix[b * 12 + k];
-				bones[b][k] = m == 0.0f || w == 0.0f ? 0.0f : ProductToFloat24((double)TruncateToFloat24(m) * w);
+			used[b] = w != 0.0f;
+			if (!used[b])
+				continue;
+			const float *m = gstate.boneMatrix + b * 12;
+			for (int j = 0; j < 4; ++j) {
+				bones[b][j] = GEMulFloat24x4(Vec4F32::Splat(w), Vec4F32::Load(m + 3 * j).WithLane3Zero());
 			}
 		}
-		auto skin = [&](const float src[3], bool translate, float dst[3]) {
-			for (int c = 0; c < 3; ++c) {
-				float acc = 0.0f;
-				auto add = [&](float term) {
-					if (term != 0.0f)
-						acc = acc == 0.0f ? TruncateToFloat24(term) : TruncateToFloat24(GEAdd(acc, term));
-				};
-				for (int b = 0; b < nweights; ++b) {
-					if (translate)
-						add(bones[b][9 + c]);
-					for (int j = 0; j < 3; ++j)
-						add(GEProduct(TruncateToFloat24(src[j]), bones[b][j * 3 + c]).Value());
-				}
-				dst[c] = acc;
+		auto skin = [&](const float src[3], bool translate, u8 *dst) {
+			Vec4F32 acc = Vec4F32::Zero();
+			for (int b = 0; b < nweights; ++b) {
+				if (!used[b])
+					continue;
+				if (translate)
+					acc = GEAddFloat24x4(acc, bones[b][3]);
+				for (int j = 0; j < 3; ++j)
+					acc = GEAddFloat24x4(acc, product(src[j], bones[b][j]));
 			}
+			alignas(16) float result[4];
+			acc.Store(result);
+			memcpy(dst, result, 3 * sizeof(float));
 		};
 		if (dec.pos) {
-			float pos[3], skinned[3];
+			float pos[3];
 			for (int i = 0; i < 3; ++i)
 				pos[i] = component(dec.posoff, i, [&](const u8 *p, int c) { return ReadRawComponent(p, dec.pos, c); });
-			skin(pos, true, skinned);
-			memcpy(out + fmt.posoff, skinned, sizeof(skinned));
+			skin(pos, true, out + fmt.posoff);
 		}
 		if (dec.nrm && fmt.nrmfmt == DEC_FLOAT_3) {
-			float nrm[3], skinned[3];
+			float nrm[3];
 			for (int i = 0; i < 3; ++i)
 				nrm[i] = component(dec.nrmoff, i, [&](const u8 *p, int c) { return ReadRawComponent(p, dec.nrm, c); });
-			skin(nrm, false, skinned);
-			memcpy(out + fmt.nrmoff, skinned, sizeof(skinned));
+			skin(nrm, false, out + fmt.nrmoff);
 		}
 	}
 }

unittest/TestGEMath.cpp

@@ -107,6 +107,89 @@ static bool TestGEDot() {
 	return true;
 }
 
+// GEAdd4 against GEAdd, bit for bit: any bits (not just float24s), close and distant exponents, signs,
+// zeros, denormals, inf and NaN.
+static bool TestGEAdd4() {
+	uint32_t state = 777;
+	auto next = [&]() {
+		state = state * 1664525u + 1013904223u;
+		return state;
+	};
+	auto randomBits = [&](int center, int spread) {
+		const uint32_t r = next();
+		uint32_t e;
+		switch (r & 15) {
+		case 0: e = 0; break;
+		case 1: e = 255; break;
+		default: e = (uint32_t)std::clamp(center + (int)(next() % (2 * spread + 1)) - spread, 1, 254); break;
+		}
+		const uint32_t bits = (e << 23) | (next() & 0x007FFFFF) | ((r & 16) ? 0x80000000 : 0);
+		float f;
+		memcpy(&f, &bits, sizeof(f));
+		return f;
+	};
+	for (int i = 0; i < 200000; ++i) {
+		const int spread = (i % 5 == 0) ? 120 : 20;
+		const int center = (i % 7 == 0) ? (int)(next() % 254) + 1 : 127;
+		alignas(16) float a[4], b[4], got[4];
+		for (int l = 0; l < 4; ++l) {
+			a[l] = randomBits(center, spread);
+			b[l] = randomBits(center, spread);
+		}
+		GEAdd4(Vec4F32::Load(a), Vec4F32::Load(b)).Store(got);
+		for (int l = 0; l < 4; ++l) {
+			const float want = GEAdd(a[l], b[l]);
+			if (memcmp(&got[l], &want, sizeof(float)) != 0) {
+				printf("GEAdd4: %a + %a = %a, want %a\n", a[l], b[l], got[l], want);
+				return false;
+			}
+		}
+	}
+	return true;
+}
+
+// GEMulFloat24x4 against ProductToFloat24, bit for bit, on the same kind of bits as TestGEAdd4.
+static bool TestGEMulFloat24x4() {
+	uint32_t state = 4321;
+	auto next = [&]() {
+		state = state * 1664525u + 1013904223u;
+		return state;
+	};
+	auto randomBits = [&](int center, int spread) {
+		const uint32_t r = next();
+		uint32_t e;
+		switch (r & 15) {
+		case 0: e = 0; break;
+		case 1: e = 255; break;
+		default: e = (uint32_t)std::clamp(center + (int)(next() % (2 * spread + 1)) - spread, 1, 254); break;
+		}
+		const uint32_t mant = (r & 32) ? 0 : (next() & 0x007FFFFF);
+		const uint32_t bits = (e << 23) | mant | ((r & 16) ? 0x80000000 : 0);
+		float f;
+		memcpy(&f, &bits, sizeof(f));
+		return f;
+	};
+	for (int i = 0; i < 200000; ++i) {
+		const int spread = (i % 5 == 0) ? 120 : 20;
+		const int center = (i % 7 == 0) ? (int)(next() % 254) + 1 : 127;
+		alignas(16) float a[4], b[4], got[4];
+		for (int l = 0; l < 4; ++l) {
+			a[l] = randomBits(center, spread);
+			b[l] = randomBits(center, spread);
+		}
+		GEMulFloat24x4(Vec4F32::Load(a), Vec4F32::Load(b)).Store(got);
+		for (int l = 0; l < 4; ++l) {
+			const float ta = TruncateToFloat24(a[l]), tb = TruncateToFloat24(b[l]);
+			const float want = ta == 0.0f || tb == 0.0f ? 0.0f : ProductToFloat24((double)ta * tb);
+			if (memcmp(&got[l], &want, sizeof(float)) != 0) {
+				printf("GEMulFloat24x4: %a * %a = %a, want %a\n", a[l], b[l], got[l], want);
+				return false;
+			}
+		}
+	}
+	return true;
+}
+
 // GERowSum4 against GERowSum, bit for bit, on random float24s: mixed signs, zeros and denormals, exponents
 // close together (so the terms' alignment matters) and far apart, including ones past the vector path.
 static bool TestGERowSum4() {
@@ -318,6 +401,8 @@ bool TestGEMath() {
 	ok = TestGEAdd() && ok;
 	ok = TestGEDot() && ok;
 	ok = TestGERowSum4() && ok;
+	ok = TestGEAdd4() && ok;
+	ok = TestGEMulFloat24x4() && ok;
 	ok = TestGESetupRecip() && ok;
 	ok = TestGELog16() && ok;
 	ok = TestTexCoordPrecision() && ok;