From 9b6727e97a3f3eaacefcd31c2a34a8419afec239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Mon, 27 Nov 2023 22:47:28 +0100 Subject: [PATCH] Allow the compiler to inline matrix4x4 multiplication. Also add a unit test. --- Common/Data/Convert/SmallDataConvert.h | 15 +---- Common/Math/fast/fast_matrix.c | 20 +----- Common/Math/fast/fast_matrix.h | 88 ++++++++++++++++++++++++-- GPU/Math3D.h | 1 + unittest/UnitTest.cpp | 75 ++++++++++++++++++++++ 5 files changed, 160 insertions(+), 39 deletions(-) diff --git a/Common/Data/Convert/SmallDataConvert.h b/Common/Data/Convert/SmallDataConvert.h index cdbfc3c5b2..f628534b73 100644 --- a/Common/Data/Convert/SmallDataConvert.h +++ b/Common/Data/Convert/SmallDataConvert.h @@ -132,13 +132,6 @@ inline void Uint8x3ToInt4(int i[4], uint32_t u) { i[3] = 0; } -inline void Uint8x3ToInt4_Alpha(int i[4], uint32_t u, uint8_t alpha) { - i[0] = ((u >> 0) & 0xFF); - i[1] = ((u >> 8) & 0xFF); - i[2] = ((u >> 16) & 0xFF); - i[3] = alpha; -} - inline void Uint8x3ToFloat4_Alpha(float f[4], uint32_t u, float alpha) { f[0] = ((u >> 0) & 0xFF) * (1.0f / 255.0f); f[1] = ((u >> 8) & 0xFF) * (1.0f / 255.0f); @@ -146,13 +139,6 @@ inline void Uint8x3ToFloat4_Alpha(float f[4], uint32_t u, float alpha) { f[3] = alpha; } -inline void Uint8x1ToFloat4(float f[4], uint32_t u) { - f[0] = ((u >> 0) & 0xFF) * (1.0f / 255.0f); - f[1] = 0.0f; - f[2] = 0.0f; - f[3] = 0.0f; -} - // These are just for readability. inline void CopyFloat2(float dest[2], const float src[2]) { @@ -206,6 +192,7 @@ inline void CopyMatrix4x4(float dest[16], const float src[16]) { memcpy(dest, src, sizeof(float) * 16); } +// WARNING: This can quietly "over-load" src by 4 bytes. inline void ExpandFloat24x3ToFloat4(float dest[4], const uint32_t src[3]) { #ifdef _M_SSE __m128i values = _mm_slli_epi32(_mm_loadu_si128((const __m128i *)src), 8); diff --git a/Common/Math/fast/fast_matrix.c b/Common/Math/fast/fast_matrix.c index 6ce87e2a6a..37fd900e81 100644 --- a/Common/Math/fast/fast_matrix.c +++ b/Common/Math/fast/fast_matrix.c @@ -4,25 +4,7 @@ #include "fast_matrix.h" -#if PPSSPP_ARCH(SSE2) - -void fast_matrix_mul_4x4_sse(float *dest, const float *a, const float *b) { - int i; - __m128 a_col_1 = _mm_loadu_ps(a); - __m128 a_col_2 = _mm_loadu_ps(&a[4]); - __m128 a_col_3 = _mm_loadu_ps(&a[8]); - __m128 a_col_4 = _mm_loadu_ps(&a[12]); - - for (i = 0; i < 16; i += 4) { - __m128 r_col = _mm_mul_ps(a_col_1, _mm_set1_ps(b[i])); - r_col = _mm_add_ps(r_col, _mm_mul_ps(a_col_2, _mm_set1_ps(b[i + 1]))); - r_col = _mm_add_ps(r_col, _mm_mul_ps(a_col_3, _mm_set1_ps(b[i + 2]))); - r_col = _mm_add_ps(r_col, _mm_mul_ps(a_col_4, _mm_set1_ps(b[i + 3]))); - _mm_storeu_ps(&dest[i], r_col); - } -} - -#elif PPSSPP_ARCH(LOONGARCH64_LSX) +#if PPSSPP_ARCH(LOONGARCH64_LSX) typedef union { diff --git a/Common/Math/fast/fast_matrix.h b/Common/Math/fast/fast_matrix.h index fb4a1b7f26..23db4a28cc 100644 --- a/Common/Math/fast/fast_matrix.h +++ b/Common/Math/fast/fast_matrix.h @@ -7,12 +7,6 @@ extern "C" { #endif // A mini library of 4x4 matrix muls. - -extern void fast_matrix_mul_4x4_c(float *dest, const float *a, const float *b); -extern void fast_matrix_mul_4x4_neon(float *dest, const float *a, const float *b); -extern void fast_matrix_mul_4x4_sse(float *dest, const float *a, const float *b); -extern void fast_matrix_mul_4x4_lsx(float *dest, const float *a, const float *b); - #if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) // Hard link to SSE implementations on x86/amd64 #define fast_matrix_mul_4x4 fast_matrix_mul_4x4_sse @@ -24,6 +18,88 @@ extern void fast_matrix_mul_4x4_lsx(float *dest, const float *a, const float *b) #define fast_matrix_mul_4x4 fast_matrix_mul_4x4_c #endif +extern void fast_matrix_mul_4x4_c(float *dest, const float *a, const float *b); + +#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) + +#include + +#include "fast_matrix.h" + +inline void fast_matrix_mul_4x4_sse(float *dest, const float *a, const float *b) { + int i; + __m128 a_col_1 = _mm_loadu_ps(a); + __m128 a_col_2 = _mm_loadu_ps(&a[4]); + __m128 a_col_3 = _mm_loadu_ps(&a[8]); + __m128 a_col_4 = _mm_loadu_ps(&a[12]); + + for (i = 0; i < 16; i += 4) { + __m128 r_col = _mm_mul_ps(a_col_1, _mm_set1_ps(b[i])); + r_col = _mm_add_ps(r_col, _mm_mul_ps(a_col_2, _mm_set1_ps(b[i + 1]))); + r_col = _mm_add_ps(r_col, _mm_mul_ps(a_col_3, _mm_set1_ps(b[i + 2]))); + r_col = _mm_add_ps(r_col, _mm_mul_ps(a_col_4, _mm_set1_ps(b[i + 3]))); + _mm_storeu_ps(&dest[i], r_col); + } +} + +#elif PPSSPP_ARCH(ARM_NEON) + +#if defined(_MSC_VER) && PPSSPP_ARCH(ARM64) +#include +#else +#include +#endif + +#if PPSSPP_ARCH(ARM) +static inline float32x4_t vfmaq_laneq_f32(float32x4_t _s, float32x4_t _a, float32x4_t _b, int lane) { + if (lane == 0) return vmlaq_lane_f32(_s, _a, vget_low_f32(_b), 0); + else if (lane == 1) return vmlaq_lane_f32(_s, _a, vget_low_f32(_b), 1); + else if (lane == 2) return vmlaq_lane_f32(_s, _a, vget_high_f32(_b), 0); + else if (lane == 3) return vmlaq_lane_f32(_s, _a, vget_high_f32(_b), 1); + else return vdupq_n_f32(0.f); +} +#endif + +// From https://developer.arm.com/documentation/102467/0100/Matrix-multiplication-example +inline void fast_matrix_mul_4x4_neon(float *C, const float *A, const float *B) { + // these are the columns A + float32x4_t A0 = vld1q_f32(A); + float32x4_t A1 = vld1q_f32(A + 4); + float32x4_t A2 = vld1q_f32(A + 8); + float32x4_t A3 = vld1q_f32(A + 12); + + // Multiply accumulate in 4x1 blocks, i.e. each column in C + float32x4_t B0 = vld1q_f32(B); + float32x4_t C0 = vmulq_laneq_f32(A0, B0, 0); + C0 = vfmaq_laneq_f32(C0, A1, B0, 1); + C0 = vfmaq_laneq_f32(C0, A2, B0, 2); + C0 = vfmaq_laneq_f32(C0, A3, B0, 3); + vst1q_f32(C, C0); + + float32x4_t B1 = vld1q_f32(B + 4); + float32x4_t C1 = vmulq_laneq_f32(A0, B1, 0); + C1 = vfmaq_laneq_f32(C1, A1, B1, 1); + C1 = vfmaq_laneq_f32(C1, A2, B1, 2); + C1 = vfmaq_laneq_f32(C1, A3, B1, 3); + vst1q_f32(C + 4, C1); + + float32x4_t B2 = vld1q_f32(B + 8); + float32x4_t C2 = vmulq_laneq_f32(A0, B2, 0); + C2 = vfmaq_laneq_f32(C2, A1, B2, 1); + C2 = vfmaq_laneq_f32(C2, A2, B2, 2); + C2 = vfmaq_laneq_f32(C2, A3, B2, 3); + vst1q_f32(C + 8, C2); + + float32x4_t B3 = vld1q_f32(B + 12); + float32x4_t C3 = vmulq_laneq_f32(A0, B3, 0); + C3 = vfmaq_laneq_f32(C3, A1, B3, 1); + C3 = vfmaq_laneq_f32(C3, A2, B3, 2); + C3 = vfmaq_laneq_f32(C3, A3, B3, 3); + vst1q_f32(C + 12, C3); +} + +#endif + #ifdef __cplusplus } // extern "C" #endif diff --git a/GPU/Math3D.h b/GPU/Math3D.h index 5a3d2f5940..073cab1337 100644 --- a/GPU/Math3D.h +++ b/GPU/Math3D.h @@ -1199,6 +1199,7 @@ inline void ConvertMatrix4x3To3x4Transposed(float *m4x4, const float *m4x3) { #endif } +// Only used in D3D9, not bothering to optimize. inline void Transpose4x4(float out[16], const float in[16]) { for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 4cca6b8ccb..da14108b17 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -81,12 +81,14 @@ #include "Common/Data/Convert/ColorConv.h" #include "Common/File/VFS/VFS.h" #include "Common/File/VFS/DirectoryReader.h" +#include "Common/Math/fast/fast_matrix.h" #include "Core/FileSystems/ISOFileSystem.h" #include "Core/MemMap.h" #include "Core/KeyMap.h" #include "Core/MIPS/MIPSVFPUUtils.h" #include "GPU/Common/TextureDecoder.h" #include "GPU/Common/GPUStateUtils.h" +#include "GPU/Math3D.h" #include "Common/File/AndroidContentURI.h" @@ -1229,6 +1231,78 @@ bool TestVolumeFunc() { EXPECT_TRUE(smaller); // printf("%d -> %f -> %d -> %f\n", i, mul, vol100, mul2); } +} + +bool TestLinAlg() { + static const float m1[16] = { + 1, 2, 3, 4, + 5, 6, 7, 8, + 9, 10, 11, 12, + 13, 14, 15, 16 + }; + static const float m2[16] = { + 56, 0, 24, 2, + 0.5f, 35, 2, 4, + 1, 6, 1, 2, + 4, 0, -1, -4 + }; + static const float correct[16] = { + 298.f, 380.f, 462.f, 544.f, + 245.5f, 287.f, 328.5f, 370.f, + 66.f, 76.f, 86.f, 96.f, + -57.f, -58.f, -59.f, -60.f, + }; + + float d[16]{}; + + fast_matrix_mul_4x4(d, m1, m2); + + for (int i = 0; i < 16; i += 4) { + // printf("%0.2f, %0.2f, %0.2f, %0.2f,\n", d[i], d[i + 1], d[i + 2], d[i + 3]); + } + + for (int i = 0; i < 16; i++) { + EXPECT_EQ_FLOAT(d[i], correct[i]); + } + + + // OK, now test 4x3 multiplication. + float a4x4[16]; + float b4x4[16]; + + ConvertMatrix4x3To4x4(a4x4, m1); + ConvertMatrix4x3To4x4(b4x4, m1); + Matrix4ByMatrix4(d, a4x4, b4x4); + + for (int i = 0; i < 16; i += 4) { + printf("%0.2f, %0.2f, %0.2f, %0.2f,\n", d[i], d[i + 1], d[i + 2], d[i + 3]); + } + + static const float correct4x4[16] = { + 30.00, 36.00, 42.00, 0.00, + 66.00, 81.00, 96.00, 0.00, + 102.00, 126.00, 150.00, 0.00, + 148.00, 182.00, 216.00, 1.00, + }; + + for (int i = 0; i < 16; i++) { + EXPECT_EQ_FLOAT(d[i], correct4x4[i]); + } + + ConvertMatrix4x3To4x4Transposed(b4x4, m1); + Matrix4ByMatrix4(d, a4x4, b4x4); + + static const float correct4x4transposed[16] = { + 14.00, 32.00, 50.00, 68.00, + 32.00, 77.00, 122.00, 167.00, + 50.00, 122.00, 194.00, 266.00, + 68.00, 167.00, 266.00, 366.00, + }; + + for (int i = 0; i < 16; i++) { + EXPECT_EQ_FLOAT(d[i], correct4x4transposed[i]); + } + // TODO: Add direct 4x3 x 4x3 multiplication return true; } @@ -1301,6 +1375,7 @@ TestItem availableTests[] = { TEST_ITEM(SIMD), TEST_ITEM(CrossSIMD), TEST_ITEM(VolumeFunc), + TEST_ITEM(LinAlg), }; int main(int argc, const char *argv[]) {